erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Object type mix: INTF, TABL, STRU, MSAG, exception tasks (harness G2, mutants, generator notes), balanced generator, kind-deficit job order, dashboard mix card

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 18:52:29 +02:00
parent 454db6626c
commit b119f1afac
39 changed files with 2329 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
harness/trainset.py
View File

@@ -19,6 +19,7 @@ from .evalset import SLOTS, RELEASES, accepted_goals
from .generator import ROOT, generate, make_k_variant
from .ledger import BudgetExceeded, spent
from . import overlap
from . import mix
POOL = os.path.join(ROOT, "tasks_gen", "train")
PLAN = os.path.join(POOL, "plan.json")
@@ -186,6 +187,137 @@ def run(part, parts, target, stop_ledger, plan_name="plan", deadline=None):
print(json.dumps(log), flush=True)
BAL_FIRST_ID = 1910
BAL_RUN_BASE = 370000 # 40 per slot; below 466560 (a digit must lead the 4-char base36 run)
BAL_ERROR_KINDS = {"CLAS": ["named-type", "long-names"], "FUNC": ["named-type"], "DDLS": ["reserved-word"],
"TABL": ["reserved-word"], "STRU": ["reserved-word"]}
BAL_HINTS = dict(((c, t), h) for c, t, _, h in SLOTS if h)
def _claims_dir():
d = os.path.join(POOL, "_claims")
os.makedirs(d, exist_ok=True)
return d
def _claim_slot():
"""Next free balanced slot number, claimed with O_EXCL (several workers). Returns (n, claim path)."""
for n in range(BAL_FIRST_ID, BAL_FIRST_ID + 600):
sid = "G%04d" % n
if os.path.exists(os.path.join(POOL, "_logs", sid + ".json")):
continue
path = os.path.join(_claims_dir(), sid + ".json")
try:
fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
except FileExistsError:
continue
os.close(fd)
return n, path
return None, None
def _kind_stats():
"""({kind: accepted}, {kind: attempted}) from the generation logs; claims of running slots count as attempted."""
acc = mix.accepted_task_counts()
att = {}
for f in glob.glob(os.path.join(POOL, "_logs", "G*.json")):
try:
l = json.load(open(f))
except (OSError, ValueError):
continue
k = l.get("kind") or l.get("object_type")
if k and l.get("category") != "K":
att[k] = att.get(k, 0) + 1
return acc, att
def run_balanced(part, parts, deadline):
"""Generation without a fixed plan: each slot takes the kind with the biggest deficit against mix.TYPE_SHARE.
A kind with 6 or more tries and an acceptance below 20 % is skipped (a harness or prompt problem: do not burn budget)."""
base_url = os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1")
evals = overlap.load_pool("eval")
while True:
while backlog() > BACKLOG_LIMIT and not os.path.exists(STOP_FLAG) and not (deadline and time.time() > deadline):
time.sleep(120)
if os.path.exists(STOP_FLAG):
print("STOP flag", flush=True)
return
if deadline and time.time() > deadline:
print("DEADLINE", flush=True)
return
acc, att = _kind_stats()
running = {}
for f in glob.glob(os.path.join(_claims_dir(), "G*.json")):
try:
k = json.load(open(f)).get("kind")
except (OSError, ValueError):
k = None
if k:
running[k] = running.get(k, 0) + 1
counts = {k: acc.get(k, 0) + running.get(k, 0) for k in set(acc) | set(running) | set(mix.TYPE_SHARE)}
blocked = {k for k in mix.TYPE_SHARE if att.get(k, 0) >= 6 and acc.get(k, 0) < 0.2 * att.get(k, 0)}
if blocked:
print("kinds skipped (low acceptance):", sorted(blocked), flush=True)
kind = mix.deficit_pick(counts, allowed=set(mix.TYPE_SHARE) - blocked)
n, claim = _claim_slot()
if n is None:
print("no free slot", flush=True)
return
sid = "G%04d" % n
json.dump({"kind": kind}, open(claim, "w"))
otype = "CLAS" if kind == "EXC" else kind
cats = mix.KIND_CATEGORIES[kind]
logs = [json.load(open(f)) for f in glob.glob(os.path.join(POOL, "_logs", "G*.json"))]
ccount = {c: sum(1 for l in logs if l.get("accepted") and l.get("category") == c) for c in cats}
tot = sum(ccount.values()) + 1
cat = max(cats, key=lambda c: tot * mix.CATEGORY_SHARE[c] / sum(mix.CATEGORY_SHARE[x] for x in cats) - ccount[c])
idx = n - BAL_FIRST_ID
error_kind = None
if idx % 5 == 2 and kind in BAL_ERROR_KINDS: # 20 % error-targeted slots
error_kind = BAL_ERROR_KINDS[kind][(idx // 5) % len(BAL_ERROR_KINDS[kind])]
cat = ERROR_CATEGORY[error_kind] if kind in ("CLAS", "FUNC") else cat
hard = cat != "H" and idx % 10 in (3, 6, 9) # 30 % hard
topic = None
if error_kind:
topic = dict(ERROR_HINTS)[error_kind]
elif kind == "EXC":
topic = "exception class (CX_...): " + ["a domain exception with context attributes and message texts",
"an exception hierarchy with a common super class",
"an exception that wraps a previous exception"][idx % 3]
elif (cat, otype) in BAL_HINTS:
h = BAL_HINTS[(cat, otype)]
topic = h[idx % len(h)]
if cat == "G":
topic = (topic + "; " if topic else "") + f"release target {RELEASES[idx % 2]}"
avoid = [g for g in accepted_goals() if g][-170:]
full = ((topic + ". ") if topic else "Choose a new, realistic business topic. ") + \
"Do not repeat these existing topics: " + "; ".join(avoid)
pool_now = evals + overlap.load_pool("train")
def extra(b, _pool=pool_now):
hits = overlap.check(overlap.load_bundle(b), _pool)
return [f"Too close to task {i} (similarity spec {sc['spec']:.2f}, rules {sc['core']:.2f}, "
f"names {sc['name']:.2f}). Choose a different business topic and different object names."
for i, sc in hits[:3]]
try:
log = generate(sid, "train", otype, cat, 3 if hard else 2, "deepseek-v4.1-flash:cloud", base_url,
BAL_RUN_BASE + 40 * idx, full, extra_check=extra)
except BudgetExceeded as e:
print("BUDGET", e, flush=True)
os.remove(claim)
return
except Exception as e: # noqa: BLE001
log = {"id": sid, "error": str(e)[:500]}
log.update(kind=kind, error_kind=error_kind, difficulty=3 if hard else 2, spent_total=spent())
os.makedirs(os.path.join(POOL, "_logs"), exist_ok=True)
json.dump(log, open(os.path.join(POOL, "_logs", sid + ".json"), "w"), indent=1)
stray = os.path.join(POOL, "generation.json")
if os.path.exists(stray):
os.remove(stray)
os.remove(claim)
print(json.dumps(log), flush=True)
K_FIRST_ID = 1300
K_RUN_BASE = 41800 # 20 per variant; above the trajectory run numbers (41000-41700)
K_COUNT = 18 # K share of the eval plan: 10 of 110 (9 %); counted inside the 200 accepted tasks
@@ -240,7 +372,7 @@ def main():
ap.add_argument("--part", type=int, default=0)
ap.add_argument("--parts", type=int, default=1)
ap.add_argument("--target", type=int, default=200)
ap.add_argument("--plan", default="plan", help="plan (first 223 slots) or plan2 (hard and error share raised)")
ap.add_argument("--plan", default="plan", help="plan (first 223 slots), plan2 (hard and error share raised) or balanced (kind with the biggest deficit)")
ap.add_argument("--deadline", help="YYYY-MM-DDTHH:MM local time: no new slot after it")
ap.add_argument("--k-count", type=int, default=K_COUNT)
ap.add_argument("--stop-ledger", type=float, default=27.0, help="ledger USD for this phase (10 USD usage = 27)")
@@ -255,6 +387,9 @@ def main():
run_k(a.k_count)
return
dl = time.mktime(time.strptime(a.deadline, "%Y-%m-%dT%H:%M")) if a.deadline else None
if a.plan == "balanced":
run_balanced(a.part, a.parts, dl)
return
run(a.part, a.parts, a.target, a.stop_ledger, a.plan, dl)