Generator training mode: train pool, eval overlap check, category mix, error-targeted tasks

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

harness/generator.py

@@ -393,7 +393,8 @@ def failure_summary(rep):
 
 
 def generate(task_id, pool, object_type, category, difficulty, model, base_url, run_base, topic=None,
-             max_repairs=3):
+             max_repairs=3, extra_check=None):
+    """extra_check(bundle) -> list of problems; it runs with the static checks (training mode: overlap)."""
     check_budget()
     pool_root = os.path.join(ROOT, "tasks_gen", pool)
     task_dir = os.path.join(pool_root, task_id)
@@ -411,6 +412,8 @@ def generate(task_id, pool, object_type, category, difficulty, model, base_url,
             b = parse_bundle(text)
             b.get("task", {}).setdefault("category", category)
             found = {"structure": check_bundle(b), "static": lint_files(b), "abaplint": abaplint_files(b)}
+            if extra_check and not found["structure"]:
+                found["extra"] = extra_check(b)
         except Exception as e:  # noqa: BLE001
             b, found = None, {"json": [f"invalid JSON: {e}"]}
         errs = [e for v in found.values() for e in v]

harness/overlap.py

@@ -0,0 +1,100 @@
+"""Overlap check between a new training task and the eval tasks (spec text and object names).
+
+A training task is rejected when its spec or its object names are too close to an eval task.
+"""
+import glob
+import json
+import math
+import os
+import re
+from collections import Counter
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+STOP = set("the a an of to in is are be and or for with that this it as on by at from not no if then each "
+           "all any must shall can will when which its into than one two use used using value values "
+           "object objects method methods class classes table tables field fields return returns".split())
+SPEC_LIMIT = 0.55   # cosine of word counts of the whole spec
+GOAL_LIMIT = 0.60   # cosine of the Goal and Business rules sections
+NAME_LIMIT = 0.60   # Jaccard of name tokens (placeholder and Z prefix removed)
+
+
+def words(text):
+    text = re.sub(r"\{\{[Pp]\}\}", " ", text)
+    return [w for w in re.findall(r"[a-z][a-z0-9]{2,}", text.lower()) if w not in STOP]
+
+
+def cosine(a, b):
+    ca, cb = Counter(a), Counter(b)
+    dot = sum(ca[k] * cb[k] for k in ca if k in cb)
+    na = math.sqrt(sum(v * v for v in ca.values()))
+    nb = math.sqrt(sum(v * v for v in cb.values()))
+    return dot / (na * nb) if na and nb else 0.0
+
+
+def section(spec, start, stop=None):
+    m = re.search(rf"(?:^|\n)#*\s*\d*\.?\s*{start}.*?\n(.*?)(?=\n#*\s*\d*\.?\s*(?:{stop})|\Z)" if stop
+                  else rf"(?:^|\n)#*\s*\d*\.?\s*{start}.*?\n(.*)", spec, re.S | re.I)
+    return m.group(1) if m else ""
+
+
+def core_text(spec):
+    return section(spec, "Goal", "Open questions|Context") + "\n" + section(spec, "Business rules", "Constraints")
+
+
+def name_tokens(name):
+    n = re.sub(r"\{\{[Pp]\}\}", "", name).lower()
+    n = re.sub(r"^(?:zcl_|zif_|zcx_|z|/\w+/)", "", n)
+    return {t for t in re.split(r"[_\W]+", n) if len(t) > 2 and t not in ("test", "tab", "cls", "intf")}
+
+
+def task_names(task):
+    return [o["name"] for k in ("contract", "seed", "reference") for o in task.get(k, []) if o.get("name")]
+
+
+def load_task(d):
+    task = json.load(open(os.path.join(d, "task.json")))
+    spec = open(os.path.join(d, "spec.md")).read()
+    return {"id": task.get("id", os.path.basename(d)), "spec": words(spec), "core": words(core_text(spec)),
+            "names": [name_tokens(n) for n in task_names(task)],
+            "main": [name_tokens(o["name"]) for o in task.get("contract", []) if o.get("name")]}
+
+
+def load_pool(pool):
+    out = []
+    for d in sorted(glob.glob(os.path.join(ROOT, "tasks_gen", pool, "G*")) + glob.glob(os.path.join(ROOT, "tasks", "T*"))):
+        try:
+            out.append(load_task(d))
+        except (OSError, ValueError):
+            pass
+    return out
+
+
+def jaccard(a, b):
+    return len(a & b) / len(a | b) if a and b else 0.0
+
+
+def compare(x, y):
+    """Scores of two loaded tasks: spec cosine, core cosine, best name Jaccard (contract names)."""
+    name = max([jaccard(a, b) for a in x["main"] for b in y["main"]] or [0.0])
+    return {"spec": cosine(x["spec"], y["spec"]), "core": cosine(x["core"], y["core"]), "name": name}
+
+
+def check(task_dir_or_loaded, pool_tasks):
+    """Return a list of problems (empty = no overlap) against a list of loaded tasks."""
+    x = task_dir_or_loaded if isinstance(task_dir_or_loaded, dict) else load_task(task_dir_or_loaded)
+    problems = []
+    for y in pool_tasks:
+        if y["id"] == x["id"]:
+            continue
+        s = compare(x, y)
+        if s["spec"] >= SPEC_LIMIT or s["core"] >= GOAL_LIMIT or s["name"] >= NAME_LIMIT:
+            problems.append((y["id"], s))
+    return problems
+
+
+def load_bundle(b):
+    """Loaded form of a generated bundle ({"task", "files"}) before it is written to disk."""
+    task, spec = b["task"], b["files"].get("spec.md", "")
+    return {"id": task.get("id", "(new)"), "spec": words(spec), "core": words(core_text(spec)),
+            "names": [name_tokens(n) for n in task_names(task)],
+            "main": [name_tokens(o["name"]) for o in task.get("contract", []) if o.get("name")]}

harness/trainset.py

@@ -0,0 +1,157 @@
+"""Training mode of the task generator (stage 2 data).
+
+Separate pool `tasks_gen/train` (ids G1000+). Same category and object type mix as the eval plan, plus
+tasks for the common errors (docs/yol-haritasi.md step 3). A new bundle is checked against all eval tasks
+and the earlier training tasks (`harness/overlap.py`); a too close bundle goes back to the model as a
+repair message. K variants are not made here (they need the generic_v0 tool schema).
+
+  python3 -m harness.trainset plan
+  python3 -m harness.trainset run --part 0 --parts 3 [--target 200] [--stop-ledger 27]
+"""
+import argparse
+import glob
+import json
+import os
+import time
+
+from .adt_client import load_env
+from .evalset import SLOTS, RELEASES, accepted_goals
+from .generator import ROOT, generate
+from .ledger import BudgetExceeded, spent
+from . import overlap
+
+POOL = os.path.join(ROOT, "tasks_gen", "train")
+PLAN = os.path.join(POOL, "plan.json")
+FIRST_ID = 1000
+RUN_BASE = 32000          # run numbers stay below 36**3 * 1 (proxy prefix rule); 40 per slot
+N_SLOTS = 224             # about 12 % more slots than the target: some tasks are not accepted
+N_ERROR = 30
+EVAL_HOLD = 3             # eval pool is loaded once per process
+
+ERROR_HINTS = [
+    ("named-type", "The contract has methods with short coded character parameters (zone code, currency code, "
+     "status flag, region key) of fixed length. A correct solution declares named types for them: "
+     "TYPE c LENGTH n is not allowed in a method signature."),
+    ("reserved-word", "The business domain uses concepts that collide with reserved or SQL words (hours, mode, "
+     "order, date, count, group, value). The contract uses safe field and parameter names (for example "
+     "work_hours, calc_mode); the solution must also avoid reserved words as field names."),
+    ("long-names", "The task has at least 10 separate business rules, so a good solution needs at least 10 "
+     "own unit tests, each for one rule. Every ABAP name stays at 30 characters or less (the model must "
+     "shorten long descriptive test method names)."),
+]
+ERROR_TYPES = ["CLAS", "CLAS", "FUNC", "DDLS", "PROG", "CLAS"]   # reserved-word fits DDLS and tables well
+
+
+CATEGORY_SHARE = {"A": 10, "B": 15, "C": 15, "D": 10, "E": 15, "F": 10, "G": 10, "I": 5, "H": 10}  # eval 8.1
+ERROR_CATEGORY = {"named-type": "A", "reserved-word": "D", "long-names": "C"}
+
+
+def build_plan():
+    """Category shares as in the eval plan; object types inside a category as in evalset.SLOTS."""
+    share_sum = sum(CATEGORY_SHARE.values())
+    items = []
+    for cat, share in CATEGORY_SHARE.items():
+        k_cat = round(N_SLOTS * share / share_sum)
+        k_err = sum(1 for k in ERROR_CATEGORY if ERROR_CATEGORY[k] == cat) * N_ERROR // 3
+        slots = [x for x in SLOTS if x[0] == cat]
+        weight = sum(n for _, _, n, _ in slots)
+        k_rest = k_cat - k_err
+        quota = [(t, hints, round(k_rest * n / weight)) for _, t, n, hints in slots]
+        quota[0] = (quota[0][0], quota[0][1], quota[0][2] + k_rest - sum(q[2] for q in quota))
+        pos = 0
+        for t, hints, k in quota:
+            for i in range(k):
+                topic = hints[i % len(hints)] if hints else None
+                if cat == "G":
+                    topic = (topic + "; " if topic else "") + f"release target {RELEASES[pos % 2]}"
+                pos += 1
+                items.append({"category": cat, "object_type": t, "topic": topic, "frac": pos / (k_cat + 1)})
+        for kind, text in ERROR_HINTS:
+            if ERROR_CATEGORY[kind] != cat:
+                continue
+            for i in range(N_ERROR // 3):
+                t = ERROR_TYPES[i % len(ERROR_TYPES)]
+                pos += 1
+                items.append({"category": cat, "object_type": t, "topic": text, "error_kind": kind,
+                              "frac": pos / (k_cat + 1)})
+    items.sort(key=lambda x: (x["frac"], x["category"], x["object_type"]))   # interleave: a cut keeps the mix
+    for n, it in enumerate(items):
+        it.pop("frac")
+        it["id"] = f"G{FIRST_ID + n:04d}"
+        it["run_base"] = RUN_BASE + 40 * n
+    return items
+
+
+def accepted_count():
+    n = 0
+    for f in glob.glob(os.path.join(POOL, "G*", "generation.json")):
+        try:
+            n += bool(json.load(open(f)).get("accepted"))
+        except (OSError, ValueError):
+            pass
+    return n
+
+
+def run(part, parts, target, stop_ledger):
+    os.makedirs(POOL, exist_ok=True)
+    if not os.path.exists(PLAN):
+        json.dump({"slots": build_plan(), "ledger_at_start": spent(), "stop_ledger": stop_ledger,
+                   "target": target, "created": time.strftime("%F %T")}, open(PLAN, "w"), indent=1)
+    plan = json.load(open(PLAN))
+    stop_at = plan["ledger_at_start"] + plan["stop_ledger"]
+    base_url = os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1")
+    evals = overlap.load_pool("eval")
+    for idx, s in enumerate(plan["slots"]):
+        if idx % parts != part:
+            continue
+        if os.path.exists(os.path.join(POOL, s["id"], "generation.json")):
+            continue
+        if accepted_count() >= plan["target"]:
+            print("TARGET reached", flush=True)
+            break
+        if spent() >= stop_at:
+            print("PHASE LIMIT", spent(), stop_at, flush=True)
+            break
+        goals = accepted_goals()
+        avoid = [g for g in goals if g]   # eval goals and all training goals
+        topic = ((s["topic"] + ". ") if s["topic"] else "Choose a new, realistic business topic. ") + \
+            "Do not repeat these existing topics: " + "; ".join(avoid[-170:])
+        pool_now = evals + overlap.load_pool("train")
+
+        def extra(b, _pool=pool_now):
+            hits = overlap.check(overlap.load_bundle(b), _pool)
+            return [f"Too close to task {i} (similarity spec {sc['spec']:.2f}, rules {sc['core']:.2f}, "
+                    f"names {sc['name']:.2f}). Choose a different business topic and different object names."
+                    for i, sc in hits[:3]]
+        try:
+            log = generate(s["id"], "train", s["object_type"], s["category"], 2, "deepseek-v4.1-flash:cloud",
+                           base_url, s["run_base"], topic, extra_check=extra)
+        except BudgetExceeded as e:
+            print("BUDGET", e, flush=True)
+            break
+        except Exception as e:  # noqa: BLE001
+            log = {"id": s["id"], "error": str(e)[:500]}
+        log.update(error_kind=s.get("error_kind"), spent_total=spent())
+        print(json.dumps(log), flush=True)
+
+
+def main():
+    load_env(os.path.join(ROOT, ".env"))
+    ap = argparse.ArgumentParser()
+    ap.add_argument("cmd", choices=["plan", "run"])
+    ap.add_argument("--part", type=int, default=0)
+    ap.add_argument("--parts", type=int, default=1)
+    ap.add_argument("--target", type=int, default=200)
+    ap.add_argument("--stop-ledger", type=float, default=27.0, help="ledger USD for this phase (10 USD usage = 27)")
+    a = ap.parse_args()
+    if a.cmd == "plan":
+        import collections
+        p = build_plan()
+        print(len(p), "slots", collections.Counter(x["category"] for x in p))
+        print(collections.Counter(x["object_type"] for x in p), collections.Counter(x.get("error_kind") for x in p))
+        return
+    run(a.part, a.parts, a.target, a.stop_ledger)
+
+
+if __name__ == "__main__":
+    main()