From 229f86260070607260da2c20f079ec32f7891a99 Mon Sep 17 00:00:00 2001 From: Kral Date: Mon, 5 Oct 2026 11:53:07 +0200 Subject: [PATCH] Generator training mode: train pool, eval overlap check, category mix, error-targeted tasks Co-Authored-By: Claude Sonnet 5.5 --- harness/generator.py | 5 +- harness/overlap.py | 100 +++++++++++++++++++++++++++ harness/trainset.py | 157 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 harness/overlap.py create mode 100644 harness/trainset.py diff --git a/harness/generator.py b/harness/generator.py index b4b7f65..fb7e8de 100644 --- a/harness/generator.py +++ b/harness/generator.py @@ -393,7 +393,8 @@ def failure_summary(rep): def generate(task_id, pool, object_type, category, difficulty, model, base_url, run_base, topic=None, - max_repairs=3): + max_repairs=3, extra_check=None): + """extra_check(bundle) -> list of problems; it runs with the static checks (training mode: overlap).""" check_budget() pool_root = os.path.join(ROOT, "tasks_gen", pool) task_dir = os.path.join(pool_root, task_id) @@ -411,6 +412,8 @@ def generate(task_id, pool, object_type, category, difficulty, model, base_url, b = parse_bundle(text) b.get("task", {}).setdefault("category", category) found = {"structure": check_bundle(b), "static": lint_files(b), "abaplint": abaplint_files(b)} + if extra_check and not found["structure"]: + found["extra"] = extra_check(b) except Exception as e: # noqa: BLE001 b, found = None, {"json": [f"invalid JSON: {e}"]} errs = [e for v in found.values() for e in v] diff --git a/harness/overlap.py b/harness/overlap.py new file mode 100644 index 0000000..410b37b --- /dev/null +++ b/harness/overlap.py @@ -0,0 +1,100 @@ +"""Overlap check between a new training task and the eval tasks (spec text and object names). + +A training task is rejected when its spec or its object names are too close to an eval task. +""" +import glob +import json +import math +import os +import re +from collections import Counter + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +STOP = set("the a an of to in is are be and or for with that this it as on by at from not no if then each " + "all any must shall can will when which its into than one two use used using value values " + "object objects method methods class classes table tables field fields return returns".split()) +SPEC_LIMIT = 0.55 # cosine of word counts of the whole spec +GOAL_LIMIT = 0.60 # cosine of the Goal and Business rules sections +NAME_LIMIT = 0.60 # Jaccard of name tokens (placeholder and Z prefix removed) + + +def words(text): + text = re.sub(r"\{\{[Pp]\}\}", " ", text) + return [w for w in re.findall(r"[a-z][a-z0-9]{2,}", text.lower()) if w not in STOP] + + +def cosine(a, b): + ca, cb = Counter(a), Counter(b) + dot = sum(ca[k] * cb[k] for k in ca if k in cb) + na = math.sqrt(sum(v * v for v in ca.values())) + nb = math.sqrt(sum(v * v for v in cb.values())) + return dot / (na * nb) if na and nb else 0.0 + + +def section(spec, start, stop=None): + m = re.search(rf"(?:^|\n)#*\s*\d*\.?\s*{start}.*?\n(.*?)(?=\n#*\s*\d*\.?\s*(?:{stop})|\Z)" if stop + else rf"(?:^|\n)#*\s*\d*\.?\s*{start}.*?\n(.*)", spec, re.S | re.I) + return m.group(1) if m else "" + + +def core_text(spec): + return section(spec, "Goal", "Open questions|Context") + "\n" + section(spec, "Business rules", "Constraints") + + +def name_tokens(name): + n = re.sub(r"\{\{[Pp]\}\}", "", name).lower() + n = re.sub(r"^(?:zcl_|zif_|zcx_|z|/\w+/)", "", n) + return {t for t in re.split(r"[_\W]+", n) if len(t) > 2 and t not in ("test", "tab", "cls", "intf")} + + +def task_names(task): + return [o["name"] for k in ("contract", "seed", "reference") for o in task.get(k, []) if o.get("name")] + + +def load_task(d): + task = json.load(open(os.path.join(d, "task.json"))) + spec = open(os.path.join(d, "spec.md")).read() + return {"id": task.get("id", os.path.basename(d)), "spec": words(spec), "core": words(core_text(spec)), + "names": [name_tokens(n) for n in task_names(task)], + "main": [name_tokens(o["name"]) for o in task.get("contract", []) if o.get("name")]} + + +def load_pool(pool): + out = [] + for d in sorted(glob.glob(os.path.join(ROOT, "tasks_gen", pool, "G*")) + glob.glob(os.path.join(ROOT, "tasks", "T*"))): + try: + out.append(load_task(d)) + except (OSError, ValueError): + pass + return out + + +def jaccard(a, b): + return len(a & b) / len(a | b) if a and b else 0.0 + + +def compare(x, y): + """Scores of two loaded tasks: spec cosine, core cosine, best name Jaccard (contract names).""" + name = max([jaccard(a, b) for a in x["main"] for b in y["main"]] or [0.0]) + return {"spec": cosine(x["spec"], y["spec"]), "core": cosine(x["core"], y["core"]), "name": name} + + +def check(task_dir_or_loaded, pool_tasks): + """Return a list of problems (empty = no overlap) against a list of loaded tasks.""" + x = task_dir_or_loaded if isinstance(task_dir_or_loaded, dict) else load_task(task_dir_or_loaded) + problems = [] + for y in pool_tasks: + if y["id"] == x["id"]: + continue + s = compare(x, y) + if s["spec"] >= SPEC_LIMIT or s["core"] >= GOAL_LIMIT or s["name"] >= NAME_LIMIT: + problems.append((y["id"], s)) + return problems + + +def load_bundle(b): + """Loaded form of a generated bundle ({"task", "files"}) before it is written to disk.""" + task, spec = b["task"], b["files"].get("spec.md", "") + return {"id": task.get("id", "(new)"), "spec": words(spec), "core": words(core_text(spec)), + "names": [name_tokens(n) for n in task_names(task)], + "main": [name_tokens(o["name"]) for o in task.get("contract", []) if o.get("name")]} diff --git a/harness/trainset.py b/harness/trainset.py new file mode 100644 index 0000000..646a690 --- /dev/null +++ b/harness/trainset.py @@ -0,0 +1,157 @@ +"""Training mode of the task generator (stage 2 data). + +Separate pool `tasks_gen/train` (ids G1000+). Same category and object type mix as the eval plan, plus +tasks for the common errors (docs/yol-haritasi.md step 3). A new bundle is checked against all eval tasks +and the earlier training tasks (`harness/overlap.py`); a too close bundle goes back to the model as a +repair message. K variants are not made here (they need the generic_v0 tool schema). + + python3 -m harness.trainset plan + python3 -m harness.trainset run --part 0 --parts 3 [--target 200] [--stop-ledger 27] +""" +import argparse +import glob +import json +import os +import time + +from .adt_client import load_env +from .evalset import SLOTS, RELEASES, accepted_goals +from .generator import ROOT, generate +from .ledger import BudgetExceeded, spent +from . import overlap + +POOL = os.path.join(ROOT, "tasks_gen", "train") +PLAN = os.path.join(POOL, "plan.json") +FIRST_ID = 1000 +RUN_BASE = 32000 # run numbers stay below 36**3 * 1 (proxy prefix rule); 40 per slot +N_SLOTS = 224 # about 12 % more slots than the target: some tasks are not accepted +N_ERROR = 30 +EVAL_HOLD = 3 # eval pool is loaded once per process + +ERROR_HINTS = [ + ("named-type", "The contract has methods with short coded character parameters (zone code, currency code, " + "status flag, region key) of fixed length. A correct solution declares named types for them: " + "TYPE c LENGTH n is not allowed in a method signature."), + ("reserved-word", "The business domain uses concepts that collide with reserved or SQL words (hours, mode, " + "order, date, count, group, value). The contract uses safe field and parameter names (for example " + "work_hours, calc_mode); the solution must also avoid reserved words as field names."), + ("long-names", "The task has at least 10 separate business rules, so a good solution needs at least 10 " + "own unit tests, each for one rule. Every ABAP name stays at 30 characters or less (the model must " + "shorten long descriptive test method names)."), +] +ERROR_TYPES = ["CLAS", "CLAS", "FUNC", "DDLS", "PROG", "CLAS"] # reserved-word fits DDLS and tables well + + +CATEGORY_SHARE = {"A": 10, "B": 15, "C": 15, "D": 10, "E": 15, "F": 10, "G": 10, "I": 5, "H": 10} # eval 8.1 +ERROR_CATEGORY = {"named-type": "A", "reserved-word": "D", "long-names": "C"} + + +def build_plan(): + """Category shares as in the eval plan; object types inside a category as in evalset.SLOTS.""" + share_sum = sum(CATEGORY_SHARE.values()) + items = [] + for cat, share in CATEGORY_SHARE.items(): + k_cat = round(N_SLOTS * share / share_sum) + k_err = sum(1 for k in ERROR_CATEGORY if ERROR_CATEGORY[k] == cat) * N_ERROR // 3 + slots = [x for x in SLOTS if x[0] == cat] + weight = sum(n for _, _, n, _ in slots) + k_rest = k_cat - k_err + quota = [(t, hints, round(k_rest * n / weight)) for _, t, n, hints in slots] + quota[0] = (quota[0][0], quota[0][1], quota[0][2] + k_rest - sum(q[2] for q in quota)) + pos = 0 + for t, hints, k in quota: + for i in range(k): + topic = hints[i % len(hints)] if hints else None + if cat == "G": + topic = (topic + "; " if topic else "") + f"release target {RELEASES[pos % 2]}" + pos += 1 + items.append({"category": cat, "object_type": t, "topic": topic, "frac": pos / (k_cat + 1)}) + for kind, text in ERROR_HINTS: + if ERROR_CATEGORY[kind] != cat: + continue + for i in range(N_ERROR // 3): + t = ERROR_TYPES[i % len(ERROR_TYPES)] + pos += 1 + items.append({"category": cat, "object_type": t, "topic": text, "error_kind": kind, + "frac": pos / (k_cat + 1)}) + items.sort(key=lambda x: (x["frac"], x["category"], x["object_type"])) # interleave: a cut keeps the mix + for n, it in enumerate(items): + it.pop("frac") + it["id"] = f"G{FIRST_ID + n:04d}" + it["run_base"] = RUN_BASE + 40 * n + return items + + +def accepted_count(): + n = 0 + for f in glob.glob(os.path.join(POOL, "G*", "generation.json")): + try: + n += bool(json.load(open(f)).get("accepted")) + except (OSError, ValueError): + pass + return n + + +def run(part, parts, target, stop_ledger): + os.makedirs(POOL, exist_ok=True) + if not os.path.exists(PLAN): + json.dump({"slots": build_plan(), "ledger_at_start": spent(), "stop_ledger": stop_ledger, + "target": target, "created": time.strftime("%F %T")}, open(PLAN, "w"), indent=1) + plan = json.load(open(PLAN)) + stop_at = plan["ledger_at_start"] + plan["stop_ledger"] + base_url = os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1") + evals = overlap.load_pool("eval") + for idx, s in enumerate(plan["slots"]): + if idx % parts != part: + continue + if os.path.exists(os.path.join(POOL, s["id"], "generation.json")): + continue + if accepted_count() >= plan["target"]: + print("TARGET reached", flush=True) + break + if spent() >= stop_at: + print("PHASE LIMIT", spent(), stop_at, flush=True) + break + goals = accepted_goals() + avoid = [g for g in goals if g] # eval goals and all training goals + topic = ((s["topic"] + ". ") if s["topic"] else "Choose a new, realistic business topic. ") + \ + "Do not repeat these existing topics: " + "; ".join(avoid[-170:]) + pool_now = evals + overlap.load_pool("train") + + def extra(b, _pool=pool_now): + hits = overlap.check(overlap.load_bundle(b), _pool) + return [f"Too close to task {i} (similarity spec {sc['spec']:.2f}, rules {sc['core']:.2f}, " + f"names {sc['name']:.2f}). Choose a different business topic and different object names." + for i, sc in hits[:3]] + try: + log = generate(s["id"], "train", s["object_type"], s["category"], 2, "deepseek-v4.1-flash:cloud", + base_url, s["run_base"], topic, extra_check=extra) + except BudgetExceeded as e: + print("BUDGET", e, flush=True) + break + except Exception as e: # noqa: BLE001 + log = {"id": s["id"], "error": str(e)[:500]} + log.update(error_kind=s.get("error_kind"), spent_total=spent()) + print(json.dumps(log), flush=True) + + +def main(): + load_env(os.path.join(ROOT, ".env")) + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["plan", "run"]) + ap.add_argument("--part", type=int, default=0) + ap.add_argument("--parts", type=int, default=1) + ap.add_argument("--target", type=int, default=200) + ap.add_argument("--stop-ledger", type=float, default=27.0, help="ledger USD for this phase (10 USD usage = 27)") + a = ap.parse_args() + if a.cmd == "plan": + import collections + p = build_plan() + print(len(p), "slots", collections.Counter(x["category"] for x in p)) + print(collections.Counter(x["object_type"] for x in p), collections.Counter(x.get("error_kind") for x in p)) + return + run(a.part, a.parts, a.target, a.stop_ledger) + + +if __name__ == "__main__": + main()