diff --git a/harness/evalset.py b/harness/evalset.py index a2d7d3b..8e5e06c 100644 --- a/harness/evalset.py +++ b/harness/evalset.py @@ -42,10 +42,13 @@ SLOTS = [ ("H", "PROG", 2, ["missing sort order or grouping rule", "contradicting filter rules"]), ("H", "DDLS", 2, ["missing rule for which records count", "contradicting join or filter rule"]), ] +# Replacements for G0009, G0013 (I) and G0016, G0023 (E): those were solvable without the legacy code +# (review 2026-10-03). Same category and object type, new definition (in place). +REGEN = [("I", "CLAS", "G0009"), ("I", "FUNC", "G0013"), ("E", "PROG", "G0016"), ("E", "CLAS", "G0023")] # Category K: variants of accepted tasks (free text or incomplete input + tool schema generic_v0) -K_SOURCES = [("G0004", "free_text"), ("G0007", "incomplete"), ("G0013", "free_text"), ("G0014", "incomplete"), +K_SOURCES = [("G0004", "free_text"), ("G0007", "incomplete"), ("G0026", "free_text"), ("G0014", "incomplete"), ("G0017", "free_text"), ("G0020", "incomplete"), ("G0022", "free_text"), ("G0012", "incomplete"), - ("G0016", "free_text"), ("G0023", "incomplete")] + ("G0003", "free_text"), ("G0010", "incomplete")] RELEASES = ["v702", "v740sp05"] @@ -77,6 +80,10 @@ def plan(): out.append({"id": f"G{FIRST_ID + n:04d}", "category": "K", "k_from": src, "style": style, "run_base": RUN_BASE + 40 * n}) n += 1 + for cat, otype, old in REGEN: + out.append({"id": f"G{FIRST_ID + n:04d}", "category": cat, "object_type": otype, "topic": None, + "replaces": old, "run_base": RUN_BASE + 40 * n}) + n += 1 return out diff --git a/harness/generator.py b/harness/generator.py index 7524827..506f144 100644 --- a/harness/generator.py +++ b/harness/generator.py @@ -174,6 +174,11 @@ def lint_files(b): errs.append(f"{rel}: name {name} has {len(name)} characters (ABAP max 30)") t = b.get("task", {}) oos = {n.upper() for n in t.get("out_of_scope", [])} + if t.get("category") in ("E", "I"): + seed_names = {o.get("name", "").upper() for o in t.get("seed", [])} + if not any(c.get("name", "").upper() in seed_names for c in t.get("contract", [])): + errs.append(f"category {t['category']}: the contract object must be the seed object that the model " + "changes in place (same name in seed, contract and reference)") test_names = {o.get("name", "").upper() for o in t.get("hidden_tests", [])} for o in t.get("reference", []): name = o.get("name", "") @@ -384,6 +389,7 @@ def generate(task_id, pool, object_type, category, difficulty, model, base_url, messages.append({"role": "assistant", "content": text}) try: b = parse_bundle(text) + b.get("task", {}).setdefault("category", category) found = {"structure": check_bundle(b), "static": lint_files(b), "abaplint": abaplint_files(b)} except Exception as e: # noqa: BLE001 b, found = None, {"json": [f"invalid JSON: {e}"]} diff --git a/harness/runner.py b/harness/runner.py index b6252ad..3c30fab 100644 --- a/harness/runner.py +++ b/harness/runner.py @@ -267,12 +267,21 @@ class Runner: if missing: g2 = False g2_detail.append(f"{c['name']}: CDS elements missing: {sorted(missing)}") + # Categories E and I: the contract object is a seed object (refactor or fix in place). It must + # change (else the null agent passes on the legacy code), and it is not out of scope. + contract_upper = {c["name"].upper() for c in contract} + for c in contract: + if c["name"].upper() in seed_src and c["type"] != "FUGR": + now = self._source(mcp, c["type"], c["name"], c.get("functionGroup")) + if _norm(now) == _norm(seed_src[c["name"].upper()]): + g2 = False + g2_detail.append(f"{c['name']}: the seed object is not changed") g["G2_contract"] = g2 and g["G1_active"] rep["contract_check"] = {"g1": g1, "g2": g2_detail} g["G4_out_of_scope"] = all( _norm(self._source(mcp, o["type"], o["name"], o.get("functionGroup"))) == _norm(seed_src[o["name"].upper()]) - for o in seed) + for o in seed if o["name"].upper() not in contract_upper) atc = [] for c in contract: if g["G1_active"]: diff --git a/tasks_gen/eval/G0009/review.json b/tasks_gen/eval/G0009/review.json index 6f4b158..3828757 100644 --- a/tasks_gen/eval/G0009/review.json +++ b/tasks_gen/eval/G0009/review.json @@ -1,6 +1,6 @@ { - "decision": "flag", - "note": "Category I is weak: the model writes a new class from complete rules; the legacy class need not be read. Options: move to A, or change to an in-place fix of the legacy class. Survived mutant (round dec 2->3) is equivalent.", + "decision": "reject", + "note": "Solvable without the legacy code. Replaced by G0188 (in-place I/E, Kral 2026-10-03).", "by": "Claude", "date": "2026-10-03", "failed_checks": { diff --git a/tasks_gen/eval/G0013/review.json b/tasks_gen/eval/G0013/review.json index 4dc8137..512a293 100644 --- a/tasks_gen/eval/G0013/review.json +++ b/tasks_gen/eval/G0013/review.json @@ -1,6 +1,6 @@ { - "decision": "flag", - "note": "Same as G0009: new FM from complete rules; old FM need not be read. Generator prompt for I fixed (in-place fix, spec gives only symptoms). Options: move to A, or regenerate as I.", + "decision": "reject", + "note": "Solvable without the legacy code. Replaced by G0189 (in-place I/E, Kral 2026-10-03).", "by": "Claude", "date": "2026-10-03", "failed_checks": { diff --git a/tasks_gen/eval/G0016/review.json b/tasks_gen/eval/G0016/review.json index fd2da9b..f16ecb7 100644 --- a/tasks_gen/eval/G0016/review.json +++ b/tasks_gen/eval/G0016/review.json @@ -1,6 +1,6 @@ { - "decision": "flag", - "note": "E is weak: new program with complete rules; legacy need not be read. Generator prompt for E fixed (in-place refactor, spec without rules). Options: keep as B-like report task, or regenerate as E.", + "decision": "reject", + "note": "Solvable without the legacy code. Replaced by G0190 (in-place I/E, Kral 2026-10-03).", "by": "Claude", "date": "2026-10-03", "failed_checks": { diff --git a/tasks_gen/eval/G0023/review.json b/tasks_gen/eval/G0023/review.json index 7e85b3c..3821f74 100644 --- a/tasks_gen/eval/G0023/review.json +++ b/tasks_gen/eval/G0023/review.json @@ -1,6 +1,6 @@ { - "decision": "flag", - "note": "E is weak: new class from complete rules; legacy need not be read. Generator prompt for E fixed. Options: move to A, or regenerate as E.", + "decision": "reject", + "note": "Solvable without the legacy code. Replaced by G0191 (in-place I/E, Kral 2026-10-03).", "by": "Claude", "date": "2026-10-03", "failed_checks": {