erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

In-place E/I: G2 needs a changed seed contract object, G4 skips it; regenerate G0009/G0013/G0016/G0023 as G0188-G0191

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 06:33:33 +02:00
parent 3550f0b651
commit fb576b0954
7 changed files with 33 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
harness/evalset.py
View File

@@ -42,10 +42,13 @@ SLOTS = [
("H", "PROG", 2, ["missing sort order or grouping rule", "contradicting filter rules"]),
("H", "DDLS", 2, ["missing rule for which records count", "contradicting join or filter rule"]),
]
# Replacements for G0009, G0013 (I) and G0016, G0023 (E): those were solvable without the legacy code
# (review 2026-10-03). Same category and object type, new definition (in place).
REGEN = [("I", "CLAS", "G0009"), ("I", "FUNC", "G0013"), ("E", "PROG", "G0016"), ("E", "CLAS", "G0023")]
# Category K: variants of accepted tasks (free text or incomplete input + tool schema generic_v0)
K_SOURCES = [("G0004", "free_text"), ("G0007", "incomplete"), ("G0013", "free_text"), ("G0014", "incomplete"),
K_SOURCES = [("G0004", "free_text"), ("G0007", "incomplete"), ("G0026", "free_text"), ("G0014", "incomplete"),
("G0017", "free_text"), ("G0020", "incomplete"), ("G0022", "free_text"), ("G0012", "incomplete"),
("G0016", "free_text"), ("G0023", "incomplete")]
("G0003", "free_text"), ("G0010", "incomplete")]
RELEASES = ["v702", "v740sp05"]
@@ -77,6 +80,10 @@ def plan():
out.append({"id": f"G{FIRST_ID + n:04d}", "category": "K", "k_from": src, "style": style,
"run_base": RUN_BASE + 40 * n})
n += 1
for cat, otype, old in REGEN:
out.append({"id": f"G{FIRST_ID + n:04d}", "category": cat, "object_type": otype, "topic": None,
"replaces": old, "run_base": RUN_BASE + 40 * n})
n += 1
return out

6
harness/generator.py
View File

@@ -174,6 +174,11 @@ def lint_files(b):
errs.append(f"{rel}: name {name} has {len(name)} characters (ABAP max 30)")
t = b.get("task", {})
oos = {n.upper() for n in t.get("out_of_scope", [])}
if t.get("category") in ("E", "I"):
seed_names = {o.get("name", "").upper() for o in t.get("seed", [])}
if not any(c.get("name", "").upper() in seed_names for c in t.get("contract", [])):
errs.append(f"category {t['category']}: the contract object must be the seed object that the model "
"changes in place (same name in seed, contract and reference)")
test_names = {o.get("name", "").upper() for o in t.get("hidden_tests", [])}
for o in t.get("reference", []):
name = o.get("name", "")
@@ -384,6 +389,7 @@ def generate(task_id, pool, object_type, category, difficulty, model, base_url,
messages.append({"role": "assistant", "content": text})
try:
b = parse_bundle(text)
b.get("task", {}).setdefault("category", category)
found = {"structure": check_bundle(b), "static": lint_files(b), "abaplint": abaplint_files(b)}
except Exception as e: # noqa: BLE001
b, found = None, {"json": [f"invalid JSON: {e}"]}

11
harness/runner.py
View File

@@ -267,12 +267,21 @@ class Runner:
if missing:
g2 = False
g2_detail.append(f"{c['name']}: CDS elements missing: {sorted(missing)}")
# Categories E and I: the contract object is a seed object (refactor or fix in place). It must
# change (else the null agent passes on the legacy code), and it is not out of scope.
contract_upper = {c["name"].upper() for c in contract}
for c in contract:
if c["name"].upper() in seed_src and c["type"] != "FUGR":
now = self._source(mcp, c["type"], c["name"], c.get("functionGroup"))
if _norm(now) == _norm(seed_src[c["name"].upper()]):
g2 = False
g2_detail.append(f"{c['name']}: the seed object is not changed")
g["G2_contract"] = g2 and g["G1_active"]
rep["contract_check"] = {"g1": g1, "g2": g2_detail}
g["G4_out_of_scope"] = all(
_norm(self._source(mcp, o["type"], o["name"], o.get("functionGroup")))
== _norm(seed_src[o["name"].upper()])
for o in seed)
for o in seed if o["name"].upper() not in contract_upper)
atc = []
for c in contract:
if g["G1_active"]:

4
tasks_gen/eval/G0009/review.json
View File

@@ -1,6 +1,6 @@
{
"decision": "flag",
"note": "Category I is weak: the model writes a new class from complete rules; the legacy class need not be read. Options: move to A, or change to an in-place fix of the legacy class. Survived mutant (round dec 2->3) is equivalent.",
"decision": "reject",
"note": "Solvable without the legacy code. Replaced by G0188 (in-place I/E, Kral 2026-10-03).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {

4
tasks_gen/eval/G0013/review.json
View File

@@ -1,6 +1,6 @@
{
"decision": "flag",
"note": "Same as G0009: new FM from complete rules; old FM need not be read. Generator prompt for I fixed (in-place fix, spec gives only symptoms). Options: move to A, or regenerate as I.",
"decision": "reject",
"note": "Solvable without the legacy code. Replaced by G0189 (in-place I/E, Kral 2026-10-03).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {

4
tasks_gen/eval/G0016/review.json
View File

@@ -1,6 +1,6 @@
{
"decision": "flag",
"note": "E is weak: new program with complete rules; legacy need not be read. Generator prompt for E fixed (in-place refactor, spec without rules). Options: keep as B-like report task, or regenerate as E.",
"decision": "reject",
"note": "Solvable without the legacy code. Replaced by G0190 (in-place I/E, Kral 2026-10-03).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {

4
tasks_gen/eval/G0023/review.json
View File

@@ -1,6 +1,6 @@
{
"decision": "flag",
"note": "E is weak: new class from complete rules; legacy need not be read. Generator prompt for E fixed. Options: move to A, or regenerate as E.",
"decision": "reject",
"note": "Solvable without the legacy code. Replaced by G0191 (in-place I/E, Kral 2026-10-03).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {