Harness: runner, generator, tasks T01 T13 T14 T15, CLAUDE.md

harness/adt_client.py

@@ -0,0 +1,72 @@
+"""Direct ADT HTTP access. Only for teardown (deletion). The model never uses it.
+
+Credentials come from the environment (or .env): A4H_URL, A4H_USER, A4H_PASSWORD, A4H_CLIENT.
+"""
+import base64
+import http.cookiejar
+import os
+import re
+import urllib.request
+from xml.sax.saxutils import quoteattr
+
+DEL_REQ = "application/vnd.sap.adt.deletion.request.v1+xml"
+DEL_RES = "application/vnd.sap.adt.deletion.response.v1+xml"
+
+
+def load_env(path=".env"):
+    if os.path.exists(path):
+        for line in open(path):
+            line = line.strip()
+            if line and not line.startswith("#") and "=" in line:
+                k, v = line.split("=", 1)
+                os.environ.setdefault(k.strip(), v.strip())
+
+
+class AdtClient:
+    def __init__(self):
+        load_env()
+        self.base = os.environ["A4H_URL"].rstrip("/")
+        self.client = os.environ.get("A4H_CLIENT", "001")
+        auth = f'{os.environ["A4H_USER"]}:{os.environ["A4H_PASSWORD"]}'
+        self.auth = "Basic " + base64.b64encode(auth.encode()).decode()
+        self.opener = urllib.request.build_opener(
+            urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
+        self.csrf = None
+
+    def _req(self, method, path, body=None, headers=None):
+        url = f"{self.base}{path}{'&' if '?' in path else '?'}sap-client={self.client}"
+        h = {"Authorization": self.auth}
+        if self.csrf:
+            h["x-csrf-token"] = self.csrf
+        h.update(headers or {})
+        req = urllib.request.Request(url, body.encode() if body else None, h, method=method)
+        with self.opener.open(req, timeout=300) as r:
+            return r.status, dict(r.headers), r.read().decode()
+
+    def fetch_csrf(self):
+        _, hdr, _ = self._req("GET", "/sap/bc/adt/discovery",
+                              headers={"x-csrf-token": "fetch", "Accept": "*/*"})
+        self.csrf = hdr.get("x-csrf-token") or hdr.get("X-CSRF-Token")
+
+    def delete(self, uris):
+        """Delete objects by ADT URI. Returns {uri: (deleted, message)}."""
+        if not uris:
+            return {}
+        if not self.csrf:
+            self.fetch_csrf()
+        objs = "".join(f"<del:object adtcore:uri={quoteattr(u)}><del:transportNumber/></del:object>"
+                       for u in uris)
+        body = ('<?xml version="1.0" encoding="UTF-8"?>'
+                '<del:deletionRequest xmlns:del="http://www.sap.com/adt/deletion" '
+                'xmlns:adtcore="http://www.sap.com/adt/core">' + objs + '</del:deletionRequest>')
+        _, _, text = self._req("POST", "/sap/bc/adt/deletion/delete", body,
+                               {"Content-Type": DEL_REQ, "Accept": DEL_RES})
+        out = {}
+        for m in re.finditer(r'<del:object\b([^>]*)>(.*?)</del:object>', text, re.S):
+            attrs, inner = m.group(1), m.group(2)
+            uri = re.search(r'adtcore:uri="([^"]+)"', attrs)
+            ok = re.search(r'del:isDeleted="([^"]+)"', attrs)
+            msg = re.search(r'<del:text>(.*?)</del:text>', inner, re.S)
+            out[uri.group(1) if uri else "?"] = (ok is not None and ok.group(1) == "true",
+                                                 msg.group(1) if msg else "")
+        return out

harness/agents.py

@@ -0,0 +1,139 @@
+"""Agents: oracle (reference solution), null (does nothing), llm (OpenAI-compatible tool loop)."""
+import json
+import os
+import time
+import urllib.request
+
+from .ledger import add_usage, check_budget
+from .proxy import BudgetExceeded
+
+SYSTEM_PROMPT = """You are an ABAP developer. You implement a plan on an SAP system with the tools.
+
+Rules:
+- Read the plan. Implement the contract exactly. Do not change a public signature in the contract.
+- You make the craft decisions: table types, keys, access paths, SQL strategy, exception design,
+  and syntax that fits the release target.
+- If a business rule is missing, or two rules contradict, stop. Do not create objects.
+  Write the gap in your report.
+- Do not change an object in the out-of-scope list.
+- Create a new object in package $TMP with sap_create_object. Then write the source with
+  sap_push_source.
+- Read an object that you do not know before you use it.
+- After each write, check the result. If the same check fails two times, stop and report.
+- Do not add behavior that the plan does not ask for.
+
+At the end, write a short report:
+1. Decisions: one line for each craft decision, with the reason.
+2. Objects: object, type, action, reason.
+3. Verification: what you checked and the result.
+4. Deviations and open points.
+"""
+
+
+class NullAgent:
+    name = "null"
+
+    def run(self, task, proxy):
+        proxy.note("final", "No action.")
+        return "No action."
+
+
+class OracleAgent:
+    """Writes the reference solution. Validates harness and scoring (expected score ~ max)."""
+    name = "oracle"
+
+    def run(self, task, proxy):
+        for o in task.objects("reference"):
+            ident = {"objectType": o["type"], "objectName": o["name"]}
+            if o.get("functionGroup"):
+                ident["functionGroup"] = o["functionGroup"]
+            proxy.call("sap_create_object", dict(ident, packageName="$TMP",
+                                                 description=o.get("description", o["name"])[:60]))
+            if o.get("source"):
+                proxy.call("sap_push_source", dict(ident, source=o["source"]))
+            if o.get("testclasses_source"):
+                proxy.call("sap_push_source", dict(ident, includeType="testclasses",
+                                                   source=o["testclasses_source"]))
+        proxy.note("final", "Reference solution written.")
+        return "Reference solution written."
+
+
+class LlmAgent:
+    """OpenAI-compatible chat completions with tool calls (Ollama, MLX server, vLLM, ...).
+
+    Local models are slow: no time limit by default (tool-call budget limits the run).
+    One request at a time; tool calls run in sequence.
+    """
+
+    def __init__(self, model, base_url=None, api_key=None, max_turns=80, temperature=0.2,
+                 max_seconds=None):
+        self.model = model
+        self.name = f"llm:{model}"
+        self.base_url = (base_url or os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1")).rstrip("/")
+        self.api_key = api_key or os.environ.get("LLM_API_KEY", "none")
+        self.max_turns = max_turns
+        self.temperature = temperature
+        self.max_seconds = max_seconds
+        self.request_timeout = 3600  # local models are slow; a hung request still ends
+
+    def _chat(self, messages, tools):
+        body = {"model": self.model, "messages": messages, "tools": tools,
+                "temperature": self.temperature, "parallel_tool_calls": False}
+        req = urllib.request.Request(f"{self.base_url}/chat/completions", json.dumps(body).encode(),
+                                     {"Content-Type": "application/json",
+                                      "Authorization": f"Bearer {self.api_key}"})
+        last = None
+        for attempt in range(4):  # model server errors (HTTP 5xx, timeouts): retry with backoff
+            try:
+                with urllib.request.urlopen(req, timeout=self.request_timeout) as r:
+                    data = json.loads(r.read().decode())
+                    return data["choices"][0]["message"], data.get("usage", {})
+            except Exception as e:  # noqa: BLE001
+                last = e
+                code = getattr(e, "code", None)
+                if code is not None and code < 500 and code != 429:
+                    raise
+                time.sleep(10 * (attempt + 1))
+        raise RuntimeError(f"model request failed after retries: {last}")
+
+    def run(self, task, proxy):
+        tools = [{"type": "function", "function": {"name": t["name"],
+                                                   "description": t.get("description", ""),
+                                                   "parameters": t.get("inputSchema", {})}}
+                 for t in proxy.schemas()]
+        messages = [{"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": task.spec}]
+        if ":cloud" in self.model:
+            check_budget()
+        final = ""
+        start = time.time()
+        for _ in range(self.max_turns):
+            if self.max_seconds and time.time() - start > self.max_seconds:
+                final = f"Stopped: time budget exceeded ({self.max_seconds} s)."
+                break
+            try:
+                msg, usage = self._chat(messages, tools)
+                add_usage(self.model, usage, kind="run", ref=proxy.prefix)
+            except RuntimeError as e:
+                final = f"Stopped: {e}"
+                break
+            proxy.note("assistant", {"content": msg.get("content"),
+                                     "tool_calls": msg.get("tool_calls"), "usage": usage})
+            messages.append({k: v for k, v in msg.items() if k in ("role", "content", "tool_calls")})
+            calls = msg.get("tool_calls") or []
+            if not calls:
+                final = msg.get("content") or ""
+                break
+            try:
+                for c in calls:
+                    fn = c["function"]
+                    args = fn.get("arguments") or "{}"
+                    args = json.loads(args) if isinstance(args, str) else args
+                    err, text = proxy.call(fn["name"], args)
+                    messages.append({"role": "tool", "tool_call_id": c.get("id", ""),
+                                     "content": ("ERROR: " if err else "") + text[:12000]})
+            except BudgetExceeded as e:
+                final = f"Stopped: budget exceeded ({e})."
+                break
+        proxy.note("final", final)
+        return final

harness/cli.py

@@ -0,0 +1,89 @@
+"""CLI.
+  python3 -m harness.cli run T01 --agent oracle|null|llm [--model NAME] [--run N] [--no-teardown]
+  python3 -m harness.cli teardown runs/<run_dir>
+  python3 -m harness.cli cleanup-list Z900001_ runs/cleanup_probe
+"""
+import argparse
+import json
+import os
+
+from .agents import LlmAgent, NullAgent, OracleAgent
+from .adt_client import load_env
+from .runner import Runner, delete_uris
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+def next_run_no():
+    runs = os.path.join(ROOT, "runs")
+    nums = [int(d[:3]) for d in os.listdir(runs) if d[:3].isdigit()]
+    return max(nums, default=0) + 1
+
+
+def main():
+    load_env(os.path.join(ROOT, ".env"))
+    ap = argparse.ArgumentParser()
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    r = sub.add_parser("run")
+    r.add_argument("task")
+    r.add_argument("--agent", default="oracle", choices=["oracle", "null", "llm"])
+    r.add_argument("--model")
+    r.add_argument("--base-url")
+    r.add_argument("--run", type=int)
+    r.add_argument("--no-teardown", action="store_true")
+    r.add_argument("--max-minutes", type=float, help="optional time limit for the llm agent (default: none)")
+    t = sub.add_parser("teardown")
+    t.add_argument("run_dir")
+    sub.add_parser("teardown-all", help="delete objects of every run that has delete_uris.json and no teardown.json")
+    rs = sub.add_parser("rescore", help="score an existing run again (objects must still exist)")
+    rs.add_argument("run_dir")
+    c = sub.add_parser("cleanup-list", help="write delete_uris.json for all objects with a prefix")
+    c.add_argument("prefix")
+    c.add_argument("out_dir")
+    a = ap.parse_args()
+
+    if a.cmd == "run":
+        agent = {"oracle": OracleAgent, "null": NullAgent}.get(a.agent)
+        agent = agent() if agent else LlmAgent(a.model, a.base_url,
+                                               max_seconds=a.max_minutes * 60 if a.max_minutes else None)
+        runner = Runner(os.path.join(ROOT, "tasks"), os.path.join(ROOT, "runs"))
+        rep, run_dir = runner.run(a.task, agent, a.run or next_run_no(), teardown=not a.no_teardown)
+        print(json.dumps({k: rep[k] for k in ("task", "agent", "prefix", "gates", "score",
+                                               "tool_calls", "seconds", "teardown")}, indent=1))
+        print("run dir:", run_dir)
+    elif a.cmd == "cleanup-list":
+        from .mcp_client import McpClient
+        from .runner import DELETE_ORDER, Runner as R
+        os.makedirs(a.out_dir, exist_ok=True)
+        with McpClient() as m:
+            objs = R("", "")._objects_with_prefix(m, a.prefix.upper())
+        objs.sort(key=lambda o: DELETE_ORDER.index(o["objectType"]) if o["objectType"] in DELETE_ORDER else 99)
+        json.dump([o["uri"] for o in objs], open(os.path.join(a.out_dir, "delete_uris.json"), "w"), indent=1)
+        print(len(objs), "objects ->", os.path.join(a.out_dir, "delete_uris.json"))
+    elif a.cmd == "rescore":
+        rd = os.path.abspath(a.run_dir)
+        name = os.path.basename(rd)
+        run_no, task_id, agent_name = name.split("_", 2)
+        runner = Runner(os.path.join(ROOT, "tasks"), os.path.join(ROOT, "runs"))
+        agent = NullAgent()
+        agent.name = agent_name
+        rep, _ = runner.run(task_id, agent, int(run_no), teardown=True, rescore_dir=rd)
+        print(json.dumps({k: rep.get(k) for k in ("task", "agent", "gates", "score", "tool_calls",
+                                                   "activations", "max_fail_streak", "agent_seconds")}, indent=1))
+    elif a.cmd == "teardown-all":
+        runs = os.path.join(ROOT, "runs")
+        for d in sorted(os.listdir(runs)):
+            rd = os.path.join(runs, d)
+            if os.path.exists(os.path.join(rd, "delete_uris.json")) and not os.path.exists(os.path.join(rd, "teardown.json")):
+                res = delete_uris(json.load(open(os.path.join(rd, "delete_uris.json"))))
+                json.dump(res, open(os.path.join(rd, "teardown.json"), "w"), indent=1)
+                print(d, sum(v["deleted"] for v in res.values()), "/", len(res), "deleted")
+    else:
+        uris = json.load(open(os.path.join(a.run_dir, "delete_uris.json")))
+        res = delete_uris(uris)
+        json.dump(res, open(os.path.join(a.run_dir, "teardown.json"), "w"), indent=1)
+        print(json.dumps(res, indent=1))
+
+
+if __name__ == "__main__":
+    main()

harness/generator.py

@@ -0,0 +1,242 @@
+"""Task generator: a cloud model writes a task bundle; the harness validates it (oracle = 100, null = 0).
+
+  python3 -m harness.generator --id G0001 --pool eval --object-type CLAS --category C --difficulty 2
+"""
+import argparse
+import json
+import os
+import re
+import time
+import urllib.request
+
+from .adt_client import load_env
+from .agents import NullAgent, OracleAgent
+from .ledger import add_usage, check_budget
+from .runner import Runner
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+EXAMPLE_FOR = {"CLAS": "T01", "INTF": "T01", "FUNC": "T13", "PROG": "T14", "DDLS": "T15", "TABL": "T15"}
+CATEGORIES = {
+    "A": "pure logic in a new class (language, OO design, Clean ABAP)",
+    "B": "database access (ABAP SQL, CDS) with test doubles",
+    "C": "internal table craft (table kind, keys, access path)",
+    "D": "exception design (class-based exceptions, messages)",
+    "E": "refactoring legacy code (seed contains legacy code; behavior must stay the same)",
+    "F": "use an unknown object: the spec names a seed object but not its signature; the model must read it",
+    "G": "old release target (v702 or v740sp05): no newer syntax",
+    "I": "bug fix: the seed contains a faulty object; the spec describes the symptom",
+}
+
+SYSTEM = """You write evaluation tasks for an ABAP developer model. Each task is a bundle of files.
+The model gets only spec.md and works on an SAP ABAP Platform 2025 system (SAP_BASIS 816, client 001)
+through ADT tools. The harness installs the seed objects, runs the model, then checks the result with
+hidden ABAP Unit tests, ATC, and abaplint.
+
+Rules for the bundle:
+- Use the placeholder {{P}} (upper case) and {{p}} (lower case) at the start of EVERY object name.
+  The harness replaces it with a run prefix of 9 characters (for example Z005P001_).
+  Name length after replacement: classes, programs, function modules, CDS entities max 30;
+  database tables max 16; function groups max 26.
+- All objects are in package $TMP. Do not use transports.
+- Do not use SAP application module data (no FI, SD, MM tables). Use generic business domains and
+  only objects that exist in every ABAP Platform system (language, ABAP SQL, CDS, CL_ABAP_*, CL_SALV_TABLE,
+  CL_OSQL_TEST_ENVIRONMENT, CL_CDS_TEST_ENVIRONMENT). Seed your own tables and data if needed.
+- No dynpro (CALL SCREEN), no SmartForms, no BAdI, no RAP behavior definitions.
+- spec.md uses Simplified Technical English and these sections in this order:
+  1. Goal, 2. Open questions (write "None."), 3. Context, 4. Contract, 5. Business rules,
+  6. Constraints (release target, coding standards, out of scope), 7. Acceptance.
+  The Contract fixes every public name the hidden tests use: object names, method signatures,
+  function module parameters, report parameters, ALV column names, CDS element names.
+  The Business rules are complete and unambiguous. Every rule is checked by at least one hidden test.
+  Do not tell the model HOW to implement (no table kinds, no SQL). Craft decisions belong to the model.
+- Seed objects: TABL as DDL source ("define table ..."), FUGR without source, FUNC with "functionGroup"
+  and full source including the signature in the FUNCTION statement, classes with full source.
+  Seed table data: a seed class that implements IF_OO_ADT_CLASSRUN, with "run": true.
+- Hidden tests: one global class, "FOR TESTING DURATION SHORT RISK LEVEL HARMLESS", 5 to 12 test
+  methods. Use only the public contract. Function modules: CALL FUNCTION with EXCEPTIONS.
+  Reports: SUBMIT ... AND RETURN with cl_salv_bs_runtime_info=>set( display = abap_false
+  metadata = abap_false data = abap_true ) and get_data_ref. CDS: cl_cds_test_environment.
+- Reference solution: correct, Clean ABAP, methods below 40 statements, passes all hidden tests,
+  no ATC priority 1 or 2 findings (for example: pass large parameters by reference).
+  Include the model's expected own tests: for classes a "testclasses_file" (local test classes);
+  for reports local test classes inside the program; for function modules and CDS a global test class.
+- task.json keys: id, category, object_type, difficulty, release_target, expected_outcome ("implement"),
+  budget {max_tool_calls, max_activations}, seed[], contract[], out_of_scope[], hidden_tests[],
+  reference[], craft_checks[]. Contract entries: CLAS {"implements"} optional; FUNC {"functionGroup",
+  "params":[{"name","type"}]}; PROG {"parameters":[...]}; DDLS {"fields":[...]}.
+
+Return ONLY one JSON object, no markdown fence:
+{"task": <task.json object>, "files": {"<relative path>": "<file content>", ...}}
+"""
+
+
+def bundle_of(task_dir):
+    files = {}
+    for base, _, names in os.walk(task_dir):
+        for n in names:
+            if n.startswith(".") or n == "generation.json":
+                continue
+            path = os.path.join(base, n)
+            rel = os.path.relpath(path, task_dir)
+            if rel != "task.json":
+                files[rel] = open(path).read()
+    return {"task": json.load(open(os.path.join(task_dir, "task.json"))), "files": files}
+
+
+def chat(model, messages, base_url):
+    body = {"model": model, "messages": messages, "temperature": 0.7}
+    req = urllib.request.Request(f"{base_url}/chat/completions", json.dumps(body).encode(),
+                                 {"Content-Type": "application/json", "Authorization": "Bearer none"})
+    for attempt in range(4):
+        try:
+            with urllib.request.urlopen(req, timeout=1800) as r:
+                data = json.loads(r.read().decode())
+            add_usage(model, data.get("usage", {}), kind="generate")
+            return data["choices"][0]["message"].get("content") or ""
+        except Exception as e:  # noqa: BLE001
+            if getattr(e, "code", 500) < 500 and getattr(e, "code", 500) != 429:
+                raise
+            time.sleep(15 * (attempt + 1))
+    raise RuntimeError("generation request failed")
+
+
+def parse_bundle(text):
+    text = text.strip()
+    text = re.sub(r"^```(json)?\s*|\s*```$", "", text)
+    start, end = text.find("{"), text.rfind("}")
+    return json.loads(text[start:end + 1])
+
+
+def check_bundle(b):
+    errs = []
+    t, files = b.get("task", {}), b.get("files", {})
+    for k in ("seed", "contract", "hidden_tests", "reference"):
+        if k not in t:
+            errs.append(f"task.json misses '{k}'")
+    if "spec.md" not in files:
+        errs.append("spec.md missing")
+    for k in ("seed", "hidden_tests", "reference"):
+        for o in t.get(k, []):
+            for fk in ("file", "testclasses_file"):
+                if fk in o and o[fk] not in files:
+                    errs.append(f"{k}: file {o[fk]} missing")
+            if not o.get("name", "").startswith("{{P}}"):
+                errs.append(f"{k}: name {o.get('name')} does not start with {{{{P}}}}")
+            limit = 16 if o.get("type") == "TABL" else 26 if o.get("type") == "FUGR" else 30
+            if len(o.get("name", "").replace("{{P}}", "Z0000000_")) > limit:
+                errs.append(f"{k}: name {o.get('name')} too long (max {limit})")
+    if not t.get("hidden_tests"):
+        errs.append("no hidden test class")
+    return errs
+
+
+def write_bundle(b, task_dir, task_id):
+    os.makedirs(task_dir, exist_ok=True)
+    b["task"]["id"] = task_id
+    json.dump(b["task"], open(os.path.join(task_dir, "task.json"), "w"), indent=2)
+    for rel, content in b["files"].items():
+        path = os.path.join(task_dir, rel)
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        open(path, "w").write(content)
+
+
+def validate(pool_root, task_id, run_base):
+    r = Runner(pool_root, os.path.join(ROOT, "runs", "gen"))
+    os.makedirs(os.path.join(ROOT, "runs", "gen"), exist_ok=True)
+    rep_o, dir_o = r.run(task_id, OracleAgent(), run_base)
+    rep_o["_dir"] = dir_o
+    rep_n, _ = r.run(task_id, NullAgent(), run_base + 1)
+    return rep_o, rep_n
+
+
+def write_errors(run_dir):
+    """Failed writes and activation messages of the oracle run."""
+    out = []
+    path = os.path.join(run_dir or "", "trajectory.jsonl")
+    if not os.path.exists(path):
+        return out
+    for line in open(path):
+        e = json.loads(line)
+        if e.get("tool") in ("sap_create_object", "sap_push_source") and (
+                e["is_error"] or '"success":false' in e["result"].replace(" ", "")):
+            out.append({"tool": e["tool"], "object": e["args"].get("objectName"),
+                        "include": e["args"].get("includeType", "main"), "result": e["result"][:1500]})
+    return out
+
+
+def failure_summary(rep):
+    out = {"reference_write_errors": write_errors(rep.get("_dir")),"gates": rep.get("gates"), "score": rep.get("score"), "setup": rep.get("setup"),
+           "atc": rep.get("atc"), "abaplint": rep.get("abaplint"), "own_tests": rep.get("own_tests"),
+           "hidden_failed": [d for d in rep.get("hidden_tests", {}).get("detail", []) if not d["ok"]],
+           "hidden_install": rep.get("hidden_tests", {}).get("install")}
+    return json.dumps(out)[:6000]
+
+
+def generate(task_id, pool, object_type, category, difficulty, model, base_url, run_base, topic=None,
+             max_repairs=3):
+    check_budget()
+    pool_root = os.path.join(ROOT, "tasks_gen", pool)
+    task_dir = os.path.join(pool_root, task_id)
+    example = bundle_of(os.path.join(ROOT, "tasks", EXAMPLE_FOR[object_type]))
+    ask = (f"Write one new task.\nObject type of the main contract object: {object_type}.\n"
+           f"Skill category {category}: {CATEGORIES[category]}.\nDifficulty {difficulty} of 3.\n"
+           + (f"Topic idea: {topic}\n" if topic else "Choose a new, realistic business topic.\n")
+           + "Here is an example bundle of a different task (same format):\n" + json.dumps(example))
+    messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": ask}]
+    log = {"id": task_id, "pool": pool, "object_type": object_type, "category": category, "attempts": []}
+    for attempt in range(max_repairs + 1):
+        text = chat(model, messages, base_url)
+        messages.append({"role": "assistant", "content": text})
+        try:
+            b = parse_bundle(text)
+            errs = check_bundle(b)
+        except Exception as e:  # noqa: BLE001
+            b, errs = None, [f"invalid JSON: {e}"]
+        if errs:
+            log["attempts"].append({"stage": "bundle", "errors": errs})
+            messages.append({"role": "user", "content": "Fix these problems and return the full bundle again:\n"
+                             + "\n".join(errs)})
+            continue
+        b["task"].setdefault("object_type", object_type)
+        b["task"].setdefault("category", category)
+        write_bundle(b, task_dir, task_id)
+        rep_o, rep_n = validate(pool_root, task_id, run_base + 2 * attempt)
+        so, sn = (rep_o.get("score") or {}).get("total"), (rep_n.get("score") or {}).get("total")
+        log["attempts"].append({"stage": "validate", "oracle": so, "null": sn})
+        if so == 100 and sn == 0:
+            log["accepted"] = True
+            break
+        messages.append({"role": "user", "content":
+                         "The harness ran your reference solution (oracle) and an empty solution (null). "
+                         f"Required: oracle 100, null 0. Result: oracle {so}, null {sn}.\n"
+                         f"Oracle report: {failure_summary(rep_o)}\n"
+                         "Fix the bundle (reference, hidden tests, seed, or contract) and return the full "
+                         "bundle again."})
+    else:
+        log["accepted"] = False
+    json.dump(log, open(os.path.join(task_dir if os.path.isdir(task_dir) else pool_root, "generation.json"), "w"),
+              indent=1)
+    return log
+
+
+def main():
+    load_env(os.path.join(ROOT, ".env"))
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--id", required=True)
+    ap.add_argument("--pool", default="eval", choices=["eval", "train"])
+    ap.add_argument("--object-type", required=True, choices=sorted(EXAMPLE_FOR))
+    ap.add_argument("--category", required=True, choices=sorted(CATEGORIES))
+    ap.add_argument("--difficulty", type=int, default=2)
+    ap.add_argument("--topic")
+    ap.add_argument("--model", default="deepseek-v4.1-flash:cloud")
+    ap.add_argument("--base-url", default=os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1"))
+    ap.add_argument("--run-base", type=int, required=True)
+    a = ap.parse_args()
+    os.makedirs(os.path.join(ROOT, "tasks_gen", a.pool), exist_ok=True)
+    log = generate(a.id, a.pool, a.object_type, a.category, a.difficulty, a.model, a.base_url,
+                   a.run_base, a.topic)
+    print(json.dumps(log))
+
+
+if __name__ == "__main__":
+    main()

harness/ledger.py

@@ -0,0 +1,42 @@
+"""Cost ledger for cloud model usage and a budget guard.
+
+.env: BUDGET_LIMIT_USD (default 45), BUDGET_CYCLE_START (YYYY-MM-DD, default 2026-10-02).
+Prices are DeepSeek V4.1 Flash list prices without cache discount (upper bound).
+"""
+import json
+import os
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+LEDGER = os.path.join(ROOT, "runs", "ledger.jsonl")
+PRICE_IN, PRICE_OUT = 0.30 / 1e6, 1.20 / 1e6
+
+
+class BudgetExceeded(Exception):
+    pass
+
+
+def add_usage(model, usage, kind, ref=""):
+    if ":cloud" not in model:
+        return
+    pin, pout = usage.get("prompt_tokens", 0), usage.get("completion_tokens", 0)
+    os.makedirs(os.path.dirname(LEDGER), exist_ok=True)
+    with open(LEDGER, "a") as f:
+        f.write(json.dumps({"t": time.time(), "day": time.strftime("%Y-%m-%d"), "model": model, "kind": kind,
+                            "ref": ref, "in": pin, "out": pout,
+                            "usd": round(pin * PRICE_IN + pout * PRICE_OUT, 5)}) + "\n")
+
+
+def spent(since=None):
+    since = since or os.environ.get("BUDGET_CYCLE_START", "2026-10-02")
+    if not os.path.exists(LEDGER):
+        return 0.0
+    return round(sum(json.loads(l)["usd"] for l in open(LEDGER) if json.loads(l)["day"] >= since), 4)
+
+
+def check_budget():
+    limit = float(os.environ.get("BUDGET_LIMIT_USD", "45"))
+    s = spent()
+    if s >= limit:
+        raise BudgetExceeded(f"cycle spend {s} USD >= limit {limit} USD")
+    return s

harness/mcp_client.py

@@ -0,0 +1,96 @@
+"""Minimal MCP streamable-http client. One session per worker."""
+import json
+import os
+import random
+import time
+import urllib.request
+import urllib.error
+
+
+class McpError(Exception):
+    pass
+
+
+class McpClient:
+    def __init__(self, url=None, token=None, timeout=300):
+        self.url = url or os.environ.get("MCP_URL", "http://127.0.0.1:3000/mcp")
+        self.token = token or os.environ.get("MCP_TOKEN", "")
+        self.timeout = timeout
+        self.sid = None
+        self._id = 0
+        self._lock_retries = 0
+        self.lock_retry_total = 0
+
+    def _post(self, body, method="POST"):
+        h = {"Content-Type": "application/json",
+             "Accept": "application/json, text/event-stream"}
+        if self.token:
+            h["Authorization"] = f"Bearer {self.token}"
+        if self.sid:
+            h["Mcp-Session-Id"] = self.sid
+        data = json.dumps(body).encode() if body is not None else None
+        req = urllib.request.Request(self.url, data, h, method=method)
+        with urllib.request.urlopen(req, timeout=self.timeout) as r:
+            sid = r.headers.get("Mcp-Session-Id")
+            raw = r.read().decode()
+        if sid:
+            self.sid = sid
+        if "data:" in raw[:40]:
+            raw = "".join(l[5:].strip() for l in raw.splitlines() if l.startswith("data:"))
+        return json.loads(raw) if raw.strip() else None
+
+    def open(self):
+        self._id += 1
+        self._post({"jsonrpc": "2.0", "id": self._id, "method": "initialize",
+                    "params": {"protocolVersion": "2025-03-26", "capabilities": {},
+                               "clientInfo": {"name": "abap-llm-harness", "version": "0.1"}}})
+        self._post({"jsonrpc": "2.0", "method": "notifications/initialized"})
+        return self
+
+    def close(self):
+        if self.sid:
+            try:
+                self._post(None, method="DELETE")
+            except Exception:
+                pass
+            self.sid = None
+
+    def list_tools(self):
+        self._id += 1
+        res = self._post({"jsonrpc": "2.0", "id": self._id, "method": "tools/list"})
+        return res["result"]["tools"]
+
+    def call(self, name, args, retry=True):
+        """Return (is_error, text)."""
+        self._id += 1
+        body = {"jsonrpc": "2.0", "id": self._id, "method": "tools/call",
+                "params": {"name": name, "arguments": args}}
+        try:
+            res = self._post(body)
+        except urllib.error.HTTPError as e:
+            if e.code == 404 and retry:  # session dropped (capacity limit)
+                self.sid = None
+                self.open()
+                return self.call(name, args, retry=False)
+            raise McpError(f"{name}: HTTP {e.code}") from e
+        if "error" in res:
+            return True, json.dumps(res["error"])
+        r = res["result"]
+        text = "\n".join(c.get("text", "") for c in r.get("content", []))
+        # The server shares one RFC connection between sessions. A parallel call gets
+        # "Concurrent call detected" instead of waiting. Retry; the caller never sees it.
+        if "Concurrent call detected" in text and self._lock_retries < 8:
+            self._lock_retries += 1
+            time.sleep(0.5 * self._lock_retries + random.random())
+            try:
+                return self.call(name, args, retry)
+            finally:
+                self._lock_retries = 0
+        self.lock_retry_total += self._lock_retries
+        return bool(r.get("isError")), text
+
+    def __enter__(self):
+        return self.open()
+
+    def __exit__(self, *a):
+        self.close()

harness/pilot.py

@@ -0,0 +1,48 @@
+"""Pilot: generate and validate a list of tasks, one after the other."""
+import json
+import os
+import sys
+
+from .adt_client import load_env
+from .generator import ROOT, generate
+from .ledger import BudgetExceeded, spent
+
+PLAN = [
+    ("CLAS", "A"), ("CLAS", "B"), ("CLAS", "C"), ("CLAS", "D"), ("CLAS", "E"),
+    ("CLAS", "F"), ("CLAS", "G"), ("CLAS", "I"), ("CLAS", "C"),
+    ("FUNC", "A"), ("FUNC", "F"), ("FUNC", "I"),
+    ("PROG", "B"), ("PROG", "B"), ("PROG", "E"),
+    ("DDLS", "B"), ("DDLS", "B"), ("DDLS", "B"), ("DDLS", "B"), ("DDLS", "B"),
+]
+DDLS_TOPICS = ["join of two tables and a calculated element", "association and path expression",
+               "aggregation with CASE and GROUP BY", "view with input parameters",
+               "UNION of two tables with a literal source element"]
+
+
+def main():
+    load_env(os.path.join(ROOT, ".env"))
+    first = int(sys.argv[1]) if len(sys.argv) > 1 else 2
+    results = []
+    d = 0
+    for k, (otype, cat) in enumerate(PLAN):
+        tid = f"G{first + k:04d}"
+        topic = None
+        if otype == "DDLS":
+            topic, d = DDLS_TOPICS[d], d + 1
+        try:
+            log = generate(tid, "eval", otype, cat, 2, "deepseek-v4.1-flash:cloud",
+                           os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1"),
+                           1100 + 10 * k, topic)
+        except BudgetExceeded as e:
+            print("BUDGET", e, flush=True)
+            break
+        except Exception as e:  # noqa: BLE001
+            log = {"id": tid, "error": str(e)[:500]}
+        log["spent_total"] = spent()
+        results.append(log)
+        print(json.dumps(log), flush=True)
+        json.dump(results, open(os.path.join(ROOT, "runs", "gen", "pilot.json"), "w"), indent=1)
+
+
+if __name__ == "__main__":
+    main()

harness/proxy.py

@@ -0,0 +1,80 @@
+"""Tool proxy between agent and MCP: whitelist, budget, prefix filter, trajectory log."""
+import json
+import re
+import time
+
+MODEL_TOOLS = {
+    "sap_search_object", "sap_pull_source", "sap_object_structure", "sap_object_members",
+    "sap_usage_references", "sap_element_info", "sap_inactive_objects", "sap_short_dumps",
+    "sap_sql_query", "sap_create_object", "sap_push_source", "sap_push_element",
+    "sap_push_message", "sap_activate", "sap_syntax_check", "sap_check_object",
+    "sap_run_unit_test", "sap_atc_run", "sap_pretty_print", "sap_run_class",
+}
+WRITE_TOOLS = {"sap_create_object", "sap_push_source", "sap_push_element",
+               "sap_push_message", "sap_activate"}
+RUN_PREFIX = re.compile(r"^Z\d[0-9A-Z]{5,6}_", re.I)
+
+
+class BudgetExceeded(Exception):
+    pass
+
+
+class ToolProxy:
+    def __init__(self, mcp, prefix, budget, log_path):
+        self.mcp = mcp
+        self.prefix = prefix.upper()
+        self.max_calls = budget.get("max_tool_calls", 60)
+        self.max_activations = budget.get("max_activations", 15)
+        self.calls = 0
+        self.activations = 0
+        self.fail_streak = 0
+        self.max_fail_streak = 0
+        self.log = open(log_path, "a")
+
+    def schemas(self):
+        return [t for t in self.mcp.list_tools() if t["name"] in MODEL_TOOLS]
+
+    def _foreign(self, name):
+        n = (name or "").upper()
+        return bool(RUN_PREFIX.match(n)) and not n.startswith(self.prefix)
+
+    def _filter(self, tool, text):
+        if tool not in ("sap_search_object", "sap_usage_references"):
+            return text
+        try:
+            data = json.loads(text)
+        except ValueError:
+            return text
+        if isinstance(data, list):
+            data = [d for d in data if not self._foreign(d.get("name") if isinstance(d, dict) else "")]
+            return json.dumps(data)
+        return text
+
+    def call(self, tool, args):
+        entry = {"t": time.time(), "tool": tool, "args": args}
+        if tool not in MODEL_TOOLS:
+            result = (True, f"Tool {tool} is not available.")
+        elif self.calls >= self.max_calls:
+            raise BudgetExceeded(f"max_tool_calls={self.max_calls}")
+        else:
+            self.calls += 1
+            name = str(args.get("objectName", "")).upper()
+            if tool in WRITE_TOOLS and self._foreign(name):
+                result = (True, f"{name} is not available.")
+            else:
+                if tool in WRITE_TOOLS and (tool == "sap_activate" or args.get("activate", True)):
+                    self.activations += 1
+                err, text = self.mcp.call(tool, args)
+                result = (err, self._filter(tool, text))
+                if tool in WRITE_TOOLS:
+                    failed = err or '"success":false' in text.replace(" ", "")
+                    self.fail_streak = self.fail_streak + 1 if failed else 0
+                    self.max_fail_streak = max(self.max_fail_streak, self.fail_streak)
+        entry["is_error"], entry["result"] = result[0], result[1][:20000]
+        self.log.write(json.dumps(entry) + "\n")
+        self.log.flush()
+        return result
+
+    def note(self, kind, content):
+        self.log.write(json.dumps({"t": time.time(), "kind": kind, "content": content}) + "\n")
+        self.log.flush()

harness/runner.py

@@ -0,0 +1,394 @@
+"""One task run: setup -> agent -> collect -> gates -> tests -> checks -> score -> teardown."""
+import json
+import os
+import re
+import subprocess
+import time
+
+from .adt_client import AdtClient, load_env
+from .mcp_client import McpClient
+from .proxy import ToolProxy
+from .task import Task, prefix_for
+
+ABAPLINT = os.path.join(os.path.dirname(__file__), "..", "node_modules", ".bin", "abaplint")
+CLEAN_RULES = {
+    "unused_variables": True, "unused_types": True, "prefer_xsdbool": True, "use_new": True,
+    "prefer_returning_to_exporting": True, "functional_writing": True,
+    "preferred_compare_operator": True, "use_line_exists": True, "line_length": {"length": 120},
+    "max_one_statement": True, "empty_statement": True, "commented_code": True,
+    "omit_parameter_name": True, "prefer_is_not": True, "exporting": True,
+}
+DELETE_ORDER = ["CLAS", "INTF", "PROG", "FUNC", "FUGR", "SRVD", "DDLX", "DCLS", "DDLS",
+                "TTYP", "TABL", "STRU", "DTEL", "DOMA", "MSAG"]
+SOURCE_TYPES = ("CLAS", "INTF", "PROG", "FUNC", "DDLS", "DCLS", "DDLX", "TABL")
+
+
+def _obj_args(otype, name, fg=None):
+    a = {"objectType": otype, "objectName": name}
+    if fg:
+        a["functionGroup"] = fg
+    return a
+
+
+def _json(text):
+    try:
+        return json.loads(text)
+    except ValueError:
+        return None
+
+
+def _norm(src):
+    return "\n".join(l.rstrip().lower() for l in (src or "").splitlines() if l.strip())
+
+
+def trajectory_stats(path):
+    """Rebuild agent statistics from a trajectory (same rules as ToolProxy)."""
+    from .proxy import WRITE_TOOLS
+    calls = activations = streak = max_streak = 0
+    final, t_first, t_last = "", None, None
+    for line in open(path):
+        e = json.loads(line)
+        t_first = t_first or e["t"]
+        t_last = e["t"]
+        if e.get("kind") == "final":
+            final = e.get("content") or ""
+        if "tool" not in e or e["result"].startswith("Tool ") and e["is_error"]:
+            continue
+        calls += 1
+        tool, args = e["tool"], e.get("args", {})
+        if tool in WRITE_TOOLS:
+            if tool == "sap_activate" or args.get("activate", True):
+                activations += 1
+            failed = e["is_error"] or '"success":false' in e["result"].replace(" ", "")
+            streak = streak + 1 if failed else 0
+            max_streak = max(max_streak, streak)
+    return {"tool_calls": calls, "activations": activations, "max_fail_streak": max_streak,
+            "final_report": final, "agent_seconds": round((t_last or 0) - (t_first or 0), 1)}
+
+
+class Runner:
+    def __init__(self, tasks_root, runs_root):
+        self.tasks_root = tasks_root
+        self.runs_root = runs_root
+
+    # ---------- helpers ----------
+    def _install(self, mcp, objs):
+        out = []
+        for o in objs:
+            fg = o.get("functionGroup")
+            cargs = dict(_obj_args(o["type"], o["name"], fg), packageName="$TMP",
+                         description=o.get("description", o["name"])[:60])
+            e1, t1 = mcp.call("sap_create_object", cargs)
+            ok, t2 = not e1, ""
+            if o.get("source"):
+                e2, t2 = mcp.call("sap_push_source", dict(_obj_args(o["type"], o["name"], fg),
+                                                          source=o["source"]))
+                ok = not e2 and (_json(t2) or {}).get("success", False)
+            if o.get("testclasses_source"):
+                e3, t3 = mcp.call("sap_push_source", dict(_obj_args(o["type"], o["name"], fg),
+                                                          includeType="testclasses",
+                                                          source=o["testclasses_source"]))
+                ok = ok and not e3 and (_json(t3) or {}).get("success", False)
+            if o.get("run"):
+                e4, t4 = mcp.call("sap_run_class", {"className": o["name"]})
+                ok = ok and not e4 and (_json(t4) or {}).get("success", False)
+            out.append({"name": o["name"], "ok": ok, "create": t1[:300], "push": t2[:500]})
+        return out
+
+    def _objects_with_prefix(self, mcp, prefix):
+        _, text = mcp.call("sap_search_object", {"query": prefix + "*", "maxResults": 200})
+        return [d for d in (_json(text) or []) if d.get("name", "").upper().startswith(prefix)
+                and d.get("objectType")]  # skips STOB entries of CDS entities
+
+    def _source(self, mcp, otype, name, fg=None):
+        err, text = mcp.call("sap_pull_source", _obj_args(otype, name, fg))
+        return None if err else text
+
+    def _abaplint(self, run_dir, task, sources):
+        d = os.path.join(run_dir, "abaplint")
+        os.makedirs(os.path.join(d, "src"), exist_ok=True)
+        ext = {"CLAS": "clas", "INTF": "intf", "PROG": "prog"}
+        for (otype, name), src in sources.items():
+            if otype not in ext:
+                continue
+            base = os.path.join(d, "src", f"{name.lower()}.{ext[otype]}")
+            open(base + ".abap", "w").write(src)
+            if otype == "PROG":
+                open(base + ".xml", "w").write(
+                    f'<?xml version="1.0" encoding="utf-8"?><abapGit version="v1.0.0" serializer="LCL_OBJECT_PROG" '
+                    f'serializer_version="v1.0.0"><asx:abap xmlns:asx="http://www.sap.com/abapxml" version="1.0">'
+                    f'<asx:values><PROGDIR><NAME>{name}</NAME><SUBC>1</SUBC><RLOAD>E</RLOAD><FIXPT>X</FIXPT>'
+                    f'<UCCHECK>X</UCCHECK></PROGDIR></asx:values></asx:abap></abapGit>')
+                continue
+            tag = "VSEOCLASS" if otype == "CLAS" else "VSEOINTERF"
+            ser = "LCL_OBJECT_CLAS" if otype == "CLAS" else "LCL_OBJECT_INTF"
+            open(base + ".xml", "w").write(
+                f'<?xml version="1.0" encoding="utf-8"?><abapGit version="v1.0.0" serializer="{ser}" '
+                f'serializer_version="v1.0.0"><asx:abap xmlns:asx="http://www.sap.com/abapxml" version="1.0">'
+                f'<asx:values><{tag}><CLSNAME>{name}</CLSNAME><LANGU>E</LANGU><DESCRIPT>x</DESCRIPT>'
+                f'<STATE>1</STATE><UNICODE>X</UNICODE></{tag}></asx:values></asx:abap></abapGit>')
+        rules = {"check_syntax": True, "unknown_types": True}
+        rules.update(CLEAN_RULES)
+        for r in task.meta.get("craft_checks", []):
+            rules[r] = {"statements": 40} if r == "method_length" else True
+        cfg = {"global": {"files": "/src/**/*.*"}, "dependencies": [],
+               "syntax": {"version": task.meta.get("release_target", "v758"),
+                          # objects of this run that abaplint cannot read (TABL, DDLS, FUNC) are not errors;
+                          # A4H syntax check (G1) covers them
+                          "errorNamespace": f"^(?!{task.prefix})(Z|Y|LCL_|TY_|LIF_|LTC_)"},
+               "rules": rules}
+        json.dump(cfg, open(os.path.join(d, "abaplint.json"), "w"), indent=1)
+        p = subprocess.run([ABAPLINT, "abaplint.json", "-f", "json"], cwd=d,
+                           capture_output=True, text=True, timeout=300)
+        try:
+            issues = json.loads(p.stdout or "[]")
+        except ValueError:
+            issues = [{"key": "abaplint_failed", "description": (p.stdout + p.stderr)[:500],
+                       "file": {"filename": ""}}]
+        def _fname(i):
+            f = i.get("file", "")
+            return f.get("filename", "") if isinstance(f, dict) else str(f)
+        return [{"rule": i.get("key"), "file": os.path.basename(_fname(i)),
+                 "line": i.get("start", {}).get("row"), "msg": i.get("description")} for i in issues]
+
+    # ---------- main ----------
+    def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None):
+        prefix = prefix_for(run_no, task_id)
+        task = Task(os.path.join(self.tasks_root, task_id), prefix)
+        run_dir = rescore_dir or os.path.join(self.runs_root, f"{int(run_no):03d}_{task_id}_{agent.name.replace(':', '_').replace('/', '_')}")
+        os.makedirs(run_dir, exist_ok=True)
+        rep = {"task": task_id, "agent": agent.name, "run": run_no, "prefix": prefix,
+               "started": time.strftime("%Y-%m-%dT%H:%M:%S")}
+        t0 = time.time()
+        with McpClient() as mcp:
+            # 1 setup
+            seed = task.objects("seed")
+            if rescore_dir:
+                seed_src = {o["name"].upper(): o["source"] for o in seed}
+            else:
+                rep["setup"] = self._install(mcp, seed)
+                if not all(x["ok"] for x in rep["setup"]):
+                    rep["setup_failed"] = True
+                    rep["score"] = {"total": None, "note": "setup failed; run not scored"}
+                    all_objs = self._objects_with_prefix(mcp, prefix)
+                    rep["teardown"] = self._teardown(all_objs, run_dir) if teardown else "skipped"
+                    json.dump(rep, open(os.path.join(run_dir, "report.json"), "w"), indent=1)
+                    return rep, run_dir
+                seed_src = {o["name"].upper(): self._source(mcp, o["type"], o["name"], o.get("functionGroup"))
+                            for o in seed}
+
+            # 2 run
+            if rescore_dir:
+                rep.update(trajectory_stats(os.path.join(run_dir, "trajectory.jsonl")))
+                rep["rescored"] = True
+            else:
+                proxy = ToolProxy(mcp, prefix, task.meta.get("budget", {}),
+                                  os.path.join(run_dir, "trajectory.jsonl"))
+                proxy.note("spec", task.spec)
+                t1 = time.time()
+                rep["final_report"] = agent.run(task, proxy)
+                rep["agent_seconds"] = round(time.time() - t1, 1)
+                rep["tool_calls"], rep["activations"] = proxy.calls, proxy.activations
+                rep["max_fail_streak"] = proxy.max_fail_streak
+
+            # 3 collect
+            hidden_names = {o["name"].upper() for o in task.objects("hidden_tests")}
+            seed_names = set(seed_src)
+            objs = self._objects_with_prefix(mcp, prefix)
+            model_objs = [o for o in objs if o["name"].upper() not in seed_names | hidden_names]
+            sources = {}
+            for o in objs:
+                if o["objectType"] in SOURCE_TYPES:
+                    src = self._source(mcp, o["objectType"], o["name"], o.get("functionGroup"))
+                    if src is not None:
+                        sources[(o["objectType"], o["name"].upper())] = src
+            os.makedirs(os.path.join(run_dir, "sources"), exist_ok=True)
+            for (t, n), s in sources.items():
+                open(os.path.join(run_dir, "sources", f"{n.lower()}.{t.lower()}.abap"), "w").write(s)
+            rep["model_objects"] = [o["name"] for o in model_objs]
+
+            # 4 gates
+            g = {}
+            _, inact = mcp.call("sap_inactive_objects", {})
+            inactive = {d.get("name", "").upper() for d in (_json(inact) or []) if isinstance(d, dict)}
+            contract = task.objects("contract")
+            g1 = []
+            for c in contract:
+                exists = any(o["name"].upper() == c["name"].upper() for o in objs)
+                ok = exists and c["name"].upper() not in inactive
+                if ok:
+                    _, st = mcp.call("sap_syntax_check", _obj_args(c["type"], c["name"], c.get("functionGroup")))
+                    ok = (_json(st) or {}).get("errorCount", 1) == 0
+                g1.append({"name": c["name"], "ok": ok})
+            g["G1_active"] = all(x["ok"] for x in g1) and bool(g1)
+            g2 = True
+            for c in contract:
+                src = sources.get((c["type"], c["name"].upper()), "")
+                if c.get("implements") and not re.search(rf"INTERFACES\s+{re.escape(c['implements'])}\b", src, re.I):
+                    g2 = False
+                if c["type"] == "FUNC":  # signature: every parameter with its type in the FUNCTION header
+                    header = src.split(".", 1)[0]
+                    for prm in c.get("params", []):
+                        if not re.search(rf"\b{re.escape(prm['name'])}\b\)?\s+TYPE\s+{re.escape(prm['type'])}\b",
+                                         header, re.I):
+                            g2 = False
+                if c["type"] == "PROG":  # selection screen parameters
+                    for prm in c.get("parameters", []):
+                        if not re.search(rf"(PARAMETERS|SELECT-OPTIONS)\s*:?[^.]*\b{re.escape(prm)}\b", src, re.I):
+                            g2 = False
+                if c["type"] == "DDLS" and c.get("fields") and g["G1_active"]:
+                    _, q = mcp.call("sap_sql_query", {"query": f"SELECT * FROM {c['name']}", "maxRows": 1})
+                    cols = {col.get("name", "").upper() for col in (_json(q) or {}).get("columns", [])}
+                    if not {f.upper() for f in c["fields"]} <= cols:
+                        g2 = False
+            g["G2_contract"] = g2 and g["G1_active"]
+            g["G4_out_of_scope"] = all(
+                _norm(self._source(mcp, o["type"], o["name"], o.get("functionGroup")))
+                == _norm(seed_src[o["name"].upper()])
+                for o in seed)
+            atc = []
+            for c in contract:
+                if g["G1_active"]:
+                    _, t = mcp.call("sap_atc_run", _obj_args(c["type"], c["name"], c.get("functionGroup")))
+                    for f in (_json(t) or []):
+                        if isinstance(f, dict):
+                            atc.append({"obj": c["name"], "prio": str(f.get("priority", "")),
+                                        "check": f.get("checkTitle") or f.get("checkId"),
+                                        "msg": (f.get("messageTitle") or f.get("text") or "")[:200]})
+            rep["atc"] = atc
+            g["G5_no_p1"] = not any(a["prio"] == "1" for a in atc)
+
+            # 5 hidden tests
+            hidden = {"total": 0, "passed": 0, "detail": []}
+            if g["G2_contract"]:
+                inst = self._install(mcp, task.objects("hidden_tests"))
+                hidden["install"] = inst
+                for o in task.objects("hidden_tests"):
+                    _, t = mcp.call("sap_run_unit_test", {"objectType": "CLAS", "objectName": o["name"]})
+                    for prog in (_json(t) or {}).get("programs", []):
+                        for tc in prog.get("testClasses", []):
+                            for m in tc.get("testMethods", []):
+                                ok = not m.get("alerts")
+                                hidden["total"] += 1
+                                hidden["passed"] += ok
+                                hidden["detail"].append({"method": m["name"], "ok": ok,
+                                                         "alerts": json.dumps(m.get("alerts"))[:300]})
+            rep["hidden_tests"] = hidden
+            g["G3_hidden_runs"] = hidden["passed"] > 0
+
+            # 6 own tests (local test include or own global test class)
+            own = {"tests": 0, "failures": 0, "coverage": None}
+            for c in contract:
+                if c["type"] == "CLAS" and g["G1_active"]:
+                    _, t = mcp.call("sap_check_object", {"objectType": "CLAS", "objectName": c["name"],
+                                                         "runAtc": False, "runUnitTest": True, "coverage": True})
+                    for st in (_json(t) or {}).get("steps", []):
+                        if st.get("step") == "unittest":
+                            own["tests"] += st.get("tests", 0)
+                            own["failures"] += st.get("failures", 0)
+                            cov = st.get("coverage", {}).get("class", {}).get("statement", {})
+                            own["coverage"] = cov.get("percent")
+                if c["type"] == "PROG" and g["G1_active"]:  # local test classes inside the report
+                    _, t = mcp.call("sap_run_unit_test", {"objectType": "PROG", "objectName": c["name"]})
+                    for prog in (_json(t) or {}).get("programs", []):
+                        for tc in prog.get("testClasses", []):
+                            for m in tc.get("testMethods", []):
+                                own["tests"] += 1
+                                own["failures"] += bool(m.get("alerts"))
+            # own global test classes (FM and CDS tasks; also classes)
+            contract_names = {c["name"].upper() for c in contract}
+            for (otype, name), src in sources.items():
+                if (otype == "CLAS" and name not in contract_names | seed_names | hidden_names
+                        and re.search(r"FOR\s+TESTING", src, re.I)):
+                    _, t = mcp.call("sap_run_unit_test", {"objectType": "CLAS", "objectName": name})
+                    for prog in (_json(t) or {}).get("programs", []):
+                        for tc in prog.get("testClasses", []):
+                            for m in tc.get("testMethods", []):
+                                own["tests"] += 1
+                                own["failures"] += bool(m.get("alerts"))
+                    own.setdefault("global_test_classes", []).append(name)
+            rep["own_tests"] = own
+
+            # 7 abaplint
+            lint_sources = {k: v for k, v in sources.items() if k[1] not in hidden_names}
+            issues = self._abaplint(run_dir, task, lint_sources)
+            model_files = {f"{n.lower()}.{t.lower()}.abap" for (t, n) in lint_sources if n not in seed_names}
+            issues = [i for i in issues if i["file"] in model_files or i["rule"] == "abaplint_failed"]
+            rep["abaplint"] = issues
+            # Release gate only on contract objects. Helper objects (smoke runners, test classes)
+            # can use standard APIs that abaplint does not know; A4H syntax check covers them.
+            contract_files = {f"{c['name'].lower()}.{c['type'].lower()}.abap" for c in contract}
+            g["G6_release"] = not any(i["rule"] == "check_syntax" and i["file"] in contract_files
+                                      for i in issues)
+
+            # 8 score
+            rep["gates"] = g
+            rep["score"] = self._score(task, rep)
+            rep["seconds"] = round(time.time() - t0, 1)
+
+            # 9 teardown
+            all_objs = self._objects_with_prefix(mcp, prefix)
+        rep["teardown"] = self._teardown(all_objs, run_dir) if teardown else "skipped"
+        json.dump(rep, open(os.path.join(run_dir, "report.json"), "w"), indent=1)
+        return rep, run_dir
+
+    def _score(self, task, rep):
+        g = rep["gates"]
+        if task.meta.get("expected_outcome") == "stop":
+            return {"total": None, "note": "stop-task scoring not implemented in skeleton"}
+        if not all(g.values()):
+            return {"total": 0, "failed_gates": [k for k, v in g.items() if not v]}
+        h = rep["hidden_tests"]
+        correctness = 40 * h["passed"] / max(h["total"], 1)
+        craft_rules = set(task.meta.get("craft_checks", []))
+        contract_files = {f"{c['name'].lower()}.{c['type'].lower()}.abap" for c in task.objects("contract")}
+        craft_n = sum(1 for i in rep["abaplint"] if i["rule"] in craft_rules and i["file"] in contract_files)
+        craft_n += sum(1 for a in rep["atc"] if a["prio"] == "2")
+        craft = 20 * max(0.0, 1 - 0.25 * craft_n)
+        clean_n = sum(1 for i in rep["abaplint"] if i["rule"] in CLEAN_RULES)
+        clean = 15 * max(0.0, 1 - 0.1 * clean_n)
+        own = rep["own_tests"]
+        own_score = 0.0
+        if own["tests"] > 0 and own["failures"] == 0:
+            own_score += 7.5
+        if own["coverage"] is not None:
+            own_score += 7.5 * min(1.0, own["coverage"] / 70.0)
+        elif own_score:  # no coverage figure for this object type (FUNC, PROG, DDLS): tests count fully
+            own_score = 15.0
+        b = task.meta.get("budget", {})
+        disc = 10.0
+        if rep["tool_calls"] > b.get("max_tool_calls", 60):
+            disc -= 5
+        if rep["activations"] > b.get("max_activations", 15):
+            disc -= 5
+        if rep.get("max_fail_streak", 0) > 2:  # rule: stop after two failures of the same check
+            disc -= 5
+        final = (rep.get("final_report") or "").strip()
+        if not final or final.startswith("Stopped:"):  # no report from the model
+            disc -= 5
+        disc = max(disc, 0.0)
+        parts = {"correctness": round(correctness, 1), "craft": round(craft, 1), "clean": round(clean, 1),
+                 "own_tests": round(own_score, 1), "discipline": disc}
+        parts["total"] = round(sum(parts.values()), 1)
+        parts["note"] = "own_tests: faulty-reference part not implemented in skeleton"
+        return parts
+
+    def _teardown(self, objs, run_dir):
+        objs = sorted(objs, key=lambda o: DELETE_ORDER.index(o["objectType"])
+                      if o["objectType"] in DELETE_ORDER else 99)
+        uris = [o["uri"] for o in objs]
+        json.dump(uris, open(os.path.join(run_dir, "delete_uris.json"), "w"), indent=1)
+        load_env()
+        if not os.environ.get("A4H_PASSWORD"):
+            return {"status": "pending", "count": len(uris),
+                    "hint": "Set A4H_URL/A4H_USER/A4H_PASSWORD in .env, then: python3 -m harness.cli teardown <run_dir>"}
+        return delete_uris(uris)
+
+
+def delete_uris(uris):
+    """Delete one by one; this keeps the dependency order explicit."""
+    adt = AdtClient()
+    res = {}
+    for u in uris:
+        res.update(adt.delete([u]))
+    return {u: {"deleted": ok, "msg": msg} for u, (ok, msg) in res.items()}

harness/task.py

@@ -0,0 +1,62 @@
+"""Task loading and prefix substitution."""
+import json
+import os
+
+
+_B36 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+
+def _b36(n, width):
+    out = ""
+    while n:
+        n, r = divmod(n, 36)
+        out = _B36[r] + out
+    return out.rjust(width, "0")[-width:]
+
+
+def prefix_for(run_no, task_id):
+    """Z + run (4 chars base36) + task (3 chars base36) + '_'.
+
+    Run numbers below 36**3 keep a digit at position 2; the proxy uses this to tell
+    harness objects from other Z objects. Old prefixes (Z201001_) match the same pattern.
+    """
+    task_no = int("".join(ch for ch in task_id if ch.isdigit()) or 0)
+    return f"Z{_b36(int(run_no), 4)}{_b36(task_no, 3)}_"
+
+
+class Task:
+    def __init__(self, task_dir, prefix):
+        self.dir = task_dir
+        self.prefix = prefix
+        self.meta = json.loads(self._read("task.json"))
+        self.id = self.meta["id"]
+
+    def _read(self, rel):
+        with open(os.path.join(self.dir, rel)) as f:
+            return f.read()
+
+    def sub(self, text):
+        return text.replace("{{P}}", self.prefix).replace("{{p}}", self.prefix.lower())
+
+    def text(self, rel):
+        return self.sub(self._read(rel))
+
+    def objects(self, key):
+        """List of dicts with substituted names and loaded source (if 'file')."""
+        out = []
+        for o in self.meta.get(key, []):
+            o = {k: self.sub(v) if isinstance(v, str) else v for k, v in o.items()}
+            if "file" in o:
+                o["source"] = self.text(o["file"])
+            if "testclasses_file" in o:
+                o["testclasses_source"] = self.text(o["testclasses_file"])
+            out.append(o)
+        return out
+
+    @property
+    def spec(self):
+        return self.text("spec.md")
+
+    @property
+    def out_of_scope(self):
+        return [self.sub(n) for n in self.meta.get("out_of_scope", [])]