erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

baseline: setup failure is not recorded as a result; repair_stats tolerates a missing trajectory

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-04 12:07:33 +02:00
parent d15e190f72
commit 5532a5b226
2 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
harness/runner.py
View File

@@ -96,6 +96,8 @@ def repair_stats(path):
import hashlib import hashlib
last = None # (object, md5, failed) last = None # (object, md5, failed)
after = changed = 0 after = changed = 0
if not os.path.exists(path): # setup failed: no agent run, no trajectory
return {"pushes_after_error": None, "pushes_changed_after_error": None, "repair_rate": None}
for line in open(path): for line in open(path):
e = json.loads(line) e = json.loads(line)
if e.get("tool") != "sap_push_source" or (e["result"].startswith("Tool ") and e["is_error"]): if e.get("tool") != "sap_push_source" or (e["result"].startswith("Tool ") and e["is_error"]):

4
train/baseline.py
View File

@@ -52,6 +52,10 @@ def main():
json.dump(res, open(out_path, "w"), indent=1) json.dump(res, open(out_path, "w"), indent=1)
print(json.dumps({"task": tid, "error": str(e)[:300]}), flush=True) print(json.dumps({"task": tid, "error": str(e)[:300]}), flush=True)
continue continue
if rep.get("setup_failed"): # no agent run; do not record, so a restart tries the task again
bad = [x for x in rep.get("setup", []) if not x.get("ok")]
print(json.dumps({"task": tid, "setup_failed": True, "details": bad, "run_dir": run_dir}, default=str)[:1500], flush=True)
continue
h = rep.get("hidden_tests") or {} h = rep.get("hidden_tests") or {}
res["tasks"][tid] = {"category": t["category"], "object_type": t["object_type"], res["tasks"][tid] = {"category": t["category"], "object_type": t["object_type"],
"score": (rep.get("score") or {}).get("total"), "parts": rep.get("score"), "score": (rep.get("score") or {}).get("total"), "parts": rep.get("score"),