From 5532a5b226a9156ca522fa1c14f5d08a8c9cc7c2 Mon Sep 17 00:00:00 2001 From: Kral Date: Sun, 4 Oct 2026 12:07:33 +0200 Subject: [PATCH] baseline: setup failure is not recorded as a result; repair_stats tolerates a missing trajectory Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat --- harness/runner.py | 2 ++ train/baseline.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/harness/runner.py b/harness/runner.py index 89a9d48..433ffe4 100644 --- a/harness/runner.py +++ b/harness/runner.py @@ -96,6 +96,8 @@ def repair_stats(path): import hashlib last = None # (object, md5, failed) after = changed = 0 + if not os.path.exists(path): # setup failed: no agent run, no trajectory + return {"pushes_after_error": None, "pushes_changed_after_error": None, "repair_rate": None} for line in open(path): e = json.loads(line) if e.get("tool") != "sap_push_source" or (e["result"].startswith("Tool ") and e["is_error"]): diff --git a/train/baseline.py b/train/baseline.py index 696cac5..c12daff 100644 --- a/train/baseline.py +++ b/train/baseline.py @@ -52,6 +52,10 @@ def main(): json.dump(res, open(out_path, "w"), indent=1) print(json.dumps({"task": tid, "error": str(e)[:300]}), flush=True) continue + if rep.get("setup_failed"): # no agent run; do not record, so a restart tries the task again + bad = [x for x in rep.get("setup", []) if not x.get("ok")] + print(json.dumps({"task": tid, "setup_failed": True, "details": bad, "run_dir": run_dir}, default=str)[:1500], flush=True) + continue h = rep.get("hidden_tests") or {} res["tasks"][tid] = {"category": t["category"], "object_type": t["object_type"], "score": (rep.get("score") or {}).get("total"), "parts": rep.get("score"),