Decisions of 2026-10-06: 60 % rule in the training script, own-test weights (fractional), 64k in the sweep, foreign-read trajectories back to the pending pool, memory test waits, foreign object scan of baselines and eval runs, 11 October check list

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-06 09:04:09 +02:00
parent a465c33e3f
commit 8ef85c2713
10 changed files with 402 additions and 41 deletions
--- a/train/requeue_foreign.py
+++ b/train/requeue_foreign.py
@@ -0,0 +1,75 @@
+"""Trajectories in which the model read another run's leftover object go back to the pending pool (Kral + Opus 2026-10-06).
+
+  python3 train/requeue_foreign.py            dry run
+  python3 train/requeue_foreign.py --apply    moves their rows from runs/traj/summary.jsonl to runs/traj/summary_excluded.jsonl
+                                              (backup: summary.jsonl.bak-<time>); the task then counts as not run and is run again
+                                              after the reset in the normal order (kind deficit). The run folders stay.
+"""
+import json
+import os
+import shutil
+import sys
+import time
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+sys.path.insert(0, os.path.join(ROOT, "train"))
+import accept as acc  # noqa: E402
+from harness import mix  # noqa: E402
+
+START = __import__("re").compile(r"^(Z\d[0-9A-Z]{6}_)", __import__("re").I)
+MID = __import__("re").compile(r"^[A-Z]{1,5}_(Z\d[0-9A-Z]{6}_)", __import__("re").I)
+READ = ("sap_pull_source", "sap_object_structure", "sap_object_members", "sap_element_info", "sap_run_unit_test", "sap_check_object",
+        "sap_syntax_check", "sap_atc_run")
+
+
+def foreign_reads(rec):
+    own = rec["prefix"].upper()
+    n = 0
+    for m in rec["messages"]:
+        if m["role"] != "assistant":
+            continue
+        for c in m.get("tool_calls") or []:
+            if c["function"]["name"] not in READ:
+                continue
+            a = c["function"].get("arguments") or "{}"
+            try:
+                a = json.loads(a) if isinstance(a, str) else a
+            except ValueError:
+                a = {}
+            name = str(a.get("objectName", "")).upper()
+            mm = START.match(name) or MID.match(name)
+            if mm and mm.group(1).upper() != own:
+                n += 1
+    return n
+
+
+def main():
+    apply = "--apply" in sys.argv
+    path = os.path.join(ROOT, "runs", "traj", "summary.jsonl")
+    rows = [json.loads(l) for l in open(path)]
+    keep, out = [], []
+    for r in rows:
+        p = os.path.join(ROOT, "runs", "traj", r.get("run_dir") or "-", "record.json")
+        if os.path.exists(p):
+            rec = json.load(open(p))
+            if acc.judge(rec, r, 80)[0]:
+                n = foreign_reads(rec)
+                if n:
+                    out.append(dict(r, excluded_reason="read of another run's object", foreign_reads=n, kind=mix.kind_of_task_dir(r["task"]),
+                                    excluded_at=time.strftime("%F %T")))
+                    continue
+        keep.append(r)
+    print(len(out), "accepted trajectories with a foreign read:", [(o["task"], o["attempt"], o["kind"], o["foreign_reads"]) for o in out])
+    if not apply:
+        return
+    shutil.copy(path, path + ".bak-" + time.strftime("%Y%m%d-%H%M%S"))
+    with open(os.path.join(ROOT, "runs", "traj", "summary_excluded.jsonl"), "a") as f:
+        for o in out:
+            f.write(json.dumps(o) + "\n")
+    open(path, "w").write("".join(json.dumps(r) + "\n" for r in keep))
+    print("moved; summary.jsonl now", len(keep), "rows")
+
+
+if __name__ == "__main__":
+    main()