76 lines
3.0 KiB
Python
76 lines
3.0 KiB
Python
"""Trajectories in which the model read another run's leftover object go back to the pending pool (Kral + Opus 2026-10-06).
|
|
|
|
python3 train/requeue_foreign.py dry run
|
|
python3 train/requeue_foreign.py --apply moves their rows from runs/traj/summary.jsonl to runs/traj/summary_excluded.jsonl
|
|
(backup: summary.jsonl.bak-<time>); the task then counts as not run and is run again
|
|
after the reset in the normal order (kind deficit). The run folders stay.
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import time
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
sys.path.insert(0, os.path.join(ROOT, "train"))
|
|
import accept as acc # noqa: E402
|
|
from harness import mix # noqa: E402
|
|
|
|
START = __import__("re").compile(r"^(Z\d[0-9A-Z]{6}_)", __import__("re").I)
|
|
MID = __import__("re").compile(r"^[A-Z]{1,5}_(Z\d[0-9A-Z]{6}_)", __import__("re").I)
|
|
READ = ("sap_pull_source", "sap_object_structure", "sap_object_members", "sap_element_info", "sap_run_unit_test", "sap_check_object",
|
|
"sap_syntax_check", "sap_atc_run")
|
|
|
|
|
|
def foreign_reads(rec):
|
|
own = rec["prefix"].upper()
|
|
n = 0
|
|
for m in rec["messages"]:
|
|
if m["role"] != "assistant":
|
|
continue
|
|
for c in m.get("tool_calls") or []:
|
|
if c["function"]["name"] not in READ:
|
|
continue
|
|
a = c["function"].get("arguments") or "{}"
|
|
try:
|
|
a = json.loads(a) if isinstance(a, str) else a
|
|
except ValueError:
|
|
a = {}
|
|
name = str(a.get("objectName", "")).upper()
|
|
mm = START.match(name) or MID.match(name)
|
|
if mm and mm.group(1).upper() != own:
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def main():
|
|
apply = "--apply" in sys.argv
|
|
path = os.path.join(ROOT, "runs", "traj", "summary.jsonl")
|
|
rows = [json.loads(l) for l in open(path)]
|
|
keep, out = [], []
|
|
for r in rows:
|
|
p = os.path.join(ROOT, "runs", "traj", r.get("run_dir") or "-", "record.json")
|
|
if os.path.exists(p):
|
|
rec = json.load(open(p))
|
|
if acc.judge(rec, r, 80)[0]:
|
|
n = foreign_reads(rec)
|
|
if n:
|
|
out.append(dict(r, excluded_reason="read of another run's object", foreign_reads=n, kind=mix.kind_of_task_dir(r["task"]),
|
|
excluded_at=time.strftime("%F %T")))
|
|
continue
|
|
keep.append(r)
|
|
print(len(out), "accepted trajectories with a foreign read:", [(o["task"], o["attempt"], o["kind"], o["foreign_reads"]) for o in out])
|
|
if not apply:
|
|
return
|
|
shutil.copy(path, path + ".bak-" + time.strftime("%Y%m%d-%H%M%S"))
|
|
with open(os.path.join(ROOT, "runs", "traj", "summary_excluded.jsonl"), "a") as f:
|
|
for o in out:
|
|
f.write(json.dumps(o) + "\n")
|
|
open(path, "w").write("".join(json.dumps(r) + "\n" for r in keep))
|
|
print("moved; summary.jsonl now", len(keep), "rows")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|