80 lines
3.5 KiB
Python
80 lines
3.5 KiB
Python
"""Read-only scan: which runs saw other runs' objects in tool results, or read them (teardown/proxy bug of 2026-10-06).
|
|
|
|
python3 train/foreign_scan.py -> prints a summary and writes runs/analysis/foreign_scan.json
|
|
"""
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import Counter
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
from harness.task import prefix_for # noqa: E402
|
|
|
|
START = re.compile(r"Z\d[0-9A-Z]{6}_")
|
|
MID_NAME = re.compile(r"^[A-Z]{1,5}_(Z\d[0-9A-Z]{6}_)", re.I)
|
|
READ = ("sap_pull_source", "sap_object_structure", "sap_object_members", "sap_element_info", "sap_run_unit_test", "sap_check_object",
|
|
"sap_syntax_check", "sap_atc_run")
|
|
GROUPS = [("official baseline Qwen (MacBook, 11 tasks)", "runs/stage1/baseline_qwen/*"),
|
|
("Devstral baseline (dropped)", "runs/archive/devstral/baseline_devstral/*"),
|
|
("older Qwen baselines (archive)", "runs/archive/qwen38/*/*"),
|
|
("smoke", "runs/stage1/smoke/*"),
|
|
("eval empirical filter (DeepSeek on eval candidates)", "runs/emp/*"),
|
|
("eval generation validation (oracle, null, mutants)", "runs/gen/*"),
|
|
("early pilot runs", "runs/2*_*"),
|
|
("training trajectories (DeepSeek)", "runs/traj/*_G*"),
|
|
("series A (local Qwen)", "runs/local_qwen/runs/*")]
|
|
|
|
|
|
def own_prefix(d):
|
|
m = re.match(r"^(\d+)_([GT]\d+)_", os.path.basename(d))
|
|
return prefix_for(int(m.group(1)), m.group(2)).upper() if m else None
|
|
|
|
|
|
def scan(d):
|
|
own = own_prefix(d)
|
|
p = os.path.join(d, "trajectory.jsonl")
|
|
if not own or not os.path.exists(p):
|
|
return None
|
|
seen, reads, tools = set(), 0, Counter()
|
|
for l in open(p):
|
|
try:
|
|
e = json.loads(l)
|
|
except ValueError:
|
|
continue
|
|
if "tool" not in e:
|
|
continue
|
|
text = (e.get("result") or "").upper()
|
|
found = {x for x in START.findall(text) if x != own and x[1].isdigit()}
|
|
found |= {m.group(1).upper() for m in re.finditer(r"\b[A-Z]{1,5}_(Z\d[0-9A-Z]{6}_)", text) if m.group(1).upper() != own}
|
|
if found:
|
|
seen |= found
|
|
tools[e["tool"]] += 1
|
|
name = str((e.get("args") or {}).get("objectName", "")).upper()
|
|
m = START.match(name) or MID_NAME.match(name)
|
|
pre = (m.group(1) if m and m.re is MID_NAME else (m.group(0) if m else "")).upper()
|
|
if e["tool"] in READ and pre and pre != own:
|
|
reads += 1
|
|
return {"run": os.path.basename(d), "foreign_prefixes_seen": len(seen), "foreign_reads": reads, "tools": dict(tools)}
|
|
|
|
|
|
def main():
|
|
out = {}
|
|
for label, pat in GROUPS:
|
|
dirs = [d for d in sorted(glob.glob(os.path.join(ROOT, pat))) if os.path.isdir(d)]
|
|
rows = [r for r in (scan(d) for d in dirs) if r]
|
|
seen = [r for r in rows if r["foreign_prefixes_seen"]]
|
|
reads = [r for r in rows if r["foreign_reads"]]
|
|
out[label] = {"runs": len(rows), "runs_with_foreign_names": len(seen), "runs_with_foreign_reads": len(reads),
|
|
"tools": dict(sum((Counter(r["tools"]) for r in seen), Counter())),
|
|
"affected": [r["run"] for r in seen][:40], "read_runs": [r["run"] for r in reads]}
|
|
print(f"{label}: {len(rows)} runs | foreign names in tool results: {len(seen)} | foreign reads: {len(reads)} | tools {out[label]['tools']}")
|
|
os.makedirs(os.path.join(ROOT, "runs", "analysis"), exist_ok=True)
|
|
json.dump(out, open(os.path.join(ROOT, "runs", "analysis", "foreign_scan.json"), "w"), indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|