"""Read-only scan: which runs saw other runs' objects in tool results, or read them (teardown/proxy bug of 2026-10-06). python3 train/foreign_scan.py -> prints a summary and writes runs/analysis/foreign_scan.json """ import glob import json import os import re import sys from collections import Counter ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) from harness.task import prefix_for # noqa: E402 START = re.compile(r"Z\d[0-9A-Z]{6}_") MID_NAME = re.compile(r"^[A-Z]{1,5}_(Z\d[0-9A-Z]{6}_)", re.I) READ = ("sap_pull_source", "sap_object_structure", "sap_object_members", "sap_element_info", "sap_run_unit_test", "sap_check_object", "sap_syntax_check", "sap_atc_run") GROUPS = [("official baseline Qwen (MacBook, 11 tasks)", "runs/stage1/baseline_qwen/*"), ("Devstral baseline (dropped)", "runs/archive/devstral/baseline_devstral/*"), ("older Qwen baselines (archive)", "runs/archive/qwen38/*/*"), ("smoke", "runs/stage1/smoke/*"), ("eval empirical filter (DeepSeek on eval candidates)", "runs/emp/*"), ("eval generation validation (oracle, null, mutants)", "runs/gen/*"), ("early pilot runs", "runs/2*_*"), ("training trajectories (DeepSeek)", "runs/traj/*_G*"), ("series A (local Qwen)", "runs/local_qwen/runs/*")] def own_prefix(d): m = re.match(r"^(\d+)_([GT]\d+)_", os.path.basename(d)) return prefix_for(int(m.group(1)), m.group(2)).upper() if m else None def scan(d): own = own_prefix(d) p = os.path.join(d, "trajectory.jsonl") if not own or not os.path.exists(p): return None seen, reads, tools = set(), 0, Counter() for l in open(p): try: e = json.loads(l) except ValueError: continue if "tool" not in e: continue text = (e.get("result") or "").upper() found = {x for x in START.findall(text) if x != own and x[1].isdigit()} found |= {m.group(1).upper() for m in re.finditer(r"\b[A-Z]{1,5}_(Z\d[0-9A-Z]{6}_)", text) if m.group(1).upper() != own} if found: seen |= found tools[e["tool"]] += 1 name = str((e.get("args") or {}).get("objectName", "")).upper() m = START.match(name) or MID_NAME.match(name) pre = (m.group(1) if m and m.re is MID_NAME else (m.group(0) if m else "")).upper() if e["tool"] in READ and pre and pre != own: reads += 1 return {"run": os.path.basename(d), "foreign_prefixes_seen": len(seen), "foreign_reads": reads, "tools": dict(tools)} def main(): out = {} for label, pat in GROUPS: dirs = [d for d in sorted(glob.glob(os.path.join(ROOT, pat))) if os.path.isdir(d)] rows = [r for r in (scan(d) for d in dirs) if r] seen = [r for r in rows if r["foreign_prefixes_seen"]] reads = [r for r in rows if r["foreign_reads"]] out[label] = {"runs": len(rows), "runs_with_foreign_names": len(seen), "runs_with_foreign_reads": len(reads), "tools": dict(sum((Counter(r["tools"]) for r in seen), Counter())), "affected": [r["run"] for r in seen][:40], "read_runs": [r["run"] for r in reads]} print(f"{label}: {len(rows)} runs | foreign names in tool results: {len(seen)} | foreign reads: {len(reads)} | tools {out[label]['tools']}") os.makedirs(os.path.join(ROOT, "runs", "analysis"), exist_ok=True) json.dump(out, open(os.path.join(ROOT, "runs", "analysis", "foreign_scan.json"), "w"), indent=1) if __name__ == "__main__": main()