d15e190f72
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
72 lines
4.1 KiB
Python
72 lines
4.1 KiB
Python
"""Stage 1 eval runs on the fixed subset (train/subset.json) with a local OpenAI-compatible server.
|
|
|
|
train/.venv not needed: python3 train/baseline.py --label baseline [--only T01] [--run-base 20000]
|
|
|
|
One task at a time. Results: runs/stage1/<label>.json (written after each task).
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
sys.path.insert(0, ROOT)
|
|
from harness.adt_client import load_env # noqa: E402
|
|
from harness.agents import LlmAgent # noqa: E402
|
|
from harness.runner import Runner, repair_stats # noqa: E402
|
|
|
|
MODEL = os.path.expanduser("~/models/Devstral-Small-2-24B-4bit")
|
|
BASE_URL = "http://127.0.0.1:8080/v1"
|
|
LOOP_GUARD = 3 # end the run after 3 identical pushes in a row (end_reason "loop"); same for after training
|
|
MAX_TOKENS = 16384 # per turn; sent in each request (the server limit stays 32768). Same for before/after.
|
|
|
|
|
|
def main():
|
|
load_env(os.path.join(ROOT, ".env"))
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--label", default="baseline")
|
|
ap.add_argument("--only", nargs="*")
|
|
ap.add_argument("--run-base", type=int, default=20000)
|
|
ap.add_argument("--model", default=MODEL, help="path of the served model (the server ignores it, it is recorded)")
|
|
ap.add_argument("--base-url", default=BASE_URL)
|
|
ap.add_argument("--enable-thinking-false", action="store_true", help="Qwen: send enable_thinking=false per request")
|
|
a = ap.parse_args()
|
|
subset = json.load(open(os.path.join(ROOT, "train", "subset.json")))["tasks"]
|
|
out_path = os.path.join(ROOT, "runs", "stage1", f"{a.label}.json")
|
|
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
|
res = json.load(open(out_path)) if os.path.exists(out_path) else {"model": a.model, "tasks": {}}
|
|
kw = {"enable_thinking": False} if a.enable_thinking_false else None
|
|
res["settings"] = {"max_tokens": MAX_TOKENS, "enable_thinking": False if a.enable_thinking_false else None, "temperature": 0.2, "tool_call_budget": 60, "loop_guard": LOOP_GUARD,
|
|
"subset": "train/subset.json", "date": "2026-10-04"}
|
|
runs_root = os.path.join(ROOT, "runs", "stage1", a.label)
|
|
for i, t in enumerate(subset):
|
|
tid = t["id"]
|
|
if (a.only and tid not in a.only) or tid in res["tasks"]:
|
|
continue
|
|
pool = os.path.join(ROOT, "tasks") if tid.startswith("T") else os.path.join(ROOT, "tasks_gen", "eval")
|
|
agent = LlmAgent(a.model, a.base_url, max_tokens=MAX_TOKENS, chat_template_kwargs=kw, loop_guard=LOOP_GUARD)
|
|
try:
|
|
rep, run_dir = Runner(pool, runs_root).run(tid, agent, a.run_base + i)
|
|
except Exception as e: # noqa: BLE001 one broken run must not stop the series
|
|
res["tasks"][tid] = {"error": str(e)[:300]}
|
|
json.dump(res, open(out_path, "w"), indent=1)
|
|
print(json.dumps({"task": tid, "error": str(e)[:300]}), flush=True)
|
|
continue
|
|
h = rep.get("hidden_tests") or {}
|
|
res["tasks"][tid] = {"category": t["category"], "object_type": t["object_type"],
|
|
"score": (rep.get("score") or {}).get("total"), "parts": rep.get("score"),
|
|
"gates": rep.get("gates"), "hidden": f"{h.get('passed')}/{h.get('total')}",
|
|
"tool_calls": rep.get("tool_calls"), "seconds": rep.get("seconds"),
|
|
"end_reason": rep.get("end_reason"), "activation_failures": rep.get("activation_failures"),
|
|
"activation_error_messages": rep.get("activation_error_messages"),
|
|
**repair_stats(os.path.join(run_dir, "trajectory.jsonl")),
|
|
"agent_seconds": rep.get("agent_seconds"), "final": (rep.get("final_report") or "")[:300],
|
|
"run_dir": os.path.relpath(run_dir, ROOT)}
|
|
json.dump(res, open(out_path, "w"), indent=1)
|
|
print(json.dumps({"task": tid, "score": res["tasks"][tid]["score"], "hidden": res["tasks"][tid]["hidden"],
|
|
"tool_calls": rep.get("tool_calls"), "seconds": rep.get("seconds")}), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|