Baseline shortened: 11-task subset (1 per category + T01), max_tokens 16384, settings in README; baseline_chain.sh

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 22:36:00 +02:00
parent 7054d9fd2e
commit 689820af4d
6 changed files with 200 additions and 97 deletions

View File

@@ -17,6 +17,7 @@ from harness.runner import Runner # noqa: E402
MODEL = os.path.expanduser("~/models/Qwen3.8-27B-4bit")
BASE_URL = "http://127.0.0.1:8080/v1"
MAX_TOKENS = 16384 # per turn; sent in each request (the server limit stays 32768). Same for before/after.
def main():
@@ -30,13 +31,15 @@ def main():
out_path = os.path.join(ROOT, "runs", "stage1", f"{a.label}.json")
os.makedirs(os.path.dirname(out_path), exist_ok=True)
res = json.load(open(out_path)) if os.path.exists(out_path) else {"model": MODEL, "tasks": {}}
res["settings"] = {"max_tokens": MAX_TOKENS, "reasoning_effort": "medium", "temperature": 0.2, "tool_call_budget": 60,
"subset": "train/subset.json", "date": "2026-10-03"}
runs_root = os.path.join(ROOT, "runs", "stage1", a.label)
for i, t in enumerate(subset):
tid = t["id"]
if (a.only and tid not in a.only) or tid in res["tasks"]:
continue
pool = os.path.join(ROOT, "tasks") if tid.startswith("T") else os.path.join(ROOT, "tasks_gen", "eval")
agent = LlmAgent(MODEL, BASE_URL)
agent = LlmAgent(MODEL, BASE_URL, max_tokens=MAX_TOKENS)
try:
rep, run_dir = Runner(pool, runs_root).run(tid, agent, a.run_base + i)
except Exception as e: # noqa: BLE001 one broken run must not stop the series