erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Series A: local Qwen on the MacBook (remote server), 12 h window with hard stop, clean pause on server loss, dashboard card, docs/remote-model.md

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 21:57:01 +02:00
parent dc8d99a913
commit eb13453097
6 changed files with 576 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
harness/runner.py
View File

@@ -209,11 +209,13 @@ class Runner:
"line": i.get("start", {}).get("row"), "msg": i.get("description")} for i in issues]
# ---------- main ----------
def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None, cds_calls=None):
def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None, cds_calls=None, tool_budget=None):
"""cds_calls: raise max_tool_calls to this value for tasks with a CDS (DDLS) contract object
(trajectory runs, 2026-10-05: the own CDS test class needs more than 60 calls; eval stays at the task value)."""
prefix = prefix_for(run_no, task_id)
task = Task(os.path.join(self.tasks_root, task_id), prefix)
if tool_budget: # fixed budget for every task (local model series: 60)
task.meta.setdefault("budget", {})["max_tool_calls"] = tool_budget
if cds_calls and any(c.get("type") == "DDLS" for c in task.meta.get("contract", [])):
b = task.meta.setdefault("budget", {})
b["max_tool_calls"] = max(b.get("max_tool_calls", 60), cds_calls)
@@ -261,6 +263,15 @@ class Runner:
if proxy.fallbacks:
rep["adt_fallbacks"] = proxy.fallbacks
if rep.get("end_reason") in ("window_end", "server_down"): # not a result: no scoring, only the teardown
rep["score"] = {"total": None, "note": f"not scored: {rep['end_reason']}"}
rep["not_a_result"] = True
rep["seconds"] = round(time.time() - t0, 1)
all_objs = self._objects_with_prefix(mcp, prefix)
rep["teardown"] = self._teardown(all_objs, run_dir) if teardown else "skipped"
json.dump(rep, open(os.path.join(run_dir, "report.json"), "w"), indent=1)
return rep, run_dir
# 3 collect
hidden_names = {o["name"].upper() for o in task.objects("hidden_tests")}
seed_names = set(seed_src)