erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trajectory runs: tool-call budget 100 for tasks with a CDS contract object (eval keeps 60)

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 14:08:33 +02:00
parent 13360292dd
commit c062df2ec5
4 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
harness/runner.py
View File

@@ -199,9 +199,14 @@ class Runner:
"line": i.get("start", {}).get("row"), "msg": i.get("description")} for i in issues]
# ---------- main ----------
def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None):
def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None, cds_calls=None):
"""cds_calls: raise max_tool_calls to this value for tasks with a CDS (DDLS) contract object
(trajectory runs, 2026-10-05: the own CDS test class needs more than 60 calls; eval stays at the task value)."""
prefix = prefix_for(run_no, task_id)
task = Task(os.path.join(self.tasks_root, task_id), prefix)
if cds_calls and any(c.get("type") == "DDLS" for c in task.meta.get("contract", [])):
b = task.meta.setdefault("budget", {})
b["max_tool_calls"] = max(b.get("max_tool_calls", 60), cds_calls)
run_dir = rescore_dir or os.path.join(self.runs_root, f"{int(run_no):03d}_{task_id}_{agent.name.replace(':', '_').replace('/', '_')}")
os.makedirs(run_dir, exist_ok=True)
rep = {"task": task_id, "agent": agent.name, "run": run_no, "prefix": prefix,
@@ -241,6 +246,7 @@ class Runner:
rep["activation_failures"] = proxy.activation_failures
rep["activation_error_messages"] = proxy.activation_errors
rep["syntax_hints"] = proxy.syntax_hints
rep["tool_budget"] = proxy.max_calls
rep["end_reason"] = getattr(agent, "end_reason", None)
if proxy.fallbacks:
rep["adt_fallbacks"] = proxy.fallbacks