erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trajectory runs: tool-call budget 100 for tasks with a CDS contract object (eval keeps 60)

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 14:08:33 +02:00
parent 13360292dd
commit c062df2ec5
4 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
harness/record.py
View File

@@ -56,7 +56,7 @@ def write_record(run_dir, task, agent, rep, pool):
"cost_ledger_usd": cost, "cost_usage_usd": round(cost / LEDGER_TO_USAGE, 5),
"end_reason": rep.get("end_reason"), "tool_calls": rep.get("tool_calls"),
"activations": rep.get("activations"), "activation_failures": rep.get("activation_failures"),
"max_fail_streak": rep.get("max_fail_streak"), "syntax_hints": rep.get("syntax_hints"),
"max_fail_streak": rep.get("max_fail_streak"), "tool_budget": rep.get("tool_budget"), "syntax_hints": rep.get("syntax_hints"),
"adt_fallbacks": bool(rep.get("adt_fallbacks")), "agent_seconds": rep.get("agent_seconds"),
"turn_usage": getattr(agent, "turn_usage", []),
"setup_failed": rep.get("setup_failed", False),

8
harness/runner.py
View File

@@ -199,9 +199,14 @@ class Runner:
"line": i.get("start", {}).get("row"), "msg": i.get("description")} for i in issues]
# ---------- main ----------
def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None):
def run(self, task_id, agent, run_no, teardown=True, rescore_dir=None, cds_calls=None):
"""cds_calls: raise max_tool_calls to this value for tasks with a CDS (DDLS) contract object
(trajectory runs, 2026-10-05: the own CDS test class needs more than 60 calls; eval stays at the task value)."""
prefix = prefix_for(run_no, task_id)
task = Task(os.path.join(self.tasks_root, task_id), prefix)
if cds_calls and any(c.get("type") == "DDLS" for c in task.meta.get("contract", [])):
b = task.meta.setdefault("budget", {})
b["max_tool_calls"] = max(b.get("max_tool_calls", 60), cds_calls)
run_dir = rescore_dir or os.path.join(self.runs_root, f"{int(run_no):03d}_{task_id}_{agent.name.replace(':', '_').replace('/', '_')}")
os.makedirs(run_dir, exist_ok=True)
rep = {"task": task_id, "agent": agent.name, "run": run_no, "prefix": prefix,
@@ -241,6 +246,7 @@ class Runner:
rep["activation_failures"] = proxy.activation_failures
rep["activation_error_messages"] = proxy.activation_errors
rep["syntax_hints"] = proxy.syntax_hints
rep["tool_budget"] = proxy.max_calls
rep["end_reason"] = getattr(agent, "end_reason", None)
if proxy.fallbacks:
rep["adt_fallbacks"] = proxy.fallbacks

3
harness/trajectories.py
View File

@@ -26,6 +26,7 @@ POOL = os.path.join(ROOT, "tasks_gen", "train")
OUT = os.path.join(ROOT, "runs", "traj")
RUN_BASE = 200000 # 200000 + (task number - 1000) * 3 + attempt (a digit must lead the 4-char base36 run: < 466560)
MODEL = "deepseek-v4.1-flash:cloud"
CDS_CALLS = 100 # tool-call budget for tasks with a CDS contract object (eval keeps 60; Kral 2026-10-05)
LOCK = threading.Lock()
@@ -76,7 +77,7 @@ def one(task_id, attempt, stop):
t0 = time.time()
row = {"task": task_id, "attempt": attempt, "run": run_no, "model": MODEL}
try:
rep, run_dir = runner.run(task_id, agent, run_no, teardown=True)
rep, run_dir = runner.run(task_id, agent, run_no, teardown=True, cds_calls=CDS_CALLS)
score = (rep.get("score") or {}).get("total")
rec = os.path.exists(os.path.join(run_dir, "record.json"))
row.update(score=score, setup_failed=bool(rep.get("setup_failed")), end_reason=rep.get("end_reason"),