erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trajectory record (messages, raw tool results, metadata; reasoning apart) and trajectory runner

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 12:03:54 +02:00
parent a4eb567e4c
commit d5e43e1a83
4 changed files with 194 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
harness/runner.py
View File

@@ -230,7 +230,8 @@ class Runner:
rep["rescored"] = True
else:
proxy = ToolProxy(mcp, prefix, task.meta.get("budget", {}),
os.path.join(run_dir, "trajectory.jsonl"), task.meta.get("tool_schema"))
os.path.join(run_dir, "trajectory.jsonl"), task.meta.get("tool_schema"),
task.meta.get("release_target"))
proxy.note("spec", task.spec)
t1 = time.time()
rep["final_report"] = agent.run(task, proxy)
@@ -239,6 +240,7 @@ class Runner:
rep["max_fail_streak"] = proxy.max_fail_streak
rep["activation_failures"] = proxy.activation_failures
rep["activation_error_messages"] = proxy.activation_errors
rep["syntax_hints"] = proxy.syntax_hints
rep["end_reason"] = getattr(agent, "end_reason", None)
if proxy.fallbacks:
rep["adt_fallbacks"] = proxy.fallbacks
@@ -409,6 +411,9 @@ class Runner:
all_objs = self._objects_with_prefix(mcp, prefix)
rep["teardown"] = self._teardown(all_objs, run_dir) if teardown else "skipped"
json.dump(rep, open(os.path.join(run_dir, "report.json"), "w"), indent=1)
if not rescore_dir:
from .record import write_record
write_record(run_dir, task, agent, rep, os.path.basename(os.path.normpath(self.tasks_root)))
return rep, run_dir
def _score(self, task, rep):