erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Loop guard (3 identical pushes), end_reason and activation error records per run; thinking off for the stage 1 baseline

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-04 07:07:44 +02:00
parent 040b9900fd
commit 0401186a6c
6 changed files with 79 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
harness/agents.py
View File

@@ -70,7 +70,7 @@ class LlmAgent:
"""
def __init__(self, model, base_url=None, api_key=None, max_turns=80, temperature=0.2,
max_seconds=None, max_tokens=None):
max_seconds=None, max_tokens=None, chat_template_kwargs=None, loop_guard=None):
self.model = model
self.name = f"llm:{model}"
self.base_url = (base_url or os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1")).rstrip("/")
@@ -83,12 +83,17 @@ class LlmAgent:
# run cost, 2026-10-03). Normal turns: p95 17.5k, max 82k. A cut turn is retried (see run).
self.max_tokens = max_tokens or (32000 if ":cloud" in (model or "") else None)
self.empty_retries = 2
self.loop_guard = loop_guard # end the run after this many identical pushes in a row (None = off)
self.end_reason = None
self.chat_template_kwargs = chat_template_kwargs # local server only, e.g. {"enable_thinking": False}
def _chat(self, messages, tools):
body = {"model": self.model, "messages": messages, "tools": tools,
"temperature": self.temperature, "parallel_tool_calls": False}
if self.max_tokens:
body["max_tokens"] = self.max_tokens
if self.chat_template_kwargs:
body["chat_template_kwargs"] = self.chat_template_kwargs
req = urllib.request.Request(f"{self.base_url}/chat/completions", json.dumps(body).encode(),
{"Content-Type": "application/json",
"Authorization": f"Bearer {self.api_key}"})
@@ -117,16 +122,19 @@ class LlmAgent:
check_budget()
final = ""
empty = 0
self.end_reason = "max_turns"
start = time.time()
for _ in range(self.max_turns):
if self.max_seconds and time.time() - start > self.max_seconds:
final = f"Stopped: time budget exceeded ({self.max_seconds} s)."
self.end_reason = "time_budget"
break
try:
msg, usage = self._chat(messages, tools)
add_usage(self.model, usage, kind="run", ref=proxy.prefix)
except RuntimeError as e:
final = f"Stopped: {e}"
self.end_reason = "model_error"
break
proxy.note("assistant", {"content": msg.get("content"),
"tool_calls": msg.get("tool_calls"), "usage": usage})
@@ -138,6 +146,7 @@ class LlmAgent:
continue
if not calls:
final = msg.get("content") or ("Stopped: empty model response." if empty else "")
self.end_reason = "report" if (msg.get("content") or "").strip() else "empty_response"
break
try:
for c in calls:
@@ -147,8 +156,15 @@ class LlmAgent:
err, text = proxy.call(fn["name"], args)
messages.append({"role": "tool", "tool_call_id": c.get("id", ""),
"content": ("ERROR: " if err else "") + text[:12000]})
if self.loop_guard and proxy.same_push_streak >= self.loop_guard:
final = f"Stopped: loop (the same source was pushed {proxy.same_push_streak} times in a row)."
self.end_reason = "loop"
break
if self.end_reason == "loop":
break
except BudgetExceeded as e:
final = f"Stopped: budget exceeded ({e})."
self.end_reason = "tool_budget"
break
proxy.note("final", final)
return final

32
harness/proxy.py
View File

@@ -1,4 +1,5 @@
"""Tool proxy between agent and MCP: whitelist, budget, prefix filter, trajectory log."""
import hashlib
import json
import re
import time
@@ -30,6 +31,23 @@ VARIANTS = {
"packageName": "package", "includeType": "include", "className": "class_name"},
},
}
def activation_messages(text, is_error=False):
"""Error messages (severity E) of a write result; the whole text if the call itself failed."""
try:
obj = json.loads(text)
except (TypeError, ValueError):
return [str(text)[:300]] if is_error else []
if not isinstance(obj, dict):
return []
msgs = (obj.get("activation") or {}).get("messages") or obj.get("messages") or []
out = [str(m.get("message", ""))[:300] for m in msgs if isinstance(m, dict) and m.get("severity") == "E"]
if not out and is_error:
out = [str(obj.get("message") or text)[:300]]
return out
RUN_PREFIX = re.compile(r"^Z\d[0-9A-Z]{5,6}_", re.I)
@@ -49,6 +67,10 @@ class ToolProxy:
self.activations = 0
self.fail_streak = 0
self.max_fail_streak = 0
self.activation_failures = 0 # write calls that failed
self.activation_errors = [] # unique error messages of failed writes
self.same_push_streak = 0 # identical sap_push_source (object + source hash) in a row
self._last_push = None
self.log = open(log_path, "a")
self.adt = None
self.fallbacks = [] # writes that needed the ADT activation fallback
@@ -125,6 +147,11 @@ class ToolProxy:
else:
if tool in WRITE_TOOLS and (tool == "sap_activate" or args.get("activate", True)):
self.activations += 1
if tool == "sap_push_source":
key = (name, str(args.get("includeType", "")),
hashlib.md5(str(args.get("source", "")).encode()).hexdigest())
self.same_push_streak = self.same_push_streak + 1 if key == self._last_push else 1
self._last_push = key
err, text = self.mcp.call(tool, args)
if tool == "sap_push_source":
if str(args.get("objectType", "")).upper() in ("PROG", "FUNC") and not args.get("includeType"):
@@ -139,6 +166,11 @@ class ToolProxy:
failed = err or '"success":false' in text.replace(" ", "")
self.fail_streak = self.fail_streak + 1 if failed else 0
self.max_fail_streak = max(self.max_fail_streak, self.fail_streak)
if failed:
self.activation_failures += 1
for m in activation_messages(text, err):
if m not in self.activation_errors and len(self.activation_errors) < 30:
self.activation_errors.append(m)
entry["is_error"], entry["result"] = result[0], result[1][:20000]
self.log.write(json.dumps(entry) + "\n")
self.log.flush()

13
harness/runner.py
View File

@@ -61,7 +61,9 @@ def _norm(src):
def trajectory_stats(path):
"""Rebuild agent statistics from a trajectory (same rules as ToolProxy)."""
from .proxy import WRITE_TOOLS
calls = activations = streak = max_streak = 0
from .proxy import activation_messages
calls = activations = streak = max_streak = fails = 0
errors = []
final, t_first, t_last = "", None, None
for line in open(path):
e = json.loads(line)
@@ -79,7 +81,13 @@ def trajectory_stats(path):
failed = e["is_error"] or '"success":false' in e["result"].replace(" ", "")
streak = streak + 1 if failed else 0
max_streak = max(max_streak, streak)
if failed:
fails += 1
for m in activation_messages(e["result"], e["is_error"]):
if m not in errors and len(errors) < 30:
errors.append(m)
return {"tool_calls": calls, "activations": activations, "max_fail_streak": max_streak,
"activation_failures": fails, "activation_error_messages": errors,
"final_report": final, "agent_seconds": round((t_last or 0) - (t_first or 0), 1)}
@@ -207,6 +215,9 @@ class Runner:
rep["agent_seconds"] = round(time.time() - t1, 1)
rep["tool_calls"], rep["activations"] = proxy.calls, proxy.activations
rep["max_fail_streak"] = proxy.max_fail_streak
rep["activation_failures"] = proxy.activation_failures
rep["activation_error_messages"] = proxy.activation_errors
rep["end_reason"] = getattr(agent, "end_reason", None)
if proxy.fallbacks:
rep["adt_fallbacks"] = proxy.fallbacks