Infrastructure outage handling (MCP/A4H down: wait, clean up, rerun), dashboard fix, budget limit 129

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

harness/infra.py

@@ -0,0 +1,62 @@
+"""Infrastructure outages: the MCP server (ADT in Eclipse, 127.0.0.1:3000) or A4H (localhost:50000) is not reachable.
+
+On 2026-10-05 21:43 the MCP server refused connections for a short time; three runs got `Connection refused` and the
+pipeline stopped because it counted three equal harness errors. An outage is not a model result and not a harness bug:
+the run is cleaned up, the work waits until the services answer, and the same run is started again.
+"""
+import errno
+import os
+import socket
+import time
+import urllib.error
+import urllib.request
+
+MCP_URL = os.environ.get("MCP_URL", "http://127.0.0.1:3000/mcp")
+A4H_URL = os.environ.get("A4H_URL", "http://localhost:50000").rstrip("/") + "/sap/public/ping"
+
+
+def is_outage(exc):
+    """True for a refused or lost connection to a local service (not for an HTTP error status)."""
+    if isinstance(exc, urllib.error.HTTPError):
+        return False
+    if isinstance(exc, urllib.error.URLError):
+        return is_outage(exc.reason) if isinstance(exc.reason, BaseException) else True
+    return isinstance(exc, (ConnectionError, socket.timeout, TimeoutError)) or \
+        (isinstance(exc, OSError) and exc.errno in (errno.ECONNREFUSED, errno.ECONNRESET, errno.EHOSTUNREACH, errno.EPIPE))
+
+
+def _answers(url):
+    try:
+        urllib.request.urlopen(url, timeout=6)
+        return True
+    except urllib.error.HTTPError:
+        return True          # 401 / 403 / 404: the server answers
+    except Exception:  # noqa: BLE001
+        return False
+
+
+def up():
+    return _answers(MCP_URL) and _answers(A4H_URL)
+
+
+def wait_until_up(max_seconds=3600, poll=30, log=print):
+    """Wait until MCP and A4H answer twice in a row (a restart shows a short flicker). False when the time is over."""
+    end, ok_in_row = time.time() + max_seconds, 0
+    while time.time() < end:
+        ok_in_row = ok_in_row + 1 if up() else 0
+        if ok_in_row >= 2:
+            return True
+        time.sleep(poll)
+    return False
+
+
+def cleanup_prefix(prefix):
+    """Delete the A4H objects of an interrupted run (best effort; returns the number of objects found)."""
+    from .mcp_client import McpClient
+    from .runner import DELETE_ORDER, Runner, delete_uris
+    with McpClient() as m:
+        objs = Runner("", "")._objects_with_prefix(m, prefix.upper())
+    objs.sort(key=lambda o: DELETE_ORDER.index(o["objectType"]) if o["objectType"] in DELETE_ORDER else 99)
+    if objs:
+        delete_uris([o["uri"] for o in objs])
+    return len(objs)