63 lines
2.4 KiB
Python
63 lines
2.4 KiB
Python
"""Infrastructure outages: the MCP server (ADT in Eclipse, 127.0.0.1:3000) or A4H (localhost:50000) is not reachable.
|
|
|
|
On 2026-10-05 21:43 the MCP server refused connections for a short time; three runs got `Connection refused` and the
|
|
pipeline stopped because it counted three equal harness errors. An outage is not a model result and not a harness bug:
|
|
the run is cleaned up, the work waits until the services answer, and the same run is started again.
|
|
"""
|
|
import errno
|
|
import os
|
|
import socket
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
MCP_URL = os.environ.get("MCP_URL", "http://127.0.0.1:3000/mcp")
|
|
A4H_URL = os.environ.get("A4H_URL", "http://localhost:50000").rstrip("/") + "/sap/public/ping"
|
|
|
|
|
|
def is_outage(exc):
|
|
"""True for a refused or lost connection to a local service (not for an HTTP error status)."""
|
|
if isinstance(exc, urllib.error.HTTPError):
|
|
return False
|
|
if isinstance(exc, urllib.error.URLError):
|
|
return is_outage(exc.reason) if isinstance(exc.reason, BaseException) else True
|
|
return isinstance(exc, (ConnectionError, socket.timeout, TimeoutError)) or \
|
|
(isinstance(exc, OSError) and exc.errno in (errno.ECONNREFUSED, errno.ECONNRESET, errno.EHOSTUNREACH, errno.EPIPE))
|
|
|
|
|
|
def _answers(url):
|
|
try:
|
|
urllib.request.urlopen(url, timeout=6)
|
|
return True
|
|
except urllib.error.HTTPError:
|
|
return True # 401 / 403 / 404: the server answers
|
|
except Exception: # noqa: BLE001
|
|
return False
|
|
|
|
|
|
def up():
|
|
return _answers(MCP_URL) and _answers(A4H_URL)
|
|
|
|
|
|
def wait_until_up(max_seconds=3600, poll=30, log=print):
|
|
"""Wait until MCP and A4H answer twice in a row (a restart shows a short flicker). False when the time is over."""
|
|
end, ok_in_row = time.time() + max_seconds, 0
|
|
while time.time() < end:
|
|
ok_in_row = ok_in_row + 1 if up() else 0
|
|
if ok_in_row >= 2:
|
|
return True
|
|
time.sleep(poll)
|
|
return False
|
|
|
|
|
|
def cleanup_prefix(prefix):
|
|
"""Delete the A4H objects of an interrupted run (best effort; returns the number of objects found)."""
|
|
from .mcp_client import McpClient
|
|
from .runner import DELETE_ORDER, Runner, delete_uris
|
|
with McpClient() as m:
|
|
objs = Runner("", "")._objects_with_prefix(m, prefix.upper())
|
|
objs.sort(key=lambda o: DELETE_ORDER.index(o["objectType"]) if o["objectType"] in DELETE_ORDER else 99)
|
|
if objs:
|
|
delete_uris([o["uri"] for o in objs])
|
|
return len(objs)
|