Lock leak: controlled reproduction attempts and method (not reproduced), enqueue reader; restart plan for 12 October

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

scripts_probe/deltest.py

@@ -0,0 +1,41 @@
+"""Delete an object through ADT while a write on it is running (what happened on 2026-10-05 19:25)."""
+import json, sys, threading, time
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness/scripts_probe")
+from lockprobe import *
+from killtest import MAIN, tc
+
+def attempt(delay, n, extra=40):
+    name = "ZPROBE0DL_%03d" % n
+    with McpClient() as m:
+        m.call("sap_create_object", {"objectType": "CLAS", "objectName": name, "packageName": "$TMP", "description": "delete race"})
+        m.call("sap_push_source", {"objectType": "CLAS", "objectName": name, "source": MAIN.format(n=name.lower())})
+    res = {}
+    def writer():
+        with McpClient() as m:
+            res["t_write_start"] = time.time()
+            e, t = m.call("sap_push_source", {"objectType": "CLAS", "objectName": name, "includeType": "testclasses", "source": tc(extra)})
+            res["write"] = t[:260]
+            res["t_write_end"] = time.time()
+    th = threading.Thread(target=writer); th.start()
+    time.sleep(delay)
+    res["delete_during_write"] = delete_uris(["/sap/bc/adt/oo/classes/" + name.lower()])
+    th.join()
+    time.sleep(2)
+    with McpClient() as m:
+        res["enqueue"] = read_locks(m)
+        e, t = m.call("sap_push_element", {"objectType": "CLAS", "objectName": name, "element": "RUN", "source": "  METHOD run.\n    rv = 2.\n  ENDMETHOD.\n"})
+        res["follow_up"] = t[:200]
+        e, t = m.call("sap_search_object", {"query": name})
+        res["still_exists"] = name in t
+    res["delete_after"] = delete_uris(["/sap/bc/adt/oo/classes/" + name.lower()]) if res["still_exists"] else "n/a"
+    res["name"], res["delay"] = name, delay
+    return res
+
+if __name__ == "__main__":
+    for i, d in enumerate(float(x) for x in sys.argv[1].split(",")):
+        r = attempt(d, int(sys.argv[2]) + i)
+        short = {k: v for k, v in r.items() if k not in ("t_write_start", "t_write_end")}
+        short["enqueue"] = short["enqueue"][:600]
+        print(json.dumps(short, indent=1))
+        if "[LOCK]" in r["follow_up"] or "enqueue entries matching the probe prefixes: 0" not in r["enqueue"]:
+            print("POSSIBLE LEAK at delay", d, r["name"]); break

scripts_probe/killcreate.py

@@ -0,0 +1,52 @@
+"""Two clients both CREATE and WRITE the same table at the same time (two controllers installing the seed of run 200273)."""
+import json, os, signal, subprocess, sys, time
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness/scripts_probe")
+from lockprobe import *
+from killtabl import DDL
+
+VICTIM2 = r'''
+import sys, json
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness")
+from harness.adt_client import load_env; load_env("/Users/erhankeseli/projects/abap-llm/harness/.env")
+from harness.mcp_client import McpClient
+a = json.loads(sys.argv[1])
+m = McpClient().open()
+print("SENT", flush=True)
+m.call("sap_create_object", {"objectType": "TABL", "objectName": a["objectName"], "packageName": "$TMP", "description": "seed"})
+print("CREATED", flush=True)
+m.call("sap_push_source", a)
+print("DONE", flush=True)
+'''
+def attempt(delay, n):
+    name = "ZPROBE0K3_%03d" % n
+    args = {"objectType": "TABL", "objectName": name, "source": DDL.format(n=name.lower(), w=20 + n)}
+    ps = [subprocess.Popen([sys.executable, "-c", VICTIM2, json.dumps(args)], stdout=subprocess.PIPE, text=True) for _ in range(2)]
+    for p in ps:
+        assert p.stdout.readline().strip() == "SENT"
+    time.sleep(delay)
+    states = []
+    for p in ps:
+        states.append("finished" if p.poll() is not None else "killed")
+        try: os.kill(p.pid, signal.SIGKILL)
+        except ProcessLookupError: pass
+        p.wait()
+    time.sleep(3)
+    out = {"name": name, "delay": delay, "victims": states}
+    with McpClient() as m:
+        out["enqueue"] = read_locks(m)
+        e, t = m.call("sap_push_source", {"objectType": "TABL", "objectName": name, "source": DDL.format(n=name.lower(), w=40 + n)})
+        out["follow_up"] = t[:300]
+        e, t = m.call("sap_search_object", {"query": name}); out["exists"] = name in t
+    out["leak"] = "[LOCK]" in out["follow_up"] or "ZPROBE0K3" in out["enqueue"]
+    if out["exists"]:
+        out["delete"] = {k.split("/")[-1]: (v["deleted"], v["msg"]) for k, v in delete_uris(["/sap/bc/adt/ddic/tables/" + name.lower()]).items()}
+    return out
+
+if __name__ == "__main__":
+    n = int(sys.argv[2])
+    for d in (float(x) for x in sys.argv[1].split(",")):
+        o = attempt(d, n); n += 1
+        o["enqueue"] = o["enqueue"][:1200]
+        print(json.dumps(o, indent=1))
+        if o["leak"]:
+            print("LEAK at delay", d, "object", o["name"]); break

scripts_probe/killtabl.py

@@ -0,0 +1,51 @@
+"""Controlled kill during the activation of a DDIC table (database table creation takes seconds)."""
+import json, os, signal, subprocess, sys, time
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness/scripts_probe")
+from lockprobe import *
+from killtest import VICTIM
+
+DDL = """@EndUserText.label : 'kill test'
+@AbapCatalog.enhancement.category : #NOT_EXTENSIBLE
+@AbapCatalog.tableCategory : #TRANSPARENT
+@AbapCatalog.deliveryClass : #A
+@AbapCatalog.dataMaintenance : #RESTRICTED
+define table {n} {{
+  key client : abap.clnt not null;
+  key item_id : abap.char(10) not null;
+  qty : abap.int4;
+  name : abap.char({w});
+}}"""
+
+def attempt(delay, n):
+    name = "ZPROBE0KT_%03d" % n
+    with McpClient() as m:
+        print("create", m.call("sap_create_object", {"objectType": "TABL", "objectName": name, "packageName": "$TMP", "description": "kill table"})[1][:50])
+        # first write alone, to learn how long it takes
+    args = {"objectType": "TABL", "objectName": name, "source": DDL.format(n=name.lower(), w=20 + n)}
+    p = subprocess.Popen([sys.executable, "-c", VICTIM, json.dumps(args)], stdout=subprocess.PIPE, text=True)
+    assert p.stdout.readline().strip() == "SENT"
+    t0 = time.time(); time.sleep(delay)
+    finished = p.poll() is not None
+    try:
+        os.kill(p.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        finished = True
+    p.wait()
+    print("table write: victim %s after %.2f s" % ("had finished" if finished else "killed", time.time() - t0))
+    time.sleep(3)
+    out = {"name": name, "delay": delay, "finished_before_kill": finished}
+    with McpClient() as m:
+        out["enqueue"] = read_locks(m)
+        e, t = m.call("sap_push_source", {"objectType": "TABL", "objectName": name, "source": DDL.format(n=name.lower(), w=40 + n)})
+        out["second_write"] = t[:300]
+    out["leak"] = "[LOCK]" in out["second_write"] or ("ZPROBE0KT" in out["enqueue"] and "enqueue entries matching the probe prefixes: 0" not in out["enqueue"])
+    out["delete"] = delete_uris(["/sap/bc/adt/ddic/tables/" + name.lower()])
+    return out
+
+if __name__ == "__main__":
+    for i, d in enumerate(float(x) for x in sys.argv[1].split(",")):
+        o = attempt(d, int(sys.argv[2]) + i)
+        o["enqueue"] = o["enqueue"][:900]
+        print(json.dumps(o, indent=1))
+        if o["leak"]:
+            print("LEAK at delay", d, "object", o["name"]); break

scripts_probe/killtest.py

@@ -0,0 +1,71 @@
+"""Controlled kill: a client sends sap_push_source (testclasses include) and is killed with SIGKILL `delay` seconds
+after the request left. Then the enqueue table is read and a second write is tried."""
+import json, os, signal, subprocess, sys, time
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness/scripts_probe")
+from lockprobe import *
+
+MAIN = """CLASS {n} DEFINITION PUBLIC FINAL CREATE PUBLIC.
+  PUBLIC SECTION.
+    METHODS run RETURNING VALUE(rv) TYPE i.
+ENDCLASS.
+CLASS {n} IMPLEMENTATION.
+  METHOD run.
+    rv = 1.
+  ENDMETHOD.
+ENDCLASS.
+"""
+def tc(extra):
+    body = "".join("    cl_abap_unit_assert=>assert_equals( act = %d exp = %d ).\n" % (i, i) for i in range(extra))
+    return ("CLASS ltc DEFINITION FINAL FOR TESTING DURATION SHORT RISK LEVEL HARMLESS.\n  PRIVATE SECTION.\n    METHODS t1 FOR TESTING.\n"
+            "ENDCLASS.\nCLASS ltc IMPLEMENTATION.\n  METHOD t1.\n" + body + "  ENDMETHOD.\nENDCLASS.\n")
+
+VICTIM = r'''
+import sys, json
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness")
+from harness.adt_client import load_env; load_env("/Users/erhankeseli/projects/abap-llm/harness/.env")
+from harness.mcp_client import McpClient
+args = json.loads(sys.argv[1])
+m = McpClient().open()
+print("SENT", flush=True)
+m.call("sap_push_source", args)
+print("DONE", flush=True)
+'''
+
+def attempt(delay, n, extra=40):
+    name = "ZPROBE0KL_%03d" % n
+    with McpClient() as m:
+        print("create", m.call("sap_create_object", {"objectType": "CLAS", "objectName": name, "packageName": "$TMP", "description": "kill test"})[1][:60])
+        print("main  ", m.call("sap_push_source", {"objectType": "CLAS", "objectName": name, "source": MAIN.format(n=name.lower())})[1][:70])
+    args = {"objectType": "CLAS", "objectName": name, "includeType": "testclasses", "source": tc(extra)}
+    p = subprocess.Popen([sys.executable, "-c", VICTIM, json.dumps(args)], stdout=subprocess.PIPE, text=True)
+    assert p.stdout.readline().strip() == "SENT"
+    t0 = time.time()
+    time.sleep(delay)
+    done_before_kill = p.poll() is not None
+    try:
+        os.kill(p.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        done_before_kill = True
+    p.wait()
+    print("victim killed %.2f s after the request was sent (finished before kill: %s)" % (time.time() - t0, done_before_kill))
+    time.sleep(3)
+    out = {"name": name, "delay": delay}
+    with McpClient() as m:
+        out["enqueue"] = read_locks(m)
+        e, t = m.call("sap_push_source", {"objectType": "CLAS", "objectName": name, "includeType": "testclasses", "source": tc(extra + 1)})
+        out["second_write"] = t[:300]
+        e, t = m.call("sap_push_element", {"objectType": "CLAS", "objectName": name, "element": "RUN", "source": "  METHOD run.\n    rv = 2.\n  ENDMETHOD.\n"})
+        out["push_element"] = t[:300]
+    out["leak"] = "currently editing" in out["second_write"] or "[LOCK]" in out["second_write"] or "[LOCK]" in out["push_element"]
+    if not out["leak"]:
+        out["delete"] = delete_uris(["/sap/bc/adt/oo/classes/" + name.lower()])
+    return out
+
+if __name__ == "__main__":
+    delays = [float(x) for x in sys.argv[1].split(",")]
+    n0 = int(sys.argv[2]) if len(sys.argv) > 2 else 1
+    for i, d in enumerate(delays):
+        o = attempt(d, n0 + i, int(sys.argv[3]) if len(sys.argv) > 3 else 40)
+        print(json.dumps({k: (v if k != "enqueue" else v[:700]) for k, v in o.items()}, indent=1))
+        if o["leak"]:
+            print("LEAK at delay", d, "- stopping, object", o["name"], "needs SM12"); break

scripts_probe/killtwo.py

@@ -0,0 +1,42 @@
+"""Two clients write the same table at the same time (what two controllers did with run 200273); both are killed."""
+import json, os, signal, subprocess, sys, time
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness/scripts_probe")
+from lockprobe import *
+from killtest import VICTIM
+from killtabl import DDL
+
+def attempt(delay, n):
+    name = "ZPROBE0K2_%03d" % n
+    with McpClient() as m:
+        m.call("sap_create_object", {"objectType": "TABL", "objectName": name, "packageName": "$TMP", "description": "two writers"})
+    args = {"objectType": "TABL", "objectName": name, "source": DDL.format(n=name.lower(), w=20 + n)}
+    ps = [subprocess.Popen([sys.executable, "-c", VICTIM, json.dumps(args)], stdout=subprocess.PIPE, text=True) for _ in range(2)]
+    for p in ps:
+        assert p.stdout.readline().strip() == "SENT"
+    time.sleep(delay)
+    states = []
+    for p in ps:
+        states.append("finished" if p.poll() is not None else "killed")
+        try:
+            os.kill(p.pid, signal.SIGKILL)
+        except ProcessLookupError:
+            pass
+        p.wait()
+    time.sleep(3)
+    out = {"name": name, "delay": delay, "victims": states}
+    with McpClient() as m:
+        out["enqueue"] = read_locks(m)
+        e, t = m.call("sap_push_source", {"objectType": "TABL", "objectName": name, "source": DDL.format(n=name.lower(), w=40 + n)})
+        out["follow_up"] = t[:300]
+    out["leak"] = "[LOCK]" in out["follow_up"] or ("ZPROBE0K2" in out["enqueue"])
+    out["delete"] = {k.split("/")[-1]: v["deleted"] for k, v in delete_uris(["/sap/bc/adt/ddic/tables/" + name.lower()]).items()}
+    return out
+
+if __name__ == "__main__":
+    n = int(sys.argv[2])
+    for d in (float(x) for x in sys.argv[1].split(",")):
+        o = attempt(d, n); n += 1
+        o["enqueue"] = o["enqueue"][:1200]
+        print(json.dumps(o, indent=1))
+        if o["leak"]:
+            print("LEAK at delay", d, "object", o["name"]); break

scripts_probe/lockprobe.py

@@ -0,0 +1,58 @@
+"""Lock leak probe helpers (no cloud, A4H only): the enqueue reader class and a controlled kill of a writing client."""
+import json, os, signal, subprocess, sys, time
+sys.path.insert(0, "/Users/erhankeseli/projects/abap-llm/harness")
+from harness.adt_client import load_env
+load_env("/Users/erhankeseli/projects/abap-llm/harness/.env")
+from harness.mcp_client import McpClient
+from harness.runner import delete_uris
+
+READER = "ZPROBE0EQ_LOCKS"
+READER_SRC = """CLASS zprobe0eq_locks DEFINITION PUBLIC FINAL CREATE PUBLIC.
+  PUBLIC SECTION.
+    INTERFACES if_oo_adt_classrun.
+ENDCLASS.
+CLASS zprobe0eq_locks IMPLEMENTATION.
+  METHOD if_oo_adt_classrun~main.
+    DATA lt_enq TYPE STANDARD TABLE OF seqg3 WITH DEFAULT KEY.
+    DATA lv_subrc TYPE sy-subrc.
+    CALL FUNCTION 'ENQUEUE_READ'
+      EXPORTING gclient = sy-mandt gname = '' garg = '' guname = ''
+      IMPORTING subrc = lv_subrc
+      TABLES enq = lt_enq
+      EXCEPTIONS communication_failure = 1 system_failure = 2 OTHERS = 3.
+    DATA(lv_n) = 0.
+    LOOP AT lt_enq ASSIGNING FIELD-SYMBOL(<l>) WHERE garg CS 'ZPROBE0' OR garg CS 'Z4AJ' OR garg CS 'Z4AE'.
+      lv_n = lv_n + 1.
+      DATA(lv_line) = ||.
+      DO.
+        ASSIGN COMPONENT sy-index OF STRUCTURE <l> TO FIELD-SYMBOL(<f>).
+        IF sy-subrc <> 0. EXIT. ENDIF.
+        DATA(lo_d) = CAST cl_abap_structdescr( cl_abap_typedescr=>describe_by_data( <l> ) ).
+        DATA(lv_name) = lo_d->components[ sy-index ]-name.
+        IF <f> IS NOT INITIAL.
+          lv_line = lv_line && lv_name && '=' && condense( |{ <f> }| ) && ' '.
+        ENDIF.
+      ENDDO.
+      out->write( lv_line ).
+    ENDLOOP.
+    out->write( |enqueue entries matching the probe prefixes: { lv_n } of { lines( lt_enq ) } (subrc { lv_subrc })| ).
+  ENDMETHOD.
+ENDCLASS.
+"""
+
+def ensure_reader(m):
+    e, t = m.call("sap_search_object", {"query": READER})
+    if READER not in t:
+        m.call("sap_create_object", {"objectType": "CLAS", "objectName": READER, "packageName": "$TMP", "description": "enqueue reader probe"})
+        e, t = m.call("sap_push_source", {"objectType": "CLAS", "objectName": READER, "source": READER_SRC})
+        if '"success":true' not in t.replace(" ", ""):
+            print("reader activation:", t[:600])
+
+def read_locks(m):
+    e, t = m.call("sap_run_class", {"className": READER})
+    return t
+
+if __name__ == "__main__":
+    with McpClient() as m:
+        ensure_reader(m)
+        print(read_locks(m)[:2500])