Overlap limit for whole spec 0.75, slot logs; STATE and roadmap: stage 2 data progress

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

harness/overlap.py

@@ -13,7 +13,7 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 STOP = set("the a an of to in is are be and or for with that this it as on by at from not no if then each "
            "all any must shall can will when which its into than one two use used using value values "
            "object objects method methods class classes table tables field fields return returns".split())
-SPEC_LIMIT = 0.55   # cosine of word counts of the whole spec
+SPEC_LIMIT = 0.75   # cosine of word counts of the whole spec (same-category specs share boilerplate: 0.63-0.67 seen)
 GOAL_LIMIT = 0.60   # cosine of the Goal and Business rules sections
 NAME_LIMIT = 0.60   # Jaccard of name tokens (placeholder and Z prefix removed)
 

harness/trainset.py

@@ -104,8 +104,8 @@ def run(part, parts, target, stop_ledger):
     for idx, s in enumerate(plan["slots"]):
         if idx % parts != part:
             continue
-        if os.path.exists(os.path.join(POOL, s["id"], "generation.json")):
-            continue
+        if os.path.exists(os.path.join(POOL, "_logs", s["id"] + ".json")):
+            continue  # done or failed: not tried again (delete the log to retry)
         if accepted_count() >= plan["target"]:
             print("TARGET reached", flush=True)
             break
@@ -132,6 +132,11 @@ def run(part, parts, target, stop_ledger):
         except Exception as e:  # noqa: BLE001
             log = {"id": s["id"], "error": str(e)[:500]}
         log.update(error_kind=s.get("error_kind"), spent_total=spent())
+        os.makedirs(os.path.join(POOL, "_logs"), exist_ok=True)
+        json.dump(log, open(os.path.join(POOL, "_logs", s["id"] + ".json"), "w"), indent=1)
+        stray = os.path.join(POOL, "generation.json")  # generate() writes here when no task dir exists
+        if os.path.exists(stray):
+            os.remove(stray)
         print(json.dumps(log), flush=True)