0ba25faeea
- mutation.py: negation mutants, skip WHILE/DO blocks (endless loop blocked RFC ~8 min) - generator: accept only after mutation check; survivors go back as repair feedback - runner/judge.py: stop tasks scored 100/30/0; gap by keywords, else judge model - proxy: tool schema variant generic_v0 (draft); generator make_k_variant - evalset.py: 88 slots for step F (A-I, H, K) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
57 lines
2.3 KiB
Python
57 lines
2.3 KiB
Python
"""Pilot: generate and validate a list of tasks, one after the other."""
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
from .adt_client import load_env
|
|
from .generator import ROOT, generate
|
|
from .ledger import BudgetExceeded, spent
|
|
|
|
PLAN = [
|
|
("CLAS", "A"), ("CLAS", "B"), ("CLAS", "C"), ("CLAS", "D"), ("CLAS", "E"),
|
|
("CLAS", "F"), ("CLAS", "G"), ("CLAS", "I"), ("CLAS", "C"),
|
|
("FUNC", "A"), ("FUNC", "F"), ("FUNC", "I"),
|
|
("PROG", "B"), ("PROG", "B"), ("PROG", "E"),
|
|
("DDLS", "B"), ("DDLS", "B"), ("DDLS", "B"), ("DDLS", "B"), ("DDLS", "B"),
|
|
]
|
|
# hard batch after the pilot: the types that failed in the pilot (2026-10-03)
|
|
HARD = [("CLAS", "D", None), ("CLAS", "E", "legacy price or freight calculation with packed numbers"),
|
|
("DDLS", "B", "view with input parameters"), ("DDLS", "B", "UNION of two tables with a literal source element"),
|
|
("FUNC", "A", None)]
|
|
DDLS_TOPICS = ["join of two tables and a calculated element", "association and path expression",
|
|
"aggregation with CASE and GROUP BY", "view with input parameters",
|
|
"UNION of two tables with a literal source element"]
|
|
|
|
|
|
def main():
|
|
"""python3 -m harness.pilot FIRST_ID [hard RUN_BASE]"""
|
|
load_env(os.path.join(ROOT, ".env"))
|
|
first = int(sys.argv[1]) if len(sys.argv) > 1 else 2
|
|
hard = len(sys.argv) > 2 and sys.argv[2] == "hard"
|
|
run_base = int(sys.argv[3]) if len(sys.argv) > 3 else 1100
|
|
name = "hard" if hard else "pilot"
|
|
plan = HARD if hard else [(o, c, None) for o, c in PLAN]
|
|
results = []
|
|
d = 0
|
|
for k, (otype, cat, topic) in enumerate(plan):
|
|
tid = f"G{first + k:04d}"
|
|
if otype == "DDLS" and not hard:
|
|
topic, d = DDLS_TOPICS[d], d + 1
|
|
try:
|
|
log = generate(tid, "eval", otype, cat, 2, "deepseek-v4.1-flash:cloud",
|
|
os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1"),
|
|
run_base + 40 * k, topic)
|
|
except BudgetExceeded as e:
|
|
print("BUDGET", e, flush=True)
|
|
break
|
|
except Exception as e: # noqa: BLE001
|
|
log = {"id": tid, "error": str(e)[:500]}
|
|
log["spent_total"] = spent()
|
|
results.append(log)
|
|
print(json.dumps(log), flush=True)
|
|
json.dump(results, open(os.path.join(ROOT, "runs", "gen", f"{name}.json"), "w"), indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|