Mutation in generator; category H (stop) scoring and judge; category K variants and tool schema variant; evalset plan

- mutation.py: negation mutants, skip WHILE/DO blocks (endless loop blocked RFC ~8 min)
- generator: accept only after mutation check; survivors go back as repair feedback
- runner/judge.py: stop tasks scored 100/30/0; gap by keywords, else judge model
- proxy: tool schema variant generic_v0 (draft); generator make_k_variant
- evalset.py: 88 slots for step F (A-I, H, K)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-03 05:37:20 +02:00
parent 32f03fcad4
commit 0ba25faeea
20 changed files with 1303 additions and 22 deletions

View File

@@ -6,6 +6,7 @@ import subprocess
import time
from .adt_client import AdtClient, load_env
from .judge import gap_named
from .mcp_client import McpClient
from .proxy import ToolProxy
from .task import Task, prefix_for
@@ -199,7 +200,7 @@ class Runner:
rep["rescored"] = True
else:
proxy = ToolProxy(mcp, prefix, task.meta.get("budget", {}),
os.path.join(run_dir, "trajectory.jsonl"))
os.path.join(run_dir, "trajectory.jsonl"), task.meta.get("tool_schema"))
proxy.note("spec", task.spec)
t1 = time.time()
rep["final_report"] = agent.run(task, proxy)
@@ -363,8 +364,11 @@ class Runner:
def _score(self, task, rep):
g = rep["gates"]
if task.meta.get("expected_outcome") == "stop":
return {"total": None, "note": "stop-task scoring not implemented in skeleton"}
if task.meta.get("expected_outcome") == "stop": # category H, docs/faz1-tasarim.md 6.3
if rep.get("model_objects"):
return {"total": 0, "stop": "implemented", "objects": rep["model_objects"]}
named, detail = gap_named(rep.get("final_report"), task.meta.get("expected_gap", {}))
return {"total": 100 if named else 30, "stop": "stopped", "gap": detail}
if not all(g.values()):
return {"total": 0, "failed_gates": [k for k, v in g.items() if not v]}
h = rep["hidden_tests"]