Mutation in generator; category H (stop) scoring and judge; category K variants and tool schema variant; evalset plan
- mutation.py: negation mutants, skip WHILE/DO blocks (endless loop blocked RFC ~8 min) - generator: accept only after mutation check; survivors go back as repair feedback - runner/judge.py: stop tasks scored 100/30/0; gap by keywords, else judge model - proxy: tool schema variant generic_v0 (draft); generator make_k_variant - evalset.py: 88 slots for step F (A-I, H, K) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ import subprocess
|
||||
import time
|
||||
|
||||
from .adt_client import AdtClient, load_env
|
||||
from .judge import gap_named
|
||||
from .mcp_client import McpClient
|
||||
from .proxy import ToolProxy
|
||||
from .task import Task, prefix_for
|
||||
@@ -199,7 +200,7 @@ class Runner:
|
||||
rep["rescored"] = True
|
||||
else:
|
||||
proxy = ToolProxy(mcp, prefix, task.meta.get("budget", {}),
|
||||
os.path.join(run_dir, "trajectory.jsonl"))
|
||||
os.path.join(run_dir, "trajectory.jsonl"), task.meta.get("tool_schema"))
|
||||
proxy.note("spec", task.spec)
|
||||
t1 = time.time()
|
||||
rep["final_report"] = agent.run(task, proxy)
|
||||
@@ -363,8 +364,11 @@ class Runner:
|
||||
|
||||
def _score(self, task, rep):
|
||||
g = rep["gates"]
|
||||
if task.meta.get("expected_outcome") == "stop":
|
||||
return {"total": None, "note": "stop-task scoring not implemented in skeleton"}
|
||||
if task.meta.get("expected_outcome") == "stop": # category H, docs/faz1-tasarim.md 6.3
|
||||
if rep.get("model_objects"):
|
||||
return {"total": 0, "stop": "implemented", "objects": rep["model_objects"]}
|
||||
named, detail = gap_named(rep.get("final_report"), task.meta.get("expected_gap", {}))
|
||||
return {"total": 100 if named else 30, "stop": "stopped", "gap": detail}
|
||||
if not all(g.values()):
|
||||
return {"total": 0, "failed_gates": [k for k, v in g.items() if not v]}
|
||||
h = rep["hidden_tests"]
|
||||
|
||||
Reference in New Issue
Block a user