3550f0b651
- docs/eval-inceleme.md checklist; harness/review.py - 19 accept, 1 fix pending (G0019), 4 flagged (I/E tasks solvable without legacy code) - generator: categories I and E fix/refactor the seed object in place Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
89 lines
3.5 KiB
Python
89 lines
3.5 KiB
Python
"""Review helper for eval tasks (docs/eval-inceleme.md).
|
|
|
|
python3 -m harness.review G0100 # print one task in a compact form
|
|
python3 -m harness.review --list # accepted tasks and their review state
|
|
python3 -m harness.review G0100 --set accept|fix|flag|reject --note "..." [--checks 2=eksik_test,8=sizinti]
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
|
|
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
POOL = os.path.join(ROOT, "tasks_gen", "eval")
|
|
|
|
|
|
def _load(tid, name, default=None):
|
|
try:
|
|
return json.load(open(os.path.join(POOL, tid, name)))
|
|
except (OSError, ValueError):
|
|
return default
|
|
|
|
|
|
def show(tid):
|
|
d = os.path.join(POOL, tid)
|
|
t = _load(tid, "task.json", {})
|
|
g = _load(tid, "generation.json", {})
|
|
m = _load(tid, "mutation.json")
|
|
print(f"== {tid} category {t.get('category')} type {t.get('object_type')} release {t.get('release_target')}"
|
|
f" outcome {t.get('expected_outcome', 'implement')}")
|
|
if t.get("base_task"):
|
|
print(f"K variant of {t['base_task']} ({t.get('input_style')}, tool schema {t.get('tool_schema')})")
|
|
if t.get("expected_gap"):
|
|
print("expected_gap:", json.dumps(t["expected_gap"]))
|
|
print("\n-- spec.md")
|
|
print(open(os.path.join(d, "spec.md")).read())
|
|
print("-- contract:", json.dumps(t.get("contract")))
|
|
print("-- seed:", [(o["type"], o["name"]) for o in t.get("seed", [])])
|
|
print("-- reference:", [(o["type"], o["name"]) for o in t.get("reference", [])])
|
|
for h in t.get("hidden_tests", []):
|
|
src = open(os.path.join(d, h["file"])).read()
|
|
names = re.findall(r"^\s*METHODS\s+(\w+)\s+FOR\s+TESTING", src, re.I | re.M)
|
|
print(f"-- hidden tests {h['name']} ({len(names)}):", ", ".join(names))
|
|
print("-- generation:", [(a.get("stage"), a.get("oracle"), a.get("null"), a.get("mutation")) for a in
|
|
g.get("attempts", [])], "accepted:", g.get("accepted"))
|
|
if m:
|
|
print(f"-- mutation: {m['killed']}/{m['valid']} ok={m['ok']}",
|
|
[x["mutant"] for x in m["mutants"] if x["status"] == "survived"])
|
|
r = _load(tid, "review.json")
|
|
if r:
|
|
print("-- review:", json.dumps(r))
|
|
|
|
|
|
def accepted():
|
|
out = []
|
|
for d in sorted(glob.glob(os.path.join(POOL, "G*"))):
|
|
tid = os.path.basename(d)
|
|
if (_load(tid, "generation.json") or {}).get("accepted"):
|
|
out.append(tid)
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("task", nargs="?")
|
|
ap.add_argument("--list", action="store_true")
|
|
ap.add_argument("--set", choices=["accept", "fix", "flag", "reject"])
|
|
ap.add_argument("--note", default="")
|
|
ap.add_argument("--checks", default="", help="failed checks, for example 2=eksik_test,8=sizinti")
|
|
a = ap.parse_args()
|
|
if a.list:
|
|
for tid in accepted():
|
|
t = _load(tid, "task.json", {})
|
|
r = _load(tid, "review.json") or {}
|
|
print(tid, t.get("category"), t.get("object_type"), r.get("decision", "-"), r.get("note", "")[:80])
|
|
return
|
|
if a.set:
|
|
rev = {"decision": a.set, "note": a.note, "by": "Claude", "date": time.strftime("%Y-%m-%d"),
|
|
"failed_checks": dict(x.split("=", 1) for x in a.checks.split(",") if "=" in x)}
|
|
json.dump(rev, open(os.path.join(POOL, a.task, "review.json"), "w"), indent=1)
|
|
print(json.dumps(rev))
|
|
return
|
|
show(a.task)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|