"""Review helper for eval tasks (docs/eval-inceleme.md). python3 -m harness.review G0100 # print one task in a compact form python3 -m harness.review --list # accepted tasks and their review state python3 -m harness.review G0100 --set accept|fix|flag|reject --note "..." [--checks 2=eksik_test,8=sizinti] """ import argparse import glob import json import os import re import time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) POOL = os.path.join(ROOT, "tasks_gen", "eval") def _load(tid, name, default=None): try: return json.load(open(os.path.join(POOL, tid, name))) except (OSError, ValueError): return default def show(tid): d = os.path.join(POOL, tid) t = _load(tid, "task.json", {}) g = _load(tid, "generation.json", {}) m = _load(tid, "mutation.json") print(f"== {tid} category {t.get('category')} type {t.get('object_type')} release {t.get('release_target')}" f" outcome {t.get('expected_outcome', 'implement')}") if t.get("base_task"): print(f"K variant of {t['base_task']} ({t.get('input_style')}, tool schema {t.get('tool_schema')})") if t.get("expected_gap"): print("expected_gap:", json.dumps(t["expected_gap"])) print("\n-- spec.md") print(open(os.path.join(d, "spec.md")).read()) print("-- contract:", json.dumps(t.get("contract"))) print("-- seed:", [(o["type"], o["name"]) for o in t.get("seed", [])]) print("-- reference:", [(o["type"], o["name"]) for o in t.get("reference", [])]) for h in t.get("hidden_tests", []): src = open(os.path.join(d, h["file"])).read() names = re.findall(r"^\s*METHODS\s+(\w+)\s+FOR\s+TESTING", src, re.I | re.M) print(f"-- hidden tests {h['name']} ({len(names)}):", ", ".join(names)) print("-- generation:", [(a.get("stage"), a.get("oracle"), a.get("null"), a.get("mutation")) for a in g.get("attempts", [])], "accepted:", g.get("accepted")) if m: print(f"-- mutation: {m['killed']}/{m['valid']} ok={m['ok']}", [x["mutant"] for x in m["mutants"] if x["status"] == "survived"]) r = _load(tid, "review.json") if r: print("-- review:", json.dumps(r)) def accepted(): out = [] for d in sorted(glob.glob(os.path.join(POOL, "G*"))): tid = os.path.basename(d) if (_load(tid, "generation.json") or {}).get("accepted"): out.append(tid) return out def main(): ap = argparse.ArgumentParser() ap.add_argument("task", nargs="?") ap.add_argument("--list", action="store_true") ap.add_argument("--set", choices=["accept", "fix", "flag", "reject"]) ap.add_argument("--note", default="") ap.add_argument("--checks", default="", help="failed checks, for example 2=eksik_test,8=sizinti") a = ap.parse_args() if a.list: for tid in accepted(): t = _load(tid, "task.json", {}) r = _load(tid, "review.json") or {} print(tid, t.get("category"), t.get("object_type"), r.get("decision", "-"), r.get("note", "")[:80]) return if a.set: rev = {"decision": a.set, "note": a.note, "by": "Claude", "date": time.strftime("%Y-%m-%d"), "failed_checks": dict(x.split("=", 1) for x in a.checks.split(",") if "=" in x)} json.dump(rev, open(os.path.join(POOL, a.task, "review.json"), "w"), indent=1) print(json.dumps(rev)) return show(a.task) if __name__ == "__main__": main()