Hard batch G0022-G0026: 5/5 accepted; budget floor, CDS checks, mutation check
- generator: budget floor (2x oracle activations, 3x calls), static checks for CDS $parameters and UNION annotation - harness/mutation.py: deterministic mutants of the reference; hidden tests must fail - G0022 revalidated with harness fixes: oracle 100, null 0 - results in docs/faz1-tasarim.md 11f Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
@@ -49,8 +49,10 @@ Rules for the bundle:
|
||||
- Method parameters need a complete type: "TYPE c LENGTH 4" is not allowed in a signature.
|
||||
Declare a type first (TYPES ty_zone TYPE c LENGTH 4) and use it.
|
||||
- Field names in tables and CDS views are not SQL reserved words (no HOURS, MODE, ORDER, DATE, COUNT ...).
|
||||
- CDS: use "define view entity". Parameters have no default value. In a UNION, all branches have the same
|
||||
key elements and element names. Hidden tests for a view with parameters pass all parameters.
|
||||
- CDS: use "define view entity". Parameters have no default value. Use a parameter as
|
||||
$parameters.p_name (not :p_name). In a UNION, all branches have the same
|
||||
key elements and element names, and the view
|
||||
has the annotation @Metadata.ignorePropagatedAnnotations: true. Hidden tests for a view with parameters pass all parameters.
|
||||
- Legacy seed code (categories E, I) must activate on SAP_BASIS 816: do not mix old and new syntax in one
|
||||
SQL statement (with new syntax, every host variable needs "@").
|
||||
- The contract lists only the objects that the hidden tests call. Test classes are never in the contract.
|
||||
@@ -178,6 +180,14 @@ def lint_files(b):
|
||||
for f in re.findall(r"^\s*(?:key\s+)?(\w+)\s*:", src, re.I | re.M):
|
||||
if f.upper() in RESERVED:
|
||||
errs.append(f"{rel}: field name {f} is a reserved word; choose another name")
|
||||
for rel, src in b.get("files", {}).items():
|
||||
if rel.endswith(".asddls") and re.search(r"define\s+(root\s+)?view\s+entity", src, re.I):
|
||||
if re.search(r"\bunion\b", src, re.I) and not re.search(r"@Metadata\.ignorePropagatedAnnotations\s*:\s*true",
|
||||
src, re.I):
|
||||
errs.append(f"{rel}: a view entity with UNION needs @Metadata.ignorePropagatedAnnotations: true")
|
||||
for p in sorted(set(re.findall(r"[=<>(,]\s*:(\w+)|\bbetween\s+:(\w+)|\band\s+:(\w+)", src, re.I))):
|
||||
name = next(x for x in p if x)
|
||||
errs.append(f"{rel}: parameter :{name}; a view entity needs $parameters.{name}")
|
||||
for k in ("seed", "reference"):
|
||||
for o in t.get(k, []):
|
||||
m = DDL_KIND.search(b.get("files", {}).get(o.get("file", ""), ""))
|
||||
@@ -279,11 +289,23 @@ def dependency_order(objs, files):
|
||||
return out
|
||||
|
||||
|
||||
def floor_budget(task):
|
||||
"""The model needs more calls than the oracle (reads, repairs). G0022: budget 12 activations,
|
||||
reference alone 13. Minimum: 2 x oracle activations, 3 x oracle tool calls."""
|
||||
ref = task.get("reference", [])
|
||||
pushes = sum(bool(o.get("file")) + bool(o.get("testclasses_file")) for o in ref)
|
||||
calls = len(ref) + pushes
|
||||
bud = task.setdefault("budget", {})
|
||||
bud["max_activations"] = max(int(bud.get("max_activations", 0)), 2 * pushes)
|
||||
bud["max_tool_calls"] = max(int(bud.get("max_tool_calls", 0)), 3 * calls)
|
||||
|
||||
|
||||
def write_bundle(b, task_dir, task_id):
|
||||
os.makedirs(task_dir, exist_ok=True)
|
||||
b["task"]["id"] = task_id
|
||||
for k in ("seed", "reference"):
|
||||
b["task"][k] = dependency_order(b["task"].get(k, []), b["files"])
|
||||
floor_budget(b["task"])
|
||||
json.dump(b["task"], open(os.path.join(task_dir, "task.json"), "w"), indent=2)
|
||||
for rel, content in b["files"].items():
|
||||
path = os.path.join(task_dir, rel)
|
||||
|
||||
188
harness/mutation.py
Normal file
188
harness/mutation.py
Normal file
@@ -0,0 +1,188 @@
|
||||
"""Mutation check (step D): the hidden tests must fail on a broken reference.
|
||||
|
||||
Small deterministic changes (mutants) go into the main source of the contract objects of the reference.
|
||||
For each mutant the oracle writes the mutated reference and the harness runs the hidden tests.
|
||||
killed = reference active and at least one hidden test fails
|
||||
survived = reference active and all hidden tests pass (a test gap or an equivalent mutant)
|
||||
invalid = the mutant does not activate (not counted)
|
||||
|
||||
python3 -m harness.mutation G0002 G0003 --pool eval --run-base 5000
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
|
||||
from .adt_client import load_env
|
||||
from .agents import OracleAgent
|
||||
from .runner import Runner
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MAX_MUTANTS = 5
|
||||
MIN_VALID = 3
|
||||
MIN_KILL_RATE = 0.75
|
||||
|
||||
# (kind, pattern, replacement); patterns match only outside comments and literals
|
||||
ABAP_OPS = [
|
||||
("rel", r"(?<=\s)>=(?=\s)", "<"), ("rel", r"(?<=\s)<=(?=\s)", ">"),
|
||||
("rel", r"(?<=\s)>(?=\s)", ">="), ("rel", r"(?<=\s)<(?=\s)", "<="),
|
||||
("rel", r"(?<=\s)<>(?=\s)", "="),
|
||||
("rel", r"\bGE\b", "LT"), ("rel", r"\bLE\b", "GT"), ("rel", r"\bGT\b", "GE"), ("rel", r"\bLT\b", "LE"),
|
||||
("rel", r"\bNE\b", "EQ"),
|
||||
("logic", r"(?<=\s)AND(?=\s)", "OR"), ("logic", r"(?<=\s)OR(?=\s)", "AND"),
|
||||
("arith", r"(?<=\s)\+(?=\s)", "-"), ("arith", r"(?<=\s)-(?=\s)", "+"), ("arith", r"(?<=\s)\*(?=\s)", "/"),
|
||||
("bool", r"\babap_true\b", "abap_false"), ("bool", r"\babap_false\b", "abap_true"),
|
||||
("const", r"(?<![\w.'-])([1-9]\d{0,5})(?![\w.'])", None), # integer literal + 1
|
||||
]
|
||||
CDS_OPS = [
|
||||
("rel", r"(?<=\s)>=(?=\s)", "<"), ("rel", r"(?<=\s)<=(?=\s)", ">"),
|
||||
("rel", r"(?<=\s)>(?=\s)", ">="), ("rel", r"(?<=\s)<(?=\s)", "<="), ("rel", r"(?<=\s)<>(?=\s)", "="),
|
||||
("logic", r"(?<=\s)and(?=\s)", "or"),
|
||||
("arith", r"(?<=\s)\+(?=\s)", "-"), ("arith", r"(?<=\s)-(?=\s)", "+"),
|
||||
("agg", r"\bsum\s*\(", "max("), ("agg", r"\bcount\s*\(", "max("), ("agg", r"\bavg\s*\(", "max("),
|
||||
("join", r"\binner\s+join\b", "left outer join"),
|
||||
("const", r"(?<![\w.'-])([1-9]\d{0,5})(?![\w.'])", None),
|
||||
]
|
||||
|
||||
|
||||
def _mask(src, cds):
|
||||
"""True for each character inside a comment or a literal."""
|
||||
m = [False] * len(src)
|
||||
pats = ([r"//[^\n]*", r"/\*.*?\*/", r"'[^']*'", r"@[^\n]*"] if cds else
|
||||
[r'"[^\n]*', r"^\*[^\n]*", r"'(?:[^']|'')*'", r"`[^`]*`", r"\|(?:[^|\\]|\\.)*\|"])
|
||||
for p in pats:
|
||||
for x in re.finditer(p, src, re.M | re.S):
|
||||
for i in range(x.start(), x.end()):
|
||||
m[i] = True
|
||||
return m
|
||||
|
||||
|
||||
def _region(src, otype):
|
||||
"""Start and end of the part that may change: the implementation, not declarations or test classes."""
|
||||
start, end = 0, len(src)
|
||||
if otype == "CLAS":
|
||||
x = re.search(r"^\s*CLASS\s+\S+\s+IMPLEMENTATION", src, re.I | re.M)
|
||||
start = x.end() if x else len(src)
|
||||
if otype == "PROG":
|
||||
x = re.search(r"^\s*CLASS\s+\S+\s+DEFINITION[^.]*FOR\s+TESTING", src, re.I | re.M)
|
||||
end = x.start() if x else len(src)
|
||||
if otype == "DDLS":
|
||||
x = re.search(r"\bas\s+select\b|\bas\s+projection\b", src, re.I)
|
||||
start = x.start() if x else 0
|
||||
return start, end
|
||||
|
||||
|
||||
def _skip_line(line):
|
||||
"""Declarations and signatures: a change there gives syntax errors or no behavior change."""
|
||||
return re.match(r"\s*(DATA|TYPES|CONSTANTS|METHODS|CLASS-METHODS|PARAMETERS|SELECT-OPTIONS|"
|
||||
r"IMPORTING|EXPORTING|RETURNING|RAISING|TABLES|FIELD-SYMBOLS)\b", line, re.I)
|
||||
|
||||
|
||||
def mutants(src, otype, seed, n=MAX_MUTANTS):
|
||||
"""Up to n mutants as (description, source). Different kinds first, spread over the source."""
|
||||
cds = otype == "DDLS"
|
||||
mask = _mask(src, cds)
|
||||
lo, hi = _region(src, otype)
|
||||
sites = []
|
||||
for kind, pat, rep in (CDS_OPS if cds else ABAP_OPS):
|
||||
for x in re.finditer(pat, src, re.I):
|
||||
if not lo <= x.start() < hi or mask[x.start()]:
|
||||
continue
|
||||
line_start = src.rfind("\n", 0, x.start()) + 1
|
||||
line_end = src.find("\n", x.start())
|
||||
line = src[line_start:line_end if line_end >= 0 else len(src)]
|
||||
if _skip_line(line) or (kind == "const" and re.search(r"\bLENGTH\b|\bDECIMALS\b|\(\s*\d+\s*,", line, re.I)):
|
||||
continue
|
||||
new = str(int(x.group(1)) + 1) if rep is None else rep
|
||||
sites.append((kind, x.start(), x.end(), new, src.count("\n", 0, x.start()) + 1, x.group(0)))
|
||||
rnd = random.Random(seed)
|
||||
rnd.shuffle(sites)
|
||||
chosen, kinds, lines = [], set(), set()
|
||||
for prefer_new in (True, False):
|
||||
for s in sites:
|
||||
if len(chosen) >= n:
|
||||
break
|
||||
if s in chosen or s[4] in lines or (prefer_new and s[0] in kinds):
|
||||
continue
|
||||
chosen.append(s)
|
||||
kinds.add(s[0])
|
||||
lines.add(s[4])
|
||||
out = []
|
||||
for kind, a, b, new, line, old in sorted(chosen, key=lambda s: s[1]):
|
||||
out.append((f"line {line}: {old.strip()} -> {new} ({kind})", src[:a] + new + src[b:]))
|
||||
return out
|
||||
|
||||
|
||||
def check_task(pool_root, task_id, run_base, n=MAX_MUTANTS, keep=False):
|
||||
"""Run the mutants of one task. Returns a summary dict; writes it to <task>/mutation.json."""
|
||||
task_dir = os.path.join(pool_root, task_id)
|
||||
meta = json.load(open(os.path.join(task_dir, "task.json")))
|
||||
contract = {c["name"].upper() for c in meta["contract"]}
|
||||
targets = [o for o in meta["reference"] if o["name"].upper() in contract and o.get("file")
|
||||
and o["type"] in ("CLAS", "FUNC", "PROG", "DDLS")]
|
||||
work = os.path.join(ROOT, "runs", "gen", "mut")
|
||||
os.makedirs(work, exist_ok=True)
|
||||
runner = Runner(os.path.join(work, "pool"), os.path.join(work, "runs"))
|
||||
# candidates per object, then round robin: objects without mutation sites (exception classes) give their share
|
||||
cand = [[(o, d, m) for d, m in mutants(open(os.path.join(task_dir, o["file"])).read(), o["type"],
|
||||
f"{task_id}:{o['name']}", n)] for o in targets]
|
||||
plan = []
|
||||
while len(plan) < n and any(cand):
|
||||
for c in cand:
|
||||
if c and len(plan) < n:
|
||||
plan.append(c.pop(0))
|
||||
results = []
|
||||
for k, (o, desc, msrc) in enumerate(plan):
|
||||
mdir = os.path.join(work, "pool", task_id)
|
||||
shutil.rmtree(mdir, ignore_errors=True)
|
||||
shutil.copytree(task_dir, mdir)
|
||||
open(os.path.join(mdir, o["file"]), "w").write(msrc)
|
||||
rep, _ = runner.run(task_id, OracleAgent(), run_base + k)
|
||||
h = rep.get("hidden_tests") or {}
|
||||
g = rep.get("gates") or {}
|
||||
if not g.get("G1_active") or not h.get("total"):
|
||||
status = "invalid"
|
||||
elif h["passed"] < h["total"]:
|
||||
status = "killed"
|
||||
else:
|
||||
status = "survived"
|
||||
results.append({"object": o["name"], "mutant": desc, "status": status,
|
||||
"hidden": f"{h.get('passed')}/{h.get('total')}",
|
||||
"failed_tests": [d["method"] for d in h.get("detail", []) if not d["ok"]]})
|
||||
if keep and status == "killed": # candidate faulty reference for own-test scoring
|
||||
fdir = os.path.join(task_dir, "faulty")
|
||||
os.makedirs(fdir, exist_ok=True)
|
||||
open(os.path.join(fdir, f"m{k}_{os.path.basename(o['file'])}"), "w").write(msrc)
|
||||
shutil.rmtree(os.path.join(work, "pool", task_id), ignore_errors=True)
|
||||
valid = [r for r in results if r["status"] != "invalid"]
|
||||
killed = [r for r in valid if r["status"] == "killed"]
|
||||
summary = {"task": task_id, "mutants": results, "valid": len(valid), "killed": len(killed),
|
||||
"kill_rate": round(len(killed) / len(valid), 2) if valid else None,
|
||||
"ok": len(valid) >= MIN_VALID and len(killed) / max(1, len(valid)) >= MIN_KILL_RATE}
|
||||
if len(valid) < MIN_VALID:
|
||||
summary["note"] = f"fewer than {MIN_VALID} valid mutants; check by review"
|
||||
json.dump(summary, open(os.path.join(task_dir, "mutation.json"), "w"), indent=1)
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
load_env(os.path.join(ROOT, ".env"))
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("tasks", nargs="+")
|
||||
ap.add_argument("--pool", default="eval")
|
||||
ap.add_argument("--run-base", type=int, required=True)
|
||||
ap.add_argument("-n", type=int, default=MAX_MUTANTS)
|
||||
ap.add_argument("--keep", action="store_true", help="store killed mutants in <task>/faulty/")
|
||||
a = ap.parse_args()
|
||||
root = os.path.join(ROOT, "tasks_gen", a.pool) if a.pool != "tasks" else os.path.join(ROOT, "tasks")
|
||||
for i, t in enumerate(a.tasks):
|
||||
s = check_task(root, t, a.run_base + 10 * i, a.n, a.keep)
|
||||
print(json.dumps({k: v for k, v in s.items() if k != "mutants"}), flush=True)
|
||||
for m in s["mutants"]:
|
||||
print(" ", m["status"], m["object"], m["mutant"], m["hidden"], flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user