Mutation in generator; category H (stop) scoring and judge; category K variants and tool schema variant; evalset plan

- mutation.py: negation mutants, skip WHILE/DO blocks (endless loop blocked RFC ~8 min)
- generator: accept only after mutation check; survivors go back as repair feedback
- runner/judge.py: stop tasks scored 100/30/0; gap by keywords, else judge model
- proxy: tool schema variant generic_v0 (draft); generator make_k_variant
- evalset.py: 88 slots for step F (A-I, H, K)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-03 05:37:20 +02:00
parent 32f03fcad4
commit 0ba25faeea
20 changed files with 1303 additions and 22 deletions

View File

@@ -21,16 +21,18 @@ from .runner import Runner
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MAX_MUTANTS = 5
MIN_VALID = 3
MIN_VALID = 2 # small references (T14, T15) have only a few mutation sites
MIN_KILL_RATE = 0.75
# (kind, pattern, replacement); patterns match only outside comments and literals
# Relational operators are negated, not shifted: a shifted boundary (< to <=) is often equivalent in
# clamp code (T13: "IF fee < 1. fee = 1."), so it says nothing about the tests.
ABAP_OPS = [
("rel", r"(?<=\s)>=(?=\s)", "<"), ("rel", r"(?<=\s)<=(?=\s)", ">"),
("rel", r"(?<=\s)>(?=\s)", ">="), ("rel", r"(?<=\s)<(?=\s)", "<="),
("rel", r"(?<=\s)<>(?=\s)", "="),
("rel", r"\bGE\b", "LT"), ("rel", r"\bLE\b", "GT"), ("rel", r"\bGT\b", "GE"), ("rel", r"\bLT\b", "LE"),
("rel", r"\bNE\b", "EQ"),
("rel", r"(?<=\s)>(?=\s)", "<="), ("rel", r"(?<=\s)<(?=\s)", ">="),
("rel", r"(?<=\s)<>(?=\s)", "="), ("eq", r"(?<=\s)=(?=\s)", "<>"),
("rel", r"\bGE\b", "LT"), ("rel", r"\bLE\b", "GT"), ("rel", r"\bGT\b", "LE"), ("rel", r"\bLT\b", "GE"),
("rel", r"\bNE\b", "EQ"), ("rel", r"\bEQ\b", "NE"),
("logic", r"(?<=\s)AND(?=\s)", "OR"), ("logic", r"(?<=\s)OR(?=\s)", "AND"),
("arith", r"(?<=\s)\+(?=\s)", "-"), ("arith", r"(?<=\s)-(?=\s)", "+"), ("arith", r"(?<=\s)\*(?=\s)", "/"),
("bool", r"\babap_true\b", "abap_false"), ("bool", r"\babap_false\b", "abap_true"),
@@ -38,13 +40,16 @@ ABAP_OPS = [
]
CDS_OPS = [
("rel", r"(?<=\s)>=(?=\s)", "<"), ("rel", r"(?<=\s)<=(?=\s)", ">"),
("rel", r"(?<=\s)>(?=\s)", ">="), ("rel", r"(?<=\s)<(?=\s)", "<="), ("rel", r"(?<=\s)<>(?=\s)", "="),
("rel", r"(?<=\s)>(?=\s)", "<="), ("rel", r"(?<=\s)<(?=\s)", ">="), ("rel", r"(?<=\s)<>(?=\s)", "="),
("eq", r"(?<=\s)=(?=\s)", "<>"),
("logic", r"(?<=\s)and(?=\s)", "or"),
("arith", r"(?<=\s)\+(?=\s)", "-"), ("arith", r"(?<=\s)-(?=\s)", "+"),
("agg", r"\bsum\s*\(", "max("), ("agg", r"\bcount\s*\(", "max("), ("agg", r"\bavg\s*\(", "max("),
("agg", r"\bsum\s*\(", "max("), ("agg", r"\bavg\s*\(", "max("),
("join", r"\binner\s+join\b", "left outer join"),
("const", r"(?<![\w.'-])([1-9]\d{0,5})(?![\w.'])", None),
]
# "=" is an assignment in ABAP; it is a comparison only in these lines
ABAP_CONDITION = re.compile(r"\s*(IF|ELSEIF|CHECK|WHILE|WHERE|AND|OR|ON)\b", re.I)
def _mask(src, cds):
@@ -74,6 +79,18 @@ def _region(src, otype):
return start, end
def _loop_blocks(src):
"""Ranges of WHILE ... ENDWHILE and DO ... ENDDO. A mutant there can make an endless loop; the hidden
test then blocks the one RFC connection of the server (G0002: 'remaining - step' to '+')."""
out, stack = [], []
for x in re.finditer(r"^\s*(WHILE|DO|ENDWHILE|ENDDO)\b", src, re.I | re.M):
if x.group(1).upper() in ("WHILE", "DO"):
stack.append(x.start())
elif stack:
out.append((stack.pop(), x.end()))
return out
def _skip_line(line):
"""Declarations and signatures: a change there gives syntax errors or no behavior change."""
return re.match(r"\s*(DATA|TYPES|CONSTANTS|METHODS|CLASS-METHODS|PARAMETERS|SELECT-OPTIONS|"
@@ -85,14 +102,19 @@ def mutants(src, otype, seed, n=MAX_MUTANTS):
cds = otype == "DDLS"
mask = _mask(src, cds)
lo, hi = _region(src, otype)
loops = [] if cds else _loop_blocks(src)
sites = []
for kind, pat, rep in (CDS_OPS if cds else ABAP_OPS):
for x in re.finditer(pat, src, re.I):
if not lo <= x.start() < hi or mask[x.start()]:
if not lo <= x.start() < hi or mask[x.start()] or any(a <= x.start() < b for a, b in loops):
continue
line_start = src.rfind("\n", 0, x.start()) + 1
line_end = src.find("\n", x.start())
line = src[line_start:line_end if line_end >= 0 else len(src)]
if kind == "eq" and not cds and not ABAP_CONDITION.match(line):
continue
if kind == "arith" and re.search(r"(\(|\bSELECT)\s*$", src[line_start:x.start()], re.I):
continue # COUNT( * ), SELECT *
if _skip_line(line) or (kind == "const" and re.search(r"\bLENGTH\b|\bDECIMALS\b|\(\s*\d+\s*,", line, re.I)):
continue
new = str(int(x.group(1)) + 1) if rep is None else rep
@@ -104,7 +126,7 @@ def mutants(src, otype, seed, n=MAX_MUTANTS):
for s in sites:
if len(chosen) >= n:
break
if s in chosen or s[4] in lines or (prefer_new and s[0] in kinds):
if s in chosen or (prefer_new and (s[4] in lines or s[0] in kinds)):
continue
chosen.append(s)
kinds.add(s[0])