Mutation in generator; category H (stop) scoring and judge; category K variants and tool schema variant; evalset plan
- mutation.py: negation mutants, skip WHILE/DO blocks (endless loop blocked RFC ~8 min) - generator: accept only after mutation check; survivors go back as repair feedback - runner/judge.py: stop tasks scored 100/30/0; gap by keywords, else judge model - proxy: tool schema variant generic_v0 (draft); generator make_k_variant - evalset.py: 88 slots for step F (A-I, H, K) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,16 +21,18 @@ from .runner import Runner
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
MAX_MUTANTS = 5
|
||||
MIN_VALID = 3
|
||||
MIN_VALID = 2 # small references (T14, T15) have only a few mutation sites
|
||||
MIN_KILL_RATE = 0.75
|
||||
|
||||
# (kind, pattern, replacement); patterns match only outside comments and literals
|
||||
# Relational operators are negated, not shifted: a shifted boundary (< to <=) is often equivalent in
|
||||
# clamp code (T13: "IF fee < 1. fee = 1."), so it says nothing about the tests.
|
||||
ABAP_OPS = [
|
||||
("rel", r"(?<=\s)>=(?=\s)", "<"), ("rel", r"(?<=\s)<=(?=\s)", ">"),
|
||||
("rel", r"(?<=\s)>(?=\s)", ">="), ("rel", r"(?<=\s)<(?=\s)", "<="),
|
||||
("rel", r"(?<=\s)<>(?=\s)", "="),
|
||||
("rel", r"\bGE\b", "LT"), ("rel", r"\bLE\b", "GT"), ("rel", r"\bGT\b", "GE"), ("rel", r"\bLT\b", "LE"),
|
||||
("rel", r"\bNE\b", "EQ"),
|
||||
("rel", r"(?<=\s)>(?=\s)", "<="), ("rel", r"(?<=\s)<(?=\s)", ">="),
|
||||
("rel", r"(?<=\s)<>(?=\s)", "="), ("eq", r"(?<=\s)=(?=\s)", "<>"),
|
||||
("rel", r"\bGE\b", "LT"), ("rel", r"\bLE\b", "GT"), ("rel", r"\bGT\b", "LE"), ("rel", r"\bLT\b", "GE"),
|
||||
("rel", r"\bNE\b", "EQ"), ("rel", r"\bEQ\b", "NE"),
|
||||
("logic", r"(?<=\s)AND(?=\s)", "OR"), ("logic", r"(?<=\s)OR(?=\s)", "AND"),
|
||||
("arith", r"(?<=\s)\+(?=\s)", "-"), ("arith", r"(?<=\s)-(?=\s)", "+"), ("arith", r"(?<=\s)\*(?=\s)", "/"),
|
||||
("bool", r"\babap_true\b", "abap_false"), ("bool", r"\babap_false\b", "abap_true"),
|
||||
@@ -38,13 +40,16 @@ ABAP_OPS = [
|
||||
]
|
||||
CDS_OPS = [
|
||||
("rel", r"(?<=\s)>=(?=\s)", "<"), ("rel", r"(?<=\s)<=(?=\s)", ">"),
|
||||
("rel", r"(?<=\s)>(?=\s)", ">="), ("rel", r"(?<=\s)<(?=\s)", "<="), ("rel", r"(?<=\s)<>(?=\s)", "="),
|
||||
("rel", r"(?<=\s)>(?=\s)", "<="), ("rel", r"(?<=\s)<(?=\s)", ">="), ("rel", r"(?<=\s)<>(?=\s)", "="),
|
||||
("eq", r"(?<=\s)=(?=\s)", "<>"),
|
||||
("logic", r"(?<=\s)and(?=\s)", "or"),
|
||||
("arith", r"(?<=\s)\+(?=\s)", "-"), ("arith", r"(?<=\s)-(?=\s)", "+"),
|
||||
("agg", r"\bsum\s*\(", "max("), ("agg", r"\bcount\s*\(", "max("), ("agg", r"\bavg\s*\(", "max("),
|
||||
("agg", r"\bsum\s*\(", "max("), ("agg", r"\bavg\s*\(", "max("),
|
||||
("join", r"\binner\s+join\b", "left outer join"),
|
||||
("const", r"(?<![\w.'-])([1-9]\d{0,5})(?![\w.'])", None),
|
||||
]
|
||||
# "=" is an assignment in ABAP; it is a comparison only in these lines
|
||||
ABAP_CONDITION = re.compile(r"\s*(IF|ELSEIF|CHECK|WHILE|WHERE|AND|OR|ON)\b", re.I)
|
||||
|
||||
|
||||
def _mask(src, cds):
|
||||
@@ -74,6 +79,18 @@ def _region(src, otype):
|
||||
return start, end
|
||||
|
||||
|
||||
def _loop_blocks(src):
|
||||
"""Ranges of WHILE ... ENDWHILE and DO ... ENDDO. A mutant there can make an endless loop; the hidden
|
||||
test then blocks the one RFC connection of the server (G0002: 'remaining - step' to '+')."""
|
||||
out, stack = [], []
|
||||
for x in re.finditer(r"^\s*(WHILE|DO|ENDWHILE|ENDDO)\b", src, re.I | re.M):
|
||||
if x.group(1).upper() in ("WHILE", "DO"):
|
||||
stack.append(x.start())
|
||||
elif stack:
|
||||
out.append((stack.pop(), x.end()))
|
||||
return out
|
||||
|
||||
|
||||
def _skip_line(line):
|
||||
"""Declarations and signatures: a change there gives syntax errors or no behavior change."""
|
||||
return re.match(r"\s*(DATA|TYPES|CONSTANTS|METHODS|CLASS-METHODS|PARAMETERS|SELECT-OPTIONS|"
|
||||
@@ -85,14 +102,19 @@ def mutants(src, otype, seed, n=MAX_MUTANTS):
|
||||
cds = otype == "DDLS"
|
||||
mask = _mask(src, cds)
|
||||
lo, hi = _region(src, otype)
|
||||
loops = [] if cds else _loop_blocks(src)
|
||||
sites = []
|
||||
for kind, pat, rep in (CDS_OPS if cds else ABAP_OPS):
|
||||
for x in re.finditer(pat, src, re.I):
|
||||
if not lo <= x.start() < hi or mask[x.start()]:
|
||||
if not lo <= x.start() < hi or mask[x.start()] or any(a <= x.start() < b for a, b in loops):
|
||||
continue
|
||||
line_start = src.rfind("\n", 0, x.start()) + 1
|
||||
line_end = src.find("\n", x.start())
|
||||
line = src[line_start:line_end if line_end >= 0 else len(src)]
|
||||
if kind == "eq" and not cds and not ABAP_CONDITION.match(line):
|
||||
continue
|
||||
if kind == "arith" and re.search(r"(\(|\bSELECT)\s*$", src[line_start:x.start()], re.I):
|
||||
continue # COUNT( * ), SELECT *
|
||||
if _skip_line(line) or (kind == "const" and re.search(r"\bLENGTH\b|\bDECIMALS\b|\(\s*\d+\s*,", line, re.I)):
|
||||
continue
|
||||
new = str(int(x.group(1)) + 1) if rep is None else rep
|
||||
@@ -104,7 +126,7 @@ def mutants(src, otype, seed, n=MAX_MUTANTS):
|
||||
for s in sites:
|
||||
if len(chosen) >= n:
|
||||
break
|
||||
if s in chosen or s[4] in lines or (prefer_new and s[0] in kinds):
|
||||
if s in chosen or (prefer_new and (s[4] in lines or s[0] in kinds)):
|
||||
continue
|
||||
chosen.append(s)
|
||||
kinds.add(s[0])
|
||||
|
||||
Reference in New Issue
Block a user