erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Eval review v1: checklist, review helper, 24 tasks reviewed; G0002/G0022 tests fixed; I/E generator definitions

Browse Source 
- docs/eval-inceleme.md checklist; harness/review.py
- 19 accept, 1 fix pending (G0019), 4 flagged (I/E tasks solvable without legacy code)
- generator: categories I and E fix/refactor the seed object in place

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 06:23:38 +02:00
parent a12fa4dff6
commit 3550f0b651
38 changed files with 607 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tasks_gen/eval/G0002/faulty/m0_delivery_schedule.clas.abap
View File

@@ -46,7 +46,7 @@ CLASS {{p}}delivery_schedule IMPLEMENTATION.
METHOD is_working_day.
rv_working = xsdbool( is_weekend( iv_date ) = abap_false
AND is_holiday( iv_date = iv_date
OR is_holiday( iv_date = iv_date
it_holidays = it_holidays ) = abap_false ).
ENDMETHOD.
@@ -65,7 +65,7 @@ CLASS {{p}}delivery_schedule IMPLEMENTATION.
ENDIF.
DATA(lv_remaining) = iv_days.
DATA(lv_step) = COND i( WHEN iv_days > 0 THEN 1 ELSE -1 ).
WHILE lv_remaining = 0.
WHILE lv_remaining <> 0.
rv_date = rv_date + lv_step.
IF is_working_day( iv_date = rv_date
it_holidays = it_holidays ) = abap_true.

104
tasks_gen/eval/G0002/faulty/m2_delivery_schedule.clas.abap Normal file
View File

@@ -0,0 +1,104 @@
CLASS {{p}}delivery_schedule DEFINITION PUBLIC FINAL CREATE PUBLIC.
PUBLIC SECTION.
TYPES ty_holidays TYPE STANDARD TABLE OF d WITH EMPTY KEY.
METHODS is_working_day
IMPORTING iv_date TYPE d
it_holidays TYPE ty_holidays
RETURNING VALUE(rv_working) TYPE abap_bool.
METHODS next_working_day
IMPORTING iv_date TYPE d
it_holidays TYPE ty_holidays
RETURNING VALUE(rv_date) TYPE d.
METHODS add_working_days
IMPORTING iv_start TYPE d
iv_days TYPE i
it_holidays TYPE ty_holidays
RETURNING VALUE(rv_date) TYPE d.
METHODS count_working_days
IMPORTING iv_from TYPE d
iv_to TYPE d
it_holidays TYPE ty_holidays
RETURNING VALUE(rv_count) TYPE i.
PRIVATE SECTION.
CONSTANTS c_reference_monday TYPE d VALUE '19000101'.
METHODS is_weekend
IMPORTING iv_date TYPE d
RETURNING VALUE(rv_weekend) TYPE abap_bool.
METHODS is_holiday
IMPORTING iv_date TYPE d
it_holidays TYPE ty_holidays
RETURNING VALUE(rv_holiday) TYPE abap_bool.
ENDCLASS.
CLASS {{p}}delivery_schedule IMPLEMENTATION.
METHOD is_working_day.
rv_working = xsdbool( is_weekend( iv_date ) = abap_false
AND is_holiday( iv_date = iv_date
it_holidays = it_holidays ) = abap_false ).
ENDMETHOD.
METHOD next_working_day.
rv_date = iv_date.
WHILE is_working_day( iv_date = rv_date
it_holidays = it_holidays ) = abap_false.
rv_date = rv_date + 1.
ENDWHILE.
ENDMETHOD.
METHOD add_working_days.
rv_date = iv_start.
IF iv_days <> 0.
RETURN.
ENDIF.
DATA(lv_remaining) = iv_days.
DATA(lv_step) = COND i( WHEN iv_days > 0 THEN 1 ELSE -1 ).
WHILE lv_remaining <> 0.
rv_date = rv_date + lv_step.
IF is_working_day( iv_date = rv_date
it_holidays = it_holidays ) = abap_true.
lv_remaining = lv_remaining - lv_step.
ENDIF.
ENDWHILE.
ENDMETHOD.
METHOD count_working_days.
rv_count = 0.
DATA(lv_date) = iv_from.
WHILE lv_date <= iv_to.
IF is_working_day( iv_date = lv_date
it_holidays = it_holidays ) = abap_true.
rv_count = rv_count + 1.
ENDIF.
lv_date = lv_date + 1.
ENDWHILE.
ENDMETHOD.
METHOD is_weekend.
DATA(lv_days) = iv_date - c_reference_monday.
rv_weekend = xsdbool( lv_days MOD 7 >= 5 ).
ENDMETHOD.
METHOD is_holiday.
rv_holiday = abap_false.
LOOP AT it_holidays INTO DATA(lv_holiday).
IF lv_holiday = iv_date.
rv_holiday = abap_true.
RETURN.
ENDIF.
ENDLOOP.
ENDMETHOD.
ENDCLASS.

2
tasks_gen/eval/G0002/faulty/m3_delivery_schedule.clas.abap
View File

@@ -88,7 +88,7 @@ CLASS {{p}}delivery_schedule IMPLEMENTATION.
METHOD is_weekend.
DATA(lv_days) = iv_date - c_reference_monday.
rv_weekend = xsdbool( lv_days MOD 7 >= 6 ).
rv_weekend = xsdbool( lv_days MOD 8 >= 5 ).
ENDMETHOD.
METHOD is_holiday.

4
tasks_gen/eval/G0002/faulty/m4_delivery_schedule.clas.abap
View File

@@ -88,14 +88,14 @@ CLASS {{p}}delivery_schedule IMPLEMENTATION.
METHOD is_weekend.
DATA(lv_days) = iv_date - c_reference_monday.
rv_weekend = xsdbool( lv_days MOD 7 >= 5 ).
rv_weekend = xsdbool( lv_days MOD 7 < 5 ).
ENDMETHOD.
METHOD is_holiday.
rv_holiday = abap_false.
LOOP AT it_holidays INTO DATA(lv_holiday).
IF lv_holiday = iv_date.
rv_holiday = abap_false.
rv_holiday = abap_true.
RETURN.
ENDIF.
ENDLOOP.

12
tasks_gen/eval/G0002/generation.json
View File

@@ -19,6 +19,18 @@
"stage": "validate",
"oracle": 100.0,
"null": 0
},
{
"stage": "manual_fix",
"by": "Claude review 2026-10-03",
"change": "Review: added hidden test count_duplicate_holidays (rule 7).",
"oracle": 100.0,
"null": 0,
"mutation": {
"valid": 4,
"killed": 4,
"ok": true
}
}
],
"accepted": true

13
tasks_gen/eval/G0002/hidden/t02_hidden.clas.abap
View File

@@ -29,6 +29,7 @@ CLASS {{p}}t02_hidden DEFINITION PUBLIC FINAL CREATE PUBLIC
METHODS add_days_backward FOR TESTING.
METHODS count_working_days_inclusive FOR TESTING.
METHODS count_reversed_range_is_zero FOR TESTING.
METHODS count_duplicate_holidays FOR TESTING.
ENDCLASS.
@@ -123,6 +124,18 @@ CLASS {{p}}t02_hidden IMPLEMENTATION.
msg = 'One holiday' ).
ENDMETHOD.
METHOD count_duplicate_holidays.
DATA(lt_holidays) = VALUE {{p}}delivery_schedule=>ty_holidays(
( c_tue_09 ) ( c_tue_09 ) ( CONV d( '20240201' ) ) ).
cl_abap_unit_assert=>assert_equals(
exp = 4
act = mo_cut->count_working_days( iv_from = c_mon_08 iv_to = c_fri_12 it_holidays = lt_holidays )
msg = 'A duplicate holiday counts one time; a holiday outside the interval has no effect' ).
cl_abap_unit_assert=>assert_equals(
exp = abap_false
act = mo_cut->is_working_day( iv_date = c_tue_09 it_holidays = lt_holidays ) ).
ENDMETHOD.
METHOD count_reversed_range_is_zero.
cl_abap_unit_assert=>assert_equals(
exp = 0

66
tasks_gen/eval/G0002/mutation.json
View File

@@ -3,9 +3,33 @@
"mutants": [
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 68: <> -> = (rel)",
"mutant": "line 49: AND -> OR (logic)",
"status": "killed",
"hidden": "9/12",
"hidden": "4/13",
"failed_tests": [
"ADD_DAYS_BACKWARD",
"ADD_DAYS_FORWARD",
"ADD_FORWARD_SKIPS_HOLIDAY",
"COUNT_DUPLICATE_HOLIDAYS",
"COUNT_WORKING_DAYS_INCLUSIVE",
"HOLIDAY_IS_NOT_WORKING",
"NEXT_DAY_FROM_WEEKEND",
"NEXT_DAY_SKIPS_HOLIDAYS",
"WEEKEND_IS_NOT_WORKING"
]
},
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 50: abap_false -> abap_true (bool)",
"status": "invalid",
"hidden": "0/0",
"failed_tests": []
},
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 63: = -> <> (eq)",
"status": "killed",
"hidden": "10/13",
"failed_tests": [
"ADD_DAYS_BACKWARD",
"ADD_DAYS_FORWARD",
@@ -14,46 +38,40 @@
},
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 72: - -> + (arith)",
"status": "invalid",
"hidden": "0/0",
"failed_tests": []
},
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 81: = -> <> (eq)",
"status": "invalid",
"hidden": "0/0",
"failed_tests": []
},
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 91: 5 -> 6 (const)",
"mutant": "line 91: 7 -> 8 (const)",
"status": "killed",
"hidden": "7/12",
"hidden": "6/13",
"failed_tests": [
"ADD_DAYS_BACKWARD",
"ADD_DAYS_FORWARD",
"ADD_FORWARD_SKIPS_HOLIDAY",
"COUNT_DUPLICATE_HOLIDAYS",
"COUNT_WORKING_DAYS_INCLUSIVE",
"NEXT_DAY_FROM_WEEKEND",
"WEEKEND_IS_NOT_WORKING"
]
},
{
"object": "{{P}}DELIVERY_SCHEDULE",
"mutant": "line 98: abap_true -> abap_false (bool)",
"mutant": "line 91: >= -> < (rel)",
"status": "killed",
"hidden": "8/12",
"hidden": "3/13",
"failed_tests": [
"ADD_DAYS_BACKWARD",
"ADD_DAYS_FORWARD",
"ADD_FORWARD_SKIPS_HOLIDAY",
"COUNT_DUPLICATE_HOLIDAYS",
"COUNT_WORKING_DAYS_INCLUSIVE",
"HOLIDAY_IS_NOT_WORKING",
"NEXT_DAY_SKIPS_HOLIDAYS"
"NEXT_DAY_FROM_WEEKEND",
"NEXT_DAY_SAME_FOR_WORKDAY",
"NEXT_DAY_SKIPS_HOLIDAYS",
"WEEKEND_IS_NOT_WORKING",
"WORKING_DAY_IS_TRUE"
]
}
],
"valid": 3,
"killed": 3,
"valid": 4,
"killed": 4,
"kill_rate": 1.0,
"ok": true
}

7
tasks_gen/eval/G0002/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Fixed: added count_duplicate_holidays; oracle 100, null 0, mutation 4/4.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0003/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Rules map to tests; mutation 5/5.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0004/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Rules map to tests; fits C (grouping, sorting).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0007/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Interface must be read; fits F.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0008/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "740sp05 target; rounding and boundary tests present.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

9
tasks_gen/eval/G0009/review.json Normal file
View File

@@ -0,0 +1,9 @@
{
"decision": "flag",
"note": "Category I is weak: the model writes a new class from complete rules; the legacy class need not be read. Options: move to A, or change to an in-place fix of the legacy class. Survived mutant (round dec 2->3) is equivalent.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {
"6": "zayif"
}
}

7
tasks_gen/eval/G0010/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Ordered stock use, case/blank rules tested; fits C.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0012/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "FM interface must be read from sample; boundaries tested; fits F.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

9
tasks_gen/eval/G0013/review.json Normal file
View File

@@ -0,0 +1,9 @@
{
"decision": "flag",
"note": "Same as G0009: new FM from complete rules; old FM need not be read. Generator prompt for I fixed (in-place fix, spec gives only symptoms). Options: move to A, or regenerate as I.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {
"6": "zayif"
}
}

7
tasks_gen/eval/G0014/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Sorted values distinct in data (120, 70, -20). Sort ties not defined in spec, but no tie in tests.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0015/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Rules map to tests; open/paid/overdue split tested. Sort ties not defined in spec.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

9
tasks_gen/eval/G0016/review.json Normal file
View File

@@ -0,0 +1,9 @@
{
"decision": "flag",
"note": "E is weak: new program with complete rules; legacy need not be read. Generator prompt for E fixed (in-place refactor, spec without rules). Options: keep as B-like report task, or regenerate as E.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {
"6": "zayif"
}
}

7
tasks_gen/eval/G0017/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Survived inner->left outer join is equivalent (WHERE on the right table).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0018/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Unmatched customer tested; mutation 2/2.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

9
tasks_gen/eval/G0019/review.json Normal file
View File

@@ -0,0 +1,9 @@
{
"decision": "fix",
"note": "sum->max survived: TOTAL_AMOUNT test has one item per region. Add a test with two items in one region.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {
"7": "bosluk"
}
}

7
tasks_gen/eval/G0020/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Survived join mutant is equivalent (WHERE on the right table).",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0021/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Union, literals and filter tested.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

85
tasks_gen/eval/G0022/faulty/m4_expense_validator.clas.abap Normal file
View File

@@ -0,0 +1,85 @@
CLASS {{p}}expense_validator DEFINITION
PUBLIC
FINAL
CREATE PUBLIC.
PUBLIC SECTION.
METHODS validate
IMPORTING is_claim TYPE {{p}}if_expense_claim=>ty_claim
RAISING {{p}}cx_claim_error.
PRIVATE SECTION.
CONSTANTS:
c_receipt_limit TYPE {{p}}if_expense_claim=>ty_amount VALUE '100.00',
c_cc_limit TYPE {{p}}if_expense_claim=>ty_amount VALUE '1000.00'.
METHODS check_amount
IMPORTING is_claim TYPE {{p}}if_expense_claim=>ty_claim
RAISING {{p}}cx_claim_error.
METHODS check_currency
IMPORTING is_claim TYPE {{p}}if_expense_claim=>ty_claim
RAISING {{p}}cx_claim_error.
METHODS check_date
IMPORTING is_claim TYPE {{p}}if_expense_claim=>ty_claim
RAISING {{p}}cx_claim_error.
METHODS check_receipt
IMPORTING is_claim TYPE {{p}}if_expense_claim=>ty_claim
RAISING {{p}}cx_claim_error.
METHODS check_cost_center
IMPORTING is_claim TYPE {{p}}if_expense_claim=>ty_claim
RAISING {{p}}cx_claim_error.
METHODS is_supported_currency
IMPORTING iv_currency TYPE {{p}}if_expense_claim=>ty_currency
RETURNING VALUE(rv_supported) TYPE abap_bool.
ENDCLASS.
CLASS {{p}}expense_validator IMPLEMENTATION.
METHOD validate.
check_amount( is_claim ).
check_currency( is_claim ).
check_date( is_claim ).
check_receipt( is_claim ).
check_cost_center( is_claim ).
ENDMETHOD.
METHOD check_amount.
IF is_claim-amount <= 0.
RAISE EXCEPTION TYPE {{p}}cx_claim_amount
EXPORTING iv_amount = is_claim-amount.
ENDIF.
ENDMETHOD.
METHOD check_currency.
IF is_supported_currency( is_claim-currency ) = abap_false.
RAISE EXCEPTION TYPE {{p}}cx_claim_currency
EXPORTING iv_currency = is_claim-currency.
ENDIF.
ENDMETHOD.
METHOD check_date.
IF is_claim-claim_date > sy-datum.
RAISE EXCEPTION TYPE {{p}}cx_claim_date
EXPORTING iv_claim_date = is_claim-claim_date.
ENDIF.
ENDMETHOD.
METHOD check_receipt.
IF is_claim-amount >= c_receipt_limit AND is_claim-has_receipt = abap_false.
RAISE EXCEPTION TYPE {{p}}cx_claim_receipt
EXPORTING iv_amount = is_claim-amount.
ENDIF.
ENDMETHOD.
METHOD check_cost_center.
IF is_claim-amount >= c_cc_limit AND is_claim-cost_center IS INITIAL.
RAISE EXCEPTION TYPE {{p}}cx_claim_costcenter
EXPORTING iv_amount = is_claim-amount.
ENDIF.
ENDMETHOD.
METHOD is_supported_currency.
rv_supported = xsdbool( iv_currency = 'EUR' OR iv_currency = 'USD'
OR iv_currency = 'GBP' AND iv_currency = 'CHF' ).
ENDMETHOD.
ENDCLASS.

12
tasks_gen/eval/G0022/generation.json
View File

@@ -37,6 +37,18 @@
"change": "budget floor: max_activations 12 -> 14 (reference alone needs 13).",
"oracle": 100.0,
"null": 0
},
{
"stage": "manual_fix",
"by": "Claude review 2026-10-03",
"change": "Review: added hidden test other_currencies_pass (USD, GBP, CHF).",
"oracle": 100.0,
"null": 0,
"mutation": {
"valid": 5,
"killed": 5,
"ok": true
}
}
],
"accepted": true

14
tasks_gen/eval/G0022/hidden/d01_hidden.clas.abap
View File

@@ -19,6 +19,7 @@ CLASS {{p}}d01_hidden DEFINITION PUBLIC FINAL CREATE PUBLIC
METHODS cost_center_not_required FOR TESTING.
METHODS amount_rule_has_priority FOR TESTING.
METHODS receipt_rule_has_priority FOR TESTING.
METHODS other_currencies_pass FOR TESTING.
METHODS build_claim
IMPORTING iv_amount TYPE ty_amount
@@ -54,6 +55,19 @@ CLASS {{p}}d01_hidden IMPLEMENTATION.
ENDTRY.
ENDMETHOD.
METHOD other_currencies_pass.
LOOP AT VALUE string_table( ( `USD` ) ( `GBP` ) ( `CHF` ) ) INTO DATA(lv_currency).
DATA(ls_claim) = build_claim( iv_amount = '250.00' iv_currency = CONV #( lv_currency )
iv_date = sy-datum iv_cc = 'CC1000'
iv_receipt = abap_true ).
TRY.
mo_cut->validate( ls_claim ).
CATCH {{p}}cx_claim_error INTO DATA(lx).
cl_abap_unit_assert=>fail( msg = |{ lv_currency }: { lx->get_message( ) }| ).
ENDTRY.
ENDLOOP.
ENDMETHOD.
METHOD zero_amount_rejected.
DATA(ls_claim) = build_claim( iv_amount = 0 iv_currency = 'EUR'
iv_date = sy-datum iv_cc = 'CC1000'

24
tasks_gen/eval/G0022/mutation.json
View File

@@ -5,11 +5,12 @@
"object": "{{P}}EXPENSE_VALIDATOR",
"mutant": "line 54: = -> <> (eq)",
"status": "killed",
"hidden": "3/12",
"hidden": "3/13",
"failed_tests": [
"COST_CENTER_NOT_REQUIRED",
"COST_CENTER_REQUIRED",
"FUTURE_DATE_REJECTED",
"OTHER_CURRENCIES_PASS",
"RECEIPT_NOT_REQUIRED_BELOW",
"RECEIPT_REQUIRED",
"RECEIPT_RULE_HAS_PRIORITY",
@@ -22,11 +23,12 @@
"object": "{{P}}EXPENSE_VALIDATOR",
"mutant": "line 54: abap_false -> abap_true (bool)",
"status": "killed",
"hidden": "3/12",
"hidden": "3/13",
"failed_tests": [
"COST_CENTER_NOT_REQUIRED",
"COST_CENTER_REQUIRED",
"FUTURE_DATE_REJECTED",
"OTHER_CURRENCIES_PASS",
"RECEIPT_NOT_REQUIRED_BELOW",
"RECEIPT_REQUIRED",
"RECEIPT_RULE_HAS_PRIORITY",
@@ -39,11 +41,12 @@
"object": "{{P}}EXPENSE_VALIDATOR",
"mutant": "line 61: > -> <= (rel)",
"status": "killed",
"hidden": "4/12",
"hidden": "4/13",
"failed_tests": [
"COST_CENTER_NOT_REQUIRED",
"COST_CENTER_REQUIRED",
"FUTURE_DATE_REJECTED",
"OTHER_CURRENCIES_PASS",
"RECEIPT_NOT_REQUIRED_BELOW",
"RECEIPT_REQUIRED",
"RECEIPT_RULE_HAS_PRIORITY",
@@ -55,10 +58,11 @@
"object": "{{P}}EXPENSE_VALIDATOR",
"mutant": "line 68: = -> <> (eq)",
"status": "killed",
"hidden": "6/12",
"hidden": "6/13",
"failed_tests": [
"COST_CENTER_NOT_REQUIRED",
"COST_CENTER_REQUIRED",
"OTHER_CURRENCIES_PASS",
"RECEIPT_REQUIRED",
"RECEIPT_RULE_HAS_PRIORITY",
"TODAY_IS_ACCEPTED",
@@ -68,13 +72,15 @@
{
"object": "{{P}}EXPENSE_VALIDATOR",
"mutant": "line 83: OR -> AND (logic)",
"status": "survived",
"hidden": "12/12",
"failed_tests": []
"status": "killed",
"hidden": "12/13",
"failed_tests": [
"OTHER_CURRENCIES_PASS"
]
}
],
"valid": 5,
"killed": 4,
"kill_rate": 0.8,
"killed": 5,
"kill_rate": 1.0,
"ok": true
}

7
tasks_gen/eval/G0022/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Fixed: added other_currencies_pass; oracle 100, null 0, mutation 5/5.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

9
tasks_gen/eval/G0023/review.json Normal file
View File

@@ -0,0 +1,9 @@
{
"decision": "flag",
"note": "E is weak: new class from complete rules; legacy need not be read. Generator prompt for E fixed. Options: move to A, or regenerate as E.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {
"6": "zayif"
}
}

7
tasks_gen/eval/G0024/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Period limits and sums tested; mutation 5/5.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0025/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Union, duplicates and source literals tested.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0026/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "All rules tested; logic in helper class; mutation 5/5.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0168/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Real, critical gap: quantity exactly 100 has no volume discount rule.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}

7
tasks_gen/eval/G0178/review.json Normal file
View File

@@ -0,0 +1,7 @@
{
"decision": "accept",
"note": "Free text keeps all rules and contract names.",
"by": "Claude",
"date": "2026-10-03",
"failed_checks": {}
}