erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mutation in generator; category H (stop) scoring and judge; category K variants and tool schema variant; evalset plan

Browse Source 
- mutation.py: negation mutants, skip WHILE/DO blocks (endless loop blocked RFC ~8 min)
- generator: accept only after mutation check; survivors go back as repair feedback
- runner/judge.py: stop tasks scored 100/30/0; gap by keywords, else judge model
- proxy: tool schema variant generic_v0 (draft); generator make_k_variant
- evalset.py: 88 slots for step F (A-I, H, K)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-03 05:37:20 +02:00
parent 32f03fcad4
commit 0ba25faeea
20 changed files with 1303 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
tasks_gen/eval/G0003/faulty/m0_course_booking.clas.abap Normal file
View File

@@ -0,0 +1,94 @@
CLASS {{P}}COURSE_BOOKING DEFINITION PUBLIC FINAL CREATE PUBLIC.
PUBLIC SECTION.
TYPES ty_course_id TYPE c LENGTH 10.
TYPES ty_category TYPE c LENGTH 10.
TYPES:
BEGIN OF ty_course_info,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
free_seats TYPE i,
END OF ty_course_info.
TYPES tt_course_info TYPE STANDARD TABLE OF ty_course_info WITH EMPTY KEY.
METHODS get_free_seats
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rv_free_seats) TYPE i.
METHODS is_bookable
IMPORTING iv_course_id TYPE ty_course_id
iv_seats TYPE i
RETURNING VALUE(rv_bookable) TYPE abap_bool.
METHODS list_bookable_courses
IMPORTING iv_category TYPE ty_category
iv_seats TYPE i
RETURNING VALUE(rt_courses) TYPE tt_course_info.
PRIVATE SECTION.
TYPES:
BEGIN OF ty_course,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
capacity TYPE i,
booked TYPE i,
END OF ty_course.
METHODS read_course
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rs_course) TYPE ty_course.
METHODS calculate_free_seats
IMPORTING is_course TYPE ty_course
RETURNING VALUE(rv_free) TYPE i.
ENDCLASS.
CLASS {{P}}COURSE_BOOKING IMPLEMENTATION.
METHOD get_free_seats.
rv_free_seats = calculate_free_seats( read_course( iv_course_id ) ).
ENDMETHOD.
METHOD is_bookable.
IF iv_seats >= 1.
rv_bookable = abap_false.
RETURN.
ENDIF.
rv_bookable = xsdbool( iv_seats <= calculate_free_seats( read_course( iv_course_id ) ) ).
ENDMETHOD.
METHOD list_bookable_courses.
DATA lt_course TYPE STANDARD TABLE OF ty_course WITH EMPTY KEY.
SELECT course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE category = @iv_category
INTO TABLE @lt_course.
LOOP AT lt_course INTO DATA(ls_course).
DATA(lv_free) = calculate_free_seats( ls_course ).
IF lv_free < iv_seats.
CONTINUE.
ENDIF.
rt_courses = VALUE #( BASE rt_courses
( course_id = ls_course-course_id
title = ls_course-title
free_seats = lv_free ) ).
ENDLOOP.
SORT rt_courses BY course_id.
ENDMETHOD.
METHOD read_course.
SELECT SINGLE course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE course_id = @iv_course_id
INTO @rs_course.
ENDMETHOD.
METHOD calculate_free_seats.
rv_free = is_course-capacity - is_course-booked.
IF rv_free < 0.
rv_free = 0.
ENDIF.
ENDMETHOD.
ENDCLASS.

94
tasks_gen/eval/G0003/faulty/m1_course_booking.clas.abap Normal file
View File

@@ -0,0 +1,94 @@
CLASS {{P}}COURSE_BOOKING DEFINITION PUBLIC FINAL CREATE PUBLIC.
PUBLIC SECTION.
TYPES ty_course_id TYPE c LENGTH 10.
TYPES ty_category TYPE c LENGTH 10.
TYPES:
BEGIN OF ty_course_info,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
free_seats TYPE i,
END OF ty_course_info.
TYPES tt_course_info TYPE STANDARD TABLE OF ty_course_info WITH EMPTY KEY.
METHODS get_free_seats
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rv_free_seats) TYPE i.
METHODS is_bookable
IMPORTING iv_course_id TYPE ty_course_id
iv_seats TYPE i
RETURNING VALUE(rv_bookable) TYPE abap_bool.
METHODS list_bookable_courses
IMPORTING iv_category TYPE ty_category
iv_seats TYPE i
RETURNING VALUE(rt_courses) TYPE tt_course_info.
PRIVATE SECTION.
TYPES:
BEGIN OF ty_course,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
capacity TYPE i,
booked TYPE i,
END OF ty_course.
METHODS read_course
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rs_course) TYPE ty_course.
METHODS calculate_free_seats
IMPORTING is_course TYPE ty_course
RETURNING VALUE(rv_free) TYPE i.
ENDCLASS.
CLASS {{P}}COURSE_BOOKING IMPLEMENTATION.
METHOD get_free_seats.
rv_free_seats = calculate_free_seats( read_course( iv_course_id ) ).
ENDMETHOD.
METHOD is_bookable.
IF iv_seats < 1.
rv_bookable = abap_true.
RETURN.
ENDIF.
rv_bookable = xsdbool( iv_seats <= calculate_free_seats( read_course( iv_course_id ) ) ).
ENDMETHOD.
METHOD list_bookable_courses.
DATA lt_course TYPE STANDARD TABLE OF ty_course WITH EMPTY KEY.
SELECT course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE category = @iv_category
INTO TABLE @lt_course.
LOOP AT lt_course INTO DATA(ls_course).
DATA(lv_free) = calculate_free_seats( ls_course ).
IF lv_free < iv_seats.
CONTINUE.
ENDIF.
rt_courses = VALUE #( BASE rt_courses
( course_id = ls_course-course_id
title = ls_course-title
free_seats = lv_free ) ).
ENDLOOP.
SORT rt_courses BY course_id.
ENDMETHOD.
METHOD read_course.
SELECT SINGLE course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE course_id = @iv_course_id
INTO @rs_course.
ENDMETHOD.
METHOD calculate_free_seats.
rv_free = is_course-capacity - is_course-booked.
IF rv_free < 0.
rv_free = 0.
ENDIF.
ENDMETHOD.
ENDCLASS.

94
tasks_gen/eval/G0003/faulty/m2_course_booking.clas.abap Normal file
View File

@@ -0,0 +1,94 @@
CLASS {{P}}COURSE_BOOKING DEFINITION PUBLIC FINAL CREATE PUBLIC.
PUBLIC SECTION.
TYPES ty_course_id TYPE c LENGTH 10.
TYPES ty_category TYPE c LENGTH 10.
TYPES:
BEGIN OF ty_course_info,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
free_seats TYPE i,
END OF ty_course_info.
TYPES tt_course_info TYPE STANDARD TABLE OF ty_course_info WITH EMPTY KEY.
METHODS get_free_seats
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rv_free_seats) TYPE i.
METHODS is_bookable
IMPORTING iv_course_id TYPE ty_course_id
iv_seats TYPE i
RETURNING VALUE(rv_bookable) TYPE abap_bool.
METHODS list_bookable_courses
IMPORTING iv_category TYPE ty_category
iv_seats TYPE i
RETURNING VALUE(rt_courses) TYPE tt_course_info.
PRIVATE SECTION.
TYPES:
BEGIN OF ty_course,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
capacity TYPE i,
booked TYPE i,
END OF ty_course.
METHODS read_course
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rs_course) TYPE ty_course.
METHODS calculate_free_seats
IMPORTING is_course TYPE ty_course
RETURNING VALUE(rv_free) TYPE i.
ENDCLASS.
CLASS {{P}}COURSE_BOOKING IMPLEMENTATION.
METHOD get_free_seats.
rv_free_seats = calculate_free_seats( read_course( iv_course_id ) ).
ENDMETHOD.
METHOD is_bookable.
IF iv_seats < 1.
rv_bookable = abap_false.
RETURN.
ENDIF.
rv_bookable = xsdbool( iv_seats <= calculate_free_seats( read_course( iv_course_id ) ) ).
ENDMETHOD.
METHOD list_bookable_courses.
DATA lt_course TYPE STANDARD TABLE OF ty_course WITH EMPTY KEY.
SELECT course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE category = @iv_category
INTO TABLE @lt_course.
LOOP AT lt_course INTO DATA(ls_course).
DATA(lv_free) = calculate_free_seats( ls_course ).
IF lv_free >= iv_seats.
CONTINUE.
ENDIF.
rt_courses = VALUE #( BASE rt_courses
( course_id = ls_course-course_id
title = ls_course-title
free_seats = lv_free ) ).
ENDLOOP.
SORT rt_courses BY course_id.
ENDMETHOD.
METHOD read_course.
SELECT SINGLE course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE course_id = @iv_course_id
INTO @rs_course.
ENDMETHOD.
METHOD calculate_free_seats.
rv_free = is_course-capacity - is_course-booked.
IF rv_free < 0.
rv_free = 0.
ENDIF.
ENDMETHOD.
ENDCLASS.

94
tasks_gen/eval/G0003/faulty/m3_course_booking.clas.abap Normal file
View File

@@ -0,0 +1,94 @@
CLASS {{P}}COURSE_BOOKING DEFINITION PUBLIC FINAL CREATE PUBLIC.
PUBLIC SECTION.
TYPES ty_course_id TYPE c LENGTH 10.
TYPES ty_category TYPE c LENGTH 10.
TYPES:
BEGIN OF ty_course_info,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
free_seats TYPE i,
END OF ty_course_info.
TYPES tt_course_info TYPE STANDARD TABLE OF ty_course_info WITH EMPTY KEY.
METHODS get_free_seats
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rv_free_seats) TYPE i.
METHODS is_bookable
IMPORTING iv_course_id TYPE ty_course_id
iv_seats TYPE i
RETURNING VALUE(rv_bookable) TYPE abap_bool.
METHODS list_bookable_courses
IMPORTING iv_category TYPE ty_category
iv_seats TYPE i
RETURNING VALUE(rt_courses) TYPE tt_course_info.
PRIVATE SECTION.
TYPES:
BEGIN OF ty_course,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
capacity TYPE i,
booked TYPE i,
END OF ty_course.
METHODS read_course
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rs_course) TYPE ty_course.
METHODS calculate_free_seats
IMPORTING is_course TYPE ty_course
RETURNING VALUE(rv_free) TYPE i.
ENDCLASS.
CLASS {{P}}COURSE_BOOKING IMPLEMENTATION.
METHOD get_free_seats.
rv_free_seats = calculate_free_seats( read_course( iv_course_id ) ).
ENDMETHOD.
METHOD is_bookable.
IF iv_seats < 1.
rv_bookable = abap_false.
RETURN.
ENDIF.
rv_bookable = xsdbool( iv_seats <= calculate_free_seats( read_course( iv_course_id ) ) ).
ENDMETHOD.
METHOD list_bookable_courses.
DATA lt_course TYPE STANDARD TABLE OF ty_course WITH EMPTY KEY.
SELECT course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE category = @iv_category
INTO TABLE @lt_course.
LOOP AT lt_course INTO DATA(ls_course).
DATA(lv_free) = calculate_free_seats( ls_course ).
IF lv_free < iv_seats.
CONTINUE.
ENDIF.
rt_courses = VALUE #( BASE rt_courses
( course_id = ls_course-course_id
title = ls_course-title
free_seats = lv_free ) ).
ENDLOOP.
SORT rt_courses BY course_id.
ENDMETHOD.
METHOD read_course.
SELECT SINGLE course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE course_id <> @iv_course_id
INTO @rs_course.
ENDMETHOD.
METHOD calculate_free_seats.
rv_free = is_course-capacity - is_course-booked.
IF rv_free < 0.
rv_free = 0.
ENDIF.
ENDMETHOD.
ENDCLASS.

94
tasks_gen/eval/G0003/faulty/m4_course_booking.clas.abap Normal file
View File

@@ -0,0 +1,94 @@
CLASS {{P}}COURSE_BOOKING DEFINITION PUBLIC FINAL CREATE PUBLIC.
PUBLIC SECTION.
TYPES ty_course_id TYPE c LENGTH 10.
TYPES ty_category TYPE c LENGTH 10.
TYPES:
BEGIN OF ty_course_info,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
free_seats TYPE i,
END OF ty_course_info.
TYPES tt_course_info TYPE STANDARD TABLE OF ty_course_info WITH EMPTY KEY.
METHODS get_free_seats
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rv_free_seats) TYPE i.
METHODS is_bookable
IMPORTING iv_course_id TYPE ty_course_id
iv_seats TYPE i
RETURNING VALUE(rv_bookable) TYPE abap_bool.
METHODS list_bookable_courses
IMPORTING iv_category TYPE ty_category
iv_seats TYPE i
RETURNING VALUE(rt_courses) TYPE tt_course_info.
PRIVATE SECTION.
TYPES:
BEGIN OF ty_course,
course_id TYPE ty_course_id,
title TYPE c LENGTH 40,
capacity TYPE i,
booked TYPE i,
END OF ty_course.
METHODS read_course
IMPORTING iv_course_id TYPE ty_course_id
RETURNING VALUE(rs_course) TYPE ty_course.
METHODS calculate_free_seats
IMPORTING is_course TYPE ty_course
RETURNING VALUE(rv_free) TYPE i.
ENDCLASS.
CLASS {{P}}COURSE_BOOKING IMPLEMENTATION.
METHOD get_free_seats.
rv_free_seats = calculate_free_seats( read_course( iv_course_id ) ).
ENDMETHOD.
METHOD is_bookable.
IF iv_seats < 1.
rv_bookable = abap_false.
RETURN.
ENDIF.
rv_bookable = xsdbool( iv_seats <= calculate_free_seats( read_course( iv_course_id ) ) ).
ENDMETHOD.
METHOD list_bookable_courses.
DATA lt_course TYPE STANDARD TABLE OF ty_course WITH EMPTY KEY.
SELECT course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE category = @iv_category
INTO TABLE @lt_course.
LOOP AT lt_course INTO DATA(ls_course).
DATA(lv_free) = calculate_free_seats( ls_course ).
IF lv_free < iv_seats.
CONTINUE.
ENDIF.
rt_courses = VALUE #( BASE rt_courses
( course_id = ls_course-course_id
title = ls_course-title
free_seats = lv_free ) ).
ENDLOOP.
SORT rt_courses BY course_id.
ENDMETHOD.
METHOD read_course.
SELECT SINGLE course_id, title, capacity, booked
FROM {{P}}COURSE
WHERE course_id = @iv_course_id
INTO @rs_course.
ENDMETHOD.
METHOD calculate_free_seats.
rv_free = is_course-capacity + is_course-booked.
IF rv_free < 0.
rv_free = 0.
ENDIF.
ENDMETHOD.
ENDCLASS.

68
tasks_gen/eval/G0003/mutation.json Normal file
View File

@@ -0,0 +1,68 @@
{
"task": "G0003",
"mutants": [
{
"object": "{{P}}COURSE_BOOKING",
"mutant": "line 52: < -> >= (rel)",
"status": "killed",
"hidden": "9/11",
"failed_tests": [
"BOOKABLE_EXACT_FREE",
"NOT_BOOKABLE_ZERO_SEATS"
]
},
{
"object": "{{P}}COURSE_BOOKING",
"mutant": "line 53: abap_false -> abap_true (bool)",
"status": "killed",
"hidden": "10/11",
"failed_tests": [
"NOT_BOOKABLE_ZERO_SEATS"
]
},
{
"object": "{{P}}COURSE_BOOKING",
"mutant": "line 69: < -> >= (rel)",
"status": "killed",
"hidden": "7/11",
"failed_tests": [
"LIST_EMPTY_WITHOUT_MATCH",
"LIST_EXCLUDES_FULL",
"LIST_FILTER_CAT_SEATS",
"LIST_SORTED_BY_COURSE"
]
},
{
"object": "{{P}}COURSE_BOOKING",
"mutant": "line 84: = -> <> (eq)",
"status": "killed",
"hidden": "6/11",
"failed_tests": [
"BOOKABLE_EXACT_FREE",
"FREE_SEATS_NEVER_NEG",
"FREE_SEATS_OPEN_COURSE",
"FREE_SEATS_UNKNOWN",
"NOT_BOOKABLE_UNKNOWN"
]
},
{
"object": "{{P}}COURSE_BOOKING",
"mutant": "line 89: - -> + (arith)",
"status": "killed",
"hidden": "4/11",
"failed_tests": [
"FREE_SEATS_NEVER_NEG",
"FREE_SEATS_OPEN_COURSE",
"LIST_EMPTY_WITHOUT_MATCH",
"LIST_EXCLUDES_FULL",
"LIST_FILTER_CAT_SEATS",
"LIST_SORTED_BY_COURSE",
"NOT_BOOKABLE_TOO_MANY"
]
}
],
"valid": 5,
"killed": 5,
"kill_rate": 1.0,
"ok": true
}