Step F first pass: 81/92 accepted; budget floor 60 calls; rejected slots moved for regeneration
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
18
tasks_gen/eval/G0185/empirical.json
Normal file
18
tasks_gen/eval/G0185/empirical.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"deepseek-v4.1-flash:cloud": {
|
||||
"score": 85.0,
|
||||
"parts": {
|
||||
"correctness": 40.0,
|
||||
"craft": 20.0,
|
||||
"clean": 15.0,
|
||||
"own_tests": 0.0,
|
||||
"discipline": 10.0,
|
||||
"total": 85.0,
|
||||
"note": "own_tests: faulty-reference part not implemented in skeleton"
|
||||
},
|
||||
"hidden": "11/11",
|
||||
"tool_calls": 17,
|
||||
"seconds": 60.2,
|
||||
"run_dir": "runs/emp/10006_G0012_llm_deepseek-v4.1-flash_cloud"
|
||||
}
|
||||
}
|
||||
16
tasks_gen/eval/G0185/generation.json
Normal file
16
tasks_gen/eval/G0185/generation.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"id": "G0185",
|
||||
"pool": "eval",
|
||||
"category": "K",
|
||||
"base_task": "G0012",
|
||||
"style": "incomplete",
|
||||
"tool_schema": "generic_v0",
|
||||
"attempts": [
|
||||
{
|
||||
"stage": "validate",
|
||||
"oracle": 100.0,
|
||||
"null": 0
|
||||
}
|
||||
],
|
||||
"accepted": true
|
||||
}
|
||||
120
tasks_gen/eval/G0185/hidden/t14_hidden.clas.abap
Normal file
120
tasks_gen/eval/G0185/hidden/t14_hidden.clas.abap
Normal file
@@ -0,0 +1,120 @@
|
||||
CLASS {{p}}t14_hidden DEFINITION PUBLIC FINAL CREATE PUBLIC
|
||||
FOR TESTING DURATION SHORT RISK LEVEL HARMLESS.
|
||||
PRIVATE SECTION.
|
||||
METHODS call
|
||||
IMPORTING iv_quantity TYPE decfloat34
|
||||
iv_scrap TYPE decfloat34
|
||||
iv_rework TYPE decfloat34 DEFAULT 0
|
||||
EXPORTING ev_pct TYPE decfloat34
|
||||
ev_severity TYPE string
|
||||
ev_rework TYPE decfloat34
|
||||
ev_subrc TYPE sysubrc.
|
||||
METHODS percent_rounds_half_up FOR TESTING.
|
||||
METHODS zero_scrap_is_ok FOR TESTING.
|
||||
METHODS low_upper_boundary FOR TESTING.
|
||||
METHODS high_lower_boundary FOR TESTING.
|
||||
METHODS high_upper_boundary FOR TESTING.
|
||||
METHODS critical_above_twenty FOR TESTING.
|
||||
METHODS rework_unchanged_for_low FOR TESTING.
|
||||
METHODS rework_set_for_high FOR TESTING.
|
||||
METHODS quantity_zero_raises FOR TESTING.
|
||||
METHODS scrap_negative_raises FOR TESTING.
|
||||
METHODS scrap_above_quantity_raises FOR TESTING.
|
||||
ENDCLASS.
|
||||
|
||||
|
||||
CLASS {{p}}t14_hidden IMPLEMENTATION.
|
||||
METHOD call.
|
||||
ev_pct = 0.
|
||||
ev_severity = ``.
|
||||
ev_rework = iv_rework.
|
||||
CALL FUNCTION '{{P}}PRD_SCRAP'
|
||||
EXPORTING
|
||||
iv_quantity = iv_quantity
|
||||
iv_scrap = iv_scrap
|
||||
IMPORTING
|
||||
ev_scrap_pct = ev_pct
|
||||
ev_severity = ev_severity
|
||||
CHANGING
|
||||
cv_rework_qty = ev_rework
|
||||
EXCEPTIONS
|
||||
invalid_quantity = 1
|
||||
scrap_too_high = 2
|
||||
OTHERS = 3.
|
||||
ev_subrc = sy-subrc.
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD percent_rounds_half_up.
|
||||
call( EXPORTING iv_quantity = 800 iv_scrap = 7
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( '0.9' ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `LOW` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD zero_scrap_is_ok.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = 0
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 0 ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `OK` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD low_upper_boundary.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = 5
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 5 ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `LOW` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD high_lower_boundary.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = '5.1'
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( '5.1' ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `HIGH` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD high_upper_boundary.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = 20
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 20 ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `HIGH` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD critical_above_twenty.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = '20.1'
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( '20.1' ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `CRITICAL` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD rework_unchanged_for_low.
|
||||
call( EXPORTING iv_quantity = 200 iv_scrap = 4 iv_rework = 99
|
||||
IMPORTING ev_rework = DATA(lv_rework) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `LOW` act = lv_sev ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 99 ) act = lv_rework ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD rework_set_for_high.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = 10 iv_rework = 99
|
||||
IMPORTING ev_rework = DATA(lv_rework) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `HIGH` act = lv_sev ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 10 ) act = lv_rework ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD quantity_zero_raises.
|
||||
call( EXPORTING iv_quantity = 0 iv_scrap = 0
|
||||
IMPORTING ev_subrc = DATA(lv_rc) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = 1 act = lv_rc ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD scrap_negative_raises.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = -1
|
||||
IMPORTING ev_subrc = DATA(lv_rc) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = 1 act = lv_rc ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD scrap_above_quantity_raises.
|
||||
call( EXPORTING iv_quantity = 100 iv_scrap = 101
|
||||
IMPORTING ev_subrc = DATA(lv_rc) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = 2 act = lv_rc ).
|
||||
ENDMETHOD.
|
||||
ENDCLASS.
|
||||
41
tasks_gen/eval/G0185/reference/prd_scrap.func.abap
Normal file
41
tasks_gen/eval/G0185/reference/prd_scrap.func.abap
Normal file
@@ -0,0 +1,41 @@
|
||||
FUNCTION {{p}}prd_scrap
|
||||
IMPORTING
|
||||
iv_quantity TYPE decfloat34
|
||||
iv_scrap TYPE decfloat34
|
||||
EXPORTING
|
||||
ev_scrap_pct TYPE decfloat34
|
||||
ev_severity TYPE string
|
||||
CHANGING
|
||||
cv_rework_qty TYPE decfloat34
|
||||
EXCEPTIONS
|
||||
invalid_quantity
|
||||
scrap_too_high.
|
||||
|
||||
IF iv_quantity <= 0 OR iv_scrap < 0.
|
||||
RAISE invalid_quantity.
|
||||
ENDIF.
|
||||
|
||||
IF iv_scrap > iv_quantity.
|
||||
RAISE scrap_too_high.
|
||||
ENDIF.
|
||||
|
||||
ev_scrap_pct = round( val = iv_scrap / iv_quantity * 100
|
||||
dec = 1
|
||||
mode = cl_abap_math=>round_half_up ).
|
||||
|
||||
IF ev_scrap_pct = 0.
|
||||
ev_severity = 'OK'.
|
||||
ELSEIF ev_scrap_pct <= 5.
|
||||
ev_severity = 'LOW'.
|
||||
ELSEIF ev_scrap_pct <= 20.
|
||||
ev_severity = 'HIGH'.
|
||||
ELSE.
|
||||
ev_severity = 'CRITICAL'.
|
||||
ENDIF.
|
||||
|
||||
CASE ev_severity.
|
||||
WHEN 'HIGH' OR 'CRITICAL'.
|
||||
cv_rework_qty = iv_scrap.
|
||||
ENDCASE.
|
||||
|
||||
ENDFUNCTION.
|
||||
90
tasks_gen/eval/G0185/reference/t14_test.clas.abap
Normal file
90
tasks_gen/eval/G0185/reference/t14_test.clas.abap
Normal file
@@ -0,0 +1,90 @@
|
||||
CLASS {{p}}t14_test DEFINITION PUBLIC FINAL CREATE PUBLIC
|
||||
FOR TESTING DURATION SHORT RISK LEVEL HARMLESS.
|
||||
PRIVATE SECTION.
|
||||
METHODS result_for
|
||||
IMPORTING iv_quantity TYPE decfloat34
|
||||
iv_scrap TYPE decfloat34
|
||||
iv_rework TYPE decfloat34 DEFAULT 0
|
||||
EXPORTING ev_pct TYPE decfloat34
|
||||
ev_severity TYPE string
|
||||
ev_rework TYPE decfloat34.
|
||||
METHODS percent_and_severity FOR TESTING.
|
||||
METHODS severity_bands FOR TESTING.
|
||||
METHODS rework_quantity FOR TESTING.
|
||||
METHODS invalid_inputs FOR TESTING.
|
||||
METHODS scrap_too_high FOR TESTING.
|
||||
ENDCLASS.
|
||||
|
||||
|
||||
CLASS {{p}}t14_test IMPLEMENTATION.
|
||||
METHOD result_for.
|
||||
ev_rework = iv_rework.
|
||||
CALL FUNCTION '{{P}}PRD_SCRAP'
|
||||
EXPORTING
|
||||
iv_quantity = iv_quantity
|
||||
iv_scrap = iv_scrap
|
||||
IMPORTING
|
||||
ev_scrap_pct = ev_pct
|
||||
ev_severity = ev_severity
|
||||
CHANGING
|
||||
cv_rework_qty = ev_rework
|
||||
EXCEPTIONS
|
||||
invalid_quantity = 1
|
||||
scrap_too_high = 2
|
||||
OTHERS = 3.
|
||||
cl_abap_unit_assert=>assert_subrc( exp = 0 ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD percent_and_severity.
|
||||
result_for( EXPORTING iv_quantity = 800 iv_scrap = 7
|
||||
IMPORTING ev_pct = DATA(lv_pct) ev_severity = DATA(lv_sev) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( '0.9' ) act = lv_pct ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `LOW` act = lv_sev ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD severity_bands.
|
||||
result_for( EXPORTING iv_quantity = 100 iv_scrap = 0
|
||||
IMPORTING ev_severity = DATA(lv_ok) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `OK` act = lv_ok ).
|
||||
result_for( EXPORTING iv_quantity = 100 iv_scrap = 5
|
||||
IMPORTING ev_severity = DATA(lv_low) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `LOW` act = lv_low ).
|
||||
result_for( EXPORTING iv_quantity = 100 iv_scrap = 20
|
||||
IMPORTING ev_severity = DATA(lv_high) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `HIGH` act = lv_high ).
|
||||
result_for( EXPORTING iv_quantity = 100 iv_scrap = 21
|
||||
IMPORTING ev_severity = DATA(lv_crit) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = `CRITICAL` act = lv_crit ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD rework_quantity.
|
||||
result_for( EXPORTING iv_quantity = 100 iv_scrap = 4 iv_rework = 99
|
||||
IMPORTING ev_rework = DATA(lv_keep) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 99 ) act = lv_keep ).
|
||||
result_for( EXPORTING iv_quantity = 100 iv_scrap = 10
|
||||
IMPORTING ev_rework = DATA(lv_set) ).
|
||||
cl_abap_unit_assert=>assert_equals( exp = CONV decfloat34( 10 ) act = lv_set ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD invalid_inputs.
|
||||
DATA lv_pct TYPE decfloat34.
|
||||
DATA lv_rework TYPE decfloat34.
|
||||
CALL FUNCTION '{{P}}PRD_SCRAP'
|
||||
EXPORTING iv_quantity = 0 iv_scrap = 0
|
||||
IMPORTING ev_scrap_pct = lv_pct
|
||||
CHANGING cv_rework_qty = lv_rework
|
||||
EXCEPTIONS invalid_quantity = 1 OTHERS = 2.
|
||||
cl_abap_unit_assert=>assert_equals( exp = 1 act = sy-subrc ).
|
||||
ENDMETHOD.
|
||||
|
||||
METHOD scrap_too_high.
|
||||
DATA lv_pct TYPE decfloat34.
|
||||
DATA lv_rework TYPE decfloat34.
|
||||
CALL FUNCTION '{{P}}PRD_SCRAP'
|
||||
EXPORTING iv_quantity = 100 iv_scrap = 101
|
||||
IMPORTING ev_scrap_pct = lv_pct
|
||||
CHANGING cv_rework_qty = lv_rework
|
||||
EXCEPTIONS scrap_too_high = 2 OTHERS = 1.
|
||||
cl_abap_unit_assert=>assert_equals( exp = 2 act = sy-subrc ).
|
||||
ENDMETHOD.
|
||||
ENDCLASS.
|
||||
7
tasks_gen/eval/G0185/review.json
Normal file
7
tasks_gen/eval/G0185/review.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"decision": "accept",
|
||||
"note": "FM interface must be read from sample; boundaries tested; fits F.",
|
||||
"by": "Claude",
|
||||
"date": "2026-10-03",
|
||||
"failed_checks": {}
|
||||
}
|
||||
18
tasks_gen/eval/G0185/seed/prd_sample.func.abap
Normal file
18
tasks_gen/eval/G0185/seed/prd_sample.func.abap
Normal file
@@ -0,0 +1,18 @@
|
||||
FUNCTION {{p}}prd_sample
|
||||
IMPORTING
|
||||
iv_quantity TYPE decfloat34
|
||||
iv_scrap TYPE decfloat34
|
||||
EXPORTING
|
||||
ev_scrap_pct TYPE decfloat34
|
||||
ev_severity TYPE string
|
||||
CHANGING
|
||||
cv_rework_qty TYPE decfloat34
|
||||
EXCEPTIONS
|
||||
invalid_quantity
|
||||
scrap_too_high.
|
||||
|
||||
" Sample confirmation module. All confirmation modules use this interface.
|
||||
ev_scrap_pct = 0.
|
||||
ev_severity = 'OK'.
|
||||
|
||||
ENDFUNCTION.
|
||||
1
tasks_gen/eval/G0185/spec.md
Normal file
1
tasks_gen/eval/G0185/spec.md
Normal file
@@ -0,0 +1 @@
|
||||
Create function module {{P}}PRD_SCRAP in function group {{P}}FG_PROD. Use the interface of {{P}}PRD_SAMPLE: importing iv_quantity and iv_scrap, exporting ev_scrap_pct and ev_severity, changing cv_rework_qty. Use the same types and exceptions as {{P}}PRD_SAMPLE. If iv_quantity is less than or equal to 0, raise INVALID_QUANTITY. If iv_scrap is less than 0, raise INVALID_QUANTITY. If iv_scrap is greater than iv_quantity, raise SCRAP_TOO_HIGH. Calculate ev_scrap_pct as iv_scrap divided by iv_quantity, multiplied by 100. Round the result half up to 1 decimal. Set ev_severity from ev_scrap_pct: equal to 0 = OK; greater than 0 and less than or equal to 5 = LOW; greater than 5 and less than or equal to 20 = HIGH; greater than 20 = CRITICAL. If ev_severity is HIGH or CRITICAL, set cv_rework_qty to iv_scrap. Otherwise do not change cv_rework_qty. Release target is 7.58. Do not change {{P}}PRD_SAMPLE. Write ABAP Unit tests in a global test class. The function module must be active and have no syntax error.
|
||||
87
tasks_gen/eval/G0185/task.json
Normal file
87
tasks_gen/eval/G0185/task.json
Normal file
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"id": "G0185",
|
||||
"category": "K",
|
||||
"object_type": "FUNC",
|
||||
"difficulty": 2,
|
||||
"release_target": "v758",
|
||||
"expected_outcome": "implement",
|
||||
"budget": {
|
||||
"max_tool_calls": 60,
|
||||
"max_activations": 15
|
||||
},
|
||||
"seed": [
|
||||
{
|
||||
"type": "FUGR",
|
||||
"name": "{{P}}FG_PROD",
|
||||
"description": "Production confirmation modules"
|
||||
},
|
||||
{
|
||||
"type": "FUNC",
|
||||
"name": "{{P}}PRD_SAMPLE",
|
||||
"functionGroup": "{{P}}FG_PROD",
|
||||
"file": "seed/prd_sample.func.abap",
|
||||
"description": "Sample confirmation module"
|
||||
}
|
||||
],
|
||||
"contract": [
|
||||
{
|
||||
"type": "FUNC",
|
||||
"name": "{{P}}PRD_SCRAP",
|
||||
"functionGroup": "{{P}}FG_PROD",
|
||||
"params": [
|
||||
{
|
||||
"name": "iv_quantity",
|
||||
"type": "decfloat34"
|
||||
},
|
||||
{
|
||||
"name": "iv_scrap",
|
||||
"type": "decfloat34"
|
||||
},
|
||||
{
|
||||
"name": "ev_scrap_pct",
|
||||
"type": "decfloat34"
|
||||
},
|
||||
{
|
||||
"name": "ev_severity",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"name": "cv_rework_qty",
|
||||
"type": "decfloat34"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"out_of_scope": [
|
||||
"{{P}}PRD_SAMPLE"
|
||||
],
|
||||
"hidden_tests": [
|
||||
{
|
||||
"type": "CLAS",
|
||||
"name": "{{P}}T14_HIDDEN",
|
||||
"file": "hidden/t14_hidden.clas.abap",
|
||||
"description": "T14 hidden tests"
|
||||
}
|
||||
],
|
||||
"reference": [
|
||||
{
|
||||
"type": "FUNC",
|
||||
"name": "{{P}}PRD_SCRAP",
|
||||
"functionGroup": "{{P}}FG_PROD",
|
||||
"file": "reference/prd_scrap.func.abap",
|
||||
"description": "Scrap evaluation module"
|
||||
},
|
||||
{
|
||||
"type": "CLAS",
|
||||
"name": "{{P}}T14_TEST",
|
||||
"file": "reference/t14_test.clas.abap",
|
||||
"description": "T14 own tests"
|
||||
}
|
||||
],
|
||||
"craft_checks": [
|
||||
"method_length"
|
||||
],
|
||||
"base_task": "G0012",
|
||||
"input_style": "incomplete",
|
||||
"tool_schema": "generic_v0"
|
||||
}
|
||||
Reference in New Issue
Block a user