erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Runner: G2 finds the FUNCTION statement after local classes; budget floor applied to all tasks

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 18:25:30 +02:00
parent 7351849a76
commit 92e30eb554
78 changed files with 1362 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
harness/runner.py
View File

@@ -248,7 +248,9 @@ class Runner:
g2 = False
g2_detail.append(f"{c['name']}: does not implement {c['implements']}")
if c["type"] == "FUNC": # signature: every parameter with its type in the FUNCTION header
header = src.split(".", 1)[0]
# the FUNCTION statement, not the first statement: local classes can come before it (G0107)
fm = re.search(rf"^\s*FUNCTION\s+{re.escape(c['name'])}\b[^.]*\.", src, re.I | re.M)
header = fm.group(0) if fm else src.split(".", 1)[0]
for prm in c.get("params", []):
if not re.search(rf"\b{re.escape(prm['name'])}\b\)?\s+TYPE\s+{re.escape(prm['type'])}\b",
header, re.I):

4
tasks/T01/task.json
View File

@@ -5,8 +5,8 @@
"release_target": "v758",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

4
tasks/T13/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v758",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

66
tasks/T14/task.json
View File

@@ -1,18 +1,60 @@
{
"id": "T14", "category": "B", "object_type": "PROG", "difficulty": 2,
"release_target": "v758", "expected_outcome": "implement",
"budget": {"max_tool_calls": 40, "max_activations": 10},
"id": "T14",
"category": "B",
"object_type": "PROG",
"difficulty": 2,
"release_target": "v758",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{"type": "TABL", "name": "{{P}}SALE", "file": "seed/sale.tabl.asabap", "description": "Sales"},
{"type": "CLAS", "name": "{{P}}T14_SEED", "file": "seed/t14_seed.clas.abap", "description": "T14 seed data", "run": true}
{
"type": "TABL",
"name": "{{P}}SALE",
"file": "seed/sale.tabl.asabap",
"description": "Sales"
},
{
"type": "CLAS",
"name": "{{P}}T14_SEED",
"file": "seed/t14_seed.clas.abap",
"description": "T14 seed data",
"run": true
}
],
"contract": [
{"type": "PROG", "name": "{{P}}SALES_SUM", "parameters": ["p_from", "s_reg"]}
{
"type": "PROG",
"name": "{{P}}SALES_SUM",
"parameters": [
"p_from",
"s_reg"
]
}
],
"out_of_scope": ["{{P}}SALE"],
"hidden_tests": [{"type": "CLAS", "name": "{{P}}T14_HIDDEN", "file": "hidden/t14_hidden.clas.abap",
"description": "T14 hidden tests"}],
"reference": [{"type": "PROG", "name": "{{P}}SALES_SUM", "file": "reference/sales_sum.prog.abap",
"description": "Sales per region"}],
"craft_checks": ["db_operation_in_loop", "method_length"]
"out_of_scope": [
"{{P}}SALE"
],
"hidden_tests": [
{
"type": "CLAS",
"name": "{{P}}T14_HIDDEN",
"file": "hidden/t14_hidden.clas.abap",
"description": "T14 hidden tests"
}
],
"reference": [
{
"type": "PROG",
"name": "{{P}}SALES_SUM",
"file": "reference/sales_sum.prog.abap",
"description": "Sales per region"
}
],
"craft_checks": [
"db_operation_in_loop",
"method_length"
]
}

69
tasks/T15/task.json
View File

@@ -1,22 +1,65 @@
{
"id": "T15", "category": "B", "object_type": "DDLS", "difficulty": 2,
"release_target": "v758", "expected_outcome": "implement",
"budget": {"max_tool_calls": 40, "max_activations": 10},
"id": "T15",
"category": "B",
"object_type": "DDLS",
"difficulty": 2,
"release_target": "v758",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{"type": "TABL", "name": "{{P}}ORD", "file": "seed/ord.tabl.asabap", "description": "Order header"},
{"type": "TABL", "name": "{{P}}ITM", "file": "seed/itm.tabl.asabap", "description": "Order item"}
{
"type": "TABL",
"name": "{{P}}ORD",
"file": "seed/ord.tabl.asabap",
"description": "Order header"
},
{
"type": "TABL",
"name": "{{P}}ITM",
"file": "seed/itm.tabl.asabap",
"description": "Order item"
}
],
"contract": [
{"type": "DDLS", "name": "{{P}}I_ORDER_TOTAL",
"fields": ["ORDER_ID", "CUSTOMER", "ITEM_COUNT", "TOTAL_AMOUNT"]}
{
"type": "DDLS",
"name": "{{P}}I_ORDER_TOTAL",
"fields": [
"ORDER_ID",
"CUSTOMER",
"ITEM_COUNT",
"TOTAL_AMOUNT"
]
}
],
"out_of_scope": [
"{{P}}ORD",
"{{P}}ITM"
],
"hidden_tests": [
{
"type": "CLAS",
"name": "{{P}}T15_HIDDEN",
"file": "hidden/t15_hidden.clas.abap",
"description": "T15 hidden tests"
}
],
"out_of_scope": ["{{P}}ORD", "{{P}}ITM"],
"hidden_tests": [{"type": "CLAS", "name": "{{P}}T15_HIDDEN", "file": "hidden/t15_hidden.clas.abap",
"description": "T15 hidden tests"}],
"reference": [
{"type": "DDLS", "name": "{{P}}I_ORDER_TOTAL", "file": "reference/i_order_total.ddls.asddls",
"description": "Order totals"},
{"type": "CLAS", "name": "{{P}}T15_TEST", "file": "reference/t15_test.clas.abap", "description": "T15 own tests"}
{
"type": "DDLS",
"name": "{{P}}I_ORDER_TOTAL",
"file": "reference/i_order_total.ddls.asddls",
"description": "Order totals"
},
{
"type": "CLAS",
"name": "{{P}}T15_TEST",
"file": "reference/t15_test.clas.abap",
"description": "T15 own tests"
}
],
"craft_checks": []
}

40
tasks_gen/eval/G0014/empirical.json
View File

@@ -1,17 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 36,
"seconds": 928.5,
"run_dir": "runs/emp/10007_G0014_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 100.0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 36,
"seconds": 928.5,
"run_dir": "runs/emp/10007_G0014_llm_deepseek-v4.1-flash_cloud"
"hidden": "6/6",
"tool_calls": 29,
"seconds": 259.2,
"run_dir": "runs/emp/15000_G0014_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0015/empirical.json
View File

@@ -1,4 +1,22 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 18,
"seconds": 687.3,
"run_dir": "runs/emp/10008_G0015_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 90.0,
"parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 18,
"seconds": 687.3,
"run_dir": "runs/emp/10008_G0015_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 59,
"seconds": 350.0,
"run_dir": "runs/emp/15001_G0015_llm_deepseek-v4.1-flash_cloud"
}
}

34
tasks_gen/eval/G0019/empirical.json
View File

@@ -1,18 +1,36 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 36,
"seconds": 680.7,
"run_dir": "runs/emp/10011_G0019_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 36,
"seconds": 680.7,
"run_dir": "runs/emp/10011_G0019_llm_deepseek-v4.1-flash_cloud"
"hidden": "9/9",
"tool_calls": 58,
"seconds": 392.1,
"run_dir": "runs/emp/15002_G0019_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0020/empirical.json
View File

@@ -1,4 +1,22 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 38,
"seconds": 658.5,
"run_dir": "runs/emp/11001_G0020_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 38,
"seconds": 658.5,
"run_dir": "runs/emp/11001_G0020_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 60,
"seconds": 154.3,
"run_dir": "runs/emp/15003_G0020_llm_deepseek-v4.1-flash_cloud"
}
}

32
tasks_gen/eval/G0024/empirical.json
View File

@@ -1,18 +1,36 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 92.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 12.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 92.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 32,
"seconds": 691.7,
"run_dir": "runs/emp/11003_G0024_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 92.0,
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 12.0,
"own_tests": 15.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 92.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 32,
"seconds": 691.7,
"run_dir": "runs/emp/11003_G0024_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 60,
"seconds": 232.0,
"run_dir": "runs/emp/15004_G0024_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0026/empirical.json
View File

@@ -1,4 +1,22 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 12,
"seconds": 656.4,
"run_dir": "runs/emp/10017_G0026_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 12,
"seconds": 656.4,
"run_dir": "runs/emp/10017_G0026_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 27,
"seconds": 113.2,
"run_dir": "runs/emp/15005_G0026_llm_deepseek-v4.1-flash_cloud"
}
}

28
tasks_gen/eval/G0107/empirical.json
View File

@@ -1,16 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 31,
"seconds": 162.0,
"run_dir": "runs/emp/10025_G0107_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@g2-func-header-bug": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 31,
"seconds": 162.0,
"run_dir": "runs/emp/10025_G0107_llm_deepseek-v4.1-flash_cloud"
}
}
}

34
tasks_gen/eval/G0108/empirical.json
View File

@@ -1,18 +1,36 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 60.0,
"parts": {
"correctness": 20.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 60.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "5/10",
"tool_calls": 20,
"seconds": 711.8,
"run_dir": "runs/emp/10026_G0108_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 60.0,
"score": 53.2,
"parts": {
"correctness": 20.0,
"correctness": 18.2,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 60.0,
"discipline": 0.0,
"total": 53.2,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "5/10",
"tool_calls": 20,
"seconds": 711.8,
"run_dir": "runs/emp/10026_G0108_llm_deepseek-v4.1-flash_cloud"
"hidden": "5/11",
"tool_calls": 60,
"seconds": 276.9,
"run_dir": "runs/emp/15006_G0108_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0109/empirical.json
View File

@@ -1,4 +1,22 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 33,
"seconds": 667.9,
"run_dir": "runs/emp/10027_G0109_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 33,
"seconds": 667.9,
"run_dir": "runs/emp/10027_G0109_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 60,
"seconds": 409.9,
"run_dir": "runs/emp/15007_G0109_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0110/empirical.json
View File

@@ -1,4 +1,22 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 17,
"seconds": 557.9,
"run_dir": "runs/emp/10028_G0110_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 17,
"seconds": 557.9,
"run_dir": "runs/emp/10028_G0110_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 57,
"seconds": 479.0,
"run_dir": "runs/emp/15008_G0110_llm_deepseek-v4.1-flash_cloud"
}
}

36
tasks_gen/eval/G0113/empirical.json
View File

@@ -1,15 +1,33 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 35,
"seconds": 917.0,
"run_dir": "runs/emp/10031_G0113_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 100.0,
"parts": {
"total": 0,
"failed_gates": [
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 35,
"seconds": 917.0,
"run_dir": "runs/emp/10031_G0113_llm_deepseek-v4.1-flash_cloud"
"hidden": "11/11",
"tool_calls": 14,
"seconds": 104.0,
"run_dir": "runs/emp/15009_G0113_llm_deepseek-v4.1-flash_cloud"
}
}

40
tasks_gen/eval/G0115/empirical.json
View File

@@ -1,17 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 563.7,
"run_dir": "runs/emp/10033_G0115_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 100.0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 563.7,
"run_dir": "runs/emp/10033_G0115_llm_deepseek-v4.1-flash_cloud"
"hidden": "10/10",
"tool_calls": 10,
"seconds": 161.8,
"run_dir": "runs/emp/15010_G0115_llm_deepseek-v4.1-flash_cloud"
}
}

30
tasks_gen/eval/G0119/empirical.json
View File

@@ -1,18 +1,36 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 42.3,
"parts": {
"correctness": 7.3,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 0.0,
"total": 42.3,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "2/11",
"tool_calls": 33,
"seconds": 752.0,
"run_dir": "runs/emp/10037_G0119_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 42.3,
"score": 49.8,
"parts": {
"correctness": 7.3,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"own_tests": 7.5,
"discipline": 0.0,
"total": 42.3,
"total": 49.8,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "2/11",
"tool_calls": 33,
"seconds": 752.0,
"run_dir": "runs/emp/10037_G0119_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 49,
"seconds": 492.0,
"run_dir": "runs/emp/15011_G0119_llm_deepseek-v4.1-flash_cloud"
}
}

36
tasks_gen/eval/G0121/empirical.json
View File

@@ -1,15 +1,33 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 24,
"seconds": 693.2,
"run_dir": "runs/emp/11007_G0121_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 70.0,
"parts": {
"total": 0,
"failed_gates": [
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 0.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 70.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 24,
"seconds": 693.2,
"run_dir": "runs/emp/11007_G0121_llm_deepseek-v4.1-flash_cloud"
"hidden": "11/11",
"tool_calls": 60,
"seconds": 686.3,
"run_dir": "runs/emp/15012_G0121_llm_deepseek-v4.1-flash_cloud"
}
}

30
tasks_gen/eval/G0123/empirical.json
View File

@@ -1,17 +1,19 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 19,
"seconds": 826.4,
"run_dir": "runs/emp/11008_G0123_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 19,
"seconds": 826.4,
"run_dir": "runs/emp/11008_G0123_llm_deepseek-v4.1-flash_cloud"
}
}
}

38
tasks_gen/eval/G0124/empirical.json
View File

@@ -1,17 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 60,
"seconds": 318.4,
"run_dir": "runs/emp/11009_G0124_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 85.5,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 10.5,
"own_tests": 15.0,
"discipline": 0.0,
"total": 85.5,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"hidden": "8/8",
"tool_calls": 60,
"seconds": 318.4,
"run_dir": "runs/emp/11009_G0124_llm_deepseek-v4.1-flash_cloud"
"seconds": 528.6,
"run_dir": "runs/emp/16014_G0124_llm_deepseek-v4.1-flash_cloud"
}
}

20
tasks_gen/eval/G0125/empirical.json
View File

@@ -1 +1,19 @@
{}
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 18,
"seconds": 786.2,
"run_dir": "runs/emp/13000_G0125_llm_deepseek-v4.1-flash_cloud"
}
}
}

18
tasks_gen/eval/G0129/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 95.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 18,
"seconds": 87.8,
"run_dir": "runs/emp/13001_G0129_llm_deepseek-v4.1-flash_cloud"
}
}

18
tasks_gen/eval/G0130/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 19,
"seconds": 162.9,
"run_dir": "runs/emp/13002_G0130_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0131/empirical.json
View File

@@ -1,4 +1,22 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 95.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 26,
"seconds": 718.5,
"run_dir": "runs/emp/10044_G0131_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 95.0,
"parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 26,
"seconds": 718.5,
"run_dir": "runs/emp/10044_G0131_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 38,
"seconds": 143.9,
"run_dir": "runs/emp/16000_G0131_llm_deepseek-v4.1-flash_cloud"
}
}

38
tasks_gen/eval/G0136/empirical.json
View File

@@ -1,16 +1,34 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 8,
"seconds": 682.7,
"run_dir": "runs/emp/10049_G0136_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 100.0,
"parts": {
"total": 0,
"failed_gates": [
"G2_contract",
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 8,
"seconds": 682.7,
"run_dir": "runs/emp/10049_G0136_llm_deepseek-v4.1-flash_cloud"
"hidden": "11/11",
"tool_calls": 17,
"seconds": 164.4,
"run_dir": "runs/emp/16001_G0136_llm_deepseek-v4.1-flash_cloud"
}
}

40
tasks_gen/eval/G0137/empirical.json
View File

@@ -1,17 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 6,
"seconds": 580.1,
"run_dir": "runs/emp/10050_G0137_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 100.0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 6,
"seconds": 580.1,
"run_dir": "runs/emp/10050_G0137_llm_deepseek-v4.1-flash_cloud"
"hidden": "12/12",
"tool_calls": 8,
"seconds": 53.5,
"run_dir": "runs/emp/16002_G0137_llm_deepseek-v4.1-flash_cloud"
}
}

18
tasks_gen/eval/G0141/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 11,
"seconds": 164.8,
"run_dir": "runs/emp/13003_G0141_llm_deepseek-v4.1-flash_cloud"
}
}

40
tasks_gen/eval/G0142/empirical.json
View File

@@ -1,17 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 7,
"seconds": 756.1,
"run_dir": "runs/emp/11014_G0142_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"score": 95.0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "0/0",
"tool_calls": 7,
"seconds": 756.1,
"run_dir": "runs/emp/11014_G0142_llm_deepseek-v4.1-flash_cloud"
"hidden": "6/6",
"tool_calls": 19,
"seconds": 199.1,
"run_dir": "runs/emp/16003_G0142_llm_deepseek-v4.1-flash_cloud"
}
}

30
tasks_gen/eval/G0154/empirical.json
View File

@@ -1,18 +1,36 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 27,
"seconds": 713.7,
"run_dir": "runs/emp/11024_G0154_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 27,
"seconds": 713.7,
"run_dir": "runs/emp/11024_G0154_llm_deepseek-v4.1-flash_cloud"
"tool_calls": 38,
"seconds": 140.0,
"run_dir": "runs/emp/16004_G0154_llm_deepseek-v4.1-flash_cloud"
}
}

32
tasks_gen/eval/G0155/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 40,
"seconds": 144.2,
"run_dir": "runs/emp/11025_G0155_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 40,
"seconds": 144.2,
"run_dir": "runs/emp/11025_G0155_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0155/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0156/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 11,
"seconds": 56.1,
"run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 11,
"seconds": 56.1,
"run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0156/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0157/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 10,
"seconds": 45.7,
"run_dir": "runs/emp/11027_G0157_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 10,
"seconds": 45.7,
"run_dir": "runs/emp/11027_G0157_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0157/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v740sp05",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0158/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 11,
"seconds": 52.5,
"run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 11,
"seconds": 52.5,
"run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0158/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0159/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 11,
"seconds": 36.2,
"run_dir": "runs/emp/11029_G0159_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 11,
"seconds": 36.2,
"run_dir": "runs/emp/11029_G0159_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0159/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v740sp05",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

35
tasks_gen/eval/G0160/empirical.json Normal file
View File

@@ -0,0 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 479.5,
"run_dir": "runs/emp/13004_G0160_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 60,
"seconds": 506.3,
"run_dir": "runs/emp/16005_G0160_llm_deepseek-v4.1-flash_cloud"
}
}

37
tasks_gen/eval/G0161/empirical.json
View File

@@ -1 +1,36 @@
{}
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 88.5,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 13.5,
"own_tests": 15.0,
"discipline": 0.0,
"total": 88.5,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 60,
"seconds": 264.3,
"run_dir": "runs/emp/13005_G0161_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 95.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 29,
"seconds": 313.1,
"run_dir": "runs/emp/16006_G0161_llm_deepseek-v4.1-flash_cloud"
}
}

44
tasks_gen/eval/G0162/empirical.json
View File

@@ -1,17 +1,35 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 40,
"seconds": 259.5,
"run_dir": "runs/emp/12025_G0162_llm_deepseek-v4.1-flash_cloud"
},
"hidden": "0/0",
"tool_calls": 40,
"seconds": 259.5,
"run_dir": "runs/emp/12025_G0162_llm_deepseek-v4.1-flash_cloud"
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 40,
"seconds": 587.0,
"run_dir": "runs/emp/16015_G0162_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0162/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0163/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 17,
"seconds": 397.6,
"run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 17,
"seconds": 397.6,
"run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0163/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

46
tasks_gen/eval/G0164/empirical.json
View File

@@ -1,18 +1,36 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 75.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 0.0,
"total": 75.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 75.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 0.0,
"total": 75.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 13,
"seconds": 1043.6,
"run_dir": "runs/emp/12023_G0164_llm_deepseek-v4.1-flash_cloud"
},
"hidden": "12/12",
"tool_calls": 13,
"seconds": 1043.6,
"run_dir": "runs/emp/12023_G0164_llm_deepseek-v4.1-flash_cloud"
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 19,
"seconds": 424.7,
"run_dir": "runs/emp/16007_G0164_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0164/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v740sp05",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0165/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 24,
"seconds": 112.4,
"run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 24,
"seconds": 112.4,
"run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0165/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

46
tasks_gen/eval/G0166/empirical.json
View File

@@ -1,18 +1,36 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 26,
"seconds": 646.5,
"run_dir": "runs/emp/12021_G0166_llm_deepseek-v4.1-flash_cloud"
},
"hidden": "8/8",
"tool_calls": 26,
"seconds": 646.5,
"run_dir": "runs/emp/12021_G0166_llm_deepseek-v4.1-flash_cloud"
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 29,
"seconds": 93.3,
"run_dir": "runs/emp/16008_G0166_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0166/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 30,
"max_activations": 8
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0167/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 9,
"seconds": 30.3,
"run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 9,
"seconds": 30.3,
"run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0167/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

43
tasks_gen/eval/G0169/empirical.json
View File

@@ -1,16 +1,35 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 30,
"parts": {
"total": 30,
"stop": "stopped",
"gap": {
"how": "empty report"
}
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 30,
"parts": {
"total": 30,
"stop": "stopped",
"gap": {
"how": "empty report"
}
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 629.3,
"run_dir": "runs/emp/12019_G0169_llm_deepseek-v4.1-flash_cloud"
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 629.3,
"run_dir": "runs/emp/12019_G0169_llm_deepseek-v4.1-flash_cloud"
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the missing precedence rule for members who are both students and under 25, matching the expected gap."
}
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 7.3,
"run_dir": "runs/emp/16009_G0169_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0169/task.json
View File

@@ -27,8 +27,8 @@
]
},
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [],
"contract": [

28
tasks_gen/eval/G0170/empirical.json
View File

@@ -1,16 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z099U04Q_STUDIO_PRICER"
]
},
"hidden": "0/0",
"tool_calls": 9,
"seconds": 73.6,
"run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z099U04Q_STUDIO_PRICER"
]
},
"hidden": "0/0",
"tool_calls": 9,
"seconds": 73.6,
"run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0170/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "stop",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

34
tasks_gen/eval/G0171/empirical.json
View File

@@ -1,19 +1,21 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the missing rule for an empty or unknown clay type and explains that the caller can pass an empty clay type, matching the expected gap."
}
},
"hidden": "0/0",
"tool_calls": 2,
"seconds": 140.7,
"run_dir": "runs/emp/12017_G0171_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the missing rule for an empty or unknown clay type and explains that the caller can pass an empty clay type, matching the expected gap."
}
},
"hidden": "0/0",
"tool_calls": 2,
"seconds": 140.7,
"run_dir": "runs/emp/12017_G0171_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0171/task.json
View File

@@ -27,8 +27,8 @@
]
},
"budget": {
"max_tool_calls": 30,
"max_activations": 5
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

37
tasks_gen/eval/G0172/empirical.json
View File

@@ -1 +1,36 @@
{}
{
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"unknown",
"code"
],
[
"other",
"code"
],
[
"not",
"defined"
],
[
"no",
"rule"
]
]
}
},
"hidden": "0/0",
"tool_calls": 4,
"seconds": 8.2,
"run_dir": "runs/emp/13006_G0172_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0172/task.json
View File

@@ -27,8 +27,8 @@
]
},
"budget": {
"max_tool_calls": 25,
"max_activations": 6
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

54
tasks_gen/eval/G0173/empirical.json
View File

@@ -1,30 +1,32 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"18",
"15"
],
[
"minimum",
"20"
],
[
"contradict",
"rule"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"18",
"15"
],
[
"minimum",
"20"
],
[
"contradict",
"rule"
]
]
]
}
},
"hidden": "0/0",
"tool_calls": 9,
"seconds": 66.6,
"run_dir": "runs/emp/12015_G0173_llm_deepseek-v4.1-flash_cloud"
}
},
"hidden": "0/0",
"tool_calls": 9,
"seconds": 66.6,
"run_dir": "runs/emp/12015_G0173_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0173/task.json
View File

@@ -23,8 +23,8 @@
]
},
"budget": {
"max_tool_calls": 30,
"max_activations": 6
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

21
tasks_gen/eval/G0174/empirical.json
View File

@@ -13,6 +13,27 @@
"tool_calls": 12,
"seconds": 665.1,
"run_dir": "runs/emp/12014_G0174_llm_deepseek-v4.1-flash_cloud"
},
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"small",
"sale",
"defin"
]
]
}
},
"hidden": "0/0",
"tool_calls": 16,
"seconds": 78.1,
"run_dir": "runs/emp/16010_G0174_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0174/task.json
View File

@@ -16,8 +16,8 @@
]
},
"budget": {
"max_tool_calls": 20,
"max_activations": 5
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

34
tasks_gen/eval/G0175/empirical.json
View File

@@ -1,19 +1,21 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the contradiction between rule 1 requiring HARVEST_DATE >= P_FROM and rule 5 requiring HARVEST_DATE < P_FROM, making both conditions unsatisfiable together."
}
},
"hidden": "0/0",
"tool_calls": 1,
"seconds": 7.9,
"run_dir": "runs/emp/12013_G0175_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the contradiction between rule 1 requiring HARVEST_DATE >= P_FROM and rule 5 requiring HARVEST_DATE < P_FROM, making both conditions unsatisfiable together."
}
},
"hidden": "0/0",
"tool_calls": 1,
"seconds": 7.9,
"run_dir": "runs/emp/12013_G0175_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0175/task.json
View File

@@ -25,8 +25,8 @@
]
},
"budget": {
"max_tool_calls": 30,
"max_activations": 6
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

40
tasks_gen/eval/G0176/empirical.json
View File

@@ -1,16 +1,32 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z099O04W_I_STATION_ENERGY"
]
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z099O04W_I_STATION_ENERGY"
]
},
"hidden": "0/0",
"tool_calls": 14,
"seconds": 534.6,
"run_dir": "runs/emp/12012_G0176_llm_deepseek-v4.1-flash_cloud"
},
"hidden": "0/0",
"tool_calls": 14,
"seconds": 534.6,
"run_dir": "runs/emp/12012_G0176_llm_deepseek-v4.1-flash_cloud"
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z0CCR04W_I_STATION_ENERGY"
]
},
"hidden": "0/0",
"tool_calls": 20,
"seconds": 102.9,
"run_dir": "runs/emp/16011_G0176_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0176/task.json
View File

@@ -27,8 +27,8 @@
]
},
"budget": {
"max_tool_calls": 20,
"max_activations": 5
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

54
tasks_gen/eval/G0177/empirical.json
View File

@@ -1,30 +1,32 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"contradiction",
"rules"
],
[
"contradict",
"rules"
],
[
"confirmed",
"cancelled"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"contradiction",
"rules"
],
[
"contradict",
"rules"
],
[
"confirmed",
"cancelled"
]
]
]
}
},
"hidden": "0/0",
"tool_calls": 6,
"seconds": 12.0,
"run_dir": "runs/emp/12011_G0177_llm_deepseek-v4.1-flash_cloud"
}
},
"hidden": "0/0",
"tool_calls": 6,
"seconds": 12.0,
"run_dir": "runs/emp/12011_G0177_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0177/task.json
View File

@@ -23,8 +23,8 @@
]
},
"budget": {
"max_tool_calls": 20,
"max_activations": 5
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

34
tasks_gen/eval/G0185/empirical.json
View File

@@ -1,18 +1,36 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 50.9,
"parts": {
"correctness": 10.9,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 50.9,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "3/11",
"tool_calls": 17,
"seconds": 589.5,
"run_dir": "runs/emp/12004_G0185_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 50.9,
"score": 85.0,
"parts": {
"correctness": 10.9,
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 50.9,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "3/11",
"tool_calls": 17,
"seconds": 589.5,
"run_dir": "runs/emp/12004_G0185_llm_deepseek-v4.1-flash_cloud"
"hidden": "11/11",
"tool_calls": 20,
"seconds": 55.8,
"run_dir": "runs/emp/16012_G0185_llm_deepseek-v4.1-flash_cloud"
}
}

18
tasks_gen/eval/G0189/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 15,
"seconds": 35.9,
"run_dir": "runs/emp/13007_G0189_llm_deepseek-v4.1-flash_cloud"
}
}

46
tasks_gen/eval/G0190/empirical.json
View File

@@ -1,18 +1,36 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 15.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 15.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 19,
"seconds": 852.8,
"run_dir": "runs/emp/12001_G0190_llm_deepseek-v4.1-flash_cloud"
},
"hidden": "7/7",
"tool_calls": 19,
"seconds": 852.8,
"run_dir": "runs/emp/12001_G0190_llm_deepseek-v4.1-flash_cloud"
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 10.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 22,
"seconds": 376.9,
"run_dir": "runs/emp/16013_G0190_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0190/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{

32
tasks_gen/eval/G0191/empirical.json
View File

@@ -1,18 +1,20 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 8,
"seconds": 87.0,
"run_dir": "runs/emp/12000_G0191_llm_deepseek-v4.1-flash_cloud"
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 8,
"seconds": 87.0,
"run_dir": "runs/emp/12000_G0191_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0191/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 40,
"max_activations": 10
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [
{