erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Runner: G2 finds the FUNCTION statement after local classes; budget floor applied to all tasks

Browse Source 
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 18:25:30 +02:00
parent 7351849a76
commit 92e30eb554
78 changed files with 1362 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
harness/runner.py
View File

@@ -248,7 +248,9 @@ class Runner:
g2 = False g2 = False
g2_detail.append(f"{c['name']}: does not implement {c['implements']}") g2_detail.append(f"{c['name']}: does not implement {c['implements']}")
if c["type"] == "FUNC": # signature: every parameter with its type in the FUNCTION header if c["type"] == "FUNC": # signature: every parameter with its type in the FUNCTION header
header = src.split(".", 1)[0] # the FUNCTION statement, not the first statement: local classes can come before it (G0107)
fm = re.search(rf"^\s*FUNCTION\s+{re.escape(c['name'])}\b[^.]*\.", src, re.I | re.M)
header = fm.group(0) if fm else src.split(".", 1)[0]
for prm in c.get("params", []): for prm in c.get("params", []):
if not re.search(rf"\b{re.escape(prm['name'])}\b\)?\s+TYPE\s+{re.escape(prm['type'])}\b", if not re.search(rf"\b{re.escape(prm['name'])}\b\)?\s+TYPE\s+{re.escape(prm['type'])}\b",
header, re.I): header, re.I):

4
tasks/T01/task.json
View File

@@ -5,8 +5,8 @@
"release_target": "v758", "release_target": "v758",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

4
tasks/T13/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v758", "release_target": "v758",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

66
tasks/T14/task.json
View File

@@ -1,18 +1,60 @@
{ {
"id": "T14", "category": "B", "object_type": "PROG", "difficulty": 2, "id": "T14",
"release_target": "v758", "expected_outcome": "implement", "category": "B",
"budget": {"max_tool_calls": 40, "max_activations": 10}, "object_type": "PROG",
"difficulty": 2,
"release_target": "v758",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [ "seed": [
{"type": "TABL", "name": "{{P}}SALE", "file": "seed/sale.tabl.asabap", "description": "Sales"}, {
{"type": "CLAS", "name": "{{P}}T14_SEED", "file": "seed/t14_seed.clas.abap", "description": "T14 seed data", "run": true} "type": "TABL",
"name": "{{P}}SALE",
"file": "seed/sale.tabl.asabap",
"description": "Sales"
},
{
"type": "CLAS",
"name": "{{P}}T14_SEED",
"file": "seed/t14_seed.clas.abap",
"description": "T14 seed data",
"run": true
}
], ],
"contract": [ "contract": [
{"type": "PROG", "name": "{{P}}SALES_SUM", "parameters": ["p_from", "s_reg"]} {
"type": "PROG",
"name": "{{P}}SALES_SUM",
"parameters": [
"p_from",
"s_reg"
]
}
], ],
"out_of_scope": ["{{P}}SALE"], "out_of_scope": [
"hidden_tests": [{"type": "CLAS", "name": "{{P}}T14_HIDDEN", "file": "hidden/t14_hidden.clas.abap", "{{P}}SALE"
"description": "T14 hidden tests"}], ],
"reference": [{"type": "PROG", "name": "{{P}}SALES_SUM", "file": "reference/sales_sum.prog.abap", "hidden_tests": [
"description": "Sales per region"}], {
"craft_checks": ["db_operation_in_loop", "method_length"] "type": "CLAS",
"name": "{{P}}T14_HIDDEN",
"file": "hidden/t14_hidden.clas.abap",
"description": "T14 hidden tests"
}
],
"reference": [
{
"type": "PROG",
"name": "{{P}}SALES_SUM",
"file": "reference/sales_sum.prog.abap",
"description": "Sales per region"
}
],
"craft_checks": [
"db_operation_in_loop",
"method_length"
]
} }

69
tasks/T15/task.json
View File

@@ -1,22 +1,65 @@
{ {
"id": "T15", "category": "B", "object_type": "DDLS", "difficulty": 2, "id": "T15",
"release_target": "v758", "expected_outcome": "implement", "category": "B",
"budget": {"max_tool_calls": 40, "max_activations": 10}, "object_type": "DDLS",
"difficulty": 2,
"release_target": "v758",
"expected_outcome": "implement",
"budget": {
"max_tool_calls": 60,
"max_activations": 15
},
"seed": [ "seed": [
{"type": "TABL", "name": "{{P}}ORD", "file": "seed/ord.tabl.asabap", "description": "Order header"}, {
{"type": "TABL", "name": "{{P}}ITM", "file": "seed/itm.tabl.asabap", "description": "Order item"} "type": "TABL",
"name": "{{P}}ORD",
"file": "seed/ord.tabl.asabap",
"description": "Order header"
},
{
"type": "TABL",
"name": "{{P}}ITM",
"file": "seed/itm.tabl.asabap",
"description": "Order item"
}
], ],
"contract": [ "contract": [
{"type": "DDLS", "name": "{{P}}I_ORDER_TOTAL", {
"fields": ["ORDER_ID", "CUSTOMER", "ITEM_COUNT", "TOTAL_AMOUNT"]} "type": "DDLS",
"name": "{{P}}I_ORDER_TOTAL",
"fields": [
"ORDER_ID",
"CUSTOMER",
"ITEM_COUNT",
"TOTAL_AMOUNT"
]
}
],
"out_of_scope": [
"{{P}}ORD",
"{{P}}ITM"
],
"hidden_tests": [
{
"type": "CLAS",
"name": "{{P}}T15_HIDDEN",
"file": "hidden/t15_hidden.clas.abap",
"description": "T15 hidden tests"
}
], ],
"out_of_scope": ["{{P}}ORD", "{{P}}ITM"],
"hidden_tests": [{"type": "CLAS", "name": "{{P}}T15_HIDDEN", "file": "hidden/t15_hidden.clas.abap",
"description": "T15 hidden tests"}],
"reference": [ "reference": [
{"type": "DDLS", "name": "{{P}}I_ORDER_TOTAL", "file": "reference/i_order_total.ddls.asddls", {
"description": "Order totals"}, "type": "DDLS",
{"type": "CLAS", "name": "{{P}}T15_TEST", "file": "reference/t15_test.clas.abap", "description": "T15 own tests"} "name": "{{P}}I_ORDER_TOTAL",
"file": "reference/i_order_total.ddls.asddls",
"description": "Order totals"
},
{
"type": "CLAS",
"name": "{{P}}T15_TEST",
"file": "reference/t15_test.clas.abap",
"description": "T15 own tests"
}
], ],
"craft_checks": [] "craft_checks": []
} }

40
tasks_gen/eval/G0014/empirical.json
View File

@@ -1,17 +1,35 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 36,
"seconds": 928.5,
"run_dir": "runs/emp/10007_G0014_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 100.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G1_active", "clean": 15.0,
"G2_contract", "own_tests": 15.0,
"G3_hidden_runs" "discipline": 10.0,
] "total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "6/6",
"tool_calls": 36, "tool_calls": 29,
"seconds": 928.5, "seconds": 259.2,
"run_dir": "runs/emp/10007_G0014_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15000_G0014_llm_deepseek-v4.1-flash_cloud"
} }
} }

24
tasks_gen/eval/G0015/empirical.json
View File

@@ -1,4 +1,22 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 18,
"seconds": 687.3,
"run_dir": "runs/emp/10008_G0015_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 90.0, "score": 90.0,
"parts": { "parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "8/8", "hidden": "8/8",
"tool_calls": 18, "tool_calls": 59,
"seconds": 687.3, "seconds": 350.0,
"run_dir": "runs/emp/10008_G0015_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15001_G0015_llm_deepseek-v4.1-flash_cloud"
} }
} }

34
tasks_gen/eval/G0019/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 36,
"seconds": 680.7,
"run_dir": "runs/emp/10011_G0019_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 80.0, "score": 90.0,
"parts": { "parts": {
"correctness": 40.0, "correctness": 40.0,
"craft": 20.0, "craft": 20.0,
"clean": 15.0, "clean": 15.0,
"own_tests": 0.0, "own_tests": 15.0,
"discipline": 5.0, "discipline": 0.0,
"total": 80.0, "total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "8/8", "hidden": "9/9",
"tool_calls": 36, "tool_calls": 58,
"seconds": 680.7, "seconds": 392.1,
"run_dir": "runs/emp/10011_G0019_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15002_G0019_llm_deepseek-v4.1-flash_cloud"
} }
} }

24
tasks_gen/eval/G0020/empirical.json
View File

@@ -1,4 +1,22 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 38,
"seconds": 658.5,
"run_dir": "runs/emp/11001_G0020_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 80.0, "score": 80.0,
"parts": { "parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "8/8", "hidden": "8/8",
"tool_calls": 38, "tool_calls": 60,
"seconds": 658.5, "seconds": 154.3,
"run_dir": "runs/emp/11001_G0020_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15003_G0020_llm_deepseek-v4.1-flash_cloud"
} }
} }

32
tasks_gen/eval/G0024/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 92.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 12.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 92.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 32,
"seconds": 691.7,
"run_dir": "runs/emp/11003_G0024_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 92.0, "score": 80.0,
"parts": { "parts": {
"correctness": 40.0, "correctness": 40.0,
"craft": 20.0, "craft": 20.0,
"clean": 12.0, "clean": 15.0,
"own_tests": 15.0, "own_tests": 0.0,
"discipline": 5.0, "discipline": 5.0,
"total": 92.0, "total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "7/7", "hidden": "7/7",
"tool_calls": 32, "tool_calls": 60,
"seconds": 691.7, "seconds": 232.0,
"run_dir": "runs/emp/11003_G0024_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15004_G0024_llm_deepseek-v4.1-flash_cloud"
} }
} }

24
tasks_gen/eval/G0026/empirical.json
View File

@@ -1,4 +1,22 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 12,
"seconds": 656.4,
"run_dir": "runs/emp/10017_G0026_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 80.0, "score": 80.0,
"parts": { "parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "10/10", "hidden": "10/10",
"tool_calls": 12, "tool_calls": 27,
"seconds": 656.4, "seconds": 113.2,
"run_dir": "runs/emp/10017_G0026_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15005_G0026_llm_deepseek-v4.1-flash_cloud"
} }
} }

28
tasks_gen/eval/G0107/empirical.json
View File

@@ -1,16 +1,18 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 0, "deepseek-v4.1-flash:cloud@g2-func-header-bug": {
"parts": { "score": 0,
"total": 0, "parts": {
"failed_gates": [ "total": 0,
"G2_contract", "failed_gates": [
"G3_hidden_runs" "G2_contract",
] "G3_hidden_runs"
}, ]
"hidden": "0/0", },
"tool_calls": 31, "hidden": "0/0",
"seconds": 162.0, "tool_calls": 31,
"run_dir": "runs/emp/10025_G0107_llm_deepseek-v4.1-flash_cloud" "seconds": 162.0,
"run_dir": "runs/emp/10025_G0107_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

34
tasks_gen/eval/G0108/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 60.0,
"parts": {
"correctness": 20.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 60.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "5/10",
"tool_calls": 20,
"seconds": 711.8,
"run_dir": "runs/emp/10026_G0108_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 60.0, "score": 53.2,
"parts": { "parts": {
"correctness": 20.0, "correctness": 18.2,
"craft": 20.0, "craft": 20.0,
"clean": 15.0, "clean": 15.0,
"own_tests": 0.0, "own_tests": 0.0,
"discipline": 5.0, "discipline": 0.0,
"total": 60.0, "total": 53.2,
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "5/10", "hidden": "5/11",
"tool_calls": 20, "tool_calls": 60,
"seconds": 711.8, "seconds": 276.9,
"run_dir": "runs/emp/10026_G0108_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15006_G0108_llm_deepseek-v4.1-flash_cloud"
} }
} }

24
tasks_gen/eval/G0109/empirical.json
View File

@@ -1,4 +1,22 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 33,
"seconds": 667.9,
"run_dir": "runs/emp/10027_G0109_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 80.0, "score": 80.0,
"parts": { "parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "8/8", "hidden": "8/8",
"tool_calls": 33, "tool_calls": 60,
"seconds": 667.9, "seconds": 409.9,
"run_dir": "runs/emp/10027_G0109_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15007_G0109_llm_deepseek-v4.1-flash_cloud"
} }
} }

24
tasks_gen/eval/G0110/empirical.json
View File

@@ -1,4 +1,22 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 17,
"seconds": 557.9,
"run_dir": "runs/emp/10028_G0110_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 80.0, "score": 80.0,
"parts": { "parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "6/6", "hidden": "6/6",
"tool_calls": 17, "tool_calls": 57,
"seconds": 557.9, "seconds": 479.0,
"run_dir": "runs/emp/10028_G0110_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15008_G0110_llm_deepseek-v4.1-flash_cloud"
} }
} }

36
tasks_gen/eval/G0113/empirical.json
View File

@@ -1,15 +1,33 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 35,
"seconds": 917.0,
"run_dir": "runs/emp/10031_G0113_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 100.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G3_hidden_runs" "clean": 15.0,
] "own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "11/11",
"tool_calls": 35, "tool_calls": 14,
"seconds": 917.0, "seconds": 104.0,
"run_dir": "runs/emp/10031_G0113_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15009_G0113_llm_deepseek-v4.1-flash_cloud"
} }
} }

40
tasks_gen/eval/G0115/empirical.json
View File

@@ -1,17 +1,35 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 563.7,
"run_dir": "runs/emp/10033_G0115_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 100.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G1_active", "clean": 15.0,
"G2_contract", "own_tests": 15.0,
"G3_hidden_runs" "discipline": 10.0,
] "total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "10/10",
"tool_calls": 0, "tool_calls": 10,
"seconds": 563.7, "seconds": 161.8,
"run_dir": "runs/emp/10033_G0115_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15010_G0115_llm_deepseek-v4.1-flash_cloud"
} }
} }

30
tasks_gen/eval/G0119/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 42.3,
"parts": {
"correctness": 7.3,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 0.0,
"total": 42.3,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "2/11",
"tool_calls": 33,
"seconds": 752.0,
"run_dir": "runs/emp/10037_G0119_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 42.3, "score": 49.8,
"parts": { "parts": {
"correctness": 7.3, "correctness": 7.3,
"craft": 20.0, "craft": 20.0,
"clean": 15.0, "clean": 15.0,
"own_tests": 0.0, "own_tests": 7.5,
"discipline": 0.0, "discipline": 0.0,
"total": 42.3, "total": 49.8,
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "2/11", "hidden": "2/11",
"tool_calls": 33, "tool_calls": 49,
"seconds": 752.0, "seconds": 492.0,
"run_dir": "runs/emp/10037_G0119_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15011_G0119_llm_deepseek-v4.1-flash_cloud"
} }
} }

36
tasks_gen/eval/G0121/empirical.json
View File

@@ -1,15 +1,33 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 24,
"seconds": 693.2,
"run_dir": "runs/emp/11007_G0121_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 70.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 0.0,
"G3_hidden_runs" "clean": 15.0,
] "own_tests": 15.0,
"discipline": 0.0,
"total": 70.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "11/11",
"tool_calls": 24, "tool_calls": 60,
"seconds": 693.2, "seconds": 686.3,
"run_dir": "runs/emp/11007_G0121_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/15012_G0121_llm_deepseek-v4.1-flash_cloud"
} }
} }

30
tasks_gen/eval/G0123/empirical.json
View File

@@ -1,17 +1,19 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 0, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 0,
"total": 0, "parts": {
"failed_gates": [ "total": 0,
"G1_active", "failed_gates": [
"G2_contract", "G1_active",
"G3_hidden_runs" "G2_contract",
] "G3_hidden_runs"
}, ]
"hidden": "0/0", },
"tool_calls": 19, "hidden": "0/0",
"seconds": 826.4, "tool_calls": 19,
"run_dir": "runs/emp/11008_G0123_llm_deepseek-v4.1-flash_cloud" "seconds": 826.4,
"run_dir": "runs/emp/11008_G0123_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

38
tasks_gen/eval/G0124/empirical.json
View File

@@ -1,17 +1,35 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 60,
"seconds": 318.4,
"run_dir": "runs/emp/11009_G0124_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 85.5,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G1_active", "clean": 10.5,
"G2_contract", "own_tests": 15.0,
"G3_hidden_runs" "discipline": 0.0,
] "total": 85.5,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "8/8",
"tool_calls": 60, "tool_calls": 60,
"seconds": 318.4, "seconds": 528.6,
"run_dir": "runs/emp/11009_G0124_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16014_G0124_llm_deepseek-v4.1-flash_cloud"
} }
} }

20
tasks_gen/eval/G0125/empirical.json
View File

@@ -1 +1,19 @@
{} {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 18,
"seconds": 786.2,
"run_dir": "runs/emp/13000_G0125_llm_deepseek-v4.1-flash_cloud"
}
}
}

18
tasks_gen/eval/G0129/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 95.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 18,
"seconds": 87.8,
"run_dir": "runs/emp/13001_G0129_llm_deepseek-v4.1-flash_cloud"
}
}

18
tasks_gen/eval/G0130/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 19,
"seconds": 162.9,
"run_dir": "runs/emp/13002_G0130_llm_deepseek-v4.1-flash_cloud"
}
}

24
tasks_gen/eval/G0131/empirical.json
View File

@@ -1,4 +1,22 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 95.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 26,
"seconds": 718.5,
"run_dir": "runs/emp/10044_G0131_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 95.0, "score": 95.0,
"parts": { "parts": {
@@ -11,8 +29,8 @@
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "10/10", "hidden": "10/10",
"tool_calls": 26, "tool_calls": 38,
"seconds": 718.5, "seconds": 143.9,
"run_dir": "runs/emp/10044_G0131_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16000_G0131_llm_deepseek-v4.1-flash_cloud"
} }
} }

38
tasks_gen/eval/G0136/empirical.json
View File

@@ -1,16 +1,34 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 8,
"seconds": 682.7,
"run_dir": "runs/emp/10049_G0136_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 100.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G2_contract", "clean": 15.0,
"G3_hidden_runs" "own_tests": 15.0,
] "discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "11/11",
"tool_calls": 8, "tool_calls": 17,
"seconds": 682.7, "seconds": 164.4,
"run_dir": "runs/emp/10049_G0136_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16001_G0136_llm_deepseek-v4.1-flash_cloud"
} }
} }

40
tasks_gen/eval/G0137/empirical.json
View File

@@ -1,17 +1,35 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 6,
"seconds": 580.1,
"run_dir": "runs/emp/10050_G0137_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 100.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G1_active", "clean": 15.0,
"G2_contract", "own_tests": 15.0,
"G3_hidden_runs" "discipline": 10.0,
] "total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "12/12",
"tool_calls": 6, "tool_calls": 8,
"seconds": 580.1, "seconds": 53.5,
"run_dir": "runs/emp/10050_G0137_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16002_G0137_llm_deepseek-v4.1-flash_cloud"
} }
} }

18
tasks_gen/eval/G0141/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 11,
"seconds": 164.8,
"run_dir": "runs/emp/13003_G0141_llm_deepseek-v4.1-flash_cloud"
}
}

40
tasks_gen/eval/G0142/empirical.json
View File

@@ -1,17 +1,35 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 7,
"seconds": 756.1,
"run_dir": "runs/emp/11014_G0142_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 0, "score": 95.0,
"parts": { "parts": {
"total": 0, "correctness": 40.0,
"failed_gates": [ "craft": 20.0,
"G1_active", "clean": 15.0,
"G2_contract", "own_tests": 15.0,
"G3_hidden_runs" "discipline": 5.0,
] "total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "0/0", "hidden": "6/6",
"tool_calls": 7, "tool_calls": 19,
"seconds": 756.1, "seconds": 199.1,
"run_dir": "runs/emp/11014_G0142_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16003_G0142_llm_deepseek-v4.1-flash_cloud"
} }
} }

30
tasks_gen/eval/G0154/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 27,
"seconds": 713.7,
"run_dir": "runs/emp/11024_G0154_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 80.0, "score": 85.0,
"parts": { "parts": {
"correctness": 40.0, "correctness": 40.0,
"craft": 20.0, "craft": 20.0,
"clean": 15.0, "clean": 15.0,
"own_tests": 0.0, "own_tests": 0.0,
"discipline": 5.0, "discipline": 10.0,
"total": 80.0, "total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "6/6", "hidden": "6/6",
"tool_calls": 27, "tool_calls": 38,
"seconds": 713.7, "seconds": 140.0,
"run_dir": "runs/emp/11024_G0154_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16004_G0154_llm_deepseek-v4.1-flash_cloud"
} }
} }

32
tasks_gen/eval/G0155/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 80.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 80.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 5.0, "own_tests": 0.0,
"total": 80.0, "discipline": 5.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 80.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "10/10", },
"tool_calls": 40, "hidden": "10/10",
"seconds": 144.2, "tool_calls": 40,
"run_dir": "runs/emp/11025_G0155_llm_deepseek-v4.1-flash_cloud" "seconds": 144.2,
"run_dir": "runs/emp/11025_G0155_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0155/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0156/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 85.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 85.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 0.0,
"total": 85.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 85.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "10/10", },
"tool_calls": 11, "hidden": "10/10",
"seconds": 56.1, "tool_calls": 11,
"run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud" "seconds": 56.1,
"run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0156/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702", "release_target": "v702",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0157/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 15.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 15.0,
"total": 100.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 100.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "10/10", },
"tool_calls": 10, "hidden": "10/10",
"seconds": 45.7, "tool_calls": 10,
"run_dir": "runs/emp/11027_G0157_llm_deepseek-v4.1-flash_cloud" "seconds": 45.7,
"run_dir": "runs/emp/11027_G0157_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0157/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v740sp05", "release_target": "v740sp05",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0158/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 85.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 85.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 0.0,
"total": 85.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 85.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "11/11", },
"tool_calls": 11, "hidden": "11/11",
"seconds": 52.5, "tool_calls": 11,
"run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud" "seconds": 52.5,
"run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0158/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702", "release_target": "v702",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0159/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 85.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 85.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 0.0,
"total": 85.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 85.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "10/10", },
"tool_calls": 11, "hidden": "10/10",
"seconds": 36.2, "tool_calls": 11,
"run_dir": "runs/emp/11029_G0159_llm_deepseek-v4.1-flash_cloud" "seconds": 36.2,
"run_dir": "runs/emp/11029_G0159_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0159/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v740sp05", "release_target": "v740sp05",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

35
tasks_gen/eval/G0160/empirical.json Normal file
View File

@@ -0,0 +1,35 @@
{
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 479.5,
"run_dir": "runs/emp/13004_G0160_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 60,
"seconds": 506.3,
"run_dir": "runs/emp/16005_G0160_llm_deepseek-v4.1-flash_cloud"
}
}

37
tasks_gen/eval/G0161/empirical.json
View File

@@ -1 +1,36 @@
{} {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 88.5,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 13.5,
"own_tests": 15.0,
"discipline": 0.0,
"total": 88.5,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 60,
"seconds": 264.3,
"run_dir": "runs/emp/13005_G0161_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": {
"score": 95.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 95.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 29,
"seconds": 313.1,
"run_dir": "runs/emp/16006_G0161_llm_deepseek-v4.1-flash_cloud"
}
}

44
tasks_gen/eval/G0162/empirical.json
View File

@@ -1,17 +1,35 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 0, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 0,
"total": 0, "parts": {
"failed_gates": [ "total": 0,
"G1_active", "failed_gates": [
"G2_contract", "G1_active",
"G3_hidden_runs" "G2_contract",
] "G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 40,
"seconds": 259.5,
"run_dir": "runs/emp/12025_G0162_llm_deepseek-v4.1-flash_cloud"
}, },
"hidden": "0/0", "deepseek-v4.1-flash:cloud@budget<60": {
"tool_calls": 40, "score": 90.0,
"seconds": 259.5, "parts": {
"run_dir": "runs/emp/12025_G0162_llm_deepseek-v4.1-flash_cloud" "correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 40,
"seconds": 587.0,
"run_dir": "runs/emp/16015_G0162_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0162/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702", "release_target": "v702",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0163/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 15.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 15.0,
"total": 100.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 100.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "11/11", },
"tool_calls": 17, "hidden": "11/11",
"seconds": 397.6, "tool_calls": 17,
"run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud" "seconds": 397.6,
"run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0163/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v702", "release_target": "v702",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

46
tasks_gen/eval/G0164/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 75.0, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 75.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 0.0, "own_tests": 0.0,
"total": 75.0, "discipline": 0.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 75.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 13,
"seconds": 1043.6,
"run_dir": "runs/emp/12023_G0164_llm_deepseek-v4.1-flash_cloud"
}, },
"hidden": "12/12", "deepseek-v4.1-flash:cloud@budget<60": {
"tool_calls": 13, "score": 100.0,
"seconds": 1043.6, "parts": {
"run_dir": "runs/emp/12023_G0164_llm_deepseek-v4.1-flash_cloud" "correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 19,
"seconds": 424.7,
"run_dir": "runs/emp/16007_G0164_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0164/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v740sp05", "release_target": "v740sp05",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0165/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 85.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 85.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 0.0,
"total": 85.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 85.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "6/6", },
"tool_calls": 24, "hidden": "6/6",
"seconds": 112.4, "tool_calls": 24,
"run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud" "seconds": 112.4,
"run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0165/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

46
tasks_gen/eval/G0166/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 80.0, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 80.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 0.0, "clean": 15.0,
"discipline": 5.0, "own_tests": 0.0,
"total": 80.0, "discipline": 5.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 26,
"seconds": 646.5,
"run_dir": "runs/emp/12021_G0166_llm_deepseek-v4.1-flash_cloud"
}, },
"hidden": "8/8", "deepseek-v4.1-flash:cloud@budget<60": {
"tool_calls": 26, "score": 85.0,
"seconds": 646.5, "parts": {
"run_dir": "runs/emp/12021_G0166_llm_deepseek-v4.1-flash_cloud" "correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 29,
"seconds": 93.3,
"run_dir": "runs/emp/16008_G0166_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0166/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 30, "max_tool_calls": 60,
"max_activations": 8 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0167/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 15.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 15.0,
"total": 100.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 100.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "11/11", },
"tool_calls": 9, "hidden": "11/11",
"seconds": 30.3, "tool_calls": 9,
"run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud" "seconds": 30.3,
"run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0167/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

43
tasks_gen/eval/G0169/empirical.json
View File

@@ -1,16 +1,35 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 30, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 30,
"total": 30, "parts": {
"stop": "stopped", "total": 30,
"gap": { "stop": "stopped",
"how": "empty report" "gap": {
} "how": "empty report"
}
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 629.3,
"run_dir": "runs/emp/12019_G0169_llm_deepseek-v4.1-flash_cloud"
}, },
"hidden": "0/0", "deepseek-v4.1-flash:cloud@budget<60": {
"tool_calls": 0, "score": 100,
"seconds": 629.3, "parts": {
"run_dir": "runs/emp/12019_G0169_llm_deepseek-v4.1-flash_cloud" "total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the missing precedence rule for members who are both students and under 25, matching the expected gap."
}
},
"hidden": "0/0",
"tool_calls": 0,
"seconds": 7.3,
"run_dir": "runs/emp/16009_G0169_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0169/task.json
View File

@@ -27,8 +27,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [], "seed": [],
"contract": [ "contract": [

28
tasks_gen/eval/G0170/empirical.json
View File

@@ -1,16 +1,18 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 0,
"total": 0, "parts": {
"stop": "implemented", "total": 0,
"objects": [ "stop": "implemented",
"Z099U04Q_STUDIO_PRICER" "objects": [
] "Z099U04Q_STUDIO_PRICER"
}, ]
"hidden": "0/0", },
"tool_calls": 9, "hidden": "0/0",
"seconds": 73.6, "tool_calls": 9,
"run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud" "seconds": 73.6,
"run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0170/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "stop", "expected_outcome": "stop",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

34
tasks_gen/eval/G0171/empirical.json
View File

@@ -1,19 +1,21 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100,
"total": 100, "parts": {
"stop": "stopped", "total": 100,
"gap": { "stop": "stopped",
"how": "judge", "gap": {
"model": "deepseek-v4.1-flash:cloud", "how": "judge",
"names_gap": true, "model": "deepseek-v4.1-flash:cloud",
"reason": "The report explicitly identifies the missing rule for an empty or unknown clay type and explains that the caller can pass an empty clay type, matching the expected gap." "names_gap": true,
} "reason": "The report explicitly identifies the missing rule for an empty or unknown clay type and explains that the caller can pass an empty clay type, matching the expected gap."
}, }
"hidden": "0/0", },
"tool_calls": 2, "hidden": "0/0",
"seconds": 140.7, "tool_calls": 2,
"run_dir": "runs/emp/12017_G0171_llm_deepseek-v4.1-flash_cloud" "seconds": 140.7,
"run_dir": "runs/emp/12017_G0171_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0171/task.json
View File

@@ -27,8 +27,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 30, "max_tool_calls": 60,
"max_activations": 5 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

37
tasks_gen/eval/G0172/empirical.json
View File

@@ -1 +1,36 @@
{} {
"_superseded": {
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"unknown",
"code"
],
[
"other",
"code"
],
[
"not",
"defined"
],
[
"no",
"rule"
]
]
}
},
"hidden": "0/0",
"tool_calls": 4,
"seconds": 8.2,
"run_dir": "runs/emp/13006_G0172_llm_deepseek-v4.1-flash_cloud"
}
}
}

4
tasks_gen/eval/G0172/task.json
View File

@@ -27,8 +27,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 25, "max_tool_calls": 60,
"max_activations": 6 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

54
tasks_gen/eval/G0173/empirical.json
View File

@@ -1,30 +1,32 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100,
"total": 100, "parts": {
"stop": "stopped", "total": 100,
"gap": { "stop": "stopped",
"how": "keywords", "gap": {
"group": [ "how": "keywords",
[ "group": [
"18", [
"15" "18",
], "15"
[ ],
"minimum", [
"20" "minimum",
], "20"
[ ],
"contradict", [
"rule" "contradict",
"rule"
]
] ]
] }
} },
}, "hidden": "0/0",
"hidden": "0/0", "tool_calls": 9,
"tool_calls": 9, "seconds": 66.6,
"seconds": 66.6, "run_dir": "runs/emp/12015_G0173_llm_deepseek-v4.1-flash_cloud"
"run_dir": "runs/emp/12015_G0173_llm_deepseek-v4.1-flash_cloud" }
} }
} }

4
tasks_gen/eval/G0173/task.json
View File

@@ -23,8 +23,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 30, "max_tool_calls": 60,
"max_activations": 6 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

21
tasks_gen/eval/G0174/empirical.json
View File

@@ -13,6 +13,27 @@
"tool_calls": 12, "tool_calls": 12,
"seconds": 665.1, "seconds": 665.1,
"run_dir": "runs/emp/12014_G0174_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/12014_G0174_llm_deepseek-v4.1-flash_cloud"
},
"deepseek-v4.1-flash:cloud@budget<60": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"small",
"sale",
"defin"
]
]
}
},
"hidden": "0/0",
"tool_calls": 16,
"seconds": 78.1,
"run_dir": "runs/emp/16010_G0174_llm_deepseek-v4.1-flash_cloud"
} }
} }
} }

4
tasks_gen/eval/G0174/task.json
View File

@@ -16,8 +16,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 20, "max_tool_calls": 60,
"max_activations": 5 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

34
tasks_gen/eval/G0175/empirical.json
View File

@@ -1,19 +1,21 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100,
"total": 100, "parts": {
"stop": "stopped", "total": 100,
"gap": { "stop": "stopped",
"how": "judge", "gap": {
"model": "deepseek-v4.1-flash:cloud", "how": "judge",
"names_gap": true, "model": "deepseek-v4.1-flash:cloud",
"reason": "The report explicitly identifies the contradiction between rule 1 requiring HARVEST_DATE >= P_FROM and rule 5 requiring HARVEST_DATE < P_FROM, making both conditions unsatisfiable together." "names_gap": true,
} "reason": "The report explicitly identifies the contradiction between rule 1 requiring HARVEST_DATE >= P_FROM and rule 5 requiring HARVEST_DATE < P_FROM, making both conditions unsatisfiable together."
}, }
"hidden": "0/0", },
"tool_calls": 1, "hidden": "0/0",
"seconds": 7.9, "tool_calls": 1,
"run_dir": "runs/emp/12013_G0175_llm_deepseek-v4.1-flash_cloud" "seconds": 7.9,
"run_dir": "runs/emp/12013_G0175_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0175/task.json
View File

@@ -25,8 +25,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 30, "max_tool_calls": 60,
"max_activations": 6 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

40
tasks_gen/eval/G0176/empirical.json
View File

@@ -1,16 +1,32 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 0, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 0,
"total": 0, "parts": {
"stop": "implemented", "total": 0,
"objects": [ "stop": "implemented",
"Z099O04W_I_STATION_ENERGY" "objects": [
] "Z099O04W_I_STATION_ENERGY"
]
},
"hidden": "0/0",
"tool_calls": 14,
"seconds": 534.6,
"run_dir": "runs/emp/12012_G0176_llm_deepseek-v4.1-flash_cloud"
}, },
"hidden": "0/0", "deepseek-v4.1-flash:cloud@budget<60": {
"tool_calls": 14, "score": 0,
"seconds": 534.6, "parts": {
"run_dir": "runs/emp/12012_G0176_llm_deepseek-v4.1-flash_cloud" "total": 0,
"stop": "implemented",
"objects": [
"Z0CCR04W_I_STATION_ENERGY"
]
},
"hidden": "0/0",
"tool_calls": 20,
"seconds": 102.9,
"run_dir": "runs/emp/16011_G0176_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0176/task.json
View File

@@ -27,8 +27,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 20, "max_tool_calls": 60,
"max_activations": 5 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

54
tasks_gen/eval/G0177/empirical.json
View File

@@ -1,30 +1,32 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100,
"total": 100, "parts": {
"stop": "stopped", "total": 100,
"gap": { "stop": "stopped",
"how": "keywords", "gap": {
"group": [ "how": "keywords",
[ "group": [
"contradiction", [
"rules" "contradiction",
], "rules"
[ ],
"contradict", [
"rules" "contradict",
], "rules"
[ ],
"confirmed", [
"cancelled" "confirmed",
"cancelled"
]
] ]
] }
} },
}, "hidden": "0/0",
"hidden": "0/0", "tool_calls": 6,
"tool_calls": 6, "seconds": 12.0,
"seconds": 12.0, "run_dir": "runs/emp/12011_G0177_llm_deepseek-v4.1-flash_cloud"
"run_dir": "runs/emp/12011_G0177_llm_deepseek-v4.1-flash_cloud" }
} }
} }

4
tasks_gen/eval/G0177/task.json
View File

@@ -23,8 +23,8 @@
] ]
}, },
"budget": { "budget": {
"max_tool_calls": 20, "max_tool_calls": 60,
"max_activations": 5 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

34
tasks_gen/eval/G0185/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"_superseded": {
"deepseek-v4.1-flash:cloud@2026-10-03": {
"score": 50.9,
"parts": {
"correctness": 10.9,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 5.0,
"total": 50.9,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "3/11",
"tool_calls": 17,
"seconds": 589.5,
"run_dir": "runs/emp/12004_G0185_llm_deepseek-v4.1-flash_cloud"
}
},
"deepseek-v4.1-flash:cloud": { "deepseek-v4.1-flash:cloud": {
"score": 50.9, "score": 85.0,
"parts": { "parts": {
"correctness": 10.9, "correctness": 40.0,
"craft": 20.0, "craft": 20.0,
"clean": 15.0, "clean": 15.0,
"own_tests": 0.0, "own_tests": 0.0,
"discipline": 5.0, "discipline": 10.0,
"total": 50.9, "total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "note": "own_tests: faulty-reference part not implemented in skeleton"
}, },
"hidden": "3/11", "hidden": "11/11",
"tool_calls": 17, "tool_calls": 20,
"seconds": 589.5, "seconds": 55.8,
"run_dir": "runs/emp/12004_G0185_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16012_G0185_llm_deepseek-v4.1-flash_cloud"
} }
} }

18
tasks_gen/eval/G0189/empirical.json Normal file
View File

@@ -0,0 +1,18 @@
{
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 15,
"seconds": 35.9,
"run_dir": "runs/emp/13007_G0189_llm_deepseek-v4.1-flash_cloud"
}
}

46
tasks_gen/eval/G0190/empirical.json
View File

@@ -1,18 +1,36 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 90.0, "deepseek-v4.1-flash:cloud@2026-10-03": {
"parts": { "score": 90.0,
"correctness": 40.0, "parts": {
"craft": 15.0, "correctness": 40.0,
"clean": 15.0, "craft": 15.0,
"own_tests": 15.0, "clean": 15.0,
"discipline": 5.0, "own_tests": 15.0,
"total": 90.0, "discipline": 5.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 19,
"seconds": 852.8,
"run_dir": "runs/emp/12001_G0190_llm_deepseek-v4.1-flash_cloud"
}, },
"hidden": "7/7", "deepseek-v4.1-flash:cloud@budget<60": {
"tool_calls": 19, "score": 85.0,
"seconds": 852.8, "parts": {
"run_dir": "runs/emp/12001_G0190_llm_deepseek-v4.1-flash_cloud" "correctness": 40.0,
"craft": 10.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 22,
"seconds": 376.9,
"run_dir": "runs/emp/16013_G0190_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0190/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {

32
tasks_gen/eval/G0191/empirical.json
View File

@@ -1,18 +1,20 @@
{ {
"deepseek-v4.1-flash:cloud": { "_superseded": {
"score": 100.0, "deepseek-v4.1-flash:cloud@budget<60": {
"parts": { "score": 100.0,
"correctness": 40.0, "parts": {
"craft": 20.0, "correctness": 40.0,
"clean": 15.0, "craft": 20.0,
"own_tests": 15.0, "clean": 15.0,
"discipline": 10.0, "own_tests": 15.0,
"total": 100.0, "discipline": 10.0,
"note": "own_tests: faulty-reference part not implemented in skeleton" "total": 100.0,
}, "note": "own_tests: faulty-reference part not implemented in skeleton"
"hidden": "12/12", },
"tool_calls": 8, "hidden": "12/12",
"seconds": 87.0, "tool_calls": 8,
"run_dir": "runs/emp/12000_G0191_llm_deepseek-v4.1-flash_cloud" "seconds": 87.0,
"run_dir": "runs/emp/12000_G0191_llm_deepseek-v4.1-flash_cloud"
}
} }
} }

4
tasks_gen/eval/G0191/task.json
View File

@@ -6,8 +6,8 @@
"release_target": "v816", "release_target": "v816",
"expected_outcome": "implement", "expected_outcome": "implement",
"budget": { "budget": {
"max_tool_calls": 40, "max_tool_calls": 60,
"max_activations": 10 "max_activations": 15
}, },
"seed": [ "seed": [
{ {