From 92e30eb55444f8ae1692f0aa024ab6549bda3fb0 Mon Sep 17 00:00:00 2001 From: Kral Date: Sat, 3 Oct 2026 18:25:30 +0200 Subject: [PATCH] Runner: G2 finds the FUNCTION statement after local classes; budget floor applied to all tasks Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat --- harness/runner.py | 4 +- tasks/T01/task.json | 4 +- tasks/T13/task.json | 4 +- tasks/T14/task.json | 68 +++++++++++++++++++++------ tasks/T15/task.json | 71 +++++++++++++++++++++++------ tasks_gen/eval/G0014/empirical.json | 40 +++++++++++----- tasks_gen/eval/G0015/empirical.json | 24 ++++++++-- tasks_gen/eval/G0019/empirical.json | 34 ++++++++++---- tasks_gen/eval/G0020/empirical.json | 24 ++++++++-- tasks_gen/eval/G0024/empirical.json | 32 ++++++++++--- tasks_gen/eval/G0026/empirical.json | 24 ++++++++-- tasks_gen/eval/G0107/empirical.json | 28 ++++++------ tasks_gen/eval/G0108/empirical.json | 34 ++++++++++---- tasks_gen/eval/G0109/empirical.json | 24 ++++++++-- tasks_gen/eval/G0110/empirical.json | 24 ++++++++-- tasks_gen/eval/G0113/empirical.json | 36 +++++++++++---- tasks_gen/eval/G0115/empirical.json | 40 +++++++++++----- tasks_gen/eval/G0119/empirical.json | 30 +++++++++--- tasks_gen/eval/G0121/empirical.json | 36 +++++++++++---- tasks_gen/eval/G0123/empirical.json | 30 ++++++------ tasks_gen/eval/G0124/empirical.json | 38 +++++++++++---- tasks_gen/eval/G0125/empirical.json | 20 +++++++- tasks_gen/eval/G0129/empirical.json | 18 ++++++++ tasks_gen/eval/G0130/empirical.json | 18 ++++++++ tasks_gen/eval/G0131/empirical.json | 24 ++++++++-- tasks_gen/eval/G0136/empirical.json | 38 +++++++++++---- tasks_gen/eval/G0137/empirical.json | 40 +++++++++++----- tasks_gen/eval/G0141/empirical.json | 18 ++++++++ tasks_gen/eval/G0142/empirical.json | 40 +++++++++++----- tasks_gen/eval/G0154/empirical.json | 30 +++++++++--- tasks_gen/eval/G0155/empirical.json | 32 +++++++------ tasks_gen/eval/G0155/task.json | 4 +- tasks_gen/eval/G0156/empirical.json | 32 +++++++------ tasks_gen/eval/G0156/task.json | 4 +- tasks_gen/eval/G0157/empirical.json | 32 +++++++------ tasks_gen/eval/G0157/task.json | 4 +- tasks_gen/eval/G0158/empirical.json | 32 +++++++------ tasks_gen/eval/G0158/task.json | 4 +- tasks_gen/eval/G0159/empirical.json | 32 +++++++------ tasks_gen/eval/G0159/task.json | 4 +- tasks_gen/eval/G0160/empirical.json | 35 ++++++++++++++ tasks_gen/eval/G0161/empirical.json | 37 ++++++++++++++- tasks_gen/eval/G0162/empirical.json | 44 ++++++++++++------ tasks_gen/eval/G0162/task.json | 4 +- tasks_gen/eval/G0163/empirical.json | 32 +++++++------ tasks_gen/eval/G0163/task.json | 4 +- tasks_gen/eval/G0164/empirical.json | 46 +++++++++++++------ tasks_gen/eval/G0164/task.json | 4 +- tasks_gen/eval/G0165/empirical.json | 32 +++++++------ tasks_gen/eval/G0165/task.json | 4 +- tasks_gen/eval/G0166/empirical.json | 46 +++++++++++++------ tasks_gen/eval/G0166/task.json | 4 +- tasks_gen/eval/G0167/empirical.json | 32 +++++++------ tasks_gen/eval/G0167/task.json | 4 +- tasks_gen/eval/G0169/empirical.json | 43 ++++++++++++----- tasks_gen/eval/G0169/task.json | 4 +- tasks_gen/eval/G0170/empirical.json | 28 ++++++------ tasks_gen/eval/G0170/task.json | 4 +- tasks_gen/eval/G0171/empirical.json | 34 +++++++------- tasks_gen/eval/G0171/task.json | 4 +- tasks_gen/eval/G0172/empirical.json | 37 ++++++++++++++- tasks_gen/eval/G0172/task.json | 4 +- tasks_gen/eval/G0173/empirical.json | 54 +++++++++++----------- tasks_gen/eval/G0173/task.json | 4 +- tasks_gen/eval/G0174/empirical.json | 21 +++++++++ tasks_gen/eval/G0174/task.json | 4 +- tasks_gen/eval/G0175/empirical.json | 34 +++++++------- tasks_gen/eval/G0175/task.json | 4 +- tasks_gen/eval/G0176/empirical.json | 40 +++++++++++----- tasks_gen/eval/G0176/task.json | 4 +- tasks_gen/eval/G0177/empirical.json | 54 +++++++++++----------- tasks_gen/eval/G0177/task.json | 4 +- tasks_gen/eval/G0185/empirical.json | 34 ++++++++++---- tasks_gen/eval/G0189/empirical.json | 18 ++++++++ tasks_gen/eval/G0190/empirical.json | 46 +++++++++++++------ tasks_gen/eval/G0190/task.json | 4 +- tasks_gen/eval/G0191/empirical.json | 32 +++++++------ tasks_gen/eval/G0191/task.json | 4 +- 78 files changed, 1362 insertions(+), 560 deletions(-) create mode 100644 tasks_gen/eval/G0129/empirical.json create mode 100644 tasks_gen/eval/G0130/empirical.json create mode 100644 tasks_gen/eval/G0141/empirical.json create mode 100644 tasks_gen/eval/G0160/empirical.json create mode 100644 tasks_gen/eval/G0189/empirical.json diff --git a/harness/runner.py b/harness/runner.py index f5ce9b6..f9a5313 100644 --- a/harness/runner.py +++ b/harness/runner.py @@ -248,7 +248,9 @@ class Runner: g2 = False g2_detail.append(f"{c['name']}: does not implement {c['implements']}") if c["type"] == "FUNC": # signature: every parameter with its type in the FUNCTION header - header = src.split(".", 1)[0] + # the FUNCTION statement, not the first statement: local classes can come before it (G0107) + fm = re.search(rf"^\s*FUNCTION\s+{re.escape(c['name'])}\b[^.]*\.", src, re.I | re.M) + header = fm.group(0) if fm else src.split(".", 1)[0] for prm in c.get("params", []): if not re.search(rf"\b{re.escape(prm['name'])}\b\)?\s+TYPE\s+{re.escape(prm['type'])}\b", header, re.I): diff --git a/tasks/T01/task.json b/tasks/T01/task.json index cb7f30b..affa31c 100644 --- a/tasks/T01/task.json +++ b/tasks/T01/task.json @@ -5,8 +5,8 @@ "release_target": "v758", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks/T13/task.json b/tasks/T13/task.json index 1a41443..aac6e4d 100644 --- a/tasks/T13/task.json +++ b/tasks/T13/task.json @@ -6,8 +6,8 @@ "release_target": "v758", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks/T14/task.json b/tasks/T14/task.json index c34013e..b6b23db 100644 --- a/tasks/T14/task.json +++ b/tasks/T14/task.json @@ -1,18 +1,60 @@ { - "id": "T14", "category": "B", "object_type": "PROG", "difficulty": 2, - "release_target": "v758", "expected_outcome": "implement", - "budget": {"max_tool_calls": 40, "max_activations": 10}, + "id": "T14", + "category": "B", + "object_type": "PROG", + "difficulty": 2, + "release_target": "v758", + "expected_outcome": "implement", + "budget": { + "max_tool_calls": 60, + "max_activations": 15 + }, "seed": [ - {"type": "TABL", "name": "{{P}}SALE", "file": "seed/sale.tabl.asabap", "description": "Sales"}, - {"type": "CLAS", "name": "{{P}}T14_SEED", "file": "seed/t14_seed.clas.abap", "description": "T14 seed data", "run": true} + { + "type": "TABL", + "name": "{{P}}SALE", + "file": "seed/sale.tabl.asabap", + "description": "Sales" + }, + { + "type": "CLAS", + "name": "{{P}}T14_SEED", + "file": "seed/t14_seed.clas.abap", + "description": "T14 seed data", + "run": true + } ], "contract": [ - {"type": "PROG", "name": "{{P}}SALES_SUM", "parameters": ["p_from", "s_reg"]} + { + "type": "PROG", + "name": "{{P}}SALES_SUM", + "parameters": [ + "p_from", + "s_reg" + ] + } ], - "out_of_scope": ["{{P}}SALE"], - "hidden_tests": [{"type": "CLAS", "name": "{{P}}T14_HIDDEN", "file": "hidden/t14_hidden.clas.abap", - "description": "T14 hidden tests"}], - "reference": [{"type": "PROG", "name": "{{P}}SALES_SUM", "file": "reference/sales_sum.prog.abap", - "description": "Sales per region"}], - "craft_checks": ["db_operation_in_loop", "method_length"] -} + "out_of_scope": [ + "{{P}}SALE" + ], + "hidden_tests": [ + { + "type": "CLAS", + "name": "{{P}}T14_HIDDEN", + "file": "hidden/t14_hidden.clas.abap", + "description": "T14 hidden tests" + } + ], + "reference": [ + { + "type": "PROG", + "name": "{{P}}SALES_SUM", + "file": "reference/sales_sum.prog.abap", + "description": "Sales per region" + } + ], + "craft_checks": [ + "db_operation_in_loop", + "method_length" + ] +} \ No newline at end of file diff --git a/tasks/T15/task.json b/tasks/T15/task.json index 719bc53..8fa19fa 100644 --- a/tasks/T15/task.json +++ b/tasks/T15/task.json @@ -1,22 +1,65 @@ { - "id": "T15", "category": "B", "object_type": "DDLS", "difficulty": 2, - "release_target": "v758", "expected_outcome": "implement", - "budget": {"max_tool_calls": 40, "max_activations": 10}, + "id": "T15", + "category": "B", + "object_type": "DDLS", + "difficulty": 2, + "release_target": "v758", + "expected_outcome": "implement", + "budget": { + "max_tool_calls": 60, + "max_activations": 15 + }, "seed": [ - {"type": "TABL", "name": "{{P}}ORD", "file": "seed/ord.tabl.asabap", "description": "Order header"}, - {"type": "TABL", "name": "{{P}}ITM", "file": "seed/itm.tabl.asabap", "description": "Order item"} + { + "type": "TABL", + "name": "{{P}}ORD", + "file": "seed/ord.tabl.asabap", + "description": "Order header" + }, + { + "type": "TABL", + "name": "{{P}}ITM", + "file": "seed/itm.tabl.asabap", + "description": "Order item" + } ], "contract": [ - {"type": "DDLS", "name": "{{P}}I_ORDER_TOTAL", - "fields": ["ORDER_ID", "CUSTOMER", "ITEM_COUNT", "TOTAL_AMOUNT"]} + { + "type": "DDLS", + "name": "{{P}}I_ORDER_TOTAL", + "fields": [ + "ORDER_ID", + "CUSTOMER", + "ITEM_COUNT", + "TOTAL_AMOUNT" + ] + } + ], + "out_of_scope": [ + "{{P}}ORD", + "{{P}}ITM" + ], + "hidden_tests": [ + { + "type": "CLAS", + "name": "{{P}}T15_HIDDEN", + "file": "hidden/t15_hidden.clas.abap", + "description": "T15 hidden tests" + } ], - "out_of_scope": ["{{P}}ORD", "{{P}}ITM"], - "hidden_tests": [{"type": "CLAS", "name": "{{P}}T15_HIDDEN", "file": "hidden/t15_hidden.clas.abap", - "description": "T15 hidden tests"}], "reference": [ - {"type": "DDLS", "name": "{{P}}I_ORDER_TOTAL", "file": "reference/i_order_total.ddls.asddls", - "description": "Order totals"}, - {"type": "CLAS", "name": "{{P}}T15_TEST", "file": "reference/t15_test.clas.abap", "description": "T15 own tests"} + { + "type": "DDLS", + "name": "{{P}}I_ORDER_TOTAL", + "file": "reference/i_order_total.ddls.asddls", + "description": "Order totals" + }, + { + "type": "CLAS", + "name": "{{P}}T15_TEST", + "file": "reference/t15_test.clas.abap", + "description": "T15 own tests" + } ], "craft_checks": [] -} +} \ No newline at end of file diff --git a/tasks_gen/eval/G0014/empirical.json b/tasks_gen/eval/G0014/empirical.json index a774b41..078f630 100644 --- a/tasks_gen/eval/G0014/empirical.json +++ b/tasks_gen/eval/G0014/empirical.json @@ -1,17 +1,35 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 36, + "seconds": 928.5, + "run_dir": "runs/emp/10007_G0014_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 100.0, "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 36, - "seconds": 928.5, - "run_dir": "runs/emp/10007_G0014_llm_deepseek-v4.1-flash_cloud" + "hidden": "6/6", + "tool_calls": 29, + "seconds": 259.2, + "run_dir": "runs/emp/15000_G0014_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0015/empirical.json b/tasks_gen/eval/G0015/empirical.json index 53e769e..f6176dd 100644 --- a/tasks_gen/eval/G0015/empirical.json +++ b/tasks_gen/eval/G0015/empirical.json @@ -1,4 +1,22 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 90.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 90.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 18, + "seconds": 687.3, + "run_dir": "runs/emp/10008_G0015_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { "score": 90.0, "parts": { @@ -11,8 +29,8 @@ "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "8/8", - "tool_calls": 18, - "seconds": 687.3, - "run_dir": "runs/emp/10008_G0015_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 59, + "seconds": 350.0, + "run_dir": "runs/emp/15001_G0015_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0019/empirical.json b/tasks_gen/eval/G0019/empirical.json index 30d1ee8..33adf67 100644 --- a/tasks_gen/eval/G0019/empirical.json +++ b/tasks_gen/eval/G0019/empirical.json @@ -1,18 +1,36 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 36, + "seconds": 680.7, + "run_dir": "runs/emp/10011_G0019_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 80.0, + "score": 90.0, "parts": { "correctness": 40.0, "craft": 20.0, "clean": 15.0, - "own_tests": 0.0, - "discipline": 5.0, - "total": 80.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 90.0, "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "8/8", - "tool_calls": 36, - "seconds": 680.7, - "run_dir": "runs/emp/10011_G0019_llm_deepseek-v4.1-flash_cloud" + "hidden": "9/9", + "tool_calls": 58, + "seconds": 392.1, + "run_dir": "runs/emp/15002_G0019_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0020/empirical.json b/tasks_gen/eval/G0020/empirical.json index 092bef7..7cfc9e6 100644 --- a/tasks_gen/eval/G0020/empirical.json +++ b/tasks_gen/eval/G0020/empirical.json @@ -1,4 +1,22 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 38, + "seconds": 658.5, + "run_dir": "runs/emp/11001_G0020_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { "score": 80.0, "parts": { @@ -11,8 +29,8 @@ "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "8/8", - "tool_calls": 38, - "seconds": 658.5, - "run_dir": "runs/emp/11001_G0020_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 60, + "seconds": 154.3, + "run_dir": "runs/emp/15003_G0020_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0024/empirical.json b/tasks_gen/eval/G0024/empirical.json index 9efc284..5fc6251 100644 --- a/tasks_gen/eval/G0024/empirical.json +++ b/tasks_gen/eval/G0024/empirical.json @@ -1,18 +1,36 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 92.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 12.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 92.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "7/7", + "tool_calls": 32, + "seconds": 691.7, + "run_dir": "runs/emp/11003_G0024_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 92.0, + "score": 80.0, "parts": { "correctness": 40.0, "craft": 20.0, - "clean": 12.0, - "own_tests": 15.0, + "clean": 15.0, + "own_tests": 0.0, "discipline": 5.0, - "total": 92.0, + "total": 80.0, "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "7/7", - "tool_calls": 32, - "seconds": 691.7, - "run_dir": "runs/emp/11003_G0024_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 60, + "seconds": 232.0, + "run_dir": "runs/emp/15004_G0024_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0026/empirical.json b/tasks_gen/eval/G0026/empirical.json index e7bce4e..cb3b25a 100644 --- a/tasks_gen/eval/G0026/empirical.json +++ b/tasks_gen/eval/G0026/empirical.json @@ -1,4 +1,22 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 12, + "seconds": 656.4, + "run_dir": "runs/emp/10017_G0026_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { "score": 80.0, "parts": { @@ -11,8 +29,8 @@ "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "10/10", - "tool_calls": 12, - "seconds": 656.4, - "run_dir": "runs/emp/10017_G0026_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 27, + "seconds": 113.2, + "run_dir": "runs/emp/15005_G0026_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0107/empirical.json b/tasks_gen/eval/G0107/empirical.json index 99dad75..d44f7db 100644 --- a/tasks_gen/eval/G0107/empirical.json +++ b/tasks_gen/eval/G0107/empirical.json @@ -1,16 +1,18 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 0, - "parts": { - "total": 0, - "failed_gates": [ - "G2_contract", - "G3_hidden_runs" - ] - }, - "hidden": "0/0", - "tool_calls": 31, - "seconds": 162.0, - "run_dir": "runs/emp/10025_G0107_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@g2-func-header-bug": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 31, + "seconds": 162.0, + "run_dir": "runs/emp/10025_G0107_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0108/empirical.json b/tasks_gen/eval/G0108/empirical.json index 135ef2a..92fe38e 100644 --- a/tasks_gen/eval/G0108/empirical.json +++ b/tasks_gen/eval/G0108/empirical.json @@ -1,18 +1,36 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 60.0, + "parts": { + "correctness": 20.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 60.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "5/10", + "tool_calls": 20, + "seconds": 711.8, + "run_dir": "runs/emp/10026_G0108_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 60.0, + "score": 53.2, "parts": { - "correctness": 20.0, + "correctness": 18.2, "craft": 20.0, "clean": 15.0, "own_tests": 0.0, - "discipline": 5.0, - "total": 60.0, + "discipline": 0.0, + "total": 53.2, "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "5/10", - "tool_calls": 20, - "seconds": 711.8, - "run_dir": "runs/emp/10026_G0108_llm_deepseek-v4.1-flash_cloud" + "hidden": "5/11", + "tool_calls": 60, + "seconds": 276.9, + "run_dir": "runs/emp/15006_G0108_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0109/empirical.json b/tasks_gen/eval/G0109/empirical.json index 9178ecb..5868d0e 100644 --- a/tasks_gen/eval/G0109/empirical.json +++ b/tasks_gen/eval/G0109/empirical.json @@ -1,4 +1,22 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 33, + "seconds": 667.9, + "run_dir": "runs/emp/10027_G0109_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { "score": 80.0, "parts": { @@ -11,8 +29,8 @@ "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "8/8", - "tool_calls": 33, - "seconds": 667.9, - "run_dir": "runs/emp/10027_G0109_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 60, + "seconds": 409.9, + "run_dir": "runs/emp/15007_G0109_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0110/empirical.json b/tasks_gen/eval/G0110/empirical.json index aa504b2..0fd4956 100644 --- a/tasks_gen/eval/G0110/empirical.json +++ b/tasks_gen/eval/G0110/empirical.json @@ -1,4 +1,22 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "6/6", + "tool_calls": 17, + "seconds": 557.9, + "run_dir": "runs/emp/10028_G0110_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { "score": 80.0, "parts": { @@ -11,8 +29,8 @@ "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "6/6", - "tool_calls": 17, - "seconds": 557.9, - "run_dir": "runs/emp/10028_G0110_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 57, + "seconds": 479.0, + "run_dir": "runs/emp/15008_G0110_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0113/empirical.json b/tasks_gen/eval/G0113/empirical.json index 3679fd9..db4b7a9 100644 --- a/tasks_gen/eval/G0113/empirical.json +++ b/tasks_gen/eval/G0113/empirical.json @@ -1,15 +1,33 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 35, + "seconds": 917.0, + "run_dir": "runs/emp/10031_G0113_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 100.0, "parts": { - "total": 0, - "failed_gates": [ - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 35, - "seconds": 917.0, - "run_dir": "runs/emp/10031_G0113_llm_deepseek-v4.1-flash_cloud" + "hidden": "11/11", + "tool_calls": 14, + "seconds": 104.0, + "run_dir": "runs/emp/15009_G0113_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0115/empirical.json b/tasks_gen/eval/G0115/empirical.json index c1e6b89..e9d4cbf 100644 --- a/tasks_gen/eval/G0115/empirical.json +++ b/tasks_gen/eval/G0115/empirical.json @@ -1,17 +1,35 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 0, + "seconds": 563.7, + "run_dir": "runs/emp/10033_G0115_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 100.0, "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 0, - "seconds": 563.7, - "run_dir": "runs/emp/10033_G0115_llm_deepseek-v4.1-flash_cloud" + "hidden": "10/10", + "tool_calls": 10, + "seconds": 161.8, + "run_dir": "runs/emp/15010_G0115_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0119/empirical.json b/tasks_gen/eval/G0119/empirical.json index 80a8a79..91ade6c 100644 --- a/tasks_gen/eval/G0119/empirical.json +++ b/tasks_gen/eval/G0119/empirical.json @@ -1,18 +1,36 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 42.3, + "parts": { + "correctness": 7.3, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 0.0, + "total": 42.3, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "2/11", + "tool_calls": 33, + "seconds": 752.0, + "run_dir": "runs/emp/10037_G0119_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 42.3, + "score": 49.8, "parts": { "correctness": 7.3, "craft": 20.0, "clean": 15.0, - "own_tests": 0.0, + "own_tests": 7.5, "discipline": 0.0, - "total": 42.3, + "total": 49.8, "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "2/11", - "tool_calls": 33, - "seconds": 752.0, - "run_dir": "runs/emp/10037_G0119_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 49, + "seconds": 492.0, + "run_dir": "runs/emp/15011_G0119_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0121/empirical.json b/tasks_gen/eval/G0121/empirical.json index 59d0133..6db6492 100644 --- a/tasks_gen/eval/G0121/empirical.json +++ b/tasks_gen/eval/G0121/empirical.json @@ -1,15 +1,33 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 24, + "seconds": 693.2, + "run_dir": "runs/emp/11007_G0121_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 70.0, "parts": { - "total": 0, - "failed_gates": [ - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 0.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 70.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 24, - "seconds": 693.2, - "run_dir": "runs/emp/11007_G0121_llm_deepseek-v4.1-flash_cloud" + "hidden": "11/11", + "tool_calls": 60, + "seconds": 686.3, + "run_dir": "runs/emp/15012_G0121_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0123/empirical.json b/tasks_gen/eval/G0123/empirical.json index 6b0b206..a88e0f1 100644 --- a/tasks_gen/eval/G0123/empirical.json +++ b/tasks_gen/eval/G0123/empirical.json @@ -1,17 +1,19 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 0, - "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] - }, - "hidden": "0/0", - "tool_calls": 19, - "seconds": 826.4, - "run_dir": "runs/emp/11008_G0123_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 19, + "seconds": 826.4, + "run_dir": "runs/emp/11008_G0123_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0124/empirical.json b/tasks_gen/eval/G0124/empirical.json index d2d8e91..1fc68df 100644 --- a/tasks_gen/eval/G0124/empirical.json +++ b/tasks_gen/eval/G0124/empirical.json @@ -1,17 +1,35 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 60, + "seconds": 318.4, + "run_dir": "runs/emp/11009_G0124_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 85.5, "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 10.5, + "own_tests": 15.0, + "discipline": 0.0, + "total": 85.5, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", + "hidden": "8/8", "tool_calls": 60, - "seconds": 318.4, - "run_dir": "runs/emp/11009_G0124_llm_deepseek-v4.1-flash_cloud" + "seconds": 528.6, + "run_dir": "runs/emp/16014_G0124_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0125/empirical.json b/tasks_gen/eval/G0125/empirical.json index 9e26dfe..853350a 100644 --- a/tasks_gen/eval/G0125/empirical.json +++ b/tasks_gen/eval/G0125/empirical.json @@ -1 +1,19 @@ -{} \ No newline at end of file +{ + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 18, + "seconds": 786.2, + "run_dir": "runs/emp/13000_G0125_llm_deepseek-v4.1-flash_cloud" + } + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0129/empirical.json b/tasks_gen/eval/G0129/empirical.json new file mode 100644 index 0000000..c49c711 --- /dev/null +++ b/tasks_gen/eval/G0129/empirical.json @@ -0,0 +1,18 @@ +{ + "deepseek-v4.1-flash:cloud": { + "score": 95.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 95.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 18, + "seconds": 87.8, + "run_dir": "runs/emp/13001_G0129_llm_deepseek-v4.1-flash_cloud" + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0130/empirical.json b/tasks_gen/eval/G0130/empirical.json new file mode 100644 index 0000000..b421b1f --- /dev/null +++ b/tasks_gen/eval/G0130/empirical.json @@ -0,0 +1,18 @@ +{ + "deepseek-v4.1-flash:cloud": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "12/12", + "tool_calls": 19, + "seconds": 162.9, + "run_dir": "runs/emp/13002_G0130_llm_deepseek-v4.1-flash_cloud" + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0131/empirical.json b/tasks_gen/eval/G0131/empirical.json index 1fbaf35..c939939 100644 --- a/tasks_gen/eval/G0131/empirical.json +++ b/tasks_gen/eval/G0131/empirical.json @@ -1,4 +1,22 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 95.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 95.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 26, + "seconds": 718.5, + "run_dir": "runs/emp/10044_G0131_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { "score": 95.0, "parts": { @@ -11,8 +29,8 @@ "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "10/10", - "tool_calls": 26, - "seconds": 718.5, - "run_dir": "runs/emp/10044_G0131_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 38, + "seconds": 143.9, + "run_dir": "runs/emp/16000_G0131_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0136/empirical.json b/tasks_gen/eval/G0136/empirical.json index c3f1c28..97ec4d1 100644 --- a/tasks_gen/eval/G0136/empirical.json +++ b/tasks_gen/eval/G0136/empirical.json @@ -1,16 +1,34 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 8, + "seconds": 682.7, + "run_dir": "runs/emp/10049_G0136_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 100.0, "parts": { - "total": 0, - "failed_gates": [ - "G2_contract", - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 8, - "seconds": 682.7, - "run_dir": "runs/emp/10049_G0136_llm_deepseek-v4.1-flash_cloud" + "hidden": "11/11", + "tool_calls": 17, + "seconds": 164.4, + "run_dir": "runs/emp/16001_G0136_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0137/empirical.json b/tasks_gen/eval/G0137/empirical.json index 5eee9e4..d4338bc 100644 --- a/tasks_gen/eval/G0137/empirical.json +++ b/tasks_gen/eval/G0137/empirical.json @@ -1,17 +1,35 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 6, + "seconds": 580.1, + "run_dir": "runs/emp/10050_G0137_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 100.0, "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 6, - "seconds": 580.1, - "run_dir": "runs/emp/10050_G0137_llm_deepseek-v4.1-flash_cloud" + "hidden": "12/12", + "tool_calls": 8, + "seconds": 53.5, + "run_dir": "runs/emp/16002_G0137_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0141/empirical.json b/tasks_gen/eval/G0141/empirical.json new file mode 100644 index 0000000..4750d00 --- /dev/null +++ b/tasks_gen/eval/G0141/empirical.json @@ -0,0 +1,18 @@ +{ + "deepseek-v4.1-flash:cloud": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "6/6", + "tool_calls": 11, + "seconds": 164.8, + "run_dir": "runs/emp/13003_G0141_llm_deepseek-v4.1-flash_cloud" + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0142/empirical.json b/tasks_gen/eval/G0142/empirical.json index b89e189..30873de 100644 --- a/tasks_gen/eval/G0142/empirical.json +++ b/tasks_gen/eval/G0142/empirical.json @@ -1,17 +1,35 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 7, + "seconds": 756.1, + "run_dir": "runs/emp/11014_G0142_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 0, + "score": 95.0, "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 95.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "0/0", - "tool_calls": 7, - "seconds": 756.1, - "run_dir": "runs/emp/11014_G0142_llm_deepseek-v4.1-flash_cloud" + "hidden": "6/6", + "tool_calls": 19, + "seconds": 199.1, + "run_dir": "runs/emp/16003_G0142_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0154/empirical.json b/tasks_gen/eval/G0154/empirical.json index 0e9d011..a794037 100644 --- a/tasks_gen/eval/G0154/empirical.json +++ b/tasks_gen/eval/G0154/empirical.json @@ -1,18 +1,36 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "6/6", + "tool_calls": 27, + "seconds": 713.7, + "run_dir": "runs/emp/11024_G0154_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 80.0, + "score": 85.0, "parts": { "correctness": 40.0, "craft": 20.0, "clean": 15.0, "own_tests": 0.0, - "discipline": 5.0, - "total": 80.0, + "discipline": 10.0, + "total": 85.0, "note": "own_tests: faulty-reference part not implemented in skeleton" }, "hidden": "6/6", - "tool_calls": 27, - "seconds": 713.7, - "run_dir": "runs/emp/11024_G0154_llm_deepseek-v4.1-flash_cloud" + "tool_calls": 38, + "seconds": 140.0, + "run_dir": "runs/emp/16004_G0154_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0155/empirical.json b/tasks_gen/eval/G0155/empirical.json index b8dfd2c..9fc8dcd 100644 --- a/tasks_gen/eval/G0155/empirical.json +++ b/tasks_gen/eval/G0155/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 80.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 5.0, - "total": 80.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "10/10", - "tool_calls": 40, - "seconds": 144.2, - "run_dir": "runs/emp/11025_G0155_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 40, + "seconds": 144.2, + "run_dir": "runs/emp/11025_G0155_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0155/task.json b/tasks_gen/eval/G0155/task.json index 0a3ec77..0bbd190 100644 --- a/tasks_gen/eval/G0155/task.json +++ b/tasks_gen/eval/G0155/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0156/empirical.json b/tasks_gen/eval/G0156/empirical.json index 029bc0e..44fba3d 100644 --- a/tasks_gen/eval/G0156/empirical.json +++ b/tasks_gen/eval/G0156/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 85.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 10.0, - "total": 85.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "10/10", - "tool_calls": 11, - "seconds": 56.1, - "run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 11, + "seconds": 56.1, + "run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0156/task.json b/tasks_gen/eval/G0156/task.json index 06408ae..b2dc19b 100644 --- a/tasks_gen/eval/G0156/task.json +++ b/tasks_gen/eval/G0156/task.json @@ -6,8 +6,8 @@ "release_target": "v702", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0157/empirical.json b/tasks_gen/eval/G0157/empirical.json index 4dd5444..0d4d1b3 100644 --- a/tasks_gen/eval/G0157/empirical.json +++ b/tasks_gen/eval/G0157/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 15.0, - "discipline": 10.0, - "total": 100.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "10/10", - "tool_calls": 10, - "seconds": 45.7, - "run_dir": "runs/emp/11027_G0157_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 10, + "seconds": 45.7, + "run_dir": "runs/emp/11027_G0157_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0157/task.json b/tasks_gen/eval/G0157/task.json index 2872fc2..208b4eb 100644 --- a/tasks_gen/eval/G0157/task.json +++ b/tasks_gen/eval/G0157/task.json @@ -6,8 +6,8 @@ "release_target": "v740sp05", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0158/empirical.json b/tasks_gen/eval/G0158/empirical.json index 73e4db9..9545e66 100644 --- a/tasks_gen/eval/G0158/empirical.json +++ b/tasks_gen/eval/G0158/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 85.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 10.0, - "total": 85.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "11/11", - "tool_calls": 11, - "seconds": 52.5, - "run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 11, + "seconds": 52.5, + "run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0158/task.json b/tasks_gen/eval/G0158/task.json index 9a0f2c6..4f6e29b 100644 --- a/tasks_gen/eval/G0158/task.json +++ b/tasks_gen/eval/G0158/task.json @@ -6,8 +6,8 @@ "release_target": "v702", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0159/empirical.json b/tasks_gen/eval/G0159/empirical.json index 83dd8cb..7a02534 100644 --- a/tasks_gen/eval/G0159/empirical.json +++ b/tasks_gen/eval/G0159/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 85.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 10.0, - "total": 85.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "10/10", - "tool_calls": 11, - "seconds": 36.2, - "run_dir": "runs/emp/11029_G0159_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 11, + "seconds": 36.2, + "run_dir": "runs/emp/11029_G0159_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0159/task.json b/tasks_gen/eval/G0159/task.json index 0ee62d3..9dc45fd 100644 --- a/tasks_gen/eval/G0159/task.json +++ b/tasks_gen/eval/G0159/task.json @@ -6,8 +6,8 @@ "release_target": "v740sp05", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0160/empirical.json b/tasks_gen/eval/G0160/empirical.json new file mode 100644 index 0000000..0d149d5 --- /dev/null +++ b/tasks_gen/eval/G0160/empirical.json @@ -0,0 +1,35 @@ +{ + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 0, + "seconds": 479.5, + "run_dir": "runs/emp/13004_G0160_llm_deepseek-v4.1-flash_cloud" + } + }, + "deepseek-v4.1-flash:cloud": { + "score": 90.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 90.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "6/6", + "tool_calls": 60, + "seconds": 506.3, + "run_dir": "runs/emp/16005_G0160_llm_deepseek-v4.1-flash_cloud" + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0161/empirical.json b/tasks_gen/eval/G0161/empirical.json index 9e26dfe..26bed92 100644 --- a/tasks_gen/eval/G0161/empirical.json +++ b/tasks_gen/eval/G0161/empirical.json @@ -1 +1,36 @@ -{} \ No newline at end of file +{ + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 88.5, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 13.5, + "own_tests": 15.0, + "discipline": 0.0, + "total": 88.5, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 60, + "seconds": 264.3, + "run_dir": "runs/emp/13005_G0161_llm_deepseek-v4.1-flash_cloud" + } + }, + "deepseek-v4.1-flash:cloud": { + "score": 95.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 95.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 29, + "seconds": 313.1, + "run_dir": "runs/emp/16006_G0161_llm_deepseek-v4.1-flash_cloud" + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0162/empirical.json b/tasks_gen/eval/G0162/empirical.json index c04a82a..78993bc 100644 --- a/tasks_gen/eval/G0162/empirical.json +++ b/tasks_gen/eval/G0162/empirical.json @@ -1,17 +1,35 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 0, - "parts": { - "total": 0, - "failed_gates": [ - "G1_active", - "G2_contract", - "G3_hidden_runs" - ] + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 40, + "seconds": 259.5, + "run_dir": "runs/emp/12025_G0162_llm_deepseek-v4.1-flash_cloud" }, - "hidden": "0/0", - "tool_calls": 40, - "seconds": 259.5, - "run_dir": "runs/emp/12025_G0162_llm_deepseek-v4.1-flash_cloud" + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 90.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 90.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 40, + "seconds": 587.0, + "run_dir": "runs/emp/16015_G0162_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0162/task.json b/tasks_gen/eval/G0162/task.json index 37a6e14..380a420 100644 --- a/tasks_gen/eval/G0162/task.json +++ b/tasks_gen/eval/G0162/task.json @@ -6,8 +6,8 @@ "release_target": "v702", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0163/empirical.json b/tasks_gen/eval/G0163/empirical.json index 24ec9df..7deb32f 100644 --- a/tasks_gen/eval/G0163/empirical.json +++ b/tasks_gen/eval/G0163/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 15.0, - "discipline": 10.0, - "total": 100.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "11/11", - "tool_calls": 17, - "seconds": 397.6, - "run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 17, + "seconds": 397.6, + "run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0163/task.json b/tasks_gen/eval/G0163/task.json index d958868..4ea324c 100644 --- a/tasks_gen/eval/G0163/task.json +++ b/tasks_gen/eval/G0163/task.json @@ -6,8 +6,8 @@ "release_target": "v702", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0164/empirical.json b/tasks_gen/eval/G0164/empirical.json index f70e441..6aa4980 100644 --- a/tasks_gen/eval/G0164/empirical.json +++ b/tasks_gen/eval/G0164/empirical.json @@ -1,18 +1,36 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 75.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 0.0, - "total": 75.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 75.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 0.0, + "total": 75.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "12/12", + "tool_calls": 13, + "seconds": 1043.6, + "run_dir": "runs/emp/12023_G0164_llm_deepseek-v4.1-flash_cloud" }, - "hidden": "12/12", - "tool_calls": 13, - "seconds": 1043.6, - "run_dir": "runs/emp/12023_G0164_llm_deepseek-v4.1-flash_cloud" + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "12/12", + "tool_calls": 19, + "seconds": 424.7, + "run_dir": "runs/emp/16007_G0164_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0164/task.json b/tasks_gen/eval/G0164/task.json index 667dc79..4273bd3 100644 --- a/tasks_gen/eval/G0164/task.json +++ b/tasks_gen/eval/G0164/task.json @@ -6,8 +6,8 @@ "release_target": "v740sp05", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0165/empirical.json b/tasks_gen/eval/G0165/empirical.json index b8cc975..f3c618a 100644 --- a/tasks_gen/eval/G0165/empirical.json +++ b/tasks_gen/eval/G0165/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 85.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 10.0, - "total": 85.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "6/6", - "tool_calls": 24, - "seconds": 112.4, - "run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "6/6", + "tool_calls": 24, + "seconds": 112.4, + "run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0165/task.json b/tasks_gen/eval/G0165/task.json index fbc7741..99046e9 100644 --- a/tasks_gen/eval/G0165/task.json +++ b/tasks_gen/eval/G0165/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0166/empirical.json b/tasks_gen/eval/G0166/empirical.json index 5eb52d8..1ad7a6a 100644 --- a/tasks_gen/eval/G0166/empirical.json +++ b/tasks_gen/eval/G0166/empirical.json @@ -1,18 +1,36 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 80.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 0.0, - "discipline": 5.0, - "total": 80.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 26, + "seconds": 646.5, + "run_dir": "runs/emp/12021_G0166_llm_deepseek-v4.1-flash_cloud" }, - "hidden": "8/8", - "tool_calls": 26, - "seconds": 646.5, - "run_dir": "runs/emp/12021_G0166_llm_deepseek-v4.1-flash_cloud" + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 29, + "seconds": 93.3, + "run_dir": "runs/emp/16008_G0166_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0166/task.json b/tasks_gen/eval/G0166/task.json index 138d8d9..bbf2966 100644 --- a/tasks_gen/eval/G0166/task.json +++ b/tasks_gen/eval/G0166/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "implement", "budget": { - "max_tool_calls": 30, - "max_activations": 8 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0167/empirical.json b/tasks_gen/eval/G0167/empirical.json index 035e198..a0ceca0 100644 --- a/tasks_gen/eval/G0167/empirical.json +++ b/tasks_gen/eval/G0167/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 15.0, - "discipline": 10.0, - "total": 100.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "11/11", - "tool_calls": 9, - "seconds": 30.3, - "run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 9, + "seconds": 30.3, + "run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0167/task.json b/tasks_gen/eval/G0167/task.json index 710471c..d9bb782 100644 --- a/tasks_gen/eval/G0167/task.json +++ b/tasks_gen/eval/G0167/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0169/empirical.json b/tasks_gen/eval/G0169/empirical.json index 5a4895c..c6e7144 100644 --- a/tasks_gen/eval/G0169/empirical.json +++ b/tasks_gen/eval/G0169/empirical.json @@ -1,16 +1,35 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 30, - "parts": { - "total": 30, - "stop": "stopped", - "gap": { - "how": "empty report" - } + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 30, + "parts": { + "total": 30, + "stop": "stopped", + "gap": { + "how": "empty report" + } + }, + "hidden": "0/0", + "tool_calls": 0, + "seconds": 629.3, + "run_dir": "runs/emp/12019_G0169_llm_deepseek-v4.1-flash_cloud" }, - "hidden": "0/0", - "tool_calls": 0, - "seconds": 629.3, - "run_dir": "runs/emp/12019_G0169_llm_deepseek-v4.1-flash_cloud" + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "judge", + "model": "deepseek-v4.1-flash:cloud", + "names_gap": true, + "reason": "The report explicitly identifies the missing precedence rule for members who are both students and under 25, matching the expected gap." + } + }, + "hidden": "0/0", + "tool_calls": 0, + "seconds": 7.3, + "run_dir": "runs/emp/16009_G0169_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0169/task.json b/tasks_gen/eval/G0169/task.json index 0fe919b..4f1e25d 100644 --- a/tasks_gen/eval/G0169/task.json +++ b/tasks_gen/eval/G0169/task.json @@ -27,8 +27,8 @@ ] }, "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [], "contract": [ diff --git a/tasks_gen/eval/G0170/empirical.json b/tasks_gen/eval/G0170/empirical.json index fcec2a7..84d7335 100644 --- a/tasks_gen/eval/G0170/empirical.json +++ b/tasks_gen/eval/G0170/empirical.json @@ -1,16 +1,18 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 0, - "parts": { - "total": 0, - "stop": "implemented", - "objects": [ - "Z099U04Q_STUDIO_PRICER" - ] - }, - "hidden": "0/0", - "tool_calls": 9, - "seconds": 73.6, - "run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 0, + "parts": { + "total": 0, + "stop": "implemented", + "objects": [ + "Z099U04Q_STUDIO_PRICER" + ] + }, + "hidden": "0/0", + "tool_calls": 9, + "seconds": 73.6, + "run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0170/task.json b/tasks_gen/eval/G0170/task.json index cb9ceb6..bab791a 100644 --- a/tasks_gen/eval/G0170/task.json +++ b/tasks_gen/eval/G0170/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "stop", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0171/empirical.json b/tasks_gen/eval/G0171/empirical.json index aa6535a..f64bb3c 100644 --- a/tasks_gen/eval/G0171/empirical.json +++ b/tasks_gen/eval/G0171/empirical.json @@ -1,19 +1,21 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100, - "parts": { - "total": 100, - "stop": "stopped", - "gap": { - "how": "judge", - "model": "deepseek-v4.1-flash:cloud", - "names_gap": true, - "reason": "The report explicitly identifies the missing rule for an empty or unknown clay type and explains that the caller can pass an empty clay type, matching the expected gap." - } - }, - "hidden": "0/0", - "tool_calls": 2, - "seconds": 140.7, - "run_dir": "runs/emp/12017_G0171_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "judge", + "model": "deepseek-v4.1-flash:cloud", + "names_gap": true, + "reason": "The report explicitly identifies the missing rule for an empty or unknown clay type and explains that the caller can pass an empty clay type, matching the expected gap." + } + }, + "hidden": "0/0", + "tool_calls": 2, + "seconds": 140.7, + "run_dir": "runs/emp/12017_G0171_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0171/task.json b/tasks_gen/eval/G0171/task.json index c2dc115..4d976a1 100644 --- a/tasks_gen/eval/G0171/task.json +++ b/tasks_gen/eval/G0171/task.json @@ -27,8 +27,8 @@ ] }, "budget": { - "max_tool_calls": 30, - "max_activations": 5 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0172/empirical.json b/tasks_gen/eval/G0172/empirical.json index 9e26dfe..6aac882 100644 --- a/tasks_gen/eval/G0172/empirical.json +++ b/tasks_gen/eval/G0172/empirical.json @@ -1 +1,36 @@ -{} \ No newline at end of file +{ + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "keywords", + "group": [ + [ + "unknown", + "code" + ], + [ + "other", + "code" + ], + [ + "not", + "defined" + ], + [ + "no", + "rule" + ] + ] + } + }, + "hidden": "0/0", + "tool_calls": 4, + "seconds": 8.2, + "run_dir": "runs/emp/13006_G0172_llm_deepseek-v4.1-flash_cloud" + } + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0172/task.json b/tasks_gen/eval/G0172/task.json index 26447c8..99965ff 100644 --- a/tasks_gen/eval/G0172/task.json +++ b/tasks_gen/eval/G0172/task.json @@ -27,8 +27,8 @@ ] }, "budget": { - "max_tool_calls": 25, - "max_activations": 6 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0173/empirical.json b/tasks_gen/eval/G0173/empirical.json index c61c363..1d532d7 100644 --- a/tasks_gen/eval/G0173/empirical.json +++ b/tasks_gen/eval/G0173/empirical.json @@ -1,30 +1,32 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100, - "parts": { - "total": 100, - "stop": "stopped", - "gap": { - "how": "keywords", - "group": [ - [ - "18", - "15" - ], - [ - "minimum", - "20" - ], - [ - "contradict", - "rule" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "keywords", + "group": [ + [ + "18", + "15" + ], + [ + "minimum", + "20" + ], + [ + "contradict", + "rule" + ] ] - ] - } - }, - "hidden": "0/0", - "tool_calls": 9, - "seconds": 66.6, - "run_dir": "runs/emp/12015_G0173_llm_deepseek-v4.1-flash_cloud" + } + }, + "hidden": "0/0", + "tool_calls": 9, + "seconds": 66.6, + "run_dir": "runs/emp/12015_G0173_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0173/task.json b/tasks_gen/eval/G0173/task.json index a928bab..64b7f0a 100644 --- a/tasks_gen/eval/G0173/task.json +++ b/tasks_gen/eval/G0173/task.json @@ -23,8 +23,8 @@ ] }, "budget": { - "max_tool_calls": 30, - "max_activations": 6 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0174/empirical.json b/tasks_gen/eval/G0174/empirical.json index 30caba7..1ebfa79 100644 --- a/tasks_gen/eval/G0174/empirical.json +++ b/tasks_gen/eval/G0174/empirical.json @@ -13,6 +13,27 @@ "tool_calls": 12, "seconds": 665.1, "run_dir": "runs/emp/12014_G0174_llm_deepseek-v4.1-flash_cloud" + }, + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "keywords", + "group": [ + [ + "small", + "sale", + "defin" + ] + ] + } + }, + "hidden": "0/0", + "tool_calls": 16, + "seconds": 78.1, + "run_dir": "runs/emp/16010_G0174_llm_deepseek-v4.1-flash_cloud" } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0174/task.json b/tasks_gen/eval/G0174/task.json index 0ac1825..7dfb281 100644 --- a/tasks_gen/eval/G0174/task.json +++ b/tasks_gen/eval/G0174/task.json @@ -16,8 +16,8 @@ ] }, "budget": { - "max_tool_calls": 20, - "max_activations": 5 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0175/empirical.json b/tasks_gen/eval/G0175/empirical.json index 0e8872e..206bca6 100644 --- a/tasks_gen/eval/G0175/empirical.json +++ b/tasks_gen/eval/G0175/empirical.json @@ -1,19 +1,21 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100, - "parts": { - "total": 100, - "stop": "stopped", - "gap": { - "how": "judge", - "model": "deepseek-v4.1-flash:cloud", - "names_gap": true, - "reason": "The report explicitly identifies the contradiction between rule 1 requiring HARVEST_DATE >= P_FROM and rule 5 requiring HARVEST_DATE < P_FROM, making both conditions unsatisfiable together." - } - }, - "hidden": "0/0", - "tool_calls": 1, - "seconds": 7.9, - "run_dir": "runs/emp/12013_G0175_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "judge", + "model": "deepseek-v4.1-flash:cloud", + "names_gap": true, + "reason": "The report explicitly identifies the contradiction between rule 1 requiring HARVEST_DATE >= P_FROM and rule 5 requiring HARVEST_DATE < P_FROM, making both conditions unsatisfiable together." + } + }, + "hidden": "0/0", + "tool_calls": 1, + "seconds": 7.9, + "run_dir": "runs/emp/12013_G0175_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0175/task.json b/tasks_gen/eval/G0175/task.json index b2983f1..262bfb2 100644 --- a/tasks_gen/eval/G0175/task.json +++ b/tasks_gen/eval/G0175/task.json @@ -25,8 +25,8 @@ ] }, "budget": { - "max_tool_calls": 30, - "max_activations": 6 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0176/empirical.json b/tasks_gen/eval/G0176/empirical.json index e9dab75..57f8ec1 100644 --- a/tasks_gen/eval/G0176/empirical.json +++ b/tasks_gen/eval/G0176/empirical.json @@ -1,16 +1,32 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 0, - "parts": { - "total": 0, - "stop": "implemented", - "objects": [ - "Z099O04W_I_STATION_ENERGY" - ] + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 0, + "parts": { + "total": 0, + "stop": "implemented", + "objects": [ + "Z099O04W_I_STATION_ENERGY" + ] + }, + "hidden": "0/0", + "tool_calls": 14, + "seconds": 534.6, + "run_dir": "runs/emp/12012_G0176_llm_deepseek-v4.1-flash_cloud" }, - "hidden": "0/0", - "tool_calls": 14, - "seconds": 534.6, - "run_dir": "runs/emp/12012_G0176_llm_deepseek-v4.1-flash_cloud" + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 0, + "parts": { + "total": 0, + "stop": "implemented", + "objects": [ + "Z0CCR04W_I_STATION_ENERGY" + ] + }, + "hidden": "0/0", + "tool_calls": 20, + "seconds": 102.9, + "run_dir": "runs/emp/16011_G0176_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0176/task.json b/tasks_gen/eval/G0176/task.json index 3b07f7d..bd9b960 100644 --- a/tasks_gen/eval/G0176/task.json +++ b/tasks_gen/eval/G0176/task.json @@ -27,8 +27,8 @@ ] }, "budget": { - "max_tool_calls": 20, - "max_activations": 5 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0177/empirical.json b/tasks_gen/eval/G0177/empirical.json index bc912e5..24d71e2 100644 --- a/tasks_gen/eval/G0177/empirical.json +++ b/tasks_gen/eval/G0177/empirical.json @@ -1,30 +1,32 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100, - "parts": { - "total": 100, - "stop": "stopped", - "gap": { - "how": "keywords", - "group": [ - [ - "contradiction", - "rules" - ], - [ - "contradict", - "rules" - ], - [ - "confirmed", - "cancelled" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "keywords", + "group": [ + [ + "contradiction", + "rules" + ], + [ + "contradict", + "rules" + ], + [ + "confirmed", + "cancelled" + ] ] - ] - } - }, - "hidden": "0/0", - "tool_calls": 6, - "seconds": 12.0, - "run_dir": "runs/emp/12011_G0177_llm_deepseek-v4.1-flash_cloud" + } + }, + "hidden": "0/0", + "tool_calls": 6, + "seconds": 12.0, + "run_dir": "runs/emp/12011_G0177_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0177/task.json b/tasks_gen/eval/G0177/task.json index c2796ac..2513d94 100644 --- a/tasks_gen/eval/G0177/task.json +++ b/tasks_gen/eval/G0177/task.json @@ -23,8 +23,8 @@ ] }, "budget": { - "max_tool_calls": 20, - "max_activations": 5 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0185/empirical.json b/tasks_gen/eval/G0185/empirical.json index e107434..ac307af 100644 --- a/tasks_gen/eval/G0185/empirical.json +++ b/tasks_gen/eval/G0185/empirical.json @@ -1,18 +1,36 @@ { + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 50.9, + "parts": { + "correctness": 10.9, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 5.0, + "total": 50.9, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "3/11", + "tool_calls": 17, + "seconds": 589.5, + "run_dir": "runs/emp/12004_G0185_llm_deepseek-v4.1-flash_cloud" + } + }, "deepseek-v4.1-flash:cloud": { - "score": 50.9, + "score": 85.0, "parts": { - "correctness": 10.9, + "correctness": 40.0, "craft": 20.0, "clean": 15.0, "own_tests": 0.0, - "discipline": 5.0, - "total": 50.9, + "discipline": 10.0, + "total": 85.0, "note": "own_tests: faulty-reference part not implemented in skeleton" }, - "hidden": "3/11", - "tool_calls": 17, - "seconds": 589.5, - "run_dir": "runs/emp/12004_G0185_llm_deepseek-v4.1-flash_cloud" + "hidden": "11/11", + "tool_calls": 20, + "seconds": 55.8, + "run_dir": "runs/emp/16012_G0185_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0189/empirical.json b/tasks_gen/eval/G0189/empirical.json new file mode 100644 index 0000000..2eec1b0 --- /dev/null +++ b/tasks_gen/eval/G0189/empirical.json @@ -0,0 +1,18 @@ +{ + "deepseek-v4.1-flash:cloud": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 15, + "seconds": 35.9, + "run_dir": "runs/emp/13007_G0189_llm_deepseek-v4.1-flash_cloud" + } +} \ No newline at end of file diff --git a/tasks_gen/eval/G0190/empirical.json b/tasks_gen/eval/G0190/empirical.json index ffa6445..cb9796a 100644 --- a/tasks_gen/eval/G0190/empirical.json +++ b/tasks_gen/eval/G0190/empirical.json @@ -1,18 +1,36 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 90.0, - "parts": { - "correctness": 40.0, - "craft": 15.0, - "clean": 15.0, - "own_tests": 15.0, - "discipline": 5.0, - "total": 90.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" + "_superseded": { + "deepseek-v4.1-flash:cloud@2026-10-03": { + "score": 90.0, + "parts": { + "correctness": 40.0, + "craft": 15.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 90.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "7/7", + "tool_calls": 19, + "seconds": 852.8, + "run_dir": "runs/emp/12001_G0190_llm_deepseek-v4.1-flash_cloud" }, - "hidden": "7/7", - "tool_calls": 19, - "seconds": 852.8, - "run_dir": "runs/emp/12001_G0190_llm_deepseek-v4.1-flash_cloud" + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 10.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "7/7", + "tool_calls": 22, + "seconds": 376.9, + "run_dir": "runs/emp/16013_G0190_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0190/task.json b/tasks_gen/eval/G0190/task.json index 3881332..aef4f05 100644 --- a/tasks_gen/eval/G0190/task.json +++ b/tasks_gen/eval/G0190/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ { diff --git a/tasks_gen/eval/G0191/empirical.json b/tasks_gen/eval/G0191/empirical.json index beb41ee..6917036 100644 --- a/tasks_gen/eval/G0191/empirical.json +++ b/tasks_gen/eval/G0191/empirical.json @@ -1,18 +1,20 @@ { - "deepseek-v4.1-flash:cloud": { - "score": 100.0, - "parts": { - "correctness": 40.0, - "craft": 20.0, - "clean": 15.0, - "own_tests": 15.0, - "discipline": 10.0, - "total": 100.0, - "note": "own_tests: faulty-reference part not implemented in skeleton" - }, - "hidden": "12/12", - "tool_calls": 8, - "seconds": 87.0, - "run_dir": "runs/emp/12000_G0191_llm_deepseek-v4.1-flash_cloud" + "_superseded": { + "deepseek-v4.1-flash:cloud@budget<60": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "12/12", + "tool_calls": 8, + "seconds": 87.0, + "run_dir": "runs/emp/12000_G0191_llm_deepseek-v4.1-flash_cloud" + } } } \ No newline at end of file diff --git a/tasks_gen/eval/G0191/task.json b/tasks_gen/eval/G0191/task.json index e0bae88..8d4a773 100644 --- a/tasks_gen/eval/G0191/task.json +++ b/tasks_gen/eval/G0191/task.json @@ -6,8 +6,8 @@ "release_target": "v816", "expected_outcome": "implement", "budget": { - "max_tool_calls": 40, - "max_activations": 10 + "max_tool_calls": 60, + "max_activations": 15 }, "seed": [ {