From 5e3baf43d3b3fab2d61cce588198b8aa9e42ab79 Mon Sep 17 00:00:00 2001 From: Kral Date: Sat, 3 Oct 2026 18:54:16 +0200 Subject: [PATCH] Empirical filter results: DeepSeek reruns W6A/W7A (partial) --- tasks_gen/eval/G0125/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0156/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0158/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0162/empirical.json | 15 +++++++++++++++ tasks_gen/eval/G0163/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0164/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0165/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0167/empirical.json | 16 ++++++++++++++++ tasks_gen/eval/G0170/empirical.json | 14 ++++++++++++++ tasks_gen/eval/G0172/empirical.json | 17 +++++++++++++++++ tasks_gen/eval/G0174/empirical.json | 21 +++++++++++++++++++++ tasks_gen/eval/G0176/empirical.json | 15 +++++++++++++++ tasks_gen/eval/G0190/empirical.json | 16 ++++++++++++++++ 13 files changed, 210 insertions(+) diff --git a/tasks_gen/eval/G0125/empirical.json b/tasks_gen/eval/G0125/empirical.json index 853350a..4a765df 100644 --- a/tasks_gen/eval/G0125/empirical.json +++ b/tasks_gen/eval/G0125/empirical.json @@ -15,5 +15,21 @@ "seconds": 786.2, "run_dir": "runs/emp/13000_G0125_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 80.0, + "parts": { + "correctness": 40.0, + "craft": 10.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 80.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "8/8", + "tool_calls": 47, + "seconds": 489.0, + "run_dir": "runs/emp/15014_G0125_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0156/empirical.json b/tasks_gen/eval/G0156/empirical.json index 44fba3d..f141884 100644 --- a/tasks_gen/eval/G0156/empirical.json +++ b/tasks_gen/eval/G0156/empirical.json @@ -16,5 +16,21 @@ "seconds": 56.1, "run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "10/10", + "tool_calls": 12, + "seconds": 56.0, + "run_dir": "runs/emp/18000_G0156_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0158/empirical.json b/tasks_gen/eval/G0158/empirical.json index 9545e66..262f345 100644 --- a/tasks_gen/eval/G0158/empirical.json +++ b/tasks_gen/eval/G0158/empirical.json @@ -16,5 +16,21 @@ "seconds": 52.5, "run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 0.0, + "discipline": 10.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 14, + "seconds": 58.9, + "run_dir": "runs/emp/18001_G0158_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0162/empirical.json b/tasks_gen/eval/G0162/empirical.json index 78993bc..2590691 100644 --- a/tasks_gen/eval/G0162/empirical.json +++ b/tasks_gen/eval/G0162/empirical.json @@ -31,5 +31,20 @@ "seconds": 587.0, "run_dir": "runs/emp/16015_G0162_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 0, + "parts": { + "total": 0, + "failed_gates": [ + "G1_active", + "G2_contract", + "G3_hidden_runs" + ] + }, + "hidden": "0/0", + "tool_calls": 36, + "seconds": 510.6, + "run_dir": "runs/emp/18002_G0162_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0163/empirical.json b/tasks_gen/eval/G0163/empirical.json index 7deb32f..ea42375 100644 --- a/tasks_gen/eval/G0163/empirical.json +++ b/tasks_gen/eval/G0163/empirical.json @@ -16,5 +16,21 @@ "seconds": 397.6, "run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 14, + "seconds": 196.7, + "run_dir": "runs/emp/17003_G0163_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0164/empirical.json b/tasks_gen/eval/G0164/empirical.json index 6aa4980..6921c19 100644 --- a/tasks_gen/eval/G0164/empirical.json +++ b/tasks_gen/eval/G0164/empirical.json @@ -32,5 +32,21 @@ "seconds": 424.7, "run_dir": "runs/emp/16007_G0164_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "12/12", + "tool_calls": 13, + "seconds": 199.0, + "run_dir": "runs/emp/18003_G0164_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0165/empirical.json b/tasks_gen/eval/G0165/empirical.json index f3c618a..00d2dd8 100644 --- a/tasks_gen/eval/G0165/empirical.json +++ b/tasks_gen/eval/G0165/empirical.json @@ -16,5 +16,21 @@ "seconds": 112.4, "run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 90.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 0.0, + "total": 90.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "6/6", + "tool_calls": 49, + "seconds": 199.6, + "run_dir": "runs/emp/17004_G0165_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0167/empirical.json b/tasks_gen/eval/G0167/empirical.json index a0ceca0..d0ea88b 100644 --- a/tasks_gen/eval/G0167/empirical.json +++ b/tasks_gen/eval/G0167/empirical.json @@ -16,5 +16,21 @@ "seconds": 30.3, "run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 100.0, + "parts": { + "correctness": 40.0, + "craft": 20.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 10.0, + "total": 100.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "11/11", + "tool_calls": 6, + "seconds": 26.6, + "run_dir": "runs/emp/17005_G0167_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0170/empirical.json b/tasks_gen/eval/G0170/empirical.json index 84d7335..018a6d9 100644 --- a/tasks_gen/eval/G0170/empirical.json +++ b/tasks_gen/eval/G0170/empirical.json @@ -14,5 +14,19 @@ "seconds": 73.6, "run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 0, + "parts": { + "total": 0, + "stop": "implemented", + "objects": [ + "Z0D4E04Q_STUDIO_PRICER" + ] + }, + "hidden": "0/0", + "tool_calls": 11, + "seconds": 80.3, + "run_dir": "runs/emp/17006_G0170_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0172/empirical.json b/tasks_gen/eval/G0172/empirical.json index 6aac882..0fcfc84 100644 --- a/tasks_gen/eval/G0172/empirical.json +++ b/tasks_gen/eval/G0172/empirical.json @@ -32,5 +32,22 @@ "seconds": 8.2, "run_dir": "runs/emp/13006_G0172_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "judge", + "model": "deepseek-v4.1-flash:cloud", + "names_gap": true, + "reason": "The report explicitly identifies the blocking gap that the business rules define behavior only for size codes 'S', 'M', 'L', and 'XL' but leave undefined what the function module must do for any other size code." + } + }, + "hidden": "0/0", + "tool_calls": 6, + "seconds": 13.2, + "run_dir": "runs/emp/17007_G0172_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0174/empirical.json b/tasks_gen/eval/G0174/empirical.json index 1ebfa79..f93ac45 100644 --- a/tasks_gen/eval/G0174/empirical.json +++ b/tasks_gen/eval/G0174/empirical.json @@ -35,5 +35,26 @@ "seconds": 78.1, "run_dir": "runs/emp/16010_G0174_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 100, + "parts": { + "total": 100, + "stop": "stopped", + "gap": { + "how": "keywords", + "group": [ + [ + "small", + "sale", + "defin" + ] + ] + } + }, + "hidden": "0/0", + "tool_calls": 17, + "seconds": 24.3, + "run_dir": "runs/emp/17008_G0174_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0176/empirical.json b/tasks_gen/eval/G0176/empirical.json index 57f8ec1..ffb15dc 100644 --- a/tasks_gen/eval/G0176/empirical.json +++ b/tasks_gen/eval/G0176/empirical.json @@ -28,5 +28,20 @@ "seconds": 102.9, "run_dir": "runs/emp/16011_G0176_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 0, + "parts": { + "total": 0, + "stop": "implemented", + "objects": [ + "Z0D4H04W_CL_STATION_ENERGY_TES", + "Z0D4H04W_I_STATION_ENERGY" + ] + }, + "hidden": "0/0", + "tool_calls": 42, + "seconds": 271.1, + "run_dir": "runs/emp/17009_G0176_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file diff --git a/tasks_gen/eval/G0190/empirical.json b/tasks_gen/eval/G0190/empirical.json index cb9796a..0856c69 100644 --- a/tasks_gen/eval/G0190/empirical.json +++ b/tasks_gen/eval/G0190/empirical.json @@ -32,5 +32,21 @@ "seconds": 376.9, "run_dir": "runs/emp/16013_G0190_llm_deepseek-v4.1-flash_cloud" } + }, + "deepseek-v4.1-flash:cloud": { + "score": 85.0, + "parts": { + "correctness": 40.0, + "craft": 10.0, + "clean": 15.0, + "own_tests": 15.0, + "discipline": 5.0, + "total": 85.0, + "note": "own_tests: faulty-reference part not implemented in skeleton" + }, + "hidden": "7/7", + "tool_calls": 17, + "seconds": 276.9, + "run_dir": "runs/emp/17010_G0190_llm_deepseek-v4.1-flash_cloud" } } \ No newline at end of file