erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Empirical filter results: DeepSeek reruns W6A/W7A (partial)

Browse Source
This commit is contained in:
Kral
2026-10-03 18:54:16 +02:00
parent d83a39a8d8
commit 5e3baf43d3
13 changed files with 210 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
tasks_gen/eval/G0125/empirical.json
View File

@@ -15,5 +15,21 @@
"seconds": 786.2, "seconds": 786.2,
"run_dir": "runs/emp/13000_G0125_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/13000_G0125_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 80.0,
"parts": {
"correctness": 40.0,
"craft": 10.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 80.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "8/8",
"tool_calls": 47,
"seconds": 489.0,
"run_dir": "runs/emp/15014_G0125_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0156/empirical.json
View File

@@ -16,5 +16,21 @@
"seconds": 56.1, "seconds": 56.1,
"run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/11026_G0156_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "10/10",
"tool_calls": 12,
"seconds": 56.0,
"run_dir": "runs/emp/18000_G0156_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0158/empirical.json
View File

@@ -16,5 +16,21 @@
"seconds": 52.5, "seconds": 52.5,
"run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/11028_G0158_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 0.0,
"discipline": 10.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 14,
"seconds": 58.9,
"run_dir": "runs/emp/18001_G0158_llm_deepseek-v4.1-flash_cloud"
} }
} }

15
tasks_gen/eval/G0162/empirical.json
View File

@@ -31,5 +31,20 @@
"seconds": 587.0, "seconds": 587.0,
"run_dir": "runs/emp/16015_G0162_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16015_G0162_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"failed_gates": [
"G1_active",
"G2_contract",
"G3_hidden_runs"
]
},
"hidden": "0/0",
"tool_calls": 36,
"seconds": 510.6,
"run_dir": "runs/emp/18002_G0162_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0163/empirical.json
View File

@@ -16,5 +16,21 @@
"seconds": 397.6, "seconds": 397.6,
"run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/12024_G0163_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 14,
"seconds": 196.7,
"run_dir": "runs/emp/17003_G0163_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0164/empirical.json
View File

@@ -32,5 +32,21 @@
"seconds": 424.7, "seconds": 424.7,
"run_dir": "runs/emp/16007_G0164_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16007_G0164_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "12/12",
"tool_calls": 13,
"seconds": 199.0,
"run_dir": "runs/emp/18003_G0164_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0165/empirical.json
View File

@@ -16,5 +16,21 @@
"seconds": 112.4, "seconds": 112.4,
"run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/12022_G0165_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 90.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 0.0,
"total": 90.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "6/6",
"tool_calls": 49,
"seconds": 199.6,
"run_dir": "runs/emp/17004_G0165_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0167/empirical.json
View File

@@ -16,5 +16,21 @@
"seconds": 30.3, "seconds": 30.3,
"run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/12020_G0167_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 100.0,
"parts": {
"correctness": 40.0,
"craft": 20.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 10.0,
"total": 100.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "11/11",
"tool_calls": 6,
"seconds": 26.6,
"run_dir": "runs/emp/17005_G0167_llm_deepseek-v4.1-flash_cloud"
} }
} }

14
tasks_gen/eval/G0170/empirical.json
View File

@@ -14,5 +14,19 @@
"seconds": 73.6, "seconds": 73.6,
"run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/12018_G0170_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z0D4E04Q_STUDIO_PRICER"
]
},
"hidden": "0/0",
"tool_calls": 11,
"seconds": 80.3,
"run_dir": "runs/emp/17006_G0170_llm_deepseek-v4.1-flash_cloud"
} }
} }

17
tasks_gen/eval/G0172/empirical.json
View File

@@ -32,5 +32,22 @@
"seconds": 8.2, "seconds": 8.2,
"run_dir": "runs/emp/13006_G0172_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/13006_G0172_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "judge",
"model": "deepseek-v4.1-flash:cloud",
"names_gap": true,
"reason": "The report explicitly identifies the blocking gap that the business rules define behavior only for size codes 'S', 'M', 'L', and 'XL' but leave undefined what the function module must do for any other size code."
}
},
"hidden": "0/0",
"tool_calls": 6,
"seconds": 13.2,
"run_dir": "runs/emp/17007_G0172_llm_deepseek-v4.1-flash_cloud"
} }
} }

21
tasks_gen/eval/G0174/empirical.json
View File

@@ -35,5 +35,26 @@
"seconds": 78.1, "seconds": 78.1,
"run_dir": "runs/emp/16010_G0174_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16010_G0174_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 100,
"parts": {
"total": 100,
"stop": "stopped",
"gap": {
"how": "keywords",
"group": [
[
"small",
"sale",
"defin"
]
]
}
},
"hidden": "0/0",
"tool_calls": 17,
"seconds": 24.3,
"run_dir": "runs/emp/17008_G0174_llm_deepseek-v4.1-flash_cloud"
} }
} }

15
tasks_gen/eval/G0176/empirical.json
View File

@@ -28,5 +28,20 @@
"seconds": 102.9, "seconds": 102.9,
"run_dir": "runs/emp/16011_G0176_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16011_G0176_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 0,
"parts": {
"total": 0,
"stop": "implemented",
"objects": [
"Z0D4H04W_CL_STATION_ENERGY_TES",
"Z0D4H04W_I_STATION_ENERGY"
]
},
"hidden": "0/0",
"tool_calls": 42,
"seconds": 271.1,
"run_dir": "runs/emp/17009_G0176_llm_deepseek-v4.1-flash_cloud"
} }
} }

16
tasks_gen/eval/G0190/empirical.json
View File

@@ -32,5 +32,21 @@
"seconds": 376.9, "seconds": 376.9,
"run_dir": "runs/emp/16013_G0190_llm_deepseek-v4.1-flash_cloud" "run_dir": "runs/emp/16013_G0190_llm_deepseek-v4.1-flash_cloud"
} }
},
"deepseek-v4.1-flash:cloud": {
"score": 85.0,
"parts": {
"correctness": 40.0,
"craft": 10.0,
"clean": 15.0,
"own_tests": 15.0,
"discipline": 5.0,
"total": 85.0,
"note": "own_tests: faulty-reference part not implemented in skeleton"
},
"hidden": "7/7",
"tool_calls": 17,
"seconds": 276.9,
"run_dir": "runs/emp/17010_G0190_llm_deepseek-v4.1-flash_cloud"
} }
} }