From fb18dc8d9db5795da835fdbc7c5b133a36f781e0 Mon Sep 17 00:00:00 2001 From: Kral Date: Sat, 3 Oct 2026 18:56:12 +0200 Subject: [PATCH] Review: 9 random high-score checks; easy candidates list --- tasks_gen/eval/G0100/review.json | 2 +- tasks_gen/eval/G0101/review.json | 2 +- tasks_gen/eval/G0111/review.json | 2 +- tasks_gen/eval/G0114/review.json | 2 +- tasks_gen/eval/G0133/review.json | 2 +- tasks_gen/eval/G0136/review.json | 2 +- tasks_gen/eval/G0146/review.json | 2 +- tasks_gen/eval/G0149/review.json | 2 +- tasks_gen/eval/G0164/review.json | 2 +- tasks_gen/eval/easy_candidates.json | 45 +++++++++++++++++++++++++++++ 10 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 tasks_gen/eval/easy_candidates.json diff --git a/tasks_gen/eval/G0100/review.json b/tasks_gen/eval/G0100/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0100/review.json +++ b/tasks_gen/eval/G0100/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0101/review.json b/tasks_gen/eval/G0101/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0101/review.json +++ b/tasks_gen/eval/G0101/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0111/review.json b/tasks_gen/eval/G0111/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0111/review.json +++ b/tasks_gen/eval/G0111/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0114/review.json b/tasks_gen/eval/G0114/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0114/review.json +++ b/tasks_gen/eval/G0114/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0133/review.json b/tasks_gen/eval/G0133/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0133/review.json +++ b/tasks_gen/eval/G0133/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0136/review.json b/tasks_gen/eval/G0136/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0136/review.json +++ b/tasks_gen/eval/G0136/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0146/review.json b/tasks_gen/eval/G0146/review.json index d24ee23..93f586e 100644 --- a/tasks_gen/eval/G0146/review.json +++ b/tasks_gen/eval/G0146/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Screened: rules map to tests, mutation ok, no craft hints. H: real gap. K: names kept.", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0149/review.json b/tasks_gen/eval/G0149/review.json index ebc3eea..93f586e 100644 --- a/tasks_gen/eval/G0149/review.json +++ b/tasks_gen/eval/G0149/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Survived mutant: round dec +1, equivalent (result field has fewer decimals).", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/G0164/review.json b/tasks_gen/eval/G0164/review.json index ebc3eea..93f586e 100644 --- a/tasks_gen/eval/G0164/review.json +++ b/tasks_gen/eval/G0164/review.json @@ -1,6 +1,6 @@ { "decision": "accept", - "note": "Survived mutant: round dec +1, equivalent (result field has fewer decimals).", + "note": "Random high-score re-check (2026-10-03): rules map to tests, no craft hints, no leak; mutation ok.", "by": "Claude", "date": "2026-10-03", "failed_checks": {} diff --git a/tasks_gen/eval/easy_candidates.json b/tasks_gen/eval/easy_candidates.json new file mode 100644 index 0000000..cfbcd8e --- /dev/null +++ b/tasks_gen/eval/easy_candidates.json @@ -0,0 +1,45 @@ +{ + "rule": "DeepSeek (budget 60) score >= 95 and tool_calls <= 20, outcome implement; candidate only until the Qwen baseline confirms", + "date": "2026-10-03", + "tasks": [ + "G0003", + "G0004", + "G0007", + "G0008", + "G0010", + "G0100", + "G0101", + "G0103", + "G0104", + "G0113", + "G0114", + "G0115", + "G0116", + "G0117", + "G0118", + "G0120", + "G0127", + "G0129", + "G0130", + "G0135", + "G0136", + "G0137", + "G0138", + "G0139", + "G0141", + "G0142", + "G0148", + "G0149", + "G0150", + "G0151", + "G0159", + "G0163", + "G0164", + "G0167", + "G0178", + "G0179", + "G0186", + "G0187", + "G0188" + ] +} \ No newline at end of file