erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K variants for training (free text, EPOD tool names), second attempt only for failed tasks, docs/epod-syntax-hint.md

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 12:50:43 +02:00
parent 35f0eb0e0c
commit 9de3911578
4 changed files with 154 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
harness/generator.py
View File

@@ -490,11 +490,12 @@ def contract_terms(task):
return out
def make_k_variant(src_id, new_id, style, model, base_url, run_base, tool_schema="generic_v0", max_repairs=2):
def make_k_variant(src_id, new_id, style, model, base_url, run_base, tool_schema="generic_v0", max_repairs=2,
pool="eval"):
"""Category K: free-text or incomplete input + other tool schema. Same reference and hidden tests
as the source task; only spec.md, category and tool_schema change."""
check_budget()
pool_root = os.path.join(ROOT, "tasks_gen", "eval")
pool_root = os.path.join(ROOT, "tasks_gen", pool)
src_dir, task_dir = os.path.join(pool_root, src_id), os.path.join(pool_root, new_id)
b = bundle_of(src_dir)
b["files"] = {k: v for k, v in b["files"].items()
@@ -507,7 +508,7 @@ def make_k_variant(src_id, new_id, style, model, base_url, run_base, tool_schema
"ALV). Keep these names exactly as written (the tests use them): "
f"{', '.join(terms)}. Keep the placeholder {{{{P}}}} in names. Use Simplified Technical English. "
f"Return only the new spec text.\n\nSpec:\n{spec}"}]
log = {"id": new_id, "pool": "eval", "category": "K", "base_task": src_id, "style": style,
log = {"id": new_id, "pool": pool, "category": "K", "base_task": src_id, "style": style,
"tool_schema": tool_schema, "attempts": []}
for attempt in range(max_repairs + 1):
text = chat(model, messages, base_url).strip()
@@ -520,7 +521,9 @@ def make_k_variant(src_id, new_id, style, model, base_url, run_base, tool_schema
+ ". Return the full spec again with all names."})
continue
b["files"]["spec.md"] = text + "\n"
b["task"].update(category="K", base_task=src_id, input_style=style, tool_schema=tool_schema)
b["task"].update(category="K", base_task=src_id, input_style=style)
if tool_schema: # None: EPOD tool names (training: free-text input only)
b["task"]["tool_schema"] = tool_schema
write_bundle(b, task_dir, new_id)
rep_o, rep_n = validate(pool_root, new_id, run_base + RUNS_PER_ATTEMPT * attempt)
so, sn = (rep_o.get("score") or {}).get("total"), (rep_n.get("score") or {}).get("total")