erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K variants for training (free text, EPOD tool names), second attempt only for failed tasks, docs/epod-syntax-hint.md

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-05 12:50:43 +02:00
parent 35f0eb0e0c
commit 9de3911578
4 changed files with 154 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
harness/trainset.py
View File

@@ -16,7 +16,7 @@ import time
from .adt_client import load_env
from .evalset import SLOTS, RELEASES, accepted_goals
from .generator import ROOT, generate
from .generator import ROOT, generate, make_k_variant
from .ledger import BudgetExceeded, spent
from . import overlap
@@ -140,10 +140,58 @@ def run(part, parts, target, stop_ledger):
print(json.dumps(log), flush=True)
K_FIRST_ID = 1300
K_RUN_BASE = 41800 # 20 per variant; above the trajectory run numbers (41000-41700)
K_COUNT = 18 # K share of the eval plan: 10 of 110 (9 %); counted inside the 200 accepted tasks
K_STYLES_CYCLE = ["free_text", "incomplete"]
def run_k(count):
"""K tasks for training: free-text or incomplete spec of an accepted training task, EPOD tool names
(no generic_v0). Same reference and hidden tests as the source task."""
base_url = os.environ.get("LLM_BASE_URL", "http://127.0.0.1:11434/v1")
for n in range(count):
new_id = f"G{K_FIRST_ID + n}"
if os.path.exists(os.path.join(POOL, "_logs", new_id + ".json")):
continue
used = {json.load(open(f)).get("base_task") for f in glob.glob(os.path.join(POOL, "_logs", "G13*.json"))}
cands = [] # accepted, not K, not H (a stop task has no free-text form), not used yet
for f in sorted(glob.glob(os.path.join(POOL, "_logs", "G1[0-2]*.json"))):
lg = json.load(open(f))
if lg.get("accepted") and lg.get("category") not in ("H", "K") and lg["id"] not in used:
cands.append(lg)
if not cands:
print("no source task left", flush=True)
return
kinds = {}
for lg in cands: # spread over object types: take the type with the fewest K variants so far
kinds.setdefault(lg["object_type"], []).append(lg)
done_types = [json.load(open(f)).get("object_type") for f in glob.glob(os.path.join(POOL, "_logs", "G13*.json"))]
otype = min(kinds, key=lambda t: done_types.count(t))
src = kinds[otype][0]
style = K_STYLES_CYCLE[n % 2]
if spent() >= json.load(open(PLAN))["ledger_at_start"] + json.load(open(PLAN))["stop_ledger"]:
print("PHASE LIMIT", flush=True)
return
try:
log = make_k_variant(src["id"], new_id, style, "deepseek-v4.1-flash:cloud", base_url,
K_RUN_BASE + 20 * n, tool_schema=None, pool="train")
except BudgetExceeded as e:
print("BUDGET", e, flush=True)
return
log.update(object_type=otype, spent_total=spent())
stray = os.path.join(POOL, "generation.json")
if os.path.exists(stray):
os.remove(stray)
os.makedirs(os.path.join(POOL, "_logs"), exist_ok=True)
json.dump(log, open(os.path.join(POOL, "_logs", new_id + ".json"), "w"), indent=1)
print(json.dumps(log), flush=True)
def main():
load_env(os.path.join(ROOT, ".env"))
ap = argparse.ArgumentParser()
ap.add_argument("cmd", choices=["plan", "run"])
ap.add_argument("cmd", choices=["plan", "run", "k"])
ap.add_argument("--part", type=int, default=0)
ap.add_argument("--parts", type=int, default=1)
ap.add_argument("--target", type=int, default=200)
@@ -155,6 +203,9 @@ def main():
print(len(p), "slots", collections.Counter(x["category"] for x in p))
print(collections.Counter(x["object_type"] for x in p), collections.Counter(x.get("error_kind") for x in p))
return
if a.cmd == "k":
run_k(K_COUNT)
return
run(a.part, a.parts, a.target, a.stop_ledger)