erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage 1 data: strict version dedup (older v* vs next newer v*); counts updated

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
This commit is contained in:
Kral
2026-10-03 22:23:04 +02:00
parent 0677d035da
commit 7b8ca01bde
5 changed files with 179 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
train/prepare.py
View File

@@ -3,8 +3,8 @@
train/.venv/bin/python train/prepare.py [--corpus PATH] [--max-len 16384] [--seed 20261003]
1. Real token counts (tokenizer of the base model).
2. Version dedup: in each family keep the newest version; keep an older version only if its source differs
from the newest by more than 5 % of lines (difflib).
2. Version dedup (strict): keep main (Cloud) and the newest v* (Standard ABAP); compare each older v* with the
next newer v*, keep it only if more than 5 % of lines differ (difflib).
3. Documents longer than max_len are split with the real tokenizer: classes at ENDMETHOD boundaries, markdown
at "##" headings (fallbacks: "###", then lines). Header lines are repeated in each piece.
4. Split 95/5 by family (all versions and pieces of one family stay in one split).
@@ -201,29 +201,49 @@ def main():
removed_versions, kept_versions, strict_extra = [], 0, 0
keep_ids = set()
n_multi = 0
for f, vers in byfam.items():
order = sorted(vers, key=lambda v: -RANK.get(v, 40))
newest = order[0]
for d in vers[newest]:
other_cmp = 0
def keep(v_docs):
for d in v_docs:
keep_ids.add(d["_id"])
kept_versions += 1
if len(order) > 1:
for f, vers in byfam.items():
# Strict rule (Kral 2026-10-03): main (ABAP Cloud) and the newest v* (Standard ABAP) are always kept.
# Each older v* is compared with the next newer v* (not with main) and kept only if > diff of lines differ.
# Other branches (oo_patterns, rap, unit_tests) are compared with main as before.
vs = sorted((v for v in vers if re.fullmatch(r"v\d+", v)), key=lambda v: -int(v[1:]))
others = [v for v in vers if v != "main" and v not in vs]
if len(vers) > 1:
n_multi += 1
base = [x for d in vers[newest] for x in norm_lines(d["text"])]
kept_lines = [base]
for v in order[1:]:
lines = [x for d in vers[v] for x in norm_lines(d["text"])]
frac = diff_fraction(base, lines)
lines_of = lambda v: [x for d in vers[v] for x in norm_lines(d["text"])]
if "main" in vers:
keep(vers["main"])
kept_versions += 1
if vs:
keep(vers[vs[0]])
kept_versions += 1
for i in range(1, len(vs)):
newer = vs[i - 1]
frac = diff_fraction(lines_of(newer), lines_of(vs[i]))
if frac > a.diff:
keep(vers[vs[i]])
kept_versions += 1
for d in vers[v]:
keep_ids.add(d["_id"])
# info only: would a stricter rule (compare with all kept versions) remove it?
if min(diff_fraction(k, lines) for k in kept_lines) <= a.diff:
strict_extra += 1
kept_lines.append(lines)
else:
removed_versions.append({"object": f, "version": v, "newest": newest, "diff": round(frac, 3),
removed_versions.append({"object": f, "version": vs[i], "newest": newer, "diff": round(frac, 3),
"tokens": sum(d["real_tokens"] for d in vers[vs[i]])})
for v in others:
ref = "main" if "main" in vers else (vs[0] if vs else None)
if ref is None:
keep(vers[v])
kept_versions += 1
continue
other_cmp += 1
frac = diff_fraction(lines_of(ref), lines_of(v))
if frac > a.diff:
keep(vers[v])
kept_versions += 1
else:
removed_versions.append({"object": f, "version": v, "newest": ref, "diff": round(frac, 3),
"tokens": sum(d["real_tokens"] for d in vers[v])})
after_dedup = [d for d in docs if d["_id"] in keep_ids]
real1 = sum(d["real_tokens"] for d in after_dedup)
@@ -282,7 +302,7 @@ def main():
"records": len(docs), "tokens_before_dedup": real0, "tokens_after_dedup": real1,
"records_after_dedup": len(after_dedup), "tokens_after_split": real2,
"objects_with_versions": n_multi, "families": len(byfam), "versions_kept": kept_versions,
"versions_removed": len(removed_versions), "strict_rule_would_remove_more": strict_extra,
"versions_removed": len(removed_versions), "other_branch_comparisons": other_cmp,
"removed_versions": removed_versions, "split_docs": len(split_info), "hard_line_splits": hard_total,
"pieces_over_limit": len(over), "train": summ(train), "valid": summ(valid), "all": summ(kept),
"share_before_dedup": share([{"types": d["types"], "tokens": d["real_tokens"]} for d in docs]),
@@ -293,13 +313,13 @@ def main():
L = ["# Stage 1 data report", "",
f"Corpus: `{a.corpus}` (SAP-samples/abap-cheat-sheets, Apache-2.0, see corpus/out/ATTRIBUTION.md).",
f"Tokenizer: `{a.model}`. Seed {a.seed}. max_seq_length {a.max_len}.", "",
"## Version dedup (keep the newest; keep an older version only if it differs by more than "
"## Version dedup (strict rule; an older version is kept only if it differs by more than "
f"{int(a.diff * 100)} % of lines)", "",
f"- Object families: {len(byfam)}; with more than one version: {n_multi}.",
f"- Versions kept: {kept_versions}. Versions removed: {len(removed_versions)}.",
f"- Records: {len(docs)} before, {len(after_dedup)} after.",
f"- Tokens before: {real0} (chars/4 estimate {est}). After dedup: {real1}. After splitting: {real2}.",
f"- Info: a stricter rule (compare also with the other kept versions) would remove {strict_extra} more kept versions.",
f"- Rule: main (Cloud) and the newest v* (Standard ABAP) always kept; each older v* compared with the next newer v*; other branches ({other_cmp} comparisons) compared with main.",
"", "## Splitting of documents over the limit", "",
f"- Documents split: {len(split_info)}; pieces: {sum(s['pieces'] for s in split_info)}; hard line splits "
f"(a block was too big): {hard_total}; pieces still over the limit: {len(over)}.", "",