erhan/abap-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overfit conversion test passed (Mac -98.5 %, GPU -99.97 %); full run started, alpha 32

Browse Source 
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Kral
2026-10-04 19:34:24 +02:00
parent 20ef7b2331
commit 0c0982f666
2 changed files with 41 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
train/hf_train.py
View File

@@ -19,6 +19,10 @@ ap.add_argument("--epochs", type=float, default=2.0)
ap.add_argument("--rank", type=int, default=16)
ap.add_argument("--alpha", type=int, default=16)
ap.add_argument("--lr", type=float, default=5e-5)
ap.add_argument("--train-file", default="train.jsonl")
ap.add_argument("--eval-file", default="valid.jsonl")
ap.add_argument("--warmup-steps", type=int, default=30)
ap.add_argument("--save-every", type=int, default=0, help="upload the adapter to <out>/step<N> every N steps")
ap.add_argument("--max-seq-length", type=int, default=16384)
a = ap.parse_args()
@@ -35,26 +39,45 @@ model = FastLanguageModel.get_peft_model(
"in_proj_qkv", "in_proj_z", "out_proj"],
use_gradient_checkpointing="unsloth", random_state=20261003)
ds = load_dataset(a.data, data_files={"train": "train.jsonl", "valid": "valid.jsonl"}, token=tok_hf)
ds = load_dataset(a.data, data_files={"train": a.train_file, "valid": a.eval_file}, token=tok_hf)
cfg = SFTConfig(
output_dir="out", per_device_train_batch_size=1, per_device_eval_batch_size=1,
gradient_accumulation_steps=1, num_train_epochs=a.epochs, max_steps=a.max_steps,
learning_rate=a.lr, lr_scheduler_type="cosine", warmup_steps=min(30, max(1, a.max_steps // 3)) if a.max_steps > 0 else 30,
learning_rate=a.lr, lr_scheduler_type="cosine", warmup_steps=a.warmup_steps,
optim="adamw_8bit", weight_decay=0.0, logging_steps=1, eval_strategy="no", save_strategy="no",
max_length=a.max_seq_length, dataset_text_field="text", packing=False, seed=20261003, report_to="none")
tr = SFTTrainer(model=model, processing_class=tok, train_dataset=ds["train"], eval_dataset=ds["valid"], args=cfg)
from transformers import TrainerCallback # noqa: E402
from huggingface_hub import HfApi # noqa: E402
api = HfApi(token=tok_hf)
class SaveCb(TrainerCallback):
def on_step_end(self, args, state, control, **kw):
n = state.global_step
if a.save_every and n % a.save_every == 0:
model.save_pretrained(f"ckpt{n}")
api.upload_folder(folder_path=f"ckpt{n}", repo_id=a.out, repo_type="model", path_in_repo=f"step{n}")
ev = tr.evaluate()
print("EVAL", json.dumps({"step": n, "eval_loss": ev.get("eval_loss")}), flush=True)
api.upload_file(path_or_fileobj=json.dumps({"step": n, "eval_loss": ev.get("eval_loss")}).encode(),
path_in_repo=f"step{n}/eval.json", repo_id=a.out, repo_type="model")
tr.add_callback(SaveCb())
ev0 = tr.evaluate()
print("EVAL", json.dumps({"step": 0, "eval_loss": ev0.get("eval_loss")}), flush=True)
t0 = time.time()
tr.train()
train_s = time.time() - t0
steps = tr.state.global_step
ev = tr.evaluate() # valid loss of the adapter, for the conversion check
info = {"steps": steps, "train_seconds": round(train_s, 1), "sec_per_step": round(train_s / max(steps, 1), 2),
info = {"eval_loss_step0": ev0.get("eval_loss"), "steps": steps, "train_seconds": round(train_s, 1), "sec_per_step": round(train_s / max(steps, 1), 2),
"eval_loss": ev.get("eval_loss"), "rank": a.rank, "alpha": a.alpha, "lr": a.lr,
"peak_gpu_gb": round(__import__("torch").cuda.max_memory_allocated() / 2**30, 1)}
print("RESULT", json.dumps(info))
model.save_pretrained("adapter")
json.dump(info, open("adapter/job_result.json", "w"))
from huggingface_hub import HfApi # noqa: E402
HfApi(token=tok_hf).upload_folder(folder_path="adapter", repo_id=a.out, repo_type="model")
api.upload_folder(folder_path="adapter", repo_id=a.out, repo_type="model")
print("PUSHED", a.out)