8d1db9c67c
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014aUaQeLnwbb1zTpN7kHeat
37 lines
1.6 KiB
Bash
Executable File
37 lines
1.6 KiB
Bash
Executable File
#!/bin/sh
|
|
# Baseline on the 11-task subset, Devstral Small 2 (2026-10-04), detached.
|
|
# Stop rule: after the first 4 tasks, if all 4 ended with "loop" and the repair rate is below 20 %, stop the run.
|
|
# Log: runs/stage1/baseline.log, chain log runs/stage1/baseline_chain.log
|
|
cd "$(dirname "$0")/.."
|
|
log() { echo "$(date '+%F %T') $*"; }
|
|
log "start baseline, 11 tasks, no thinking mode (Devstral), max_tokens 16384, loop guard 3"
|
|
python3 train/baseline.py --label baseline_devstral --run-base 21000 > runs/stage1/baseline.log 2>&1 &
|
|
BP=$!
|
|
log "baseline PID $BP"
|
|
CHECKED=0
|
|
while kill -0 $BP 2>/dev/null; do
|
|
if [ "$CHECKED" = 0 ]; then
|
|
R=$(python3 - <<'PY' 2>/dev/null
|
|
import json
|
|
t = json.load(open("runs/stage1/baseline_devstral.json"))["tasks"]
|
|
if len(t) < 4:
|
|
print("wait")
|
|
else:
|
|
f = list(t.values())[:4]
|
|
after = sum(x.get("pushes_after_error") or 0 for x in f)
|
|
ch = sum(x.get("pushes_changed_after_error") or 0 for x in f)
|
|
rate = ch / after if after else 0
|
|
loops = sum(x.get("end_reason") == "loop" for x in f)
|
|
print("stop" if loops == 4 and rate < 0.2 else "go", loops, round(rate, 3))
|
|
PY
|
|
)
|
|
case "$R" in
|
|
stop*) log "STOP RULE: $R"; kill $BP; osascript -e 'display notification "Stop rule: 4 loops, repair rate below 20 %" with title "stage1"'; exit 0 ;;
|
|
go*) log "first 4 tasks checked: $R"; CHECKED=1 ;;
|
|
esac
|
|
fi
|
|
sleep 60
|
|
done
|
|
log "baseline ended: $(python3 -c "import json;print(len(json.load(open('runs/stage1/baseline_devstral.json'))['tasks']))") tasks"
|
|
osascript -e 'display notification "Baseline Devstral (11 tasks) ended" with title "stage1"'
|