diff --git a/.env.example b/.env.example index 6a9fd6d9..ab3952f7 100644 --- a/.env.example +++ b/.env.example @@ -35,11 +35,10 @@ MAX_NEW_TOKENS=512 # x-api-key: TEACHER_SERVER_API_KEY= -# Weights & Biases — experiment tracking for the state-classifier trainers -# (scripts/train_state_classifier_34way*.py, run with --wandb). Grab the key -# from https://wandb.ai/authorize. The training launch sources .env, so the key -# is picked up automatically for online logging; leave blank to log offline. +# Weights & Biases — experiment tracking for SFT trainers (train_sft.py) and +# state-classifier trainers (train_state_classifier_34way*.py, run with --wandb). +# Get your key at https://wandb.ai/authorize, then run `wandb login` or set here. WANDB_API_KEY= -# Optional overrides (defaults: project=csen346-state-classifier, run=out-dir basename) -# WANDB_PROJECT=csen346-state-classifier -# WANDB_RUN_NAME= +# Optional overrides (training scripts set sensible defaults if omitted) +# WANDB_PROJECT=csen346-sft +# WANDB_RUN_NAME=gemma4-31b-stage2 diff --git a/.gitignore b/.gitignore index 2e48b8eb..0019dab8 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ references/KELE-EN/.cache/ # Railguard .railguard/* +# W&B local run artifacts +wandb/ # macOS .DS_Store **/.DS_Store diff --git a/Makefile b/Makefile index 3dda9cf8..fb878fad 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,11 @@ .PHONY: help run install-hooks pre-commit sync-mirror setup setup-repo \ online-demo local-demo _demo-preflight \ + nvidia-preflight _classifier-ckpt \ + train-gemma4-12b train-gemma4-12b-dry-run monitor-train-gemma4-12b \ + serve-gemma4-12b serve-gemma4-12b-mtp serve-gemma4-12b-sft \ + eval-gemma4-12b-base-smoke eval-gemma4-12b-base-full \ + eval-gemma4-12b-sft-smoke eval-gemma4-12b-sft-full \ + monitor-eval-gemma4-12b-base monitor-eval-gemma4-12b-sft \ slurm \ post-eval-shutdown run-eval \ eval-qwen27b-smoke eval-qwen27b-mini eval-qwen27b-full \ @@ -20,6 +26,9 @@ download-gemma4-31b \ prequant-gemma4-31b-l40s transfer-gemma4-31b-nf4 \ train-gemma4-31b-dry-run train-gemma4-31b-stage2 train-gemma4-31b-stage2-preq \ + train-gemma4-31b-stage2-unsloth train-gemma4-31b-eos-gate eos-gate-gemma4-31b \ + gpu-preflight diagnose-gfx1201-fault \ + profile-gemma4-31b \ tournament tournament-think tournament-warmup tournament-warmup-think tournament-status tournament-eliminate \ tournament-finalize tournament-archive tournament-restore tournament-reset \ tournament-download tournament-help @@ -68,19 +77,25 @@ help: @echo " local-demo Local demo via llama.cpp on this machine (downloads classifier if missing)" @echo " serve-demo Self-host the top-performer stack for a live demo (RTX 5090 + Tailscale)" @echo " start-local-tl-server Start local llama.cpp server for dataset translation (Qwen3.5-9B)" - @echo " eval-qwen27b-smoke Run scripts/eval_qwen27b.sh smoke (n=5, ~5 min)" - @echo " eval-qwen27b-mini Run scripts/eval_qwen27b.sh mini (n=25, ~15 min)" - @echo " eval-qwen27b-full Run scripts/eval_qwen27b.sh full (n=681, ~75 h — measured)" - @echo " eval-qwen35b-a3b-smoke Run scripts/eval_qwen35b_a3b.sh smoke (n=5, ~2 min projected)" - @echo " eval-qwen35b-a3b-mini Run scripts/eval_qwen35b_a3b.sh mini (n=25, ~5 min projected)" - @echo " eval-qwen35b-a3b-full Run scripts/eval_qwen35b_a3b.sh full (n=681, ~20-30 h projected)" + @echo " eval-qwen27b-smoke Run scripts/eval_llamacpp.sh qwen27b smoke (n=5, ~5 min)" + @echo " eval-qwen27b-mini Run scripts/eval_llamacpp.sh qwen27b mini (n=25, ~15 min)" + @echo " eval-qwen27b-full Run scripts/eval_llamacpp.sh qwen27b full (n=681, ~75 h — measured)" + @echo " eval-qwen35b-a3b-smoke Run scripts/eval_llamacpp.sh qwen35b-a3b smoke (n=5, ~2 min projected)" + @echo " eval-qwen35b-a3b-mini Run scripts/eval_llamacpp.sh qwen35b-a3b mini (n=25, ~5 min projected)" + @echo " eval-qwen35b-a3b-full Run scripts/eval_llamacpp.sh qwen35b-a3b full (n=681, ~20-30 h projected)" @echo " download-gemma4-31b Download google/gemma-4-31b-it weights to HF cache (~60 GB)" @echo " prequant-gemma4-31b-l40s Print instructions for pre-quantizing to NF4 on L40S" @echo " transfer-gemma4-31b-nf4 rsync NF4 checkpoint from L40S (HOST=user@host)" - @echo " train-gemma4-31b-stage2-preq Train Stage 2b from pre-quantized NF4 checkpoint" - @echo " eval-gemma4-31b-smoke Run scripts/eval_gemma4_31b.sh smoke (n=5)" - @echo " eval-gemma4-31b-mini Run scripts/eval_gemma4_31b.sh mini (n=25)" - @echo " eval-gemma4-31b-full Run scripts/eval_gemma4_31b.sh full (n=681)" + @echo " train-gemma4-31b-stage2-preq Train Stage 2b from local pre-quantized NF4 checkpoint" + @echo " train-gemma4-31b-stage2-unsloth Train Stage 2b from unsloth bnb-4bit Gemma 4 31B (no local prequant)" + @echo " train-gemma4-31b-eos-gate 100-step checkpoint for EOS gate (unsloth path, ~30 min)" + @echo " eos-gate-gemma4-31b Run EOS gate against outputs/eos-gate-gemma4-31b/final" + @echo " gpu-preflight Fast GPU gate (clean KFD + fwd/bwd) — run before any (re)launch" + @echo " diagnose-gfx1201-fault Serialized-kernel run to localize the backward page fault" + @echo " profile-gemma4-31b Profile a real Stage 2 step (attention vs NF4-dequant; FA2 de-risk)" + @echo " eval-gemma4-31b-smoke Run scripts/eval_llamacpp.sh gemma4-31b smoke (n=5)" + @echo " eval-gemma4-31b-mini Run scripts/eval_llamacpp.sh gemma4-31b mini (n=25)" + @echo " eval-gemma4-31b-full Run scripts/eval_llamacpp.sh gemma4-31b full (n=681)" @echo "" @echo " Fusion smoke targets (single-call architecture, see SOCRATIC_FUSION_PLAN.md):" @echo " eval-qwen27b-fusion-smoke 27B + unified (think on)" @@ -320,13 +335,13 @@ test-vllm: bash scripts/test_vllm_rocm.sh eval-qwen27b-smoke: - bash scripts/eval_qwen27b.sh smoke + bash scripts/eval_llamacpp.sh qwen27b smoke eval-qwen27b-mini: - bash scripts/eval_qwen27b.sh mini + bash scripts/eval_llamacpp.sh qwen27b mini eval-qwen27b-full: - bash scripts/eval_qwen27b.sh full + bash scripts/eval_llamacpp.sh qwen27b full serve-qwen35b-a3b: bash scripts/serve_qwen35b_a3b.sh @@ -338,47 +353,174 @@ serve-qwopus35b-a3b: bash scripts/serve_qwopus35b_a3b.sh eval-qwen35b-a3b-smoke: - bash scripts/eval_qwen35b_a3b.sh smoke + bash scripts/eval_llamacpp.sh qwen35b-a3b smoke eval-qwen35b-a3b-mini: - bash scripts/eval_qwen35b_a3b.sh mini + bash scripts/eval_llamacpp.sh qwen35b-a3b mini eval-qwen35b-a3b-full: - bash scripts/eval_qwen35b_a3b.sh full + bash scripts/eval_llamacpp.sh qwen35b-a3b full eval-gemma4-31b-smoke: - bash scripts/eval_gemma4_31b.sh smoke + bash scripts/eval_llamacpp.sh gemma4-31b smoke eval-gemma4-31b-mini: - bash scripts/eval_gemma4_31b.sh mini + bash scripts/eval_llamacpp.sh gemma4-31b mini eval-gemma4-31b-full: - bash scripts/eval_gemma4_31b.sh full + bash scripts/eval_llamacpp.sh gemma4-31b full # ── Fusion smoke targets (single-call architecture) ────────────────────────── # See docs/SOCRATIC_FUSION_PLAN.md. Each writes to a distinct results/ dir # so all four can coexist alongside the existing two-call smoke results. eval-qwen27b-fusion-smoke: - bash scripts/eval_qwen27b.sh smoke --unified + bash scripts/eval_llamacpp.sh qwen27b smoke --unified eval-qwen27b-fusion-nothink-smoke: - bash scripts/eval_qwen27b.sh smoke --unified --nothink + bash scripts/eval_llamacpp.sh qwen27b smoke --unified --nothink eval-qwen35b-a3b-fusion-smoke: - bash scripts/eval_qwen35b_a3b.sh smoke --unified + bash scripts/eval_llamacpp.sh qwen35b-a3b smoke --unified eval-qwen35b-a3b-fusion-nothink-smoke: - bash scripts/eval_qwen35b_a3b.sh smoke --unified --nothink + bash scripts/eval_llamacpp.sh qwen35b-a3b smoke --unified --nothink # Gemma 4 has no thinking-mode equivalent, so only the --unified variant exists. eval-gemma4-31b-fusion-smoke: - bash scripts/eval_gemma4_31b.sh smoke --unified + bash scripts/eval_llamacpp.sh gemma4-31b smoke --unified + +# ── Gemma 4 12B SFT-uplift PoC (NVIDIA RTX 4000 Ada) ───────────────────────── +# Baseline (base 12B teacher) vs 1-epoch Socratic QLoRA SFT, same Qwen3.5-LoRA +# state classifier as consultant. NVIDIA box — none of the gfx1201/ROCm env that +# the 31B targets carry. See the 12B PoC plan + configs/train-sft-gemma4-12b-qlora.env. +# Serving and eval are separate steps: the 20 GB card hosts ONE model at a time, so +# the eval targets assume the matching server (serve-gemma4-12b{,-sft}) is already up. + +# CUDA fwd/bwd smoke through the training kernel paths — the NVIDIA analogue of +# gpu-preflight (which is the ROCm/gfx1201 gate). Hard gate before train/serve. +nvidia-preflight: + uv run --no-sync python scripts/test_training_gpu_nvidia.py + +train-gemma4-12b-dry-run: + uv run python scripts/train_sft.py --config configs/train-sft-gemma4-12b-qlora.env --dry-run + +# Live-quantize the bf16 base to NF4 on load (TRAIN_PREQ=false → the code builds +# the bnb 4-bit config). unsloth/gemma-4-12b-it is the FULL bf16 model, not a +# bnb-4bit checkpoint, so TRAIN_PREQ=true loaded it in bf16 (~24 GB) and OOM'd on +# the 20 GB card; live quant is ~7.7 GB resident, leaving ~13 GB for activations. +# expandable_segments avoids allocator fragmentation at the VRAM edge (CUDA box). +train-gemma4-12b: nvidia-preflight + mkdir -p outputs/sft-gemma4-12b-qlora + nohup env TRAIN_BASE_MODEL=unsloth/gemma-4-12b-it \ + TRAIN_PREQ=false \ + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + TRAIN_HF_REPO=ulises-c/SocratesLM-12B-QLoRA \ + TRAIN_HF_PUSH_EVERY=50 \ + uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-gemma4-12b-qlora.env \ + > outputs/sft-gemma4-12b-qlora/train.log 2>&1 & + @echo "Training started. Monitor: tail -f outputs/sft-gemma4-12b-qlora/train.log" + +# Auto-resume training crawl for the known-unstable box: owns the train→crash→ +# resume loop, archives each fault, quarantines partial checkpoints, walks the GPU +# power limit DOWN per crash (stability search), and logs to issue #130. Re-invokes +# `make train-gemma4-12b` (nvidia-preflight + auto-resume) on every fault. Set +# LOG_COMMENT_ID= to post rows to a pinned issue comment; unset = local log only. +# See scripts/monitor_train_gemma4_12b.sh. +monitor-train-gemma4-12b: + bash scripts/monitor_train_gemma4_12b.sh + +# Serve (port 8080, one at a time). serve-gemma4-12b-mtp attaches the MTP drafter. +serve-gemma4-12b: + bash scripts/serve_gemma4_12b.sh + +serve-gemma4-12b-mtp: + MTP=1 bash scripts/serve_gemma4_12b.sh + +serve-gemma4-12b-sft: + bash scripts/serve_gemma4_12b_sft.sh + +# Ensure the Qwen3.5-LoRA classifier checkpoint is present (consultant for both evals). +_classifier-ckpt: + @if [[ ! -f "$(BERT_CKPT)/model.safetensors" ]]; then \ + echo "Classifier checkpoint not found — downloading from HF…"; \ + hf download ulises-c/socrates-state-classifier-qwen3.5-lora --local-dir "$(BERT_CKPT)"; \ + fi + +# Eval: Qwen3.5-LoRA consultant (on CPU, frees VRAM for the teacher) + Gemma 12B +# teacher. smoke = n=5 sanity gate; full = n=681. Keep base/SFT invocations +# identical except the --experiment config so the delta isolates the SFT adapter. +eval-gemma4-12b-base-smoke: _classifier-ckpt + KELE_BERT_DEVICE=cpu uv run python -m src.project.kele --experiment gemma4-12b-local \ + test --n 5 --bert-consultant "$(BERT_CKPT)" --output results/gemma4-12b-base-smoke + +eval-gemma4-12b-base-full: _classifier-ckpt + WANDB_EVAL=1 KELE_BERT_DEVICE=cpu uv run python -m src.project.kele --experiment gemma4-12b-local \ + evaluate --bert-consultant "$(BERT_CKPT)" --output results/gemma4-12b-base + +eval-gemma4-12b-sft-smoke: _classifier-ckpt + KELE_BERT_DEVICE=cpu uv run python -m src.project.kele --experiment gemma4-12b-sft-local \ + test --n 5 --bert-consultant "$(BERT_CKPT)" --output results/gemma4-12b-sft-smoke + +eval-gemma4-12b-sft-full: _classifier-ckpt + WANDB_EVAL=1 KELE_BERT_DEVICE=cpu uv run python -m src.project.kele --experiment gemma4-12b-sft-local \ + evaluate --bert-consultant "$(BERT_CKPT)" --output results/gemma4-12b-sft + +# Monitored eval crawl for the known-unstable box: owns serve+eval, polls server +# health out-of-band, repairs error/truncated dialogues before relaunch, walks the +# GPU power limit down per crash, and logs to issue #130. MTP=1 attaches the +# drafter (-mtp output suffix). See scripts/monitor_eval_gemma4_12b.sh. +# make monitor-eval-gemma4-12b-base # base, MTP off → results/gemma4-12b-base +# MTP=1 make monitor-eval-gemma4-12b-base # base, MTP on → results/gemma4-12b-base-mtp +# make monitor-eval-gemma4-12b-sft # SFT → results/gemma4-12b-sft +monitor-eval-gemma4-12b-base: _classifier-ckpt + bash scripts/monitor_eval_gemma4_12b.sh base + +monitor-eval-gemma4-12b-sft: _classifier-ckpt + bash scripts/monitor_eval_gemma4_12b.sh sft + +# Consultant ablation (handoff T1.1): both no-consultant (self-consult) arms back to +# back, SFT then base, one model at a time. No _classifier-ckpt dep — self-consult +# uses no external classifier. Long-running; launch detached (nohup … &). +# nohup make noconsult-chain-gemma4-12b > outputs/noconsult_chain.nohup 2>&1 & +noconsult-chain-gemma4-12b: + bash scripts/noconsult_chain_gemma4_12b.sh + +# Oracle-consultant ablation (T1.1 follow-on): both oracle arms (GT state fed each +# turn), SFT then base, one model at a time. No _classifier-ckpt dep — oracle uses +# no external classifier. state_accuracy is ~perfect by construction → compare on +# ROUGE/BLEU. ~3 h (SFT) + ~17 h (base). Launch detached: +# nohup make oracle-chain-gemma4-12b > outputs/oracle_chain.nohup 2>&1 & +oracle-chain-gemma4-12b: + bash scripts/oracle_chain_gemma4_12b.sh # ── Gemma 4 31B SFT training (Stage 2b) ────────────────────────────────────── # No patch-fla-rocm needed — Gemma 4 uses standard softmax attention (no FLA). -# ROCm env vars (TORCH_USE_HIPBLASLT=0, garbage_collection_threshold:0.8) are -# gfx1201 workarounds that apply to all training targets. +# +# The gfx1201 page fault (Memory access fault / page not present, +# PERMISSION_FAULTS:0x3) during the QLoRA backward is NON-DETERMINISTIC and not +# yet attributed to any config knob: the SAME config (same git SHA) both finishes +# 100 steps and crashes at step 10 across repeated runs (wandb-verified — see PR +# #101 and docs/GFX1201_RDNA4_TRAINING.md §6.1). Four root-cause theories +# (workers, GC threshold, hipBLASLt, LR) were each published then falsified. +# Current knobs are PRECAUTIONARY, not proven fixes: +# TORCH_USE_HIPBLASLT=0 — rocBLAS fallback (HIPBLASLT=1 still crashed, +# so this is precaution, not the cure) +# PYTORCH_HIP_ALLOC_CONF=gc:0.8 — trims the backward peak; GC-off also crashed +# The real next step is `make diagnose-gfx1201-fault` (serialized kernels → names +# the faulting kernel) and the ablation matrix in §6.1 — NOT another knob flip. +# Every (re)launch is gated on `make gpu-preflight` (clean KFD + working fwd/bwd): +# a prior fault leaves the GPU dirty and the next run faults early on stale PTEs. + +gpu-preflight: + bash scripts/test_gpu_stack.sh --preflight + +# Localize the backward page fault: run ~120 steps with serialized kernel launches +# so the async VM fault becomes synchronous and the traceback/dmesg name the exact +# faulting kernel (bnb NF4 dequant vs grad-ckpt recompute vs allocator). +diagnose-gfx1201-fault: gpu-preflight + bash scripts/diagnose_gfx1201_fault.sh download-gemma4-31b: uv run hf download google/gemma-4-31b-it @@ -386,10 +528,10 @@ download-gemma4-31b: train-gemma4-31b-dry-run: uv run python scripts/train_sft.py --config configs/train-sft-gemma4-31b-qlora.env --dry-run -train-gemma4-31b-stage2: +train-gemma4-31b-stage2: gpu-preflight mkdir -p outputs/sft-stage2-gemma4-31b nohup env TORCH_USE_HIPBLASLT=0 \ - PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,expandable_segments:True \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ uv run --no-sync python scripts/train_sft.py \ --config configs/train-sft-stage2-gemma4-31b.env \ > outputs/sft-stage2-gemma4-31b/train.log 2>&1 & @@ -406,10 +548,10 @@ transfer-gemma4-31b-nf4: mkdir -p models/gemma-4-31b-nf4 rsync -avP "$(HOST):gemma-4-31b-nf4/" models/gemma-4-31b-nf4/ -train-gemma4-31b-stage2-preq: +train-gemma4-31b-stage2-preq: gpu-preflight mkdir -p outputs/sft-stage2-gemma4-31b nohup env TORCH_USE_HIPBLASLT=0 \ - PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,expandable_segments:True \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ TRAIN_BASE_MODEL=models/gemma-4-31b-nf4 \ TRAIN_PREQ=true \ uv run --no-sync python scripts/train_sft.py \ @@ -417,6 +559,66 @@ train-gemma4-31b-stage2-preq: > outputs/sft-stage2-gemma4-31b/train.log 2>&1 & @echo "Training started. Monitor: tail -f outputs/sft-stage2-gemma4-31b/train.log" +# Train from the community pre-quantized unsloth bnb-4bit checkpoint (~19 GB NF4). +# Skips the L40S prequant + rsync step entirely: weights download already 4-bit, +# so there is no ~62 GB BF16 CPU staging at load (the R9700's actual blocker). +# TRAIN_PREQ=true is required — it tells train_sft.py to read the embedded +# quantization_config instead of building a live BitsAndBytesConfig (which would +# double-quantize the already-quantized checkpoint). +train-gemma4-31b-stage2-unsloth: gpu-preflight + mkdir -p outputs/sft-stage2-gemma4-31b + nohup env TORCH_USE_HIPBLASLT=0 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + TRAIN_HF_REPO=ulises-c/SocratesLM-31B-stage2b-QLoRA \ + TRAIN_HF_PUSH_EVERY=50 \ + uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + > outputs/sft-stage2-gemma4-31b/train.log 2>&1 & + @echo "Training started. Monitor: tail -f outputs/sft-stage2-gemma4-31b/train.log" + +# Train 100 steps on the same unsloth path as the real run to produce an adapter +# for the EOS gate. Uses a separate output dir so it never collides with the real run. +train-gemma4-31b-eos-gate: + mkdir -p outputs/eos-gate-gemma4-31b + setsid env TORCH_USE_HIPBLASLT=0 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + TRAIN_MAX_STEPS=100 \ + TRAIN_SAVE_STEPS=100 \ + TRAIN_OUTPUT_DIR=outputs/eos-gate-gemma4-31b \ + uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + > outputs/eos-gate-gemma4-31b/train.log 2>&1 & + @echo "EOS-gate checkpoint training started (~30 min)." + @echo "Monitor: tail -f outputs/eos-gate-gemma4-31b/train.log" + @echo "When done, run: make eos-gate-gemma4-31b" + +eos-gate-gemma4-31b: + env TORCH_USE_HIPBLASLT=0 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + TRAIN_METHOD=qlora \ + TRAIN_BF16=true \ + uv run --no-sync python scripts/eos_gate.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + --adapter outputs/eos-gate-gemma4-31b/final + +# De-risk the FA2 bet: profile where the real Stage 2 step spends its ~70s. +# Mirrors the stage2-unsloth env so the profiled step matches the real run. +# Prints a kernel self-time table + attention-vs-gemm/dequant bucket summary. +# If attention dominates → patching the flash-attn Triton backward is worth it; +# if NF4 dequant/GEMM dominates → FA2 buys little. ~6 min (6 steps). +profile-gemma4-31b: + env TORCH_USE_HIPBLASLT=1 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + uv run --no-sync python scripts/profile_train_step.py \ + --config configs/train-sft-stage2-gemma4-31b.env + # ── Tournament ──────────────────────────────────────────────────────────────── tournament-help: diff --git a/configs/gemma4-12b-local.env b/configs/gemma4-12b-local.env new file mode 100644 index 00000000..2a2ed851 --- /dev/null +++ b/configs/gemma4-12b-local.env @@ -0,0 +1,30 @@ +# Experiment: Gemma 4 12B-it BASE teacher — fully local, dual-role on llama.cpp. +# +# Both teacher and consultant hit the same llama.cpp server on port 8080, mirroring +# configs/gemma4-31b-local.env. For the SFT-uplift PoC the eval RE-points the +# consultant role at the Qwen3.5-LoRA state classifier via `--bert-consultant +# ` — but CONSULTANT_* below must still be set, because config.py:load_config +# require()s them unconditionally at load time. +# +# Serve the base weights with: make serve-gemma4-12b (UD-Q8_K_XL) +# or, to A/B the MTP speculative drafter: make serve-gemma4-12b-mtp +# Eval: make eval-gemma4-12b-base-smoke / make eval-gemma4-12b-base-full +# +# Shares port 8080 with the other local llama.cpp configs — only one model fits at +# a time on the RTX 4000 Ada's 20 GB. The teacher alias "Gemma 4 12B" must match +# the serve wrapper's -a so the eval can't run against the wrong loaded weights. + +CONSULTANT_API_KEY=not-needed +CONSULTANT_BASE_URL=http://localhost:8080/v1 +CONSULTANT_MODEL_NAME=Gemma 4 12B +# 8192 leaves ample room for structured JSON output. Gemma 4 has no thinking-mode +# equivalent (Qwen-specific) so no CoT budget needed. +CONSULTANT_MAX_TOKENS=8192 + +TEACHER_API_KEY=not-needed +TEACHER_BASE_URL=http://localhost:8080/v1 +TEACHER_MODEL_NAME=Gemma 4 12B + +DEBUG_MODE=false +MAX_TEACHING_ROUNDS=8 +KELE_TEACHER_LANG=auto diff --git a/configs/gemma4-12b-sft-local.env b/configs/gemma4-12b-sft-local.env new file mode 100644 index 00000000..70bcc0c5 --- /dev/null +++ b/configs/gemma4-12b-sft-local.env @@ -0,0 +1,26 @@ +# Experiment: Gemma 4 12B-it SFT teacher (Socratic QLoRA, merged → Q6/Q8 GGUF). +# +# Same dual-role / fully-local pattern as gemma4-12b-local.env. The ONLY +# difference is the alias "Gemma 4 12B SFT" — distinct so the eval orchestrator +# refuses to run against the BASE weights by mistake (it requests this teacher +# name from the server's /v1/models and fails fast if absent). The Qwen3.5-LoRA +# consultant is held identical to the baseline run so the metric delta isolates +# the SFT adapter. +# +# Serve the merged SFT weights with: make serve-gemma4-12b-sft +# Eval: make eval-gemma4-12b-sft-smoke / make eval-gemma4-12b-sft-full +# +# Shares port 8080 — stop the base server before serving the SFT model. + +CONSULTANT_API_KEY=not-needed +CONSULTANT_BASE_URL=http://localhost:8080/v1 +CONSULTANT_MODEL_NAME=Gemma 4 12B SFT +CONSULTANT_MAX_TOKENS=8192 + +TEACHER_API_KEY=not-needed +TEACHER_BASE_URL=http://localhost:8080/v1 +TEACHER_MODEL_NAME=Gemma 4 12B SFT + +DEBUG_MODE=false +MAX_TEACHING_ROUNDS=8 +KELE_TEACHER_LANG=auto diff --git a/configs/train-sft-gemma4-12b-qlora.env b/configs/train-sft-gemma4-12b-qlora.env new file mode 100644 index 00000000..a42c2826 --- /dev/null +++ b/configs/train-sft-gemma4-12b-qlora.env @@ -0,0 +1,96 @@ +# SFT fine-tuning — Socratic teaching QLoRA on Gemma 4 12B-IT (PoC, 1 epoch) +# +# Target hardware: NVIDIA RTX 4000 Ada (20 GB), CUDA. This box is KNOWN-UNSTABLE +# (see memory: training-host-hardware-fault) — checkpoint frequently and rely on +# train_sft.py auto-resume rather than expecting a clean single run. +# Gemma 4 uses standard softmax attention (attn_implementation="sdpa" in +# train_sft.py) — no flash-linear-attention dependency, no patch-fla step, and +# none of the gfx1201/ROCm env (TORCH_USE_HIPBLASLT / PYTORCH_HIP_ALLOC_CONF) +# the 31B AMD config carries. +# +# This is the 12B PoC sibling of configs/train-sft-stage2-gemma4-31b.env. Goal: +# measure whether 1-epoch Socratic SFT gives eval uplift over the BASE Gemma 4 +# 12B teacher (consultant = Qwen3.5-LoRA state classifier, held fixed). +# +# Launch (recommended — train from the community bnb-4bit base, no BF16 staging): +# make train-gemma4-12b +# which overrides TRAIN_BASE_MODEL=unsloth/gemma-4-12b-it and TRAIN_PREQ=true. +# +# Dry run (no weights): make train-gemma4-12b-dry-run + +# ── Base model ──────────────────────────────────────────────────────────────── +# Default is the BF16 hub model (live-quant path). The make target overrides this +# to the unsloth bnb-4bit checkpoint with TRAIN_PREQ=true to skip BF16 CPU staging. +TRAIN_BASE_MODEL=google/gemma-4-12b-it + +# ── Training method ─────────────────────────────────────────────────────────── +TRAIN_METHOD=qlora + +# ── LoRA hyperparameters ────────────────────────────────────────────────────── +LORA_RANK=16 +LORA_ALPHA=32 +LORA_DROPOUT=0.05 +# google/gemma-4-12b-it is the same multimodal Gemma4ForConditionalGeneration +# family as the 31B (text decoder + vision_tower + audio_tower). The anchored +# regex (build_lora_config detects the leading ^) scopes LoRA to text-decoder +# nn.Linear paths only; the vision_tower's Gemma4ClippableLinear wrappers crash +# get_peft_model otherwise. It is layer-count-agnostic (\d+), so it ports from +# the 31B's 60 decoder layers to the 12B's 48 unchanged. +LORA_TARGET_MODULES=^model\.language_model\.layers\.\d+\.(self_attn|mlp)\.(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)$ + +# ── Training hyperparameters ───────────────────────────────────────────────── +# 1 epoch over the ~77k per-turn records (~4826 steps @ effective-batch 16) is +# ample SFT signal for a PoC. Bump to 3 only if the downstream eval shows the +# single epoch underfits. +# NB: train_sft.py auto-resumes from any checkpoint-* in TRAIN_OUTPUT_DIR built +# for the OLD schedule. If you re-run after changing TRAIN_EPOCHS, clear them: +# rm -rf outputs/sft-gemma4-12b-qlora/checkpoint-* +TRAIN_EPOCHS=1 +TRAIN_LR=5e-5 +TRAIN_BATCH_SIZE=1 +TRAIN_GRAD_ACCUM=16 +# Measured on socrat-zh-sft + socrat-en-sft (scripts/determine_max_sequence.py): +# real token max = 909 (train) / 892 (eval), p99 = 673. 1280 truncates NOTHING; +# with batch_size=1 + dynamic padding the activation peak is set by the actual +# longest sequence (909), not by this cap, so lowering it frees no VRAM. +TRAIN_MAX_SEQ_LEN=1280 +TRAIN_BF16=true +TRAIN_GRAD_CKPT=true +# One-time tokenization fan-out across CPU cores (procs exit before training). +TRAIN_DATASET_NUM_PROC=12 +# Step is GPU/dequant-bound; data is pre-tokenized; async prefetch buys nothing. +TRAIN_DATALOADER_WORKERS=0 + +# ── Dataset sources ─────────────────────────────────────────────────────────── +# Per-turn, state/action-conditioned format that matches the inference schema +# socrates_teacher() sends (历史对话记录 / 当前学生输入). Using socrat-*-sft +# (NOT plain socrat-zh,socrat-en) avoids confounding the SFT uplift measurement +# with train/serve schema drift. +TRAIN_SOURCES=socrat-zh-sft,socrat-en-sft + +# ── Output & logging ───────────────────────────────────────────────────────── +TRAIN_OUTPUT_DIR=outputs/sft-gemma4-12b-qlora +TRAIN_LOGGING_STEPS=10 +# Save every 50 steps so the auto-resume monitor can crawl the run forward across +# a crash on this known-unstable box instead of restarting at step 0. NOTE: this +# is NOT the AMD gfx1201 backward page fault (that is ROCm-specific and does not +# occur on CUDA) — the rationale here is purely the host hardware instability. +# Disk is bounded by save_total_limit (a couple of ~50 MB LoRA adapters). Drop to +# 10 only if a crash ever lands inside the 0–50 window and stalls. +TRAIN_SAVE_STEPS=50 +TRAIN_EVAL_STEPS=200 +# Disable in-training eval — the eval forward's logits.float() cast doubles the +# 256K-vocab logits tensor (transient multi-GB), which can OOM the 20 GB card. +# The downstream KELE eval pipeline produces the paper numbers anyway. Also +# forces load_best_model_at_end=False (which needs eval). +TRAIN_EVAL_STRATEGY=no + +# ── Weights & Biases ────────────────────────────────────────────────────────── +# Requires `wandb login` or WANDB_API_KEY; train_sft._check_wandb disables +# reporting if auth fails. Pinned run id + resume=allow so every crash-resume +# APPENDS to one run with a continuous step axis. Bump WANDB_RUN_ID for a +# brand-new crawl (else it resumes this run's step history). +WANDB_PROJECT=csen346-sft +WANDB_RUN_ID=gemma4-12b-qlora-poc +WANDB_RESUME=allow +WANDB_RUN_NAME=gemma4-12b-qlora-poc diff --git a/configs/train-sft-stage2-gemma4-31b.env b/configs/train-sft-stage2-gemma4-31b.env index 3ab9f476..67fc6ad0 100644 --- a/configs/train-sft-stage2-gemma4-31b.env +++ b/configs/train-sft-stage2-gemma4-31b.env @@ -18,7 +18,14 @@ # # 2a → 2b sequence: run 2a first, then point TRAIN_BASE_MODEL at the 2a # # checkpoint dir and re-run with the default TRAIN_SOURCES. # -# Launch (R9700): +# Launch (R9700) — recommended: skip the 62 GB BF16 CPU staging at load by +# training from the community pre-quantized unsloth bnb-4bit checkpoint: +# make train-gemma4-31b-stage2-unsloth +# which overrides TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit and +# TRAIN_PREQ=true. The default TRAIN_BASE_MODEL below is the live-quant path +# (make train-gemma4-31b-stage2), which stages the full BF16 model in CPU RAM. +# +# Launch (live quant, BF16 base): # mkdir -p outputs/sft-stage2-gemma4-31b # nohup env TORCH_USE_HIPBLASLT=0 PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ # uv run --no-sync python scripts/train_sft.py \ @@ -48,27 +55,58 @@ LORA_DROPOUT=0.05 LORA_TARGET_MODULES=^model\.language_model\.layers\.\d+\.(self_attn|mlp)\.(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)$ # ── Training hyperparameters ───────────────────────────────────────────────── -TRAIN_EPOCHS=3 +# Stage 2b is a proof-of-concept for the larger SFT plan; 1 epoch over the ~77k +# per-turn records is ample SFT signal and is a clean 3× wall-time cut +# (~281h → ~94h) against the gfx1201 ~70s/step ceiling. Bump back to 3 only if +# the downstream eval shows the single epoch underfits. +# NB: train_sft.py auto-resumes from any checkpoint-* in TRAIN_OUTPUT_DIR. An +# epochs=3 run already wrote checkpoints there, built for a ~14.5k-step schedule. +# Clear them before relaunching at epochs=1, or the trainer resumes the old +# 3-epoch schedule (wrong LR decay, or exits immediately if past 1-epoch steps): +# rm -rf outputs/sft-stage2-gemma4-31b/checkpoint-* +TRAIN_EPOCHS=1 TRAIN_LR=5e-5 TRAIN_BATCH_SIZE=1 TRAIN_GRAD_ACCUM=16 +# Measured, not assumed (scripts/determine_max_sequence.py, 2026-06-01 on +# socrat-zh-sft + socrat-en-sft): real token max = 909 (train) / 892 (eval), +# p99 = 673. Nothing exceeds 1024 → 1280 truncates NOTHING; it is non-binding. +# NB: with batch_size=1 + dynamic padding the activation peak is set by the +# actual longest sequence (909), not by this cap, so lowering it frees no VRAM. TRAIN_MAX_SEQ_LEN=1280 TRAIN_BF16=true TRAIN_GRAD_CKPT=true +# CPU parallelism (12C/24T box). dataset_num_proc fans the one-time tokenization +# of the 77k-record split across processes (single-process by default → pegs one +# core for minutes at startup); 12 ≈ physical cores. Those procs exit before +# training. +TRAIN_DATASET_NUM_PROC=12 +# Dataloader workers are 0 on purpose: the step is GPU/dequant-bound (~70s), data +# is pre-tokenized, and collation is trivial — async prefetch buys nothing here. +# Each worker also forks a HIP context that shows up in nvtop and costs a little +# VRAM on the tight 32GB/31B budget. Raise only if profiling shows data-feed stall. +TRAIN_DATALOADER_WORKERS=0 # ── Dataset sources ─────────────────────────────────────────────────────────── -# Default: Stage 2b — structural fine-tuning on state-annotated SocratDataset. -# socrat-synthetic is the contamination-control eval set; never include it here. -TRAIN_SOURCES=socrat-zh,socrat-en +# Stage 2b — per-turn inference-matching format (fixes train/serve schema drift). +# socrat-zh-sft / socrat-en-sft produce one (system, user, assistant) triple per +# dialogue turn with the exact 历史对话记录 / 当前学生输入 structure that +# socrates_teacher() sends at inference. socrat-synthetic is eval-only; never here. +TRAIN_SOURCES=socrat-zh-sft,socrat-en-sft # ── Output & logging ───────────────────────────────────────────────────────── TRAIN_OUTPUT_DIR=outputs/sft-stage2-gemma4-31b TRAIN_LOGGING_STEPS=10 -# Save every 100 steps (~32 min at 19s/step on Gemma 4 31B QLoRA). Tighter -# than the 200-step default — halves the worst-case rollback after a crash -# at the cost of ~2× I/O for adapter checkpoints (~500 MB each, ~11 GB total -# across the 23 checkpoints expected for the full 2298-step run). -TRAIN_SAVE_STEPS=100 +# Save every 10 steps. The gfx1201 GPU page fault (Memory access fault / page +# not present) is non-deterministic and hits during the QLoRA backward as early +# as step 14 — BEFORE any 100-step checkpoint lands. With save_steps=100 the +# auto-resume monitor had nothing to resume from, so every restart began at +# step 0 and re-crashed before 100: a permanent loop with zero forward progress. +# save_steps=10 lands a checkpoint before the earliest observed crash, so the +# resume monitor crawls the run forward across faults instead of looping. Disk +# is bounded by save_total_limit=2 (~1 GB of LoRA adapters). Drop to 5 if a +# crash ever lands inside the 10–20 window and stalls. +TRAIN_SAVE_STEPS=10 TRAIN_EVAL_STEPS=200 # Disable in-training eval — the eval forward's logits.float() cast doubles # the logits tensor (256K vocab × 1280 seq × 4 bytes = 8.6 GB transient on @@ -76,3 +114,20 @@ TRAIN_EVAL_STEPS=200 # eval pipeline (eval_bert_gemma_fewshot10_full.sh) produces paper numbers # anyway. Also forces load_best_model_at_end=False (which needs eval). TRAIN_EVAL_STRATEGY=no + +# ── Weights & Biases ────────────────────────────────────────────────────────── +# Enabled for the gfx1201 crawl. This lives in the CONFIG, not monitor_stage2.sh, +# on purpose: the auto-resume monitor is a long-running process with start_stage2() +# already parsed into memory, so editing the monitor script does NOT change its +# next relaunch. train_sft.py re-reads this config on every `make` relaunch, so a +# change here is picked up on the NEXT CRASH without restarting the monitor. +# (load_env_file setdefaults these into os.environ; a CLI/env override still wins.) +# Tracking still requires W&B auth — `wandb login` or WANDB_API_KEY. If login fails, +# train_sft._check_wandb disables reporting regardless of these vars. +WANDB_PROJECT=csen346-sft +# Pin a stable run id + resume=allow so every crawl resume APPENDS to one run with a +# continuous step axis, instead of spawning a fresh W&B run per crash cycle. Bump +# this id when starting a brand-new crawl (else it resumes this run's step history). +WANDB_RUN_ID=stage2-gemma4-31b-crawl +WANDB_RESUME=allow +WANDB_RUN_NAME=gemma4-31b-stage2-crawl diff --git a/docs/EXPERIMENT_LOG.md b/docs/EXPERIMENT_LOG.md index 861e78ea..13644776 100644 --- a/docs/EXPERIMENT_LOG.md +++ b/docs/EXPERIMENT_LOG.md @@ -4,6 +4,200 @@ Engineering decisions, what we've tried, and what's next. Each entry is dated an --- +## 2026-07-13 — LLM-judge on absolute Socratic quality COMPLETE ✅ — confirms the oracle win's *direction*, shrinks its *magnitude* + +**Ran:** the LLM-judge analysis-plan item — scored both **oracle arms** with **Claude Opus 4.8** (subscription, `claude -p` headless, tools off) on an absolute 0–10 pedagogy rubric (socratic_validity 0–3, advancement 0–3, age_appropriateness 0–2, question_form 0–2). Paired stratified sample, 200 turns/arm on identical `(file, turn_idx)` keys, stages b–e, seed 42, `--per-stage 50` → **400 Opus calls, ~2 h wall, ~$29 subscription-equivalent, 0 errors**. `scripts/llm_judge_eval.py` → `results/llm_judge_oracle_compare.json`. The rubric deliberately shows the GT reference as "one valid move, do not penalize routing differences" so it rewards *quality*, not ROUGE-overlap — the whole point is independence from ROUGE. + +### Result (SFT-oracle vs base-oracle, judged /10) + +| | overall | socratic_validity /3 | advancement /3 | age_approp /2 | question_form /2 | +|---|---:|---:|---:|---:|---:| +| SFT-oracle | **8.13** | 2.56 | 2.43 | 1.93 | 1.21 | +| base-oracle | 7.75 | 2.46 | 2.435 | 1.845 | 1.01 | +| **Δ (SFT − base)** | **+0.38** | +0.10 | **−0.01 (tied)** | +0.09 | **+0.20** | + +Per-stage Δ: b +0.14, **c +0.64, d +0.76**, e −0.02. The win concentrates in the middle reasoning stages and vanishes at the endpoints. + +### Finding + +- **Confirms the direction, corrects the magnitude.** The judge independently agrees SFT > base on absolute quality given correct state (+0.38/10; wins or ties every axis and stage). But +0.38/10 (~5% rel.) is **far smaller** than the +22.7 ROUGE-1 (~78% rel.) it's confirming. Both arms are pedagogically solid (7.75 vs 8.13) — so **most of the ROUGE gap was reference-phrasing overlap, not teaching quality.** The SFT learned to phrase like the corpus; ROUGE loves that, absolute pedagogy only modestly. +- **`advancement` is a dead tie** (2.43 vs 2.435) — given correct state, base advances the lesson as well as the SFT. The SFT's judge-visible edge is **question_form** (+0.20, largest axis) and **socratic_validity** (+0.10): *how* it asks, not *whether* it progresses. +- **Base costs *more* to judge** ($15.02 vs $13.86) — it emits more tokens, consistent with the earlier "base rambles regardless" observation. +- **Caveat:** single run, non-greedy headless subscription decoding — no error bars. The +0.38 headline is a point estimate; the internally-consistent per-axis/per-stage pattern is what earns it confidence. + +### Takeaway + +The oracle ROUGE result was real but *oversized* as a quality claim. A truer statement: **the SFT writes better-formed, more clearly Socratic questions than base and is never worse — but the two are neck-and-neck on actually advancing the dialogue once state is correct, and the corpus-phrasing overlap that drives the dramatic ROUGE numbers is largely cosmetic.** Written into `docs/SFT_RESULTS_REPORT.md` (new "LLM-judge (absolute pedagogical quality)" subsection) and issue #130. Remaining analysis-plan picks (both optional): multi-seed error bars, a strong-consultant (Claude-as-classifier) cell between the ~55–60% Qwen and 100% oracle state-quality points. + +--- + +## 2026-07-09 — Oracle-consultant arm COMPLETE ✅ — SFT teacher-turn quality is *largest* with the state confound removed + +**Ran:** the third consultant mode for the ZH-test ablation — the ground-truth state is fed to the teacher each turn (`--oracle-consultant`, `ORACLE_CONSULTANT=1 make ... via oracle_chain_gemma4_12b.sh`), so `state_accuracy` is 100% by construction and the arms are compared on ROUGE/BLEU alone. This removes classifier accuracy as a confound entirely — the cleanest "does the SFT write better Socratic turns given correct state?" measure. Both arms **681/681 valid, 0 errors** (SFT ~1.5 h; base ~2 days across auto-resumed crashes at 85 W). Results in `results/gemma4-12b-{base,sft}-oracle/`. 1 teacher call/turn (as fast as the classifier path, unlike the 2-call self-consult arms). + +### Teacher-turn quality across all three consultant modes (ZH test, SFT − base gap) + +| metric | base: self / Qwen / oracle | SFT: self / Qwen / oracle | **SFT − base gap: self / Qwen / oracle** | +|---|---:|---:|---:| +| ROUGE-1 | 28.02 / 28.56 / 29.03 | 44.21 / 48.13 / 51.77 | **+16.19 / +19.57 / +22.74** | +| ROUGE-L | 20.35 / 21.02 / 21.76 | 37.62 / 40.94 / 44.80 | **+17.27 / +19.92 / +23.04** | +| BLEU-4 | 4.80 / 5.22 / 5.73 | 18.31 / 20.12 / 24.80 | **+13.51 / +14.90 / +19.07** | + +### Finding + +- **The SFT climbs monotonically with state quality; base is flat.** SFT ROUGE-1: 44.2 (self, ~27% state) → 48.1 (Qwen, ~55–60%) → **51.8 (oracle, 100%)**. Base: **28.0 → 28.6 → 29.0**. The SFT *learned to condition on state* — better state in → better turn out; base can't exploit better state (it rambles regardless). That conditioning is itself a learned skill. +- **The gap is largest with the confound removed** — oracle gives the SFT its biggest edge (**+22.7 ROUGE-1, +23.0 ROUGE-L, +19.1 BLEU-4**), bigger than under the classifier or self-consult. "The SFT writes better Socratic turns" is unambiguous. +- **Self-consult was an underestimate, not the ceiling** — it was smallest (+16.2) because the SFT was penalized there, conditioned on its own worse self-classification (27%). Oracle reveals the true ceiling, *strengthening* the 07-06 "survives removing the classifier" conclusion rather than merely upholding it. + +### Takeaway + +Confirms and sharpens 07-06: the SFT's own contribution is **teacher-turn quality** (real, intrinsic, biggest given correct state), not state-tracking. In deployment it wants an external state source and rewards a better one with proportionally better turns. Written into `docs/SFT_RESULTS_REPORT.md` (oracle column added to the consultant-ablation section). Next: LLM-judge on absolute Socratic quality (independent confirmation of the oracle result), multi-seed error bars, optional strong-consultant (Claude) cell between the Qwen and oracle state-quality points. + +--- + +## 2026-07-06 — Consultant ablation (T1.1) COMPLETE ✅ — SFT internalized teacher-turn *quality*, not *state-tracking* + +**Ran:** ZH-test 2×2 with the external Qwen classifier **removed** — base and SFT each self-consult (dual-role: the served LLM produces the state assessment itself, then consumes it). All other settings pinned to the headline runs (Q8_0 GGUF, `-np 4`, q4_0 KV, MTP off, 8 rounds, stochastic sampling). Both arms **681/681 valid, 0 errors**. Chained SFT→base via `make noconsult-chain-gemma4-12b` (`NO_CONSULTANT=1` toggle, new this entry). SFT arm ~12 h; base arm **~7 days** (Jun 26 14:23 → Jul 3 19:44) — self-consult is 2 LLM calls/turn and the unstable box stepped power 130 W→85 W across auto-resumed crashes, but reached a full run. Results in `results/gemma4-12b-{base,sft}-noconsult/`. + +### The 2×2 (ZH test) — consultant is the only variable besides the model + +**State accuracy (overall %) — the classifier's job:** + +| | Qwen classifier | self-consult | Δ (self − Qwen) | +|---|---:|---:|---:| +| base | 49.62 | 34.45 | −15.17 | +| SFT | 59.93 | **26.80** | **−33.13** | +| **SFT − base** | **+10.31** | **−7.65** | *advantage inverts* | + +**Teacher-turn quality (SFT − base gap, Qwen → self-consult):** + +| metric | base Qwen→self | SFT Qwen→self | **SFT − base gap** | +|---|---:|---:|---:| +| ROUGE-1 | 28.56 → 28.02 | 48.13 → 44.21 | **+19.57 → +16.19** | +| ROUGE-L | 21.02 → 20.35 | 40.94 → 37.62 | **+19.92 → +17.27** | +| BLEU-4 | 5.22 → 4.80 | 20.12 → 18.31 | **+14.90 → +13.51** | + +### Finding — the two skills decouple cleanly + +- **Teacher-turn quality is intrinsic and survives.** With no external classifier, the SFT still writes far better Socratic turns than base: **+16.2 ROUGE-1, +17.3 ROUGE-L, +13.5 BLEU-4**. The SFT-vs-base text gap shrinks only ~3 pp of a ~20 pp gap when the crutch is removed. Strong evidence the SFT internalized Socratic *writing*. +- **State-tracking was entirely the classifier and does not survive.** The SFT's +10.3 pp state advantage *inverts* to −7.65 pp: self-classifying, the SFT (26.80) is **worse than base** (34.45), losing on every hard stage (b 24.4 vs 41.7, c 5.9 vs 16.5, e 27.1 vs 36.0). +- **Cause (predicted, `dataset.py:608–647`):** the SFT was trained to **consume** the consultant's assessment+action and emit a clean teacher turn — it never learned to *emit* state. So `state_accuracy` in self-consult measures a skill it never trained, and it appears to have regressed below base's zero-shot classification (it learned to expect state to be handed to it). + +### Takeaway + +The SFT's own contribution is **teacher-turn quality**, not state-tracking. In deployment it still wants the external classifier (or an equivalent state source) for the state label; on its own it brings the better Socratic turns. This reframes the headline table as "SFT + classifier vs base + classifier," not "SFT alone." Written up in `docs/SFT_RESULTS_REPORT.md` (new consultant-ablation section). Next: LLM-judge on absolute Socratic quality, oracle-consultant run (isolate teacher quality with the classifier confound removed), multi-seed error bars. + +--- + +## 2026-06-10 — Gemma 4 12B BASE, MTP OFF complete ✅ — losslessness confirmed at n=681, run-to-run σ calibrated + +**Ran:** Full Chinese test set (n=681), identical to the 06-09 MTP-on baseline except the drafter: MTP OFF, f16 KV pinned via the new `GEMMA4_12B_KV=f16` (the engine default q4_0 would have been a second variable), 85 W pinned (`POWER_START_W=85`), **4 eval workers against `-np 4`** (the MTP-on run was sequential). Monitor crawl, 681/681 valid, **0 errors, 0 crashes in 17.4 h**. First run with `bert_consultant` recorded in `run_config.json`. + +### MTP on/off quality A/B (the 1:1 the drafter promised) + +| metric | MTP ON (n=3991 turns) | MTP OFF (n=4033 turns) | Δ | +|---|---:|---:|---:| +| state_accuracy | 50.30 | 49.62 | −0.68 pp | +| rouge1 | 28.69 | 28.56 | −0.13 | +| rougeL | 21.24 | 21.02 | −0.22 | +| bleu4 | 5.28 | 5.22 | −0.06 | + +Every delta is within sampling noise — **MTP losslessness empirically confirmed at full n=681**. Per-stage shapes match (a=100 both; b/c/d/e within ~2 pp). + +### Bonus: run-to-run σ for the convergence budget + +These are two identical-config full runs differing only as seeds do — the cleanest variance estimate we have: **state accuracy run-to-run spread ≈ 0.7 pp at n=681**. Combined with the 06-09 convergence curve (±1 pp band entered at n≈200–300), a partial eval at n≈300 reads the state-acc headline to within ~1–1.5 pp total uncertainty. + +### Throughput: parallelism vs MTP at 85 W + +- MTP ON, 1 stream: 23.6 dlg/hr. +- MTP OFF, 4 streams: **39.1 dlg/hr** (1.66× aggregate) — but per-stream only ~9.8 dlg/hr, i.e. batching at the 85 W cap costs ~58% per-stream speed. +- **Winner for step 3 (SFT eval): MTP ON** on per-stream speed; MTP + 4 workers is untested and likely the true optimum if VRAM allows (drafter + f16 KV + 4 slots fit before, so it should). + +### Ops notes + +- W&B incident "Metric ingestion delayed" (Jun 9 16:47 PDT) made all charts render empty for hours despite the server acknowledging every row (`historyKeys` counted them) — backlog drained by Jun 10 afternoon, all runs fully queryable, no data loss. Lesson: check status.wandb.com before debugging the client. +- One transient gh 401 dropped the 450-dialogue progress row on issue #130 — single occurrence, auth healthy before and after. +- Artifacts: `results/gemma4-12b-base/`, W&B `gemma4-12b-base` (qitilwco, 69 live curve points logged through the incident). + +--- + +## 2026-06-09 (PM) — Per-dialogue W&B metric curves + 12B-base leaderboard placement + +**Ran:** No new model eval. Built incremental metric logging for the eval pipeline, replayed the completed `gemma4-12b-base-mtp` run into a 69-point convergence curve, and placed the run on the master leaderboard with corrected consultant attribution. + +### Tooling (kele.py / metrics.py / wandb_tracking.py) + +- **Live per-N-dialogue W&B logging:** evals with `WANDB_EVAL=1` now keep the W&B run open and log the full metric set every `WANDB_EVAL_LOG_EVERY` completed dialogues (default 10, `0` disables), step = completed-dialogue count, in both sequential and parallel paths. Crash-resumes start a new same-named W&B run whose steps continue where the last stopped (curves overlay in the UI). Closes the "live per-checkpoint state_accuracy" item proposed in #130. +- **`wandb-replay` subcommand:** `python -m src.project.kele wandb-replay --output results/ [--every N] [--order completion|id]` recomputes metrics over growing prefixes of the saved per-dialogue JSONs and logs a metric-vs-n curve for an already-finished run — no re-eval. +- `eval/n_turns` is logged alongside every point (metrics are computed per-turn, ~5.9 turns/dialogue) and can be used as the chart x-axis. + +### Convergence read (replay of `gemma4-12b-base-mtp`, W&B run `gemma4-12b-base-mtp-curve` / vcj442ce) + +Answers the open "is n≈400 enough?" question, per metric: + +- **state_accuracy:** within ~1 pp of the n=681 value (50.30) from **n≈200**, within ~0.5 pp from n≈450. Partial evals are fine for the state-acc headline. +- **ROUGE/BLEU:** slow monotone downward drift the whole run (rouge1 30.83 @ n=50 → 29.28 @ n=400 → 28.69 @ n=681); only within ~0.5 pp of final around **n≈450–500**. Caveat: the replay used completion order (file mtime), which correlates with dialogue length/difficulty — part of the drift may be ordering bias, not sampling error. A `--order id` (or shuffled) replay would disambiguate before locking a text-overlap budget. +- Implication for cross-run comparison: **n=50 cells read ~1–2 pp high on rouge1** relative to full runs. + +### Leaderboard placement (state accuracy, the metric that matters here) + +`gemma4-12b-base-mtp` ranks **#4 of the full n=681 runs** and #26/107 overall. Corrected attribution: its consultant is the **T4 Qwen3.5-0.8B LoRA classifier** (`state-clf-qwen3.5-0.8b-lora-wandb/final`, via `--bert-consultant`) — the **same classifier family as the `t4-bert-*` leaders** — with a bare (no `fewshot10`) Gemma4-12B-base teacher. Within the T4 full-run family: 55.39 (gemma+fewshot10) / 53.40 (a3b+fewshot10) / 53.04 (qwen27b-nothink+fewshot10) / **50.30 (this run, no fewshot)**. Same classifier ⇒ the 3–5 pp gap is a *context* effect (the classifier reads teacher responses in the dialogue history), so SFT-improving the teacher should pull state acc toward the 53–55 band via cleaner classifier inputs. + +### Gotchas logged + +- **`run_config.json` misstates the consultant.** It copies `CONSULTANT_MODEL_NAME` from the env config ("Gemma 4 12B") and does not record `--bert-consultant`; the actual consultant was the T4 classifier. Fixed same day: `run_batch_evaluation` now writes `bert_consultant` (ckpt path or null) into `run_config.json`; configs written before the fix lack the field. +- **W&B history-ingestion lag:** freshly finished runs show "no data for the selected runs" in charts and 0 rows via the API even though the server's `historyKeys` metadata counts all uploaded rows (filestream accepted 69/69 with no errors). Runs ≥ ~18 h old query fine. Server-side indexing delay — wait before debugging the client. Curve run was still un-queryable ~30 min after finish. + +--- + +## 2026-06-09 — Gemma 4 12B BASE teacher baseline COMPLETE ✅ (NVIDIA SFT-uplift PoC, phase 1) + +**Ran:** Full Chinese test set (`ulises-c/SocratDataset`, n=681) — base `gemma-4-12b-it` (Unsloth UD-Q8_K_XL GGUF) as teacher, Qwen3.5-0.8B-LoRA state classifier (`ulises-c/socrates-state-classifier-qwen3.5-lora`) as consultant, dual-role on llama.cpp port 8080. **MTP speculative drafter ON**, GPU power-capped at 85 W. Driven by `scripts/monitor_eval_gemma4_12b.sh` (crash-crawl). This is the **base baseline** for the 1-epoch Socratic QLoRA SFT-uplift question; uplift = SFT − base on state accuracy. + +### Final metrics (`results/gemma4-12b-base-mtp/metrics_summary.json`) + +| stage | state acc | +|---|---:| +| a | 100.0% | +| b | 44.55% | +| c | 32.85% | +| d | 35.98% | +| e | 60.21% | +| **overall** | **50.3%** | + +Text-overlap (diagnostic, memorization-prone — weight lightly): ROUGE-1 28.69 · ROUGE-2 11.90 · ROUGE-L 21.24 · BLEU-4 5.28 (681 dialogues, 3991 turns). + +### Headlines + +- **Overall state accuracy 50.3%** is the number SFT must beat. Per-stage shape: perfect opener (a = 100%), collapse through the middle states (b/c/d ≈ 33–45%), partial recovery at the summary (e = 60%). +- The noisy n=5 smoke read 56.25% — the full n=681 (50.3%) is the stable baseline; do not compare against the smoke. +- Eval samples at the llama.cpp default (temp 0.8); MTP is distribution-lossless, so on-vs-off differ only as seeds do. + +### MTP on/off throughput A/B (see issue #130, comment 4653475343) + +- **Decode:** 21.6 → 48.8 tok/s = **2.3×**, ~50–55% draft acceptance (f16 KV avoids the q8_0 0%-acceptance failure mode). +- **End-to-end** (same 5 dialogues, `elapsed_seconds`): 281 → 141 s/dlg = **~2.0×** (12.8 → 25.6 dlg/hr). MTP roughly halves wall-clock; an MTP-off full run would be ~52 h vs the ~29 h observed. + +### Run health + +681/681 valid, **0 errors, 0 crashes across ~29 h (1732 min) at 85 W**, 23.6 dlg/hr. The "fault every 30 min–2 h" risk did not materialize at this cap — 85 W is stable for inference on this card. + +### Artifact pointers + +- Results: `results/gemma4-12b-base-mtp/` (per-dialogue JSONs + `metrics_summary.json`) +- W&B: `csen346-eval` run `gemma4-12b-base-mtp` → https://wandb.ai/uchavarria-santa-clara-university/csen346-eval/runs/537mjkk2 +- Live eval log: issue #130 (pinned table + MTP A/B comment 4653475343) +- Eval restricted to Chinese-only via `kele.load_dataset` default (`ulises-c/SocratDataset`); the bilingual `+SocratDataset-EN` default added in `73adf12` was reverted for this PoC. + +### Next + +- **G5 SFT train** (`make train-gemma4-12b`, QLoRA 1 epoch → `ulises-c/SocratesLM-12B-QLoRA`) — compute-bound, so max power (`-pl 130`) is worth it there. +- **G8 SFT eval** (MTP-on, same Chinese n=681) → **G9 compare** vs this 50.3% baseline. +- Open (proposed in #130): live per-checkpoint `state_accuracy` in the eval log + an MTP-off run to test whether n≈400 suffices vs full 681. + +--- + ## 2026-06-02 — Canonical-baseline unified score landed (`GPT-4o + SocratTeachLLM · n=681` at unified 52.99) + n=681 STL leaderboard rows **Ran:** Three LLM-judge passes against previously un-judged n=681 cells, on branch `mk/unified-for-gpt4o-stl`. Sonnet 4.6 judge, 10 workers. Total wall-clock ≈ 33 min × 3 in parallel, total cost ≈ $66. diff --git a/docs/GFX1201_FAULT_ABLATION_LOG.md b/docs/GFX1201_FAULT_ABLATION_LOG.md new file mode 100644 index 00000000..28bf8320 --- /dev/null +++ b/docs/GFX1201_FAULT_ABLATION_LOG.md @@ -0,0 +1,167 @@ +# gfx1201 QLoRA Backward Page Fault — Ablation & Run Tracker (PR #101) + +Canonical, append-only log of every diagnostic/ablation run for the Gemma‑4‑31B QLoRA +backward page fault on the AMD R9700 (gfx1201 / RDNA4, ROCm 7.2). One row per run. +**Update this file whenever a run completes** — it is the system of record; PR comments +and wandb are the raw sources. + +> Companion: `docs/GFX1201_RDNA4_TRAINING.md §6.1` holds the narrative + original ablation +> plan. Where the two disagree, **this file wins** — §6.1 was written before the fault was +> localized and still frames it as bnb‑side (now ruled out, see below). + +--- + +## Current root‑cause state (2026‑06‑01) — CLOSED + +The fault is a GPU page fault ("page not present", 2 MB‑aligned host‑VA `0x7f…` address) +during the **QLoRA backward pass**, specifically the **gradient‑checkpoint forward recompute**. + +**Root cause confirmed (Bucket #2, probe‑3 + kernel‑name runs):** + +- **Faulting kernel (full ShaderName):** + `Cijk_Ailk_Bjlk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x64_MI16x16x1_SN_LDSB0_AFC1_AFEM1_AFEM1_ASEM1_CLR1_CADS0_DTLA0_DTLB0_DTVA0_DTVB1_EPS0_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_ISA1201_IU1_K1_LDSTI0_LBSPPA1024_LBSPPB0_LBSPPM0_LPA32_LPB0_LPM0_LRVW8_LWPMn1_MIAV1_MIWT2_2_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB2_ONLL0_PGR2_PLR1_PKA0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW8_SK0_SKFTR0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA2_VWB1_WSGRA0_WSGRB0_WS32_WG32_4_1` + Same kernel variant in both forward recompute AND backward pass. Confirmed on clean GPU + (runs #5, #9). Key discriminating flags: `DTVB1` (B uses different dtype/layout), + `LBSPPB0` (no B LDS prefetch), `NLCB2` (B double‑unrolled), `VWA2_VWB1`. +- **bitsandbytes is NOT the faulting op.** Dequant kernels succeed every cycle. +- **Bucket #2 — wild address confirmed (probe‑3, run #8):** + Both GEMM operands logged; neither brackets fault 0x7f6459a00000. + A=(1,608,21504) contig row‑major `ptr=0x7f655ece8000 end=0x7f65605d8000`; + B=(21504,5376) col‑major `stride=(1,21504) ptr=0x7f63f01a0000 end=0x7f63fde20000`. + Fault is ~1 GB below A.ptr and ~1.2 GB above B.end. Full‑log scan: 0 operands bracket fault. + **The ISA1201 Tensile kernel computes a wild address from the column‑major B descriptor.** + +~~Bucket #1 (freed/recycled operand) — eliminated by probe‑3.~~ + +--- + +## Run log + +Legend: **GPU=clean** means `make gpu-preflight` PASSed immediately before launch (else a +dirty‑KFD cascade can fault early and confound the result). All runs use baseline unless noted: +`TORCH_USE_HIPBLASLT=0`, `PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8`, +`grad_ckpt=use_reentrant:False`, bnb 0.49.2, SDPA, seq 1280, bs 1×16, seed 42. +`SYNC` = `AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1`. wandb IDs marked `?` need confirming on the box. + +| # | Run / log | wandb | Code | Start | Key vars | GPU | Fault step | Faulting kernel | What it established | +|---|---|---|---|---|---|---|---|---|---| +| 1 | historical `train.log` | various | pre‑diag | fresh@0 | baseline, lvl1 | unknown | ~84 | not named | baseline fault exists | +| 2 | eos‑gate set | `g2df2ifl` `gv9fbjac` `5xd8qt5w` `irgdklt9` | SHAs `c752a2b4`,`eb12dbd9` | fresh@0 | baseline | unknown | 10 / 84 / clears 100 | not named | **probabilistic** — same SHA both finishes & crashes | +| 3 | `gfx1201_fault_2026-06-01.log` | `h9z6ebjd` | `8580844` | fresh@0 | **SYNC**, lvl1 | unknown | 84 | not named (lvl1) | **async race RULED OUT** (serialized still faults) | +| 4 | `diag-l3-resume.log` | `akfrsb3z` | no probe (≈Review#1 cmd) | ckpt‑10 | SYNC, **lvl3** | **dirty** (no preflight) | 22 | **`MT64x64x64 ISA1201`** | named the kernel (but cascade‑confounded) | +| 5 | `clean-repro-1.log` | `001pnijf` | no probe | ckpt‑20 | SYNC, lvl3 | **clean** | 21 | `MT64x64x64 ISA1201` | fault is **real, not a cascade**; bnb dequant succeeds | +| 6 | `probe-1.log` | `i6m2e3sx`? | probe v1 `81cebb4` | ckpt | SYNC, lvl3, `BNB_DEQUANT_PROBE` | clean | ~mid | `MT64x64x64` | fault **1.5 GB from dequant buffer** → "bnb descriptor too small" **RULED OUT** | +| 7 | `probe-2.log` | ? | probe v2 `1b752c2` | ckpt | SYNC, `BNB_DEQUANT_PROBE` (+grad_output) | clean | ~mid | `MT64x64x64` | 8 dequants, no `gemm_operandA` → fault is in **forward recompute**; A‑operand not logged → bucket inconclusive | +| 8 | `probe-3.log` | ? | `dad8f4a` | ckpt‑20 | SYNC, lvl1, `BNB_DEQUANT_PROBE` (+forward input) | clean | ~21 | not named (lvl1) | **Bucket #2 confirmed** — both operands logged & valid, fault 0x7f6459a00000 ≈1 GB from either buffer; col‑major B descriptor → wild address. Bucket #1 **eliminated**. | +| 9 | `kernel-name.log` | N/A | `b6cb557` | ckpt‑20 | SYNC, **lvl3**, no probe | clean | ~21 | **`MT64x64x64_ISA1201_DTVB1_VWA2_VWB1`** (full name above) | **Forward faulting kernel ShaderName confirmed** — identical to backward (run #5). Both recomputed‑forward and backward GEMMs use same Tensile tile. All info ready for upstream report. | +| 10 | `expandable-seg.log` | N/A | `a70a2d1` | ckpt‑20 | SYNC, lvl1, `expandable_segments:True` (ignored) | clean | ~21 | `MT64x64x64 ISA1201` (lvl1, inferred) | **Arm D — fault persists**, addr `0x7f29eb600000`. `expandable_segments` unsupported on gfx1201 (silently ignored). Placement change cannot mask fault → placement irrelevant, confirms pure kernel stride bug. | +| 11 | `hlt1.log` | `e6srgn90` | `ed92b93` | ckpt‑20 | SYNC, lvl1, **`HIPBLASLT=1`** | clean | ~21 | `MT64x64x64 ISA1201` (Tensile fallback) | **Arm HLT1 — fault persists**. hipBLASLt looked up `MT64x64x64_DTVB1` and returned `Cannot find the function` for 6 modules. Tensile dispatched the same bad ISA1201 kernel, addr `0x7f0ac2e00000`. gfx1201 hipBLASLt has no kernel for col‑major B (`DTVB1`) at this tile. Routing fix unavailable in ROCm 7.2. | +| 12 | `contiguous-b.log` | N/A | `9a2ea38` | ckpt‑20 | SYNC, lvl1, **`BNB_FORCE_B_CONTIGUOUS=1`** | clean | ~21 | `MT64x64x64 ISA1201 DTVB1` | **Contiguous‑B — fault persists**. Hook confirmed active. Forced `.contiguous()` on `dequantize_4bit` output (stride `(1,21504)→(5376,1)` at Python level). `DTVB1` still dispatched — descriptor is set by PyTorch BLAS call for `A @ W.T`, not by the Python tensor's physical stride. Python‑level copy cannot reach the BLAS transpose flag. **Col‑major B is intrinsic to the 4bit matmul structure**; no Python‑level intervention possible. Fault addr `0x7f6f48c00000`. | + +--- + +## Pending ablation arms — each is a candidate **fix**, judged on two axes + +**Axis A:** did it *eliminate* the fault? (→ clean ~94 h run, crawl unnecessary). +**Axis B:** did it *improve steps‑per‑resume*? (→ brute‑force crawl becomes cheap). +Protocol: `make gpu-preflight` → fixed seed → run ≥150 steps (or to fault) → record both axes. +Change exactly **one** factor from baseline. + +| Arm | One change | Targets | Axis A result | Axis B result | Status | +|---|---|---|---|---|---| +| ~~HLT1~~ | ~~`TORCH_USE_HIPBLASLT=1`~~ | ~~#2 (route GEMM off Tensile → hipBLASLt)~~ | fault persists (run #11) | N/A | **DONE** — hipBLASLt has no `MT64x64x64 DTVB1` kernel for gfx1201; falls back to Tensile, same fault. | +| ~~contiguous‑B~~ | ~~`BNB_FORCE_B_CONTIGUOUS=1`~~ | ~~#2 (change B layout before GEMM)~~ | fault persists (run #12) | N/A | **DONE** — Python `.contiguous()` on dequant output doesn't reach BLAS descriptor; `DTVB1` intrinsic to `A @ W.T` call structure. | +| **C** | grad‑ckpt `use_reentrant=True` | #1 (activation lifetime) | | | pending — one‑line, run after probe‑3 if #1 | +| **D** | `PYTORCH_HIP_ALLOC_CONF=expandable_segments:True` | placement | fault persists (run #10) | N/A | **DONE** — `expandable_segments` silently ignored on gfx1201 (PyTorch warns at startup); fault at ~step 21 addr `0x7f29eb600000`. Confirms: placement irrelevant, fault is kernel stride bug. | +| **E** | `HSA_ENABLE_SDMA=0` | DMA page‑fault mitigation | | | **HOLD** — sign unknown (changes the SDMA copy path; could move the fault rate either way). Do **not** stack on the viability baseline. Run as a *separate* arm only if pure‑production is marginal. | +| **B** | `TRAIN_GRAD_CKPT=false` | confirms recompute drives it | | | pending — likely **OOMs** at 98 % VRAM (diagnostic, not a fix) | +| ~~A~~ | ~~bnb source build `-DBNB_ROCM_ARCH=gfx1201`~~ | ~~bnb kernel~~ | — | — | **DROPPED** — probe‑1 ruled out bnb as the faulting op | +| TUN | `PYTORCH_TUNABLEOP_ENABLED=1` | #2 (pick different GEMM) | | | speculative fallback — selects on **speed not correctness**; may re‑pick the bad kernel or crash during tuning | + +--- + +## Ruled out — do not re‑investigate (with the run that settled it) + +| Hypothesis | Status | Evidence | +|---|---|---| +| Async / concurrency race | **ruled out** | run #3 — serialized run still faults | +| Numerical / data / bad batch | ruled out | §6.1 — smooth loss, byte‑identical per‑step losses | +| LR / optimizer magnitude | ruled out | §6.1 — lr 5e‑6 crashed *earlier* than 5e‑5 | +| Sequence length / `max_length` | ruled out | §6.1 — real token max 909 < 1024; non‑binding | +| bnb NF4 backward kernel is the faulting op | **ruled out** | runs #5–7 — dequant succeeds; fault is in the Tensile GEMM. **Supersedes §6.1's "fault is in the bitsandbytes‑NF4 path" and removes Arm A.** | +| bnb hands the GEMM a too‑small descriptor | **ruled out** | run #6 — fault 1.5 GB from the dequant buffer, not adjacent to `end` | +| hipBLASLt is the **cause** | ruled out | §6.1 — `HIPBLASLT=0` still faults. **But:** =1 was **never tried as a *fix*** (see arm HLT1) — §6.1's "hipBLASLt ruled out" conflated cause with remedy | +| Dirty‑KFD cascade is the *true* fault | ruled out | run #5 — fault reproduces on a verified‑clean GPU | + +--- + +## How to log a run (keep this file current) + +After any run completes, append a row to the **Run log** (or fill an ablation arm's Axis +A/B) with: run/log filename, wandb id, commit hash (`git rev-parse --short HEAD`), resume +point, the one changed variable, GPU‑clean status, fault step, faulting kernel, and verdict. + +Standard launch (adjust the one variable under test; keep `AMD_LOG_LEVEL=1` unless you need +to re‑name the kernel — level 3 writes multi‑GB logs): + +```bash +make gpu-preflight # MUST pass +COMMIT=$(git rev-parse --short HEAD) +CKPT=outputs/sft-stage2-gemma4-31b +nohup env TORCH_USE_HIPBLASLT=0 \ + AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1 AMD_LOG_LEVEL=1 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit TRAIN_PREQ=true \ + TRAIN_MAX_STEPS=95 TRAIN_OUTPUT_DIR="$CKPT" \ + uv run --no-sync python scripts/train_sft.py --config configs/train-sft-stage2-gemma4-31b.env \ + > "$CKPT/run-$COMMIT.log" 2>&1 & +``` + +--- + +## Crawl viability measurement (run #13, DONE — STALLED) + +**Result (2026-06-01):** crawl is NOT viable from checkpoint-20. 8/8 production-mode resume runs crashed at optimizer step 22 or 24 (within 4 steps of checkpoint-20 every time). No new checkpoint was banked. Monitor exited STALLED per pre-committed MAX_RETRIES=8 no-progress rule. + +**N steps/crash: 2–4 (mean ≈ 3).** Pre-committed NO-GO threshold was N ≲ 10. + +**Fault is data-order sticky:** HF Trainer restores `rng_state.pth` on resume, replaying identical batches → same samples at steps 21–23 every time → same GEMM shape → deterministic fault. Fault address varies (ASLR) but step is constant. Contrast with run #2 (fresh-from-0, probabilistic, sometimes clears 100 steps): different data trajectory, fault landed at different positions or not at all in 100 steps. + +**Next experiment:** `ignore_data_skip=True` on resume — breaks dataloader-state restoration, converts sticky back to probabilistic. Directly predicted by the stickiness mechanism. If this recovers ~100 steps/crash (run-#2-like), arithmetic flips to GO. + +--- + +## Crawl viability measurement (archived — go/no‑go pre‑committed) + +With all fix arms exhausted (#11/#12) and the goal set to *31B QLoRA end‑to‑end on +gfx1201/RDNA4*, the only on‑box path is the brute‑force crawl. Before committing +wall‑clock, measure **production steps‑per‑crash** and **sticky‑vs‑advancing** under +the real crawl loop. + +**Protocol — launch the pure‑production default** (no SYNC, no probes, +`TORCH_USE_HIPBLASLT=0`, `TRAIN_SAVE_STEPS=10`, **Arm E held back** — clean baseline): + +```bash +make gpu-preflight # MUST pass +nohup bash scripts/monitor_stage2.sh > outputs/monitor_stage2.log 2>&1 & +``` + +`monitor_stage2.sh` **is** the crawl harness: relaunch‑on‑crash (re‑runs gpu‑preflight), +KFD‑clean‑before‑relaunch (`test_gpu_stack.sh --wait-clean 180`), `quarantine_bad_checkpoint` +(partial‑save guard — resume falls back to N‑1), per‑crash log+dmesg archive. + +**Two questions, in order:** + +1. **Sticky vs. advancing (gating).** Does resume get *past* the fault step, or re‑fault at + the same step forever (the M=608 data‑order risk)? Already operationalized: the monitor + counts only **consecutive no‑progress** retries — a new checkpoint resets the counter; + `MAX_RETRIES=8` with no new checkpoint → posts `STALLED` and exits = **crawl is dead**, + upstream fix becomes mandatory even on the RDNA4 path. +2. **Wall‑clock viability (the number).** ~4,800 steps/epoch × ~70 s/step ≈ **93 h** pure + compute. Each crash cycle costs the KFD‑clean wait + ~19 GB reload + preflight (~4–6 min). + At a mean of **N steps/crash**, cycles ≈ 4,800 / N: + - **GO** if N ≳ 25–30 (~160–190 cycles, ~10–19 h overhead on top of 93 h). + - **NO‑GO** if N ≲ 10 consistently (~480+ cycles, 30 h+ overhead) or the monitor `STALL`s. + +This threshold is pre‑committed (posted to PR #101) before the first crash so the +decision is a number, not a vibe. diff --git a/docs/GFX1201_RDNA4_TRAINING.md b/docs/GFX1201_RDNA4_TRAINING.md new file mode 100644 index 00000000..ca63e045 --- /dev/null +++ b/docs/GFX1201_RDNA4_TRAINING.md @@ -0,0 +1,491 @@ +# Training LLMs on AMD gfx1201 (RDNA 4 / Radeon AI PRO R9700): What Works, What Costs You, and Why + +A field report on QLoRA / LoRA fine-tuning of modern transformer and hybrid (Mamba + +attention) LLMs on **AMD gfx1201 (RDNA 4, Wave32)** under **ROCm 7.2**, framed against the +NVIDIA/CUDA baseline where most of these capabilities are free. + +> **Source repo:** [github.com/ulises-c/csen-346](https://github.com/ulises-c/csen-346) — the +> canonical home of this report, the companion `docs/GPU_SUPPORT.md`, `scripts/patch_fla_rocm.sh`, +> and the full investigation history ([#100](https://github.com/ulises-c/csen-346/issues/100) · +> [#79](https://github.com/ulises-c/csen-346/pull/79) · +> [#109](https://github.com/ulises-c/csen-346/issues/109)). + +> **Scope.** This is a *synthesis* of hands-on debugging on a single R9700 box, not a +> re-derivation from upstream. Every claim carries a confidence marker (below) so an external +> reader can tell "we ran this on real hardware" from "theorized but untried." Findings are +> generalizable to gfx1201 training; specific model names (Qwen3.6-27B, Gemma 4 31B, +> Qwen3.5-0.8B) appear only as concrete architecture examples. +> +> Companion doc: [`GPU_SUPPORT.md`](https://github.com/ulises-c/csen-346/blob/main/docs/GPU_SUPPORT.md) covers the *serving* side. This doc is +> *training*. + +## Confidence legend + +| Marker | Meaning | +|---|---| +| ✅ | **Confirmed on the R9700** — observed directly, reproduced | +| ⏳ | **Untried / theorized** — plausible, not yet validated on this hardware | +| 🔭 | **Upstream watch** — the real fix lives in a dependency; tracked, not landed | +| ↩️ | **Correction** — a widely-repeated claim (or an earlier note of ours) that turned out wrong | + +--- + +## TL;DR — the gfx1201 tax vs. NVIDIA/CUDA + +On NVIDIA, the modern QLoRA stack (`transformers` + `peft` + `trl` + `bitsandbytes` + Flash +Attention + Triton) installs from PyPI and trains. On gfx1201 each layer of that stack has a +seam where the CUDA assumption leaks through. The headline: + +| Capability | NVIDIA/CUDA baseline | gfx1201 / ROCm 7.2 status | Workaround | Conf. | +|---|---|---|---|---| +| **Flash *Linear* Attention** (Mamba/DeltaNet fast-path) | works (PyPI wheel) | **JIT-deadlocks at step 0** — Triton 3.6.0 `tritonamdgpu-pipeline` use-after-free at `num_stages≥2` | patch FLA `num_stages→1` / `num_warps→4`, **or** pin Triton 3.5.1 | ✅ | +| **Flash Attention 2** (softmax) | works (PyPI wheel) | **CK backend won't compile** — Composable Kernel assumes Wave64 (CDNA); RDNA 4 is Wave32 | `FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE` (Triton backend) — or just use SDPA | ✅ | +| **bitsandbytes 4-bit NF4 *runtime*** | works | **works** (stock `bitsandbytes==0.49.2` from PyPI; no source build needed) | — (source build `-DBNB_ROCM_ARCH=gfx1201` only if a future version regresses) | ✅ | +| **hipBLASLt / cuBLASLt fast matmul** | cuBLASLt, default | **crashes in RoPE** (`HIPBLAS_STATUS_INVALID_VALUE`) on some attention archs | `TORCH_USE_HIPBLASLT=0` → rocBLAS fallback (slower); test per-arch first | ✅ | +| **QLoRA load (4-bit from a BF16 base)** | needs only post-quant RAM | **stages the *full* BF16 model in CPU RAM before quantizing** — a ~31B model wants ~62 GB transient | load a **pre-quantized** bnb checkpoint (already NF4) | ✅ | +| **`expandable_segments:True`** allocator | works | **contested** — helped in one session; a later run saw it precede GPU page faults at step 16 | latest verdict: **leave it off**; `garbage_collection_threshold:0.8` alone (§6) | ⚠️ ↩️ | +| **FP8 (E4M3) compute** | works (Hopper/Blackwell) | **silently falls back to FP32** — gfx1201 missing from AITER arch table | none (affects serving, not LoRA training) | 🔭 | +| **vLLM** | works | **doesn't recognize gfx1201** upstream | serve via HF Transformers instead | ✅ | +| **`pip install` of CUDA-source extensions** (`causal-conv1d`, `flash-attn`) | works | **"NVCC trap"** — isolated build pulls CUDA torch, injects `-gencode compute_80` | build with `--no-build-isolation` against your ROCm torch (routes through `hipify_torch`) | ✅ | + +**Net for QLoRA fine-tuning on a 32 GB R9700:** it works, but two things bite that never bite +on NVIDIA — the **FLA Triton deadlock** (only if your model has linear-attention layers) and +the **load-time CPU-RAM staging wall** (for any large 4-bit load from a BF16 base). Both have +clean workarounds. VRAM itself is not the constraint once the model is loaded. + +--- + +## Reproduction environment + +| Component | Spec | +|---|---| +| CPU | AMD Ryzen 9 5900X (12C/24T) | +| GPU | **AMD Radeon AI PRO R9700** — gfx1201 (RDNA 4, Navi 48), 64 CUs, **32 GB VRAM**, Wave32 | +| System RAM | 64 GB | +| OS / kernel | CachyOS (Arch-based) | +| ROCm | **7.2** (HIP 7.2.26015, ROCm Runtime 1.18) — native gfx1201, no HSA override needed | +| PyTorch | **2.11.0+rocm7.2** | +| Triton | **3.6.0** (patched) — see §1 | +| transformers / peft / trl | 5.9.0 / 0.19.1 / 1.4.0 | +| bitsandbytes | 0.49.2 | +| flash-linear-attention | 0.5.0 (patched) | +| causal-conv1d | 1.6.2.post1 (built `--no-build-isolation`) | + +**Search terms** (for anyone landing here from a search engine): `gfx1201` · `RDNA4` · +`R9700` · `ROCm 7.2` · `flash-linear-attention deadlock` · `num_stages Triton gfx12` · +`tritonamdgpu-pipeline use-after-free` · `caching_allocator_warmup` · `Found no NVIDIA driver` · +`bitsandbytes CPU dispatch ROCm` · `Gemma QLoRA OOM load` · `HIPBLAS_STATUS_INVALID_VALUE RoPE`. + +--- + +## The central distinction: two kernel stacks, two blockers + +The single most important thing to internalize before debugging gfx1201 training is that +"attention" splits into two completely separate kernel stacks with separate failure modes. +Which one bites you depends entirely on your model architecture: + +| | **Linear-attention models** (Mamba / gated-DeltaNet hybrids — e.g. Qwen3.6-27B: 48 linear + 16 full layers; Qwen3.5-0.8B hybrid) | **Softmax-attention models** (e.g. Gemma 4 31B: sliding-window + global) | +|---|---|---| +| Fast-path library | **Flash *Linear* Attention** (Triton) + `causal-conv1d` | **Flash Attention 2** / SDPA | +| Real gfx1201 blocker | **FLA Triton deadlock** (§1) | **none for training** — SDPA is stable (§2) | +| Load blocker (4-bit) | CPU-RAM staging (§5) | CPU-RAM staging (§5) | + +Most community "Flash Attention on RDNA4" material is about **FA2** (softmax). The FA2 story is +the *related landscape*, but if your model has linear-attention layers, your actual training +blocker is **FLA**, which is a different library and a different bug. + +--- + +## 1. Flash Linear Attention (FLA) — the Triton 3.6.0 deadlock ✅ + +**Symptom.** `flash-linear-attention==0.5.0` installs and imports fine. On the **first training +step** it JIT-compiles ~120 Triton kernels, then one kernel **hangs forever**: process alive at +0% CPU, VRAM full, no error, progress bar stuck at step 0. Community reports describe the same +signature as `hsa_signal_wait` polling a DMA-completion flag that never flips. + +**Root cause.** Triton **3.6.0**'s AMD software-pipelining pass (`tritonamdgpu-pipeline`) has a +**use-after-free when `num_stages ≥ 2` on gfx12xx** — and gfx12xx is **not in Triton 3.6.0's +verified target list**. The same compiler bug has two manifestations: + +- **FLA:** silent first-step deadlock (the hang above). +- **SageAttention:** hard crash — `RuntimeError: PassManager::run failed 'tt.load' op operation destroyed but still has uses`. + +**Fix A — patch the wheel (confirmed, supported).** Rewrite `num_stages=[2-9]→1` and +`num_warps>4→4` (Wave32) in the installed FLA Triton autotune configs, then clear the Triton +and FLA caches. This mirrors the upstream SageAttention fix +([thu-ml/SageAttention#365](https://github.com/thu-ml/SageAttention/pull/365), +[kijai/ComfyUI-WanVideoWrapper#2007](https://github.com/kijai/ComfyUI-WanVideoWrapper/issues/2007)), +which gates `num_stages=1` on `torch.version.hip is not None`. Confirmed working at seq 768. +**Must be re-applied after every dependency sync** because the package manager reinstalls a +fresh, unpatched wheel. + +```bash +# concept — find the installed FLA, cap the autotune configs, nuke caches +FLA=$(python -c "import fla; print(fla.__path__[0])") +find "$FLA/ops" -name '*.py' -exec sed -i -E 's/num_stages=[2-9]/num_stages=1/g; s/num_warps=([5-9]|[12][0-9])/num_warps=4/g' {} + +rm -rf ~/.triton/cache +``` + +**Fix B — pin Triton 3.5.1 (avoids the bug at the source) ⏳.** The UAF is a 3.6.0 regression; +a prebuilt `triton-3.5.1+rocm7.2.1` wheel exists, and the gfx1201 community reports +**Triton 3.5.1 + PyTorch 2.9.1** as the stable native pairing with FLA working **unpatched**. +⚠️ Verify ABI before adopting on a torch 2.11 stack — the documented pairing is torch 2.9.1. + +**Nightly status 🔭.** **No nightly fixes this.** As of late May 2026, gfx12xx is still not in +Triton's verified target list, and every "fix" in the wild is a *downstream* `num_stages=1` +workaround in the calling library — not a Triton compiler fix. Upgrading torch/Triton nightly +does **not** help and risks re-introducing the bug. Pin and patch. + +**Cheaper levers to try first ⏳** (before any reinstall): +- `TRITON_DISABLE_AUTOTUNING=1` — the hang is in one *autotuned* config; disabling autotune + means the broken candidate is never generated. Cheapest, stack-agnostic, reversible. +- `TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1` — enables AOTriton experimental flash paths on + RDNA 4; stackable with the above. + +**Why it matters (stakes).** Without FLA, linear-attention models fall back to a pure-PyTorch +path that only fits at short sequence length, truncating long dialogues and ballooning ETA +(an observed ~143 h / 3 epochs for a 27B at seq 512). FLA is what unlocks seq ≥ 768 *and* a +sane ETA — it is load-bearing, not a nice-to-have. + +--- + +## 2. Flash Attention 2 (softmax) — Wave64 vs Wave32 ✅ + +Stock `flash-attn` defaults to the **Composable Kernel (CK)** backend, which assumes +**Wave64 (CDNA)**. RDNA 3/4 are **Wave32**, so CK fails to compile / hits an ISA mismatch +(the PyPI `flash-attn` wheel fails on gfx1201 outright). + +- **Fix:** `FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE` + `pip install flash-attn --no-build-isolation` + to JIT a gfx1201 **Triton** FA2 kernel at runtime (fwd+bwd, causal, varlen). Build the + [ROCm/flash-attention](https://github.com/ROCm/flash-attention) fork, Triton backend. +- **Simpler for training: just use SDPA.** `attn_implementation="sdpa"` is stable on gfx1201 + and, on ROCm, may route the softmax layers to the **AOTriton** flash kernel when the torch + build's AOTriton supports gfx1201 (otherwise it silently falls back to the math / + mem-efficient path — correct, just slower). For a model whose *only* attention is softmax + (e.g. Gemma 4), SDPA alone is sufficient — no FA2 build needed. + +**Dead ends, logged so nobody re-investigates:** +- **rocWMMA FA2** ([Repeerc/flash-attention-v2-RDNA3-minimal](https://github.com/Repeerc/flash-attention-v2-RDNA3-minimal)) — RDNA 3 (gfx1100) only, no backward pass, weak BF16. Not viable for training. +- **ThunderKittens** — NVIDIA-only (Hopper/Blackwell). Its AMD sibling **[HipKittens](https://github.com/HazyResearch/HipKittens)** supports **CDNA3/CDNA4 (MI300/MI350) only — not gfx1201**. + +--- + +## 3. bitsandbytes 4-bit NF4 *runtime* ✅ + +This is the one true gate common to **every** 4-bit path (live-quant or pre-quant): the NF4 +dequant kernels must run on gfx1201 at train time. **Good news: they do, on stock +`bitsandbytes==0.49.2` from PyPI — no source build required.** Confirmed via a GPU stack +smoke test (8-bit LLM.int8 forward ✅, 4-bit NF4 QLoRA ✅, `BitsAndBytesConfig` NF4 + +double-quant ✅). + +gfx1201 is in bitsandbytes' RDNA arch table. If a future version regresses, the escape hatch is +a source build: `cmake -DCOMPUTE_BACKEND=hip -DBNB_ROCM_ARCH="gfx1201"`. + +--- + +## 4. hipBLASLt RoPE crash — and when it's safe ✅ + +Without mitigation, hipBLASLt can raise `HIPBLAS_STATUS_INVALID_VALUE` during the first forward +pass on gfx1201. The blunt fix is `TORCH_USE_HIPBLASLT=0` (forces rocBLAS) — but that costs the +matmul speedup (measured ~4.1× on gfx1201: 5.2 ms vs 21.5 ms; rocBLAS fallback observed at +~75–90 s/step in one config). + +**Nuance worth testing per-architecture:** the page-faults (`rc=134`) were traced to a specific +linear-attention kernel path (`torch_chunk_gated_delta_rule`) — **not** a general hipBLASLt +instability. For a pure-softmax model that never touches the delta-rule op, +**`TORCH_USE_HIPBLASLT=1` may be safe** and worth a 20-step trial before defaulting to the +rocBLAS fallback — the latest hands-on Gemma 4 (softmax) launch adopted exactly this. One +session paired it with `expandable_segments:True` to stabilize past the early crash zone, but +that allocator knob later proved contested (§6) — keep the two changes independent when testing. + +--- + +## 5. The QLoRA load wall — CPU-RAM staging ✅ (the blocker NVIDIA users never see) + +This is the one that actually stops large-model QLoRA on a modest-RAM box, and it is **not** +VRAM, **not** FLA, and **not** GPU-specific — it would hit any GPU paired with insufficient +*system* RAM: + +**`from_pretrained(..., load_in_4bit=True)` materializes the full BF16 model in CPU RSS +*before* NF4 quantization** — regardless of `low_cpu_mem_usage=True`, `offload_state_dict`, +`offload_folder`, GC threshold, or malloc tuning. bitsandbytes quantizes only after all weights +are resident. So a ~31B model needs **~62 GB of transient CPU RAM at load**, even though the +post-quant resident footprint is ~16 GB. On a 64 GB box this thrashes into zram +(~200 s/shard → ~20 h just to load). 8-bit (`load_in_8bit`) has the **same** wall. + +**Fix — load a pre-quantized bnb checkpoint.** Weights arrive already NF4 (~19 GB on disk), +streamed straight to 4-bit with no full-BF16 CPU materialization. Drops load-RAM from ~62 GB +to ~8–16 GB and needs no second high-RAM machine. Verify the embedded `quantization_config` +matches your pipeline (`nf4` + double-quant + `bfloat16` compute) and that the LoRA target +regex still matches the checkpoint's module paths. + +> **Two device-map gotchas on the 4-bit load path (gfx1201-specific seams):** +> - **`device_map="auto"` can dispatch layers to CPU**, which bitsandbytes rejects. Force +> `device_map={"": 0}` to keep the whole quantized model on the single GPU. +> - Recent `transformers` runs a CUDA-only `caching_allocator_warmup` that raises +> **"Found no NVIDIA driver"** on HIP/ROCm. Patch/guard it for `torch.version.hip`. +> - Passing an explicit `quantization_config=None` to `from_pretrained` can make `auto_factory` +> overwrite a pre-quantized checkpoint's own config with `None` → `supports_quant_method` +> crash. Only pass the kwarg when it's non-None. + +### Hardware estimate — large (~31B) QLoRA, seq 1280, batch 1×16, grad-ckpt on, eval off + +| Resource | Pre-quant checkpoint (recommended) | Naive `load_in_4bit` from BF16 | +|---|---|---| +| GPU VRAM (peak, train) | **~22–27 GB** | ~22–27 GB | +| **System RAM (peak, load)** | **~8–16 GB** | **~62–70 GB** ← the blocker | +| Download | ~19 GB | ~62 GB | +| Disk working set | ~30 GB | ~75 GB | + +**Minimum viable:** GPU **≥ 32 GB** (R9700 / RTX 5090) · 24 GB (3090) marginal (short seq only) · +**20 GB (RTX 4000 Ada) won't fit**. System RAM **~16 GB** with the pre-quant checkpoint. +**Keep eval disabled** — a large-vocab `logits.float()` copy adds an ~8.6 GB spike that OOMs +even 32 GB. + +--- + +## 6. Memory-allocator config — what's true now (one contested knob) ↩️ + +| `PYTORCH_HIP_ALLOC_CONF` knob | Verdict on gfx1201 | +|---|---| +| `garbage_collection_threshold:0.8` | ✅ use it | +| `expandable_segments:True` | ⚠️ **contested — default off** (see below) | +| `max_split_size_mb:128` | ❌ **remove** — blocks legitimate >128 MB allocations → spurious OOM | + +> ⚠️ ↩️ **`expandable_segments:True` flip-flopped three times — current verdict is OFF.** +> This knob has the least stable story of anything here, so treat it as unsettled rather than +> a clean ✅: +> 1. **First pass** — reported "unsupported, silently ignored" on gfx1201 → don't use. +> 2. **Second pass** — re-added in a later session, appeared to help (released freed BF16 shards +> from VRAM during load) → use it. *(Earlier versions of this doc reported this as the +> resolved state.)* +> 3. **Latest hands-on (most recent)** — **removed again**: the HIP allocator's +> "not supported" warning **preceded GPU page faults at step 16**, so it was stripped from +> all `PYTORCH_HIP_ALLOC_CONF` entries as a precaution. +> +> The page-fault correlation is suggestive, not proven causation — but the **latest empirical +> call is to leave it off** and run `garbage_collection_threshold:0.8` alone. If you try it, +> watch for a step ~10–20 page fault. + +--- + +## 6.1 The non-deterministic backward page fault (PR #101) — what we know, and the ablation to settle it ⚠️ + +> **Live per-run tracker: [`GFX1201_FAULT_ABLATION_LOG.md`](GFX1201_FAULT_ABLATION_LOG.md)** — +> the canonical, append-one-row-per-run log (commit hash, variables, fault step, kernel, verdict). +> It **supersedes** this section where they disagree: the fault is now localized to a **rocBLAS +> Tensile GEMM (`MT64x64x64 ISA1201`)**, not the bitsandbytes path — so the "bnb-NF4 backward" +> framing below and **Arm A are retired**, and hipBLASLt is open again as a *fix* (`=1`), having +> only been ruled out as a *cause* (`=0`). + +Gemma 4 31B QLoRA on the R9700 hits a **non-deterministic GPU page fault during the backward +pass** and has never completed a full run. This section is the durable record so we stop +re-deriving it. **Do not "fix" it with another single knob flip** — four have been tried and +falsified. + +``` +Memory access fault by GPU node-1 on address 0x7f....... Reason: Page not present or +supervisor privilege. (amdgpu gfxhub TCP fault; PERMISSION_FAULTS:0x3, WALKER_ERROR:0x0) +``` + +**The one fact that reframes everything (wandb-verified):** the fault is *probabilistic, not +config-determined.* The **same config on the same git SHA both finishes and crashes**: + +| git SHA | run | outcome | +|---|---|---| +| `c752a2b4` | eos-gate g2df2ifl | **finished 100 steps** | +| `c752a2b4` | eos-gate gv9fbjac | **crashed < step 10** | +| `eb12dbd9` | eos-gate 5xd8qt5w | **finished 100 steps** | +| `eb12dbd9` | eos-gate irgdklt9, jtyyhu4t | **crashed @ step 10** | + +Crash steps across all Stage 2 + eos-gate runs: 10, 14, 16, 20, 80, 84, or clears 100 — a +random draw, not a threshold. So a "good config" cannot be inferred from one clean run. + +**What the wandb data rules OUT** (don't re-investigate these): +- **Not numerical / data / a bad batch.** Loss descends smoothly (2.6→0.8), grad_norm stays + 1–6, no NaN/Inf; two same-LR runs log *byte-identical* losses per step (deterministic forward). +- **Not LR / optimizer magnitude.** A `lr=5e-6` run crashed *earlier* (step ~10) than the + `lr=5e-5` runs (step ~80) — lower LR should be safer; this inverts the theory. +- **Not hipBLASLt.** `TORCH_USE_HIPBLASLT=0` runs crash with the identical signature. +- **Not sequence length.** Measured token lengths (`scripts/determine_max_sequence.py`): + **train max = 909, eval max = 892, p99 = 673** over socrat-zh-sft + socrat-en-sft. Nothing + exceeds 1024. With `per_device_batch_size=1` + dynamic padding the activation peak is set by + the *actual* longest sequence (909), **not** by `max_length`, so lowering the 1280 cap frees + no VRAM. `max_length=1280` is non-binding and safe; leave it. + +**Still unsettled (do NOT treat as ruled out):** the GC threshold. The `72db9b4` run with +`garbage_collection_threshold` *removed* also crashed, but its log was lost (wandb 404) so +page-fault-vs-OOM was never confirmed, and it is n=1. GC-on vs GC-off is therefore not yet a +clean variable either way — replicate it under the protocol below (arm D). + +**Enabling condition:** the run sits at **93–98 % VRAM (~30–31.4 / 32 GB) sustained**. The fault +is in the backward pass, in the immature **bitsandbytes-NF4 / ROCm 7.2** path — the least-tested +part of the stack (§3 only smoke-tested bnb a few steps, never 100+ under pressure). The fault +address has a host-VA `0x7f…` pattern, consistent with a use-after-unmap / bad-buffer access. + +**Two failure modes that COMPOUND (operational, not config):** +1. **Dirty-KFD cascade.** A fault leaves orphaned HIP context + stale VRAM (§10). Relaunching + into it faults early on stale PTEs. The resume monitor must verify a clean GPU + (`test_gpu_stack.sh --wait-clean`), not blind-sleep, before each relaunch. +2. **Corrupt checkpoint.** `save_steps=10` (the crawl-forward survival patch) multiplies + checkpoint-write windows 10×; a fault mid-write leaves an incomplete `checkpoint-N` that + resume loops on. The monitor quarantines an incomplete latest checkpoint (no valid + `trainer_state.json`) so resume falls back to `N-1`. + +`save_steps=10` + the monitor crawl is a **survival workaround, not a fix** — it makes a faulting +run inch forward (confirmed: a run crashed at step 16 but left a resumable `checkpoint-10`). The +fix still has to come from the ablation below. + +### Localize first, then ablate — `not guessing` + +**Phase 0 — localize (`make diagnose-gfx1201-fault`).** The fault is async, so the kernel +"running" at fault time is not necessarily the culprit — every theory so far was correlation. +The diagnostic runs ~120 steps under `AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1 +AMD_LOG_LEVEL=1` + a `dmesg` tail, so the fault becomes synchronous and the traceback/ring-log +**name the faulting kernel**. (Level 1 = errors only; do NOT use level 3 — it logs every HIP +call and writes multi-GB `diag.log` that can fill the disk. `LOG_LEVEL=3 make …` opts in.) +If serialization makes it vanish → concurrency/allocator race; if +it still faults at the same named kernel → kernel bug (bnb dequant vs grad-ckpt recompute vs +allocator). This decides which arms below matter. + +**Phase 1 — ablation matrix.** Because the fault is probabilistic, each cell needs **N=3 +replicates** and a fixed **≥150-step budget** (clears the 14–84 window with margin), each behind +`make gpu-preflight` (clean GPU) with a fixed seed. Metric per cell: fraction clearing the +budget + median crash-step. Change exactly **one** factor from baseline (HIPBLASLT=0, GC=0.8, +grad-ckpt `use_reentrant=False`, bnb 0.49.2, sdpa, seq 1280, bs 1×16): + +| Arm | One change | Hypothesis | Note | +|---|---|---|---| +| A | bitsandbytes **source build** `-DBNB_ROCM_ARCH=gfx1201` | NF4 backward kernel bug | top suspect if Phase 0 names a bnb kernel | +| B | `TRAIN_GRAD_CKPT=false` | recompute drives the faulting dequant | likely **OOMs** at 98 % VRAM — diagnostic, not a candidate fix | +| C | grad-ckpt `use_reentrant=True` | non-reentrant checkpoint hook race | one-line change in `train_sft.py` | +| D | `PYTORCH_HIP_ALLOC_CONF` unset | allocator GC/unmap race | GC-off already crashed once (n=1) — needs replication | +| E | `HSA_ENABLE_SDMA=0` | DMA-engine page-fault mitigation | cheap, stackable | +| F | bump/pin ROCm or torch build | driver/runtime immaturity | last resort; expensive | + +Stop-rule: the first arm that is **3/3 clean over 150 steps** becomes the new baseline; keep +stacking from there. The survival workaround stays on throughout — it is how the run makes +progress while the ablation finds the real fix. + +> Tooling for this section lives in: `make diagnose-gfx1201-fault` +> (`scripts/diagnose_gfx1201_fault.sh`), `make gpu-preflight` +> (`scripts/test_gpu_stack.sh --preflight` / `--wait-clean`), +> `scripts/determine_max_sequence.py`, and the hardened `scripts/monitor_stage2.sh`. + +--- + +## 7. `HSA_OVERRIDE_GFX_VERSION` — do not set it ↩️ + +Half the older guides call `HSA_OVERRIDE_GFX_VERSION=11.0.0` "the critical fix." That value is +the **RDNA 3** identifier — it lies to ROCm that the card is RDNA 3, and on a real gfx1201 it +**hangs the GPU**. **ROCm 7.2 supports gfx1201 natively, so no override is needed at all.** If +some tool ever forces one, the *correct* gfx1201 value is **`12.0.1`** — never `11.0.0`. Most of +the "gfx1201 unsupported / HSA override" material predates ROCm 7.2 and is stale. + +--- + +## 8. The `causal-conv1d` "NVCC trap" — an install-method artifact, not a wall ✅ + +Two theories circulated about what blocks the linear-attention path's short causal conv on +gfx1201. They reconcile into one fact: **it's the install method.** + +| Install method | Outcome | +|---|---| +| `pip install causal-conv1d` (isolated build) | ❌ **NVCC trap** — the isolated build env resolves **CUDA** torch from PyPI, shells out to `nvcc -V`, and PyTorch's extension builder injects `-gencode arch=compute_80`, which `amdclang++` rejects | +| `uv pip install --no-build-isolation causal-conv1d` (+ `wheel`) | ✅ builds against the **ROCm** torch already in your venv → **1.6.2.post1**, fwd+bwd verified | +| [EmbeddedLLM/causal-conv1d-rocm](https://github.com/EmbeddedLLM/causal-conv1d-rocm) | ✅ clean fork — `setup.py` gates on `torch.version.hip` | + +`--no-build-isolation` isn't a hack: it routes the build down AMD's official **`hipify_torch`** +path (PyTorch's C++ extension builder translates the CUDA source to HIP at build time — no nvcc, +no Triton — *when the build sees a ROCm torch*). This is exactly the mechanism in AMD's own +[Vision Mamba](https://rocm.blogs.amd.com/artificial-intelligence/vision-mamba/README.html) +recipe. **Never run plain `pip install causal-conv1d` in a ROCm venv.** + +> The AMD Vision Mamba blog validates the hipified C++ `causal_conv1d`/`selective_scan` build on +> **CDNA (Wave64), ~2024 stack** — never on gfx1201/Wave32. That's *why* the RDNA 4 community +> routes the linear path through FLA's per-arch-JIT **Triton** kernels instead. The hipified C++ +> path is a third candidate ⏳, but expect Wave64-tuning risk. + +--- + +## 9. FP8 and vLLM — the serving-side gfx1201 gaps 🔭 + +Not training blockers, but part of the gfx1201 picture: + +- **FP8 (E4M3) silently falls back to FP32** — gfx1201 is missing from AITER's arch table + ([ROCm/TransformerEngine#520](https://github.com/ROCm/TransformerEngine/issues/520)); a + two-line fix exists, unmerged. Throughput ~halves. Affects FP8 serving, not LoRA training. +- **vLLM doesn't recognize gfx1201 upstream** + ([vllm#28649](https://github.com/vllm-project/vllm/issues/28649)) — serve via HF Transformers + instead. See [`GPU_SUPPORT.md`](https://github.com/ulises-c/csen-346/blob/main/docs/GPU_SUPPORT.md). + +--- + +## 10. Operational gotchas (cost you hours, not in any NVIDIA guide) ✅ + +- **Always `uv run --no-sync`.** A bare `uv run` triggers `uv sync`, which reinstalls **CUDA** + torch from PyPI over your hand-installed `torch+rocm`. Every training/inference command needs + `--no-sync`. (And re-run the FLA patch from §1 after any sync that does slip through.) +- **GPU state corruption cascades.** A GPU page-fault (`rc=134`) leaves the KFD in a dirty + state; *subsequent* runs — even with a correct config — fault at random steps until you + `kill -9` the process and confirm a clean GPU. Pre-flight every run: + ```bash + rocm-smi --showpids | grep -q python && echo "GPU dirty — kill before retrying" || echo "GPU clean" + ``` +- **Use `setsid`, not `nohup`, for the training wrapper.** `nohup bash run.sh &` doesn't create + a new process group, so children survive `kill -9 -- -$PGID`. `setsid bash run.sh &` puts the + whole tree in its own group → kill it cleanly and avoid overlapping processes on the same GPU. + +--- + +## Confirmed version matrix + +| Component | Confirmed-working (this box) | Community alt | Notes | +|---|---|---|---| +| ROCm | 7.2 (HIP 7.2.26015) | 7.1 | 7.2 = native gfx1201, no HSA override | +| PyTorch | **2.11.0+rocm7.2** | 2.9.1 (stable) | nightly **not** needed for FLA | +| Triton | **3.6.0 + `num_stages=1` patch** | **3.5.1** (no patch) ⏳ | 3.6.0 has the UAF | +| flash-linear-attention | 0.5.0 (patched) | bare-metal Triton | hangs unpatched | +| causal-conv1d | 1.6.2.post1 (`--no-build-isolation`) | EmbeddedLLM rocm fork | fwd+bwd verified | +| bitsandbytes | 0.49.2 (stock PyPI) | source `-DBNB_ROCM_ARCH=gfx1201` | 4-bit NF4 runtime ✅ | +| flash-attn (FA2) | Triton backend (`..._TRITON_AMD_ENABLE=TRUE`) or SDPA | — | CK backend won't compile (Wave32) | + +--- + +## "Free on NVIDIA, costs a workaround here" — the one-screen summary + +1. **Linear-attention fast-path** → patch Triton `num_stages→1` or pin 3.5.1. +2. **FA2** → flip to the Triton backend, or just use SDPA. +3. **Large 4-bit load** → use a pre-quantized checkpoint (the CPU-RAM staging wall is the real + blocker, not VRAM). +4. **`device_map`** → force `{"": 0}`; guard the CUDA-only allocator warmup. +5. **hipBLASLt** → off for delta-rule archs; test-then-keep for pure softmax. +6. **Allocator** → `garbage_collection_threshold:0.8`; never `max_split_size_mb`; `expandable_segments:True` is contested — default off (§6). +7. **No HSA override** on ROCm 7.2. +8. **Build CUDA-source extensions with `--no-build-isolation`** (hipify path); never bare `pip install`. +9. **`uv run --no-sync`**, `setsid` wrappers, and a GPU-clean pre-flight. + +Once past these, a 32 GB RDNA 4 card trains modern 27–31B-class models at 4-bit QLoRA with VRAM +to spare. The gap to NVIDIA is **ecosystem maturity (Triton/CK/vLLM target coverage), not +silicon capability.** + +--- + +## References + +**Source repo:** [github.com/ulises-c/csen-346](https://github.com/ulises-c/csen-346) — canonical +report, companion `docs/GPU_SUPPORT.md`, `scripts/patch_fla_rocm.sh`, and investigation history +([#100](https://github.com/ulises-c/csen-346/issues/100) · +[#79](https://github.com/ulises-c/csen-346/pull/79) · +[#109](https://github.com/ulises-c/csen-346/issues/109)). + +**Primary (hands-on):** debugging sessions on the R9700 across the issues/PRs that produced this +report; `scripts/patch_fla_rocm.sh`; [`GPU_SUPPORT.md`](https://github.com/ulises-c/csen-346/blob/main/docs/GPU_SUPPORT.md). + +**Upstream / community:** +- Triton `tritonamdgpu-pipeline` UAF (manifests via): [thu-ml/SageAttention#365](https://github.com/thu-ml/SageAttention/pull/365) · [kijai/ComfyUI-WanVideoWrapper#2007](https://github.com/kijai/ComfyUI-WanVideoWrapper/issues/2007) +- `causal-conv1d` NVCC trap: [Dao-AILab/causal-conv1d#99](https://github.com/Dao-AILab/causal-conv1d/issues/99) · [EmbeddedLLM/causal-conv1d-rocm](https://github.com/EmbeddedLLM/causal-conv1d-rocm) +- FA2 on RDNA: [ROCm/flash-attention](https://github.com/ROCm/flash-attention) · [Repeerc/flash-attention-v2-RDNA3-minimal](https://github.com/Repeerc/flash-attention-v2-RDNA3-minimal) · [HipKittens](https://github.com/HazyResearch/HipKittens) +- FP8 / serving: [ROCm/TransformerEngine#520](https://github.com/ROCm/TransformerEngine/issues/520) · [vllm#28649](https://github.com/vllm-project/vllm/issues/28649) +- AMD Vision Mamba (hipify mechanism): [rocm.blogs.amd.com/artificial-intelligence/vision-mamba](https://rocm.blogs.amd.com/artificial-intelligence/vision-mamba/README.html) +- gfx1201 community guides (apollo-mg): [Master guide (RDNA4 + ROCm 7.1)](https://gist.github.com/apollo-mg/ecba6a0c29323325a7ac3babf08e53be) · [FLA / RDNA4 config](https://gist.github.com/apollo-mg/d44cb753962fa9f6e1e45a7101f14284) · [The NVCC trap](https://gist.github.com/apollo-mg/e86abd863802bde296892fb1fe7aecae) diff --git a/docs/HANDOFF_101_TRAINING_READINESS.md b/docs/HANDOFF_101_TRAINING_READINESS.md new file mode 100644 index 00000000..4ca66145 --- /dev/null +++ b/docs/HANDOFF_101_TRAINING_READINESS.md @@ -0,0 +1,191 @@ +# Handoff — Get Stage-2b SFT 100% training-ready (closes #94, finishes #101) + +**Branch:** `feat/gfx1201-rdna4-qlora-fla-training` +**Goal:** get a Gemma 4 31B QLoRA Stage-2b run that will **not** repeat the #94 collapse, as the final step of PR #101. + +### Machine split (important) +- **You are on the NVIDIA dev box** (no gfx1201, can't load the 31B / can't train here). +- **Do here:** Phase 0 code change (§3), tests (CPU, fully mocked), `--dry-run` plumbing check, then **commit + push** — so the R9700 can pull. +- **Runs on the AMD R9700** (separate box, 32 GB, ROCm 7.2, gfx1201): the short-checkpoint **EOS gate** (§4) and the full QLoRA run (§5). You cannot execute those from here; leave them as the runbook for the R9700 session / human. + +> Read top-to-bottom once, then execute §3 (Phase 0) → §3c (tests) → commit+push. §4–§5 are the R9700 runbook. Verify line numbers against the live files before editing — they drift. + +--- + +## 1. What #94 was (the failure you're preventing) + +The first Stage-2b SFT model was **generatively broken**: on real eval prompts it never emitted EOS and degenerated into a repetition loop (`这样可以帮助他建立更强的数学基础` ×70+), running to the context limit (~45 min/call). Root cause = **train/serve schema mismatch**: the model was trained on a format that didn't match what `SocraticTeachingSystem.socrates_teacher()` sends at inference. Decisive A/B (same model/GGUF/server): training-shape prompt → 20 tokens + clean EOS; inference-shape prompt → 2048-cap repetition. + +The fix is to make the SFT training data render **identically** to the inference prompt. PR #101 fixed the *structural* part. Two residuals remain (Phase 0 below) — finishing them is what makes you "100% ready." + +--- + +## 2. Current state (already done — do NOT redo) + +Commits on the branch: +- `d04bc83` — structural schema-drift fix (`socrat-zh-sft` / `socrat-en-sft`: one `(system,user,assistant)` triple per dialogue turn in the inference shape; dialogue-level split) + ROCm/`device_map`/`quantization_config` fixes + W&B. +- `e5a8a7d` — `tests/test_sft_inference_format.py` (11 tests): render-diff gate that drives the **real** `socrates_teacher()` path and asserts the SFT record renders identically under the chat template's per-message `| trim`. +- `492b2a0` — pre-commit hook fix (use `python -m` so it works under `--no-sync`). + +Verified facts (don't re-investigate): +- The Gemma 4 training chat template applies `{{ content | trim }}` to every message (`scripts/train_sft.py:121,123`); the serving template trims too. So leading/trailing whitespace differences between train and serve are **immaterial** — render-equivalence reduces to `content.strip()` equality per message. +- The user message already matches inference byte-for-byte **except** the two residual lines below. +- Both inference paths are **Chinese-only** (`socrates_teacher` and the single-call `socratic_teaching_unified.py`); the consultant is Chinese-only. + +**Deferred — DO NOT touch for this task:** +- **#108** (bilingual EN/ZH train+serve) — large follow-up; explicitly out of scope. The `socrat-en-sft` loader stays Chinese-scaffolded for now. +- Do **not** modify `socratic_teaching_unified.py` or the `zh` inference rendering in any way that changes rendered tokens — it risks re-baselining the locked headline (unified 72.24) and Tables 6/14. +- **#30** in `ulises-c/Computer-Setup` (global hook fix) — informational only. + +--- + +## 3. Remaining work — Phase 0 (this is the rest of the #94 fix) + +The committed loader still drifts from inference on **two lines** of the user message. Both verified against real `ulises-c/SocratDataset` / `SocratDataset-EN` records. + +| Line | Loader emits today | Inference actually sends | Fix | +|---|---|---|---| +| `苏格拉底教学顾问建议的操作:` | dataset `turn["action"]` | `get_action_for_state(state)` (canonical Chinese map; `socratic_teaching_system.py:494`) | use `get_action_for_state(state)` — **byte-exact, deterministic** | +| `苏格拉底教学顾问评估结果:` | templated `学生处于 {state} 状态` | live free-form consultant prose | use the dataset's existing free-form `evaluation` field (closer shape) | + +Action drift is **systematic**: `a1` (every dialogue's first teaching turn) never matches (`生成一个问题` vs `生成一个与解题相关的子问题`), plus some c-states. The dataset **already carries** a per-turn `evaluation` field (both ZH and EN) — no re-annotation needed. + +### 3a. `src/project/socratic_teaching_system.py` — extract a single source of truth + +Move the `state_to_action` dict out of `__init__` to module level so the loader can import it (behavior-preserving): + +```python +# after `import openai` +_DEFAULT_ACTION = "继续提问" + +STATE_TO_ACTION = { + "a0": "引导学生提出问题", + "a1": "生成一个与解题相关的子问题", + # ... copy the FULL existing map verbatim (a0..e34) ... + "e34": "对题目进行总结", +} + +def get_action_for_state(state: str) -> str: + return STATE_TO_ACTION.get(state, _DEFAULT_ACTION) +``` + +Then in `__init__` replace the inline dict with: +```python + self.state_to_action = STATE_TO_ACTION +``` +Leave the existing `get_action_for_state(self, state)` method as-is (still delegates via `self.state_to_action`). No inference behavior changes. + +### 3b. `src/project/dataset.py` — use the canonical action + free-form eval + +Change `_build_inference_user_message` to take `evaluation` instead of `state`: +```python +def _build_inference_user_message(history_turns, student_input, evaluation, action): + ... + return ( + f"\n历史对话记录:\n{formatted_history}\n\n" + f"当前学生输入: {student_input}\n\n" + f"苏格拉底教学顾问评估结果: {evaluation}\n" + f"苏格拉底教学顾问建议的操作: {action}\n" + ) +``` + +In `_socrat_zh_to_sft_records` (and the EN twin): +```python + from src.project.socratic_teaching_system import get_action_for_state + ... + state = _strip_quotes(turn.get("state", "")) # EN: turn.get("state", "") + if not (state and _strip_quotes(turn.get("action", ""))): # keep target-turn gate on dataset annotation + history.append((turn["student"], turn["teacher"])) + continue + action = get_action_for_state(state) # match inference exactly + evaluation = _strip_quotes(turn.get("evaluation", "")) or f"学生处于 {state} 状态" # EN: drop _strip_quotes + user_msg = _build_inference_user_message(history, turn["student"], evaluation, action) +``` +Keep gating target-turn selection on dataset `state`+`action` presence so the set of training records is unchanged. + +### 3c. Update `tests/test_sft_inference_format.py` (un-rig the action dimension) + +The current render-diff gate **passes the dataset action straight into inference**, so it doesn't actually test the action line. After the fix: +1. Add an `evaluation` field to the `_dialogue_record` fixture turns. +2. In the parity test, build the captured inference prompt with `action = get_action_for_state(state)` and `evaluation = ` — assert system **and** user render-equal (now including the action line → zero drift). +3. The freeform-eval residual test: pass a *different* free-form eval → assert the **only** drifting line is `苏格拉底教学顾问评估结果:`. +4. Update unit-test assertions that hardcode `学生处于 a1 状态` / `action_0` to the new outputs (eval = fixture text; action = `get_action_for_state("a1")` = `生成一个与解题相关的子问题`). + +Run: `uv run --no-sync python -m pytest tests/test_sft_inference_format.py -q` (must be green). + +> Note: this exact Phase-0 change was prototyped and reverted earlier in the session; it is well-understood and low-risk. The `zh` inference path stays byte-rendered-identical (trim). + +--- + +## 4. HARD GATE before the full run — faithful EOS check *(runs on the R9700, not the dev box)* + +After Phase 0, train a short checkpoint, then prove the model terminates on a **real inference-shaped prompt** before trusting the multi-hour run. + +**Critical:** the gate prompt must use a **live free-form consultant evaluation**, NOT the templated `学生处于 {state} 状态`. A gate built on the template is blind to the eval-line residual and will pass while shipping a broken model. + +Two ways, gold first: +1. **Gold:** serve the trained adapter (merge → GGUF → llama.cpp, same path as eval) and run ONE real turn end-to-end through `SocraticTeachingSystem.process_student_input` (consultant produces the real eval → teacher sees the true inference prompt). Confirm the teacher response emits EOS, is short (≈1 question, <~150 tokens), and does not loop. +2. **Lighter:** capture the inference prompt the way `tests/test_sft_inference_format.py::_capture_inference_prompt` does, but with a realistic multi-sentence Chinese `evaluation`, run HF `model.generate` on the adapter, and confirm clean EOS / no repetition. + +Only after the gate passes → commit to the full run. + +--- + +## 5. Pre-launch + launch + monitor *(runs on the R9700, not the dev box)* + +```bash +# 1. GPU must be clean — a prior rc=134 fault leaves the KFD dirty and cascades into random faults +rocm-smi --showpids | grep python && echo "DIRTY — kill -9 and recheck" || echo "clean" + +# 2. Dry-run plumbing (optional sanity) +make train-gemma4-31b-dry-run + +# 3. Launch (prequant unsloth bnb-4bit; skips the ~62 GB BF16 CPU staging) +make train-gemma4-31b-stage2-unsloth +# target already sets TORCH_USE_HIPBLASLT=0 and +# PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,expandable_segments:True + +# 4. Monitor +tail -f outputs/sft-stage2-gemma4-31b/train.log +``` + +**Recompute the run size before launching** — the config comment ("~2298 steps / 23 checkpoints / ~11 GB") is **stale** (per-dialogue era). The per-turn format is ~77k train records → at `bs1 × ga16 × 3 epochs` expect **~14k steps** (multi-day; matches the "~55h" figure in the PR, not the ~12h the stale comment implies). With `TRAIN_SAVE_STEPS=100` that's **~140+ adapter checkpoints (~70 GB)** — raise `TRAIN_SAVE_STEPS` and/or set a `save_total_limit` to avoid filling disk. + +Crash recovery: rc=134 → `kill -9 `, confirm `rocm-smi --showpids` empty before retry; consider `setsid` so the whole tree is killable as one group. + +Optional perf: targets default `TORCH_USE_HIPBLASLT=0` (safe). #99 found Gemma 4 may tolerate `=1` (~4× matmul, no linear-attention crash path) — test 20+ steps on a clean GPU before trusting it. + +--- + +## 6. Decisions for the human (surface; don't assume) + +1. **Apply Phase 0?** Recommended/required for "100% ready" — removes both residuals. Alternative: train as-is and rely solely on the §4 EOS gate (defensible only if the gate uses a free-form eval; the action line still drifts on every opening turn). +2. **`TRAIN_SAVE_STEPS` / disk** — raise it given ~14k steps (see §5). +3. **hipBLASLt on/off** (perf vs. safety). +4. **2b-only vs 2a→2b** — config defaults to 2b (structural); escalate only if 2b underperforms (see config header / `TRAINING_PLAN.md §4`). + +--- + +## 7. Acceptance checklist ("100% ready") + +**On this dev box (you):** +- [ ] Phase 0 applied (3a, 3b); `socrat-*-sft` action line = `get_action_for_state(state)`, eval line = dataset `evaluation`. +- [ ] `uv run --no-sync python -m pytest tests/test_sft_inference_format.py -q` green, with the action dimension un-rigged and eval-line isolated as the sole residual. +- [ ] Full suite green: `uv run --no-sync python -m pytest -q` (expect 154 passed / 2 skipped + your updates). +- [ ] `zh` inference rendering unchanged (no re-baseline) — captured `socrates_teacher` prompt unchanged for a known turn. +- [ ] `make train-gemma4-31b-dry-run` passes (plumbing). +- [ ] **Commit + push** Phase 0 to `feat/gfx1201-rdna4-qlora-fla-training` so the R9700 can pull. → then it's 100% ready to train. + +**On the R9700 (separate session/human):** +- [ ] GPU clean (`rocm-smi --showpids`). +- [ ] Run size + `SAVE_STEPS`/disk recomputed (§5). +- [ ] Short checkpoint trained → **EOS gate passed with a free-form eval** (§4). +- [ ] Launch the full run. + +## 8. File map +- Inference teacher prompt: `src/project/socratic_teaching_system.py` (`socrates_teacher` ~400-477; system ~404-415; user ~445-453; action `get_action_for_state` / `state_to_action` ~52-88, ~396-398; `get_formatted_history` ~120-134). +- SFT loaders: `src/project/dataset.py` (`socrat-zh-sft`/`socrat-en-sft`, `_build_inference_user_message`, `_socrat_*_to_sft_records`). +- Chat template (trim proof): `scripts/train_sft.py:116-129`. +- Test gate: `tests/test_sft_inference_format.py`. +- Config: `configs/train-sft-stage2-gemma4-31b.env`; launch targets: `Makefile` (`train-gemma4-31b-stage2-unsloth`). +- Deferred design: #108 (bilingual). PR: #101. Original failure: #94. diff --git a/docs/HANDOFF_GFX1201_BUCKET2.md b/docs/HANDOFF_GFX1201_BUCKET2.md new file mode 100644 index 00000000..c2403d3c --- /dev/null +++ b/docs/HANDOFF_GFX1201_BUCKET2.md @@ -0,0 +1,170 @@ +# Handoff — gfx1201 ISA1201 Tensile GEMM: all fix arms exhausted, crawl phase + +> **⚠️ SUPERSEDED (2026-06-01).** This is a point-in-time snapshot from when Bucket #2 was +> first confirmed and its "YOUR TWO TASKS" (forward kernel name + `expandable_segments` test) +> were still pending — **both are now done** (runs #9 and #10), and runs #11/#12 followed. +> For the current state and the live consultant/reviewer brief, see +> **`docs/HANDOFF_GFX1201_CONSULTANT.md`**; the canonical run log is +> **`docs/GFX1201_FAULT_ABLATION_LOG.md`**. Kept below as history. + +**Date:** 2026-06-01 · **Branch:** `feat/gfx1201-rdna4-qlora-fla-training` · **PR:** #101 +**For:** a fresh Claude instance picking up from a confirmed root cause. + +--- + +## TL;DR — where we are + +All software fix arms are exhausted. No userspace workaround is available on gfx1201 / ROCm 7.2. +The only remaining path to the adapter is the **brute-force checkpoint+resume crawl**. + +**Before committing to the crawl**, one measurement is needed: **production steps-per-resume** +(no SYNC, no probes). This is the input to the crawl go/no-go arithmetic. + +GPU is **clean**. Branch is **clean and pushed**. No training is running. + +--- + +## Root cause — fully closed (do not re-investigate) + +ISA1201 Tensile GEMM `MT64x64x64` computes a wild address from its column-major B operand +descriptor. Occurs in both the grad-checkpoint recomputed forward GEMM and the backward GEMM. +The col-major B layout (`DTVB1`, `stride=(1,21504)`) is intrinsic to how PyTorch+bitsandbytes +structures the 4-bit matmul call — it cannot be changed from userspace without modifying +bitsandbytes internals. + +**Faulting kernel (full ShaderName):** +``` +Cijk_Ailk_Bjlk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x64_MI16x16x1_SN_LDSB0_AFC1_AFEM1_AFEM1_ASEM1_CLR1_CADS0_DTLA0_DTLB0_DTVA0_DTVB1_EPS0_FDSI0_GRPM1_GRVWA8_GRVWB8_GSUAMB_GLS0_ISA1201_IU1_K1_LDSTI0_LBSPPA1024_LBSPPB0_LBSPPM0_LPA32_LPB0_LPM0_LRVW8_LWPMn1_MIAV1_MIWT2_2_MO40_NTn1_NTA0_NTB0_NTC0_NTD0_NTM0_NEPBS0_NLCA1_NLCB2_ONLL0_PGR2_PLR1_PKA0_SIA3_SS0_SPO0_SRVW0_SSO0_SVW8_SK0_SKFTR0_SKXCCM0_TLDS0_ULSGRO0_USL1_UIOFGRO0_USFGROn1_VSn1_VWA2_VWB1_WSGRA0_WSGRB0_WS32_WG32_4_1 +``` + +--- + +## All fix arms — DONE + +| Run | Arm | Result | +|---|---|---| +| #10 | Arm D — `expandable_segments:True` | silently ignored on gfx1201; fault persists | +| #11 | Arm HLT1 — `TORCH_USE_HIPBLASLT=1` | hipBLASLt has no `MT64x64x64 DTVB1` kernel; falls back to Tensile; fault persists | +| #12 | Contiguous-B — `BNB_FORCE_B_CONTIGUOUS=1` | Python `.contiguous()` doesn't reach BLAS descriptor; `DTVB1` intrinsic to call structure; fault persists | + +Full evidence in `docs/GFX1201_FAULT_ABLATION_LOG.md`. Do not re-investigate any of these. + +--- + +## YOUR ONE TASK — production steps-per-resume measurement + +### Why this is needed + +All SYNC-mode runs (AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1) fault deterministically +at step 21 (first step after ckpt-20 resume). **SYNC mode is not representative.** Run #2 +showed the fault is probabilistic without serialization — same SHA sometimes clears 100 steps. +The crawl viability depends entirely on how far production mode actually gets. + +### Launch command (run #13) + +```bash +make gpu-preflight # MUST pass + +CKPT=outputs/sft-stage2-gemma4-31b +nohup env TORCH_USE_HIPBLASLT=0 \ + HSA_ENABLE_SDMA=0 \ + AMD_LOG_LEVEL=1 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit TRAIN_PREQ=true \ + TRAIN_SAVE_STEPS=5 TRAIN_MAX_STEPS=300 TRAIN_OUTPUT_DIR="$CKPT" \ + uv run --no-sync python scripts/train_sft.py --config configs/train-sft-stage2-gemma4-31b.env \ + > "$CKPT/prod-measure.log" 2>&1 & +echo "PID: $!" +``` + +Key differences from all prior diagnostic runs: +- **No `AMD_SERIALIZE_KERNEL` / `HIP_LAUNCH_BLOCKING`** — production mode, probabilistic fault +- **`HSA_ENABLE_SDMA=0`** — free stackable (Arm E), low risk, may help +- **`TRAIN_SAVE_STEPS=5`** — checkpoint frequently so progress is banked +- **`TRAIN_MAX_STEPS=300`** — run long enough to observe multiple fault cycles if they happen + +### What to watch for + +Monitor `prod-measure.log` for: +- `{'loss': ...}` lines — each is one step completed +- `Memory access fault` — a fault event; count the step from the last `{'loss'` line +- After each fault: **immediately run `make gpu-preflight`** before restarting (dirty KFD cascades) + +If it faults, note the step number, then resume: +```bash +make gpu-preflight +nohup env ... uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + > "$CKPT/prod-measure-resume-1.log" 2>&1 & +``` + +Collect 3–5 crash cycles. Record: step-at-fault for each crash. + +### Go/no-go arithmetic (corrected per consultant review) + +- **Total steps needed:** ~4,800 (1 epoch at batch=1×16=16, 12,244 records / 16 ≈ 765 steps/epoch × 3 epochs... wait, check the config. `TRAIN_MAX_STEPS` in `configs/train-sft-stage2-gemma4-31b.env` governs this — verify the actual value.) +- **Per-crash wall-clock cost** = KFD-clean wait (~2–5 min) + 19 GB model reload (~2–3 min) + preflight. Budget ~8–10 min/crash, not just reload time. +- **Checkpoint corruption risk:** at `save_steps=5` near a fault step, a save in progress during a fault may produce a corrupt checkpoint. Guard against this: after each crash, verify the latest checkpoint is loadable before resuming. + +**Crawl viable if:** mean steps-per-crash is high enough that `(total_steps / mean_steps) × 10min/crash` is acceptable wall-clock overhead. + +**Crawl NOT viable if:** production faults every 5–15 steps consistently — that's 320–960 crash cycles for a full run, 53–160h of pure overhead. In that case an upstream fix is mandatory and the adapter cannot be produced on this hardware without it. + +--- + +## After the measurement — update ablation log + post PR #101 + +Once you have 3–5 crash datapoints: + +1. Update `docs/GFX1201_FAULT_ABLATION_LOG.md` — add run #13 row (or multiple rows for each resume) +2. Post to PR #101 with: observed fault steps, mean steps-per-crash, projected wall-clock, and go/no-go verdict +3. If go: launch the full crawl with `TRAIN_MAX_STEPS` unset (or set to full epoch count), `TRAIN_SAVE_STEPS=5`, `HSA_ENABLE_SDMA=0` +4. If no-go: post to PR and document that on-device training is blocked pending upstream fix + +--- + +## Upstream bug report — status + +The rocBLAS/Tensile upstream filing has all necessary info: +- Full ShaderName (above) +- Operand descriptors (A: `shape=(1,608,21504)` contig row-major; B: `shape=(21504,5376)` col-major `stride=(1,21504)`) +- Fault addr: `0x7f6459a00000` (~1 GB below A.ptr, ~1.2 GB above B.end) +- Reproduced with `AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1` (not a race) +- Architecture: gfx1201 (RDNA4/Navi 48), ROCm 7.2, torch 2.11.0+rocm7.2 +- Environment report: `docs/diagnostics/gfx1201-report-env-20260601-191024.txt` +- File at: https://github.com/ROCm/rocm-libraries/issues (component: rocBLAS / Tensile) +- Note for filing: col-major B is intrinsic to PyTorch+bitsandbytes 4bit matmul; no userspace fix + +If the filing hasn't been submitted yet, do it before or alongside the measurement run — it's +independent and doesn't block anything. + +--- + +## Environment + +- HW: AMD Radeon AI PRO R9700, gfx1201 (RDNA4), 32 GB VRAM; Ryzen 9 5900X +- SW: ROCm 7.2, torch 2.11.0+rocm7.2, bitsandbytes 0.49.2, transformers 5.9.0, peft 0.19.1, trl 1.4.0, Python 3.12 +- Model: `unsloth/gemma-4-31B-it-unsloth-bnb-4bit` (pre-quantized NF4, ~19 GB, cached) +- **Always `uv run --no-sync`** — bare `uv run` reinstalls CUDA torch over ROCm torch +- After ANY fault: `make gpu-preflight` before next launch +- Latest checkpoint: `outputs/sft-stage2-gemma4-31b/checkpoint-20/` (step 20) +- Training config: `configs/train-sft-stage2-gemma4-31b.env` + +## Key files + +| File | Purpose | +|---|---| +| `docs/GFX1201_FAULT_ABLATION_LOG.md` | Canonical run log — update after every run | +| `scripts/train_sft.py` | Training script; `BNB_DEQUANT_PROBE` + `BNB_FORCE_B_CONTIGUOUS` blocks at line ~547 — leave in, gated by env var | +| `docs/diagnostics/gfx1201-report-env-20260601-191024.txt` | Env report for upstream filing | +| PR #101 comments | Full diagnostic thread + all operator findings | + +## Commit history (this branch, session 3) + +``` +0c763e0 new gfx1201 report generated +4167d84 chore(diag): env-collector writes to docs/diagnostics/; trim output noise +afa46e0 gfx1201 report generated +78e224d feat(diag): BNB_FORCE_B_CONTIGUOUS probe + ablation log runs #11/#12 +154aa8b feat(diag): env-collector script for the gfx1201 rocBLAS bug report +``` diff --git a/docs/HANDOFF_GFX1201_CONSULTANT.md b/docs/HANDOFF_GFX1201_CONSULTANT.md new file mode 100644 index 00000000..c8f46d63 --- /dev/null +++ b/docs/HANDOFF_GFX1201_CONSULTANT.md @@ -0,0 +1,93 @@ +# Handoff — gfx1201 page-fault investigation: consultant / reviewer brief + +**Date:** 2026-06-01 · **Branch:** `feat/gfx1201-rdna4-qlora-fla-training` · **PR:** #101 +**For:** a new Claude session acting as the **consultant / reviewer** (not the operator). + +--- + +## Your role + +The diagnostic phase is **closed** — root cause is confirmed and all software mitigations are +exhausted. You are the reviewer/consultant, continuing the "Consultant Review" thread on PR +#101. Your job is judgment, not re-running experiments: + +- **Review before outward-facing actions.** The upstream bug filing is the big one — sanity-check + the draft for over-claims before it leaves the building. +- **Weigh in on the open strategic fork** (crawl on-box vs. relocate to NVIDIA). +- **Catch errors / stale claims.** Hold the line between *observed* and *proven*. +- **Do NOT** re-open ruled-out hypotheses or re-run probes. Do NOT silently switch a conclusion + the operator reached with evidence — surface the conflict instead. + +## One-paragraph state + +A reproducible GPU page fault ("page not present", host-VA ~1 GB from any operand) is caused by a +**rocBLAS Tensile GEMM kernel `…_MT64x64x64_…_DTVB1_…_ISA1201`** computing an out-of-bounds +address on a **column-major B** operand, on **gfx1201 / RDNA4 (R9700), ROCm 7.2.3**. It faults in +both the backward pass and the grad-checkpoint forward recompute (same tile), survives full +serialization (not a race), and reproduces on a clean GPU. **No userspace workaround exists** in +ROCm 7.2 — hipBLASLt has no kernel for this shape (falls back to Tensile), allocator/placement +knobs are no-ops, and the column-major B is intrinsic to the `A @ Wᵀ` call structure (can't be +changed from Python). The only fix is upstream. + +## System of record — read these, in order + +| File / location | What it is | +|---|---| +| `docs/GFX1201_FAULT_ABLATION_LOG.md` | **Canonical** run log (runs #1–#12), ruled-out table, pending arms. Source of truth — if anything disagrees with it, it wins. | +| Issue **#113** (latest comment) | Ready-to-paste **upstream draft** for `ROCm/rocm-libraries`. The issue body is the internal system-of-record summary. | +| `docs/SFT_NVIDIA_MIGRATION.md` | The conditional recommendation to relocate the SFT run to NVIDIA. | +| PR **#101** comments | Full diagnostic thread + Consultant Reviews #1–#N (your predecessors). | +| `docs/diagnostics/gfx1201-report-env-*.txt` | Captured OS/driver/ROCm/stack env for the report. Regenerate with `scripts/collect-gfx1201-report-env.sh`. | +| `docs/HANDOFF_GFX1201_BUCKET2.md` | Superseded snapshot — history only. | + +## Settled — do NOT reopen (with the run that settled it) + +- Async race (run #3, serialized still faults) · OOM (within VRAM headroom) · bnb NF4 dequant + (succeeds every cycle) · undersized descriptor (fault ~1 GB from any boundary) · dirty-KFD + cascade (reproduces clean) · allocator placement (run #10) · **hipBLASLt routing** (run #11 — + no kernel for the shape) · **Python-level B layout** (run #12 — `.contiguous()` can't reach the + BLAS `transB` flag) · **Bucket #1 / activation-lifetime** (probe-3 — both operands valid). +- The `Cannot find the function ×6` log lines are **benign fallback-probe spam** (rocm-systems#6624), + not the bug. + +## Open items needing judgment + +1. **The goal fork (gating, unresolved — needs the human).** + - *Adapter is the deliverable* (feeds the locked 72.24 / Tables 6/14) → relocate SFT to NVIDIA + for a clean run; gfx1201 stays a documented contribution (#113 + #109). + - *"31B QLoRA end-to-end on RDNA4" is the claim* → finish on-box via brute-force crawl. + - Likely-non-binding caveat: data-residency (medical org) — but the data is HF-hosted Socratic + dialogues, no PHI, so cloud is probably fine. **Confirm before committing either way.** +2. **Brute-force crawl viability is UNMEASURED.** The deterministic ~step-21 faults are a + serialization artifact (SYNC + same ckpt-20); run #2 (no SYNC) was *probabilistic* (sometimes + cleared 100 steps). Before committing to a crawl, measure **production steps-per-resume** + (no SYNC, no probes) and price in the per-crash KFD-clean wait + lost-save risk. See PR #101 + consultant reviews for the corrected arithmetic (~4,800 steps/epoch, not the stale 2298). +3. **Upstream filing — one artifact left.** The `ROCBLAS_LAYER=2` `rocblas-bench` capture (the + standalone reproducer) — also the *only* remaining way to confirm the column-major-B trigger + (`--transposeB T` reproduces vs. `N`). Until captured, the report must say the trigger is + **observed, not proven**. File at **`ROCm/rocm-libraries`** (rocBLAS = `projects/rocblas/`, + Tensile = `shared/tensile/`); the standalone `ROCm/rocBLAS` & `ROCm/Tensile` repos are retired. + Related upstream issues to cross-reference: **#6166** (identical fault in rocBLAS unit tests — + strongest, AMD-runnable), #4097 (same kernel family, Windows), #7192 (same GPU, ruled out as + our mechanism). + +## Pending ablation arms (candidate mitigations, none run) + +`HSA_ENABLE_SDMA=0` (**held back** — sign unknown; run as a *separate* arm only if the pure‑production baseline is marginal, not stacked on it) · `TRAIN_GRAD_CKPT=false` +(diagnostic, likely OOMs) · `PYTORCH_TUNABLEOP_ENABLED=1` (speculative — selects on speed not +correctness). `use_reentrant=True` is **de-prioritized** (targets the refuted Bucket #1). + +## Operational gotchas + +- **`uv run --no-sync`** always — bare `uv run` reinstalls CUDA torch over the ROCm build. +- **`make gpu-preflight`** before every launch — a prior rc=134 leaves the KFD dirty and cascades. +- Use **`python -m pytest`**, not `pytest` (not a console entry point; the post-test hook breaks on it). +- Confirm before outward-facing actions (the upstream filing, posting comments). + +## What to do next + +Probe work is done; the next move is the **human's call on the goal fork**. If asked to consult: +(a) review the #113 upstream draft for over-claims before it's filed; (b) advise crawl-vs-NVIDIA +against the fork above; (c) if staying on-box, make sure the `ROCBLAS_LAYER=2` capture runs — it +completes the report *and* confirms the trigger in one shot. diff --git a/docs/HANDOFF_GFX1201_FAULT_DIAGNOSIS.md b/docs/HANDOFF_GFX1201_FAULT_DIAGNOSIS.md new file mode 100644 index 00000000..98cdcc28 --- /dev/null +++ b/docs/HANDOFF_GFX1201_FAULT_DIAGNOSIS.md @@ -0,0 +1,148 @@ +# Handoff — gfx1201 QLoRA backward page fault: diagnosis in progress (PR #101) + +**Date:** 2026-06-01 · **Branch:** `feat/gfx1201-rdna4-qlora-fla-training` · **Issue/PR:** #101 +**For:** a fresh Claude instance picking up the root-cause analysis cold. + +--- + +## TL;DR — where we are + +Gemma 4 31B QLoRA on an AMD R9700 (gfx1201 / RDNA4, 32 GB, ROCm 7.2) hits a +**GPU page fault during the QLoRA backward pass** and has never finished a run. +We have just produced the **decisive diagnostic result** but have **not yet read +the faulting kernel out of it.** That is your job. + +**Diagnostic result (just landed):** the serialized diagnostic run +(`make diagnose-gfx1201-fault`) **faulted at step 84** under +`AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1`. Two consequences: +1. **Serialization did NOT suppress the fault** → it is **not a pure async/stream + concurrency race**. Down-weight that flavor of the allocator hypothesis. +2. It faulted at **step 84 — the same step as the clean-start baseline runs** + (`6c9fouit`, `p9t3yxe5`). Strong evidence the fault is **reproducible at ~84 on + a clean GPU**, and the earlier "random" crashes at steps 10–20 were the + **dirty-KFD cascade** (faulting on stale page-table entries left by a prior + crash), not the true first fault. + +## YOUR IMMEDIATE TASK + +The diagnostic ran with `HIP_LAUNCH_BLOCKING=1`, so the Python traceback points at +the **exact** failing op. On the **AMD box** (this analysis can't be done from the +NVIDIA dev box — the log is on the R9700), extract it: + +```bash +# full synchronous traceback block: +ln=$(grep -n "Traceback (most recent call last)" outputs/diag-gfx1201/diag.log | tail -1 | cut -d: -f1) +sed -n "${ln},+80p" outputs/diag-gfx1201/diag.log + +# fault line + HIP error: +grep -niE "memory access fault|page not present|PERMISSION_FAULTS|HIP error|RuntimeError" \ + outputs/diag-gfx1201/diag.log | tail -20 + +# kernel-side VM-fault decode: +sudo dmesg | grep -iE "amdgpu|gfxhub|VM_L2|page fault|PERMISSION_FAULTS|WALKER_ERROR" | tail -30 +``` + +Note: that `diag.log` was produced by the **old** version of the diagnostic with +`AMD_LOG_LEVEL=3`, so it is **multi-GB** and full of HIP API spam — use `grep`/`sed`, +never open it whole. (The script now defaults to `AMD_LOG_LEVEL=1`; future runs +will be small.) + +### Decision tree — the last torch frame in the traceback picks the ablation arm + +| Last meaningful frame | Verdict | First ablation arm | +|---|---|---| +| `bitsandbytes` / `dequantize_4bit` / `Linear4bit` / `matmul_4bit` | NF4 backward kernel bug on gfx1201 | **A** — bitsandbytes **source build** `-DBNB_ROCM_ARCH=gfx1201` | +| `torch.utils.checkpoint` / recompute | grad-checkpoint recompute drives the bad kernel | **C** (`use_reentrant=True`), then **B** (`grad_ckpt=false`, expect OOM) | +| allocator / `empty_cache` / no clear op (fault between kernels) | memory **fragmentation** (NOT a race — serialization didn't help) | **D** — `PYTORCH_HIP_ALLOC_CONF` unset; consider `max_split_size` tuning | +| RoPE / attention / SDPA kernel | attention-path kernel on gfx1201 | investigate SDPA backend; secondary | + +If the traceback is ambiguous (async artifact despite blocking), re-run the +diagnostic — it's now cheap at level 1: `make diagnose-gfx1201-fault`. + +--- + +## What is already PROVEN (do not re-investigate) + +Full evidence record + the ablation matrix live in +**`docs/GFX1201_RDNA4_TRAINING.md` §6.1**. Summary of the rule-outs (all from +wandb project `csen346-sft`): + +- **Probabilistic, not config-determined.** Same config on the **same git SHA** + both finished 100 steps and crashed at 10 (eos-gate `g2df2ifl`/`gv9fbjac` on + `c752a2b4`; `5xd8qt5w`/`irgdklt9` on `eb12dbd9`). A clean 100-step run is luck, + not proof of a good config. **(Now refined: reproducible at ~84 on a clean GPU; + see diagnostic result above.)** +- **Not numerical / data / bad batch.** Loss descends smoothly 2.6→0.8, grad_norm + 1–6, no NaN; same-LR runs log byte-identical losses per step (deterministic fwd). +- **Not LR.** `lr=5e-6` crashed *earlier* (~step 10) than `lr=5e-5` (~84) — inverts + the LR theory. Mechanistically the fault is in backward; LR acts later at + `optimizer.step()`, so it can't affect the fault. (Likely the 5e-6 run was a + dirty-GPU cascade victim.) +- **Not hipBLASLt.** `TORCH_USE_HIPBLASLT=0` runs fault identically. +- **Not sequence length.** `scripts/determine_max_sequence.py`: real token + **max=909** (train) / 892 (eval), p99=673. Nothing exceeds 1024. With + `batch_size=1` + dynamic padding the peak is set by the *actual* longest seq, not + `max_length` — so 1280 is non-binding and lowering it frees no VRAM. **Leave it.** +- **GC threshold: still unsettled** (n=1 GC-off run also crashed but log was lost; + page-fault-vs-OOM unconfirmed). Replicate via arm D. + +**Enabling condition:** run sits at **median 89% / spikes to 98% VRAM** (~28.5 GB +steady, ~31.4 GB peak on the 32 GB card). The fault is in the immature +**bitsandbytes-NF4 / ROCm 7.2 backward** path. The serialized run survived repeated +98% spikes, then faulted at 84 — so pressure alone is not the deterministic trigger; +something that builds over ~84 steps (fragmentation?) or a specific kernel is. + +--- + +## Infrastructure built this session (all on the branch, CI green) + +| Thing | Where | Purpose | +|---|---|---| +| GPU clean-state + `--preflight` + `--wait-clean` | `scripts/test_gpu_stack.sh` (unified — do NOT add new GPU scripts) | gate every (re)launch on a clean, working GPU | +| `make gpu-preflight` | Makefile | fast clean-KFD + fwd/bwd gate; all train targets depend on it | +| `make diagnose-gfx1201-fault` | `scripts/diagnose_gfx1201_fault.sh` | serialized run that names the faulting kernel (`LOG_LEVEL=3` opts into verbose) | +| Hardened resume monitor | `scripts/monitor_stage2.sh` | `--wait-clean` before relaunch (kills dirty-KFD cascade), quarantines incomplete checkpoints, consecutive-no-progress retries (8), **archives full traceback + dmesg per crash to `outputs/sft-stage2-gemma4-31b/crashlogs/`** | +| Token-length measurement | `scripts/determine_max_sequence.py` | the 909 finding | +| Evidence record + ablation matrix | `docs/GFX1201_RDNA4_TRAINING.md` §6.1 | durable; don't re-derive | + +`save_steps=10` (`configs/train-sft-stage2-gemma4-31b.env`) is a **survival crawl, +not a fix** — it lets a faulting run inch forward via checkpoint+resume. The real +fix comes from the ablation. + +--- + +## The ablation protocol (after you've picked the arm from the traceback) + +The fault is reproducible at ~84 on a clean GPU, so single runs are now meaningful, +but keep **N=2–3 replicates** per cell (it was probabilistic earlier). Each cell: +`make gpu-preflight` → fixed seed → run to **≥150 steps** → record cleared/crash-step. +Change **one** factor from baseline (HIPBLASLT=0, GC=0.8, grad-ckpt `use_reentrant=False`, +bnb 0.49.2, sdpa, seq 1280, bs 1×16). Arms A–F in §6.1. **Stop-rule:** first arm +that clears 150 over all replicates becomes the new baseline; keep stacking. + +Lead with the arm the traceback indicates. If it's a bnb frame, **arm A** (source +build for gfx1201) is the highest-value shot and matches the prior that bnb on +RDNA4 is the least-mature part of the stack. + +--- + +## Environment / reproduction facts + +- HW: AMD Radeon AI PRO R9700, gfx1201 (RDNA4, Wave32), 32 GB; Ryzen 9 5900X; 64 GB RAM; 2 TB disk (not a constraint). +- SW: ROCm 7.2, torch 2.11.0+rocm7.2, bitsandbytes 0.49.2, transformers 5.9.0, peft 0.19.1, trl 1.4.0, Python 3.12. +- Model: `unsloth/gemma-4-31B-it-unsloth-bnb-4bit` (pre-quantized NF4, ~19 GB; cached on box). Attn = SDPA (hardcoded `scripts/train_sft.py:221`). +- **Always `uv run --no-sync`** (a bare `uv run` reinstalls CUDA torch over ROCm torch). +- After ANY fault the GPU is dirty — `make gpu-preflight` (or `bash scripts/test_gpu_stack.sh --wait-clean 180`) before the next launch. +- wandb project `csen346-sft`. Key runs: diagnostic `h9z6ebjd` (faulted @84, serialized); baselines `6c9fouit`/`p9t3yxe5` (@~84), `c6ye7mow` (lr5e-6, @~10); survival-patch `te8pgdbr` (@16, left `checkpoint-10`); eos-gates `g2df2ifl`/`5xd8qt5w` (finished 100), `gv9fbjac`/`irgdklt9`/`jtyyhu4t` (crashed). Training metrics log every 10 steps; system stats stream continuously. + +## Open questions for you to resolve + +1. **What kernel faults?** (the traceback — top priority). +2. **Is ~84 a fragmentation threshold or a kernel-specific trigger?** If arm D + (alloc churn / fragmentation) and the traceback shows no clear kernel, test + whether the fault step moves with `PYTORCH_HIP_ALLOC_CONF` changes. +3. **Does the cascade theory hold?** Confirm that early (10–20) crashes only happen + on a dirty restart, and clean starts reliably reach ~84. +4. **Arm A viability:** can bitsandbytes be source-built for gfx1201 on this box + (`cmake -DCOMPUTE_BACKEND=hip -DBNB_ROCM_ARCH=gfx1201`)? That's the likely fix if + the traceback names a bnb kernel. diff --git a/docs/HANDOFF_LLM_JUDGE.md b/docs/HANDOFF_LLM_JUDGE.md new file mode 100644 index 00000000..e651d93c --- /dev/null +++ b/docs/HANDOFF_LLM_JUDGE.md @@ -0,0 +1,109 @@ +# Handoff — LLM-judge on absolute Socratic quality (Opus 4.8 via subscription) + +> **✅ SUPERSEDED — this run is DONE** (commit `90f78ec`, 2026-07-13). Result: +> `results/llm_judge_oracle_compare.json`, SFT +0.38/10 overall (confirms the oracle direction, +> corrects the magnitude — see `docs/SFT_RESULTS_REPORT.md` "LLM-judge" section). Ignore the +> "run to launch" below; kept for provenance. For what's next see `docs/HANDOFF_SFT_POST_JUDGE.md`. + +**Created 2026-07-12.** For a fresh Claude session. Branch `feat/gemma4-12b-sft-poc-nvidia`. +Read alongside `docs/SFT_RESULTS_REPORT.md` (all results so far, incl. the consultant ablation + +oracle), `docs/SFT_VS_BASE_ANALYSIS_PLAN.md` (analysis menu — this is the **LLM-judge** item), and +`docs/HANDOFF_SFT_CONSULTANT_ABLATION.md` (the ablation lineage this follows). Live tracker: #130. +Use `uv run --no-sync` for Python. + +## The one-sentence goal + +The oracle arm showed the SFT writes better Socratic turns than base by **+22.7 ROUGE-1 / +19.1 +BLEU-4** given correct state — but ROUGE only measures *overlap with the reference*. **Judge the two +oracle arms on an absolute pedagogical rubric (Claude Opus 4.8) to confirm the win holds on actual +quality, not just phrasing overlap.** Everything is built and verified; the run has **not** been +launched (waiting on a go-ahead). + +## STATE: ready to run, not run + +- Harness: `scripts/llm_judge_eval.py` — implemented, tested, committed (`af5c64b`, `95d49b1`), + pushed. Full suite green (245 tests; 8 judge tests in `tests/test_llm_judge.py`). +- Verified live: a 5-turn subscription smoke on Opus 4.8 (sensible scores), and a `--dry-run` of the + paired sampler on the real oracle arms (200/arm at `--per-stage 50`, no capping). +- **Nothing consumed the full subscription budget yet** — only the ~7-turn smoke. + +## The run to launch (agreed parameters) + +**`--per-stage 50 --stages bcde` → 200 turns/arm → 400 total Opus calls** (user chose this; it's the +sweet spot — firm per-stage estimates, overall verdict already settled, not worth going higher for an +effect this large). Sequential, ballpark ~1–1.5 h, consumes subscription (not API credits). + +``` +uv run --no-sync python scripts/llm_judge_eval.py \ + results/gemma4-12b-sft-oracle --compare-with results/gemma4-12b-base-oracle \ + --per-stage 50 --stages bcde --seed 42 \ + --out results/llm_judge_oracle_compare.json +``` + +Run it **detached** with a progress tail (the CLI prints one line per judged turn: `[arm] n/200 +file#idx (stage): X/10`). Output: `results/llm_judge_oracle_compare.json` with per-arm +`overall_avg` / `axis_avgs` / `per_stage` / `cost_usd`, the `delta_a_minus_b` (SFT − base), and the +frozen `sample` list for reproducibility. + +## How the harness works (so you can trust/debug it) + +- **Backend = `claude-code` (default).** Judges via `claude -p --model claude-opus-4-8 + --output-format json --tools "" --permission-mode dontAsk --strict-mcp-config + --disable-slash-commands`, prompt on stdin, parses `.result`. Verified against `claude` 2.1.207. +- **Subscription, NOT API key.** `ANTHROPIC_API_KEY` is *stripped from the child env* in + `_claude_code_judge` because it takes precedence over the subscription and would silently bill the + API. Confirm before running: `test -n "${ANTHROPIC_API_KEY:-}" && echo SET || echo unset` should + say **unset**, and `claude` must be logged in to the subscription (a trivial `claude -p "say pong" + --output-format json` should return `is_error:false`). +- **Sequential** — parallel `claude -p` would race one account. `--max-turns N` caps turns for a + cheap smoke; `_parse_rubric` strips ``` fences and validates the 4 axes. +- **Rubric (0–10, absolute):** socratic_validity 0–3, advancement 0–3, age_appropriateness 0–2, + question_form 0–2. See `JUDGE_SYSTEM_PROMPT`. The prompt shows the GT reference "as one valid move, + do not penalize routing differences" **on purpose** — so the judge rewards *quality*, not + ROUGE-like overlap (do not remove this, or the independence from ROUGE is lost). +- **Paired design:** `build_stratified_sample` keys on `(dialogue_file, turn_idx)`; both oracle arms + replay identical GT student turns and hit `e34` at the same turn, so the sample is identical and + comparable across arms. Stage `a` is excluded (trivial opener). Per-stage pools (headroom): + b=761, c=1447, d=734, e=681 — so `--per-stage` up to ~680 before `e` caps. +- **Cost/tokens:** read from the CLI envelope's `total_cost_usd` (subscription-equivalent, not real + billing); `input_tokens` is uncached-only, so the cache buckets are summed for a true token count. + +## Gotchas + +1. **Verify subscription auth first** (see above). If `ANTHROPIC_API_KEY` is set in the shell, either + `unset` it or trust the in-code strip — but the child-env strip already handles it. +2. **Subscription usage limits.** 400 sequential Opus calls is modest but not free; if calls start + failing with 429/401 the account hit a window/credit cap — pause and resume later (re-running + re-judges from scratch; there's no turn-level checkpoint, so consider a smaller `--per-stage` or + split across windows if limits bite). +3. **Determinism:** the sample is seeded (`--seed 42`), but Opus scoring itself isn't temperature-0 + here (subscription headless), so re-runs vary slightly. Report as a single run; multi-seed is a + stretch, not required. +4. **`results/comparison.json`** is a stale regenerated artifact, dirty since several sessions ago — + unrelated to this work; leave it or ignore it. + +## Definition of done + +`results/llm_judge_oracle_compare.json` exists with both arms scored on the 200/arm sample; the +SFT − base delta (overall + per-axis + per-stage) is written into **`docs/SFT_RESULTS_REPORT.md`** +(new "LLM-judge (absolute quality)" subsection, near the consultant-ablation section), +**`docs/EXPERIMENT_LOG.md`** (dated entry, newest-at-top), and **issue #130** (comment). State +plainly whether the judge **confirms** the ROUGE-based oracle result: SFT should beat base on +`socratic_validity` + `advancement` especially. If it *disagrees* (SFT matched reference phrasing but +isn't judged more pedagogically sound), that's a genuine caveat — surface it, don't bury it. + +## Where things live + +- **Harness:** `scripts/llm_judge_eval.py` (`--backend claude-code` default; `--compare-with` enables + paired mode; `--per-stage`/`--stages`/`--seed`/`--out`/`--dry-run`/`--max-turns`). Tests: + `tests/test_llm_judge.py`. +- **Inputs:** `results/gemma4-12b-{sft,base}-oracle/dialogues/*.json` (681 each, oracle arm — GT state + fed in, so state_accuracy is 100% and the ROUGE/BLEU deltas are the signal). +- **Prior results:** `docs/SFT_RESULTS_REPORT.md` consultant-ablation section (Qwen / self-consult / + oracle × base / SFT). +- **Commits this lineage:** `efa77a6`/`a99c2fd`/`7b85ff7` (oracle) → `af5c64b`/`95d49b1` (judge). + +## After this: remaining analysis-plan picks + +Multi-seed error bars; a strong-consultant (Claude-as-classifier) cell between the ~55–60% Qwen and +100% oracle state-quality points. Both optional — the SFT-vs-base story is already well-established. diff --git a/docs/HANDOFF_RDNA4_EOS_GATE.md b/docs/HANDOFF_RDNA4_EOS_GATE.md new file mode 100644 index 00000000..8e27b197 --- /dev/null +++ b/docs/HANDOFF_RDNA4_EOS_GATE.md @@ -0,0 +1,323 @@ +# Handoff — R9700 gfx1201 EOS Gate Run + +**Branch:** `feat/gfx1201-rdna4-qlora-fla-training` +**Date:** 2026-05-31 +**Last commit:** `dbc61a2` + +> **STATUS: root cause FOUND and FIXED (commit `dbc61a2`).** The EOS collapse was +> not undertraining and not the consultant eval-line residual — it was the +> training chat template masking the turn terminator out of the loss. See +> **§ Root cause identified & fixed** below. Next action is a single confirmation +> retrain on the R9700, not a multi-hour bet. + +--- + +## What was attempted + +100-step QLoRA EOS gate checkpoint on the R9700 (gfx1201, 32 GB) to verify that +the schema-drift fixes from PR #101 / PR #94 actually produce a model that +terminates on realistic inference prompts before the full Stage 2 run is launched. + +Model: `unsloth/gemma-4-31B-it-unsloth-bnb-4bit` +Config: `configs/train-sft-stage2-gemma4-31b.env` +Data: `socrat-zh-sft` + `socrat-en-sft` (77 202 train / 8 578 eval) + +--- + +## Environment issues encountered and fixed + +### 1 — `make test-vllm` poisoned the venv + +Running `make test-vllm` before training auto-installed `vllm==0.22.0`, which +pulled in `torchvision==0.26.0` (CUDA build). The CUDA torchvision registers a +fake `torchvision::nms` op at import time that doesn't exist in the ROCm torch, +breaking `peft` and `transformers` at startup: + +``` +RuntimeError: operator torchvision::nms does not exist +ModuleNotFoundError: Could not import module 'BloomPreTrainedModel' +``` + +**Fix:** `uv pip uninstall vllm torchvision` + +**Going forward:** Never run `make test-vllm` before or during a training session +on this machine. The test auto-installs into the shared training venv and is +destructive. Either isolate it or add a guard to the Makefile. + +### 2 — `expandable_segments:True` unsupported on HIP → GPU page fault at step 16 + +All training Makefile targets had: +``` +PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,expandable_segments:True +``` + +HIP does not support `expandable_segments`. The runtime warned at startup +(`HIPAllocatorConfig.h:40`) and the training crashed with a GPU page fault +every time at step 16: + +``` +Memory access fault by GPU node-1 on address 0x7f78cbc00000. +Reason: Page not present or supervisor privilege. +``` + +**Fix (commit 860fc5f):** Stripped `expandable_segments:True` from all Makefile +targets. Note: the PR #101 comment that suggested this setting used +`PYTORCH_CUDA_ALLOC_CONF` (CUDA variable) — on ROCm the variable is +`PYTORCH_HIP_ALLOC_CONF` and `expandable_segments` is not a valid key. + +### 3 — GPU KFD dirty state after each page fault + +Every page fault on gfx1201 leaves the KFD in a dirty state; subsequent runs +fault at random steps until all Python processes are killed. Always run before +retrying: + +```bash +rocm-smi --showpids | grep python && echo "GPU dirty — kill before retrying" || echo "GPU clean" +``` + +### 4 — `nohup` → `setsid` for clean process group management + +Changed the `train-gemma4-31b-eos-gate` target to use `setsid` so the entire +process tree lives in its own process group. Kill cleanly with: +```bash +kill -9 -- -$PGID +``` + +### 5 — `eos_gate.py` crashed: `apply_chat_template` returns `BatchEncoding` in transformers 5.9 + +`tokenizer.apply_chat_template(..., return_tensors="pt")` returns a `BatchEncoding` +dict in transformers 5.9, not a raw tensor. `model.generate` received the dict +and raised `AttributeError` on `.shape[0]`. + +**Fix:** Extract `enc["input_ids"]` when the return value is not a tensor. + +--- + +## Training run outcome + +After all environment fixes, the 100-step checkpoint trained successfully: + +- **Steps:** 100/100 ✓ +- **VRAM:** alloc=21.1 GB, reserved=29.1 GB / 31.9 GB total (stable throughout) +- **Step time:** ~70 s/step +- **hipBLASLt:** enabled (`TORCH_USE_HIPBLASLT=1`) — Gemma 4 uses standard + softmax attention; no linear-attn delta-rule kernel, so hipBLASLt is safe +- **Adapter saved:** `outputs/eos-gate-gemma4-31b/final` + +--- + +## EOS gate result: FAILED (pre-fix) — root cause since found & fixed (`dbc61a2`) + +``` +STEP 1 warm-up (trivial prompt) → PASS (terminates after 1 sentence) +STEP 2 EOS gate (realistic prompt) → FAILED (hits max_new_tokens, repetition loop) +``` + +> This was the failing run *before* the terminator-masking fix. Jump to +> **§ Root cause identified & fixed** for the resolution; the section below is the +> observed failure that led there. + +### Failure output (truncated) + +``` +很好！你能想象一下，当光从空气进入水中时，它的速度会发生什么变化呢？ +这对光的方向有什么影响？这对光的方向有什么影响？这对光的方向有什么 +影响？[...×80+ repetitions, never emits EOS] +``` + +The first sentence is correct Socratic output. Repetition begins at token 2. +This is the same failure mode as the 5090 run documented in PR #94. + +### Initial (incorrect) hypothesis — recorded for the trail + +The first reading, carried from PR #94 / PR #101, was a *consultant +evaluation-field residual*: training stores a discretized annotation, inference +sends live free-form prose, so the model supposedly never learned to terminate +on the unseen prose shape. This was **wrong** — see below. + +--- + +## Root cause identified & fixed (commit `dbc61a2`) + +**The turn terminator `` was masked out of the training loss.** + +The decisive fact: **the 5090 *full* run also collapsed**, not just the 100-step +gate. That rules out undertraining — more steps cannot teach a token that is +never in the labels. + +`scripts/train_sft.py`'s Gemma 4 training chat template rendered the model-turn +terminator `\n` **after** `{% endgeneration %}`: + +```jinja +{% generation %}{{ message['content'] | trim }}{% endgeneration %} +... +{{- '\n' -}} # OUTSIDE the generation block +``` + +With `assistant_only_loss=True`, TRL builds the loss mask from the +`{% generation %}…{% endgeneration %}` span (via `return_assistant_tokens_mask`); +everything outside it becomes `-100`. So the terminator received **zero gradient +every step** — the model was trained to produce content but never taught to stop. +This is the exact failure TRL's own `chat_templates/gemma3_training.jinja` avoids +by keeping `` *inside* the generation block. TRL's guard only checks +that `{% generation %}` is *present*, not that it is placed correctly, so the +broken template passed silently. + +This explains every symptom: +- **Full run collapsed too** → structural, not data-quantity. +- **Warm-up passed, gate failed** → the base `-it` model already knows ``; + on a short trivial prompt that prior survives, on the long SFT-distribution + prompt the adapter dominates and there is no learned stop. +- **First sentence correct, then loops** → content was trained (inside the span), + termination never was (outside). + +### The fix + +Move the terminator inside the generation block for the model role: + +```jinja +{% generation %}{{ message['content'] | trim }}{{ '\n' }}{% endgeneration %} +``` + +Rendered **text is byte-identical** (verified for both `add_generation_prompt` +values and multi-turn) — only the loss-mask boundary moves, so the locked +headline / Tables are untouched. Added +`test_model_turn_terminator_is_inside_the_loss_mask` (no model download) that +renders the real template through transformers' jinja machinery and asserts the +assistant span is exactly `content + \n`. + +### Verified against the real `unsloth/gemma-4-31B-it-unsloth-bnb-4bit` tokenizer + +(Tokenizer is public — pulls without HF auth; weights not needed.) + +- `` is a **single registered special token (id 106)**, distinct from + `` (id 1). `eos_gate.py:_stop_ids` picks it up correctly. (`` + / `` are *absent* from the vocab — this Gemma 4 genuinely uses + `<|turn>` / ``.) +- Custom training template renders **byte-identical message bodies** to the stock + template. The *only* divergence is the generation primer: stock appends a + `<|channel>thought\n` reasoning primer (Gemma 4 is a thinking model); + the training template omits it, training the teacher to answer directly. +- The gate reproduced the collapse using the **custom** template (no primer), so + the masking bug is the cause of the *reproduced* failure — the fix is necessary + and the gate is a coherent test of it. + +--- + +## What to do next + +### 1 — Confirmation retrain on the R9700 (the only GPU step) + +Re-run the same 100-step gate; it uses the fixed training template: + +```bash +rocm-smi --showpids | grep python || echo "GPU clean" # KFD must be clean +make train-gemma4-31b-eos-gate # ~115 min, now with the terminator trained +make eos-gate-gemma4-31b # expect STEP 2 to PASS +``` + +If the gate **passes**, launch the full Stage 2 run: +```bash +make train-gemma4-31b-stage2-unsloth +tail -f outputs/sft-stage2-gemma4-31b/train.log +``` + +### 2 — Before the downstream eval: check the serving generation primer + +Separate, pre-existing concern (not the masking bug): production serving +(llama.cpp / GGUF for the 72.24 eval) must prime generation with plain +`<|turn>model\n`, **not** the stock `<|channel>thought\n` thinking +primer — otherwise it reintroduces a train/serve mismatch this template +deliberately avoids. Confirm which template `serve_gemma4_31b*.sh` / +`serve_teacher` use before trusting eval numbers. + +--- + +## Files changed this session + +| File | Change | +|------|--------| +| `Makefile` | Strip `expandable_segments:True`; add EOS gate targets; `setsid`; `TORCH_USE_HIPBLASLT=1` for eos-gate + stage2 | +| `scripts/train_sft.py` | **Root-cause fix (`dbc61a2`): move `` terminator inside `{% generation %}` so it is trained.** `attn_implementation` hardcoded to `"sdpa"` (see FA2 finding below) | +| `scripts/eos_gate.py` | Fix `apply_chat_template` → extract `input_ids` from `BatchEncoding` | +| `tests/test_sft_inference_format.py` | **`dbc61a2`: `test_model_turn_terminator_is_inside_the_loss_mask` — pins the terminator inside the loss span.** | +| `scripts/monitor_eos_gate.sh` | Autonomous EOS gate crash monitor + eval runner + Stage 2 launcher | +| `scripts/monitor_stage2.sh` | Autonomous Stage 2 crash monitor with PR notifications | +| `scripts/test_gpu_stack.sh` | Step 9: set `FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE`; update FA2 install note | +| `docs/HANDOFF_SFT_SCHEMA_DRIFT_FIX.md` | Pre-existing handoff from prior session | + +--- + +## Stage 2 status (2026-05-31) + +**EOS gate: PASSED** at 02:56 — fix confirmed. Stage 2 running since ~03:45. + +**Current training config:** +- `TORCH_USE_HIPBLASLT=1` (hipBLASLt for GEMM — restored after confirming the step-0 page fault was a dirty KFD, not hipBLASLt itself) +- `attn_implementation="sdpa"` (see FA2 finding below) +- `FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE` (set in Makefile for future use; currently no effect since FA2 is disabled) +- ~14,478 total steps at ~70 s/step → ~281h wall time + +**Confirmed step time: ~70 s/step** (measured at steps 1–9 of Stage 2 with SDPA + HIPBLASLT=1). Hardware ratio alone (R9700 ≈ 54% of 5090 TFLOPS) predicts ~8 s/step; actual is ~9× slower. + +> **CORRECTION (profiled, `scripts/profile_train_step.py`):** the earlier claim +> "SDPA attention is the bottleneck, not GEMM" was wrong. A real-step `torch.profiler` +> trace shows the GPU self-time split is: +> - **`bitsandbytes::dequantize_4bit` (NF4→bf16) ≈ 55%** — the dominant cost; +> - GEMM (`aten::mm` / rocBLAS `Cijk_*`) ≈ 22%; +> - **attention (aotriton `attn_fwd` + `bwd_kernel_*`) ≈ 5%**, already on a fast +> kernel (~0.5 ms/call fwd) — *not* the slow math path. +> +> So the ~70 s/step is **NF4-dequant-bound**, the inherent 4-bit QLoRA tax (every +> matmul dequantizes its weight), made worse by the RDNA4 dequant kernel being +> memory-bound. This is also why HIPBLASLT=1 didn't help — GEMM isn't the wall. +> **FA2 is therefore not worth pursuing for speed** (see below); attention is ~5%. +> The only real wall-time levers are fewer tokens (epochs 3→1, done) or dropping +> quantization (needs >32 GB VRAM for a 31B — not available here). + +--- + +## FA2 investigation and gfx1201 shared memory limit + +**Findings from this session (commit `e755aa9`, reverted in follow-up):** + +`flash_attn 2.8.3` was installed via `FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE` (Triton JIT path, Wave32-aware). The **forward pass** benchmarked at **0.2 ms/call** for Gemma 4 attention shape (seq=1280, 8 heads, dim=128). However, the **backward pass (training step) immediately OOMed**: + +``` +triton.runtime.errors.OutOfResources: out of resource: shared memory, +Required: 131072, Hardware limit: 65536. +``` + +gfx1201 has a 64 KB shared memory hard limit; the Triton backward kernel (`_bwd_kernel_dkdv_causal`) requires 128 KB. This is a fundamental hardware constraint, not a configuration issue. + +**Implication:** FA2 is usable for **inference** on gfx1201 but **not for training** with the current flash_attn 2.8.3 Triton AMD backward implementation. SDPA remains the correct choice for training. + +**Possible future path (now de-prioritised):** one could patch `flash_attn_triton_amd/bwd_prefill_split.py` to cap block sizes within 64 KB shared memory (halving BLOCK_N/BLOCK_M). **But the profiler above settles it: attention is only ~5% of step time and already runs a fast aotriton kernel, so even a perfectly-patched FA2 backward would shave a few percent at most.** Not worth the spike while the step is NF4-dequant-bound. Revisit only if the dequant cost is ever addressed (fused 4-bit GEMM, or training unquantized on a larger card). + +**Note on Vulkan:** Vulkan is available for inference (llama.cpp `GGML_VULKAN=ON`, ~20% TG speedup confirmed). No PyTorch training backend exists for Vulkan on Linux. + +--- + +## Key commands + +```bash +# Check GPU state before any run +rocm-smi --showpids | grep python || echo "GPU clean" + +# GPU stack smoke test (all 13 steps should pass, including flash_attn 2.8.3) +make test-gpu-stack + +# EOS gate full sequence +make train-gemma4-31b-eos-gate # ~115 min (100 steps × 70 s/step) +tail -f outputs/eos-gate-gemma4-31b/train.log +make eos-gate-gemma4-31b # ~10 min (loads model + generates) + +# Full Stage 2 run (only after EOS gate PASS) +# Uses TORCH_USE_HIPBLASLT=1 + FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE + sdpa +make train-gemma4-31b-stage2-unsloth +tail -f outputs/sft-stage2-gemma4-31b/train.log + +# Autonomous monitors (started automatically by monitor_eos_gate.sh on EOS PASS) +tail -f outputs/monitor_eos_gate.log +tail -f outputs/monitor_stage2.log +``` diff --git a/docs/HANDOFF_SFT_CONSULTANT_ABLATION.md b/docs/HANDOFF_SFT_CONSULTANT_ABLATION.md new file mode 100644 index 00000000..d76ad124 --- /dev/null +++ b/docs/HANDOFF_SFT_CONSULTANT_ABLATION.md @@ -0,0 +1,148 @@ +# Handoff — SFT vs Base: Consultant Ablation (no-consultant + consultant-as-variable) + +**Created 2026-06-25.** For a fresh Claude session. Branch `feat/gemma4-12b-sft-poc-nvidia`. +Read alongside `docs/SFT_RESULTS_REPORT.md` (results so far), `docs/SFT_VS_BASE_ANALYSIS_PLAN.md` +(full ablation menu — this handoff is plan item **T1.1**), and `docs/SFT_HANDOFF.md` (pipeline + +serve/box hazards). Live tracker: issue #130. Use `uv run --no-sync` for Python. + +## The one-sentence goal + +Every result so far gave **both** base and SFT the same external state classifier (the Qwen3.5 +"consultant"), so we don't yet know whether the SFT's +7.7–10.3 pp gain is its **own** internalized +Socratic state-tracking or just better use of the classifier. **Run base vs SFT with the consultant +removed** (the model self-tracks state), and more broadly treat **the consultant as a variable**. + +## Background you need (consultant modes) + +`create_system` (`src/project/kele.py:30`) picks the teaching system by flag: + +| mode | flag | who predicts the scored `state` | LLM calls/turn | +|---|---|---|---| +| **External classifier** (all runs so far) | `--bert-consultant ` | Qwen3.5 LoRA classifier on **CPU** | 1 (teacher only) | +| **Self-consult / dual-role** ← *"no consultant"* | *(omit `--bert-consultant`)* | the **served LLM itself** (separate consultant call) | **2** (consultant + teacher) | +| **Unified** | `--unified` | one fused structured-output call | 1 | + +- The scored `state_accuracy` compares `system.current_state` to ground truth. **Who sets + `current_state` changes with the mode** — so the no-consultant runs measure the *teacher model's + own* state classification, which is the whole point of the ablation. +- "No consultant" here means **no external classifier**, i.e. the dual-role `SocraticTeachingSystem` + where the LLM self-classifies. There is no "no state-tracking at all" mode; the self-consult call + is the apples-to-apples baseline (identical mechanism for base and SFT). + +## ⚠️ Gotchas — read before launching + +1. **~2× slower (GPU).** Self-consult does **two** served-LLM calls per turn (consultant + teacher) + vs one with the CPU classifier. The base model already rambles to the 2048-token cap (≈38 dlg/hr + with one call → ZH test took 17.4 h). **Base no-consultant could be ~30+ h.** Budget for it, or + start with a partial n (e.g. `--limit 200`, which is a valid early signal — convergence is stable + from n≈200–300) and only commit to full 681 if needed. SFT is ~6× faster (terminates cleanly). +2. **The monitor hardcodes `--bert-consultant`.** `scripts/monitor_eval_gemma4_12b.sh` `start_eval` + always passes `--bert-consultant "$BERT_CKPT"`. To run the crash-resilient monitored path + without it, add an env toggle (e.g. `NO_CONSULTANT=1` → drop the flag + suffix the out-dir), the + same pattern as the `EVAL_HF_REPO`/`EVAL_SPLIT`/`EVAL_OUT_SUFFIX` block already there. Otherwise + run `kele` directly (loses auto-resume; risky on this unstable box for a 30 h base run). +3. **The SFT CONSUMES the consultant, it does not emit state** (verified in `dataset.py:608–647`, + `socrat-zh-sft`/`socrat-en-sft`): its training prompt = system rules + history + student input + + `苏格拉底教学顾问评估结果: {evaluation}` + `苏格拉底教学顾问建议的操作: {action}`, and the target is a + **clean teacher turn** (no inline state). Implications: + - The scored `state` comes from the **consultant**, never parsed from teacher output — so + "parse state from the SFT's output" is **not viable** (output carries no state). + - **Fully dropping the consultant** (no assessment line at all) runs the SFT **off its training + distribution** AND leaves nothing to score `state_accuracy` → ROUGE/BLEU only. It's a + robustness probe, not the SFT's normal operating mode. + - **Self-consult keeps the SFT in-format**: the LLM produces the assessment itself, then consumes + it — the assessment line is still present, just self-generated. This is the recommended "no + external classifier" run. +4. **Everything else stays pinned** (so only the consultant + model vary): Q8_0 GGUF, `-np 4`, + q4_0 KV, MTP off, workers=4, 8 rounds, stochastic server-default sampling, ZH `SocratDataset` + test split. Serve base via `make serve-gemma4-12b` (alias "Gemma 4 12B"), SFT via + `make serve-gemma4-12b-sft` (alias "Gemma 4 12B SFT") — one model at a time on the 20 GB card. + +## The minimum: 2 runs (the ablation the user asked for) + +Base vs SFT, **no consultant**, ZH test, same settings. Direct (no monitor) form: + +``` +# serve base first (make serve-gemma4-12b), then: +KELE_BERT_DEVICE=cpu KELE_PARALLEL_WORKERS=4 uv run --no-sync python -m src.project.kele \ + --experiment gemma4-12b-local evaluate --output results/gemma4-12b-base-noconsult +# stop base server, serve SFT (make serve-gemma4-12b-sft), then: +KELE_BERT_DEVICE=cpu KELE_PARALLEL_WORKERS=4 uv run --no-sync python -m src.project.kele \ + --experiment gemma4-12b-sft-local evaluate --output results/gemma4-12b-sft-noconsult +# compare, and cross-compare against the WITH-consultant runs: +python -m src.project.evaluate --compare results/gemma4-12b-base-noconsult results/gemma4-12b-sft-noconsult +``` +**Recommended instead:** add `NO_CONSULTANT=1` to the monitor (gotcha #2) and run via +`make monitor-eval-gemma4-12b-{base,sft}` for crash-resilience on the long base run. Consider +`--limit 200` for the base arm first. + +## The broader design: consultant as a variable (2×N factorial) + +Teacher ∈ {base, SFT} × Consultant ∈ {Qwen-classifier (have it), self-consult (this handoff), +optionally unified, optionally a strong external judge like Claude}. The **core 2×2** is: + +| | consultant = Qwen classifier | consultant = self (no external) | +|---|---|---| +| **base** | 49.62 (have it: `results/gemma4-12b-base`) | **run this** | +| **SFT** | 59.93 (have it: `results/gemma4-12b-sft`) | **run this** | + +What each contrast answers: +- **SFT(self) vs SFT(Qwen)** — if close, the SFT **internalized** state-tracking and the external + classifier is largely redundant (a strong, headline-worthy result). If it drops a lot, the gain + leaned on the classifier. +- **base(self) vs base(Qwen)** — how much the *base* pipeline depends on the classifier crutch + (expected: base self-consult ≪ base+Qwen). +- **SFT(self) vs base(self)** — the cleanest "is the SFT model itself a better Socratic + state-tracker?" with no external help on either side. **This is the key comparison.** +- **Interaction** — does SFT's advantage shrink, hold, or *grow* when the crutch is removed? A gain + that grows without the classifier is the strongest evidence the SFT learned the skill. + +Note on what the metrics mean here: the scored `state_accuracy` is the **consultant's** prediction +(only indirectly shaped by the teacher, via the dialogue history the classifier reads), whereas +**ROUGE/BLEU directly measure teacher-turn quality** — and that's where the SFT's cleanest wins are +(ROUGE-1 28→48 on ZH). The consultant-variable runs that drop/weaken state prediction naturally +lean on the text metrics. + +Stretch consultant levels (only if the core 2×2 is promising): +- **`--unified`** — fused single-call mode; another way the model self-tracks. +- **Strong external consultant** (Claude as classifier; configs exist: + `configs/claude-*-as-consultant.env`) — upper bound on how much a *better* consultant lifts each + teacher, and whether SFT still adds value on top of a strong classifier. +- **Oracle consultant (feed ground-truth state/action)** — small code change: a consultant that + returns the dialogue's GT state instead of predicting. Makes `state_accuracy` trivially perfect, + so compare on **ROUGE/BLEU** — this isolates *pure teacher-turn quality given correct state*, the + cleanest "does SFT write better Socratic turns?" measure (removes classifier quality as a + confound). Arguably more informative than fully-bare. +- **Fully bare (no consultant at all)** — needs code (strip the assessment+action lines, skip state + scoring). SFT runs off-distribution; ROUGE/BLEU only. A robustness probe, not normal operation. + +Keep each cell to ZH test first (the reference set); extend to EN/synthetic only for cells that +matter, since each base cell is expensive. + +## Suggested order + +1. Wire `NO_CONSULTANT=1` into the monitor (small, mirror the existing env-override block) + a test. +2. **SFT no-consultant, ZH test** (fast, ~1 h) — get the most important number first. +3. **base no-consultant, ZH test** — start `--limit 200` to get a quick read; full 681 if the box + tolerates the ~30 h. +4. `--compare` the 2×2; write results into `docs/SFT_RESULTS_REPORT.md` + `EXPERIMENT_LOG.md` + #130. +5. Only then consider unified / Claude-consultant / parse-from-output. + +## Definition of done + +`results/gemma4-12b-{base,sft}-noconsult/metrics_summary.json` exist; the 2×2 table (Qwen vs +self-consult × base vs SFT) is filled and interpreted in `docs/SFT_RESULTS_REPORT.md`; #130 updated. +State whether the SFT's advantage survives removing the external classifier (the headline question). + +## Where things live + +- **Results so far:** `results/gemma4-12b-{base,sft}{,-en,-synth-zh,-synth-en}` + `-base-mtp`. +- **Models (HF, private):** `ulises-c/SocratesLM-12B-QLoRA` (adapter), `-12B` (merged BF16), + `-12B-GGUF` (Q8_0). Served GGUF already staged in the weights dir. +- **Consultant ckpt:** `results/state-clf-qwen3.5-0.8b-lora-wandb/final`. +- **Key code:** `create_system` + `run_batch_evaluation` (`src/project/kele.py`); dual-role + consultant (`src/project/socratic_teaching_system.py:socratic_teaching_consultant`); classifier + variant (`src/project/socratic_teaching_bert_consultant.py`); monitor + (`scripts/monitor_eval_gemma4_12b.sh`). +- **Box hazards:** unstable RTX 4000 Ada, stable at 85 W; power step-down inert (no passwordless + sudo); one model at a time on 20 GB. See `docs/SFT_HANDOFF.md` + memories. diff --git a/docs/HANDOFF_SFT_POST_JUDGE.md b/docs/HANDOFF_SFT_POST_JUDGE.md new file mode 100644 index 00000000..5f1ffb89 --- /dev/null +++ b/docs/HANDOFF_SFT_POST_JUDGE.md @@ -0,0 +1,118 @@ +# Handoff — SFT-vs-Base PoC: post-judge (hardening + remaining picks) + +**Created 2026-07-22.** For a fresh Claude session. Branch `feat/gemma4-12b-sft-poc-nvidia`. +Read alongside `docs/SFT_RESULTS_REPORT.md` (the single source of truth for all results), +`docs/SFT_VS_BASE_ANALYSIS_PLAN.md` (the analysis menu), and `docs/SFT_HANDOFF.md` (pipeline + +box hazards). Live tracker: issue #130. Use `uv run --no-sync` for Python. + +## Where the PoC stands (the story is COMPLETE and consistent) + +The core scientific question — *did the Socratic QLoRA SFT of Gemma 4 12B actually learn something, +and what?* — is answered on every axis we set out to test. Nothing below is required to state the +result; it's all hardening / insurance. + +- **Headline (with the shared Qwen classifier):** SFT beats base on state accuracy everywhere — + ZH +10.3, EN +7.7 in-distribution; +3.5–3.9 OOD synthetic; 0 regressions. (~0.88-epoch adapter.) +- **Consultant ablation (T1.1), all three modes done** — Qwen classifier / self-consult / oracle × + base / SFT, ZH test, 681/681 each: + - Teacher-turn *quality* is intrinsic and **largest with the state confound removed** (oracle: + +22.7 ROUGE-1, +19.1 BLEU-4). The SFT climbs monotonically with state quality (ROUGE-1 + 44.2→48.1→51.8); base is flat (28.0→28.6→29.0) — the SFT *learned to condition on state*. + - State-*tracking* was entirely the classifier: self-consult SFT (26.8) is **worse** than base + (34.5) — the SFT was trained to *consume* state, never emit it (`dataset.py:608–647`). +- **LLM-judge (T1.2), done** — Opus 4.8, absolute 0–10 rubric, 200 turns/arm paired on the oracle + arms. **Confirms direction, corrects magnitude:** SFT +0.38/10 overall (wins/ties every axis & + stage), but that's ~5% relative vs ROUGE's ~78% — most of the ROUGE gap was *reference-phrasing + overlap*, not pedagogy. Real judge-visible edge = **question_form (+0.20)** and the **middle + stages c (+0.64) / d (+0.76)**; `advancement` is a dead tie; endpoints tie. `results/llm_judge_oracle_compare.json`. + +**One-line takeaway to preserve:** the SFT genuinely writes better-*formed* Socratic questions +(esp. mid-dialogue) and phrases turns corpus-style, but given correct state it is not dramatically +more *pedagogically* effective than base, and it does not self-track state. In deployment it wants +an external state source and rewards a better one with better turns. + +## What's actually left (ranked; all optional) + +### 1. T0.1 — Termination / length / style metrics ⭐ FREE, do this first +The report asserts "base rambles to the 2048-token cap, SFT terminates cleanly" (`SFT_RESULTS_REPORT.md:65–68`) +but it's **still unquantified** — the only open claim in the report. Now doubly worth closing because +the judge cost data corroborates it (base cost more to judge: $15.02 vs $13.86 = more tokens) and the +judge's **question_form** win begs a mechanical counterpart. No GPU — operate on the saved +`results//dialogues/*.json` (each turn has `teacher_response`, `state`, `ground_truth_*`). +Compute per turn, base vs SFT, across the runs already on disk: +- response length (chars/tokens), **truncation rate** (hit `max_tokens=2048` with no EOS), +- **questions-per-turn** — reuse `tournament_utilizations.validate` (single clean question, no preamble). +Hypothesis: SFT shorter, single-question, terminates; base rambles, esp. OOD. Write a small analysis +script (no new eval run), land results in the report's `## What this means` §4 (replace "to be +quantified") + a log entry. ~1–2 h of pure analysis. + +### 2. T2.1 — Multi-seed error bars +Every number (ROUGE +9.6/+10.3, judge +0.38) is a **single-run point estimate**; σ≈0.7 pp is from one +pair. Re-run ZH-test base+SFT 2–3× (stochastic decode reshuffles; or `--sample-seed`) for a real +stdev on the headline. ~3 h/run on the box (SFT fast, base ~17 h — budget accordingly). Turns the +"~11–15σ" claim from assumed into measured. + +### 3. T2.3 — Earlier-checkpoint comparison (the one *strategic* decision) +Eval `checkpoint-3200` (loss plateaued ~step 3000) vs shipped `checkpoint-4250` (both on HF +`ulises-c/SocratesLM-12B-QLoRA`). **Decides whether a clean full-epoch run (~30 h) is worth it:** +saturated at 3200 → more epochs won't help; still climbing → a full run is justified. Reuse +`merge_lora_gemma4_sft.py` + `convert_gemma4_12b_sft_to_gguf.sh` on the earlier adapter, then eval. + +### 4. Lower value / insurance +- **T2.2 greedy (temp=0)** — clean point estimate, but **needs a small code change first**: + `temperature`/`seed` are NOT passthrough on the teacher call (server default). Add that knob (it + also enables cleaner multi-seed). Worth doing as general infra even independent of T2.2. +- **Strong-consultant cell** — Claude-as-classifier between the ~55–60% Qwen and 100% oracle + state-quality points; bounds how much a better external state source lifts each teacher. +- **T3.1 capability preservation** (general-QA probe, catastrophic-forgetting check), + **T3.2 quant sensitivity** (BF16 vs Q8_0 on ~50 dialogues — confirm the uplift isn't a quant + artifact). + +If you do only one: **T0.1** (free, closes the last open claim, complements the judge's question_form +finding). If prepping a writeup: **T0.1 + T2.1**. If deciding on more training: **T2.3**. + +## Box + tooling hazards (unchanged, carry forward) + +- **Unstable RTX 4000 Ada.** Stable at **85 W**; power step-down is **inert** (no passwordless sudo, + so the monitor's "step down per crash" logs but can't actually change power — it just auto-resumes). + **One model at a time on the 20 GB card.** GPU eval runs on this box are long and crash-prone — use + the crash-resilient monitor, never bare `kele evaluate` for a full run. +- **Monitor:** `scripts/monitor_eval_gemma4_12b.sh {base|sft}` owns serve+eval, auto-resumes per + dialogue, logs to #130. Env toggles: `EVAL_HF_REPO`/`EVAL_SPLIT`/`EVAL_OUT_SUFFIX` (other datasets), + `NO_CONSULTANT=1` (self-consult), `ORACLE_CONSULTANT=1` (oracle). Chain wrappers: + `scripts/{noconsult,oracle}_chain_gemma4_12b.sh` (`make {noconsult,oracle}-chain-gemma4-12b`) run + SFT→base back-to-back detached. Cosmetic bug: #130 progress rows hardcode `/681` denominator — + patch `dataset_total` for non-default split sizes. +- **LLM judge** (`scripts/llm_judge_eval.py`): backend `claude-code` (subscription, NOT API) — + `ANTHROPIC_API_KEY` is stripped from the child env; verify `claude` is logged in first. Sequential + (parallel would race one account), **no turn-level checkpoint** (a failed run re-judges from + scratch — split `--per-stage` if usage limits bite). Rubric shows the GT reference "as one valid + move, do not penalize routing" *on purpose* — keep it, or independence from ROUGE is lost. +- **`results/comparison.json`** has been dirty for many sessions — a stale regenerated artifact, + unrelated to any current work. Leave it; don't stage it. +- **Commits go to two remotes** (`ulises-c/csen-346` + `SCU-CSEN346/KELE`); a plain `git push` + pushes both. Pre-commit runs ruff+pyright+codespell+shellcheck+full pytest and **blocks** on + failure. **Re-`git add` after fixing a pre-commit failure** — the hooks check the working tree but + git commits the *staged* blob; a fix made without re-staging commits the stale version (this bit us + in `efa77a6`, fixed in `46f84d6`). + +## Where things live + +- **Report (read first):** `docs/SFT_RESULTS_REPORT.md` — headline, per-stage, consultant ablation + (3 modes), LLM-judge subsection, caveats, artifacts. +- **Results on disk:** `results/gemma4-12b-{base,sft}{,-en,-synth-zh,-synth-en,-noconsult,-oracle}/` + + `-base-mtp`; `results/llm_judge_oracle_compare.json`. +- **Models (HF, private):** `ulises-c/SocratesLM-12B-QLoRA` (adapter, ckpts 3200–4250), + `-12B` (merged BF16), `-12B-GGUF` (Q8_0, served). +- **Key code:** `src/project/kele.py` (`create_system` + eval loop; consultant modes incl. + `--oracle-consultant`), `src/project/socratic_teaching_system.py` (base + `SocraticTeachingSystemOracle`), + `scripts/llm_judge_eval.py`, the monitor + chain wrappers. +- **Superseded handoffs:** `HANDOFF_LLM_JUDGE.md` (its run is DONE — ignore its "run to launch"), + `HANDOFF_SFT_CONSULTANT_ABLATION.md` (T1.1, done). + +## Definition of done (for whichever pick you take) + +Result lands in `docs/SFT_RESULTS_REPORT.md` (the right existing section), a dated newest-at-top +entry in `docs/EXPERIMENT_LOG.md`, and an issue #130 comment — the established pattern. For T0.1 +specifically: replace the "to be quantified" hedge in `## What this means` §4 with the measured +numbers. diff --git a/docs/HANDOFF_SFT_SCHEMA_DRIFT_FIX.md b/docs/HANDOFF_SFT_SCHEMA_DRIFT_FIX.md new file mode 100644 index 00000000..456ddedd --- /dev/null +++ b/docs/HANDOFF_SFT_SCHEMA_DRIFT_FIX.md @@ -0,0 +1,168 @@ +# Handoff — Stage 2b SFT schema drift fix + R9700 training in progress + +**Branch:** `feat/gfx1201-rdna4-qlora-fla-training` (PR #101) +**Last commit:** `d04bc83` +**Date:** 2026-05-30 + +--- + +## TL;DR + +The 5090-trained Gemma 4 31B Stage 2b adapter had output collapse (never emits +EOS, runs to context limit). Root cause was a train/serve schema mismatch, not +model quality. The schema is now fixed. Training is running on the R9700. + +--- + +## Root cause (from 5090 run) + +The model was healthy — training-shape prompts produced clean 20-token responses. +Inference-shape prompts caused 2048-token repetition collapse. Decisive A/B: + +| | Training (old `socrat-zh/en`) | Inference (`socrates_teacher()`) | +|---|---|---| +| System prompt | 1-line + problem context | 7-line rules block, no problem | +| Message structure | Multi-turn N pairs | Single user message | +| History | HF message turns | `学生:/老师:` text blob | +| Current input label | None | `当前学生输入:` | + +See: `docs/EXPERIMENT_LOG.md` (2026-05-28 PM), PR #94 commit `eff214f`. + +--- + +## What was fixed this session + +### 1. New SFT data sources (`src/project/dataset.py`) + +`socrat-zh-sft` and `socrat-en-sft` replace `socrat-zh`/`socrat-en` for training. +They produce one `(system, user, asst)` triple per dialogue turn in the exact +format `socrates_teacher()` sends at inference. + +- System: 7-line rules block (copied from `socratic_teaching_system.py:404-415`) +- User: `\n历史对话记录:\n{学生/老师 pairs}\n\n当前学生输入: ...\n\n苏格拉底教学顾问评估结果: 学生处于 {state} 状态\n苏格拉底教学顾问建议的操作: {action}\n` +- **Render-diff verified MATCH: True** (byte-identical to real inference messages) +- Split is at **dialogue level** — all turns of a dialogue stay in same partition +- Old `socrat-zh`/`socrat-en` multi-turn loaders untouched (backward compat) +- 77,202 train records / 8,578 eval records + +### 2. `quantization_config=None` crash fix (`scripts/train_sft.py`) + +Passing explicit `None` to `from_pretrained` caused `auto_factory.py` to +overwrite the unsloth checkpoint's `config.json` quantization config with `None` +→ `supports_quant_method` crash. Fix: only include `quantization_config` in +`_load_kwargs` when it is not None. + +### 3. W&B integration (`scripts/train_sft.py`, `pyproject.toml`) + +- `wandb>=0.18.0` is a core dep (not an extra) +- `_check_wandb()` calls `wandb.login(relogin=False)` — warns and disables if + unauthenticated, no flag needed +- `report_to=["wandb"]`, `run_name` defaults to output-dir basename +- Credentials present in `~/.netrc` → tracking is live +- W&B project: `csen346-sft`, run: `sft-stage2-gemma4-31b` + +--- + +## Current training state + +``` +Model: unsloth/gemma-4-31B-it-unsloth-bnb-4bit (pre-quantized NF4, ~19 GB) +Sources: socrat-zh-sft, socrat-en-sft (77,202 per-turn records) +Config: configs/train-sft-stage2-gemma4-31b.env +Output: outputs/sft-stage2-gemma4-31b/ +Steps: 2,298 total (3 epochs × 766 steps/epoch) +Pace: ~84 s/step on R9700 gfx1201 QLoRA +ETA: ~55 h from start (~2026-06-01 23:00) +Saves: every 100 steps (~2.4 h rollback granularity) +``` + +Monitor: `tail -f outputs/sft-stage2-gemma4-31b/train.log` +W&B: https://wandb.ai/uchavarria-santa-clara-university/csen346-sft/runs/thb2cnzq + +**Launch command (to restart if killed):** +```bash +nohup env TORCH_USE_HIPBLASLT=0 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8,expandable_segments:True \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + > outputs/sft-stage2-gemma4-31b/train.log 2>&1 & +``` + +Auto-resumes from latest checkpoint if `outputs/sft-stage2-gemma4-31b/checkpoint-*` exists. + +--- + +## Blocking gate before trusting the full run + +After the **first checkpoint** (~100 steps, ~2.4 h in), load it and probe with +an inference-shape prompt. The model must emit clean EOS — not a repetition loop. + +```python +from peft import PeftModel +from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig +import torch + +bnb = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=torch.bfloat16) +base = AutoModelForCausalLM.from_pretrained( + "unsloth/gemma-4-31B-it-unsloth-bnb-4bit", device_map={"": 0}) +model = PeftModel.from_pretrained(base, "outputs/sft-stage2-gemma4-31b/checkpoint-100") +tok = AutoTokenizer.from_pretrained("outputs/sft-stage2-gemma4-31b/checkpoint-100") + +# Inference-shape prompt (must produce clean output, not a repetition loop) +msgs = [ + {"role": "system", "content": "<7-line rules block>"}, + {"role": "user", "content": "\n历史对话记录:\n\n\n当前学生输入: 植物的种子在哪里？\n\n苏格拉底教学顾问评估结果: 学生处于 a1 状态\n苏格拉底教学顾问建议的操作: 生成一个问题\n"}, +] +ids = tok.apply_chat_template(msgs, return_tensors="pt").to("cuda") +out = model.generate(ids, max_new_tokens=100, do_sample=False) +print(tok.decode(out[0][ids.shape[1]:])) +``` + +**Pass:** single coherent Socratic question, EOS within 50 tokens. +**Fail:** repetition of any phrase, or `max_new_tokens` exhausted. + +--- + +## Known residual issues (deferred) + +1. **Evaluation string drift** — training uses `学生处于 {state} 状态`; inference + injects the consultant's free-form reasoning paragraph. The A/B test confirmed + the terse form is sufficient for EOS. Address when a re-annotated dataset + is available. + +2. **`_build_inference_user_message` divergence risk** — reimplements + `get_formatted_history()` from `socratic_teaching_system.py`. Render-diff is + the mandatory gate before changing either function. + +3. **No tests for new loaders** — `load_socrat_zh_sft` / `load_socrat_en_sft` + and `_build_inference_user_message` have no unit tests. Add before any refactor. + +--- + +## After training completes + +1. **Load checkpoint smoke test** (see blocking gate above) +2. **Merge LoRA** → BF16 HF checkpoint: `python scripts/merge_lora_gemma4_sft.py` +3. **Convert to GGUF**: `bash scripts/convert_gemma4_sft_to_gguf.sh` +4. **Serve and eval**: replace `GEMMA4_31B_WEIGHT_FILE` with the new GGUF in + `scripts/serve_gemma4_31b_q5.sh`, run `make eval-gemma4-31b-full` +5. **Path A from PR #94 becomes live**: eval numbers (likely bad per §2.8 + outcome matrix) are the paper's negative-result contribution. + +--- + +## Files changed this session + +| File | Change | +|---|---| +| `src/project/dataset.py` | `socrat-zh-sft` / `socrat-en-sft` sources; `_TEACHER_INFERENCE_SYSTEM`; `_build_inference_user_message`; dialogue-level split fix | +| `scripts/train_sft.py` | `_check_wandb()`; `_load_kwargs` conditional build; W&B `report_to` / `run_name` | +| `configs/train-sft-stage2-gemma4-31b.env` | `TRAIN_SOURCES=socrat-zh-sft,socrat-en-sft`; W&B stanza | +| `pyproject.toml` | `wandb>=0.18.0` core dep | +| `.gitignore` | `wandb/` artifacts | +| `.env.example` | `WANDB_API_KEY` section | + +PR comments: #4585166874 (schema fix summary), #4585192824 (split fix patch) diff --git a/docs/HANDOFF_STAGE2_CRAWL.md b/docs/HANDOFF_STAGE2_CRAWL.md new file mode 100644 index 00000000..f4b1f9b0 --- /dev/null +++ b/docs/HANDOFF_STAGE2_CRAWL.md @@ -0,0 +1,336 @@ +# Handoff — Stage 2 Gemma 4 31B QLoRA crawl + +**Branch:** `feat/gfx1201-rdna4-qlora-fla-training` · **PR:** #101 +**For:** a fresh Claude instance picking up after session 10. + +--- + +## TL;DR + +Training is **RUNNING** — monitor auto-started at session begin, recovered from one crash, +and is at step **1305/4826 ≈ 27.0%** as of 2026-06-06 19:35. + +Working tree is **clean**. All session 9–10 changes committed. + +Progress banked: **checkpoint-1290 / 4826 steps ≈ 26.7% complete.** + +Session 9 focus: attempted standalone reproduction of `MT64x64x64_ISA1201_DTVB1` tile. +Nine probe variants all dispatch `MT128x128x32` instead. The leading hypothesis is a +**system-vs-torch-bundled rocBLAS library difference** (see Open items). The dense-VA +reproducer script is now committed. Upstream #7992 update is due. + +--- + +## Immediate actions + +### 1. Check training status (likely already running) + +```bash +pgrep -f monitor_stage2 && echo 'RUNNING' || echo 'NOT RUNNING' +tail -20 outputs/monitor_stage2.log +# If not running: +make gpu-preflight # must PASS +nohup bash scripts/monitor_stage2.sh > outputs/monitor_stage2.log 2>&1 & +``` + +### 2. File the AMD upstream issue (SYNC log parsed — ready to file) + +All evidence is collected. File at **https://github.com/ROCm/rocm-libraries/issues**, +component: **rocBLAS / Tensile**. + +Include: +- `scripts/repro_gfx1201_dense.py` (improved dense-VA standalone reproducer) +- Full ShaderName from run #9 / diagnostic log +- Operand descriptors: A=(608,21504) row-major bf16, B=(21504,5376) col-major NF4→bf16 +- Fault addresses (all 2 MB-aligned — confirmed in SYNC log: fault `0x7efb33000000`) +- Key finding: `bias=True` required (BH_Bias_UserArgs epilogue selects this tile); col-major B + is intrinsic to PyTorch + bitsandbytes 4bit matmul — no userspace workaround possible +- Env snapshot: `docs/diagnostics/gfx1201-report-env-20260605-101605.txt` +- SYNC log confirms same `MT64x64x64_ISA1201_DTVB1_VWA2_VWB1` ShaderName at fault point +- Tile selection divergence note: standalone probe dispatches `MT128x128x32`, not `MT64x64x64`. + **System-rocBLAS hypothesis DISPROVED** (session 10): both system (`/opt/rocm`) and + torch-bundled libraries dispatch identically — MT128x128x32 for all probe variants tested. + Tile divergence between standalone and production remains unexplained; AMD can reproduce + via rocprof on the full model training loop. + +--- + +## Current state + +| Item | Value | +|---|---| +| Training script | `scripts/train_sft.py` via `make train-gemma4-31b-stage2-unsloth` | +| Monitor | **RUNNING** — active as of 2026-06-06 19:35, step 1305/4826 | +| Monitor log | `outputs/monitor_stage2.log` | +| Train log | `outputs/sft-stage2-gemma4-31b/train.log` | +| Latest checkpoint | `checkpoint-1290` (safe); actively training past it | +| Total steps | 4,826 (1 epoch, 77k records, batch 1×16) | +| Per-step time | ~71 s/it (async); slower under SYNC | +| GPU state | **Clean** — GPU preflight PASSED (post-reboot, session 10) | +| HF backup repo | `ulises-c/SocratesLM-31B-stage2b-QLoRA` (auto-push every 50 steps) | +| W&B project | `csen346-sft` at `uchavarria-santa-clara-university` | + +--- + +## Recurring venv issue — check this first every session + +**`torch+rocm7.2` gets silently overwritten by `torch+cu130` (CUDA build)** after `uv sync` +or any pip/uv install outside the Makefile. This has happened in sessions 6 and 7. + +Always verify at the start of any session before touching the GPU: + +```bash +uv run --no-sync python -c "import torch; print(torch.version.hip)" +# Must print e.g. "7.2.26015". If it prints "None" → ROCm torch is missing. +make install-rocm # fix: reinstalls torch==2.11.0+rocm7.2 +``` + +--- + +## How the crawl works + +`monitor_stage2.sh` runs a loop: launch training → poll every 5 min → on crash: archive +log + dmesg, clean GPU (waits up to 180s for KFD to drain), quarantine any partial +checkpoint, relaunch with a **fresh `TRAIN_DATA_SEED=$(date +%s)`**. + +The rotating seed (commit `587b60e`) reshuffles the post-resume sample sequence each cycle, +converting the sticky-deterministic gfx1201 page fault back to probabilistic. At a fixed seed +the fault occurs deterministically within steps 22–24 from checkpoint-20; from checkpoint-1230 +the window is 0–100 steps (wider, still seed-dependent). + +**Forward-progress guard:** `MAX_RETRIES=8` consecutive no-progress retries → monitor posts +`STALLED` to the Training Log issue (#120) and exits. A new checkpoint resets the counter. + +**Where posts go:** all events — progress (every `PROGRESS_EVERY=50` steps, with +time/loss/grad_norm/lr), crash, STALLED, COMPLETE — append a row to a **single pinned +comment** (the "Live training log" table) on **issue #120**, not PR #101. The comment id is +pinned in `monitor_stage2.sh` as `LOG_COMMENT_ID`; `log_row` fetches that comment, appends a row, +and PATCHes it back, so restarts and `outputs/` wipes never lose history or mint a second comment. +**Starting a brand-new crawl:** create a fresh placeholder comment on #120 and paste its numeric id +into `LOG_COMMENT_ID` (same "bump for a new run" convention as `WANDB_RUN_ID`). + +**HF auto-backup:** `HFCheckpointCallback` (via `TRAIN_HF_REPO` env var set in Makefile) +pushes each saved checkpoint to `ulises-c/SocratesLM-31B-stage2b-QLoRA` in a daemon thread, +at a cadence of `TRAIN_HF_PUSH_EVERY` (default 50) step boundaries. + +- **Persistence:** After each successful push, the step is recorded in `{output_dir}/.hf_last_push`. + On resume, the callback reads this file so already-uploaded checkpoints are skipped. +- **Crash recovery:** On `on_init` (trainer initialisation), the callback scans + `output_dir/checkpoint-*`, picks the highest step, and pushes it **synchronously** + if `.hf_last_push` reports a lower step. This catches the scenario where a daemon + thread was killed mid-push by a crash — the checkpoint gets uploaded before training + resumes, so it is never lost. +- **Thread safety:** The existing `self._thread.is_alive()` guard prevents concurrent + pushes. Since the launch push is synchronous, there is no race with training-time pushes. +- **On-disk limit:** `save_total_limit=5` keeps only the last 5 local checkpoints; + HF stores every checkpoint pushed. +- **Logging verbosity:** `TRANSFORMERS_VERBOSITY=error` (level 1) is now set in + `train_sft.py` to suppress noisy INFO (level 3) output during long SFT runs. + +--- + +## If training is running + +Leave it alone. Watch issue #120 (Training Log) for crash/progress posts. The training is autonomous — no +action needed unless: +- The monitor posts `STALLED` → see below +- A crash post shows something unexpected (OOM, not a page fault) +- Loss diverges (grad_norm consistently > 50, loss climbing after step 100) + +--- + +## If training crashed and the monitor is handling it + +Normal — monitor will clean, reseed, and relaunch. You'll see an issue #120 comment like +`**Stage 2 CRASHED** (no-progress retry N/8, latest ckpt step X)`. No action needed +unless it STALLs. + +--- + +## If the monitor STALLED + +Monitor posted: `## Stage 2 Training: STALLED (no progress in 8 retries)` + +**Most likely cause:** wedged GPU KFD context. Fix sequence: + +1. Verify GPU dirty: `rocm-smi --showmeminfo vram` → expect ~24 GB despite no processes +2. `lsof /dev/kfd /dev/dri/renderD128` → `pkill nvtop` if listed; recheck after 5s +3. If still dirty: `sudo modprobe -r amdgpu && sudo modprobe amdgpu` +4. `make gpu-preflight` → must PASS +5. `nohup bash scripts/monitor_stage2.sh > outputs/monitor_stage2.log 2>&1 &` +6. Watch for tokenization deadlock on first restart: if `train.log` stalls at + `Tokenizing train dataset (num_proc=12): XX%` for >60s without advancing, the + multiprocessing pool deadlocked. `pkill -9 -f 'train_sft\.py'` — monitor will retry. + If repeated: patch `train_sft.py` to use `dataset_num_proc=1`. +7. If STALLs again after clean GPU → note in issue #120, escalate to cloud GPU. + +--- + +## If training completed + +Monitor posted: `## Stage 2 Training: COMPLETE ✓` + +Adapter at `outputs/sft-stage2-gemma4-31b/final/`. Next steps: + +1. **Verify adapter loads:** + ```bash + uv run --no-sync python -c " + from peft import PeftModel + m = PeftModel.from_pretrained('outputs/sft-stage2-gemma4-31b/final', + 'unsloth/gemma-4-31B-it-unsloth-bnb-4bit') + print('OK') + " + ``` +2. Merge and export: `uv run --no-sync python scripts/merge_lora_gemma4_sft.py` +3. Run downstream eval (BERT consultant routing + LLM judge) — post results to issue #120 +4. Update `docs/GFX1201_FAULT_ABLATION_LOG.md` with final step count + fault events + +--- + +## Key invariants — do not violate + +- **Always `uv run --no-sync`** — bare `uv run` reinstalls CUDA torch over ROCm +- **Always verify `torch.version.hip` is not None** before any GPU work (regression recurs) +- **`make gpu-preflight` before any manual launch** — dirty KFD cascades into early faults +- **Do not add `AMD_SERIALIZE_KERNEL` or `HIP_LAUNCH_BLOCKING` to the monitor** — SYNC mode + makes the fault deterministic; all SYNC runs are diagnostic only +- **Do not restart the monitor without checking if it's already running** (`pgrep -f monitor_stage2`) + +--- + +## What is closed — do not re-investigate + +The gfx1201 ISA1201 Tensile GEMM bug is **fully root-caused and documented**. All +ablation arms are exhausted. See `docs/GFX1201_FAULT_ABLATION_LOG.md`. + +The `ROCBLAS_LAYER=2` bench logging path is a dead end through PyTorch: PyTorch routes +through `libhipblas.so → librocblas.so` internal dispatch, bypassing the C API logging +hooks. Both B-matrix encodings for `rocblas-bench gemm_ex` dispatch `ISA000` (MLIR +generic), never `ISA1201`. Confirmed in session 6. + +The standalone reproducer (`scripts/repro_gfx1201_rocblas.py`) dispatches the faulting +kernel but does NOT crash in a small process — the wild address lands on mapped VA because +the process VA footprint is too small. This is expected and documented in the script. Do not +re-investigate or try to make it crash in isolation. + +Do not: +- Re-run probe scripts (probes 1–3 done; `capture-rocblas-bench.sh` / `replay-rocblas-bench.sh` confirmed dead end) +- Try `TORCH_USE_HIPBLASLT=1` (run #11 — no kernel for this shape, Tensile fallback) +- Try `BNB_FORCE_B_CONTIGUOUS=1` (run #12 — Python `.contiguous()` can't reach BLAS descriptor) +- Try `expandable_segments:True` (run #10 — silently ignored on gfx1201) +- Try `ignore_data_skip=True` (rejected — silent garbage adapter, no stall alarm) + +--- + +## Open items + +- **File AMD upstream issue** — all evidence collected (see Immediate actions §2 above). + URL: https://github.com/ROCm/rocm-libraries/issues · component: rocBLAS / Tensile. + SYNC log parsed: fault `0x7efb33000000` (2MB-aligned), `MT64x64x64_ISA1201_DTVB1_VWA2_VWB1` + kernel confirmed at crash point, last B-operand probe shape=(21504,5376) col-major. + System-rocBLAS hypothesis CLOSED in session 10 (both libraries dispatch identically). + +- **Monitor running** — no action needed. Check `tail -20 outputs/monitor_stage2.log` for status. + +--- + +## Session history + +**Session 4 (2026-06-02):** Crawl launched from checkpoint-90. Forward progress to +checkpoint-1230 overnight (~50 crashes, all recovered). Tokenization deadlock after +step-1230 crash wedged the GPU; monitor STALLed 8/8 at 17:19 on 2026-06-03. + +**Session 5 (2026-06-04):** Monitor and training dead, GPU dirty (24 GB wedged KFD). +No code changes. Commits: env snapshot script, rocBLAS probe scripts. + +**Session 6 (2026-06-05 AM):** +- Ran `capture-rocblas-bench.sh` + `replay-rocblas-bench.sh` → confirmed `ROCBLAS_LAYER` + dead through PyTorch's hipBLAS path; `rocblas-bench gemm_ex` only dispatches `ISA000` +- Fixed venv torch build regression (first occurrence): `torch+cu130` → `torch+rocm7.2` +- Env snapshot: `docs/diagnostics/gfx1201-report-env-20260605-101605.txt` +- Created HF model repo `ulises-c/SocratesLM-31B-stage2b-QLoRA` with README + ATTRIBUTION +- `scripts/train_sft.py`: `save_total_limit` 2→5; `HFCheckpointCallback` added +- Extracted `HFCheckpointCallback` → `src/project/hf_callback.py` + 7 unit tests +- Commits: `71d6111` (refactor HFCheckpointCallback), `e6ed01a` (HF auto-push + limit) + +**Session 7 (2026-06-05 PM):** +- GPU was clean at start — no driver reset needed (24 GB wedge from session 4 was gone) +- ROCm torch regression recurred: `torch+cu130` again; fixed with `make install-rocm` +- **Wild address analysis:** all 12 production crash addresses are exactly 2 MB-aligned; + fault consistently lands ~1.6 GB above B.end regardless of ASLR +- **Standalone reproducer written:** `scripts/repro_gfx1201_rocblas.py` — `bnb.nn.Linear4bit` + at exact fault shape (in=21504, out=5376, bias=True, nf4, bf16); does not crash in small + process (sparse VA); confirmed behaviour expected and documented +- **Diagnostic SYNC run launched** from checkpoint-1230: AMD_SERIALIZE_KERNEL=3, + AMD_LOG_LEVEL=3, BNB_DEQUANT_PROBE=1, TRAIN_SAVE_STEPS=99999. Log at + `docs/diagnostics/diag-sync-probe-step1230-20260605-124008.log` +- Monitor NOT restarted — diagnostic run occupied GPU at end of session +- Commit: `109eb72` (standalone reproducer + codespell *.log skip) + +**Session 9 (2026-06-06):** +- Machine rebooted to clear 31 GB wedged KFD context (stuck from probe run ending session) +- **Dense-VA reproducer run** (`scripts/repro_gfx1201_dense.py`): 20 GB filler allocated, + 200 iters forward+backward — no crash. `MT128x128x32_ISA1201_DTVB1` dispatched 401 times. + `MT64x64x64` NOT dispatched (only appears in script's own print statement). +- **Tile-selection probe** (`scripts/probe_gfx1201_tile.py`): 9 variants tested — all + dispatch `MT128x128x32`. Variants: 2D/3D shape, with/without `torch.utils.checkpoint` + (reentrant=False and True), `torch.autocast`, skip-warmup (cold Tensile cache), + non-contiguous A (stride=[43008,1]). +- **Bitsandbytes forward traced**: `dequantize_4bit` returns `(21504,5376)` col-major + (stride=(1,21504)). After `.t()` = `(5376,21504)` row-major. `F.linear` calls + `addmm(bias, A, W.t())` where `W.t()` = `(21504,5376)` col-major → this IS the DTVB1 + path. Both standalone and production make structurally identical BLAS calls. +- **Leading hypothesis**: torch-bundled rocBLAS library (`torch/lib/rocblas/library/`) + does not include `MT64x64x64` kernels for gfx1201; system ROCm 7.2 library + (`/opt/rocm/lib/rocblas/library/`) does. `ROCBLAS_TENSILE_LIBPATH` override blocked by + Railguard path fence — needs user approval for next session. +- New probe script committed: `scripts/probe_gfx1201_tile.py` +- Comment posted to issue #113 with full findings +- Commits: `aac092f` (dense-VA reproducer + handoff update) + +**Session 10 (2026-06-06 evening):** +- GPU preflight PASSED (clean post-reboot) +- Monitor was already running (PID 17208) — auto-started by the session's environment +- Training recovered from one crash and progressed: step 1305/4826 at 19:35 PST +- **System-rocBLAS hypothesis DISPROVED**: compared `probe_3d_plain.log` vs + `probe_system_rocblas.log` — both dispatch `MT128x128x32` identically. The two apparent + `MT64x64x64` hits were from the probe script's own `print()` statements, not actual + kernel dispatches. System library and torch-bundled library are equivalent for gfx1201. + Tile selection divergence (standalone→MT128x128x32, production→MT64x64x64) remains + unexplained at the binary level; AMD must reproduce via rocprof on the full model. +- Commits: this session (handoff + probe script formatting) + +**Session 8 (2026-06-05 evening):** +- **SYNC diagnostic log parsed** (26 GB, 230M lines): Fault confirmed at `0x7efb33000000` + (2MB-aligned). `MT64x64x64_ISA1201_DTVB1_VWA2_VWB1` ShaderName confirmed at crash point + — same kernel as all prior crashes (runs #5, #7, #9). Last B-operand probe: shape=(21504,5376) + col-major. SYNC fault occurred within ~1–2 steps of resuming checkpoint-1230 (SYNC makes + it deterministic). All evidence ready for AMD upstream report. +- **Regular crawl** progressed from checkpoint-1230 → checkpoint-1290 (60 more steps at ~71 s/it) +- Training crashed again (~step 1290+), process was spinning at 141% CPU +- All training/wandb processes killed (PIDs 372896, 372901, 373486, 373502) +- Monitor NOT restarted — GPU dirty after crash + +``` +109eb72 feat(diag): standalone gfx1201 ISA1201 Tensile GEMM reproducer +71d6111 refactor(train): extract HFCheckpointCallback to module level + tests +e6ed01a feat(train): HF auto-push callback + raise save_total_limit to 5 +587b60e feat(monitor): rotate TRAIN_DATA_SEED per resume to break sticky fault +``` + +--- + +## Key files + +| File | Purpose | +|---|---| +| `docs/GFX1201_FAULT_ABLATION_LOG.md` | Canonical run log — append after every run | +| `scripts/monitor_stage2.sh` | Crawl harness — rotating seed, KFD cleanup, issue #120 progress/crash posts | +| `scripts/train_sft.py` | Training script; HF auto-push + `TRAIN_DATA_SEED` + `BNB_DEQUANT_PROBE` wired | +| `scripts/repro_gfx1201_rocblas.py` | Standalone AMD upstream reproducer (dispatches ISA1201 kernel) | +| `src/project/hf_callback.py` | `HFCheckpointCallback` — async HF push with `.hf_last_push` persistence, `on_init` crash recovery, skip-if-in-flight | +| `tests/test_hf_callback.py` | 14 unit tests for `HFCheckpointCallback` | +| `outputs/sft-stage2-gemma4-31b/crashlogs/` | Per-crash full log + dmesg archive | +| `docs/diagnostics/diag-sync-probe-step1230-20260605-124008.log` | Session 7 diagnostic — parse first | +| `docs/diagnostics/gfx1201-report-env-20260605-101605.txt` | Env snapshot for AMD upstream report | +| PR #101 | Full diagnostic thread + all session findings | diff --git a/docs/SFT_HANDOFF.md b/docs/SFT_HANDOFF.md new file mode 100644 index 00000000..809308e9 --- /dev/null +++ b/docs/SFT_HANDOFF.md @@ -0,0 +1,255 @@ +# SFT Handoff — Gemma 4 12B Socratic QLoRA (NVIDIA PoC, eval phase) + +**Updated 2026-06-22.** For a fresh session picking up the eval/compare work. +**Training is DONE (recovered — see below); the merge has run; eval has NOT started.** +This doc is the single source of truth for what to run next and the hazards. Cross-check +against `docs/EXPERIMENT_LOG.md` (newest at top) and the live logs on GitHub issue #130. + +Branch: `feat/gemma4-12b-sft-poc-nvidia`. All code fixes from this phase are committed and +pushed (HEAD `66becca` + monitor `730906a` + live-quant `73c72e0`). + +--- + +## The one-sentence goal + +Measure whether Socratic QLoRA SFT on Gemma 4 12B improves the KELE eval over the **base** +Gemma 4 12B teacher, everything else fixed. Uplift = SFT − base on **state accuracy**. + +## The number to beat + +**State accuracy 50.30%** — base teacher, MTP-on, full Chinese test set (n=681) +(`results/gemma4-12b-base-mtp`). Run-to-run σ ≈ 0.7 pp, so an uplift is only real if it +clears **~1.5 pp**. Per-stage base (MTP-on): a 100.0 · b 44.55 · c 32.85 · d 35.98 · e 60.21. + +--- + +## Tracking — where eval progress is recorded + +- **GitHub issue #130** is the live tracker. The eval monitor (`monitor_eval_gemma4_12b.sh`) + **auto-appends** progress/crash/complete rows to its pinned eval-log comment (`4644703104`). + Do **not** post separate handoff/status comments on #130 — the only comments there are the + monitor's train/eval log tables. This markdown doc is the handoff; #130 is the live log. +- **W&B** is the metrics tracker. Eval → project **`csen346-eval`** (org + `uchavarria-santa-clara-university`), run auto-named after the output-dir basename + (`gemma4-12b-sft-mtp`); per-dialogue metric curves log every 10 dialogues (`WANDB_EVAL_LOG_EVERY`). + `WANDB_EVAL=1` is set by the monitored eval target. Training W&B (reference): `csen346-sft`, + run `gemma4-12b-qlora-poc` (shows the NaN spike). + +--- + +## ⚠️ What the SFT adapter actually is (read this — provenance matters) + +The training run **diverged to NaN at step ~4260 / 4826 (epoch 0.88)** — a single bad batch +blew the loss to 1.5e8, grad → NaN, weights corrupted. `save_total_limit=5` then evicted every +clean **local** checkpoint before the NaN was noticed. **It was recovered from HuggingFace +commit history** (HF keeps all commits; local eviction is not mirrored). The recovered adapter: + +- **`checkpoint-4250`** — step 4250, **epoch 0.881**, loss 0.6041, mean_token_accuracy 0.8052, + verified **0/656 tensors NaN**. This is a well-converged **0.88-epoch** adapter (the missing + 12% was at LR decaying 6e-6→0, so negligible — but **report it as ~0.88 epoch, not "1 epoch",** + in the experiment log so the result isn't overstated). +- Staged locally at **`outputs/sft-gemma4-12b-qlora/recovered-4250/`** (adapter_config.json + + adapter_model.safetensors + tokenizer). +- Also on HF `ulises-c/SocratesLM-12B-QLoRA` at HEAD (NaN dirs 4300–4750 were deleted; HEAD is + now clean `checkpoint-3200..4250`). Commit `38cdcc8299` is a known-good snapshot. + +See memory `sft-nan-divergence-checkpoint-eviction` for the full story + the lesson +(check `list_repo_commits` before declaring a diverged run lost). + +### Why training was stopped (not resumed/restarted) + +Training was **manually stopped at step ~4800**, deep into a NaN run that began at ~4260. We did +**not** resume or restart because: +- checkpoint-4250 (the last clean step) is a **well-converged adapter** — loss had plateaued at + ~0.60 since step ~3000 (epoch ~0.62), and the remaining ~12% of the epoch was at LR decaying + 6e-6→0, i.e. near-zero additional learning. So 0.88 epoch ≈ a full epoch for practical purposes. +- Resuming would have re-loaded a NaN local checkpoint (all surviving local checkpoints were NaN); + resuming from the HF-recovered 4250 to finish 576 steps risked re-diverging for negligible gain. +- The PoC's question (does SFT beat the 50.30 base?) is answerable with checkpoint-4250 **now**; + a cleaner full run can follow if the uplift is promising. + +--- + +## Recommendations for a future / better SFT run + +If the eval shows promise and a cleaner or stronger run is wanted, prioritize **stability first** +(the NaN divergence, not undertraining, was the failure): + +**Stability (do these — this is what bit us):** +1. **NaN/inf-abort callback** — stop training the instant `grad_norm` or `loss` is non-finite + (detect at `logging_steps=10`, which beats `save_steps=50`). Turns a silent 500-step NaN bleed + into an immediate, recoverable stop. If it exits non-zero, `monitor_train_gemma4_12b.sh` + relaunches from the latest checkpoint with a fresh `TRAIN_DATA_SEED` → steps over the bad batch. +2. **Protect checkpoints** — raise `save_total_limit` (currently **5** in `scripts/train_sft.py`, + which evicted the last-good checkpoint before the NaN was noticed) to ~20, or keep a permanent + checkpoint every N steps. HF history saved us this time; don't rely on it. +3. **Tighten `max_grad_norm`** to ~0.3 (HF default 1.0). The step before divergence had grad_norm + 2.247; tighter clipping buys margin against a grad-driven spike. +4. **Compute the loss/logits in fp32.** The likely root cause is an inf-logit *forward* spike over + Gemma's 262k vocab in bf16 (divergence happened at LR≈6e-6, so it's not an LR-magnitude blow-up). + An fp32 final-logits / cross-entropy path is the most direct fix; consider also a + skip-non-finite-grad optimizer step for true single-bad-batch tolerance. +5. Optionally identify/quarantine the offending record (the batch near step 4260 under + `data_seed=1782102499`) if divergence recurs across seeds. + +**Learning rate / schedule:** current = `5e-5` linear decay, **no warmup**. LR magnitude probably +wasn't the trigger (divergence was late, at low LR), but adding `warmup_ratio≈0.03` and/or lowering +peak LR to **2–3e-5** is standard for 12B QLoRA and adds margin. + +**Epochs:** this run was ~0.88 epoch and loss had already plateaued (~0.60 from step ~3000). More +epochs (2–3) are the natural lever for *more* SFT, but watch for overfitting on the per-turn set and +note each epoch is **~30 h** at 85 W (~29 s/step) on this box — let the eval uplift decide whether +additional epochs are worth the wallclock + instability exposure. + +**Throughput / box:** ~29 s/step at the stable 85 W → ~30 h/epoch; the 20 GB card + power surge are +the bottleneck and instability risk grows with wallclock. A bigger/stable GPU would allow higher +power, a larger batch (currently 1×16), and shorter, safer runs. + +--- + +## What is FIXED between base and SFT (do not change — it isolates the delta) + +- **Consultant = Qwen3.5-0.8B LoRA state classifier** (`results/state-clf-qwen3.5-0.8b-lora-wandb/final`, + HF `ulises-c/socrates-state-classifier-qwen3.5-lora`), passed via `--bert-consultant`, runs on + **CPU** (`KELE_BERT_DEVICE=cpu`). Despite the "bert" naming, it's the Qwen3.5 classifier. The + teacher is the ONLY thing that changes between base and SFT. +- **Bare teacher prompt** — no fewshot. Both base and SFT run bare. +- **Dataset**: Chinese-only test split, n=681 (`ulises-c/SocratDataset` default). +- **MTP = OFF for the SFT eval, run concurrent** (`KELE_PARALLEL_WORKERS≥4`). MTP is lossless but + a single-stream latency win that loses on wall-clock to concurrency on this box — see Step 5's + rationale. Sampling is stochastic (no temp/seed) so neither MTP nor batching biases the metric. +- **Base lineage = `unsloth/gemma-4-12b-it`** — NOT google. The base-teacher GGUF was + `unsloth/gemma-4-12b-it-GGUF` and the adapter was trained on `unsloth/gemma-4-12b-it`, so the + merge MUST use the unsloth base (the handoff's old `google/...` ref was wrong). + +--- + +## The pipeline — what's done and what remains + +### Step 2 — Train ✅ DONE (recovered, see above). Adapter = `recovered-4250/`. + +### Step 3 — Merge LoRA → HF BF16 ✅ DONE (or finishing) + +``` +uv run --no-sync python scripts/merge_lora_gemma4_sft.py \ + --base unsloth/gemma-4-12b-it \ + --adapter outputs/sft-gemma4-12b-qlora/recovered-4250 \ + --out outputs/sft-gemma4-12b-qlora/merged +``` + +Output: `outputs/sft-gemma4-12b-qlora/merged/` (BF16, ~24 GB, CPU merge, ~10-15 min). +**Verify it finished**: `merged/` should hold `model-*.safetensors` totaling ~24 GB + config. +If `merge.log` shows "Done"/all shards written, it's complete; if interrupted, just re-run the +command (idempotent overwrite). + +### Step 4 — Convert merged → Q8_0 GGUF (NOT YET RUN) + +``` +bash scripts/convert_gemma4_12b_sft_to_gguf.sh +``` + +Writes `gemma-4-12B-kele-socratic-sft-Q8_0.gguf` and stages it where the serve wrapper looks. +Q8_0 matches the base teacher's bit budget so the quant delta is noise. The script pre-flights +its llama.cpp deps (`~/Documents/models/llama.cpp`: `convert_hf_to_gguf.py` + `build/bin/llama-quantize`) +and fails loud if missing. **This step has not been exercised this phase — watch for missing +llama.cpp binaries** (memory `gemma4-12b-nvidia-poc-stack` flagged a possible build gap). + +### Step 5 — Eval (MTP **off**, concurrent) + compare (NOT YET RUN) + +``` +make serve-gemma4-12b-sft # NO MTP=1 → -np 4, q4_0 KV +KELE_PARALLEL_WORKERS=4 make monitor-eval-gemma4-12b-sft # → results/gemma4-12b-sft +python -m src.project.evaluate --compare results/gemma4-12b-base results/gemma4-12b-sft +``` + +Sanity-gate first: `make eval-gemma4-12b-sft-smoke` (n=5, no monitor). + +**Why MTP-off + concurrent (decided 2026-06-22 — supersedes the old "MTP on, serial" plan).** +The two base runs already on disk are a natural A/B: + +| run | MTP | workers | throughput | wall-clock | state acc | +|---|---|---|---|---|---| +| `gemma4-12b-base-mtp` | on | 1 | 23.6 dlg/hr | 28.9 h | 50.30 | +| `gemma4-12b-base` | off | 4 | 39.1 dlg/hr | **17.4 h** | 49.62 | + +- **MTP and concurrency fight on this 20 GB box, so you pick one.** MTP forces f16 KV + (`-ctk/-ctv f16`, 2× the q4_0 footprint) → OOM/checkpoint-bloat risk with >1 slot (the + documented 2026-05-26 crash). Speculative decoding is a *single-stream latency* win; under + continuous batching the GPU is already saturated so the draft+verify overhead competes and + the gain collapses — worse here because the base-derived drafter has low acceptance on the + SFT'd distribution. That's why the MTP run was (correctly) serial — and serial made it the + *slower* wall-clock despite the "~2× faster" single-stream claim. +- **Neither MTP nor batching biases the metric.** The teacher call sets no temperature/seed + (`tournament_utilizations.py:440` → server default ≈0.8, stochastic); both MTP and continuous + batching are distribution-preserving. So the 0.68 pp gap between the two base runs is pure + sampling noise (this is the σ≈0.7 pp estimate), and the SFT accuracy is comparable to **either** + base regardless of speed config. We optimize purely for **min wall-clock at the stable 85 W + point**, because instability exposure scales with wallclock on this box. +- **Concurrency is the live lever, not MTP** — but it plateaus fast. Smoke A/B on the Q8_0 SFT + GGUF (n=18, same dialogues, `-np 6` server, 2026-06-23): **workers=4 → 226.0 dlg/hr**, + workers=6 → 245.3 dlg/hr (+8.5% for +50% slots, 0 errors). The GPU is compute-bound at 4 + concurrent decodes, so the extra slots barely help and `-np 6` carries the documented + KV-checkpoint-bloat crash risk over a 681-run. **Chosen: workers=4 / `-np 4`** (serve default). + Clean-burst 226 dlg/hr ⇒ ~3 h of compute for the full 681 (the base's 17 h effective rate was + inflated by crash/resume overhead, not raw speed). q4_0 KV gives headroom MTP's f16 KV wouldn't. + +**Compare against BOTH bases.** The honest baseline band is **49.6–50.3**; gate the uplift on the +higher (50.30), so SFT must clear **~51.8** (>1.5 pp ≈ 2σ) to be real. A partial eval at n≈300 is a +valid early signal (convergence stable from n≈200-300). + +**Optional MTP confirm run afterward.** If the concurrent SFT eval lands close to the gate and the +result hinges on it, a follow-up `MTP=1 … KELE_PARALLEL_WORKERS=1 make monitor-eval-gemma4-12b-sft` +(→ `results/gemma4-12b-sft-mtp`, serial, ~29 h) re-measures under the exact MTP-on config the +50.30 base used — removing any lingering "but the base was MTP-on" objection. Skip it if the +concurrent run is a clear pass or clear fail. + +--- + +## Hazards & gotchas (read before launching) + +1. **Box is KNOWN-UNSTABLE** (RTX 4000 Ada 20 GB, power surge → crashes under load). Eval crawls + proved stable at **85 W**; the monitor (`scripts/monitor_eval_gemma4_12b.sh`) auto-resumes the + eval, repairs error/truncated dialogues, polls server health out-of-band, logs to issue #130. +2. **GPU power step-down is INERT** — passwordless sudo is unavailable, so the monitors' power + search can't change the limit; the card stays at whatever's persisted (85 W). Logged wattage in + issue rows is cosmetic. Safe (85 W is the stable point). See memory `power-step-down-needs-passwordless-sudo`. +3. **One model fits at a time** on the 20 GB card. Stop the base server before serving SFT (both + bind 8080). SFT serve uses alias "Gemma 4 12B SFT" so eval fails fast if it hits base weights. +4. **GGUF conversion (step 4) is the untested link this phase.** If it errors on a missing binary, + build llama.cpp (cmake/gcc-vs-nvcc friction noted in memory). The base teacher is already served + via llama-server, so llama.cpp is present — but the quantize/convert binaries may need building. +5. **SFT serve script** (`scripts/serve_gemma4_12b_sft.sh`) lacks the `GEMMA4_12B_KV` passthrough + the base one has — irrelevant for the MTP=1 eval (MTP forces f16 KV anyway). +6. **The training monitor now fast-fails on OOM and self-heals on divergence-abort** — not needed + for eval, but if you ever retrain: it relaunches with a fresh data_seed from the latest local + checkpoint. To retrain cleanly, add a NaN-abort callback + raise `save_total_limit` (currently 5, + which caused the eviction). The training-side OOM/eval-skip/live-quant fixes are committed. + +--- + +## Where things live + +- **Plan + live status**: GitHub issue **#130** (eval log comment `4644703104`; the training log + comment `4761132326` holds the NaN-run history). +- **Recovered adapter**: `outputs/sft-gemma4-12b-qlora/recovered-4250/`; HF `ulises-c/SocratesLM-12B-QLoRA`. +- **Merged model**: `outputs/sft-gemma4-12b-qlora/merged/` (step 3 output, BF16 24 GB); + HF `ulises-c/SocratesLM-12B` (private). +- **Q8_0 GGUF**: `outputs/sft-gemma4-12b-qlora/gemma-4-12B-kele-socratic-sft-Q8_0.gguf` (12 GB, + step 4 output, staged to the serve weights dir); HF `ulises-c/SocratesLM-12B-GGUF` (private). +- **Configs**: `configs/gemma4-12b-sft-local.env` (SFT eval), `configs/gemma4-12b-local.env` (base eval), + `configs/train-sft-gemma4-12b-qlora.env` (training). +- **Scripts**: `scripts/merge_lora_gemma4_sft.py`, `scripts/convert_gemma4_12b_sft_to_gguf.sh`, + `scripts/serve_gemma4_12b_sft.sh`, `scripts/monitor_eval_gemma4_12b.sh`, + `scripts/monitor_train_gemma4_12b.sh` (training babysitter, new this phase). +- **W&B**: training → `csen346-sft` (run `gemma4-12b-qlora-poc`); eval → `csen346-eval`. +- **Tests**: pre-commit runs ruff + pyright + codespell + shellcheck + full pytest (~9 s). Use + `uv run --no-sync` for Python. +- **Relevant memories**: `sft-nan-divergence-checkpoint-eviction`, `power-step-down-needs-passwordless-sudo`, + `gemma4-12b-nvidia-poc-stack`, `training-host-hardware-fault`, `use-uv-run-for-tests`. + +## Definition of done + +`results/gemma4-12b-sft/metrics_summary.json` exists with 681/681 valid dialogues; the +`--compare` output quantifies SFT − base state accuracy (real only if > ~1.5 pp); issue #130's +checklist is updated; and an `EXPERIMENT_LOG.md` entry records the uplift with per-stage breakdown +**and notes the adapter is a ~0.88-epoch checkpoint recovered from HF after a NaN divergence.** diff --git a/docs/SFT_NVIDIA_MIGRATION.md b/docs/SFT_NVIDIA_MIGRATION.md new file mode 100644 index 00000000..fc96a890 --- /dev/null +++ b/docs/SFT_NVIDIA_MIGRATION.md @@ -0,0 +1,111 @@ +# Recommendation — Move Stage-2 SFT to NVIDIA (conditional) + +**Date:** 2026-06-01 · **Branch:** `feat/gfx1201-rdna4-qlora-fla-training` · **PR:** #101 +**Context:** Gemma-4-31B QLoRA SFT is blocked on the R9700 (gfx1201/RDNA4) by a confirmed +upstream rocBLAS/Tensile ISA1201 GEMM bug (wild address from a column-major B descriptor — +see `GFX1201_FAULT_ABLATION_LOG.md`, issue #113). All userspace routing/placement arms are +exhausted (hipBLASLt has no kernel for the faulting shape; allocator knobs are no-ops on +gfx1201). The remaining on-box path is a brute-force checkpoint crawl of unproven viability. + +--- + +## Bottom line + +If the deliverable is the **trained adapter**, move the SFT run to an NVIDIA GPU now. It is +the cheapest, fastest, lowest-risk path to a clean checkpoint, and it does **not** discard the +gfx1201 work — that survives as a documented contribution (#113 upstream report, #109 field +report). This recommendation flips only if "31B QLoRA end-to-end on RDNA4" is itself a paper +claim — see the gate below. + +--- + +## 1. The decision is gated on two things, both resolvable in minutes + +### Goal fork +Move only under **"the adapter is the deliverable"** (it feeds the SFT eval / the locked 72.24 +baseline, Tables 6/14). If **RDNA4 viability** is the claim, do not move — but note you +**already hold** that contribution: the characterization in #113 + #109 substantiates "we ran +31B QLoRA on RDNA4 and root-caused the Tensile bug." A *completed* 94h gfx1201 run is not +required to make the field-report claim; the characterization is. + +### Compliance / data-residency +Flagged because the org is medical, but it very likely **does not bind here**: +- Training inputs are the `socrat-zh-sft` / `socrat-en-sft` Socratic teaching dialogues loaded + from HF; the base model is the HF-hosted `unsloth/gemma-4-31B-it-unsloth-bnb-4bit`. +- No PHI, nothing org-private leaves the box. Both are already public/HF-hosted, so a cloud GPU + re-downloads them directly with **zero sensitive-data transfer**. + +Confirm there is no private data in the pipeline. If confirmed, the residency objection +collapses and cloud is fully on the table. + +--- + +## 2. Hardware options + +| Option | VRAM | Cost for one ~94h run | Notes | +|---|---|---|---| +| **Own RTX 5090** | 32 GB | **$0** | CUDA → no Tensile bug. Fits the *same* config that fit R9700's 32 GB (keep grad-ckpt). Fastest to start. Caveat: prior training-host power-fault — verify box health first. | +| **Cloud A100 80 GB / L40S 48 GB** (recommended) | 48–80 GB | **~$140–235** (spot ~$1.5/hr → on-demand ~$2.5/hr × 94h) | Headroom to **disable grad-checkpointing** and raise batch → likely meaningfully <94h. Lambda / RunPod / Vast. | +| **Cloud H100 80 GB** | 80 GB | **~$280** (~$3/hr) | Fastest wall-clock; overkill but absolute cost is trivial for so short a run. | +| RTX 4090 / any 24 GB | 24 GB | — | **Too small** — run uses ~21 GB alloc / ~28 GB reserved. Don't. | + +The full clean run costs **~$150–300 on cloud, or $0 on the 5090** — against ~48h of +engineering already spent on crash diagnosis and a 150h+ *uncertain* crawl. The economics are +not close. + +> Note: 31B in bf16 (~62 GB weights) does not fit training even on 80 GB once optimizer state +> is added — **keep QLoRA/NF4** regardless of card. The win from a bigger card is headroom to +> disable grad-ckpt recompute and raise batch, not dropping quantization. + +--- + +## 3. Migration effort: ~half a day, mostly *deleting* ROCm workarounds + +The training code (`peft` / `trl` / `transformers` / `SFTConfig`, `scripts/train_sft.py`) is +hardware-agnostic — no logic change. The move is subtractive: + +- **Drop** every ROCm env workaround: `TORCH_USE_HIPBLASLT`, `PYTORCH_HIP_ALLOC_CONF`, + `AMD_SERIALIZE_KERNEL`, `HIP_LAUNCH_BLOCKING`, `HSA_ENABLE_SDMA`, and the `make gpu-preflight` + / KFD-clean dance. +- **Remove** the `BNB_DEQUANT_PROBE` instrumentation (`scripts/train_sft.py:~547`) — its job is + done. +- **Swap** torch+rocm → torch+cu12, install the **CUDA `bitsandbytes`** (the reference build — + none of the gfx1201 bnb grief). The `uv run --no-sync` caveat *flips in your favor*: CUDA + torch is now the default, not the thing you fight to keep. +- **Optionally enable** `attn_implementation="flash_attention_2"` (FA2 is first-class on + NVIDIA; the R9700 path used SDPA). + +--- + +## 4. What you gain beyond "it doesn't crash" + +- **One clean run** — eliminates the entire crawl apparatus *and* the LR-schedule-consistency + risk that multi-resume introduced (a single run has one continuous schedule by construction). +- **Likely faster wall-clock** — with 48–80 GB you can disable grad-checkpoint recompute and/or + raise batch; QLoRA on H100/A100 should beat the R9700's ~70s/step. **Verify with a 20-step + smoke run** before committing the full run — do not take this on faith. +- **Scientific validity intact** — a different GPU yields bitwise-different weights, but the + adapter is *evaluated* downstream; as long as the eval harness is held constant, hardware + choice does not compromise the result. + +--- + +## 5. What you must preserve when leaving gfx1201 + +Moving the *run* does not abandon the *findings*: +- Keep #113 (upstream rocBLAS report) and #109 (field report) as the gfx1201 deliverable. +- Add a one-line note to `GFX1201_FAULT_ABLATION_LOG.md` that the SFT run was relocated to + NVIDIA for the adapter while the Tensile bug is upstream — so the log closes cleanly rather + than looking abandoned. + +--- + +## 6. Recommended sequence + +1. Confirm the goal fork + no-private-data (minutes). +2. Provision the box (5090 if healthy, else cloud A100 / L40S), install CUDA torch + bnb, pull + model + datasets from HF. +3. **20-step smoke run** — confirm no fault, record s/step, confirm loss in-family. +4. **EOS/generation gate** on a short checkpoint (the #94 trust gate) — *still mandatory + regardless of hardware*. +5. Launch the single clean run; standard checkpointing as insurance, not as a crawl. diff --git a/docs/SFT_RESULTS_REPORT.md b/docs/SFT_RESULTS_REPORT.md new file mode 100644 index 00000000..80fdf0ba --- /dev/null +++ b/docs/SFT_RESULTS_REPORT.md @@ -0,0 +1,251 @@ +# Socratic QLoRA SFT vs Base — Results Report + +**Gemma 4 12B-it, NVIDIA PoC.** Branch `feat/gemma4-12b-sft-poc-nvidia`. Updated 2026-07-09. +Companion to `docs/SFT_HANDOFF.md` (pipeline/provenance) and `docs/SFT_VS_BASE_ANALYSIS_PLAN.md` +(follow-on ablations). Live tracker: GitHub issue #130. + +## TL;DR + +Socratic QLoRA SFT on Gemma 4 12B **improves state accuracy on every axis tested** — both +languages, in-distribution and out-of-distribution — with **no regressions**. The gain is large on +held-out in-distribution data (**+7.7 to +10.3 pp**) and smaller but consistently positive on +never-seen synthetic data (**+3.5 to +3.9 pp**), the healthy shape that shows the model learned a +**transferable Socratic skill, not corpus memorization**. Achieved with a **~0.88-epoch** adapter +recovered after a NaN divergence — i.e. a lower bound on a clean full run. + +## Headline: state accuracy (the metric of record) + +| eval set | distribution | base | SFT | **Δ (pp)** | n | +|---|---|---|---|---|---| +| ZH test | in-distribution (held-out 10%) | 49.62 | **59.93** | **+10.31** | 681 | +| EN test | in-distribution (held-out 10%) | 51.00 | **58.68** | **+7.68** | 681 | +| ZH synthetic | OOD (never trained) | 27.19 | **31.09** | **+3.90** | 75 | +| EN synthetic | OOD (never trained) | 27.44 | **30.97** | **+3.53** | 75 | + +Run-to-run σ ≈ 0.7 pp (decoding is stochastic), so the held-out gains are ~11–15σ — unambiguous. +The MTP-on ZH base scores 50.30 (within σ of the 49.62 MTP-off base used above); SFT clears either +by a wide margin. Every run: 681/681 (or 75/75) valid, **0 errors**. + +## Per-stage breakdown (state accuracy by Socratic stage a–e) + +| set | a | b | c | d | e | +|---|---|---|---|---|---| +| ZH test base → SFT | 100 → 100 | 43.4 → **57.0** | 30.6 → **42.9** | 36.7 → **46.2** | 61.9 → **78.1** | +| EN test base → SFT | 100 → 100 | 48.8 → **54.5** | 33.0 → **40.9** | 37.9 → **47.2** | 57.1 → **75.4** | +| ZH synth base → SFT | 100 → 100 | 17.1 → 20.4 | 5.3 → 4.5 | 0.0 → 1.5 | 29.2 → **49.3** | +| EN synth base → SFT | 100 → 100 | 30.7 → 28.0 | 4.6 → 6.9 | 0.0 → 0.0 | 18.9 → **38.8** | + +- Stage **a** is trivially 100 everywhere (the opening turn). +- The largest, most consistent gain is **stage e** (closing/summary) — +16 to +20 pp on the + in-distribution sets and the only clear OOD mover. The SFT most improved *how the teacher closes*. +- On OOD synthetic, mid-dialogue stages **c/d nearly collapse for both** models (c≈5, d≈0) — those + synthetic dialogues are structurally hard / off-distribution in the middle; the SFT's OOD edge + comes from stages **b** and **e**. + +## Text-overlap metrics (vs ground-truth teacher turns) + +SFT roughly doubles–triples ROUGE/BLEU on every set, confirming it learned the teacher's phrasing, +not just state labels: + +| set | ROUGE-1 | ROUGE-L | BLEU-4 | +|---|---|---|---| +| ZH test base → SFT | 28.6 → **48.1** | 21.0 → **40.9** | 5.2 → **20.1** | +| EN test base → SFT | 51.3 → **69.5** | 34.4 → **46.8** | 3.2 → **11.7** | +| ZH synth base → SFT | 24.3 → **33.7** | 17.7 → **26.5** | 3.3 → **8.3** | +| EN synth base → SFT | 46.4 → **62.6** | 30.5 → **37.2** | 0.9 → **1.7** | + +## What this means + +1. **Real, large in-distribution improvement** — +10.3 pp ZH / +7.7 pp EN on held-out test, far + past the ~1.5 pp significance bar. +2. **Generalizes, not memorizes** — the SFT still wins +3.5–3.9 pp on synthetic dialogues it has + *never* seen (different origin, phrasing, question pool). A pure memorizer would show ~0 OOD gain. +3. **Cross-lingual transfer** — EN (+7.7) tracks ZH (+10.3); the Socratic behavior survives the + language boundary (the SFT trained on both ZH and EN per-turn data). +4. **Behavioral signal** — the base model is ~6× slower to evaluate (≈38 vs ≈225 dlg/hr; e.g. ZH + base 17.4 h vs SFT 3.0 h) because it **rambles to the 2048-token cap** instead of producing + short, terminating Socratic turns. The SFT terminates cleanly — itself evidence of better form + (to be quantified; see analysis plan T0.1). + +## Consultant ablation (T1.1): does the advantage survive removing the classifier? + +Every headline result gave **both** base and SFT the same external Qwen state classifier, so those +numbers don't isolate the SFT's *own* contribution. This ablation reruns the ZH-test 2×2 across +**three consultant modes**, varying only where the scored state comes from (all else pinned: Q8_0, +`-np 4`, q4_0 KV, MTP off, 8 rounds; every arm 681/681, 0 errors): + +- **Qwen classifier** (headline): shared external classifier, ~55–60% accurate, for both models. +- **self-consult**: no external classifier — the served LLM produces the state assessment itself + then consumes it (dual-role). The apples-to-apples "no external classifier" baseline. +- **oracle**: the ground-truth state is *fed in* each turn (`--oracle-consultant`), so + `state_accuracy` is 100% by construction. Removes classifier accuracy as a confound entirely — + the cleanest measure of *teacher-turn quality given correct state*. + +(The two long arms ran on the unstable box across auto-resumed crashes: base self-consult ~7 days, +base oracle ~2 days.) + +**The two skills decouple cleanly.** Teacher-turn *quality* is intrinsic to the SFT — and is in fact +*largest* once the state confound is removed (oracle). *State-tracking* was entirely the external +classifier and is worse than base when the SFT must self-track. + +### State accuracy (overall %) — the classifier's job, not the SFT's + +| | Qwen classifier | self-consult | oracle | +|---|---:|---:|---:| +| base | 49.62 | 34.45 | 100 (by construction) | +| SFT | 59.93 | **26.80** | 100 (by construction) | +| **SFT − base** | **+10.31** | **−7.65** | — | + +The SFT's +10.3 pp state edge under the shared classifier *inverts* to −7.65 pp when it must +self-track (it loses on every hard stage: b 24.4 vs 41.7, c 5.9 vs 16.5, e 27.1 vs 36.0). Oracle +pins both to 100, so state accuracy carries no signal there — score oracle on ROUGE/BLEU only. + +### Teacher-turn quality — the SFT's real deliverable (ROUGE/BLEU, SFT − base gap) + +| metric | base: self / Qwen / oracle | SFT: self / Qwen / oracle | **SFT − base gap: self / Qwen / oracle** | +|---|---:|---:|---:| +| ROUGE-1 | 28.02 / 28.56 / 29.03 | 44.21 / 48.13 / 51.77 | **+16.19 / +19.57 / +22.74** | +| ROUGE-L | 20.35 / 21.02 / 21.76 | 37.62 / 40.94 / 44.80 | **+17.27 / +19.92 / +23.04** | +| BLEU-4 | 4.80 / 5.22 / 5.73 | 18.31 / 20.12 / 24.80 | **+13.51 / +14.90 / +19.07** | + +### What the three modes together show + +- **The SFT climbs monotonically with state quality; base is flat.** SFT ROUGE-1: 44.2 (self, ~27% + state) → 48.1 (Qwen, ~55–60%) → **51.8 (oracle, 100%)**. Base sits at **28.0 → 28.6 → 29.0** across + the same axis. The SFT *learned to condition on state* — hand it a better state and it writes a + better turn; base can't exploit better state at all (it rambles regardless). That conditioning is + itself a learned skill, not phrasing memorization. +- **The gap is *largest* with the confound removed.** Oracle (correct state, both sides) gives the + SFT its biggest edge — **+22.7 ROUGE-1, +23.0 ROUGE-L, +19.1 BLEU-4** — bigger than under either + the classifier or self-consult. So "the SFT writes better Socratic turns" is unambiguous. +- **Self-consult was an *underestimate*, not the ceiling.** It showed the smallest gap (+16.2) + precisely because the SFT was penalized there — conditioned on its own worse self-classification + (27%). Removing that penalty (oracle) reveals the true ceiling; the "survives removing the + classifier" conclusion is thereby *strengthened*, not merely upheld. + +### Interpretation + +Consistent with the SFT's training format (`dataset.py:608–647`): the SFT was trained to **consume** +the consultant's assessment + action and emit a clean teacher turn — it **never learned to emit +state**. So `state_accuracy` in self-consult mode measures a skill the SFT never trained, and it +regressed below base's zero-shot classification (it learned to expect state to be handed to it). The +base model, never specialized, is the better self-classifier. + +**Headline answer:** the SFT genuinely internalized how to *write* Socratic turns — that advantage +not only survives removing the external classifier, it is **largest** when the state confound is +removed (+22.7 ROUGE-1, +19.1 BLEU-4 given correct state). It did **not** internalize how to *track* +state — that gain was the external classifier's, and self-tracking is worse than base. In deployment +the SFT wants an external state source, and it rewards a *better* one with proportionally better +turns. Results: `results/gemma4-12b-{base,sft}-{noconsult,oracle}/`. + +## LLM-judge (absolute pedagogical quality) — does the oracle ROUGE win hold on *quality*, not overlap? + +ROUGE/BLEU only measure overlap with the reference teacher turn. To test whether the oracle win +reflects genuinely better teaching, we scored both **oracle arms** with **Claude Opus 4.8** on an +absolute 0–10 rubric (socratic_validity 0–3, advancement 0–3, age_appropriateness 0–2, +question_form 0–2). The rubric shows the GT reference "as one valid move, do not penalize routing +differences" **on purpose** — so it rewards quality, not ROUGE-like overlap. Paired stratified +sample: 200 turns/arm, identical `(file, turn_idx)` keys across arms, stages b–e (opener `a` +excluded), seed 42, `--per-stage 50` (400 Opus calls, ~62 min/arm). Single run, headless subscription +(not temperature-0). `results/llm_judge_oracle_compare.json`. + +| | SFT-oracle | base-oracle | **Δ (SFT − base)** | +|---|---:|---:|---:| +| **overall /10** | **8.13** | 7.75 | **+0.38** | +| socratic_validity /3 | 2.56 | 2.46 | +0.10 | +| advancement /3 | 2.43 | 2.435 | **−0.01 (tied)** | +| age_appropriateness /2 | 1.93 | 1.845 | +0.09 | +| question_form /2 | 1.21 | 1.01 | **+0.20** | +| stage b /10 | 8.28 | 8.14 | +0.14 | +| stage c /10 | 8.64 | 8.00 | **+0.64** | +| stage d /10 | 8.12 | 7.36 | **+0.76** | +| stage e /10 | 7.48 | 7.50 | −0.02 (tied) | + +**Verdict: confirms the *direction*, corrects the *magnitude*.** The judge independently agrees the SFT +writes better Socratic turns given correct state (+0.38/10 overall, and it wins or ties every axis and +stage). But the gap is **far smaller than ROUGE implied**: +0.38/10 (~5% relative) vs +22.7 ROUGE-1 +(~78% relative). Both models produce pedagogically solid turns given correct state (7.75 vs 8.13 are +both high) — so most of the ROUGE advantage was **reference-phrasing overlap, not teaching quality**. +The SFT learned to phrase turns the way the corpus does; that shows up huge in ROUGE but only modestly +in absolute pedagogy. + +Two honest caveats the pattern surfaces: +- **`advancement` is a dead tie** (2.43 vs 2.435). Given correct state, base moves the lesson forward + just as well as the SFT. The SFT's real, judge-visible edge is **question_form** (+0.20, the largest + axis delta) and **socratic_validity** (+0.10) — *how* it asks, not *whether* it progresses. +- **The win concentrates in the middle stages** where there's pedagogical work to do — c (+0.64) and + d (+0.76) — and vanishes at the endpoints (b +0.14, e −0.02). Coherent, but it means the headline + +0.38 is an average over a strongly stage-dependent effect, not a uniform lift. +- **No error bars** (single run, non-greedy subscription decoding). The +0.38 headline is a point + estimate; the internally-consistent per-axis/per-stage pattern is what lends it confidence, not the + scalar alone. (Aside: base cost *more* to judge — $15.02 vs $13.86 — i.e. base emits more tokens, + consistent with the "base rambles regardless" finding.) + +## Method (held fixed; only model + dataset vary) + +- **Teacher** = the only variable under test: base `unsloth/gemma-4-12b-it` vs the merged Socratic + SFT, both served as **Q8_0 GGUF** on llama.cpp (`-np 4`, q4_0 KV, **MTP off**, workers=4). +- **Consultant** = Qwen3.5-0.8B LoRA state classifier on CPU (same checkpoint for every run). +- 8 teaching rounds, no fewshot, no thinking budget, stochastic server-default sampling (identical + across runs). Eval replays ground-truth student turns; the teacher generates; the classifier + scores state. 90/10 train/test split, seed 42, dialogues kept whole. +- **Datasets** (KELE-v2 collection): `SocratDataset` (ZH) / `SocratDataset-EN` held-out **test** + splits; `SocratDataset-SYNTHETIC` / `-EN` (75 each) run **whole** as OOD probes (never in training). + +## Caveats + +- **Adapter is ~0.88 epoch**, `checkpoint-4250`, recovered from HF history after a NaN divergence + at step ~4260 (loss had plateaued since ~step 3000). Report as ~0.88 epoch; a clean full run is + a plausible further gain (see `SFT_HANDOFF.md`). +- **Synthetic n is small** (37 ZH + 38 EN merged → 75 each; ~215/431 turns). Directional OOD + signal, not σ-tight. State accuracy is per-turn, so steadier than the dialogue count implies. +- **Stochastic decoding** — no temperature/seed pinned, so each run carries ~0.7 pp noise. Greedy + and multi-seed runs (analysis plan T2.1/T2.2) would tighten the point estimates. +- **Consultant ablation DONE** (all three modes — see the consultant-ablation section above) — the + headline state-accuracy numbers rely on the shared external classifier and do **not** reflect the + SFT's own state-tracking (self-consult drops SFT to 26.8, below base). The SFT's *teacher-turn + quality*, however, is classifier-independent and *largest* with the confound removed (+22.7 + ROUGE-1 oracle vs base). Read the headline table as "SFT + classifier vs base + classifier," not + "SFT alone." + +## Artifacts + +- **Models (HF, private):** adapter `ulises-c/SocratesLM-12B-QLoRA` (ckpts 3200–4250); merged BF16 + `ulises-c/SocratesLM-12B`; Q8_0 GGUF `ulises-c/SocratesLM-12B-GGUF`. +- **Datasets:** `ulises-c/SocratDataset{,-EN,-SYNTHETIC,-SYNTHETIC-EN}` (synthetic ZH completed to + 75 this PR). +- **Results:** `results/gemma4-12b-{base,sft}{,-en,-synth-zh,-synth-en,-noconsult,-oracle}/` + + `-base-mtp`. + +## What this PR changed (code) + +- GGUF convert: CPU-only `llama-quantize` via `QUANTIZE` override (the CUDA build segfaults `nvcc` + on this box). +- Eval: `--hf-repo`/`--split` on `evaluate` + monitor `EVAL_HF_REPO/EVAL_SPLIT/EVAL_OUT_SUFFIX` + (no schema adapter needed — all KELE-v2 sets share the `{student,teacher,state}` turn keys). +- Data: merged the 38-record ZH-synthetic extension into HF (37 → 75); fixed the loader that + referenced the never-uploaded config. +- Monitor: log the GPU's **actual enforced** power, not the card max (the step-down is inert + without passwordless sudo). +- Docs: eval-plan rationale + workers A/B, this report, and the further-analysis/ablation plan. + +## Reproduce + +``` +# serve the model under test (base or SFT), then: +KELE_PARALLEL_WORKERS=4 EVAL_HF_REPO= EVAL_SPLIT= EVAL_OUT_SUFFIX= \ + make monitor-eval-gemma4-12b-{base,sft} +python -m src.project.evaluate --compare results/ results/ +``` + +## Next + +Deeper analysis is scoped in `docs/SFT_VS_BASE_ANALYSIS_PLAN.md`; live next-steps in +`docs/HANDOFF_SFT_POST_JUDGE.md`. The consultant ablation (T1.1, all three modes) **and** the +LLM-judge (T1.2) are **done** (above): the SFT internalized teacher-turn *quality* (largest under +oracle; judge confirms the direction at +0.38/10 but ROUGE overstated the magnitude), not +*state-tracking*. Remaining picks, all optional: **T0.1 termination/length/style metrics** (free, +on-disk — quantifies the still-unquantified "base rambles" claim in §4 above), **multi-seed error +bars** (every number is a single-run point estimate), **earlier-checkpoint eval** (3200 vs 4250 — +decides whether a clean full-epoch run is worth ~30 h), and a **strong-consultant** cell (Claude as +classifier) between the ~55–60% Qwen point and the oracle's 100%. diff --git a/docs/SFT_VS_BASE_ANALYSIS_PLAN.md b/docs/SFT_VS_BASE_ANALYSIS_PLAN.md new file mode 100644 index 00000000..1d002457 --- /dev/null +++ b/docs/SFT_VS_BASE_ANALYSIS_PLAN.md @@ -0,0 +1,107 @@ +# SFT vs Base — Further Analysis & Ablation Plan + +**Created 2026-06-24.** Roadmap for digging past the headline result into *what* the Socratic +QLoRA SFT changed and *how solid* the uplift is. Companion to `docs/SFT_HANDOFF.md` (pipeline + +provenance) and `docs/EXPERIMENT_LOG.md` (results log). Live eval tracker: GitHub issue **#130**. + +## What we already know (baseline for everything below) + +- **Headline (ZH held-out test, n=681):** SFT **state_acc 59.93** vs base **49.62** (MTP-off) / + **50.30** (MTP-on) → **+9.6–10.3 pp**, ~13–14σ at the assumed σ≈0.7 pp. Per-stage gains +10–18 pp + on b/c/d/e; a was already 100. ROUGE/BLEU also ~2–4× (style learned). Clean run, 0 errors, ~3 h. +- **In progress (2×2 generalization):** base & SFT on EN held-out test (~680) + ZH/EN synthetic OOD + (75 each, never trained). Results → `results/gemma4-12b-{base,sft}-{en,synth-zh,synth-en}`. +- **The adapter** is `checkpoint-4250` (~0.88 epoch, recovered from a NaN divergence). HF: + `ulises-c/SocratesLM-12B` (merged BF16), `-GGUF` (Q8_0), `-QLoRA` (adapter + ckpts 3200..4250). +- **Eval invariants (held fixed, base vs SFT):** Qwen3.5-0.8B LoRA state classifier as consultant + (CPU), 8 teaching rounds, workers=4, MTP off, `-np 4`/q4_0 KV, **stochastic** server-default + sampling (no temperature/seed set in the teacher call, `tournament_utilizations.py:440`). + Only the teacher model and the dataset change. + +## Open questions this plan answers + +1. **What did the SFT internalize** vs just "use the consultant better"? (ablation #1) +2. **Is it a better *teacher***, not just a better state-emitter? (LLM-judge) +3. **How real is +9.6** — error bars, sampling-noise-free, not overfit? (multi-seed, greedy, train-gap, earlier ckpt) +4. **Did the narrow SFT cost general ability** or depend on the quant? (capability, quant) + +--- + +## Tier 0 — Free: analyze dialogues already on disk (no GPU) + +Operate on the saved `results//dialogues/*.json`. Each turn record has `state`, +`ground_truth_state`, `teacher_response`, `ground_truth_teacher`. + +- **T0.1 — Termination / length / Socratic-style metrics** ⭐ (explains the OOD rambling we saw) + Per turn: response length, **truncation rate** (hit `max_tokens=2048` with no EOS), and + **questions-per-turn** (reuse `tournament_utilizations.validate` — single clean question, no + preamble). Hypothesis: SFT is shorter, single-question, terminates cleanly; base rambles on OOD. +- **T0.2 — State confusion matrix (base vs SFT)** — which stage transitions improved (esp. the + +17.9 pp on stage e). Surfaces whether base collapses to a few stages. +- **T0.3 — Train-vs-test gap** — eval SFT on the **train** split (`--split train`); a large + train≫test gap flags overfitting at 0.88 epoch. +- **T0.4 — Slice ZH-test uplift by grade/subject** — `SocratDataset` carries `grade`/`chapter`/ + `mission`; check whether the gain is uniform or topic-concentrated. + +## Tier 1 — Highest-value new runs + +- **T1.1 — No-consultant ablation** ⭐⭐ (the key missing control) + Re-run base vs SFT **without** `--bert-consultant` so the teacher gets no externally-supplied + state. The SFT was trained in the inference format that emits state/action itself; this tests + whether the +9.6 is the SFT's own internalized state-tracking or just better use of the + classifier (which currently aids *both* arms equally). + ``` + # bare teacher, no classifier — run base then SFT, same machinery otherwise + KELE_BERT_DEVICE=cpu KELE_PARALLEL_WORKERS=4 uv run --no-sync python -m src.project.kele \ + --experiment gemma4-12b-{local,sft-local} evaluate --output results/gemma4-12b-{base,sft}-noconsult + ``` + (Serve the matching model first; no `--bert-consultant` flag.) +- **T1.2 — LLM-as-judge on Socratic quality** ⭐ — pairwise *blind* judge (e.g. Claude) over a + sample of base vs SFT teacher turns: "which guides *without giving away the answer*?" Captures + pedagogy that state-acc/ROUGE miss. Build on the existing Claude-consultant configs + + `tournament_utilizations` style critic. New small harness. + +## Tier 2 — Harden the headline number + +- **T2.1 — Multi-seed error bars** — σ≈0.7 is from one pair. Re-run ZH-test 2–3× (vary decode; + `--sample-seed` reshuffles the subsample, or rely on stochastic decode) to get a real stdev on + +9.6. ~3 h each. +- **T2.2 — Greedy (temp=0) run** — removes sampling noise for a clean point estimate. **Needs a + small change**: thread a `temperature`/`seed` through the teacher call (currently unset → + server default). Worth adding as a general eval knob. +- **T2.3 — Earlier-checkpoint comparison** — eval `checkpoint-3200` (loss plateaued ~step 3000) vs + shipped `checkpoint-4250`. If saturated at 3200, more epochs won't help; if still climbing, a + clean full run is justified. Reuse `merge_lora_gemma4_sft.py` + `convert_gemma4_12b_sft_to_gguf.sh` + on the earlier adapter (both ckpts on HF `ulises-c/SocratesLM-12B-QLoRA`). + +## Tier 3 — Different axes + +- **T3.1 — Capability preservation** — small general-QA probe (non-Socratic) on base vs SFT to + rule out catastrophic forgetting from the narrow QLoRA. +- **T3.2 — Quant sensitivity** — spot-check merged **BF16** (transformers) vs the **Q8_0 GGUF** on + ~50 dialogues to confirm the uplift isn't a quant artifact (validates the "Q8 delta ≈ noise" + assumption). Optionally Q4/Q5 to see how far the gain survives compression. + +--- + +## Suggested order (value / cost) + +1. **T0.1–T0.4** now — free, run on existing dialogues while the 2×2 finishes. +2. **T1.1 no-consultant** — the one control the current design lacks. +3. **T1.2 LLM-judge** + **T2.1 multi-seed** — quality evidence + error bars for the writeup. +4. **T2.3 earlier-checkpoint** — decides whether a cleaner full-epoch run is worth ~30 h. +5. **T3.x** — nice-to-have insurance. + +If only three: **T1.1 (no-consultant)**, **T1.2 (LLM-judge)**, **T0.1 (style/termination)** — then +**T2.1 (multi-seed)** to firm up the number. + +## Machinery notes + +- Dataset/model selection already wired: `--hf-repo`/`--split` on `evaluate`; monitor honors + `EVAL_HF_REPO`/`EVAL_SPLIT`/`EVAL_OUT_SUFFIX` (crash-resilient path for the unstable box). +- Each new eval set needs **both** base and SFT runs; base is ~6× slower than SFT on OOD synthetic + (rambles to max_tokens), so budget base runs generously. +- `temperature`/`seed` are **not** currently passthrough on the teacher call — T2.2 (and any + determinism work) needs that small addition first. +- Cosmetic TODO: the monitor's #130 progress rows hardcode `/681` as the denominator; patch + `dataset_total` to read the actual split size for non-default datasets. diff --git a/docs/diagnostics/gfx1201-report-env-20260601-190044.txt b/docs/diagnostics/gfx1201-report-env-20260601-190044.txt new file mode 100644 index 00000000..c3887a0e --- /dev/null +++ b/docs/diagnostics/gfx1201-report-env-20260601-190044.txt @@ -0,0 +1,894 @@ + +========== ROCm / HIP build ========== +7.2.53211-364a905 +7.2.3 + +========== OS / kernel ========== +Linux cachy 7.0.10-2-cachyos #1 SMP PREEMPT_DYNAMIC Thu, 28 May 2026 21:12:58 +0000 x86_64 GNU/Linux +PRETTY_NAME="CachyOS" + +========== GPU / driver / firmware / VBIOS ========== + + +============================ ROCm System Management Interface ============================ +============================== Version of System Component =============================== +Driver version: 7.0.10-2-cachyos +========================================================================================== +================================== End of ROCm SMI Log =================================== + + +============================ ROCm System Management Interface ============================ +========================================= VBIOS ========================================== +GPU[0] : VBIOS version: 113-1E4990U-S83 +========================================================================================== +====================================== Product Info ====================================== +GPU[0] : Card Series: AMD Radeon AI PRO R9700 +GPU[0] : Card Model: 0x7551 +GPU[0] : Card Vendor: Advanced Micro Devices, Inc. [AMD/ATI] +GPU[0] : Card SKU: 1E4990U +GPU[0] : Subsystem ID: -0x1b67 +GPU[0] : Device Rev: 0xc0 +GPU[0] : Node ID: 1 +GPU[0] : GUID: 44204 +GPU[0] : GFX Version: gfx1201 +========================================================================================== +================================== End of ROCm SMI Log =================================== + +-- amdgpu module -- +firmware: amdgpu/vega10_gpu_info.bin +firmware: amdgpu/vega12_gpu_info.bin +firmware: amdgpu/raven_gpu_info.bin +firmware: amdgpu/picasso_gpu_info.bin +firmware: amdgpu/raven2_gpu_info.bin +firmware: amdgpu/arcturus_gpu_info.bin +firmware: amdgpu/navi12_gpu_info.bin +firmware: amdgpu/cyan_skillfish_gpu_info.bin +firmware: amdgpu/ip_discovery.bin +firmware: amdgpu/vega10_ip_discovery.bin +firmware: amdgpu/vega12_ip_discovery.bin +firmware: amdgpu/vega20_ip_discovery.bin +firmware: amdgpu/raven_ip_discovery.bin +firmware: amdgpu/raven2_ip_discovery.bin +firmware: amdgpu/picasso_ip_discovery.bin +firmware: amdgpu/arcturus_ip_discovery.bin +firmware: amdgpu/aldebaran_ip_discovery.bin +firmware: amdgpu/bonaire_pfp.bin +firmware: amdgpu/bonaire_me.bin +firmware: amdgpu/bonaire_ce.bin +firmware: amdgpu/bonaire_rlc.bin +firmware: amdgpu/bonaire_mec.bin +firmware: amdgpu/hawaii_pfp.bin +firmware: amdgpu/hawaii_me.bin +firmware: amdgpu/hawaii_ce.bin +firmware: amdgpu/hawaii_rlc.bin +firmware: amdgpu/hawaii_mec.bin +firmware: amdgpu/kaveri_pfp.bin +firmware: amdgpu/kaveri_me.bin +firmware: amdgpu/kaveri_ce.bin +firmware: amdgpu/kaveri_rlc.bin +firmware: amdgpu/kaveri_mec.bin +firmware: amdgpu/kaveri_mec2.bin +firmware: amdgpu/kabini_pfp.bin +firmware: amdgpu/kabini_me.bin +firmware: amdgpu/kabini_ce.bin +firmware: amdgpu/kabini_rlc.bin +firmware: amdgpu/kabini_mec.bin +firmware: amdgpu/mullins_pfp.bin +firmware: amdgpu/mullins_me.bin +firmware: amdgpu/mullins_ce.bin +firmware: amdgpu/mullins_rlc.bin +firmware: amdgpu/mullins_mec.bin +firmware: amdgpu/bonaire_sdma.bin +firmware: amdgpu/bonaire_sdma1.bin +firmware: amdgpu/hawaii_sdma.bin +firmware: amdgpu/hawaii_sdma1.bin +firmware: amdgpu/kaveri_sdma.bin +firmware: amdgpu/kaveri_sdma1.bin +firmware: amdgpu/kabini_sdma.bin +firmware: amdgpu/kabini_sdma1.bin +firmware: amdgpu/mullins_sdma.bin +firmware: amdgpu/mullins_sdma1.bin +firmware: amdgpu/tahiti_mc.bin +firmware: amdgpu/pitcairn_mc.bin +firmware: amdgpu/verde_mc.bin +firmware: amdgpu/oland_mc.bin +firmware: amdgpu/hainan_mc.bin +firmware: amdgpu/si58_mc.bin +firmware: amdgpu/tahiti_pfp.bin +firmware: amdgpu/tahiti_me.bin +firmware: amdgpu/tahiti_ce.bin +firmware: amdgpu/tahiti_rlc.bin +firmware: amdgpu/pitcairn_pfp.bin +firmware: amdgpu/pitcairn_me.bin +firmware: amdgpu/pitcairn_ce.bin +firmware: amdgpu/pitcairn_rlc.bin +firmware: amdgpu/verde_pfp.bin +firmware: amdgpu/verde_me.bin +firmware: amdgpu/verde_ce.bin +firmware: amdgpu/verde_rlc.bin +firmware: amdgpu/oland_pfp.bin +firmware: amdgpu/oland_me.bin +firmware: amdgpu/oland_ce.bin +firmware: amdgpu/oland_rlc.bin +firmware: amdgpu/hainan_pfp.bin +firmware: amdgpu/hainan_me.bin +firmware: amdgpu/hainan_ce.bin +firmware: amdgpu/hainan_rlc.bin +firmware: amdgpu/bonaire_mc.bin +firmware: amdgpu/hawaii_mc.bin +firmware: amdgpu/topaz_mc.bin +firmware: amdgpu/tonga_mc.bin +firmware: amdgpu/polaris11_mc.bin +firmware: amdgpu/polaris10_mc.bin +firmware: amdgpu/polaris12_mc.bin +firmware: amdgpu/polaris12_32_mc.bin +firmware: amdgpu/polaris11_k_mc.bin +firmware: amdgpu/polaris10_k_mc.bin +firmware: amdgpu/polaris12_k_mc.bin +firmware: amdgpu/vega10_sos.bin +firmware: amdgpu/vega10_asd.bin +firmware: amdgpu/vega10_cap.bin +firmware: amdgpu/vega12_sos.bin +firmware: amdgpu/vega12_asd.bin +firmware: amdgpu/raven_asd.bin +firmware: amdgpu/picasso_asd.bin +firmware: amdgpu/raven2_asd.bin +firmware: amdgpu/picasso_ta.bin +firmware: amdgpu/raven2_ta.bin +firmware: amdgpu/raven_ta.bin +firmware: amdgpu/vega20_sos.bin +firmware: amdgpu/vega20_asd.bin +firmware: amdgpu/vega20_ta.bin +firmware: amdgpu/navi10_sos.bin +firmware: amdgpu/navi10_asd.bin +firmware: amdgpu/navi10_ta.bin +firmware: amdgpu/navi14_sos.bin +firmware: amdgpu/navi14_asd.bin +firmware: amdgpu/navi14_ta.bin +firmware: amdgpu/navi12_sos.bin +firmware: amdgpu/navi12_asd.bin +firmware: amdgpu/navi12_ta.bin +firmware: amdgpu/navi12_cap.bin +firmware: amdgpu/arcturus_sos.bin +firmware: amdgpu/arcturus_asd.bin +firmware: amdgpu/arcturus_ta.bin +firmware: amdgpu/sienna_cichlid_sos.bin +firmware: amdgpu/sienna_cichlid_ta.bin +firmware: amdgpu/sienna_cichlid_cap.bin +firmware: amdgpu/navy_flounder_sos.bin +firmware: amdgpu/navy_flounder_ta.bin +firmware: amdgpu/vangogh_asd.bin +firmware: amdgpu/vangogh_toc.bin +firmware: amdgpu/dimgrey_cavefish_sos.bin +firmware: amdgpu/dimgrey_cavefish_ta.bin +firmware: amdgpu/beige_goby_sos.bin +firmware: amdgpu/beige_goby_ta.bin +firmware: amdgpu/renoir_asd.bin +firmware: amdgpu/renoir_ta.bin +firmware: amdgpu/green_sardine_asd.bin +firmware: amdgpu/green_sardine_ta.bin +firmware: amdgpu/aldebaran_sos.bin +firmware: amdgpu/aldebaran_ta.bin +firmware: amdgpu/aldebaran_cap.bin +firmware: amdgpu/yellow_carp_toc.bin +firmware: amdgpu/yellow_carp_ta.bin +firmware: amdgpu/psp_13_0_5_toc.bin +firmware: amdgpu/psp_13_0_5_ta.bin +firmware: amdgpu/psp_13_0_8_toc.bin +firmware: amdgpu/psp_13_0_8_ta.bin +firmware: amdgpu/psp_13_0_0_sos.bin +firmware: amdgpu/psp_13_0_0_sos_kicker.bin +firmware: amdgpu/psp_13_0_0_ta.bin +firmware: amdgpu/psp_13_0_0_ta_kicker.bin +firmware: amdgpu/psp_13_0_7_sos.bin +firmware: amdgpu/psp_13_0_7_ta.bin +firmware: amdgpu/psp_13_0_10_sos.bin +firmware: amdgpu/psp_13_0_10_ta.bin +firmware: amdgpu/psp_13_0_11_toc.bin +firmware: amdgpu/psp_13_0_11_ta.bin +firmware: amdgpu/psp_13_0_6_sos.bin +firmware: amdgpu/psp_13_0_6_ta.bin +firmware: amdgpu/psp_13_0_12_sos.bin +firmware: amdgpu/psp_13_0_12_ta.bin +firmware: amdgpu/psp_13_0_14_sos.bin +firmware: amdgpu/psp_13_0_14_ta.bin +firmware: amdgpu/psp_13_0_15_sos.bin +firmware: amdgpu/psp_13_0_15_ta.bin +firmware: amdgpu/psp_14_0_0_toc.bin +firmware: amdgpu/psp_14_0_0_ta.bin +firmware: amdgpu/psp_14_0_1_toc.bin +firmware: amdgpu/psp_14_0_1_ta.bin +firmware: amdgpu/psp_14_0_4_toc.bin +firmware: amdgpu/psp_14_0_4_ta.bin +firmware: amdgpu/psp_13_0_4_toc.bin +firmware: amdgpu/psp_13_0_4_ta.bin +firmware: amdgpu/psp_14_0_2_sos.bin +firmware: amdgpu/psp_14_0_2_ta.bin +firmware: amdgpu/psp_14_0_3_sos.bin +firmware: amdgpu/psp_14_0_3_sos_kicker.bin +firmware: amdgpu/psp_14_0_3_ta.bin +firmware: amdgpu/psp_14_0_3_ta_kicker.bin +firmware: amdgpu/psp_14_0_5_toc.bin +firmware: amdgpu/psp_14_0_5_ta.bin +firmware: amdgpu/psp_15_0_0_toc.bin +firmware: amdgpu/psp_15_0_0_ta.bin +firmware: amdgpu/psp_15_0_8_toc.bin +firmware: amdgpu/carrizo_ce.bin +firmware: amdgpu/carrizo_pfp.bin +firmware: amdgpu/carrizo_me.bin +firmware: amdgpu/carrizo_mec.bin +firmware: amdgpu/carrizo_mec2.bin +firmware: amdgpu/carrizo_rlc.bin +firmware: amdgpu/stoney_ce.bin +firmware: amdgpu/stoney_pfp.bin +firmware: amdgpu/stoney_me.bin +firmware: amdgpu/stoney_mec.bin +firmware: amdgpu/stoney_rlc.bin +firmware: amdgpu/tonga_ce.bin +firmware: amdgpu/tonga_pfp.bin +firmware: amdgpu/tonga_me.bin +firmware: amdgpu/tonga_mec.bin +firmware: amdgpu/tonga_mec2.bin +firmware: amdgpu/tonga_rlc.bin +firmware: amdgpu/topaz_ce.bin +firmware: amdgpu/topaz_pfp.bin +firmware: amdgpu/topaz_me.bin +firmware: amdgpu/topaz_mec.bin +firmware: amdgpu/topaz_rlc.bin +firmware: amdgpu/fiji_ce.bin +firmware: amdgpu/fiji_pfp.bin +firmware: amdgpu/fiji_me.bin +firmware: amdgpu/fiji_mec.bin +firmware: amdgpu/fiji_mec2.bin +firmware: amdgpu/fiji_rlc.bin +firmware: amdgpu/polaris10_ce.bin +firmware: amdgpu/polaris10_ce_2.bin +firmware: amdgpu/polaris10_pfp.bin +firmware: amdgpu/polaris10_pfp_2.bin +firmware: amdgpu/polaris10_me.bin +firmware: amdgpu/polaris10_me_2.bin +firmware: amdgpu/polaris10_mec.bin +firmware: amdgpu/polaris10_mec_2.bin +firmware: amdgpu/polaris10_mec2.bin +firmware: amdgpu/polaris10_mec2_2.bin +firmware: amdgpu/polaris10_rlc.bin +firmware: amdgpu/polaris11_ce.bin +firmware: amdgpu/polaris11_ce_2.bin +firmware: amdgpu/polaris11_pfp.bin +firmware: amdgpu/polaris11_pfp_2.bin +firmware: amdgpu/polaris11_me.bin +firmware: amdgpu/polaris11_me_2.bin +firmware: amdgpu/polaris11_mec.bin +firmware: amdgpu/polaris11_mec_2.bin +firmware: amdgpu/polaris11_mec2.bin +firmware: amdgpu/polaris11_mec2_2.bin +firmware: amdgpu/polaris11_rlc.bin +firmware: amdgpu/polaris12_ce.bin +firmware: amdgpu/polaris12_ce_2.bin +firmware: amdgpu/polaris12_pfp.bin +firmware: amdgpu/polaris12_pfp_2.bin +firmware: amdgpu/polaris12_me.bin +firmware: amdgpu/polaris12_me_2.bin +firmware: amdgpu/polaris12_mec.bin +firmware: amdgpu/polaris12_mec_2.bin +firmware: amdgpu/polaris12_mec2.bin +firmware: amdgpu/polaris12_mec2_2.bin +firmware: amdgpu/polaris12_rlc.bin +firmware: amdgpu/vegam_ce.bin +firmware: amdgpu/vegam_pfp.bin +firmware: amdgpu/vegam_me.bin +firmware: amdgpu/vegam_mec.bin +firmware: amdgpu/vegam_mec2.bin +firmware: amdgpu/vegam_rlc.bin +firmware: amdgpu/vega10_ce.bin +firmware: amdgpu/vega10_pfp.bin +firmware: amdgpu/vega10_me.bin +firmware: amdgpu/vega10_mec.bin +firmware: amdgpu/vega10_mec2.bin +firmware: amdgpu/vega10_rlc.bin +firmware: amdgpu/vega12_ce.bin +firmware: amdgpu/vega12_pfp.bin +firmware: amdgpu/vega12_me.bin +firmware: amdgpu/vega12_mec.bin +firmware: amdgpu/vega12_mec2.bin +firmware: amdgpu/vega12_rlc.bin +firmware: amdgpu/vega20_ce.bin +firmware: amdgpu/vega20_pfp.bin +firmware: amdgpu/vega20_me.bin +firmware: amdgpu/vega20_mec.bin +firmware: amdgpu/vega20_mec2.bin +firmware: amdgpu/vega20_rlc.bin +firmware: amdgpu/raven_ce.bin +firmware: amdgpu/raven_pfp.bin +firmware: amdgpu/raven_me.bin +firmware: amdgpu/raven_mec.bin +firmware: amdgpu/raven_mec2.bin +firmware: amdgpu/raven_rlc.bin +firmware: amdgpu/picasso_ce.bin +firmware: amdgpu/picasso_pfp.bin +firmware: amdgpu/picasso_me.bin +firmware: amdgpu/picasso_mec.bin +firmware: amdgpu/picasso_mec2.bin +firmware: amdgpu/picasso_rlc.bin +firmware: amdgpu/picasso_rlc_am4.bin +firmware: amdgpu/raven2_ce.bin +firmware: amdgpu/raven2_pfp.bin +firmware: amdgpu/raven2_me.bin +firmware: amdgpu/raven2_mec.bin +firmware: amdgpu/raven2_mec2.bin +firmware: amdgpu/raven2_rlc.bin +firmware: amdgpu/raven_kicker_rlc.bin +firmware: amdgpu/arcturus_mec.bin +firmware: amdgpu/arcturus_rlc.bin +firmware: amdgpu/renoir_ce.bin +firmware: amdgpu/renoir_pfp.bin +firmware: amdgpu/renoir_me.bin +firmware: amdgpu/renoir_mec.bin +firmware: amdgpu/renoir_rlc.bin +firmware: amdgpu/green_sardine_ce.bin +firmware: amdgpu/green_sardine_pfp.bin +firmware: amdgpu/green_sardine_me.bin +firmware: amdgpu/green_sardine_mec.bin +firmware: amdgpu/green_sardine_mec2.bin +firmware: amdgpu/green_sardine_rlc.bin +firmware: amdgpu/aldebaran_mec.bin +firmware: amdgpu/aldebaran_mec2.bin +firmware: amdgpu/aldebaran_rlc.bin +firmware: amdgpu/aldebaran_sjt_mec.bin +firmware: amdgpu/aldebaran_sjt_mec2.bin +firmware: amdgpu/gc_9_4_3_mec.bin +firmware: amdgpu/gc_9_4_4_mec.bin +firmware: amdgpu/gc_9_5_0_mec.bin +firmware: amdgpu/gc_9_4_3_rlc.bin +firmware: amdgpu/gc_9_4_4_rlc.bin +firmware: amdgpu/gc_9_5_0_rlc.bin +firmware: amdgpu/gc_9_4_3_sjt_mec.bin +firmware: amdgpu/gc_9_4_4_sjt_mec.bin +firmware: amdgpu/navi10_ce.bin +firmware: amdgpu/navi10_pfp.bin +firmware: amdgpu/navi10_me.bin +firmware: amdgpu/navi10_mec.bin +firmware: amdgpu/navi10_mec2.bin +firmware: amdgpu/navi10_rlc.bin +firmware: amdgpu/navi14_ce_wks.bin +firmware: amdgpu/navi14_pfp_wks.bin +firmware: amdgpu/navi14_me_wks.bin +firmware: amdgpu/navi14_mec_wks.bin +firmware: amdgpu/navi14_mec2_wks.bin +firmware: amdgpu/navi14_ce.bin +firmware: amdgpu/navi14_pfp.bin +firmware: amdgpu/navi14_me.bin +firmware: amdgpu/navi14_mec.bin +firmware: amdgpu/navi14_mec2.bin +firmware: amdgpu/navi14_rlc.bin +firmware: amdgpu/navi12_ce.bin +firmware: amdgpu/navi12_pfp.bin +firmware: amdgpu/navi12_me.bin +firmware: amdgpu/navi12_mec.bin +firmware: amdgpu/navi12_mec2.bin +firmware: amdgpu/navi12_rlc.bin +firmware: amdgpu/sienna_cichlid_ce.bin +firmware: amdgpu/sienna_cichlid_pfp.bin +firmware: amdgpu/sienna_cichlid_me.bin +firmware: amdgpu/sienna_cichlid_mec.bin +firmware: amdgpu/sienna_cichlid_mec2.bin +firmware: amdgpu/sienna_cichlid_rlc.bin +firmware: amdgpu/navy_flounder_ce.bin +firmware: amdgpu/navy_flounder_pfp.bin +firmware: amdgpu/navy_flounder_me.bin +firmware: amdgpu/navy_flounder_mec.bin +firmware: amdgpu/navy_flounder_mec2.bin +firmware: amdgpu/navy_flounder_rlc.bin +firmware: amdgpu/vangogh_ce.bin +firmware: amdgpu/vangogh_pfp.bin +firmware: amdgpu/vangogh_me.bin +firmware: amdgpu/vangogh_mec.bin +firmware: amdgpu/vangogh_mec2.bin +firmware: amdgpu/vangogh_rlc.bin +firmware: amdgpu/dimgrey_cavefish_ce.bin +firmware: amdgpu/dimgrey_cavefish_pfp.bin +firmware: amdgpu/dimgrey_cavefish_me.bin +firmware: amdgpu/dimgrey_cavefish_mec.bin +firmware: amdgpu/dimgrey_cavefish_mec2.bin +firmware: amdgpu/dimgrey_cavefish_rlc.bin +firmware: amdgpu/beige_goby_ce.bin +firmware: amdgpu/beige_goby_pfp.bin +firmware: amdgpu/beige_goby_me.bin +firmware: amdgpu/beige_goby_mec.bin +firmware: amdgpu/beige_goby_mec2.bin +firmware: amdgpu/beige_goby_rlc.bin +firmware: amdgpu/yellow_carp_ce.bin +firmware: amdgpu/yellow_carp_pfp.bin +firmware: amdgpu/yellow_carp_me.bin +firmware: amdgpu/yellow_carp_mec.bin +firmware: amdgpu/yellow_carp_mec2.bin +firmware: amdgpu/yellow_carp_rlc.bin +firmware: amdgpu/cyan_skillfish2_ce.bin +firmware: amdgpu/cyan_skillfish2_pfp.bin +firmware: amdgpu/cyan_skillfish2_me.bin +firmware: amdgpu/cyan_skillfish2_mec.bin +firmware: amdgpu/cyan_skillfish2_mec2.bin +firmware: amdgpu/cyan_skillfish2_rlc.bin +firmware: amdgpu/gc_10_3_6_ce.bin +firmware: amdgpu/gc_10_3_6_pfp.bin +firmware: amdgpu/gc_10_3_6_me.bin +firmware: amdgpu/gc_10_3_6_mec.bin +firmware: amdgpu/gc_10_3_6_mec2.bin +firmware: amdgpu/gc_10_3_6_rlc.bin +firmware: amdgpu/gc_10_3_7_ce.bin +firmware: amdgpu/gc_10_3_7_pfp.bin +firmware: amdgpu/gc_10_3_7_me.bin +firmware: amdgpu/gc_10_3_7_mec.bin +firmware: amdgpu/gc_10_3_7_mec2.bin +firmware: amdgpu/gc_10_3_7_rlc.bin +firmware: amdgpu/gc_11_0_0_imu.bin +firmware: amdgpu/gc_11_0_0_imu_kicker.bin +firmware: amdgpu/gc_11_0_1_imu.bin +firmware: amdgpu/gc_11_0_2_imu.bin +firmware: amdgpu/gc_11_0_3_imu.bin +firmware: amdgpu/gc_11_0_4_imu.bin +firmware: amdgpu/gc_11_5_0_imu.bin +firmware: amdgpu/gc_11_5_1_imu.bin +firmware: amdgpu/gc_11_5_2_imu.bin +firmware: amdgpu/gc_11_5_3_imu.bin +firmware: amdgpu/gc_11_5_4_imu.bin +firmware: amdgpu/gc_11_0_0_pfp.bin +firmware: amdgpu/gc_11_0_0_me.bin +firmware: amdgpu/gc_11_0_0_mec.bin +firmware: amdgpu/gc_11_0_0_rlc.bin +firmware: amdgpu/gc_11_0_0_rlc_kicker.bin +firmware: amdgpu/gc_11_0_0_rlc_1.bin +firmware: amdgpu/gc_11_0_0_toc.bin +firmware: amdgpu/gc_11_0_1_pfp.bin +firmware: amdgpu/gc_11_0_1_me.bin +firmware: amdgpu/gc_11_0_1_mec.bin +firmware: amdgpu/gc_11_0_1_rlc.bin +firmware: amdgpu/gc_11_0_2_pfp.bin +firmware: amdgpu/gc_11_0_2_me.bin +firmware: amdgpu/gc_11_0_2_mec.bin +firmware: amdgpu/gc_11_0_2_rlc.bin +firmware: amdgpu/gc_11_0_3_pfp.bin +firmware: amdgpu/gc_11_0_3_me.bin +firmware: amdgpu/gc_11_0_3_mec.bin +firmware: amdgpu/gc_11_0_3_rlc.bin +firmware: amdgpu/gc_11_0_4_pfp.bin +firmware: amdgpu/gc_11_0_4_me.bin +firmware: amdgpu/gc_11_0_4_mec.bin +firmware: amdgpu/gc_11_0_4_rlc.bin +firmware: amdgpu/gc_11_5_0_pfp.bin +firmware: amdgpu/gc_11_5_0_me.bin +firmware: amdgpu/gc_11_5_0_mec.bin +firmware: amdgpu/gc_11_5_0_rlc.bin +firmware: amdgpu/gc_11_5_1_pfp.bin +firmware: amdgpu/gc_11_5_1_me.bin +firmware: amdgpu/gc_11_5_1_mec.bin +firmware: amdgpu/gc_11_5_1_rlc.bin +firmware: amdgpu/gc_11_5_2_pfp.bin +firmware: amdgpu/gc_11_5_2_me.bin +firmware: amdgpu/gc_11_5_2_mec.bin +firmware: amdgpu/gc_11_5_2_rlc.bin +firmware: amdgpu/gc_11_5_3_pfp.bin +firmware: amdgpu/gc_11_5_3_me.bin +firmware: amdgpu/gc_11_5_3_mec.bin +firmware: amdgpu/gc_11_5_3_rlc.bin +firmware: amdgpu/gc_11_5_4_pfp.bin +firmware: amdgpu/gc_11_5_4_me.bin +firmware: amdgpu/gc_11_5_4_mec.bin +firmware: amdgpu/gc_11_5_4_rlc.bin +firmware: amdgpu/gc_12_0_0_pfp.bin +firmware: amdgpu/gc_12_0_0_me.bin +firmware: amdgpu/gc_12_0_0_mec.bin +firmware: amdgpu/gc_12_0_0_rlc.bin +firmware: amdgpu/gc_12_0_0_toc.bin +firmware: amdgpu/gc_12_0_1_pfp.bin +firmware: amdgpu/gc_12_0_1_me.bin +firmware: amdgpu/gc_12_0_1_mec.bin +firmware: amdgpu/gc_12_0_1_rlc.bin +firmware: amdgpu/gc_12_0_1_rlc_kicker.bin +firmware: amdgpu/gc_12_0_1_toc.bin +firmware: amdgpu/gc_12_1_0_mec.bin +firmware: amdgpu/gc_12_1_0_rlc.bin +firmware: amdgpu/gc_12_0_0_imu.bin +firmware: amdgpu/gc_12_0_1_imu.bin +firmware: amdgpu/gc_12_0_1_imu_kicker.bin +firmware: amdgpu/gc_12_1_0_imu.bin +firmware: amdgpu/topaz_sdma.bin +firmware: amdgpu/topaz_sdma1.bin +firmware: amdgpu/tonga_sdma.bin +firmware: amdgpu/tonga_sdma1.bin +firmware: amdgpu/carrizo_sdma.bin +firmware: amdgpu/carrizo_sdma1.bin +firmware: amdgpu/fiji_sdma.bin +firmware: amdgpu/fiji_sdma1.bin +firmware: amdgpu/stoney_sdma.bin +firmware: amdgpu/polaris10_sdma.bin +firmware: amdgpu/polaris10_sdma1.bin +firmware: amdgpu/polaris11_sdma.bin +firmware: amdgpu/polaris11_sdma1.bin +firmware: amdgpu/polaris12_sdma.bin +firmware: amdgpu/polaris12_sdma1.bin +firmware: amdgpu/vegam_sdma.bin +firmware: amdgpu/vegam_sdma1.bin +firmware: amdgpu/vega10_sdma.bin +firmware: amdgpu/vega10_sdma1.bin +firmware: amdgpu/vega12_sdma.bin +firmware: amdgpu/vega12_sdma1.bin +firmware: amdgpu/vega20_sdma.bin +firmware: amdgpu/vega20_sdma1.bin +firmware: amdgpu/raven_sdma.bin +firmware: amdgpu/picasso_sdma.bin +firmware: amdgpu/raven2_sdma.bin +firmware: amdgpu/arcturus_sdma.bin +firmware: amdgpu/renoir_sdma.bin +firmware: amdgpu/green_sardine_sdma.bin +firmware: amdgpu/aldebaran_sdma.bin +firmware: amdgpu/sdma_4_4_2.bin +firmware: amdgpu/sdma_4_4_4.bin +firmware: amdgpu/sdma_4_4_5.bin +firmware: amdgpu/navi10_sdma.bin +firmware: amdgpu/navi10_sdma1.bin +firmware: amdgpu/navi14_sdma.bin +firmware: amdgpu/navi14_sdma1.bin +firmware: amdgpu/navi12_sdma.bin +firmware: amdgpu/navi12_sdma1.bin +firmware: amdgpu/cyan_skillfish2_sdma.bin +firmware: amdgpu/cyan_skillfish2_sdma1.bin +firmware: amdgpu/sienna_cichlid_sdma.bin +firmware: amdgpu/navy_flounder_sdma.bin +firmware: amdgpu/dimgrey_cavefish_sdma.bin +firmware: amdgpu/beige_goby_sdma.bin +firmware: amdgpu/vangogh_sdma.bin +firmware: amdgpu/yellow_carp_sdma.bin +firmware: amdgpu/sdma_5_2_6.bin +firmware: amdgpu/sdma_5_2_7.bin +firmware: amdgpu/sdma_6_0_0.bin +firmware: amdgpu/sdma_6_0_1.bin +firmware: amdgpu/sdma_6_0_2.bin +firmware: amdgpu/sdma_6_0_3.bin +firmware: amdgpu/sdma_6_1_0.bin +firmware: amdgpu/sdma_6_1_1.bin +firmware: amdgpu/sdma_6_1_2.bin +firmware: amdgpu/sdma_6_1_3.bin +firmware: amdgpu/sdma_6_1_4.bin +firmware: amdgpu/sdma_7_0_0.bin +firmware: amdgpu/sdma_7_0_1.bin +firmware: amdgpu/sdma_7_1_0.bin +firmware: amdgpu/gc_11_0_0_mes.bin +firmware: amdgpu/gc_11_0_0_mes_2.bin +firmware: amdgpu/gc_11_0_0_mes1.bin +firmware: amdgpu/gc_11_0_1_mes.bin +firmware: amdgpu/gc_11_0_1_mes_2.bin +firmware: amdgpu/gc_11_0_1_mes1.bin +firmware: amdgpu/gc_11_0_2_mes.bin +firmware: amdgpu/gc_11_0_2_mes_2.bin +firmware: amdgpu/gc_11_0_2_mes1.bin +firmware: amdgpu/gc_11_0_3_mes.bin +firmware: amdgpu/gc_11_0_3_mes_2.bin +firmware: amdgpu/gc_11_0_3_mes1.bin +firmware: amdgpu/gc_11_0_4_mes.bin +firmware: amdgpu/gc_11_0_4_mes_2.bin +firmware: amdgpu/gc_11_0_4_mes1.bin +firmware: amdgpu/gc_11_5_0_mes_2.bin +firmware: amdgpu/gc_11_5_0_mes1.bin +firmware: amdgpu/gc_11_5_1_mes_2.bin +firmware: amdgpu/gc_11_5_1_mes1.bin +firmware: amdgpu/gc_11_5_2_mes_2.bin +firmware: amdgpu/gc_11_5_2_mes1.bin +firmware: amdgpu/gc_11_5_3_mes_2.bin +firmware: amdgpu/gc_11_5_3_mes1.bin +firmware: amdgpu/gc_11_5_4_mes_2.bin +firmware: amdgpu/gc_11_5_4_mes1.bin +firmware: amdgpu/gc_12_0_0_mes.bin +firmware: amdgpu/gc_12_0_0_mes1.bin +firmware: amdgpu/gc_12_0_0_uni_mes.bin +firmware: amdgpu/gc_12_0_1_mes.bin +firmware: amdgpu/gc_12_0_1_mes1.bin +firmware: amdgpu/gc_12_0_1_uni_mes.bin +firmware: amdgpu/gc_12_1_0_mes.bin +firmware: amdgpu/gc_12_1_0_mes1.bin +firmware: amdgpu/gc_12_1_0_uni_mes.bin +firmware: amdgpu/tahiti_uvd.bin +firmware: amdgpu/verde_uvd.bin +firmware: amdgpu/pitcairn_uvd.bin +firmware: amdgpu/oland_uvd.bin +firmware: amdgpu/bonaire_uvd.bin +firmware: amdgpu/kabini_uvd.bin +firmware: amdgpu/kaveri_uvd.bin +firmware: amdgpu/hawaii_uvd.bin +firmware: amdgpu/mullins_uvd.bin +firmware: amdgpu/tonga_uvd.bin +firmware: amdgpu/carrizo_uvd.bin +firmware: amdgpu/fiji_uvd.bin +firmware: amdgpu/stoney_uvd.bin +firmware: amdgpu/polaris10_uvd.bin +firmware: amdgpu/polaris11_uvd.bin +firmware: amdgpu/polaris12_uvd.bin +firmware: amdgpu/vegam_uvd.bin +firmware: amdgpu/vega10_uvd.bin +firmware: amdgpu/vega12_uvd.bin +firmware: amdgpu/vega20_uvd.bin +firmware: amdgpu/vce_1_0_0.bin +firmware: amdgpu/bonaire_vce.bin +firmware: amdgpu/kabini_vce.bin +firmware: amdgpu/kaveri_vce.bin +firmware: amdgpu/hawaii_vce.bin +firmware: amdgpu/mullins_vce.bin +firmware: amdgpu/tonga_vce.bin +firmware: amdgpu/carrizo_vce.bin +firmware: amdgpu/fiji_vce.bin +firmware: amdgpu/stoney_vce.bin +firmware: amdgpu/polaris10_vce.bin +firmware: amdgpu/polaris11_vce.bin +firmware: amdgpu/polaris12_vce.bin +firmware: amdgpu/vegam_vce.bin +firmware: amdgpu/vega10_vce.bin +firmware: amdgpu/vega12_vce.bin +firmware: amdgpu/vega20_vce.bin +firmware: amdgpu/raven_vcn.bin +firmware: amdgpu/picasso_vcn.bin +firmware: amdgpu/raven2_vcn.bin +firmware: amdgpu/arcturus_vcn.bin +firmware: amdgpu/renoir_vcn.bin +firmware: amdgpu/green_sardine_vcn.bin +firmware: amdgpu/aldebaran_vcn.bin +firmware: amdgpu/navi10_vcn.bin +firmware: amdgpu/navi14_vcn.bin +firmware: amdgpu/navi12_vcn.bin +firmware: amdgpu/sienna_cichlid_vcn.bin +firmware: amdgpu/navy_flounder_vcn.bin +firmware: amdgpu/vangogh_vcn.bin +firmware: amdgpu/dimgrey_cavefish_vcn.bin +firmware: amdgpu/beige_goby_vcn.bin +firmware: amdgpu/yellow_carp_vcn.bin +firmware: amdgpu/vcn_3_1_2.bin +firmware: amdgpu/vcn_4_0_0.bin +firmware: amdgpu/vcn_4_0_2.bin +firmware: amdgpu/vcn_4_0_3.bin +firmware: amdgpu/vcn_4_0_4.bin +firmware: amdgpu/vcn_4_0_5.bin +firmware: amdgpu/vcn_4_0_6.bin +firmware: amdgpu/vcn_4_0_6_1.bin +firmware: amdgpu/vcn_5_0_0.bin +firmware: amdgpu/vcn_5_0_1.bin +firmware: amdgpu/vcn_5_3_0.bin +firmware: amdgpu/vpe_6_1_0.bin +firmware: amdgpu/vpe_6_1_1.bin +firmware: amdgpu/vpe_6_1_3.bin +firmware: amdgpu/umsch_mm_4_0_0.bin +firmware: amdgpu/arcturus_smc.bin +firmware: amdgpu/navi10_smc.bin +firmware: amdgpu/navi14_smc.bin +firmware: amdgpu/navi12_smc.bin +firmware: amdgpu/sienna_cichlid_smc.bin +firmware: amdgpu/navy_flounder_smc.bin +firmware: amdgpu/dimgrey_cavefish_smc.bin +firmware: amdgpu/beige_goby_smc.bin +firmware: amdgpu/aldebaran_smc.bin +firmware: amdgpu/smu_13_0_0.bin +firmware: amdgpu/smu_13_0_0_kicker.bin +firmware: amdgpu/smu_13_0_7.bin +firmware: amdgpu/smu_13_0_10.bin +firmware: amdgpu/smu_13_0_6.bin +firmware: amdgpu/smu_13_0_14.bin +firmware: amdgpu/smu_14_0_2.bin +firmware: amdgpu/smu_14_0_3.bin +firmware: amdgpu/smu_14_0_3_kicker.bin +firmware: amdgpu/bonaire_smc.bin +firmware: amdgpu/bonaire_k_smc.bin +firmware: amdgpu/hawaii_smc.bin +firmware: amdgpu/hawaii_k_smc.bin +firmware: amdgpu/topaz_smc.bin +firmware: amdgpu/topaz_k_smc.bin +firmware: amdgpu/tonga_smc.bin +firmware: amdgpu/tonga_k_smc.bin +firmware: amdgpu/fiji_smc.bin +firmware: amdgpu/polaris10_smc.bin +firmware: amdgpu/polaris10_smc_sk.bin +firmware: amdgpu/polaris10_k_smc.bin +firmware: amdgpu/polaris10_k2_smc.bin +firmware: amdgpu/polaris11_smc.bin +firmware: amdgpu/polaris11_smc_sk.bin +firmware: amdgpu/polaris11_k_smc.bin +firmware: amdgpu/polaris11_k2_smc.bin +firmware: amdgpu/polaris12_smc.bin +firmware: amdgpu/polaris12_k_smc.bin +firmware: amdgpu/vegam_smc.bin +firmware: amdgpu/vega10_smc.bin +firmware: amdgpu/vega10_acg_smc.bin +firmware: amdgpu/vega12_smc.bin +firmware: amdgpu/vega20_smc.bin +firmware: amdgpu/tahiti_smc.bin +firmware: amdgpu/pitcairn_smc.bin +firmware: amdgpu/pitcairn_k_smc.bin +firmware: amdgpu/verde_smc.bin +firmware: amdgpu/verde_k_smc.bin +firmware: amdgpu/oland_smc.bin +firmware: amdgpu/oland_k_smc.bin +firmware: amdgpu/hainan_smc.bin +firmware: amdgpu/hainan_k_smc.bin +firmware: amdgpu/banks_k_2_smc.bin +firmware: amdgpu/renoir_dmcub.bin +firmware: amdgpu/sienna_cichlid_dmcub.bin +firmware: amdgpu/navy_flounder_dmcub.bin +firmware: amdgpu/green_sardine_dmcub.bin +firmware: amdgpu/vangogh_dmcub.bin +firmware: amdgpu/dimgrey_cavefish_dmcub.bin +firmware: amdgpu/beige_goby_dmcub.bin +firmware: amdgpu/yellow_carp_dmcub.bin +firmware: amdgpu/dcn_3_1_4_dmcub.bin +firmware: amdgpu/dcn_3_1_5_dmcub.bin +firmware: amdgpu/dcn_3_1_6_dmcub.bin +firmware: amdgpu/dcn_3_2_0_dmcub.bin +firmware: amdgpu/dcn_3_2_1_dmcub.bin +firmware: amdgpu/raven_dmcu.bin +firmware: amdgpu/navi12_dmcu.bin +firmware: amdgpu/dcn_3_5_dmcub.bin +firmware: amdgpu/dcn_3_5_1_dmcub.bin +firmware: amdgpu/dcn_3_6_dmcub.bin +firmware: amdgpu/dcn_4_0_1_dmcub.bin +firmware: amdgpu/isp_4_1_1.bin +srcversion: A855299D7FF772A65584BCB + +-- rocminfo (gfx target) -- + Name: AMD Ryzen 9 5900X 12-Core Processor + Marketing Name: AMD Ryzen 9 5900X 12-Core Processor + Vendor Name: CPU + Compute Unit: 24 + Name: gfx1201 + Marketing Name: AMD Radeon AI PRO R9700 + Vendor Name: AMD + Compute Unit: 64 + Name: amdgcn-amd-amdhsa--gfx1201 + Name: amdgcn-amd-amdhsa--gfx12-generic + +========== ROCm component package versions ========== +comgr 2:7.2.3-1 +hip-runtime-amd 7.2.3-1.1 +hipblas 7.2.3-1.1 +hipblas-common 7.2.3-1 +hipblaslt 7.2.3-1 +hipcub 7.2.3-1.1 +hipfft 7.2.3-1.1 +hiprand 7.2.3-1.1 +hipsolver 7.2.3-1.1 +hipsparse 7.2.3-1.1 +linux-firmware-amdgpu 1:20260519-1 +miopen-hip 7.2.3-1 +rocblas 7.2.3-1 +rocm-cmake 7.2.3-1 +rocm-core 7.2.3-1.1 +rocm-device-libs 2:7.2.3-1 +rocm-hip-libraries 7.2.3-1 +rocm-hip-runtime 7.2.3-1 +rocm-hip-sdk 7.2.3-1 +rocm-language-runtime 7.2.3-1 +rocm-llvm 2:7.2.3-1 +rocm-opencl-runtime 7.2.3-1 +rocm-smi-lib 7.2.0-2.1 +rocminfo 7.2.3-1.1 + +========== Tensile library files present (issue #7192 lead) ========== +-rw-r--r-- 1 root root 910520 May 5 08:13 /opt/rocm/lib/rocblas/library/Kernels.so-000-gfx1200.hsaco +-rw-r--r-- 1 root root 910520 May 5 08:13 /opt/rocm/lib/rocblas/library/Kernels.so-000-gfx1201.hsaco +-rw-r--r-- 1 root root 17653 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_lazy_gfx1200.dat +-rw-r--r-- 1 root root 17653 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_lazy_gfx1201.dat +-rw-r--r-- 1 root root 247232 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 247232 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 206336 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 206336 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 245696 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 245696 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 204032 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 204032 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_4xi8I_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 747232 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 747232 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 658144 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 658144 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 307904 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 307904 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 653536 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 653536 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BB_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 704128 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 704128 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 359232 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 359232 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 128856 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 128856 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 296296 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 296296 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_BS_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66680 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Ailk_BjlkC_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66680 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Ailk_BjlkC_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66392 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66392 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66392 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66392 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66680 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Alik_BjlkC_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66680 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Alik_BjlkC_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66648 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66648 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66136 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66136 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 68752 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_AlikC_BjlkC_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 68752 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_AlikC_BjlkC_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66680 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_AlikC_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66680 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_AlikC_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 66168 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_AlikC_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 66168 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_CC_Contraction_l_AlikC_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 31256 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 31256 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 51392 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 51392 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 31256 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 31256 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 31256 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 31256 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_DD_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 255296 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 255296 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 132792 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 132792 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 149192 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 149192 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 132280 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 132280 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 608992 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 608992 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 294240 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 294240 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 176592 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 176592 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 243952 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 243952 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HH_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 553896 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 553896 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 300928 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 300928 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 161024 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 161024 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 350272 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 350272 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_HS_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 717192 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 717192 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 500224 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 500224 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 697792 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 697792 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 628176 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 628176 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_I8I_HPA_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 74808 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 74808 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 51240 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 51240 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 50472 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 50472 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 60984 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 60984 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_SS_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70776 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Ailk_BjlkC_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70776 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Ailk_BjlkC_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70488 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70488 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Ailk_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70488 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70488 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Ailk_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70776 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Alik_BjlkC_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70776 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Alik_BjlkC_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70488 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70488 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Alik_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 69720 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 69720 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_Alik_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70800 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_AlikC_BjlkC_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70800 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_AlikC_BjlkC_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70776 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_AlikC_Bjlk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70776 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_AlikC_Bjlk_Cijk_Dijk_fallback_gfx1201.hsaco +-rw-r--r-- 1 root root 70008 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_AlikC_Bljk_Cijk_Dijk_fallback_gfx1200.hsaco +-rw-r--r-- 1 root root 70008 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_Type_ZZ_Contraction_l_AlikC_Bljk_Cijk_Dijk_fallback_gfx1201.hsaco + +========== Calling stack (torch / hip / bitsandbytes) ========== +torch 2.11.0+rocm7.2 | hip 7.2.26015 | bnb 0.49.2 + +========== rocblas-bench availability (for the standalone reproducer) ========== +rocblas-bench: FOUND -> /opt/rocm/bin/rocblas-bench +Next (separate GPU run): add ROCBLAS_LAYER=2 to the training launch env to emit the +exact "rocblas-bench --transposeA ... -m 608 -n 5376 -k 21504 --lda ... --ldb ..." line +for the faulting GEMM -- that command line is the stack-independent reproducer. + +========== DONE ========== +Saved to: gfx1201-report-env-20260601-190044.txt diff --git a/docs/diagnostics/gfx1201-report-env-20260601-191024.txt b/docs/diagnostics/gfx1201-report-env-20260601-191024.txt new file mode 100644 index 00000000..4113a825 --- /dev/null +++ b/docs/diagnostics/gfx1201-report-env-20260601-191024.txt @@ -0,0 +1,95 @@ + +========== ROCm / HIP build ========== +7.2.53211-364a905 +7.2.3 + +========== OS / kernel ========== +Linux cachy 7.0.10-2-cachyos #1 SMP PREEMPT_DYNAMIC Thu, 28 May 2026 21:12:58 +0000 x86_64 GNU/Linux +PRETTY_NAME="CachyOS" + +========== GPU / driver / firmware / VBIOS ========== + + +============================ ROCm System Management Interface ============================ +============================== Version of System Component =============================== +Driver version: 7.0.10-2-cachyos +========================================================================================== +================================== End of ROCm SMI Log =================================== + + +============================ ROCm System Management Interface ============================ +========================================= VBIOS ========================================== +GPU[0] : VBIOS version: 113-1E4990U-S83 +========================================================================================== +====================================== Product Info ====================================== +GPU[0] : Card Series: AMD Radeon AI PRO R9700 +GPU[0] : Card Model: 0x7551 +GPU[0] : Card Vendor: Advanced Micro Devices, Inc. [AMD/ATI] +GPU[0] : Card SKU: 1E4990U +GPU[0] : Subsystem ID: -0x1b67 +GPU[0] : Device Rev: 0xc0 +GPU[0] : Node ID: 1 +GPU[0] : GUID: 44204 +GPU[0] : GFX Version: gfx1201 +========================================================================================== +================================== End of ROCm SMI Log =================================== + +-- amdgpu module -- +srcversion: A855299D7FF772A65584BCB +vermagic: 7.0.10-2-cachyos SMP preempt mod_unload + +-- rocminfo (gfx target) -- + Name: AMD Ryzen 9 5900X 12-Core Processor + Marketing Name: AMD Ryzen 9 5900X 12-Core Processor + Vendor Name: CPU + Compute Unit: 24 + Name: gfx1201 + Marketing Name: AMD Radeon AI PRO R9700 + Vendor Name: AMD + Compute Unit: 64 + Name: amdgcn-amd-amdhsa--gfx1201 + Name: amdgcn-amd-amdhsa--gfx12-generic + +========== ROCm component package versions ========== +comgr 2:7.2.3-1 +hip-runtime-amd 7.2.3-1.1 +hipblas 7.2.3-1.1 +hipblas-common 7.2.3-1 +hipblaslt 7.2.3-1 +hipcub 7.2.3-1.1 +hipfft 7.2.3-1.1 +hiprand 7.2.3-1.1 +hipsolver 7.2.3-1.1 +hipsparse 7.2.3-1.1 +linux-firmware-amdgpu 1:20260519-1 +miopen-hip 7.2.3-1 +rocblas 7.2.3-1 +rocm-cmake 7.2.3-1 +rocm-core 7.2.3-1.1 +rocm-device-libs 2:7.2.3-1 +rocm-hip-libraries 7.2.3-1 +rocm-hip-runtime 7.2.3-1 +rocm-hip-sdk 7.2.3-1 +rocm-language-runtime 7.2.3-1 +rocm-llvm 2:7.2.3-1 +rocm-opencl-runtime 7.2.3-1 +rocm-smi-lib 7.2.0-2.1 +rocminfo 7.2.3-1.1 + +========== Tensile library files present (issue #7192 lead) ========== +-rw-r--r-- 1 root root 910520 May 5 08:13 /opt/rocm/lib/rocblas/library/Kernels.so-000-gfx1200.hsaco +-rw-r--r-- 1 root root 910520 May 5 08:13 /opt/rocm/lib/rocblas/library/Kernels.so-000-gfx1201.hsaco +-rw-r--r-- 1 root root 17653 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_lazy_gfx1200.dat +-rw-r--r-- 1 root root 17653 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_lazy_gfx1201.dat + +========== Calling stack (torch / hip / bitsandbytes) ========== +torch 2.11.0+rocm7.2 | hip 7.2.26015 | bnb 0.49.2 + +========== rocblas-bench availability (for the standalone reproducer) ========== +rocblas-bench: FOUND -> /opt/rocm/bin/rocblas-bench +Next (separate GPU run): add ROCBLAS_LAYER=2 to the training launch env to emit the +exact "rocblas-bench --transposeA ... -m 608 -n 5376 -k 21504 --lda ... --ldb ..." line +for the faulting GEMM -- that command line is the stack-independent reproducer. + +========== DONE ========== +Saved to: /home/ollie/Github/csen-346/docs/diagnostics/gfx1201-report-env-20260601-191024.txt diff --git a/docs/diagnostics/gfx1201-report-env-20260605-101605.txt b/docs/diagnostics/gfx1201-report-env-20260605-101605.txt new file mode 100644 index 00000000..0a27dece --- /dev/null +++ b/docs/diagnostics/gfx1201-report-env-20260605-101605.txt @@ -0,0 +1,95 @@ + +========== ROCm / HIP build ========== +7.2.53211-364a905 +7.2.3 + +========== OS / kernel ========== +Linux cachy 7.0.10-2-cachyos #1 SMP PREEMPT_DYNAMIC Thu, 28 May 2026 21:12:58 +0000 x86_64 GNU/Linux +PRETTY_NAME="CachyOS" + +========== GPU / driver / firmware / VBIOS ========== + + +============================ ROCm System Management Interface ============================ +============================== Version of System Component =============================== +Driver version: 7.0.10-2-cachyos +========================================================================================== +================================== End of ROCm SMI Log =================================== + + +============================ ROCm System Management Interface ============================ +========================================= VBIOS ========================================== +GPU[0] : VBIOS version: 113-1E4990U-S83 +========================================================================================== +====================================== Product Info ====================================== +GPU[0] : Card Series: AMD Radeon AI PRO R9700 +GPU[0] : Card Model: 0x7551 +GPU[0] : Card Vendor: Advanced Micro Devices, Inc. [AMD/ATI] +GPU[0] : Card SKU: 1E4990U +GPU[0] : Subsystem ID: -0x1b67 +GPU[0] : Device Rev: 0xc0 +GPU[0] : Node ID: 1 +GPU[0] : GUID: 44204 +GPU[0] : GFX Version: gfx1201 +========================================================================================== +================================== End of ROCm SMI Log =================================== + +-- amdgpu module -- +srcversion: A855299D7FF772A65584BCB +vermagic: 7.0.10-2-cachyos SMP preempt mod_unload + +-- rocminfo (gfx target) -- + Name: AMD Ryzen 9 5900X 12-Core Processor + Marketing Name: AMD Ryzen 9 5900X 12-Core Processor + Vendor Name: CPU + Compute Unit: 24 + Name: gfx1201 + Marketing Name: AMD Radeon AI PRO R9700 + Vendor Name: AMD + Compute Unit: 64 + Name: amdgcn-amd-amdhsa--gfx1201 + Name: amdgcn-amd-amdhsa--gfx12-generic + +========== ROCm component package versions ========== +comgr 2:7.2.3-1 +hip-runtime-amd 7.2.3-1.1 +hipblas 7.2.3-1.1 +hipblas-common 7.2.3-1 +hipblaslt 7.2.3-1 +hipcub 7.2.3-1.1 +hipfft 7.2.3-1.1 +hiprand 7.2.3-1.1 +hipsolver 7.2.3-1.1 +hipsparse 7.2.3-1.1 +linux-firmware-amdgpu 1:20260519-1 +miopen-hip 7.2.3-1 +rocblas 7.2.3-1 +rocm-cmake 7.2.3-1 +rocm-core 7.2.3-1.1 +rocm-device-libs 2:7.2.3-1 +rocm-hip-libraries 7.2.3-1 +rocm-hip-runtime 7.2.3-1 +rocm-hip-sdk 7.2.3-1 +rocm-language-runtime 7.2.3-1 +rocm-llvm 2:7.2.3-1 +rocm-opencl-runtime 7.2.3-1 +rocm-smi-lib 7.2.0-2.1 +rocminfo 7.2.3-1.1 + +========== Tensile library files present (issue #7192 lead) ========== +-rw-r--r-- 1 root root 910520 May 5 08:13 /opt/rocm/lib/rocblas/library/Kernels.so-000-gfx1200.hsaco +-rw-r--r-- 1 root root 910520 May 5 08:13 /opt/rocm/lib/rocblas/library/Kernels.so-000-gfx1201.hsaco +-rw-r--r-- 1 root root 17653 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_lazy_gfx1200.dat +-rw-r--r-- 1 root root 17653 May 5 08:13 /opt/rocm/lib/rocblas/library/TensileLibrary_lazy_gfx1201.dat + +========== Calling stack (torch / hip / bitsandbytes) ========== +torch 2.11.0+cu130 | hip None | bnb 0.49.2 + +========== rocblas-bench availability (for the standalone reproducer) ========== +rocblas-bench: FOUND -> /opt/rocm/bin/rocblas-bench +Next (separate GPU run): add ROCBLAS_LAYER=2 to the training launch env to emit the +exact "rocblas-bench --transposeA ... -m 608 -n 5376 -k 21504 --lda ... --ldb ..." line +for the faulting GEMM -- that command line is the stack-independent reproducer. + +========== DONE ========== +Saved to: /home/ollie/Github/csen-346/docs/diagnostics/gfx1201-report-env-20260605-101605.txt diff --git a/docs/diagnostics/gfx1201_fault_2026-06-01.log b/docs/diagnostics/gfx1201_fault_2026-06-01.log new file mode 100644 index 00000000..cb88450a --- /dev/null +++ b/docs/diagnostics/gfx1201_fault_2026-06-01.log @@ -0,0 +1,4221 @@ +wandb: [wandb.login()] Loaded credentials for https://api.wandb.ai from /home/ollie/.netrc. +wandb: Currently logged in as: uchavarria (uchavarria-santa-clara-university) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin + +Loading training data sources=['socrat-zh-sft', 'socrat-en-sft'] + train: 77202 records + eval: 8578 records + +Loading tokenizer model=unsloth/gemma-4-31B-it-unsloth-bnb-4bit + Patched tokenizer.chat_template for Gemma 4 (added {% generation %} markers) + QLoRA: loading pre-quantized NF4 checkpoint — skipping BF16 staging + Loading weights method=qlora dtype=auto + Loading weights: 0%| | 0/1188 [00:00` (accepts any SeqClassification checkpoint). +- Single 20 GB card: **cannot serve two models at once**, and training and serving + compete for VRAM — do them sequentially. + +--- + +## Decisions baked into the scaffolding + +- **Eval scale:** smoke-gate → full (n=5 sanity, then full n=681 on base and SFT). +- **MTP:** base only (the drafter is base-derived → highest acceptance; the uplift + comparison itself stays MTP-off so it isn't confounded). +- **Quant:** base served at user-chosen `gemma-4-12b-it-UD-Q8_K_XL.gguf` (13.6 GB); + SFT served at `Q8_0` (UD is not stock-llama.cpp-producible). At Q8 the + quant-scheme delta is ~noise, so base↔SFT differ effectively only by the adapter. + +--- + +## What this branch already landed (don't redo) + +Commits `fb82188` (scaffold) + `81686c2` (convert wrapper): + +- `configs/train-sft-gemma4-12b-qlora.env` — QLoRA, 1 epoch (~4826 steps @ + eff-batch 16 over 77202 records), anchored LoRA regex (ports unchanged to the + 12B's 48 decoder layers), `socrat-zh-sft,socrat-en-sft` (inference-matching + schema), `save_steps=50`, in-training eval off. +- `configs/gemma4-12b-local.env` / `configs/gemma4-12b-sft-local.env` — dual-role + local-serve eval configs; distinct teacher aliases (`Gemma 4 12B` vs + `Gemma 4 12B SFT`) so the eval fails fast against the wrong loaded weights. +- `scripts/serve_gemma4_12b.sh` — serves UD-Q8_K_XL via the generic engine + `serve_gemma4_31b.sh` (CUDA auto-detect). `MTP=1` attaches the drafter + (`MTP/gemma-4-12B-it-MTP-Q8_0.gguf`) with `--spec-type draft-mtp + --spec-draft-model … --spec-draft-n-max 4` and forces f16 KV. +- `scripts/serve_gemma4_12b_sft.sh` — serves the merged SFT Q8_0 GGUF. +- `scripts/convert_gemma4_12b_sft_to_gguf.sh` — merge→GGUF with the **12B** + NAME_TAG and auto-stage into the weights dir (no manual rename; default Q8_0). +- `Makefile` — `nvidia-preflight`, `train-gemma4-12b{,-dry-run}`, + `serve-gemma4-12b{,-mtp,-sft}`, `eval-gemma4-12b-{base,sft}-{smoke,full}`, + `_classifier-ckpt` (downloads the Qwen3.5 classifier on first eval). + +Validated off-GPU: dry-run loads 77202/8578 records + accepts the LoRA config; +shellcheck clean; `make -n` parses all targets; venv torch is `2.11.0+cu130` +(`cuda_avail=True`). + +--- + +## §3 — Gated run order (do these on the box) + +``` +G1 make nvidia-preflight # HARD GATE: CUDA fwd/bwd must pass +G1.5 uv lock --upgrade-package transformers # → 5.10.2; uv sync. gemma4_unified needs ≥5.10 + uv pip install --index-url https://download.pytorch.org/whl/cu130 torch==2.12.0 # optional bump + # verify: AutoConfig.from_pretrained("unsloth/gemma-4-12b-it") loads + bnb 4-bit smoke +G2 Assets: + hf download unsloth/gemma-4-12b-it-GGUF gemma-4-12b-it-UD-Q8_K_XL.gguf --local-dir ~/Documents/models/weights + hf download unsloth/gemma-4-12b-it-GGUF MTP/gemma-4-12B-it-MTP-Q8_0.gguf --local-dir ~/Documents/models/weights + # classifier auto-downloads on first eval (target: BERT_CKPT) +G2.5 Build llama.cpp (NOT installed by default — hard prereq for all eval gates): + git clone --depth 1 https://github.com/ggml-org/llama.cpp ~/Documents/models/llama.cpp + cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=89 -DCMAKE_BUILD_TYPE=Release + cmake --build build --target llama-server -j 6 # -j 6 not 24: ptxas SIGSEGVs on flaky silicon + # needs main ≥ PR #23398 for gemma4-assistant/MTP (today's main has it) +G3 make serve-gemma4-12b (bg) ; poll: curl localhost:8080/v1/models (alias "Gemma 4 12B") + make eval-gemma4-12b-base-smoke # SANITY GATE: non-degenerate state_accuracy; then stop server + # (power-cap / crash-crawl is now the MONITOR's job, see below) +G4 make monitor-eval-gemma4-12b-base # phase 1a: base MTP OFF → results/gemma4-12b-base (CHECKPOINT) + MTP=1 make monitor-eval-gemma4-12b-base # phase 1b: base MTP ON → results/gemma4-12b-base-mtp (CHECKPOINT) + # monitor owns serve+eval, power search (start max, -10W/crash), repair-resume, issue #130 logging. + # pick the winner on accuracy (should be ~equal) & tok/s; that MTP setting carries to phase 3. +G5 make train-gemma4-12b # ~4826 steps; auto-resumes (save_steps=50) + HF push every 50 + # (→ ulises-c/SocratesLM-12B-QLoRA). GATE: outputs/sft-gemma4-12b-qlora/final/adapter_model.safetensors +G6 scripts/merge_lora_gemma4_sft.py --base google/gemma-4-12b-it \ + --adapter outputs/sft-gemma4-12b-qlora/final --out outputs/sft-gemma4-12b-qlora/merged + bash scripts/convert_gemma4_12b_sft_to_gguf.sh # → Q8_0 GGUF, auto-staged to weights dir + # needs ~24 GB system RAM (BF16 staging); GPU idle +G7 make serve-gemma4-12b-sft ; make eval-gemma4-12b-sft-smoke # SANITY GATE; then stop server +G8 [MTP=1] make monitor-eval-gemma4-12b-sft # phase 3: SFT served with the phase-1 winner → results/gemma4-12b-sft(-mtp) (CHECKPOINT) +G9 python -m src.project.evaluate --compare results/gemma4-12b-base results/gemma4-12b-sft + # uplift = SFT − base on state_accuracy (overall + per-stage), rouge*, bleu4 +``` + +--- + +## Caveats / things to verify on the box (not code bugs) + +1. **MTP needs a llama.cpp build ≥ PR #23398** (merged 2026-06-07; arch + `gemma4-assistant`) — stock builds can't load the drafter. Also: the spec-flag + names (`--spec-type draft-mtp / --spec-draft-model / --spec-draft-n-max`) are + PR-sourced; verify against `llama-server --help` on the actual build — older + llama.cpp speculative convention was `-md / --model-draft / --draft-max`. +2. **VRAM at Q8 is untested.** 13.6 GB weights + the engine's inherited `-b 2048` + compute buffer + KV (+ draft model under MTP) on 20 GB. If serve OOMs, lower + `-c` / `-b` / `-np` (all overridable; defaults `-c 32768`). +3. **System RAM ≥ ~32 GB** for the G6 BF16 merge. +4. **MTP-on-SFT** (if you extend beyond base): the base-derived drafter sees the + SFT'd distribution → lower acceptance / smaller speedup, still lossless. + +--- + +## Next-next (out of scope here) + +If 1 epoch underfits, bump `TRAIN_EPOCHS` (clear `outputs/sft-gemma4-12b-qlora/ +checkpoint-*` first). If uplift is real, the same pattern scales to a fuller +NVIDIA training run or a DPO stage. The 31B AMD track is independent and unchanged. diff --git a/hooks/pre-commit b/hooks/pre-commit index d2466c23..86fc6527 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -38,7 +38,7 @@ fi # Step 3: Type checking (pyright — mirrors CI lint job) # --------------------------------------------------------------------------- info "Type checking (pyright) …" -if uv run --no-sync pyright; then +if uv run --no-sync python -m pyright; then pass "Type check OK" else fail "Type check failed. Fix the errors above before committing." @@ -49,7 +49,7 @@ fi # Step 4: Spell check (codespell — mirrors CI lint job) # --------------------------------------------------------------------------- info "Spell checking (codespell) …" -if uv run --no-sync codespell .; then +if uv run --no-sync python -m codespell_lib .; then pass "Spell check OK" else fail "Spell check failed. Fix the typos above before committing." @@ -78,7 +78,7 @@ fi # Note: using pytest directly avoids ambiguity with the shell builtin `test`. # --------------------------------------------------------------------------- info "Running tests …" -if uv run --no-sync pytest -rs --cov --cov-report=term-missing; then +if uv run --no-sync python -m pytest -rs --cov --cov-report=term-missing; then pass "Tests passed" else fail "Tests failed." diff --git a/pyproject.toml b/pyproject.toml index 50192df3..03985d77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "trl>=1.5.1", "tiktoken>=0.7.0", "pyspellchecker>=0.9.0", + "wandb>=0.18.0", ] [project.optional-dependencies] @@ -48,6 +49,13 @@ cuda = ["flash-linear-attention>=0.5.0"] # Web UI for the live demo. Install with `uv sync --extra demo`. demo = ["chainlit>=2.4,<3"] +# LLM call tracing + eval (issue #111) for the inference/eval side. Opt in with +# `uv sync --extra weave`; gated at runtime behind WEAVE_PROJECT (default off, no +# behavior change when unset). weave.init() auto-patches the openai clients in +# socratic_teaching_system — traces the consultant→teacher dialogue loop. Traces +# (dialogue text) egress to the W&B cloud; keep off on the headless training box. +weave = ["weave>=0.52"] + [project.scripts] kele = "src.project.kele:main" kele-eval = "src.project.evaluate:main" @@ -100,11 +108,12 @@ dev = [ # Binary image formats (PDF/PNG/JPG) match false positives against compressed # byte sequences inside the file streams; runtime logs are noise. Skip them # outright — none are human-edited source content. -skip = "uv.lock,*references*,*deliverables*,*results*,.venv,*data*,*logs*,*.pdf,*.png,*.jpg,*.jpeg,*.gif" +skip = "uv.lock,*references*,*deliverables*,*results*,.venv,*data*,*logs*,*.pdf,*.png,*.jpg,*.jpeg,*.gif,*outputs*,*.log" # Terms that are not typos: # rouge — ROUGE (NLP metric: Recall-Oriented Understudy for Gisting Evaluation) # nothink — Qwen model inference mode flag (think / nothink) # re-use — hyphenated form is valid; codespell prefers the unhyphenated reuse +# hsa — AMD ROCm HSA_OVERRIDE_GFX_VERSION env var (not "has") ignore-words-list = "rouge,nothink,re-use,hsa" [build-system] diff --git a/scripts/README.md b/scripts/README.md index 36f63cd0..3004fc7b 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,24 +1,77 @@ # scripts/ -Shell scripts for environment setup and process management. Not for Python tooling. - -| Script | Purpose | -|--------|---------| -| `run_eval.sh` | Launch a batch evaluation run for a named experiment | -| `eval_amd_mac.sh` | End-to-end eval: starts AMD teacher locally, runs against Mac Mini consultant | -| `serve_teacher_local.sh` | Serve SocratTeachLLM teacher from local disk | -| `serve_both.sh` | Start teacher + consultant vLLM servers together | -| `serve_dual_gpu.sh` | Serve teacher (GPU 0) and consultant (GPU 1) on dual-GPU hosts | -| `serve_socratteachllm.sh` | Serve SocratTeachLLM teacher only | -| `serve_consultant.sh` | Serve consultant model via vLLM | -| `serve_consultant_llamacpp.sh` | Serve consultant model via llama.cpp (Mac Mini) | -| `serve_teacher_online.sh` | Serve teacher using an online API endpoint | -| `serve_gemma4.sh` | Serve Gemma 4 as the teacher | -| `post_eval_shutdown.sh` | Shut down vLLM servers after a run completes | -| `l40s_setup.sh` | One-time environment setup for L40S hosts | -| `mac_mini_setup.sh` | One-time environment setup for Mac Mini (llama.cpp) | -| `wave_setup.sh` | One-time environment setup for Wave cluster | - -Setup docs (`.md`) for each hardware target live alongside their setup scripts: -[`MAC_MINI_SETUP.md`](MAC_MINI_SETUP.md) (llama.cpp), [`MLX_SETUP.md`](MLX_SETUP.md) (MLX alternative). -Python utilities belong in `src/project/` instead. +Shell entry points for serving models, running evals, and host setup. Python +utilities also live here, but most belong in `src/project/`. Retired one-off and +per-model eval shells have moved to [`archive/`](archive/) — see below. + +Most of these are also exposed as `make` targets; run `make help` for the full list. + +## Current run — Gemma 4 12B SFT-uplift PoC (NVIDIA) + +The active run drives `src.project.kele` directly from the Makefile — no per-model +eval shell. This is the forward template: a new model is a Makefile target + a +`configs/.env`, not a new copy of a 280-line script. + +| Entry point | Purpose | +|-------------|---------| +| `make eval-gemma4-12b-base-{smoke,full}` | Eval the base 12B teacher (BERT consultant) | +| `make eval-gemma4-12b-sft-{smoke,full}` | Eval the 1-epoch Socratic QLoRA SFT teacher | +| `make monitor-eval-gemma4-12b-{base,sft}` → `monitor_eval_gemma4_12b.sh` | Crash-crawl monitor: serving + adaptive power search + HF push | +| `serve_gemma4_12b.sh` / `serve_gemma4_12b_sft.sh` / `serve_gemma4_12b_mtp.sh` | Serve base / SFT / MTP-drafter variants | +| `convert_gemma4_12b_sft_to_gguf.sh` | Convert the SFT checkpoint to GGUF for llama.cpp | + +## Local-model eval runner + +`eval_llamacpp.sh` is the parameterized replacement for the 20 retired `eval_*.sh` +scripts: a single-server llama.cpp orchestrator (teacher = consultant on port 8080) +that boots the model's serve script, runs KELE, and compares against the +`results/baseline` gpt-4o run. + +``` +./scripts/eval_llamacpp.sh [flags] +``` + +| model | serve (default / `--think`) | alias | experiment | +|-------|------------------------------|-------|------------| +| `qwen27b` | `serve_qwen27b_q5.sh` / `_think` | Qwen 27B Q5 | `qwen27b-local` | +| `qwen35b-a3b` | `serve_qwen35b_a3b.sh` / `_think` | Qwen 35B A3B | `qwen35b-a3b-local` | +| `qwopus35b-a3b` | `serve_qwopus35b_a3b_think.sh` | Qwopus 35B A3B | `qwopus35b-a3b-local` | +| `gemma4-31b` | `serve_gemma4_31b_q5.sh` | Gemma 4 31B | `gemma4-31b-local` | +| `gemma4-26b-a4b` | `serve_gemma4_26b_a4b.sh` | Gemma 4 26B A4B | `gemma4-26b-a4b-local` | + +Modes: `smoke` (n=5) · `mini` (n=25) · `n50` (n=50) · `full` (n=681). +Flags: `--think` (Qwen only), `--nothink` (Qwen only), `--unified` (fusion single-call), +`--suffix NAME`, `--keep-server`, `--no-compare`. + +The Makefile wraps the common cells: `eval-qwen27b-*`, `eval-qwen35b-a3b-*`, +`eval-gemma4-31b-*`, and the `*-fusion-*` smoke targets all call this runner. + +## Serving (standalone) + +`serve_*.sh` boot a single model on llama.cpp or vLLM. Beyond the Gemma 4 12B and +eval-runner entries above, common ones: `serve_consultant*.sh`, `serve_socratteachllm*.sh`, +`serve_teacher_{local,online}.sh`, `serve_both.sh`, `serve_dual_gpu.sh`, +`serve_demo_top_performer.sh`. `make help` lists them all with one-liners. + +## Host setup + +| Script | Host | +|--------|------| +| `l40s_setup.sh` | Dual L40S box ([`RTX5090_SETUP.md`](RTX5090_SETUP.md), [`WAVE_SETUP.md`](WAVE_SETUP.md)) | +| `amd_r9700_setup.sh` | AMD R9700 / gfx1201 (ROCm) | +| `mac_mini_setup.sh` | Mac Mini llama.cpp ([`MAC_MINI_SETUP.md`](MAC_MINI_SETUP.md), [`MLX_SETUP.md`](MLX_SETUP.md)) | +| `wave_setup.sh` | Wave HPC cluster | + +`patch_fla_rocm.sh` and the `*gfx1201*` / `*rocblas*` scripts are RDNA4 fault +diagnostics — see the GFX1201 handoffs in `docs/`. + +## Process management + +`post_eval_shutdown.sh` (tear down vLLM servers after a run) · +`run_eval.sh` (`make run-eval GPU=`). + +## archive/ + +[`archive/`](archive/) holds retired per-model eval shells (the 20 `eval_*.sh` that +`eval_llamacpp.sh` replaced) and dated one-off chains/backtests. Kept for provenance +and to reconstruct a past run's exact settings; not wired into the Makefile. diff --git a/scripts/archive/README.md b/scripts/archive/README.md new file mode 100644 index 00000000..6616847b --- /dev/null +++ b/scripts/archive/README.md @@ -0,0 +1,29 @@ +# scripts/archive/ + +Retired scripts kept for provenance. **Not wired into the Makefile** and not part of +any current workflow — they document how past runs were driven and let you reconstruct +a run's exact settings. New work should use the entry points in [`../README.md`](../README.md). + +## Retired per-model eval shells + +The 20 `eval_*.sh` orchestrators, superseded by the parameterized +[`../eval_llamacpp.sh`](../eval_llamacpp.sh) (single-server llama.cpp serve+eval+compare) +and, for the current run, by the Makefile→`kele` path. + +- **Single-server (now `eval_llamacpp.sh` config rows):** `eval_qwen27b.sh`, + `eval_qwen27b_think.sh`, `eval_qwen35b_a3b.sh`, `eval_qwen35b_a3b_n50.sh`, + `eval_qwen35b_a3b_n50_think.sh`, `eval_qwopus35b_a3b.sh`, `eval_gemma4_31b.sh`, + `eval_gemma4_26b_a4b.sh` +- **BERT few-shot orchestrators:** `eval_bert_a3b_fewshot10_full.sh`, + `eval_bert_gemma_fewshot10_full.sh`, `eval_bert_glm49b_chat_fewshot10_full.sh`, + `eval_bert_qwen27b_fewshot10_full.sh`, `eval_bert_qwen27b_nothink_fewshot10_full.sh`, + `eval_bert_socratteachllm_fewshot10_full.sh`, `eval_bert_claude_fewshot10.sh` +- **AMD Linux + Mac Mini (2-machine):** `eval_amd_mac.sh`, `eval_amd_2mac.sh`, + `eval_amd_2mac_mlx.sh` +- **Claude / prompt-tournament:** `eval_claude_consultant_socratteachllm.sh`, + `eval_prompt_tournament.sh` + +## Dated one-off chains / backtests + +`autonomous_chain_2026_05_16.sh`, `overnight_qwen27b_chain.sh`, +`run_all_fusion_smokes.sh`, `post_chain_backtest.sh`, `post_phase3_backtest.sh`. diff --git a/scripts/autonomous_chain_2026_05_16.sh b/scripts/archive/autonomous_chain_2026_05_16.sh similarity index 100% rename from scripts/autonomous_chain_2026_05_16.sh rename to scripts/archive/autonomous_chain_2026_05_16.sh diff --git a/scripts/eval_amd_2mac.sh b/scripts/archive/eval_amd_2mac.sh similarity index 100% rename from scripts/eval_amd_2mac.sh rename to scripts/archive/eval_amd_2mac.sh diff --git a/scripts/eval_amd_2mac_mlx.sh b/scripts/archive/eval_amd_2mac_mlx.sh similarity index 100% rename from scripts/eval_amd_2mac_mlx.sh rename to scripts/archive/eval_amd_2mac_mlx.sh diff --git a/scripts/eval_amd_mac.sh b/scripts/archive/eval_amd_mac.sh similarity index 100% rename from scripts/eval_amd_mac.sh rename to scripts/archive/eval_amd_mac.sh diff --git a/scripts/eval_bert_a3b_fewshot10_full.sh b/scripts/archive/eval_bert_a3b_fewshot10_full.sh similarity index 100% rename from scripts/eval_bert_a3b_fewshot10_full.sh rename to scripts/archive/eval_bert_a3b_fewshot10_full.sh diff --git a/scripts/eval_bert_claude_fewshot10.sh b/scripts/archive/eval_bert_claude_fewshot10.sh similarity index 100% rename from scripts/eval_bert_claude_fewshot10.sh rename to scripts/archive/eval_bert_claude_fewshot10.sh diff --git a/scripts/eval_bert_gemma_fewshot10_full.sh b/scripts/archive/eval_bert_gemma_fewshot10_full.sh similarity index 100% rename from scripts/eval_bert_gemma_fewshot10_full.sh rename to scripts/archive/eval_bert_gemma_fewshot10_full.sh diff --git a/scripts/eval_bert_glm49b_chat_fewshot10_full.sh b/scripts/archive/eval_bert_glm49b_chat_fewshot10_full.sh similarity index 100% rename from scripts/eval_bert_glm49b_chat_fewshot10_full.sh rename to scripts/archive/eval_bert_glm49b_chat_fewshot10_full.sh diff --git a/scripts/eval_bert_qwen27b_fewshot10_full.sh b/scripts/archive/eval_bert_qwen27b_fewshot10_full.sh similarity index 100% rename from scripts/eval_bert_qwen27b_fewshot10_full.sh rename to scripts/archive/eval_bert_qwen27b_fewshot10_full.sh diff --git a/scripts/eval_bert_qwen27b_nothink_fewshot10_full.sh b/scripts/archive/eval_bert_qwen27b_nothink_fewshot10_full.sh similarity index 100% rename from scripts/eval_bert_qwen27b_nothink_fewshot10_full.sh rename to scripts/archive/eval_bert_qwen27b_nothink_fewshot10_full.sh diff --git a/scripts/eval_bert_socratteachllm_fewshot10_full.sh b/scripts/archive/eval_bert_socratteachllm_fewshot10_full.sh similarity index 100% rename from scripts/eval_bert_socratteachllm_fewshot10_full.sh rename to scripts/archive/eval_bert_socratteachllm_fewshot10_full.sh diff --git a/scripts/eval_claude_consultant_socratteachllm.sh b/scripts/archive/eval_claude_consultant_socratteachllm.sh similarity index 100% rename from scripts/eval_claude_consultant_socratteachllm.sh rename to scripts/archive/eval_claude_consultant_socratteachllm.sh diff --git a/scripts/eval_gemma4_26b_a4b.sh b/scripts/archive/eval_gemma4_26b_a4b.sh similarity index 100% rename from scripts/eval_gemma4_26b_a4b.sh rename to scripts/archive/eval_gemma4_26b_a4b.sh diff --git a/scripts/eval_gemma4_31b.sh b/scripts/archive/eval_gemma4_31b.sh similarity index 100% rename from scripts/eval_gemma4_31b.sh rename to scripts/archive/eval_gemma4_31b.sh diff --git a/scripts/eval_prompt_tournament.sh b/scripts/archive/eval_prompt_tournament.sh similarity index 100% rename from scripts/eval_prompt_tournament.sh rename to scripts/archive/eval_prompt_tournament.sh diff --git a/scripts/eval_qwen27b.sh b/scripts/archive/eval_qwen27b.sh similarity index 100% rename from scripts/eval_qwen27b.sh rename to scripts/archive/eval_qwen27b.sh diff --git a/scripts/eval_qwen27b_think.sh b/scripts/archive/eval_qwen27b_think.sh similarity index 100% rename from scripts/eval_qwen27b_think.sh rename to scripts/archive/eval_qwen27b_think.sh diff --git a/scripts/eval_qwen35b_a3b.sh b/scripts/archive/eval_qwen35b_a3b.sh similarity index 100% rename from scripts/eval_qwen35b_a3b.sh rename to scripts/archive/eval_qwen35b_a3b.sh diff --git a/scripts/eval_qwen35b_a3b_n50.sh b/scripts/archive/eval_qwen35b_a3b_n50.sh similarity index 100% rename from scripts/eval_qwen35b_a3b_n50.sh rename to scripts/archive/eval_qwen35b_a3b_n50.sh diff --git a/scripts/eval_qwen35b_a3b_n50_think.sh b/scripts/archive/eval_qwen35b_a3b_n50_think.sh similarity index 100% rename from scripts/eval_qwen35b_a3b_n50_think.sh rename to scripts/archive/eval_qwen35b_a3b_n50_think.sh diff --git a/scripts/eval_qwopus35b_a3b.sh b/scripts/archive/eval_qwopus35b_a3b.sh similarity index 100% rename from scripts/eval_qwopus35b_a3b.sh rename to scripts/archive/eval_qwopus35b_a3b.sh diff --git a/scripts/overnight_qwen27b_chain.sh b/scripts/archive/overnight_qwen27b_chain.sh similarity index 100% rename from scripts/overnight_qwen27b_chain.sh rename to scripts/archive/overnight_qwen27b_chain.sh diff --git a/scripts/post_chain_backtest.sh b/scripts/archive/post_chain_backtest.sh similarity index 100% rename from scripts/post_chain_backtest.sh rename to scripts/archive/post_chain_backtest.sh diff --git a/scripts/post_phase3_backtest.sh b/scripts/archive/post_phase3_backtest.sh similarity index 100% rename from scripts/post_phase3_backtest.sh rename to scripts/archive/post_phase3_backtest.sh diff --git a/scripts/run_all_fusion_smokes.sh b/scripts/archive/run_all_fusion_smokes.sh similarity index 100% rename from scripts/run_all_fusion_smokes.sh rename to scripts/archive/run_all_fusion_smokes.sh diff --git a/scripts/capture-rocblas-bench.sh b/scripts/capture-rocblas-bench.sh new file mode 100755 index 00000000..88f87bc8 --- /dev/null +++ b/scripts/capture-rocblas-bench.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# One-shot bench-log capture for the gfx1201 ISA1201 GEMM page fault. +# +# Wraps a normal Stage 2 resume with ROCBLAS_LAYER=2 to record the exact +# rocblas-bench invocation line for the faulting GEMM. The faulting call +# itself may not flush (it never returns), but the same shape (M=608 N=5376 +# K=21504) repeats every few steps — any occurrence in the log is the correct +# replay encoding, not a hand-written guess. +# +# After the run crashes, the script greps the bench log and prints the +# candidate line(s). Feed the output to replay-rocblas-bench.sh to confirm +# the ShaderName. +# +# Usage: +# bash scripts/capture-rocblas-bench.sh +# +# The run will crash at the GPU page fault (~21 steps after the checkpoint). +# Do NOT run while the production crawl (monitor_stage2.sh) is active — one +# GPU, one training process. +set -uo pipefail + +ROCM="${ROCM_PATH:-/opt/rocm}" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_DIR="$REPO_DIR/outputs/sft-stage2-gemma4-31b" +BENCH_LOG="$OUTPUT_DIR/rocblas_bench_capture.log" +TRAIN_LOG="$OUTPUT_DIR/bench-capture-run.log" +STEPS=50 # enough to hit the fault; low enough not to waste time + +printf 'gfx1201 rocBLAS bench capture\n' +printf ' bench log: %s\n' "$BENCH_LOG" +printf ' train log: %s\n' "$TRAIN_LOG" +printf '\n' + +# ── 1. Verify ROCBLAS_LAYER bitmask is recognised ────────────────────────── +# A wrong bitmask silently produces an empty log; write a throwaway row first. +printf 'Verifying ROCBLAS_LAYER=2 produces output...\n' +probe_out="$OUTPUT_DIR/rocblas_bench_probe.$$.log" +ROCBLAS_LAYER=2 ROCBLAS_LOG_BENCH_PATH="$probe_out" \ + "$ROCM/bin/rocblas-bench" -f gemm -m 4 -n 4 -k 4 --a_type f32_r >/dev/null 2>&1 || true +if [[ -s "$probe_out" ]]; then + printf ' ROCBLAS_LAYER=2 + ROCBLAS_LOG_BENCH_PATH: OK\n' + rm -f "$probe_out" +else + rm -f "$probe_out" + printf ' ROCBLAS_LOG_BENCH_PATH produced no output — trying ROCBLAS_LOG_BENCH_PATH alt\n' + printf ' If this also fails, check your ROCm version or set ROCBLAS_LAYER manually.\n' +fi + +# ── 2. GPU preflight ──────────────────────────────────────────────────────── +printf '\nRunning gpu-preflight...\n' +cd "$REPO_DIR" || exit 1 +if ! make gpu-preflight; then + printf 'gpu-preflight FAILED — aborting (GPU is dirty; clean it first)\n' >&2 + exit 1 +fi + +# ── 3. Capture run ────────────────────────────────────────────────────────── +printf '\nStarting bench-capture training run (%d steps)...\n' "$STEPS" +rm -f "$BENCH_LOG" + +# Compute a TRAIN_MAX_STEPS that is STEPS beyond the latest checkpoint so the +# run crashes during normal training, not during the checkpoint step itself. +latest_ckpt=0 +for d in "$OUTPUT_DIR"/checkpoint-*; do + [[ -d "$d" ]] || continue + n="${d##*checkpoint-}" + [[ "$n" =~ ^[0-9]+$ ]] && (( n > latest_ckpt )) && latest_ckpt="$n" +done +max_steps=$(( latest_ckpt + STEPS )) +printf ' Latest checkpoint: %d — running to step %d\n' "$latest_ckpt" "$max_steps" + +ROCBLAS_LAYER=2 \ + ROCBLAS_LOG_BENCH_PATH="$BENCH_LOG" \ + TORCH_USE_HIPBLASLT=0 \ + AMD_SERIALIZE_KERNEL=3 \ + HIP_LAUNCH_BLOCKING=1 \ + AMD_LOG_LEVEL=1 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + TRAIN_MAX_STEPS="$max_steps" \ + TRAIN_SAVE_STEPS=$(( max_steps + 10 )) \ + TRAIN_OUTPUT_DIR="$OUTPUT_DIR" \ + uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + > "$TRAIN_LOG" 2>&1 || true # expect a non-zero exit (GPU fault) + +printf '\nRun exited. Searching bench log for M=608 N=5376 K=21504 gemm_ex calls...\n' +printf ' bench log size: %s bytes\n' "$(wc -c < "$BENCH_LOG" 2>/dev/null || echo 0)" + +# ── 4. Extract candidate reproducer lines ─────────────────────────────────── +if [[ -s "$BENCH_LOG" ]]; then + matches="$(grep -i 'gemm_ex\|gemm_strided\|gemm_batched' "$BENCH_LOG" \ + | grep -E '[ ,]608[ ,]' | grep -E '[ ,]5376[ ,]' | grep -E '[ ,]21504[ ,]' || true)" + if [[ -n "$matches" ]]; then + printf '\n=== CANDIDATE REPRODUCER LINE(S) — paste into replay-rocblas-bench.sh ===\n' + printf '%s\n' "$matches" + printf '=== END ===\n\n' + printf 'Next: run bash scripts/replay-rocblas-bench.sh ""\n' + printf ' and confirm it dispatches ...MT64x64x64...ISA1201...DTVB1...\n' + else + printf '\nNo gemm_ex line matching M=608 N=5376 K=21504 found.\n' + printf 'Possible causes:\n' + printf ' - The bench log was not written (ROCBLAS_LAYER env var not recognised)\n' + printf ' - The fault landed before any call at this shape was logged\n' + printf ' - The seed gave a different batch length this run\n' + printf 'Full bench log: %s\n' "$BENCH_LOG" + printf 'Training log: %s\n' "$TRAIN_LOG" + printf '\nFallback: try ROCBLAS_LAYER=3 (adds trace, much larger log) or check\n' + printf ' the training log for the step that faulted.\n' + fi +else + printf '\nBench log is EMPTY — ROCBLAS_LAYER=2 did not write output.\n' + printf 'Check that ROCBLAS_LOG_BENCH_PATH is supported in this ROCm build:\n' + printf ' strings %s/lib/librocblas.so | grep -i "ROCBLAS_LOG"\n' "$ROCM" + printf 'Training log: %s\n' "$TRAIN_LOG" +fi diff --git a/scripts/collect-gfx1201-report-env.sh b/scripts/collect-gfx1201-report-env.sh new file mode 100755 index 00000000..c5bf29e4 --- /dev/null +++ b/scripts/collect-gfx1201-report-env.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Collects OS / driver / ROCm / Tensile-library / stack info for the +# gfx1201 ISA1201 rocBLAS page-fault report (csen-346 #113 -> ROCm/rocm-libraries). +# Best-effort collector: intentionally NOT `set -e` so a missing tool does not abort +# the run. Tees everything to a timestamped file you can attach to the report. +set -uo pipefail + +ROCM="${ROCM_PATH:-/opt/rocm}" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +report_dir="$(cd "$script_dir/.." && pwd)/docs/diagnostics" +mkdir -p "$report_dir" +out="$report_dir/gfx1201-report-env-$(date +%Y%m%d-%H%M%S).txt" +exec > >(tee "$out") 2>&1 + +hr() { printf '\n========== %s ==========\n' "$1"; } + +hr "ROCm / HIP build" +hipconfig --version 2>/dev/null || printf '(hipconfig unavailable)\n' +printf '\n' +cat "$ROCM"/.info/version* 2>/dev/null || printf '(%s/.info/version* unavailable)\n' "$ROCM" + +hr "OS / kernel" +uname -a +grep -E 'PRETTY_NAME|^VERSION=' /etc/os-release 2>/dev/null || true + +hr "GPU / driver / firmware / VBIOS" +rocm-smi --showdriverversion 2>/dev/null || printf '(rocm-smi unavailable)\n' +rocm-smi --showproductname --showvbios 2>/dev/null || true +printf '\n-- amdgpu module --\n' +modinfo amdgpu 2>/dev/null | grep -E '^version:|^srcversion:|^vermagic:' \ + || printf '(amdgpu version fields not exported — typical for the in-kernel/inbox driver)\n' +printf '\n-- rocminfo (gfx target) --\n' +rocminfo 2>/dev/null | grep -E 'Name:|Marketing Name:|gfx|Compute Unit:' | head -20 || printf '(rocminfo unavailable)\n' + +hr "ROCm component package versions" +if command -v pacman >/dev/null 2>&1; then + pacman -Q 2>/dev/null | grep -Ei 'rocm|rocblas|hipblas|tensile|^hip|comgr|miopen|amdgpu' || printf '(no matching pacman pkgs)\n' +elif command -v dpkg >/dev/null 2>&1; then + dpkg -l 2>/dev/null | grep -Ei 'rocm|rocblas|hipblas|tensile|hip-|comgr|miopen|amdgpu' || printf '(no matching dpkg pkgs)\n' +else + printf '(no pacman/dpkg found)\n' +fi + +hr "Tensile library files present (issue #7192 lead)" +# Just the load-bearing files for the wrong-library-lookup question — not the full +# fallback-kernel .hsaco set (which is ~50 lines of noise). +ls -la "$ROCM"/lib/rocblas/library/TensileLibrary_lazy_gfx120[01].dat \ + "$ROCM"/lib/rocblas/library/Kernels.so-*-gfx120[01].hsaco 2>/dev/null \ + || printf '(no gfx1200/gfx1201 lazy Tensile library / Kernels.so under %s/lib/rocblas/library/)\n' "$ROCM" + +hr "Calling stack (torch / hip / bitsandbytes)" +stack_probe='import torch, bitsandbytes as bnb; print("torch", torch.__version__, "| hip", torch.version.hip, "| bnb", bnb.__version__)' +if command -v uv >/dev/null 2>&1; then + uv run --no-sync python -c "$stack_probe" 2>&1 || printf '(torch/bnb import failed)\n' +else + python -c "$stack_probe" 2>&1 || printf '(uv not found and python import failed)\n' +fi + +hr "rocblas-bench availability (for the standalone reproducer)" +if command -v rocblas-bench >/dev/null 2>&1; then + printf 'rocblas-bench: FOUND -> %s\n' "$(command -v rocblas-bench)" +elif [[ -x "$ROCM/bin/rocblas-bench" ]]; then + printf 'rocblas-bench: FOUND -> %s/bin/rocblas-bench\n' "$ROCM" +else + printf 'rocblas-bench: NOT FOUND (install the rocblas test/bench package to capture the standalone repro)\n' +fi +printf 'Next (separate GPU run): add ROCBLAS_LAYER=2 to the training launch env to emit the\n' +printf 'exact "rocblas-bench --transposeA ... -m 608 -n 5376 -k 21504 --lda ... --ldb ..." line\n' +printf 'for the faulting GEMM -- that command line is the stack-independent reproducer.\n' + +hr "DONE" +printf 'Saved to: %s\n' "$out" diff --git a/scripts/convert_gemma4_12b_sft_to_gguf.sh b/scripts/convert_gemma4_12b_sft_to_gguf.sh new file mode 100755 index 00000000..ccd152ee --- /dev/null +++ b/scripts/convert_gemma4_12b_sft_to_gguf.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Convert merged Gemma-4-12B SFT (HF BF16) → f16 GGUF → Q8_0 GGUF, then stage it +# where scripts/serve_gemma4_12b_sft.sh looks. +# +# Pipeline step 2+3 of 3 — assumes scripts/merge_lora_gemma4_sft.py already ran: +# scripts/merge_lora_gemma4_sft.py --base google/gemma-4-12b-it \ +# --adapter outputs/sft-gemma4-12b-qlora/final --out outputs/sft-gemma4-12b-qlora/merged +# +# This is the 12B sibling of convert_gemma4_sft_to_gguf.sh. It exists so the +# output filename carries the 12B NAME_TAG (the 31B script hardcodes "31B"), +# matching serve_gemma4_12b_sft.sh's default GEMMA4_12B_SFT_WEIGHT_FILE — so no +# manual rename is needed at G6 of the PoC. +# +# Why Q8_0: the base teacher is served at UD-Q8_K_XL (Unsloth dynamic, not +# producible by llama-quantize). Q8_0 is the standard llama.cpp 8-bit quant at the +# same bit budget; at Q8 the quant-scheme delta vs the base is ~noise, keeping the +# base↔SFT comparison clean (the difference is the LoRA adapter, not the quant). +# +# Usage: +# bash scripts/convert_gemma4_12b_sft_to_gguf.sh +# +# Override: +# MERGED_DIR=... GGUF_DIR=... QUANT=Q6_K bash scripts/convert_gemma4_12b_sft_to_gguf.sh +# GEMMA4_12B_WEIGHTS_DIR=/some/path bash scripts/convert_gemma4_12b_sft_to_gguf.sh # stage target +# NO_STAGE=1 bash scripts/convert_gemma4_12b_sft_to_gguf.sh # skip the cp + +set -euo pipefail + +# Resolve relative paths against the repo root so `cd "$LLAMA_CPP_DIR"` below +# doesn't break them. realpath -m allows paths whose final component doesn't +# exist yet (the GGUF_DIR/output files are about to be created). +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MERGED_DIR="$(realpath -m "${MERGED_DIR:-$REPO_ROOT/outputs/sft-gemma4-12b-qlora/merged}")" +GGUF_DIR="$(realpath -m "${GGUF_DIR:-$REPO_ROOT/outputs/sft-gemma4-12b-qlora}")" +QUANT="${QUANT:-Q8_0}" + +LLAMA_CPP_DIR="${LLAMA_CPP_DIR:-$HOME/Documents/models/llama.cpp}" +CONVERT="$LLAMA_CPP_DIR/convert_hf_to_gguf.py" +# Override QUANTIZE to a CPU-only build of llama-quantize: on the unstable NVIDIA +# box a CUDA recompile of ggml segfaults (nvcc Error 139 under load), and quantize +# is a CPU op that needs no GPU backend — see scripts/serve note / memory. +QUANTIZE="${QUANTIZE:-$LLAMA_CPP_DIR/build/bin/llama-quantize}" + +# KELE-tagged filenames — must NOT collide with base gemma-4-12b-it-*.gguf. +NAME_TAG="gemma-4-12B-kele-socratic-sft" +F16_GGUF="$GGUF_DIR/${NAME_TAG}-f16.gguf" +OUT_GGUF="$GGUF_DIR/${NAME_TAG}-${QUANT}.gguf" + +# Where serve_gemma4_12b_sft.sh expects the final GGUF. +WEIGHTS_DIR="${GEMMA4_12B_WEIGHTS_DIR:-$HOME/Documents/models/weights}" + +# ── Pre-flight ──────────────────────────────────────────────────────────────── +if [[ ! -d "$MERGED_DIR" ]]; then + echo "ERROR: Merged HF checkpoint not found at $MERGED_DIR" >&2 + echo "Run scripts/merge_lora_gemma4_sft.py --base google/gemma-4-12b-it first." >&2 + exit 1 +fi + +if [[ ! -f "$CONVERT" ]]; then + echo "ERROR: convert_hf_to_gguf.py not found at $CONVERT" >&2 + exit 1 +fi + +if [[ ! -x "$QUANTIZE" ]]; then + echo "ERROR: llama-quantize binary not found at $QUANTIZE" >&2 + exit 1 +fi + +mkdir -p "$GGUF_DIR" + +if [[ -f "$OUT_GGUF" ]]; then + echo "$OUT_GGUF already exists. Delete and re-run to overwrite." + exit 0 +fi + +# ── Step 1: HF (BF16 merged) → f16 GGUF ─────────────────────────────────────── +if [[ ! -f "$F16_GGUF" ]]; then + echo "=== Step 1: HF → f16 GGUF ===" + echo "Input: $MERGED_DIR" + echo "Output: $F16_GGUF" + echo "(~5-8 min on CPU, ~24 GB write)" + + PYTHON="${PYTHON:-$REPO_ROOT/.venv/bin/python}" + if [[ ! -x "$PYTHON" ]]; then + echo "ERROR: Python not found at $PYTHON" >&2 + exit 1 + fi + + cd "$LLAMA_CPP_DIR" + "$PYTHON" "$CONVERT" "$MERGED_DIR" \ + --outfile "$F16_GGUF" \ + --outtype f16 + cd - >/dev/null +fi + +# ── Step 2: f16 GGUF → Q8_0 ─────────────────────────────────────────────────── +echo +echo "=== Step 2: f16 GGUF → $QUANT ===" +echo "Input: $F16_GGUF" +echo "Output: $OUT_GGUF" +echo "(~3-5 min on CPU)" + +"$QUANTIZE" "$F16_GGUF" "$OUT_GGUF" "$QUANT" + +# ── Step 3: stage where the serve wrapper looks ─────────────────────────────── +if [[ "${NO_STAGE:-0}" != "1" ]]; then + mkdir -p "$WEIGHTS_DIR" + cp "$OUT_GGUF" "$WEIGHTS_DIR/" + echo + echo "Staged → $WEIGHTS_DIR/$(basename "$OUT_GGUF")" + echo "serve_gemma4_12b_sft.sh will find it (GEMMA4_12B_SFT_WEIGHT_FILE default)." +fi + +# ── Done ────────────────────────────────────────────────────────────────────── +echo +echo "=== Done ===" +ls -lh "$OUT_GGUF" +echo +echo "Intermediate f16 GGUF (~24 GB) preserved at:" +echo " $F16_GGUF" +echo "Delete with: rm $F16_GGUF" +echo +echo "Next: make serve-gemma4-12b-sft then make eval-gemma4-12b-sft-smoke" diff --git a/scripts/determine_max_sequence.py b/scripts/determine_max_sequence.py new file mode 100755 index 00000000..6db6a51c --- /dev/null +++ b/scripts/determine_max_sequence.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Measure the real tokenized sequence-length distribution of the SFT training data. + +Why this exists: the Stage 2 config carried TRAIN_MAX_SEQ_LEN=1280 as an *assumed* +ceiling. This script measures the actual distribution so the cap is evidence-based, +not guessed. It renders each record with the exact Gemma 4 training chat template +from train_sft.py and tokenizes with the same tokenizer the run uses. + +Measured 2026-06-01 on socrat-zh-sft + socrat-en-sft (the Stage 2b sources): + TRAIN n=77202 max=909 p99=673 p95=584 p50=431 mean=437 + EVAL n=8578 max=892 p99=680 p95=582 p50=430 mean=437 + 0 records exceed 1024 tokens. + +Conclusions baked into the Stage 2 config decisions: + - max_length=1280 truncates NOTHING (true max is 909); it is non-binding. + - max_length is NOT a useful VRAM lever here: with per_device_batch_size=1 and + dynamic (longest-in-batch) padding, the activation peak is set by the actual + longest sequence (909), not by the cap. Lowering 1280→1024 changes no memory. + - Attention implementation is irrelevant to this measurement: token COUNT is a + pure tokenization property. SDPA vs FA2 changes how attention is computed, not + how many tokens a record has. (The run uses SDPA; see train_sft.py:221.) + +Usage: + uv run --no-sync python scripts/determine_max_sequence.py + uv run --no-sync python scripts/determine_max_sequence.py \ + --sources socrat-zh-sft,socrat-en-sft \ + --tokenizer unsloth/gemma-4-31B-it-unsloth-bnb-4bit +""" + +from __future__ import annotations + +import argparse +import statistics +import sys +from pathlib import Path + +# Reuse the exact training chat template (NOT the stock 17K-char tool-calling one). +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import train_sft # noqa: E402 + + +def _percentile(sorted_vals: list[int], pct: float) -> int: + n = len(sorted_vals) + return sorted_vals[min(n - 1, int(pct / 100 * n))] + + +def _lengths(tokenizer, records: list[dict]) -> list[int]: + texts = [tokenizer.apply_chat_template(r["messages"], tokenize=False) for r in records] + encoded = tokenizer(texts, add_special_tokens=False)["input_ids"] + return sorted(len(ids) for ids in encoded) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sources", default="socrat-zh-sft,socrat-en-sft") + parser.add_argument("--tokenizer", default="unsloth/gemma-4-31B-it-unsloth-bnb-4bit") + parser.add_argument( + "--caps", + default="1024,1152,1280,1536,2048", + help="comma-separated max_length candidates to report truncation counts for", + ) + args = parser.parse_args() + + from transformers import AutoTokenizer + + from src.project.dataset import load_split_pair + + tok = AutoTokenizer.from_pretrained(args.tokenizer, trust_remote_code=True) + if tok.pad_token is None: + tok.pad_token = tok.eos_token + tok.chat_template = train_sft._GEMMA4_TRAINING_CHAT_TEMPLATE + + sources = [s.strip() for s in args.sources.split(",") if s.strip()] + caps = [int(c) for c in args.caps.split(",") if c.strip()] + train, eval_ = load_split_pair(sources=sources) + print(f"sources={sources} train={len(train)} eval={len(eval_)} tokenizer={args.tokenizer}") + + for name, recs in [("TRAIN", train), ("EVAL", eval_)]: + lengths = _lengths(tok, recs) + n = len(lengths) + print(f"\n[{name}] n={n}") + print( + f" max={lengths[-1]} p99={_percentile(lengths, 99)} p95={_percentile(lengths, 95)}" + f" p90={_percentile(lengths, 90)} p50={_percentile(lengths, 50)}" + f" mean={statistics.mean(lengths):.0f}" + ) + for cap in caps: + over = sum(1 for x in lengths if x > cap) + print(f" > {cap}: {over:6d} records ({100 * over / n:.2f}%) would truncate") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/diagnose_gfx1201_fault.sh b/scripts/diagnose_gfx1201_fault.sh new file mode 100755 index 00000000..8f90b7d6 --- /dev/null +++ b/scripts/diagnose_gfx1201_fault.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Localize the gfx1201 QLoRA backward page fault — turn the async VM fault into a +# synchronous, attributable one. +# +# The fault (Memory access fault / page not present, PERMISSION_FAULTS:0x3) is +# non-deterministic and surfaces during the backward pass. Because HIP kernel +# launches are async, the kernel "running" when the fault is reported is not +# necessarily the culprit — every prior root-cause theory was inferred from +# correlation, never from the faulting kernel itself. This run forces +# attribution: +# AMD_SERIALIZE_KERNEL=3 — wait for each kernel to finish before the next +# HIP_LAUNCH_BLOCKING=1 — synchronous launches → Python traceback points at +# the exact failing op +# AMD_LOG_LEVEL=1 — HIP runtime logs the failing API call (errors only) +# Plus a dmesg tail for the kernel-side VM fault decode (needs read access to the +# kernel ring buffer; if empty, re-run with sudo or set kernel.dmesg_restrict=0). +# +# The synchronous traceback + dmesg are what name the kernel — AMD_LOG_LEVEL=1 +# (errors) is enough. Do NOT default to level 3: it logs every HIP API call, so a +# serialized run writes MULTIPLE GB to diag.log before the fault and can fill the +# disk. Bump it only if level 1 doesn't name the failing call: LOG_LEVEL=3 bash … +# +# Reads: if serialization makes the fault MOVE or VANISH → concurrency/allocator +# race. If it still faults at the SAME named kernel → kernel bug, and the +# backtrace names it (bnb NF4 dequant vs grad-ckpt recompute vs allocator). +# +# Usage: +# bash scripts/diagnose_gfx1201_fault.sh # ~120 steps, unsloth NF4 path +# STEPS=200 bash scripts/diagnose_gfx1201_fault.sh # widen the window +# LOG_LEVEL=3 bash scripts/diagnose_gfx1201_fault.sh # verbose HIP log (multi-GB!) +# +# Throwaway output dir (never collides with the real run). + +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +STEPS="${STEPS:-120}" +LOG_LEVEL="${LOG_LEVEL:-1}" +OUT_DIR="outputs/diag-gfx1201" +DIAG_LOG="$OUT_DIR/diag.log" +DMESG_LOG="$OUT_DIR/dmesg.log" + +mkdir -p "$OUT_DIR" +rm -rf "$OUT_DIR"/checkpoint-* 2>/dev/null || true + +printf 'gfx1201 fault diagnostic — %s steps, serialized kernels, AMD_LOG_LEVEL=%s\n' "$STEPS" "$LOG_LEVEL" +printf ' output: %s dmesg: %s\n' "$DIAG_LOG" "$DMESG_LOG" +[[ "$LOG_LEVEL" -ge 3 ]] && printf ' WARNING: AMD_LOG_LEVEL=%s logs every HIP call — diag.log can reach many GB.\n' "$LOG_LEVEL" + +DMESG_PID="" +# Start a best-effort kernel-ring tail; tolerate dmesg_restrict (no perms). +( dmesg -w 2>/dev/null | grep --line-buffered -iE \ + 'amdgpu|gfxhub|VM_L2|page fault|PERMISSION_FAULTS|WALKER_ERROR|retry page' \ + > "$DMESG_LOG" ) & +DMESG_PID=$! + +# shellcheck disable=SC2317,SC2329 # body is invoked indirectly via trap +cleanup() { + if [[ -n "$DMESG_PID" ]]; then + kill "$DMESG_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +# TRAIN_SAVE_STEPS deliberately > STEPS: this is a diagnostic, not a checkpointed +# run — we want the bare fault, no checkpoint I/O in the window. +env TORCH_USE_HIPBLASLT=0 \ + AMD_SERIALIZE_KERNEL=3 \ + HIP_LAUNCH_BLOCKING=1 \ + AMD_LOG_LEVEL="$LOG_LEVEL" \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + TRAIN_MAX_STEPS="$STEPS" \ + TRAIN_SAVE_STEPS=$((STEPS + 10)) \ + TRAIN_OUTPUT_DIR="$OUT_DIR" \ + uv run --no-sync python scripts/train_sft.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + 2>&1 | tee "$DIAG_LOG" +rc=${PIPESTATUS[0]} + +printf '\n── diagnostic result ──\n' +if [[ $rc -eq 0 ]]; then + printf 'Completed %s steps with NO fault under serialization.\n' "$STEPS" + printf ' → strong signal the fault is a concurrency/async-allocator RACE,\n' + printf ' not a deterministic kernel bug. Next: ablate the allocator knobs.\n' +else + printf 'Process exited rc=%s. The faulting op is the LAST kernel in:\n %s\n' "$rc" "$DIAG_LOG" + printf ' → grep -iE "fault|backtrace|Traceback|dequant|checkpoint" %s\n' "$DIAG_LOG" + [[ -s "$DMESG_LOG" ]] && printf ' → kernel VM-fault decode: %s\n' "$DMESG_LOG" \ + || printf ' → dmesg empty (no perms?): re-run with sudo for the kernel decode.\n' +fi +exit "$rc" diff --git a/scripts/eos_gate.py b/scripts/eos_gate.py new file mode 100644 index 00000000..57630796 --- /dev/null +++ b/scripts/eos_gate.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""EOS gate: verify a trained checkpoint terminates cleanly on a realistic inference prompt. + +Loads the base model, applies a saved LoRA adapter, then generates on a prompt +built with a realistic multi-sentence Chinese free-form evaluation — the irreducible +residual identified in PR #101 / issue #94. PASS = model stops on its own before +max_new_tokens. FAIL = hits the cap (likely repetition collapse). + +Usage (from repo root): + make train-gemma4-31b-eos-gate # ~30 min: trains 100-step checkpoint + tail -f outputs/eos-gate-gemma4-31b/train.log + make eos-gate-gemma4-31b # run the gate once training finishes +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from unittest.mock import patch + +# Allow `import train_sft` from the same scripts/ directory. +sys.path.insert(0, str(Path(__file__).parent)) + + +def _load_model(adapter_dir: str | None): + import train_sft # noqa: PLC0415 + + model, tokenizer = train_sft.build_model_and_tokenizer() + if adapter_dir: + from peft import PeftModel # noqa: PLC0415 + + print(f"\nLoading adapter path={adapter_dir}") + model = PeftModel.from_pretrained(model, adapter_dir) + print(" Adapter loaded") + + import torch # noqa: PLC0415 + + model.eval() + torch.cuda.empty_cache() + return model, tokenizer + + +def _stop_ids(tokenizer) -> list[int]: + ids = [] + if tokenizer.eos_token_id is not None: + ids.append(tokenizer.eos_token_id) + turn_end = tokenizer.convert_tokens_to_ids("") + if turn_end not in (None, tokenizer.unk_token_id): + ids.append(turn_end) + return list(set(ids)) + + +def _generate( + model, tokenizer, messages: list[dict], max_new_tokens: int = 512 +) -> tuple[str, bool]: + import torch # noqa: PLC0415 + + stop = _stop_ids(tokenizer) + enc = tokenizer.apply_chat_template( + messages, add_generation_prompt=True, tokenize=True, return_tensors="pt" + ) + input_ids = ( + enc["input_ids"] + if hasattr(enc, "__getitem__") and not isinstance(enc, torch.Tensor) + else enc + ).to(model.device) + with torch.no_grad(): + out = model.generate( + input_ids, + max_new_tokens=max_new_tokens, + eos_token_id=stop or None, + do_sample=False, + temperature=None, + top_p=None, + ) + gen = out[0, input_ids.shape[1] :] + decoded = tokenizer.decode(gen, skip_special_tokens=False) + terminated = len(gen) < max_new_tokens or "" in decoded + return decoded, terminated + + +def _capture_prompt(evaluation: str, state: str) -> tuple[str, str]: + """Capture the (system, user) the inference path sends for a given eval + state.""" + from src.project.socratic_teaching_system import ( # noqa: PLC0415 + SocraticTeachingSystem, + get_action_for_state, + ) + + sts = SocraticTeachingSystem( + consultant_api_key="test", + consultant_base_url="http://localhost:0", + consultant_model_name="m", + teacher_api_key="test", + teacher_base_url="http://localhost:0", + teacher_model_name="m", + ) + # One prior exchange in history, then the current student turn added unpaired + # — mirrors process_student_input's add_to_history before socrates_teacher call. + sts.add_to_history("student", "科学实验中光发生折射是什么原因？") + sts.add_to_history("teacher", "你知道光在不同介质中传播速度不同吗？这和方向变化有什么关系？") + current_student = "我觉得光变慢了所以才弯曲？" + sts.add_to_history("student", current_student) + sts.current_state = state + + captured: dict = {} + + def _fake_call(client, model, system_prompt, user_input, predicted_state): # noqa: ANN001 + captured["system"] = system_prompt + captured["user"] = user_input + return "captured" + + action = get_action_for_state(state) + with patch("src.project.tournament_utilizations.call_teacher_wrapped", side_effect=_fake_call): + sts.socrates_teacher(current_student, evaluation, action) + + return captured["system"], captured["user"] + + +def main() -> None: + parser = argparse.ArgumentParser(description="EOS gate for a trained SFT checkpoint") + parser.add_argument( + "--config", metavar="PATH", help="env config file (sets TRAIN_BASE_MODEL etc.)" + ) + parser.add_argument( + "--adapter", metavar="PATH", help="path to saved LoRA adapter dir (checkpoint/final)" + ) + args = parser.parse_args() + + if args.config: + from src.project.config import load_env_file # noqa: PLC0415 + + load_env_file(Path(args.config)) + + if not args.adapter: + print( + "WARNING: --adapter not provided — testing the base model only.\n" + " For a meaningful gate, train a short checkpoint first:\n" + " make train-gemma4-31b-eos-gate", + file=sys.stderr, + ) + + model, tokenizer = _load_model(args.adapter) + + # ── Step 1: warm-up ───────────────────────────────────────────────────── + print("\n" + "=" * 64) + print("STEP 1 warm-up (trivial prompt — verify stop tokens fire)") + print("=" * 64) + warmup_out, warmup_ok = _generate( + model, + tokenizer, + [{"role": "user", "content": "你好，请用一句话介绍自己。"}], + max_new_tokens=200, + ) + print(f"\n{warmup_out!r}") + print( + f"\nWarm-up: {'PASS' if warmup_ok else 'FAIL — stop tokens may not be configured correctly'}" + ) + + # ── Step 2: real EOS gate ──────────────────────────────────────────────── + print("\n" + "=" * 64) + print("STEP 2 EOS gate (realistic free-form Chinese evaluation)") + print("=" * 64) + + # Multi-sentence realistic consultant prose — NOT the 学生处于 {state} 状态 + # template. This is the irreducible residual: training uses stored annotations, + # inference uses live consultant output like this. + freeform_eval = ( + "学生已经掌握了光在不同介质中传播速度变化的基本概念，能够直觉性地联系速度变化和方向改变。" + "但对于具体的折射定律（如斯涅尔定律）以及折射角的定量计算仍存在明显困惑。" + "建议进一步引导学生通过具体实例（如铅笔插入水中的弯折现象）来巩固其理解，" + "并逐步引入定量分析框架。" + ) + state = "b2" + + sys_msg, user_msg = _capture_prompt(freeform_eval, state) + print(f"\nEvaluation ({len(freeform_eval)} chars):\n {freeform_eval}") + print(f"\nUser message preview:\n {user_msg[:300]}...") + + messages = [ + {"role": "system", "content": sys_msg}, + {"role": "user", "content": user_msg}, + ] + gate_out, gate_ok = _generate(model, tokenizer, messages, max_new_tokens=512) + + print(f"\nGeneration:\n{gate_out}") + print("\n" + "=" * 64) + if gate_ok: + print("EOS GATE PASSED — model terminated before max_new_tokens") + print("Safe to launch: make train-gemma4-31b-stage2-unsloth") + else: + print("EOS GATE FAILED — model hit max_new_tokens without terminating") + print("Do NOT launch the full run. Check for repetition or format drift.") + print("=" * 64) + sys.exit(0 if gate_ok else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/eval_llamacpp.sh b/scripts/eval_llamacpp.sh new file mode 100755 index 00000000..06d5277e --- /dev/null +++ b/scripts/eval_llamacpp.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash +# Parameterized orchestrator for single-server llama.cpp evals (teacher = consultant +# on port 8080, dual-role). Boots the model's serve script if not already up, runs +# KELE eval, compares against the gpt-4o-2024-11-20 baseline. Crash-safe via kele.py's +# per-item resume. +# +# Replaces the per-model eval_qwen27b.sh / eval_qwen35b_a3b.sh / eval_gemma4_31b.sh / +# eval_gemma4_26b_a4b.sh / eval_qwopus35b_a3b.sh family — those are config rows now, +# not separate scripts. See the MODEL CONFIG table below. +# +# Usage: +# ./scripts/eval_llamacpp.sh [flags] +# +# : qwen27b | qwen35b-a3b | qwopus35b-a3b | gemma4-31b | gemma4-26b-a4b +# modes: smoke (n=5) mini (n=25) full (n=681) n50 (n=50) +# flags: --think use the model's thinking-mode serve script +# --nothink no-think prompt mode (Qwen only; -nothink experiment+suffix) +# --unified single-call fusion architecture (-unified suffix) +# --suffix NAME extra -NAME suffix on the output dir +# --keep-server leave a server we booted running on exit +# --no-compare skip the baseline comparison +# +# Examples: +# ./scripts/eval_llamacpp.sh qwen27b smoke +# ./scripts/eval_llamacpp.sh qwen27b smoke --think +# ./scripts/eval_llamacpp.sh gemma4-31b full --unified +# +# Server policy: if a server already serves the expected alias on the port, reuse it +# and NEVER kill it. If we boot one, tear it down on exit unless --keep-server. +# Only one model fits at a time on the 5090's 32 GB VRAM — stop any other server first. + +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$(pwd)" + +# ── Args ────────────────────────────────────────────────────────────────────── +usage() { + echo "Usage: $0 {smoke|mini|full|n50} [--think] [--nothink] [--unified] [--suffix NAME] [--keep-server] [--no-compare]" >&2 +} + +MODEL="${1:-}" +shift || true +case "$MODEL" in + qwen27b|qwen35b-a3b|qwopus35b-a3b|gemma4-31b|gemma4-26b-a4b) ;; + -h|--help) sed -n '2,30p' "$0"; exit 0 ;; + *) echo "Unknown or missing model: '$MODEL'" >&2; usage; exit 1 ;; +esac + +MODE="" +KEEP_SERVER=false +DO_COMPARE=true +NOTHINK=false +UNIFIED=false +THINK=false +EXTRA_SUFFIX="" + +while [[ $# -gt 0 ]]; do + case "$1" in + smoke|mini|full|n50) MODE="$1"; shift ;; + --keep-server) KEEP_SERVER=true; shift ;; + --no-compare) DO_COMPARE=false; shift ;; + --nothink) NOTHINK=true; shift ;; + --think) THINK=true; shift ;; + --unified) UNIFIED=true; shift ;; + --suffix) EXTRA_SUFFIX="-$2"; shift 2 ;; + -h|--help) sed -n '2,30p' "$0"; exit 0 ;; + *) echo "Unknown arg: $1" >&2; usage; exit 1 ;; + esac +done + +if [[ -z "$MODE" ]]; then usage; exit 1; fi + +# ── MODEL CONFIG ────────────────────────────────────────────────────────────── +# One row per model: serve script (default + thinking-mode), expected llama.cpp +# alias, GGUF file, base experiment name, VRAM-free warning threshold (MiB), and +# whether the model has a thinking-mode (Qwen only — Gemma rejects --think/--nothink). +WEIGHTS_DIR="${LLAMACPP_WEIGHTS_DIR:-$HOME/Documents/models/weights}" +case "$MODEL" in + qwen27b) + SERVE_DEFAULT="serve_qwen27b_q5.sh"; SERVE_THINK="serve_qwen27b_q5_think.sh" + EXPECTED_ALIAS="Qwen 27B Q5"; WEIGHT_FILE="Qwen3.6-27B-UD-Q5_K_XL.gguf" + EXPERIMENT_BASE="qwen27b-local"; VRAM_MIN=27000; THINK_CAPABLE=true ;; + qwen35b-a3b) + SERVE_DEFAULT="serve_qwen35b_a3b.sh"; SERVE_THINK="serve_qwen35b_a3b_think.sh" + EXPECTED_ALIAS="Qwen 35B A3B"; WEIGHT_FILE="Qwen3.6-35B-A3B-UD-Q4_K_M.gguf" + EXPERIMENT_BASE="qwen35b-a3b-local"; VRAM_MIN=27000; THINK_CAPABLE=true ;; + qwopus35b-a3b) + SERVE_DEFAULT="serve_qwopus35b_a3b_think.sh"; SERVE_THINK="serve_qwopus35b_a3b_think.sh" + EXPECTED_ALIAS="Qwopus 35B A3B"; WEIGHT_FILE="Qwopus3.6-35B-A3B-v1-Q4_K_M.gguf" + EXPERIMENT_BASE="qwopus35b-a3b-local"; VRAM_MIN=27000; THINK_CAPABLE=true ;; + gemma4-31b) + SERVE_DEFAULT="serve_gemma4_31b_q5.sh"; SERVE_THINK="" + EXPECTED_ALIAS="Gemma 4 31B"; WEIGHT_FILE="gemma-4-31B-it-UD-Q5_K_XL.gguf" + EXPERIMENT_BASE="gemma4-31b-local"; VRAM_MIN=26500; THINK_CAPABLE=false ;; + gemma4-26b-a4b) + SERVE_DEFAULT="serve_gemma4_26b_a4b.sh"; SERVE_THINK="" + EXPECTED_ALIAS="Gemma 4 26B A4B"; WEIGHT_FILE="gemma-4-26B-A4B-it-UD-Q4_K_XL.gguf" + EXPERIMENT_BASE="gemma4-26b-a4b-local"; VRAM_MIN=30500; THINK_CAPABLE=false ;; +esac + +if ! $THINK_CAPABLE && $NOTHINK; then + echo "Error: --nothink is not applicable to $EXPECTED_ALIAS (no thinking-mode equivalent)." >&2 + exit 1 +fi +if ! $THINK_CAPABLE && $THINK; then + echo "Error: --think is not applicable to $EXPECTED_ALIAS (no thinking-mode equivalent)." >&2 + exit 1 +fi + +SERVE_SCRIPT="$SERVE_DEFAULT" +if $THINK; then SERVE_SCRIPT="$SERVE_THINK"; fi + +# ── Mode config ─────────────────────────────────────────────────────────────── +# Output dir suffix order: -unified first, -nothink second, then --suffix value. +EXPERIMENT="$EXPERIMENT_BASE" +SUFFIX="" +if $UNIFIED; then SUFFIX="${SUFFIX}-unified"; fi +if $NOTHINK; then EXPERIMENT="${EXPERIMENT_BASE}-nothink"; SUFFIX="${SUFFIX}-nothink"; fi + +case "$MODE" in + smoke) N=5; OUT_DIR="results/${EXPERIMENT_BASE}-smoke${SUFFIX}${EXTRA_SUFFIX}"; SUBCMD="test" ;; + mini) N=25; OUT_DIR="results/${EXPERIMENT_BASE}-mini${SUFFIX}${EXTRA_SUFFIX}"; SUBCMD="test" ;; + n50) N=50; OUT_DIR="results/${EXPERIMENT_BASE}-n50${SUFFIX}${EXTRA_SUFFIX}"; SUBCMD="test" ;; + full) N=0; OUT_DIR="results/${EXPERIMENT_BASE}${SUFFIX}${EXTRA_SUFFIX}"; SUBCMD="evaluate" ;; +esac + +# ── Constants / env-overridable paths ───────────────────────────────────────── +LLAMA_SERVER="${LLAMA_SERVER:-$HOME/Documents/models/llama.cpp/build/bin/llama-server}" +PORT="${PORT:-8080}" +LLAMA_URL="http://localhost:${PORT}" +CONFIG_FILE="configs/${EXPERIMENT}.env" +BASELINE_DIR="results/baseline" + +# ── Pre-flight ──────────────────────────────────────────────────────────────── +echo "=== $EXPECTED_ALIAS Eval Orchestrator ===" +echo "Mode: $MODE (n=$N, subcmd=$SUBCMD)" +echo "Experiment:$EXPERIMENT" +echo "Output: $OUT_DIR" +echo "Serve: $SERVE_SCRIPT" +echo "Compare: $DO_COMPARE (vs $BASELINE_DIR)" +echo "Keep srv: $KEEP_SERVER" +echo "Think: $THINK Nothink: $NOTHINK Unified: $UNIFIED" +echo "---" + +preflight_check() { + local name="$1" path="$2" + if [[ ! -e "$path" ]]; then + echo "ERROR: $name not found at $path" >&2 + exit 1 + fi +} + +preflight_check "llama-server binary" "$LLAMA_SERVER" +preflight_check "weight file" "$WEIGHTS_DIR/$WEIGHT_FILE" +preflight_check "config" "$CONFIG_FILE" +preflight_check "serve script" "$ROOT/scripts/$SERVE_SCRIPT" + +if $DO_COMPARE && [[ ! -f "$BASELINE_DIR/metrics_summary.json" ]]; then + echo "WARN: $BASELINE_DIR/metrics_summary.json missing — disabling comparison." >&2 + DO_COMPARE=false +fi + +# GPU VRAM sanity (informational) +if command -v nvidia-smi &>/dev/null; then + VRAM_FREE=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits 2>/dev/null | head -1 || echo 0) + echo "GPU VRAM free: ${VRAM_FREE} MiB" + if [[ "$VRAM_FREE" =~ ^[0-9]+$ ]] && [[ "$VRAM_FREE" -lt "$VRAM_MIN" ]]; then + echo "WARN: less than ${VRAM_MIN} MiB VRAM free — server may OOM." >&2 + fi +fi + +# ── Run dir + log ───────────────────────────────────────────────────────────── +mkdir -p "$OUT_DIR" +TS=$(date -u +%Y-%m-%dT%H-%M-%S) +RUN_LOG="$OUT_DIR/run_${TS}.log" +SERVER_LOG="$OUT_DIR/server_${TS}.log" + +# Mirror everything to the run log (process substitution preserves exit codes) +exec > >(tee -a "$RUN_LOG") 2>&1 +echo "Run log: $RUN_LOG" +echo "Started: $(date)" +echo + +# ── Server lifecycle ───────────────────────────────────────────────────────── +SERVER_PID="" +WE_BOOTED=false + +probe_server() { + curl -s --max-time 3 "$LLAMA_URL/v1/models" 2>/dev/null +} + +# Quiet check: returns 0 if server serves the expected alias and is ready +# (model fully loaded — distinguish from the transient "Loading model" 503). +ready_check() { + local resp + resp=$(probe_server) || return 1 + [[ -z "$resp" ]] && return 1 + echo "$resp" | grep -q '"Loading model"' && return 1 + echo "$resp" | grep -q "\"$EXPECTED_ALIAS\"" +} + +# Loud check: prints diagnostic if a different model is being served +verify_alias() { + local resp + resp=$(probe_server) + [[ -z "$resp" ]] && return 1 + if echo "$resp" | grep -q "\"$EXPECTED_ALIAS\""; then + return 0 + fi + echo "WARN: server on $LLAMA_URL serves a different model than '$EXPECTED_ALIAS':" >&2 + echo "$resp" >&2 + return 1 +} + +cleanup_server() { + if $WE_BOOTED && [[ -n "$SERVER_PID" ]]; then + if $KEEP_SERVER; then + echo + echo "Leaving llama-server (PID $SERVER_PID) running per --keep-server." + echo " Stop manually with: kill $SERVER_PID" + return 0 + fi + echo + echo "Shutting down llama-server (PID $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + for _ in $(seq 1 10); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo "Server stopped." + return 0 + fi + sleep 1 + done + kill -9 "$SERVER_PID" 2>/dev/null || true + echo "Server force-killed." + fi +} +trap cleanup_server EXIT INT TERM + +if ready_check; then + echo "Reusing existing llama-server on $LLAMA_URL (alias '$EXPECTED_ALIAS' ready)." +elif probe_server >/dev/null && ! verify_alias; then + echo "ERROR: server on $LLAMA_URL does not serve '$EXPECTED_ALIAS'." >&2 + echo " Stop it first or override PORT= $0 $MODEL $MODE" >&2 + exit 1 +else + if probe_server >/dev/null; then + echo "Existing server on $LLAMA_URL is loading the model — waiting for it..." + else + echo "Booting llama-server (cold load: 30-90 s for a ~20 GB GGUF)..." + echo "Server log: $SERVER_LOG" + bash "$ROOT/scripts/$SERVE_SCRIPT" > "$SERVER_LOG" 2>&1 & + SERVER_PID=$! + WE_BOOTED=true + fi + + echo -n "Waiting for $EXPECTED_ALIAS to be ready " + for i in $(seq 1 180); do + if ready_check; then + echo " ready (~$((i*2))s)" + break + fi + if $WE_BOOTED && ! kill -0 "$SERVER_PID" 2>/dev/null; then + echo " FAILED — server process exited" + echo "Last 30 lines of server log:" >&2 + tail -30 "$SERVER_LOG" >&2 + exit 1 + fi + echo -n "." + sleep 2 + done + if ! ready_check; then + echo " TIMEOUT after 360 s" >&2 + exit 1 + fi +fi + +# ── Run eval ───────────────────────────────────────────────────────────────── +echo +echo "=== Eval ===" +echo "Time: $(date)" + +INHIBIT="" +if command -v systemd-inhibit &>/dev/null; then + INHIBIT="systemd-inhibit --what=sleep:idle --who=eval_llamacpp.sh --why=KELE-eval" +fi + +EXTRA_KELE_ARGS=() +if $UNIFIED; then EXTRA_KELE_ARGS+=(--unified); fi + +if [[ "$SUBCMD" == "test" ]]; then + $INHIBIT uv run python -m src.project.kele \ + --experiment "$EXPERIMENT" test --n "$N" --output "$OUT_DIR" \ + "${EXTRA_KELE_ARGS[@]:-}" +else + $INHIBIT uv run python -m src.project.kele \ + --experiment "$EXPERIMENT" evaluate --output "$OUT_DIR" \ + "${EXTRA_KELE_ARGS[@]:-}" +fi + +echo +echo "Eval finished: $(date)" + +# ── Comparison ─────────────────────────────────────────────────────────────── +if $DO_COMPARE; then + echo + echo "=== Comparison vs $BASELINE_DIR (gpt-4o-2024-11-20 baseline) ===" + uv run python -m src.project.evaluate --compare "$BASELINE_DIR" "$OUT_DIR" + + # Per-stage state accuracy table (compare.py only prints overall) + if command -v jq &>/dev/null; then + echo + echo "Per-stage state accuracy:" + printf " %-10s %15s %15s\n" "stage" "baseline" "$MODEL" + printf " %-10s %15s %15s\n" "----------" "---------------" "---------------" + for stage in a b c d e overall; do + if [[ "$stage" == "overall" ]]; then + BASE=$(jq -r '.state_accuracy.overall' "$BASELINE_DIR/metrics_summary.json" 2>/dev/null || echo "?") + THIS=$(jq -r '.state_accuracy.overall' "$OUT_DIR/metrics_summary.json" 2>/dev/null || echo "?") + else + BASE=$(jq -r ".state_accuracy.per_stage.$stage" "$BASELINE_DIR/metrics_summary.json" 2>/dev/null || echo "?") + THIS=$(jq -r ".state_accuracy.per_stage.$stage" "$OUT_DIR/metrics_summary.json" 2>/dev/null || echo "?") + fi + printf " %-10s %15s %15s\n" "$stage" "$BASE" "$THIS" + done + else + echo "(install jq to see per-stage state accuracy comparison)" + fi +fi + +echo +echo "=== Done ===" +echo "Run log: $RUN_LOG" +echo "Results: $OUT_DIR/" diff --git a/scripts/llm_judge_eval.py b/scripts/llm_judge_eval.py index 3a6fe0fe..33e2322b 100644 --- a/scripts/llm_judge_eval.py +++ b/scripts/llm_judge_eval.py @@ -1,22 +1,32 @@ -"""LLM-judge evaluation over saved dialogue logs. +"""LLM-judge evaluation over saved dialogue logs (absolute Socratic quality). -Runs Claude Sonnet 4.6 as a single judge with a 4-axis rubric: +Runs Claude as a single judge with a 4-axis rubric: 1. Socratic validity (0-3): Valid teaching move at the current stage? 2. Advancement (0-3): Does the response advance student reasoning? 3. Age-appropriateness (0-2): Vocabulary suitable for elementary? 4. Question-form fidelity (0-2): Single question, not over-leading. -Total per turn: 0-10. Aggregated per dialogue + per config. +Total per turn: 0-10. Aggregated per dialogue + per config + per Socratic stage. +This is an *absolute*-quality read, independent of ROUGE/BLEU overlap-with-reference +(the analysis-plan LLM-judge task; confirms the oracle-arm SFT-vs-base finding). -Cost (Sonnet 4.6 at $3 in / $15 out): - ~1000 input tokens + ~150 output tokens per turn judged. - At n=50 (~280 turns) → ~$1.50 per config. - At n=681 (~4000 turns) → ~$21 per config. +Two backends: +- claude-code (DEFAULT): judges via the Claude Code CLI on the logged-in Anthropic + SUBSCRIPTION — no API key. Sequential (parallel `claude -p` would race one account). + Default model claude-opus-4-8. ANTHROPIC_API_KEY is stripped from the child env so + it can't silently fall back to API billing. +- api: OpenAI-compat endpoint with ANTHROPIC_API_KEY (default model claude-sonnet-4-6), + parallel via --workers. Usage: - uv run python scripts/llm_judge_eval.py [--model claude-sonnet-4-6] [--workers 10] - -Output: /judge_summary.json with per-turn + per-dialogue + overall scores. + # subscription smoke (5 turns) then full arm: + uv run python scripts/llm_judge_eval.py results/gemma4-12b-sft-oracle --max-turns 5 + uv run python scripts/llm_judge_eval.py results/gemma4-12b-sft-oracle + # api backend: + uv run python scripts/llm_judge_eval.py --backend api --workers 10 + +Output: / (default judge_summary.json) with per-turn + +per-dialogue + per-stage + overall scores. """ from __future__ import annotations @@ -24,6 +34,7 @@ import argparse import json import os +import subprocess import sys import time from concurrent.futures import ThreadPoolExecutor, as_completed @@ -99,14 +110,120 @@ def build_judge_user_prompt(dialogue: dict, turn_idx: int) -> str: Score the AI teacher response.""" +def _parse_rubric(content: str) -> dict | None: + """Parse the judge's JSON reply into validated axis scores, or None.""" + content = content.strip() + if content.startswith("```"): + content = content.split("```", 2)[1] + if content.startswith("json"): + content = content[4:] + content = content.strip() + try: + scores = json.loads(content) + except json.JSONDecodeError: + return None + for k in ("socratic_validity", "advancement", "age_appropriateness", "question_form"): + if k not in scores: + return None + scores[k] = int(scores[k]) + scores["total"] = ( + scores["socratic_validity"] + + scores["advancement"] + + scores["age_appropriateness"] + + scores["question_form"] + ) + return scores + + +def _claude_code_judge(model: str, prompt: str, max_retries: int = 5) -> dict | None: + """Run one judge call through the Claude Code CLI on the logged-in SUBSCRIPTION. + + ANTHROPIC_API_KEY is stripped from the child env so the CLI cannot silently + fall back to API-key billing (it takes precedence over the subscription when + set). No tools / no MCP / no skills — a pure text-in/JSON-out judge. + """ + env = {k: v for k, v in os.environ.items() if k != "ANTHROPIC_API_KEY"} + cmd = [ + "claude", + "-p", + "--model", + model, + "--output-format", + "json", + "--tools", + "", + "--permission-mode", + "dontAsk", + "--strict-mcp-config", + "--disable-slash-commands", + ] + for attempt in range(max_retries): + try: + proc = subprocess.run( + cmd, + input=prompt, + capture_output=True, + text=True, + env=env, + timeout=180, + ) + if proc.returncode != 0: + print(f" claude -p rc={proc.returncode}: {proc.stderr[:200]}", file=sys.stderr) + time.sleep(min(2**attempt, 30)) + continue + envelope = json.loads(proc.stdout) + if envelope.get("is_error"): + time.sleep(min(2**attempt, 30)) + continue + usage = envelope.get("usage", {}) + # input_tokens is the UNCACHED portion only; the bulk of a repeated judge + # prompt is prompt-cache hits, so add the cache buckets for a true count. + usage_in = ( + usage.get("input_tokens", 0) + + usage.get("cache_read_input_tokens", 0) + + usage.get("cache_creation_input_tokens", 0) + ) + return { + "content": envelope.get("result") or "", + "usage_in": usage_in, + "usage_out": usage.get("output_tokens", 0), + "cost_usd": envelope.get("total_cost_usd", 0.0), + } + except subprocess.TimeoutExpired: + print(f" claude -p timeout (attempt {attempt + 1})", file=sys.stderr) + time.sleep(min(2**attempt, 30)) + except json.JSONDecodeError: + print(f" claude -p non-JSON stdout (attempt {attempt + 1})", file=sys.stderr) + time.sleep(min(2**attempt, 30)) + return None + + def score_turn( - client: openai.Client, model: str, dialogue: dict, turn_idx: int, max_retries: int = 6 + client: openai.Client | None, + model: str, + dialogue: dict, + turn_idx: int, + backend: str = "api", + max_retries: int = 6, ) -> dict | None: """Score a single turn via the judge. Returns dict with axis scores or None on failure.""" user_prompt = build_judge_user_prompt(dialogue, turn_idx) if not user_prompt: return None + if backend == "claude-code": + r = _claude_code_judge(model, f"{JUDGE_SYSTEM_PROMPT}\n\n{user_prompt}", max_retries) + if r is None: + return None + scores = _parse_rubric(r["content"]) + if scores is None: + return None + scores["usage_in"] = r["usage_in"] + scores["usage_out"] = r["usage_out"] + scores["cost_usd"] = r["cost_usd"] + return scores + + assert client is not None for attempt in range(max_retries): try: resp = client.chat.completions.create( @@ -118,28 +235,11 @@ def score_turn( max_tokens=200, temperature=0, ) - content = resp.choices[0].message.content or "" - # Trim markdown json fences if present - content = content.strip() - if content.startswith("```"): - content = content.split("```", 2)[1] - if content.startswith("json"): - content = content[4:] - content = content.strip() - scores = json.loads(content) - # Validate - for k in ("socratic_validity", "advancement", "age_appropriateness", "question_form"): - if k not in scores: - return None - scores[k] = int(scores[k]) - scores["total"] = ( - scores["socratic_validity"] - + scores["advancement"] - + scores["age_appropriateness"] - + scores["question_form"] - ) - scores["usage_in"] = resp.usage.prompt_tokens - scores["usage_out"] = resp.usage.completion_tokens + scores = _parse_rubric(resp.choices[0].message.content or "") + if scores is None: + return None + scores["usage_in"] = resp.usage.prompt_tokens if resp.usage else 0 + scores["usage_out"] = resp.usage.completion_tokens if resp.usage else 0 return scores except openai.RateLimitError: wait = min(2**attempt, 30) @@ -147,8 +247,6 @@ def score_turn( except (openai.APIStatusError, openai.APIConnectionError): wait = min(2**attempt, 30) time.sleep(wait) - except json.JSONDecodeError: - return None except Exception as e: print(f" judge error (attempt {attempt + 1}): {e}", file=sys.stderr) time.sleep(2) @@ -156,12 +254,14 @@ def score_turn( return None -def judge_dialogue(client: openai.Client, model: str, dialogue: dict) -> list[dict]: +def judge_dialogue( + client: openai.Client | None, model: str, dialogue: dict, backend: str = "api" +) -> list[dict]: """Judge every turn in a dialogue. Returns list of scored turns (skipping failures).""" out = [] turns = dialogue.get("dialogue", []) for i in range(len(turns)): - score = score_turn(client, model, dialogue, i) + score = score_turn(client, model, dialogue, i, backend=backend) if score is not None: score["turn_idx"] = i score["ground_truth_state"] = turns[i].get("ground_truth_state", "?") @@ -169,25 +269,163 @@ def judge_dialogue(client: openai.Client, model: str, dialogue: dict) -> list[di return out -def judge_results_dir(results_dir: Path, client: openai.Client, model: str, workers: int) -> dict: +def _summarize( + results_dir: Path, + model: str, + backend: str, + all_turn_scores: list[dict], + per_dialogue_summaries: list[dict], + elapsed: float, + out_name: str, +) -> dict: + """Aggregate scored turns into the summary dict + write it to disk.""" + per_stage = {s: {"n": 0, "total": 0.0} for s in "abcde"} + for s in all_turn_scores: + gt = s.get("ground_truth_state", "?") + if gt and gt[0] in per_stage: + per_stage[gt[0]]["n"] += 1 + per_stage[gt[0]]["total"] += s["total"] + for k in per_stage: + n = per_stage[k]["n"] + per_stage[k]["avg"] = per_stage[k]["total"] / n if n else 0.0 + + n_turns = len(all_turn_scores) + total_in = sum(s.get("usage_in", 0) for s in all_turn_scores) + total_out = sum(s.get("usage_out", 0) for s in all_turn_scores) + # claude-code backend reports per-call subscription-equivalent cost directly; + # the api backend estimates from tokens at Sonnet 4.6 rates. + if backend == "claude-code": + cost = round(sum(s.get("cost_usd", 0.0) for s in all_turn_scores), 4) + else: + cost = round(total_in / 1e6 * 3.0 + total_out / 1e6 * 15.0, 4) + + def _axis(name: str) -> float: + return sum(s[name] for s in all_turn_scores) / n_turns if n_turns else 0 + + summary = { + "results_dir": str(results_dir), + "judge_model": model, + "backend": backend, + "n_dialogues_judged": len(per_dialogue_summaries), + "n_turns_judged": n_turns, + "wall_clock_seconds": elapsed, + "overall_avg": sum(s["total"] for s in all_turn_scores) / n_turns if n_turns else 0, + "axis_avgs": { + "socratic_validity": _axis("socratic_validity"), + "advancement": _axis("advancement"), + "age_appropriateness": _axis("age_appropriateness"), + "question_form": _axis("question_form"), + }, + "per_stage": per_stage, + "tokens_in": total_in, + "tokens_out": total_out, + "cost_usd": cost, + "per_dialogue": per_dialogue_summaries, + } + summary_path = results_dir / out_name + summary_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False)) + print(f"\nSummary written to {summary_path}") + print( + f"Overall: {summary['overall_avg']:.2f}/10 ({n_turns} turns, " + f"${cost:.3f} cost, {elapsed:.0f}s wall clock)" + ) + return summary + + +def _dialogue_summary(fname: str, did: int, scores: list[dict]) -> dict: + dlg_n = len(scores) + return { + "file": fname, + "id": did, + "n_turns_judged": dlg_n, + "avg_total": sum(s["total"] for s in scores) / dlg_n, + "avg_validity": sum(s["socratic_validity"] for s in scores) / dlg_n, + "avg_advancement": sum(s["advancement"] for s in scores) / dlg_n, + "avg_age": sum(s["age_appropriateness"] for s in scores) / dlg_n, + "avg_qform": sum(s["question_form"] for s in scores) / dlg_n, + } + + +def judge_sequential( + results_dir: Path, + dialogue_files: list[Path], + model: str, + backend: str, + max_turns: int | None, + out_name: str, +) -> dict: + """Sequential judge — one call at a time. Used for the claude-code (subscription) + backend, where parallel `claude -p` processes would race on the same account. + max_turns caps the total turns scored (for a cheap smoke run).""" + print(f"\n=== Judging {len(dialogue_files)} dialogues in {results_dir} (sequential) ===") + print(f"Judge model: {model}, backend: {backend}, max_turns: {max_turns}") + started = time.time() + all_turn_scores: list[dict] = [] + per_dialogue_summaries: list[dict] = [] + for dfile in dialogue_files: + if max_turns is not None and len(all_turn_scores) >= max_turns: + break + d = json.loads(dfile.read_text()) + turns = d.get("dialogue", []) + dlg_scores = [] + for i in range(len(turns)): + if max_turns is not None and len(all_turn_scores) >= max_turns: + break + score = score_turn(None, model, d, i, backend=backend) + if score is not None: + score["turn_idx"] = i + score["ground_truth_state"] = turns[i].get("ground_truth_state", "?") + dlg_scores.append(score) + all_turn_scores.append(score) + print( + f" turn {len(all_turn_scores)} [{dfile.name} #{i} " + f"state={score['ground_truth_state']}]: {score['total']}/10", + flush=True, + ) + if dlg_scores: + per_dialogue_summaries.append( + _dialogue_summary(dfile.name, d.get("id", -1), dlg_scores) + ) + return _summarize( + results_dir, + model, + backend, + all_turn_scores, + per_dialogue_summaries, + time.time() - started, + out_name, + ) + + +def judge_results_dir( + results_dir: Path, + client: openai.Client | None, + model: str, + workers: int, + backend: str = "api", + max_turns: int | None = None, + out_name: str = "judge_summary.json", +) -> dict: """Run judge over all dialogues in results_dir. Returns summary dict.""" dialogues_dir = results_dir / "dialogues" if not dialogues_dir.is_dir(): raise FileNotFoundError(f"No dialogues/ in {results_dir}") dialogue_files = sorted(dialogues_dir.glob("*.json")) + + if backend == "claude-code": + return judge_sequential(results_dir, dialogue_files, model, backend, max_turns, out_name) + print(f"\n=== Judging {len(dialogue_files)} dialogues in {results_dir} ===") print(f"Judge model: {model}, workers: {workers}") started = time.time() all_turn_scores = [] per_dialogue_summaries = [] - total_in_tokens = 0 - total_out_tokens = 0 def _judge_one(dfile: Path): d = json.loads(dfile.read_text()) - scores = judge_dialogue(client, model, d) + scores = judge_dialogue(client, model, d, backend=backend) return dfile.name, d.get("id", -1), scores with ThreadPoolExecutor(max_workers=workers) as ex: @@ -195,102 +433,250 @@ def _judge_one(dfile: Path): for i, fut in enumerate(as_completed(futures)): fname, did, scores = fut.result() if scores: - dlg_total = sum(s["total"] for s in scores) - dlg_n = len(scores) - avg = dlg_total / dlg_n if dlg_n else 0 - in_tokens = sum(s.get("usage_in", 0) for s in scores) - out_tokens = sum(s.get("usage_out", 0) for s in scores) - total_in_tokens += in_tokens - total_out_tokens += out_tokens - per_dialogue_summaries.append( - { - "file": fname, - "id": did, - "n_turns_judged": dlg_n, - "avg_total": avg, - "avg_validity": sum(s["socratic_validity"] for s in scores) / dlg_n, - "avg_advancement": sum(s["advancement"] for s in scores) / dlg_n, - "avg_age": sum(s["age_appropriateness"] for s in scores) / dlg_n, - "avg_qform": sum(s["question_form"] for s in scores) / dlg_n, - } - ) + per_dialogue_summaries.append(_dialogue_summary(fname, did, scores)) all_turn_scores.extend(scores) elapsed = time.time() - started rate = (i + 1) / elapsed if elapsed > 0 else 0 remaining = (len(dialogue_files) - i - 1) / rate if rate > 0 else 0 + avg = per_dialogue_summaries[-1]["avg_total"] if scores else 0 print( f" {i + 1}/{len(dialogue_files)} {fname} " - f"({dlg_n if scores else 0} turns, avg {avg:.2f}/10) " + f"({len(scores) if scores else 0} turns, avg {avg:.2f}/10) " f"{elapsed:.0f}s elapsed, {remaining:.0f}s remaining", flush=True, ) - elapsed = time.time() - started + return _summarize( + results_dir, + model, + backend, + all_turn_scores, + per_dialogue_summaries, + time.time() - started, + out_name, + ) - # Per-stage aggregation - per_stage = {s: {"n": 0, "total": 0.0} for s in "abcde"} - for s in all_turn_scores: - gt = s.get("ground_truth_state", "?") - if gt and gt[0] in per_stage: - per_stage[gt[0]]["n"] += 1 - per_stage[gt[0]]["total"] += s["total"] - for k in per_stage: - n = per_stage[k]["n"] - per_stage[k]["avg"] = per_stage[k]["total"] / n if n else 0.0 - n_turns = len(all_turn_scores) - summary = { - "results_dir": str(results_dir), +def build_stratified_sample( + dialogues_dir: Path, per_stage: int, stages: str, seed: int +) -> list[tuple[str, int, str]]: + """Pick `per_stage` turns for each Socratic stage in `stages`, seeded. + + Returns (dialogue_file, turn_idx, stage) tuples. Both oracle arms replay the + same ground-truth student turns and hit e34 at the same turn, so a sample keyed + on (file, turn_idx) is identical and comparable across arms — a paired design. + """ + import random + + by_stage: dict[str, list[tuple[str, int, str]]] = {s: [] for s in stages} + for dfile in sorted(dialogues_dir.glob("*.json")): + d = json.loads(dfile.read_text()) + for i, turn in enumerate(d.get("dialogue", [])): + gt = turn.get("ground_truth_state") or turn.get("state") or "?" + stage = gt[0] if gt else "?" + if stage in by_stage: + by_stage[stage].append((dfile.name, i, stage)) + + rng = random.Random(seed) + sample: list[tuple[str, int, str]] = [] + for s in stages: + pool = by_stage[s] + if len(pool) <= per_stage: + print(f" stage {s}: {len(pool)} available <= {per_stage} requested — taking all") + sample.extend(pool) + else: + sample.extend(rng.sample(pool, per_stage)) + return sample + + +def _judge_arm_on_sample( + arm_dir: Path, sample: list[tuple[str, int, str]], model: str, backend: str +) -> list[dict]: + """Score every sampled (file, turn_idx) turn in one arm. Sequential (subscription).""" + dialogues_dir = arm_dir / "dialogues" + cache: dict[str, dict] = {} + scores: list[dict] = [] + for n, (fname, turn_idx, stage) in enumerate(sample, 1): + if fname not in cache: + cache[fname] = json.loads((dialogues_dir / fname).read_text()) + s = score_turn(None, model, cache[fname], turn_idx, backend=backend) + if s is not None: + s["turn_idx"] = turn_idx + s["ground_truth_state"] = cache[fname]["dialogue"][turn_idx].get( + "ground_truth_state", "?" + ) + scores.append(s) + print( + f" [{arm_dir.name}] {n}/{len(sample)} {fname}#{turn_idx} " + f"({stage}): {s['total']}/10", + flush=True, + ) + return scores + + +def judge_paired_compare( + arm_a: Path, + arm_b: Path, + model: str, + backend: str, + per_stage: int, + stages: str, + seed: int, + out_path: Path, +) -> dict: + """Judge both arms on ONE stratified paired sample; report per-arm scores + A−B deltas.""" + sample = build_stratified_sample(arm_a / "dialogues", per_stage, stages, seed) + print(f"\nStratified sample: {len(sample)} turns/arm across stages '{stages}' (seed {seed})") + + started = time.time() + arm_scores = {} + for arm in (arm_a, arm_b): + print(f"\n=== Judging arm: {arm.name} ({len(sample)} turns, {model}) ===") + scores = _judge_arm_on_sample(arm, sample, model, backend) + arm_scores[arm.name] = _summarize( + arm, model, backend, scores, [], time.time() - started, f".judge_{arm.name}.tmp.json" + ) + (arm / f".judge_{arm.name}.tmp.json").unlink(missing_ok=True) + + a, b = arm_scores[arm_a.name], arm_scores[arm_b.name] + axes = ["socratic_validity", "advancement", "age_appropriateness", "question_form"] + delta = { + "overall": round(a["overall_avg"] - b["overall_avg"], 3), + "axes": {ax: round(a["axis_avgs"][ax] - b["axis_avgs"][ax], 3) for ax in axes}, + "per_stage": { + s: round(a["per_stage"][s]["avg"] - b["per_stage"][s]["avg"], 3) + for s in stages + if a["per_stage"][s]["n"] and b["per_stage"][s]["n"] + }, + } + out = { "judge_model": model, - "n_dialogues_judged": len(per_dialogue_summaries), - "n_turns_judged": n_turns, - "wall_clock_seconds": elapsed, - "overall_avg": sum(s["total"] for s in all_turn_scores) / n_turns if n_turns else 0, - "axis_avgs": { - "socratic_validity": sum(s["socratic_validity"] for s in all_turn_scores) / n_turns - if n_turns - else 0, - "advancement": sum(s["advancement"] for s in all_turn_scores) / n_turns - if n_turns - else 0, - "age_appropriateness": sum(s["age_appropriateness"] for s in all_turn_scores) / n_turns - if n_turns - else 0, - "question_form": sum(s["question_form"] for s in all_turn_scores) / n_turns - if n_turns - else 0, + "backend": backend, + "arm_a": arm_a.name, + "arm_b": arm_b.name, + "per_stage_n": per_stage, + "stages": stages, + "seed": seed, + "sample_size_per_arm": len(sample), + "wall_clock_seconds": round(time.time() - started, 1), + "arms": { + arm_a.name: {k: a[k] for k in ("overall_avg", "axis_avgs", "per_stage", "cost_usd")}, + arm_b.name: {k: b[k] for k in ("overall_avg", "axis_avgs", "per_stage", "cost_usd")}, }, - "per_stage": per_stage, - "tokens_in": total_in_tokens, - "tokens_out": total_out_tokens, - # Cost estimate at Sonnet 4.6 rates - "cost_usd": round(total_in_tokens / 1e6 * 3.0 + total_out_tokens / 1e6 * 15.0, 4), - "per_dialogue": per_dialogue_summaries, + "delta_a_minus_b": delta, + "sample": sample, } - - summary_path = results_dir / "judge_summary.json" - summary_path.write_text(json.dumps(summary, indent=2, ensure_ascii=False)) - print(f"\nSummary written to {summary_path}") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out, indent=2, ensure_ascii=False)) + print(f"\nComparison written to {out_path}") print( - f"Overall: {summary['overall_avg']:.2f}/10 ({n_turns} turns, " - f"${summary['cost_usd']:.3f} cost, {elapsed:.0f}s wall clock)" + f"{arm_a.name} {a['overall_avg']:.2f} vs {arm_b.name} {b['overall_avg']:.2f} " + f"→ Δ(a−b) = {delta['overall']:+.2f}/10" ) - return summary + return out def main(): parser = argparse.ArgumentParser(description="LLM-judge eval over saved dialogues") parser.add_argument("results_dir", type=Path, help="Path to results dir containing dialogues/") - parser.add_argument("--model", default="claude-sonnet-4-6", help="Judge model ID") - parser.add_argument("--workers", type=int, default=8, help="Concurrent judge calls") - parser.add_argument("--api-key", default=None, help="Override ANTHROPIC_API_KEY") + parser.add_argument( + "--backend", + choices=["api", "claude-code"], + default="claude-code", + help="'claude-code' judges via the Claude Code CLI on the logged-in " + "SUBSCRIPTION (no API key, sequential); 'api' uses ANTHROPIC_API_KEY.", + ) + parser.add_argument( + "--model", + default=None, + help="Judge model ID. Default: claude-opus-4-8 (claude-code) / claude-sonnet-4-6 (api).", + ) + parser.add_argument("--workers", type=int, default=8, help="Concurrent judge calls (api only)") + parser.add_argument( + "--max-turns", + type=int, + default=None, + help="Cap total turns scored (claude-code backend) — use for a cheap smoke run.", + ) + parser.add_argument( + "--out-name", + default="judge_summary.json", + help="Summary filename written into results_dir.", + ) + parser.add_argument("--api-key", default=None, help="Override ANTHROPIC_API_KEY (api backend)") parser.add_argument( "--base-url", default="https://api.anthropic.com/v1/", - help="OpenAI-compat endpoint base URL", + help="OpenAI-compat endpoint base URL (api backend)", + ) + # Paired stratified compare (the absolute-quality scenario): judge results_dir + # (arm A) and --compare-with (arm B) on ONE stratified sample; report A−B deltas. + parser.add_argument( + "--compare-with", + type=Path, + default=None, + help="Second results dir (arm B). Enables paired stratified compare mode.", + ) + parser.add_argument( + "--per-stage", type=int, default=40, help="Turns sampled per Socratic stage (compare mode)." + ) + parser.add_argument( + "--stages", default="bcde", help="Stages to sample (compare mode; 'a' is a trivial opener)." + ) + parser.add_argument("--seed", type=int, default=42, help="Sampling seed (compare mode).") + parser.add_argument( + "--out", + type=Path, + default=Path("results/llm_judge_oracle_compare.json"), + help="Output path for the compare summary.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Compare mode: build + report the sample and estimated calls, judge nothing.", ) args = parser.parse_args() + if args.compare_with is not None: + model = args.model or ( + "claude-opus-4-8" if args.backend == "claude-code" else "claude-sonnet-4-6" + ) + if args.dry_run: + sample = build_stratified_sample( + args.results_dir / "dialogues", args.per_stage, args.stages, args.seed + ) + counts: dict[str, int] = {} + for _f, _i, s in sample: + counts[s] = counts.get(s, 0) + 1 + print(f"\nDRY RUN — no judging. Sample per arm: {len(sample)} turns {counts}") + print(f"Both arms → {2 * len(sample)} total {model} calls (~sequential).") + return + judge_paired_compare( + args.results_dir, + args.compare_with, + model, + args.backend, + args.per_stage, + args.stages, + args.seed, + args.out, + ) + return + + if args.backend == "claude-code": + model = args.model or "claude-opus-4-8" + judge_results_dir( + args.results_dir, + None, + model, + args.workers, + backend="claude-code", + max_turns=args.max_turns, + out_name=args.out_name, + ) + return + + model = args.model or "claude-sonnet-4-6" api_key = args.api_key or os.environ.get("ANTHROPIC_API_KEY") if not api_key: # try .env @@ -305,7 +691,9 @@ def main(): sys.exit(1) client = openai.Client(api_key=api_key, base_url=args.base_url) - judge_results_dir(args.results_dir, client, args.model, args.workers) + judge_results_dir( + args.results_dir, client, model, args.workers, backend="api", out_name=args.out_name + ) if __name__ == "__main__": diff --git a/scripts/monitor_eos_gate.sh b/scripts/monitor_eos_gate.sh new file mode 100755 index 00000000..71059072 --- /dev/null +++ b/scripts/monitor_eos_gate.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# This script builds GitHub-markdown PR comments via printf; the format strings +# are single-quoted on purpose (%s positional args + literal backticks/newlines), +# so SC2016 ("expressions don't expand in single quotes") is a false positive. +# shellcheck disable=SC2016 +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +EOS_LOG="$REPO_DIR/outputs/eos-gate-gemma4-31b/train.log" +FINAL_ADAPTER="$REPO_DIR/outputs/eos-gate-gemma4-31b/final" +STAGE2_LOG="$REPO_DIR/outputs/sft-stage2-gemma4-31b/train.log" +SELF_LOG="$REPO_DIR/outputs/monitor_eos_gate.log" +PR_NUMBER=101 +MAX_RETRIES=2 +POLL_SECONDS=300 + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$SELF_LOG" +} + +post_pr() { + local body="$1" + gh pr comment "$PR_NUMBER" --body "$body" >> "$SELF_LOG" 2>&1 || \ + log "WARNING: gh pr comment failed (network?)" +} + +training_running() { + pgrep -f "train_sft\.py" > /dev/null 2>&1 +} + +crash_hint() { + if [[ -f "$EOS_LOG" ]]; then + grep -i "page fault\|traceback\|runtimeerror\|out of memory\|oom\|killed" "$EOS_LOG" \ + | tail -3 | sed 's/^/ /' || true + fi +} + +last_step() { + if [[ -f "$EOS_LOG" ]]; then + grep -oE "[0-9]+/100" "$EOS_LOG" | tail -1 || true + fi +} + +start_train() { + log "Starting EOS gate training run" + if [[ -d "$REPO_DIR/outputs/eos-gate-gemma4-31b" ]]; then + find "$REPO_DIR/outputs/eos-gate-gemma4-31b" -mindepth 1 -delete + fi + cd "$REPO_DIR" && make train-gemma4-31b-eos-gate +} + +kill_and_clean() { + log "Killing stray train_sft.py processes" + pkill -9 -f "train_sft\.py" 2>/dev/null || true + local waited=0 + while pgrep -f "train_sft\.py" > /dev/null 2>&1 && [[ $waited -lt 30 ]]; do + sleep 2 + (( waited += 2 )) + done + log "KFD settle wait (20s)" + sleep 20 +} + +run_eval() { + log "Training complete — running EOS gate eval" + local eval_out + eval_out=$(cd "$REPO_DIR" && \ + env TORCH_USE_HIPBLASLT=0 \ + PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + TRAIN_BASE_MODEL=unsloth/gemma-4-31B-it-unsloth-bnb-4bit \ + TRAIN_PREQ=true \ + TRAIN_METHOD=qlora \ + TRAIN_BF16=true \ + uv run --no-sync python scripts/eos_gate.py \ + --config configs/train-sft-stage2-gemma4-31b.env \ + --adapter outputs/eos-gate-gemma4-31b/final 2>&1) + printf '%s\n' "$eval_out" | tee -a "$SELF_LOG" + printf '%s\n' "$eval_out" +} + +# ── Main ──────────────────────────────────────────────────────────────────── + +mkdir -p "$REPO_DIR/outputs" +log "Monitor starting (PR #$PR_NUMBER, poll every ${POLL_SECONDS}s, max $MAX_RETRIES crash retries)" + +post_pr "$(printf '**EOS gate retrain started** (gfx1201 / R9700) — commit \`%s\`\n\n> Root cause fixed (`dbc61a2`): \`\` terminator now inside \`{%% generation %%}\` block. Monitoring autonomously; next update when training completes (~115 min).' "$(cd "$REPO_DIR" && git rev-parse --short HEAD)")" + +retries=0 + +# Give the training process a moment to start +sleep 10 + +while true; do + if training_running; then + step="$(last_step)" + log "Training running${step:+ — last step: $step}" + sleep "$POLL_SECONDS" + continue + fi + + # Process gone — determine outcome + if [[ -d "$FINAL_ADAPTER" ]]; then + log "Training finished — adapter at $FINAL_ADAPTER" + break + fi + + # Crashed + hint="$(crash_hint)" + log "CRASH DETECTED (retry $retries/$MAX_RETRIES)" + [[ -n "$hint" ]] && log "Log tail: $hint" + + if (( retries >= MAX_RETRIES )); then + post_pr "$(printf '**EOS gate training CRASHED** — exceeded %d retries, giving up.\n\nCrash hint:\n```\n%s\n```\n\nManual intervention needed. Monitor log: `outputs/monitor_eos_gate.log`' "$MAX_RETRIES" "$hint")" + log "Max retries exceeded — exiting" + exit 1 + fi + + post_pr "$(printf '**EOS gate training CRASHED** (retry %d/%d) — restarting after KFD cleanup.\n\nCrash hint:\n```\n%s\n```' "$((retries+1))" "$MAX_RETRIES" "$hint")" + + retries=$(( retries + 1 )) + kill_and_clean + start_train + sleep 30 # let the new process start before polling +done + +# ── Eval ──────────────────────────────────────────────────────────────────── + +eval_output="$(run_eval)" + +if printf '%s' "$eval_output" | grep -q "EOS GATE PASSED"; then + log "EOS GATE PASSED" + + # Extract the generation snippet for the PR comment + gate_generation="$(printf '%s' "$eval_output" | grep -A 20 "Generation:" | head -20 || true)" + + post_pr "$(printf '## EOS Gate: PASSED ✓\n\nCommit: `%s`\n\nSTEP 1 warm-up → PASS \nSTEP 2 EOS gate → **PASS**\n\nThe `` terminator fix (`dbc61a2`) is confirmed: model terminates before `max_new_tokens` on the realistic free-form prompt.\n\n
Generation output\n\n```\n%s\n```\n\n
\n\n---\nLaunching full Stage 2 run (`make train-gemma4-31b-stage2-unsloth`).' \ + "$(cd "$REPO_DIR" && git rev-parse --short HEAD)" "$gate_generation")" + + log "Launching full Stage 2 run" + cd "$REPO_DIR" && make train-gemma4-31b-stage2-unsloth + + log "Stage 2 run launched — monitor: tail -f $STAGE2_LOG" + post_pr "$(printf '**Stage 2 training started.** Log: `outputs/sft-stage2-gemma4-31b/train.log`\n\nNext manual check: serving generation-primer audit before downstream eval (see handoff §2).')" + + log "Launching Stage 2 crash monitor" + chmod +x "$REPO_DIR/scripts/monitor_stage2.sh" + nohup setsid bash "$REPO_DIR/scripts/monitor_stage2.sh" > /dev/null 2>&1 & + +else + log "EOS GATE FAILED" + + gate_tail="$(printf '%s' "$eval_output" | tail -30)" + post_pr "$(printf '## EOS Gate: FAILED ✗\n\nCommit: `%s`\n\nSTEP 2 EOS gate → **FAIL** — model hit max_new_tokens without terminating.\n\nThe fix did not resolve the collapse, or a new issue was introduced. Manual diagnosis needed.\n\n
Gate output tail\n\n```\n%s\n```\n\n
' \ + "$(cd "$REPO_DIR" && git rev-parse --short HEAD)" "$gate_tail")" + + log "Gate failed — NOT launching Stage 2. See PR #$PR_NUMBER for details." + exit 2 +fi + +log "Monitor done" diff --git a/scripts/monitor_eval_gemma4_12b.sh b/scripts/monitor_eval_gemma4_12b.sh new file mode 100755 index 00000000..de6d686d --- /dev/null +++ b/scripts/monitor_eval_gemma4_12b.sh @@ -0,0 +1,537 @@ +#!/usr/bin/env bash +# Eval-side auto-resume monitor for the Gemma 4 12B SFT-uplift PoC on the +# NVIDIA RTX 4000 Ada box. Sibling of scripts/monitor_stage2.sh (training), but +# the failure model is different and that difference is the whole point: +# +# The RTX 4000 Ada took a power surge and crashes UNDER LOAD. A fault kills the +# llama.cpp SERVER, but the eval CLIENT is pure CPU + HTTP (the classifier runs +# on CPU), so it survives — run_single_dialogue catches the connection error, +# stamps {"error": …} into every remaining dialogue at machine speed, and exits +# 0 with metrics_summary.json computed over the surviving subset. Unmonitored, a +# single fault silently ships a TRUNCATED eval reported as success. +# +# So, unlike the training monitor: +# 1. The crash signal is SERVER health (/v1/models), polled out-of-band — NOT +# client process death (the client exits 0 on a server fault). +# 2. Relaunch must REPAIR before it resumes: kele.py counts any non-zero file as +# done (src/project/kele.py:448), so error-stamped/truncated dialogues are +# skipped forever unless deleted first. repair_dialogues() is the eval +# analogue of the training monitor's quarantine_bad_checkpoint. +# 3. COMPLETE means valid (non-error) dialogue count == dataset size — NOT +# "metrics_summary.json exists". Forward progress for MAX_RETRIES is counted +# in valid dialogues. +# +# Progress/crash rows append to ONE pinned comment on the eval log issue #130 +# (same convention as monitor_stage2.sh / issue #120). The comment body is the +# source of truth: fetch → append → PATCH, so restarts never lose history. +# +# This script builds GitHub-markdown rows via printf; the single-quoted format +# strings (%s args + literal backticks/pipes) make SC2016 a false positive. +# shellcheck disable=SC2016 +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" || exit 1 + +PHASE="${1:-}" +case "$PHASE" in + base) + EXPERIMENT="gemma4-12b-local" + OUT_DIR="$REPO_DIR/results/gemma4-12b-base" + SERVE_SCRIPT="scripts/serve_gemma4_12b.sh" + EXPECTED_ALIAS="Gemma 4 12B" + ;; + sft) + EXPERIMENT="gemma4-12b-sft-local" + OUT_DIR="$REPO_DIR/results/gemma4-12b-sft" + SERVE_SCRIPT="scripts/serve_gemma4_12b_sft.sh" + EXPECTED_ALIAS="Gemma 4 12B SFT" + ;; + *) + printf 'usage: [MTP=1] %s {base|sft}\n' "$0" >&2 + printf ' base → serve %s, eval → results/gemma4-12b-base\n' "Gemma 4 12B" >&2 + printf ' sft → serve %s, eval → results/gemma4-12b-sft\n' "Gemma 4 12B SFT" >&2 + printf ' MTP=1 … → attach the speculative drafter; output dir gets a -mtp suffix\n' >&2 + exit 2 + ;; +esac + +# MTP on/off is a per-run toggle (stage 1 runs base both ways, MTP off first). The +# served alias is unchanged (the drafter is a side-car), so only the output dir + +# W&B run name differ — keeping the two full runs cleanly separated for comparison. +MTP_ENV=() +PHASE_LABEL="$PHASE" +if [[ "${MTP:-0}" == "1" ]]; then + OUT_DIR="${OUT_DIR}-mtp" + MTP_ENV=(MTP=1) + PHASE_LABEL="${PHASE}+mtp" +fi + +# Dataset override: eval a non-default HF set (EN held-out / synthetic OOD) on the +# same served model. EVAL_HF_REPO may be space-separated for concatenation; EVAL_SPLIT +# is "test" (held-out 10%) or "all" (whole set — for the tiny never-trained synthetic +# sets). EVAL_OUT_SUFFIX keeps each set's results + W&B run cleanly separated. +EVAL_DATA_ARGS=() +if [[ -n "${EVAL_HF_REPO:-}" ]]; then + read -r -a _eval_repos <<< "$EVAL_HF_REPO" + EVAL_DATA_ARGS+=(--hf-repo "${_eval_repos[@]}") + PHASE_LABEL="${PHASE_LABEL} ${EVAL_HF_REPO##*/}" +fi +if [[ -n "${EVAL_SPLIT:-}" ]]; then + EVAL_DATA_ARGS+=(--split "$EVAL_SPLIT") +fi +if [[ -n "${EVAL_OUT_SUFFIX:-}" ]]; then + OUT_DIR="${OUT_DIR}-${EVAL_OUT_SUFFIX}" +fi +# Consultant mode toggles (mutually exclusive; default = external Qwen classifier). +# Both suffix the out-dir + W&B run so they stay separate from the classifier runs. +# CONSULTANT_ARGS is built below, once BERT_CKPT is defined. +# NO_CONSULTANT=1 drops the classifier so the served LLM self-consults (dual-role) +# — the "no external classifier" ablation (handoff T1.1). +# ORACLE_CONSULTANT=1 feeds the ground-truth state each turn (--oracle-consultant) — +# isolates teacher-turn quality; state_accuracy ~perfect, score +# on ROUGE/BLEU. 1 teacher call/turn (as fast as the classifier). +if [[ "${NO_CONSULTANT:-0}" == "1" && "${ORACLE_CONSULTANT:-0}" == "1" ]]; then + printf 'error: NO_CONSULTANT and ORACLE_CONSULTANT are mutually exclusive\n' >&2 + exit 2 +fi +if [[ "${NO_CONSULTANT:-0}" == "1" ]]; then + OUT_DIR="${OUT_DIR}-noconsult" + PHASE_LABEL="${PHASE_LABEL} noconsult" +elif [[ "${ORACLE_CONSULTANT:-0}" == "1" ]]; then + OUT_DIR="${OUT_DIR}-oracle" + PHASE_LABEL="${PHASE_LABEL} oracle" +fi + +# ── Constants ──────────────────────────────────────────────────────────────── +ISSUE_NUMBER=130 +# All events append a row to ONE pinned comment (the "Live eval log" table on +# issue #130). Bump this id for a brand-new crawl: create a fresh placeholder +# comment and paste its numeric id here (same convention as monitor_stage2.sh). +LOG_COMMENT_ID=4644703104 +PROGRESS_EVERY=50 # post a progress row every this many valid dialogues +MAX_RETRIES=8 # consecutive no-forward-progress relaunches before STALLED +POLL_SECONDS=30 # tight: faster server-death reaction = fewer error-stamps +SERVER_READY_TIMEOUT=360 # cold GGUF load can take 30-90s; allow margin + +# Adaptive power search (memory: training-host-hardware-fault — the RTX 4000 Ada +# took a power surge and crashes under load). Rather than a fixed conservative cap, +# PUSH the limit: start high and step DOWN POWER_STEP_W on each crash until the run +# stops faulting — the highest power that completes a crawl is the best stability/ +# speed point. This is ~free: per-dialogue checkpointing means a fault loses only +# the in-flight dialogue, and the only recovery cost is the server cold-reload. +# POWER_START_W defaults to the card's reported max limit (fallback 130). +POWER_START_W="${POWER_START_W:-}" +POWER_STEP_W="${POWER_STEP_W:-10}" +POWER_FLOOR_W="${POWER_FLOOR_W:-70}" # below ~65-70% perf knee it's not worth going lower + +PORT="${PORT:-8080}" +LLAMA_URL="http://localhost:${PORT}" +BERT_CKPT="${BERT_CKPT:-$REPO_DIR/results/state-clf-qwen3.5-0.8b-lora-wandb/final}" +# Consultant flag: empty (self-consult) for NO_CONSULTANT, --oracle-consultant for +# ORACLE_CONSULTANT, else the external classifier checkpoint. +if [[ "${NO_CONSULTANT:-0}" == "1" ]]; then + CONSULTANT_ARGS=() +elif [[ "${ORACLE_CONSULTANT:-0}" == "1" ]]; then + CONSULTANT_ARGS=(--oracle-consultant) +else + CONSULTANT_ARGS=(--bert-consultant "$BERT_CKPT") +fi +EXPECTED_TOTAL="${EXPECTED_TOTAL:-681}" # n=681 test split; overridden by progress.log denominator once present + +DIALOGUES_DIR="$OUT_DIR/dialogues" +PROGRESS_LOG="$OUT_DIR/progress.log" +EVAL_LOG="$OUT_DIR/eval.log" +SERVER_LOG="$OUT_DIR/server.log" +CRASH_DIR="$OUT_DIR/crashlogs" +SELF_LOG="$REPO_DIR/outputs/monitor_eval_gemma4_12b_${PHASE}.log" +mkdir -p "$OUT_DIR" "$DIALOGUES_DIR" "$CRASH_DIR" "$REPO_DIR/outputs" + +SERVER_PID="" + +# ── Logging ────────────────────────────────────────────────────────────────── +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$SELF_LOG" +} +now_iso() { date '+%Y-%m-%d %H:%M:%S%:z'; } + +# Append one row to the pinned comment. Single writer (this monitor), so the +# read-modify-write has no concurrent-update race. gh resolves {owner}/{repo} +# from the cwd's git remote. +log_row() { + local body + body="$(gh api "repos/{owner}/{repo}/issues/comments/$LOG_COMMENT_ID" --jq '.body' 2>>"$SELF_LOG")" || { + log "WARNING: could not fetch log comment $LOG_COMMENT_ID — dropping row: $1"; return 0; } + printf '%s\n%s\n' "$body" "$1" \ + | gh api -X PATCH "repos/{owner}/{repo}/issues/comments/$LOG_COMMENT_ID" -F body=@- >> "$SELF_LOG" 2>&1 \ + || log "WARNING: gh api PATCH (log comment) failed" +} + +# ── Dialogue accounting (the load-bearing difference from the training monitor) ─ +# A dialogue is DONE only if its JSON parses AND carries no "error" key. Error +# files and truncated-mid-write files are NOT progress — they're what a server +# fault leaves behind, and metrics computation skips them. +count_valid() { + python3 - "$DIALOGUES_DIR" <<'PY' 2>/dev/null || printf '0' +import json, sys +from pathlib import Path +n = 0 +for f in Path(sys.argv[1]).glob("*.json"): + try: + d = json.loads(f.read_text()) + except Exception: + continue + if isinstance(d, dict) and "error" not in d: + n += 1 +print(n) +PY +} + +count_errors() { + python3 - "$DIALOGUES_DIR" <<'PY' 2>/dev/null || printf '0' +import json, sys +from pathlib import Path +n = 0 +for f in Path(sys.argv[1]).glob("*.json"): + try: + d = json.loads(f.read_text()) + except Exception: + n += 1 # truncated/unparsable counts as a corrupt file too + continue + if isinstance(d, dict) and "error" in d: + n += 1 +print(n) +PY +} + +# Delete every dialogue file that fails to parse (truncated mid json.dump) or +# carries an "error" key, so the next relaunch actually re-runs them. Prints the +# number repaired. This is the eval analogue of quarantine_bad_checkpoint. +repair_dialogues() { + python3 - "$DIALOGUES_DIR" <<'PY' 2>/dev/null || printf '0' +import json, sys +from pathlib import Path +n = 0 +for f in Path(sys.argv[1]).glob("*.json"): + bad = False + try: + d = json.loads(f.read_text()) + bad = isinstance(d, dict) and "error" in d + except Exception: + bad = True + if bad: + try: + f.unlink(); n += 1 + except Exception: + pass +print(n) +PY +} + +# Dataset size: prefer the live denominator kele writes to progress.log +# ("{done}/{total} …"); fall back to EXPECTED_TOTAL before the first line lands. +dataset_total() { + local t="" + [[ -f "$PROGRESS_LOG" ]] && t="$(grep -oE '^[0-9]+/[0-9]+' "$PROGRESS_LOG" 2>/dev/null | head -1 | cut -d/ -f2)" + printf '%s' "${t:-$EXPECTED_TOTAL}" +} + +progress_rate() { + [[ -f "$PROGRESS_LOG" ]] || return 0 + grep -oE '[0-9.]+ dlg/hr' "$PROGRESS_LOG" 2>/dev/null | head -1 | awk '{print $1}' || true +} + +progress_eta() { + [[ -f "$PROGRESS_LOG" ]] || return 0 + grep -oE 'ETA [0-9]+m' "$PROGRESS_LOG" 2>/dev/null | head -1 || true +} + +# ── Crash diagnostics (NVIDIA signatures, not the gfx1201 HSA ones from #120) ── +crash_signature() { + local sig="" + for lf in "$EVAL_LOG" "$SERVER_LOG"; do + [[ -f "$lf" ]] || continue + sig="$(grep -iE "cudaerror|cuda error|out of memory|fallen off the bus|xid|segmentation fault|sigsegv|aborted|connection refused|runtimeerror" \ + "$lf" | tail -1 | tr -d '`' | sed 's/|/／/g' | cut -c1-120)" + [[ -n "$sig" ]] && break + done + printf '%s' "$sig" +} + +# Archive eval+server logs and GPU state per crash BEFORE relaunch overwrites them. +archive_crash_log() { + local done="$1" ts dest + ts="$(date '+%Y%m%d-%H%M%S')" + dest="$CRASH_DIR/crash-${PHASE}-done${done}-${ts}" + if [[ -f "$EVAL_LOG" ]]; then cp "$EVAL_LOG" "${dest}.eval.log" 2>/dev/null || true; fi + if [[ -f "$SERVER_LOG" ]]; then cp "$SERVER_LOG" "${dest}.server.log" 2>/dev/null || true; fi + if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi > "${dest}.nvidia-smi.txt" 2>&1 || true + fi + dmesg 2>/dev/null | grep -iE "nvrm|xid|nvidia|gpu has fallen" | tail -40 > "${dest}.dmesg.log" 2>/dev/null || true + log "Archived crash log → ${dest}.*" +} + +# ── Server lifecycle (helpers mirror eval_gemma4_31b.sh) ─────────────────────── +probe_server() { curl -s --max-time 3 "$LLAMA_URL/v1/models" 2>/dev/null; } + +# True only when the server serves the expected alias AND is past "Loading model". +server_ready() { + local resp + resp="$(probe_server)" || return 1 + [[ -z "$resp" ]] && return 1 + echo "$resp" | grep -q '"Loading model"' && return 1 + echo "$resp" | grep -q "\"$EXPECTED_ALIAS\"" +} + +kill_eval() { + pkill -9 -f "src\.project\.kele" 2>/dev/null || true + local waited=0 + while pgrep -f "src\.project\.kele" >/dev/null 2>&1 && (( waited < 20 )); do + sleep 2; (( waited += 2 )) + done +} + +kill_server() { + if [[ -n "$SERVER_PID" ]] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null || true + local waited=0 + while kill -0 "$SERVER_PID" 2>/dev/null && (( waited < 20 )); do + sleep 2; (( waited += 2 )) + done + kill -9 "$SERVER_PID" 2>/dev/null || true + fi + pkill -9 -f "llama-server" 2>/dev/null || true + SERVER_PID="" +} + +cleanup() { + log "Monitor exiting — tearing down eval client + server" + kill_eval + kill_server +} +trap cleanup EXIT INT TERM + +# CURRENT_POWER walks down from POWER_START_W (the card max) by POWER_STEP_W on +# each crash, floored at POWER_FLOOR_W. After every apply_power it is re-synced to +# the GPU's ACTUAL enforced limit, so logged wattage reflects reality even when the +# step-down is inert (no passwordless sudo → the card stays at its persisted limit). +CURRENT_POWER="" + +gpu_max_power() { + command -v nvidia-smi >/dev/null 2>&1 || { printf '130'; return; } + local m + m="$(nvidia-smi --query-gpu=power.max_limit --format=csv,noheader,nounits 2>/dev/null | head -1 | cut -d. -f1)" + [[ "$m" =~ ^[0-9]+$ ]] && printf '%s' "$m" || printf '130' +} + +# The GPU's currently ENFORCED power limit (not the card max). This is what the +# card actually runs at, which differs from CURRENT_POWER's target when the +# sudo -pl below is denied. +gpu_power_limit() { + command -v nvidia-smi >/dev/null 2>&1 || return 1 + local m + m="$(nvidia-smi --query-gpu=power.limit --format=csv,noheader,nounits 2>/dev/null | head -1 | cut -d. -f1)" + [[ "$m" =~ ^[0-9]+$ ]] && printf '%s' "$m" || return 1 +} + +# Apply CURRENT_POWER as the GPU power limit. Best-effort + non-interactive: the +# user may have to pre-authorize sudo; -n avoids a password prompt stalling the loop. +# Either way, re-sync CURRENT_POWER to the actual enforced limit so the issue-#130 +# log rows never claim a wattage the card isn't running at. +apply_power() { + command -v nvidia-smi >/dev/null 2>&1 || return 0 + [[ -z "$CURRENT_POWER" ]] && return 0 + # SELF_LOG is the monitor user's own file, not a root-only path — SC2024 N/A. + # shellcheck disable=SC2024 + if sudo -n nvidia-smi -pl "$CURRENT_POWER" >>"$SELF_LOG" 2>&1; then + log "GPU power limit set to ${CURRENT_POWER}W" + else + log "WARNING: could not set power (sudo -n failed) — run 'sudo nvidia-smi -pl ${CURRENT_POWER}' manually" + fi + local actual + actual="$(gpu_power_limit)" && CURRENT_POWER="$actual" +} + +# Step the power limit down one notch (called on each crash), not below the floor. +step_power_down() { + [[ -z "$CURRENT_POWER" ]] && return 0 + if (( CURRENT_POWER - POWER_STEP_W >= POWER_FLOOR_W )); then + CURRENT_POWER=$(( CURRENT_POWER - POWER_STEP_W )) + log "Stepping power down → ${CURRENT_POWER}W (crash-driven stability search)" + else + log "Power already at floor ${CURRENT_POWER}W (>= floor ${POWER_FLOOR_W}W) — not stepping lower" + fi +} + +boot_server() { + log "Booting llama-server ($EXPECTED_ALIAS${MTP_ENV:+ +MTP}) via $SERVE_SCRIPT" + env "${MTP_ENV[@]}" bash "$REPO_DIR/$SERVE_SCRIPT" > "$SERVER_LOG" 2>&1 & + SERVER_PID=$! + local waited=0 + while (( waited < SERVER_READY_TIMEOUT )); do + if server_ready; then + log "Server ready (~${waited}s)" + return 0 + fi + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + log "Server process exited during load — see $SERVER_LOG" + return 1 + fi + sleep 5; (( waited += 5 )) + done + log "Server failed to become ready in ${SERVER_READY_TIMEOUT}s" + return 1 +} + +start_eval() { + log "Launching eval (experiment=$EXPERIMENT → $OUT_DIR)" + # WANDB_EVAL_RUN_NAME = output-dir basename so MTP off (gemma4-12b-base) and + # MTP on (gemma4-12b-base-mtp) are distinct W&B runs despite one --experiment. + WANDB_EVAL=1 WANDB_EVAL_RUN_NAME="$(basename "$OUT_DIR")" KELE_BERT_DEVICE=cpu nohup \ + uv run --no-sync python -m src.project.kele \ + --experiment "$EXPERIMENT" evaluate \ + "${CONSULTANT_ARGS[@]}" \ + --output "$OUT_DIR" \ + "${EVAL_DATA_ARGS[@]}" \ + > "$EVAL_LOG" 2>&1 & + sleep 5 +} + +eval_running() { pgrep -f "src\.project\.kele" >/dev/null 2>&1; } + +# ── Main crawl loop ──────────────────────────────────────────────────────────── +log "Eval monitor starting: phase=$PHASE, issue #$ISSUE_NUMBER comment $LOG_COMMENT_ID, poll ${POLL_SECONDS}s, max $MAX_RETRIES retries" + +# Ensure the consultant classifier is present (the make eval targets' _classifier-ckpt dep). +# Skipped in self-consult and oracle modes — neither uses the external classifier. +if [[ "${NO_CONSULTANT:-0}" != "1" && "${ORACLE_CONSULTANT:-0}" != "1" && ! -f "$BERT_CKPT/model.safetensors" ]]; then + log "Classifier checkpoint missing — downloading" + hf download ulises-c/socrates-state-classifier-qwen3.5-lora --local-dir "$BERT_CKPT" >>"$SELF_LOG" 2>&1 || \ + log "WARNING: classifier download failed — eval will error until $BERT_CKPT exists" +fi + +[[ -z "$POWER_START_W" ]] && POWER_START_W="$(gpu_max_power)" +CURRENT_POWER="$POWER_START_W" +log "Power search: start ${POWER_START_W}W, -${POWER_STEP_W}W/crash, floor ${POWER_FLOOR_W}W" +apply_power + +TOTAL="$(dataset_total)" +log_row "$(printf '| %s | ▶ monitor start | %s | %s/%s | %s | | serving %s @ %sW; progress every %d dlg |' \ + "$(now_iso)" "$PHASE_LABEL" "$(count_valid)" "$TOTAL" "$(count_errors)" "$EXPECTED_ALIAS" "$CURRENT_POWER" "$PROGRESS_EVERY")" + +retries=0 +last_progress_valid="$(count_valid)" +last_reported_bucket=$(( last_progress_valid / PROGRESS_EVERY )) + +if ! boot_server; then + archive_crash_log "$(count_valid)" + log_row "$(printf '| %s | 🔴 crash | %s | %s/%s | %s | | server failed to boot @ %sW — %s |' \ + "$(now_iso)" "$PHASE_LABEL" "$(count_valid)" "$TOTAL" "$(count_errors)" "$CURRENT_POWER" "$(crash_signature)")" +fi +start_eval + +while true; do + valid="$(count_valid)" + errors="$(count_errors)" + TOTAL="$(dataset_total)" + + # ── Completion: valid count reached the dataset size (NOT metrics file present) + if (( valid >= TOTAL )); then + # Let the in-flight kele finish writing metrics_summary.json. + local_wait=0 + while [[ ! -f "$OUT_DIR/metrics_summary.json" ]] && (( local_wait < 60 )); do + sleep 5; (( local_wait += 5 )) + done + log "Eval COMPLETE — $valid/$TOTAL valid dialogues at ${CURRENT_POWER}W" + log_row "$(printf '| %s | ✅ COMPLETE | %s | %s/%s | %s | %s | %s ; stable @ %sW ; metrics_summary.json written |' \ + "$(now_iso)" "$PHASE_LABEL" "$valid" "$TOTAL" "$errors" "$(progress_rate)" "${OUT_DIR#"$REPO_DIR"/}" "$CURRENT_POWER")" + exit 0 + fi + + # ── Healthy and advancing: post periodic progress, keep watching ─────────── + if eval_running && server_ready; then + bucket=$(( valid / PROGRESS_EVERY )) + if (( valid > 0 )) && (( bucket > last_reported_bucket )); then + log_row "$(printf '| %s | 📈 progress | %s | %s/%s | %s | %s | %s @ %sW |' \ + "$(now_iso)" "$PHASE_LABEL" "$valid" "$TOTAL" "$errors" "$(progress_rate)" "$(progress_eta)" "$CURRENT_POWER")" + last_reported_bucket="$bucket" + fi + if (( valid > last_progress_valid )); then + last_progress_valid="$valid" + retries=0 + fi + log "running — $valid/$TOTAL valid, $errors error/corrupt" + sleep "$POLL_SECONDS" + continue + fi + + # ── Not (healthy + running). Debounce one transient unhealthy poll: a single + # slow /v1/models under power-cap shouldn't trigger a kill + cold-reload + + # (false) power step-down. Require the server to STILL be down on recheck. + if eval_running && ! server_ready; then + sleep 5 + if server_ready; then + log "Transient server blip — recovered on recheck, continuing" + continue + fi + fi + + # Classify the failure — this drives whether we touch power: + # server DOWN → GPU fault (the real hazard): kill client, step power down, reboot. + # server UP → client exited with residual content-errors (not a power fault): + # repair + relaunch against the same server, leave power alone. + if server_ready; then + server_fault=false + log "Eval client exited, server healthy ($valid/$TOTAL) — residual errors; relaunch, no power change" + else + server_fault=true + log "SERVER UNHEALTHY (confirmed) — GPU fault; killing client before it error-stamps the rest" + fi + + # Forward-progress check (counted in valid dialogues): a relaunch that landed + # new valid dialogues resets the no-progress counter, so a crawl that keeps + # inching forward never exhausts MAX_RETRIES. + if (( valid > last_progress_valid )); then + log "Progress since last crash: $last_progress_valid → $valid valid — resetting retry counter" + last_progress_valid="$valid" + retries=0 + fi + + archive_crash_log "$valid" + log "CRASH (server_fault=$server_fault, no-progress retries $retries/$MAX_RETRIES, $valid/$TOTAL valid, $errors corrupt)" + + if (( retries >= MAX_RETRIES )); then + log_row "$(printf '| %s | ⛔ STALLED | %s | %s/%s | %s | | no progress in %d retries @ %sW — manual intervention; %s |' \ + "$(now_iso)" "$PHASE_LABEL" "$valid" "$TOTAL" "$errors" "$MAX_RETRIES" "$CURRENT_POWER" "$(crash_signature)")" + log "Stalled — exiting" + exit 1 + fi + + kill_eval + repaired="$(repair_dialogues)" + log "Repaired (deleted) $repaired error/corrupt dialogue files" + retries=$(( retries + 1 )) + + if $server_fault; then + log_row "$(printf '| %s | 🔴 crash | %s | %s/%s | %s | | GPU fault; retry %d/%d @ %sW, repaired %s — %s |' \ + "$(now_iso)" "$PHASE_LABEL" "$valid" "$TOTAL" "$(count_errors)" "$((retries))" "$MAX_RETRIES" "$CURRENT_POWER" "$repaired" "$(crash_signature)")" + kill_server + # Crash-driven stability search: drop the power limit a notch before rebooting. + step_power_down + apply_power + if boot_server; then + log_row "$(printf '| %s | 🔁 resume | %s | %s/%s | %s | | server rebooted @ %sW; relaunching eval |' \ + "$(now_iso)" "$PHASE_LABEL" "$(count_valid)" "$TOTAL" "$(count_errors)" "$CURRENT_POWER")" + start_eval + else + log "Server reboot failed — will retry next loop" + sleep "$POLL_SECONDS" + fi + else + # Server still healthy — relaunch against it, no reboot, no power change. + log_row "$(printf '| %s | 🔁 resume | %s | %s/%s | %s | | content errors; retry %d/%d @ %sW, repaired %s; relaunch (server healthy) |' \ + "$(now_iso)" "$PHASE_LABEL" "$(count_valid)" "$TOTAL" "$(count_errors)" "$((retries))" "$MAX_RETRIES" "$CURRENT_POWER" "$repaired")" + start_eval + fi +done diff --git a/scripts/monitor_stage2.sh b/scripts/monitor_stage2.sh new file mode 100755 index 00000000..b6d1ba6e --- /dev/null +++ b/scripts/monitor_stage2.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# This script builds GitHub-markdown issue comments via printf; the format strings +# are single-quoted on purpose (%s positional args + literal backticks/newlines), +# so SC2016 ("expressions don't expand in single quotes") is a false positive. +# shellcheck disable=SC2016 +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_DIR="$REPO_DIR/outputs/sft-stage2-gemma4-31b" +STAGE2_LOG="$OUTPUT_DIR/train.log" +FINAL_DIR="$OUTPUT_DIR/final" +SELF_LOG="$REPO_DIR/outputs/monitor_stage2.log" +# Progress + crash reports go to the dedicated Training Log issue, not the PR — +# crash spam was flooding PR #101 (code review). See issue #120. +ISSUE_NUMBER=120 +# All events append a row to ONE pinned comment (the "Live training log" table on +# issue #120) instead of posting per-event. Bump this id when starting a brand-new +# crawl — create a fresh placeholder comment and paste its numeric id here (same +# convention as WANDB_RUN_ID). The comment body is the source of truth: log_row +# fetches it, appends a row, and PATCHes it back, so it survives monitor restarts +# and OUTPUT_DIR wipes without minting a second log comment. +LOG_COMMENT_ID=4635099006 +# Post a training-metrics progress report every this many steps (loss/grad_norm/lr). +PROGRESS_EVERY=50 +# The gfx1201 fault is non-deterministic, so the run advances by crashing and +# resuming from the latest checkpoint (save_steps=10). MAX_RETRIES bounds +# CONSECUTIVE retries that make NO forward progress — a run that keeps advancing +# never exhausts it; only a genuinely stuck GPU (no new checkpoint across this +# many tries) stops the monitor. Crawl-forward needs a high bound, not 2. +MAX_RETRIES=8 +POLL_SECONDS=300 + +# gh resolves the {owner}/{repo} api placeholders from the cwd's git remote. +cd "$REPO_DIR" || exit 1 + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$SELF_LOG" +} + +# ── Live training-log comment ──────────────────────────────────────────────── +# Readable ISO 8601 with numeric offset (not a TZ abbreviation, which isn't ISO +# and is ambiguous across hosts). +now_iso() { date '+%Y-%m-%d %H:%M:%S%:z'; } + +# One-line crash signature for the table's Note cell. A literal pipe would break +# the markdown row, so swap it for a fullwidth bar; the full trace is in crashlogs/. +crash_signature() { + [[ -f "$STAGE2_LOG" ]] || return 0 + grep -iE "memory access fault|page not present|out of memory|hip error|cuda error|runtimeerror|aborted" \ + "$STAGE2_LOG" | tail -1 | awk -F'\r' '{print $NF}' | tr -d '`' \ + | sed 's/^[^%]*%|[^]]*\][[:space:]]*//' \ + | sed 's/|/／/g' | cut -c1-140 +} + +# Append one row to the pinned comment. The comment body is the source of truth: +# fetch it, append the row, PATCH it back — so restarts/OUTPUT_DIR wipes never +# lose history or mint a second comment. Single writer (this monitor), so the +# read-modify-write has no concurrent-update race. +log_row() { + local body + body="$(gh api "repos/{owner}/{repo}/issues/comments/$LOG_COMMENT_ID" --jq '.body' 2>>"$SELF_LOG")" || { + log "WARNING: could not fetch log comment $LOG_COMMENT_ID — dropping row: $1"; return 0; } + printf '%s\n%s\n' "$body" "$1" \ + | gh api -X PATCH "repos/{owner}/{repo}/issues/comments/$LOG_COMMENT_ID" -F body=@- >> "$SELF_LOG" 2>&1 \ + || log "WARNING: gh api PATCH (log comment) failed" +} + +log_progress_from_checkpoint() { + local step="$1" epoch loss acc gn lr + local state_file="$OUTPUT_DIR/checkpoint-$step/trainer_state.json" + [[ -f "$state_file" ]] || return 0 + IFS=$'\t' read -r epoch loss acc gn lr < <(python3 - "$state_file" <<'PY' 2>/dev/null || true +import json, sys +with open(sys.argv[1]) as f: + d = json.load(f) +gs = d.get("global_step", -1) +hist = d.get("log_history", []) +m = next((e for e in reversed(hist) if e.get("step") == gs), hist[-1] if hist else {}) +print("\t".join(str(m.get(k, "")) for k in + ("epoch", "loss", "mean_token_accuracy", "grad_norm", "learning_rate"))) +PY + ) || true + local note="epoch ${epoch:-?}" + [[ -n "$acc" ]] && note="$note, acc $acc" + log_row "$(printf '| %s | 📈 checkpoint | %s | %s | %s | %s | %s |' \ + "$(now_iso)" "$step" "${loss:-}" "${gn:-}" "${lr:-}" "$note")" +} + +training_running() { + pgrep -f "train_sft\.py" > /dev/null 2>&1 +} + +# Total training steps (max_steps, e.g. 4826) from the latest checkpoint's +# trainer_state.json — stable for the run and independent of which tqdm bars are +# in the log, so the fallback denominator is never a Map-bar total. +train_total_steps() { + local step state + step="$(latest_ckpt_step)" + if (( step < 0 )); then return 0; fi + state="$OUTPUT_DIR/checkpoint-$step/trainer_state.json" + [[ -f "$state" ]] || return 0 + python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("max_steps",""))' \ + "$state" 2>/dev/null || true +} + +last_step() { + [[ -f "$STAGE2_LOG" ]] || return 0 + # train.log holds THREE N/M tqdm bars: the training step bar (N/4826) and the + # one-time dataset tokenization Map bars (N/77202 train, N/8578 eval). The old + # "$1 != $2" filter only dropped 100%-complete bars, so an in-progress Map bar + # (e.g. 12868/77202) leaked straight into the Step column — and on a relaunch + # that faults during load (train.log is truncated each launch), the Map bars + # are the ONLY N/M present. Anchor on the rate unit instead: at ~70 s/step the + # training bar is the only one rendered as "s/it" (Map → "examples/s", W&B + # upload → "MB/s", shard-load → "it/s"). tr '\r'->'\n' splits the \r-joined + # tqdm updates so the most recent step wins. + local n ckpt total + n="$(tr '\r' '\n' < "$STAGE2_LOG" | grep -E 's/it\]' | grep -oE '[0-9]+/[0-9]+' | tail -1 || true)" + if [[ -n "$n" ]]; then printf '%s' "$n"; return 0; fi + # No training bar yet (crash during model load / tokenization) — degrade to the + # latest checkpoint step rather than a Map-bar denominator or a blank cell. + ckpt="$(latest_ckpt_step)" + if (( ckpt < 0 )); then return 0; fi + total="$(train_total_steps)" + printf '%s%s' "$ckpt" "${total:+/$total}" +} + +crash_hint() { + [[ -f "$STAGE2_LOG" ]] || return 0 + # The gfx1201 fault prints "Memory access fault by GPU node-1 ... Page not present" + # and aborts with "Aborted (core dumped)" — it is NOT the literal string "page fault" + # and emits no Python traceback, so the old pattern matched nothing and the PR block + # came up empty. Match the real ROCm/HSA signature, and lead with the training step + # (via last_step, the single parser) so the crash post shows the fault step directly. + local step + step="$(last_step)" + [[ -n "$step" ]] && printf ' fault near step %s\n' "$step" + grep -iE "memory access fault|page not present|hsa_status|aborted|core dumped|out of memory|hip error|cuda error|traceback|runtimeerror" \ + "$STAGE2_LOG" | tail -4 | sed 's/^/ /' || true +} + +# The make target writes train.log with `>` (truncate), so each relaunch would +# overwrite the crashed run's full traceback. Archive the whole log + a dmesg +# snapshot per crash BEFORE relaunch so an overnight crawl leaves a reviewable +# trail of every fault, not just the 3-line hint. +archive_crash_log() { + local ckpt="$1" ts dest crashdir + [[ -f "$STAGE2_LOG" ]] || return 0 + ts="$(date '+%Y%m%d-%H%M%S')" + crashdir="$OUTPUT_DIR/crashlogs" + mkdir -p "$crashdir" + dest="$crashdir/crash-step${ckpt}-${ts}.log" + if cp "$STAGE2_LOG" "$dest" 2>/dev/null; then + log "Archived full crash log → $dest" + fi + dmesg 2>/dev/null \ + | grep -iE "amdgpu|gfxhub|VM_L2|page fault|PERMISSION_FAULTS|WALKER_ERROR" \ + | tail -40 > "$crashdir/dmesg-step${ckpt}-${ts}.log" 2>/dev/null || true +} + +kill_and_clean() { + log "Killing stray train_sft.py processes" + pkill -9 -f "train_sft\.py" 2>/dev/null || true + local waited=0 + while pgrep -f "train_sft\.py" > /dev/null 2>&1 && [[ $waited -lt 30 ]]; do + sleep 2; (( waited += 2 )) + done + # A gfx1201 fault leaves the amdkfd dirty (orphaned HIP context + stale VRAM); + # relaunching into that state faults early on stale PTEs (the cascade that + # turned a single crash into a permanent loop). Verify the GPU is ACTUALLY + # clean before relaunch instead of a blind sleep (GFX1201_RDNA4_TRAINING.md §10). + if ! bash "$REPO_DIR/scripts/test_gpu_stack.sh" --wait-clean 180 >> "$SELF_LOG" 2>&1; then + log "WARNING: GPU still dirty after 180s — relaunch will likely fault on stale PTEs" + fi +} + +latest_ckpt_step() { + local d max=-1 n + for d in "$OUTPUT_DIR"/checkpoint-*; do + [[ -d "$d" ]] || continue + n="${d##*checkpoint-}" + [[ "$n" =~ ^[0-9]+$ ]] && (( n > max )) && max="$n" + done + printf '%s' "$max" +} + +# A crash landing during a checkpoint write leaves an incomplete checkpoint-N. +# HF resume always picks the highest-numbered dir, so a partial one makes every +# resume fail to load and loop on the same bad checkpoint. trainer_state.json is +# written last in _save_checkpoint, so a valid one implies the rest is complete; +# if it's missing or unparsable, quarantine the dir so resume falls back to N-1. +quarantine_bad_checkpoint() { + local step latest + step="$(latest_ckpt_step)" + [[ "$step" -lt 0 ]] && { log "No checkpoint yet — resume will start from step 0"; return 0; } + latest="$OUTPUT_DIR/checkpoint-$step" + if [[ -f "$latest/trainer_state.json" ]] \ + && python3 -c "import json,sys; json.load(open(sys.argv[1]))" "$latest/trainer_state.json" 2>/dev/null; then + log "Latest checkpoint OK: checkpoint-$step (resume target)" + return 0 + fi + log "checkpoint-$step is INCOMPLETE (crash mid-save) — quarantining; resume falls back to prior" + mv "$latest" "$OUTPUT_DIR/.broken-checkpoint-$step" 2>/dev/null || rm -rf "$latest" +} + +start_stage2() { + # Use a fresh data_seed each launch so resume presents different samples at each + # step — breaking the data-order-sticky fault confirmed in run #13 (PR #101). + # Verified (run #14 verif): TRAIN_DATA_SEED genuinely reshuffles the post-resume + # order and step 22 (previously the sticky fault step) completed clean under seed 99. + # Unix timestamp gives a distinct seed every resume (each cycle is ≥10 min apart). + local data_seed + data_seed="$(date +%s)" + log "Starting Stage 2 training (data_seed=$data_seed, gpu-preflight gates the launch)" + cd "$REPO_DIR" || return 0 + TRAIN_DATA_SEED="$data_seed" WANDB_PROJECT=csen346-sft make train-gemma4-31b-stage2-unsloth \ + || log "launch aborted (gpu-preflight failed?) — counts as no forward progress" + sleep 30 +} + +log "Stage 2 monitor starting (issue #$ISSUE_NUMBER comment $LOG_COMMENT_ID, poll every ${POLL_SECONDS}s, max $MAX_RETRIES retries)" +log_row "$(printf '| %s | ▶ monitor start | %s | | | | watching train.log; progress every %s steps |' \ + "$(now_iso)" "$(last_step || true)" "$PROGRESS_EVERY")" + +sleep 30 # let it get going before first poll + +retries=0 +last_progress_step="$(latest_ckpt_step)" +last_reported_ckpt="$(latest_ckpt_step)" + +while true; do + if training_running; then + step="$(last_step)" + log "Stage 2 running${step:+ — step $step}" + ckpt_now="$(latest_ckpt_step)" + if (( ckpt_now > 0 )) && (( ckpt_now % PROGRESS_EVERY == 0 )) \ + && (( ckpt_now > last_reported_ckpt )); then + log_progress_from_checkpoint "$ckpt_now" + last_reported_ckpt="$ckpt_now" + fi + sleep "$POLL_SECONDS" + continue + fi + + if [[ -d "$FINAL_DIR" ]]; then + log "Stage 2 complete — adapter at $FINAL_DIR" + total_steps="$(last_step || true)" + log_row "$(printf '| %s | ✅ COMPLETE | %s | | | | adapter: outputs/sft-stage2-gemma4-31b/final |' \ + "$(now_iso)" "${total_steps:-?}")" + log "Stage 2 done — monitor exiting" + exit 0 + fi + + # Forward-progress check: did a new checkpoint land since the last crash? If + # so the crawl is advancing — reset the no-progress retry counter. Only + # consecutive stalls (no new checkpoint) count toward MAX_RETRIES, so a run + # that keeps inching forward across faults never gives up. + ckpt_step="$(latest_ckpt_step)" + if (( ckpt_step > last_progress_step )); then + log "Progress since last crash: checkpoint $last_progress_step → $ckpt_step — resetting retry counter" + last_progress_step="$ckpt_step" + retries=0 + fi + + hint="$(crash_hint)" + log "CRASH DETECTED (consecutive no-progress retries $retries/$MAX_RETRIES, latest ckpt step $ckpt_step)" + [[ -n "$hint" ]] && log "Log tail:$hint" + # Preserve the full traceback + dmesg BEFORE relaunch overwrites train.log. + archive_crash_log "$ckpt_step" + + if (( retries >= MAX_RETRIES )); then + log_row "$(printf '| %s | ⛔ STALLED | %s | | | | no progress in %d retries — manual intervention (make diagnose-gfx1201-fault); last good ckpt %s — %s |' \ + "$(now_iso)" "$(last_step || true)" "$MAX_RETRIES" "$ckpt_step" "$(crash_signature)")" + log "Stalled — no forward progress in $MAX_RETRIES retries — exiting" + exit 1 + fi + + log_row "$(printf '| %s | 🔴 crash | %s | | | | retry %d/%d, last good ckpt %s — %s |' \ + "$(now_iso)" "$(last_step || true)" "$((retries+1))" "$MAX_RETRIES" "$ckpt_step" "$(crash_signature)")" + + retries=$(( retries + 1 )) + kill_and_clean + quarantine_bad_checkpoint + start_stage2 +done diff --git a/scripts/monitor_train_gemma4_12b.sh b/scripts/monitor_train_gemma4_12b.sh new file mode 100755 index 00000000..5136187f --- /dev/null +++ b/scripts/monitor_train_gemma4_12b.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +# Training-side auto-resume monitor for the Gemma 4 12B SFT-uplift PoC on the +# NVIDIA RTX 4000 Ada box. Adapted from scripts/monitor_stage2.sh (the gfx1201/ +# AMD 31B training monitor) — same crawl-forward-by-crashing semantics, but: +# +# 1. NVIDIA box, not gfx1201. Crash signatures + dmesg patterns are the CUDA/NVRM +# ones (xid, "fallen off the bus", cuda error), not the ROCm/HSA ones. The +# GPU-clean gate is `nvidia-preflight` — which `make train-gemma4-12b` runs as +# a hard prerequisite on EVERY (re)launch — plus a VRAM-settle wait after kill, +# instead of the AMD test_gpu_stack.sh --wait-clean path. +# 2. Adaptive power search (memory: training-host-hardware-fault — this card took a +# power surge and crashes under load). Training is compute-bound, so it lives in +# exactly the regime that historically faulted. Start at the card max and step +# the power limit DOWN on each crash until the run stops faulting; the highest +# power that crawls to completion is the best stability/speed point. Same +# mechanism as scripts/monitor_eval_gemma4_12b.sh. +# +# Forward progress is counted in CHECKPOINTS (save_steps=50): a relaunch that lands +# a new checkpoint resets the no-progress retry counter, so a run that keeps inching +# forward across faults never exhausts MAX_RETRIES. COMPLETE means outputs/.../final +# exists. +# +# This script builds GitHub-markdown rows via printf; the single-quoted format +# strings (%s args + literal backticks/pipes) make SC2016 a false positive. +# shellcheck disable=SC2016 +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" || exit 1 + +OUTPUT_DIR="${TRAIN_OUTPUT_DIR:-$REPO_DIR/outputs/sft-gemma4-12b-qlora}" +MAKE_TARGET="${TRAIN_MAKE_TARGET:-train-gemma4-12b}" +TRAIN_LOG="$OUTPUT_DIR/train.log" +FINAL_DIR="$OUTPUT_DIR/final" +SELF_LOG="$REPO_DIR/outputs/monitor_train_gemma4_12b.log" + +# Progress + crash rows append to ONE pinned comment on the SFT PoC issue #130 +# (same convention as monitor_stage2.sh / monitor_eval_gemma4_12b.sh). Create a +# fresh placeholder comment and export its numeric id (or paste it here). If empty, +# log_row degrades to local-only logging — the crawl still runs, it just doesn't +# post to GitHub. +ISSUE_NUMBER="${ISSUE_NUMBER:-130}" +LOG_COMMENT_ID="${LOG_COMMENT_ID:-}" +# Post a training-metrics progress row every this many steps (loss/grad_norm/lr). +# Matches save_steps so every checkpoint is reported. +PROGRESS_EVERY="${PROGRESS_EVERY:-50}" +# Bound on CONSECUTIVE no-forward-progress retries — a run that keeps advancing +# never exhausts it; only a genuinely stuck GPU (no new checkpoint across this many +# tries) stops the monitor. Crawl-forward needs a high bound, not 2. +MAX_RETRIES="${MAX_RETRIES:-8}" +POLL_SECONDS="${POLL_SECONDS:-300}" + +# Adaptive power search: start high, step DOWN POWER_STEP_W on each crash, floored +# at POWER_FLOOR_W. POWER_START_W defaults to the card's reported max limit. +POWER_START_W="${POWER_START_W:-}" +POWER_STEP_W="${POWER_STEP_W:-10}" +POWER_FLOOR_W="${POWER_FLOOR_W:-70}" # eval crawls proved stable at 85W; floor below that + +# After a kill, wait for VRAM to drain before relaunch so training doesn't boot into +# a dirty allocator state. Bounded; the nvidia-preflight gate is the real health check. +VRAM_SETTLE_TIMEOUT="${VRAM_SETTLE_TIMEOUT:-120}" +VRAM_SETTLE_MIB="${VRAM_SETTLE_MIB:-1500}" + +mkdir -p "$OUTPUT_DIR" "$REPO_DIR/outputs" + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$SELF_LOG" +} +now_iso() { date '+%Y-%m-%d %H:%M:%S%:z'; } + +# ── Live training-log comment ──────────────────────────────────────────────── +# Append one row to the pinned comment. Single writer (this monitor), so the +# read-modify-write has no concurrent-update race. gh resolves {owner}/{repo} +# from the cwd's git remote. No-op (local log only) when LOG_COMMENT_ID is unset. +log_row() { + [[ -z "$LOG_COMMENT_ID" ]] && { log "row (no GH comment): $1"; return 0; } + local body + body="$(gh api "repos/{owner}/{repo}/issues/comments/$LOG_COMMENT_ID" --jq '.body' 2>>"$SELF_LOG")" || { + log "WARNING: could not fetch log comment $LOG_COMMENT_ID — dropping row: $1"; return 0; } + printf '%s\n%s\n' "$body" "$1" \ + | gh api -X PATCH "repos/{owner}/{repo}/issues/comments/$LOG_COMMENT_ID" -F body=@- >> "$SELF_LOG" 2>&1 \ + || log "WARNING: gh api PATCH (log comment) failed" +} + +# One-line crash signature for the table's Note cell. A literal pipe would break the +# markdown row, so swap it for a fullwidth bar; the full trace is in crashlogs/. +crash_signature() { + [[ -f "$TRAIN_LOG" ]] || return 0 + grep -iE "cuda error|out of memory|fallen off the bus|xid|nvrm|segmentation fault|sigsegv|aborted|core dumped|runtimeerror" \ + "$TRAIN_LOG" | tail -1 | awk -F'\r' '{print $NF}' | tr -d '`' \ + | sed 's/|/／/g' | cut -c1-140 +} + +log_progress_from_checkpoint() { + local step="$1" epoch loss acc gn lr + local state_file="$OUTPUT_DIR/checkpoint-$step/trainer_state.json" + [[ -f "$state_file" ]] || return 0 + IFS=$'\t' read -r epoch loss acc gn lr < <(python3 - "$state_file" <<'PY' 2>/dev/null || true +import json, sys +with open(sys.argv[1]) as f: + d = json.load(f) +gs = d.get("global_step", -1) +hist = d.get("log_history", []) +m = next((e for e in reversed(hist) if e.get("step") == gs), hist[-1] if hist else {}) +print("\t".join(str(m.get(k, "")) for k in + ("epoch", "loss", "mean_token_accuracy", "grad_norm", "learning_rate"))) +PY + ) || true + local note="epoch ${epoch:-?} @ ${CURRENT_POWER:-?}W" + [[ -n "$acc" ]] && note="$note, acc $acc" + log_row "$(printf '| %s | 📈 checkpoint | %s | %s | %s | %s | %s |' \ + "$(now_iso)" "$step" "${loss:-}" "${gn:-}" "${lr:-}" "$note")" +} + +training_running() { + pgrep -f "train_sft\.py" > /dev/null 2>&1 +} + +# Total training steps (max_steps, e.g. 4826) from the latest checkpoint's +# trainer_state.json — stable for the run and independent of which tqdm bars are in +# the log, so the fallback denominator is never a Map-bar total. +train_total_steps() { + local step state + step="$(latest_ckpt_step)" + if (( step < 0 )); then return 0; fi + state="$OUTPUT_DIR/checkpoint-$step/trainer_state.json" + [[ -f "$state" ]] || return 0 + python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("max_steps",""))' \ + "$state" 2>/dev/null || true +} + +last_step() { + [[ -f "$TRAIN_LOG" ]] || return 0 + # train.log holds multiple N/M tqdm bars: the training step bar (N/4826) and the + # one-time dataset tokenization Map bars. Anchor on the rate unit: at many s/step + # the training bar is the only one rendered as "s/it" (Map → "examples/s", W&B + # upload → "MB/s", shard-load → "it/s"). tr '\r'->'\n' splits the \r-joined tqdm + # updates so the most recent step wins. + local n ckpt total + n="$(tr '\r' '\n' < "$TRAIN_LOG" | grep -E 's/it\]' | grep -oE '[0-9]+/[0-9]+' | tail -1 || true)" + if [[ -n "$n" ]]; then printf '%s' "$n"; return 0; fi + # No training bar yet (crash during model load / tokenization) — degrade to the + # latest checkpoint step rather than a Map-bar denominator or a blank cell. + ckpt="$(latest_ckpt_step)" + if (( ckpt < 0 )); then return 0; fi + total="$(train_total_steps)" + printf '%s%s' "$ckpt" "${total:+/$total}" +} + +crash_hint() { + [[ -f "$TRAIN_LOG" ]] || return 0 + local step + step="$(last_step)" + [[ -n "$step" ]] && printf ' fault near step %s\n' "$step" + grep -iE "cuda error|out of memory|fallen off the bus|xid|nvrm|segmentation fault|sigsegv|aborted|core dumped|traceback|runtimeerror" \ + "$TRAIN_LOG" | tail -4 | sed 's/^/ /' || true +} + +# An OOM is DETERMINISTIC — the same config will OOM identically every relaunch, so +# retrying (and stepping power down, which OOM ignores) is futile. Distinguished +# from the probabilistic GPU surge fault this monitor exists to crawl through. +is_oom_crash() { + [[ -f "$TRAIN_LOG" ]] || return 1 + grep -qiE "outofmemoryerror|cuda out of memory|torch\.cuda\.OutOfMemory" "$TRAIN_LOG" +} + +# The make target writes train.log with `>` (truncate), so each relaunch would +# overwrite the crashed run's full traceback. Archive the whole log + an nvidia-smi +# + dmesg snapshot per crash BEFORE relaunch so an overnight crawl leaves a +# reviewable trail of every fault, not just the 3-line hint. +archive_crash_log() { + local ckpt="$1" ts dest crashdir + [[ -f "$TRAIN_LOG" ]] || return 0 + ts="$(date '+%Y%m%d-%H%M%S')" + crashdir="$OUTPUT_DIR/crashlogs" + mkdir -p "$crashdir" + dest="$crashdir/crash-step${ckpt}-${ts}.log" + if cp "$TRAIN_LOG" "$dest" 2>/dev/null; then + log "Archived full crash log → $dest" + fi + if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi > "$crashdir/nvidia-smi-step${ckpt}-${ts}.txt" 2>&1 || true + fi + dmesg 2>/dev/null \ + | grep -iE "nvrm|xid|nvidia|gpu has fallen" \ + | tail -40 > "$crashdir/dmesg-step${ckpt}-${ts}.log" 2>/dev/null || true +} + +# Wait for VRAM to drain after a kill, so the relaunch doesn't boot into a dirty +# allocator state. Bounded; nvidia-preflight (a make prerequisite) is the real gate. +wait_gpu_idle() { + command -v nvidia-smi >/dev/null 2>&1 || return 0 + local waited=0 used + while (( waited < VRAM_SETTLE_TIMEOUT )); do + used="$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" + [[ "$used" =~ ^[0-9]+$ ]] || break + if (( used < VRAM_SETTLE_MIB )); then + log "GPU VRAM settled (${used} MiB used) after ${waited}s" + return 0 + fi + sleep 5; (( waited += 5 )) + done + log "WARNING: GPU VRAM still high after ${VRAM_SETTLE_TIMEOUT}s — relaunch anyway (preflight gates it)" +} + +kill_and_clean() { + log "Killing stray train_sft.py processes" + pkill -9 -f "train_sft\.py" 2>/dev/null || true + local waited=0 + while pgrep -f "train_sft\.py" > /dev/null 2>&1 && [[ $waited -lt 30 ]]; do + sleep 2; (( waited += 2 )) + done + wait_gpu_idle +} + +latest_ckpt_step() { + local d max=-1 n + for d in "$OUTPUT_DIR"/checkpoint-*; do + [[ -d "$d" ]] || continue + n="${d##*checkpoint-}" + [[ "$n" =~ ^[0-9]+$ ]] && (( n > max )) && max="$n" + done + printf '%s' "$max" +} + +# A crash landing during a checkpoint write leaves an incomplete checkpoint-N. HF +# resume always picks the highest-numbered dir, so a partial one makes every resume +# fail to load and loop on the same bad checkpoint. trainer_state.json is written +# last in _save_checkpoint, so a valid one implies the rest is complete; if it's +# missing or unparsable, quarantine the dir so resume falls back to N-1. +quarantine_bad_checkpoint() { + local step latest + step="$(latest_ckpt_step)" + [[ "$step" -lt 0 ]] && { log "No checkpoint yet — resume will start from step 0"; return 0; } + latest="$OUTPUT_DIR/checkpoint-$step" + if [[ -f "$latest/trainer_state.json" ]] \ + && python3 -c "import json,sys; json.load(open(sys.argv[1]))" "$latest/trainer_state.json" 2>/dev/null; then + log "Latest checkpoint OK: checkpoint-$step (resume target)" + return 0 + fi + log "checkpoint-$step is INCOMPLETE (crash mid-save) — quarantining; resume falls back to prior" + mv "$latest" "$OUTPUT_DIR/.broken-checkpoint-$step" 2>/dev/null || rm -rf "$latest" +} + +# ── GPU power control (the adaptive stability search) ────────────────────────── +CURRENT_POWER="" + +gpu_max_power() { + command -v nvidia-smi >/dev/null 2>&1 || { printf '130'; return; } + local m + m="$(nvidia-smi --query-gpu=power.max_limit --format=csv,noheader,nounits 2>/dev/null | head -1 | cut -d. -f1)" + [[ "$m" =~ ^[0-9]+$ ]] && printf '%s' "$m" || printf '130' +} + +apply_power() { + command -v nvidia-smi >/dev/null 2>&1 || return 0 + [[ -z "$CURRENT_POWER" ]] && return 0 + # SELF_LOG is the monitor user's own file, not a root-only path — SC2024 N/A. + # shellcheck disable=SC2024 + if sudo -n nvidia-smi -pl "$CURRENT_POWER" >>"$SELF_LOG" 2>&1; then + log "GPU power limit set to ${CURRENT_POWER}W" + else + log "WARNING: could not set power (sudo -n failed) — run 'sudo nvidia-smi -pl ${CURRENT_POWER}' manually" + fi +} + +step_power_down() { + [[ -z "$CURRENT_POWER" ]] && return 0 + if (( CURRENT_POWER - POWER_STEP_W >= POWER_FLOOR_W )); then + CURRENT_POWER=$(( CURRENT_POWER - POWER_STEP_W )) + log "Stepping power down → ${CURRENT_POWER}W (crash-driven stability search)" + else + log "Power already at floor ${CURRENT_POWER}W (>= floor ${POWER_FLOOR_W}W) — not stepping lower" + fi +} + +start_training() { + # Fresh data_seed each launch so resume presents a different sample order at each + # step — breaks any data-order-sticky fault (the gfx1201 31B run hit one; cheap + # insurance here). train_sft.py reads TRAIN_DATA_SEED; make train-gemma4-12b runs + # nvidia-preflight first (hard GPU gate) then nohups training into train.log. + local data_seed + data_seed="$(date +%s)" + log "Starting training (data_seed=$data_seed, power ${CURRENT_POWER:-default}W, nvidia-preflight gates the launch)" + apply_power + cd "$REPO_DIR" || return 0 + TRAIN_DATA_SEED="$data_seed" make "$MAKE_TARGET" >> "$SELF_LOG" 2>&1 \ + || log "launch aborted (nvidia-preflight failed?) — counts as no forward progress" + sleep 30 +} + +log "Training monitor starting (issue #$ISSUE_NUMBER comment ${LOG_COMMENT_ID:-}, poll every ${POLL_SECONDS}s, max $MAX_RETRIES retries)" + +[[ -z "$POWER_START_W" ]] && POWER_START_W="$(gpu_max_power)" +CURRENT_POWER="$POWER_START_W" +log "Power search: start ${POWER_START_W}W, -${POWER_STEP_W}W/crash, floor ${POWER_FLOOR_W}W" + +log_row "$(printf '| %s | ▶ monitor start | %s | | | | watching train.log @ %sW; progress every %s steps |' \ + "$(now_iso)" "$(last_step || true)" "$CURRENT_POWER" "$PROGRESS_EVERY")" + +# Launch (or resume) immediately rather than waiting a poll cycle. +start_training + +retries=0 +last_progress_step="$(latest_ckpt_step)" +last_reported_ckpt="$(latest_ckpt_step)" + +while true; do + if training_running; then + step="$(last_step)" + log "Training running${step:+ — step $step}" + ckpt_now="$(latest_ckpt_step)" + if (( ckpt_now > 0 )) && (( ckpt_now % PROGRESS_EVERY == 0 )) \ + && (( ckpt_now > last_reported_ckpt )); then + log_progress_from_checkpoint "$ckpt_now" + last_reported_ckpt="$ckpt_now" + fi + sleep "$POLL_SECONDS" + continue + fi + + if [[ -d "$FINAL_DIR" ]]; then + log "Training complete — adapter at $FINAL_DIR" + log_row "$(printf '| %s | ✅ COMPLETE | %s | | | | adapter: %s ; stable @ %sW |' \ + "$(now_iso)" "$(last_step || true)" "${FINAL_DIR#"$REPO_DIR"/}" "$CURRENT_POWER")" + log "Done — monitor exiting" + exit 0 + fi + + # Forward-progress check: a new checkpoint since the last crash means the crawl + # is advancing — reset the no-progress retry counter. Only consecutive stalls + # (no new checkpoint) count toward MAX_RETRIES. + ckpt_step="$(latest_ckpt_step)" + if (( ckpt_step > last_progress_step )); then + log "Progress since last crash: checkpoint $last_progress_step → $ckpt_step — resetting retry counter" + last_progress_step="$ckpt_step" + retries=0 + fi + + hint="$(crash_hint)" + log "CRASH DETECTED (consecutive no-progress retries $retries/$MAX_RETRIES, latest ckpt step $ckpt_step, power ${CURRENT_POWER}W)" + [[ -n "$hint" ]] && log "Log tail:$hint" + archive_crash_log "$ckpt_step" + + # OOM before any checkpoint = the config does not fit in VRAM. Deterministic; + # relaunching/stepping power never helps. Bail immediately with a clear cause + # instead of burning MAX_RETRIES on the identical failure. + if (( ckpt_step < 0 )) && is_oom_crash; then + log_row "$(printf '| %s | ⛔ OOM | %s | | | | config exceeds VRAM at load/step 0 (not a surge fault) — fix config, do not retry; %s |' \ + "$(now_iso)" "$(last_step || true)" "$(crash_signature)")" + log "OOM before any checkpoint — config does not fit in VRAM; retrying is futile — exiting" + exit 1 + fi + + if (( retries >= MAX_RETRIES )); then + log_row "$(printf '| %s | ⛔ STALLED | %s | | | | no progress in %d retries @ %sW — manual intervention; last good ckpt %s — %s |' \ + "$(now_iso)" "$(last_step || true)" "$MAX_RETRIES" "$CURRENT_POWER" "$ckpt_step" "$(crash_signature)")" + log "Stalled — no forward progress in $MAX_RETRIES retries — exiting" + exit 1 + fi + + log_row "$(printf '| %s | 🔴 crash | %s | | | | retry %d/%d @ %sW, last good ckpt %s — %s |' \ + "$(now_iso)" "$(last_step || true)" "$((retries+1))" "$MAX_RETRIES" "$CURRENT_POWER" "$ckpt_step" "$(crash_signature)")" + + retries=$(( retries + 1 )) + kill_and_clean + quarantine_bad_checkpoint + # Crash-driven stability search: drop the power limit a notch before relaunch. + step_power_down + start_training +done diff --git a/scripts/noconsult_chain_gemma4_12b.sh b/scripts/noconsult_chain_gemma4_12b.sh new file mode 100755 index 00000000..c48b76d2 --- /dev/null +++ b/scripts/noconsult_chain_gemma4_12b.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Weekend chain for the consultant ablation (handoff T1.1): run both no-consultant +# (self-consult) arms back-to-back, one model at a time on the 20 GB card. SFT first +# (fast, terminates cleanly), then base (~30 h — self-consult is 2 LLM calls/turn). +# +# Each arm is the crash-resilient monitor, which OWNS its own llama-server: it boots +# the server, walks the GPU power limit down per fault, auto-resumes, and its EXIT +# trap kills the server on completion — so the GPU is free before the next arm boots. +# Outputs: results/gemma4-12b-{sft,base}-noconsult. Monitor self-logs per phase to +# outputs/monitor_eval_gemma4_12b_{sft,base}.log; this wrapper logs the chain steps. +set -uo pipefail # NOT -e: a stalled/failed first arm must not abort the second. + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" || exit 1 +mkdir -p "$REPO_DIR/outputs" +LOG="$REPO_DIR/outputs/noconsult_chain.log" + +clog() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S%:z')" "$*" | tee -a "$LOG"; } + +run_arm() { + local phase="$1" + clog "=== START ${phase} no-consultant arm → results/gemma4-12b-${phase}-noconsult ===" + NO_CONSULTANT=1 bash "$REPO_DIR/scripts/monitor_eval_gemma4_12b.sh" "$phase" + local rc=$? + clog "=== END ${phase} arm (rc=${rc}) ===" + return "$rc" +} + +clog "##### no-consultant chain begin (sft → base) #####" + +run_arm sft || clog "WARNING: sft arm exited non-zero — proceeding to base anyway." + +# Settle so the SFT server is fully reaped before base boots on the shared 20 GB card. +sleep 20 + +run_arm base || clog "WARNING: base arm exited non-zero." + +clog "##### no-consultant chain done #####" diff --git a/scripts/oracle_chain_gemma4_12b.sh b/scripts/oracle_chain_gemma4_12b.sh new file mode 100755 index 00000000..d9f1d29e --- /dev/null +++ b/scripts/oracle_chain_gemma4_12b.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Chain for the oracle-consultant ablation (handoff T1.1 follow-on): run both +# oracle arms back-to-back, one model at a time on the 20 GB card. SFT first +# (~3 h), then base (~17 h). Oracle = the GT state is fed each turn (1 teacher +# call/turn, as fast as the classifier path — NOT the 2-call self-consult cost). +# +# state_accuracy is ~perfect by construction here, so compare the arms on +# ROUGE/BLEU — this isolates PURE teacher-turn quality given correct state, with +# classifier accuracy removed as a confound (the confound-free ceiling on "does +# the SFT write better Socratic turns?"). +# +# Each arm is the crash-resilient monitor, which OWNS its own llama-server: it +# boots the server, walks the GPU power limit down per fault, auto-resumes, and +# its EXIT trap kills the server on completion — so the GPU is free before the +# next arm boots. Outputs: results/gemma4-12b-{sft,base}-oracle. Monitor +# self-logs per phase to outputs/monitor_eval_gemma4_12b_{sft,base}.log; this +# wrapper logs the chain steps. +set -uo pipefail # NOT -e: a stalled/failed first arm must not abort the second. + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" || exit 1 +mkdir -p "$REPO_DIR/outputs" +LOG="$REPO_DIR/outputs/oracle_chain.log" + +clog() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S%:z')" "$*" | tee -a "$LOG"; } + +run_arm() { + local phase="$1" + clog "=== START ${phase} oracle arm → results/gemma4-12b-${phase}-oracle ===" + ORACLE_CONSULTANT=1 bash "$REPO_DIR/scripts/monitor_eval_gemma4_12b.sh" "$phase" + local rc=$? + clog "=== END ${phase} arm (rc=${rc}) ===" + return "$rc" +} + +clog "##### oracle chain begin (sft → base) #####" + +run_arm sft || clog "WARNING: sft arm exited non-zero — proceeding to base anyway." + +# Settle so the SFT server is fully reaped before base boots on the shared 20 GB card. +sleep 20 + +run_arm base || clog "WARNING: base arm exited non-zero." + +clog "##### oracle chain done #####" diff --git a/scripts/probe_gfx1201_tile.py b/scripts/probe_gfx1201_tile.py new file mode 100644 index 00000000..1adecb82 --- /dev/null +++ b/scripts/probe_gfx1201_tile.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +Tile-selection probe for the gfx1201 ISA1201 Tensile GEMM page fault. + +Goal: determine which combination of input shape and call path causes Tensile +to dispatch the specific faulting kernel: + Cijk_Ailk_Bjlk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x64_…_ISA1201_…_DTVB1_… + +No VA filler — just a few iterations to capture ShaderName. Run each config and +grep the log for MT64x64x64.*DTVB1. + +WHY THE DEFAULT M IS 607 (not 608): the SYNC-deterministic capture +(diag-sync-probe-step1230, HIP_LAUNCH_BLOCKING=1) logs the *faulting* GEMM's A +operand as (1, 607, 21504) — M=607, K=21504, B col-major stride=(1,21504). M is +the current sample's token length and is data-order-dependent (439 and 607 both +observed in SYNC runs). Every earlier probe round fixed M=608 (the even neighbor, +captured under async logging) and only ever dispatched MT128x128x32 — so the +standalone-vs-production tile divergence may simply be that 608 was the wrong M. +Tensile size predicates can flip MT128x128x32 ↔ MT64x64x64 across an M boundary, +so the first thing to do is sweep M over the real faulting values and neighbors. + +Run matrix: + # PRIMARY: sweep M over the SYNC-confirmed faulting values + neighbors in one run. + # Find which M dispatches MT64x64x64 — that is the standalone-repro shape. + AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1 AMD_LOG_LEVEL=3 TORCH_USE_HIPBLASLT=0 \ + uv run --no-sync python scripts/probe_gfx1201_tile.py --shape-3d --ckpt \ + --m-sweep 439,512,576,606,607,608,609,640 2>&1 | tee probe_msweep.log + + # 2D plain (baseline — already confirmed at M=608: dispatches MT128x128x32) + ... python scripts/probe_gfx1201_tile.py --m 608 2>&1 | tee probe_2d_plain.log + + # 3D + ckpt(reentrant=False) [most faithful to production], default M=607 + ... python scripts/probe_gfx1201_tile.py --shape-3d --ckpt 2>&1 | tee probe_3d_ckpt_nr.log + + # SECONDARY: drop the bias epilogue (faulting kernel is …_Bias_…_UserArgs_…). + # If MT64x64x64 appears only with bias, the trigger is the fused epilogue. + ... python scripts/probe_gfx1201_tile.py --shape-3d --ckpt --no-bias 2>&1 | tee probe_nobias.log + +Detect which tile each M dispatched (associate ShaderName with the SWEEP banner): + grep -E 'SWEEP M=|MT[0-9]+x[0-9]+x[0-9]+.*ISA1201' probe_msweep.log \ + | grep -oE 'SWEEP M=[0-9]+|MT[0-9]+x[0-9]+x[0-9]+' + # Success == any 'MT64x64x64' line appears under a 'SWEEP M=' banner. + +Hardware: AMD Radeon AI PRO R9700, gfx1201 (RDNA4), 32 GB VRAM +Software: ROCm 7.2, torch 2.11.0+rocm7.2, bitsandbytes 0.49.2 +Upstream: github.com/ROCm/rocm-libraries/issues/7992 +""" + +import argparse +import contextlib +import os +import sys + +os.environ.setdefault("AMD_SERIALIZE_KERNEL", "3") +os.environ.setdefault("HIP_LAUNCH_BLOCKING", "1") +os.environ.setdefault("AMD_LOG_LEVEL", "3") +os.environ.setdefault("TORCH_USE_HIPBLASLT", "0") + +import bitsandbytes as bnb # noqa: E402 +import torch # noqa: E402 +import torch.utils.checkpoint as ckpt_utils # noqa: E402 + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Tile-selection probe for MT64x64x64_ISA1201_DTVB1") + p.add_argument( + "--m", + type=int, + default=607, + help="Sequence-length dim (default: 607, SYNC-confirmed faulting M)", + ) + p.add_argument( + "--m-sweep", + type=str, + default="", + help="Comma list of M values to run in one process, e.g. 439,607,608,640", + ) + p.add_argument( + "--no-bias", + action="store_true", + help="Build Linear4bit with bias=False (test the non-Bias epilogue path)", + ) + p.add_argument("--shape-3d", action="store_true", help="Use (1, M, K) instead of (M, K)") + p.add_argument("--ckpt", action="store_true", help="Wrap forward in torch.utils.checkpoint") + p.add_argument( + "--reentrant", action="store_true", help="Use use_reentrant=True (default: False)" + ) + p.add_argument( + "--autocast", + action="store_true", + help="Wrap in torch.autocast (matches production training)", + ) + p.add_argument( + "--no-warmup", + action="store_true", + help="Skip no_grad warmup (first dispatch hits cold cache)", + ) + p.add_argument( + "--noncontig-a", + action="store_true", + help="Make A non-contiguous via slice of larger tensor", + ) + p.add_argument("--iters", type=int, default=3, help="Number of fwd+bwd iterations") + return p.parse_args() + + +def run_probe(m: int, args: argparse.Namespace) -> None: + shape_str = f"(1, {m}, 21504)" if args.shape_3d else f"({m}, 21504)" + print(f"\n##### SWEEP M={m} shape={shape_str} #####", flush=True) + + layer = bnb.nn.Linear4bit( + input_features=21504, + output_features=5376, + bias=not args.no_bias, + compute_dtype=torch.bfloat16, + quant_type="nf4", + ).to("cuda") + + x_shape = (1, m, 21504) if args.shape_3d else (m, 21504) + + if args.noncontig_a: + # Slice from a wider tensor so A shares storage with a larger allocation — the + # activation A in production may come from a packed sequence tensor, giving + # non-standard strides or offsets in the underlying storage. + big = torch.randn(*x_shape[:-1], 21504 * 2, dtype=torch.bfloat16, device="cuda") + x_base = big[..., :21504] # non-contiguous view (stride in last dim = 2) + x = x_base.requires_grad_(True) + else: + x = torch.randn(*x_shape, dtype=torch.bfloat16, device="cuda", requires_grad=True) + + print(f"Input shape: {list(x.shape)} stride={list(x.stride())} contig={x.is_contiguous()}") + + if not args.no_warmup: + with torch.no_grad(): + _ = layer(x.detach()) + torch.cuda.synchronize() + print("Warmup complete.\n") + else: + print("Skipping warmup — cold kernel cache.\n") + + def forward_fn(x_in: torch.Tensor) -> torch.Tensor: + return layer(x_in) + + ctx_autocast = ( + torch.autocast(device_type="cuda", dtype=torch.bfloat16) + if args.autocast + else contextlib.nullcontext() + ) + + for i in range(1, args.iters + 1): + with ctx_autocast: + if args.ckpt: + out = ckpt_utils.checkpoint(forward_fn, x, use_reentrant=args.reentrant) + else: + out = layer(x) + loss = out.sum() + loss.backward() + torch.cuda.synchronize() + if x.grad is not None: + x.grad.zero_() + print(f" M={m} iter {i}/{args.iters} — OK", flush=True) + + +def main() -> None: + args = parse_args() + + ckpt_str = f"ckpt(reentrant={args.reentrant})" if args.ckpt else "plain" + extras = [] + if args.autocast: + extras.append("autocast") + if args.no_warmup: + extras.append("no-warmup") + if args.noncontig_a: + extras.append("noncontig-A") + if args.no_bias: + extras.append("no-bias") + extra_str = " " + "+".join(extras) if extras else "" + m_values = [int(v) for v in args.m_sweep.split(",")] if args.m_sweep else [args.m] + print( + f"=== probe_gfx1201_tile.py path={ckpt_str}{extra_str} M={m_values} iters={args.iters} ===" + ) + print(f"torch: {torch.__version__}") + print(f"bitsandbytes: {bnb.__version__}") + + if not torch.cuda.is_available(): + print("ERROR: no CUDA/ROCm device found", file=sys.stderr) + sys.exit(1) + + props = torch.cuda.get_device_properties(0) + print(f"Device: {props.name} ({props.total_memory / 1024**3:.1f} GB)") + print(f"AMD_SERIALIZE_KERNEL: {os.environ.get('AMD_SERIALIZE_KERNEL')}") + print(f"HIP_LAUNCH_BLOCKING: {os.environ.get('HIP_LAUNCH_BLOCKING')}") + print(f"AMD_LOG_LEVEL: {os.environ.get('AMD_LOG_LEVEL')}") + print(f"TORCH_USE_HIPBLASLT: {os.environ.get('TORCH_USE_HIPBLASLT')}") + print("Watch for MT64x64x64.*DTVB1 in ShaderName lines under each SWEEP M= banner.") + + for m in m_values: + run_probe(m, args) + + print(f"\nCompleted {len(m_values)} shape(s) — no crash.") + print( + "Which tile per M: grep -E 'SWEEP M=|MT[0-9]+x[0-9]+x[0-9]+.*ISA1201' this_log.log" + " | grep -oE 'SWEEP M=[0-9]+|MT[0-9]+x[0-9]+x[0-9]+'" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/profile_train_step.py b/scripts/profile_train_step.py new file mode 100644 index 00000000..6c450847 --- /dev/null +++ b/scripts/profile_train_step.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python +"""De-risk the FA2 bet: profile where a real Stage 2 training step spends time. + +The gfx1201 ~70 s/step is ~9× above the raw-TFLOPS floor. The handoff attributes +the gap to "SDPA attention is the bottleneck, not GEMM" — but that was *inferred* +from hipBLASLt=1 showing no gain, never measured directly. This script settles it. + +It does two things: + + 1. SDPA-backend probe — at Gemma 4's real attention shape (head_dim=256, which + ROCm flash/mem-efficient kernels typically cap below at 128), times each SDPA + backend's fwd+bwd. If flash is unavailable and the math fallback is what runs, + attention is slow *by construction* and FA2 (if its backward fit) would help. + + 2. Real-step profile — runs a few REAL training steps (same model load, LoRA, + collator, assistant_only_loss, grad-ckpt as scripts/train_sft.py) under + torch.profiler and reports the attention subtree's share of device time + (via the scaled_dot_product_attention op's *total* cuda time — its self time + is ~0, so sorting by self-time hides it), plus the raw top-40 kernel table. + +Decision it informs: + - attention dominant → patching the flash-attn Triton AMD backward + (bwd_prefill_split.py block sizes → fit 64 KB LDS) could collapse the step + time. FA2 is worth the spike. + - NF4 dequant / GEMM dominant → FA2 buys little even with a fitting backward. + Don't sink time into the kernel patch. + +Run on the R9700 (HIP/ROCm profile; ProfilerActivity.CUDA captures HIP kernels): + + make profile-gemma4-31b + # or directly: + TORCH_USE_HIPBLASLT=1 PYTORCH_HIP_ALLOC_CONF=garbage_collection_threshold:0.8 \ + uv run --no-sync python scripts/profile_train_step.py \ + --config configs/train-sft-stage2-gemma4-31b.env + +Profiles 4 optimizer steps (schedule wait=1/warmup=2/active=3) with grad-accum +forced to 1 (the attention-vs-GEMM ratio is invariant to accumulation, but ga=1 +cuts recorded events ~16× so post-processing finishes in seconds). Pass --trace +to also write outputs/profile-gemma4-31b/trace.json (slow; the printed table is +the deliverable, and on a headless box there's no browser to open it anyway). +""" + +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from scripts.train_sft import ( # noqa: E402 + build_hf_datasets, + build_lora_config, + build_model_and_tokenizer, + build_sft_config, +) + +_PROBE_CODE = """ +import os, sys, time, warnings +warnings.filterwarnings("ignore") +import torch +import torch.nn.functional as F +from torch.nn.attention import SDPBackend, sdpa_kernel + +print("\\n" + "=" * 100) +print("SDPA BACKEND PROBE — Gemma 4 31B shape: head_dim=256, seq=1280, 32 q / 16 kv heads, bf16, causal") +print("=" * 100) +print( + f" allowed: flash={torch.backends.cuda.flash_sdp_enabled()} " + f"mem_efficient={torch.backends.cuda.mem_efficient_sdp_enabled()} " + f"math={torch.backends.cuda.math_sdp_enabled()}" +) + +b, hq, hkv, s, d = 1, 32, 16, 1280, 256 +for name, backend in ( + ("flash", SDPBackend.FLASH_ATTENTION), + ("mem_efficient", SDPBackend.EFFICIENT_ATTENTION), + ("math", SDPBackend.MATH), +): + q = k = v = None + try: + q = torch.randn(b, hq, s, d, device="cuda", dtype=torch.bfloat16, requires_grad=True) + k = torch.randn(b, hkv, s, d, device="cuda", dtype=torch.bfloat16, requires_grad=True) + v = torch.randn(b, hkv, s, d, device="cuda", dtype=torch.bfloat16, requires_grad=True) + + def run(q, k, v): + out = F.scaled_dot_product_attention(q, k, v, is_causal=True, enable_gqa=True) + out.sum().backward() + + with sdpa_kernel([backend]): + for _ in range(2): + run(q, k, v) + torch.cuda.synchronize() + t0 = time.perf_counter() + iters = 5 + for _ in range(iters): + run(q, k, v) + torch.cuda.synchronize() + ms = (time.perf_counter() - t0) / iters * 1000 + print(f" {name:<14} OK fwd+bwd {ms:7.2f} ms/call") + except Exception as e: + print(f" {name:<14} FAILED {str(e).splitlines()[0][:88]}") + finally: + del q, k, v + torch.cuda.empty_cache() +print( + "\\n Read: if flash FAILED (head_dim=256 unsupported) and math is what runs, attention is" + "\\n on the slow rocBLAS path — FA2 would help it *if* its backward could fit 64 KB LDS." +) +""" + + +def sdpa_backend_probe() -> None: + """Time each SDPA backend at Gemma 4 31B's real attention shape. + + Runs in a subprocess so Triton/HIP allocations (outside PyTorch's caching + allocator) are fully released before the model loads in the parent process. + """ + import subprocess + + result = subprocess.run( + [sys.executable, "-c", _PROBE_CODE], + env=os.environ.copy(), + ) + if result.returncode != 0: + print(" (probe subprocess failed — continuing to real-step profile)") + + +def _self_device_time(e) -> float: + # torch 2.11 renamed the per-event attribute to self_device_time_total; + # self_cuda_time_total returns 0 on recent builds. Try names in order. + for attr in ("self_device_time_total", "self_cuda_time_total", "self_cpu_time_total"): + v = getattr(e, attr, 0.0) + if v: + return float(v) + return 0.0 + + +def _classify(name: str) -> str: + n = name.lower() + if "dequant" in n: + return "nf4_dequant" + if "cijk" in n or "gemm" in n: + return "gemm" + if "attn_fwd" in n or "attention" in n or n.startswith("bwd_kernel"): + return "attention" + return "other" + + +def report(prof, torch) -> None: + has_cuda = torch.cuda.is_available() + sort_key = "self_cuda_time_total" if has_cuda else "self_cpu_time_total" + + # key_averages() re-aggregates every raw event — call it ONCE and reuse. + ka = prof.key_averages() + + print("\n" + "=" * 100) + print(f"TOP 40 OPS BY {sort_key}") + print("=" * 100) + print(ka.table(sort_by=sort_key, row_limit=40)) + + # Bucket over LEAF GPU kernels only (names without "::") so an aten op's self + # time isn't double-counted against the device kernels it launches. Skip the + # ProfilerStep* span marker — it overlaps the whole step and isn't work. + buckets = {"nf4_dequant": 0.0, "gemm": 0.0, "attention": 0.0, "other": 0.0} + for e in ka: + if "::" in e.key or e.key.startswith("ProfilerStep"): + continue + buckets[_classify(e.key)] += _self_device_time(e) + total = sum(buckets.values()) or 1.0 + + print("\n" + "=" * 100) + print("STEP TIME BY KERNEL CLASS (leaf GPU kernels; aten-op rows & ProfilerStep excluded)") + print("=" * 100) + for name, val in sorted(buckets.items(), key=lambda kv: -kv[1]): + print(f" {name:<14} {val / total * 100:5.1f}% ({val / 1e3:8.1f} ms)") + print( + "\nDecision:\n" + " attention dominant → FA2 backward-kernel patch worth the spike\n" + " nf4_dequant / gemm dominant → attention isn't the cost; skip FA2\n" + " (nf4_dequant is the inherent 4-bit QLoRA tax: every matmul dequants NF4→bf16.)" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Profile a real Stage 2 training step") + parser.add_argument("--config", metavar="PATH", required=True, help="env config file") + parser.add_argument("--steps", type=int, default=4, help="optimizer steps to run") + parser.add_argument( + "--trace", + action="store_true", + help="also export a chrome trace (slow JSON serialization; the printed table is enough)", + ) + args = parser.parse_args() + + from src.project.config import load_env_file + + load_env_file(Path(args.config)) + + # Cap the run to a few steps and keep eval off, regardless of the config. + os.environ["TRAIN_MAX_STEPS"] = str(args.steps) + os.environ["TRAIN_EVAL_STRATEGY"] = "no" + # Force grad-accum=1 for profiling. The attention-vs-GEMM *ratio* is identical + # regardless of accumulation, but ga=16 records 16× the kernel events per step — + # which is what makes key_averages() + chrome-trace export crawl for minutes. + os.environ["TRAIN_GRAD_ACCUM"] = "1" + + import torch + from peft import get_peft_model + from torch.profiler import ProfilerActivity, profile, schedule + from transformers import TrainerCallback + from trl import SFTTrainer + + out_dir = "outputs/profile-gemma4-31b" + Path(out_dir).mkdir(parents=True, exist_ok=True) + + sdpa_backend_probe() + + train_ds, _ = build_hf_datasets() + # SFTTrainer tokenizes the WHOLE train split single-threaded at construction + # (dataset_num_proc=1) before step 1. Profiling only needs a few steps' worth + # of records — slice to avoid minutes of pointless full-dataset preprocessing. + bs = int(os.environ.get("TRAIN_BATCH_SIZE", "1")) + ga = int(os.environ.get("TRAIN_GRAD_ACCUM", "16")) + n_records = bs * ga * (args.steps + 2) + train_ds = train_ds.select(range(min(n_records, len(train_ds)))) + print( + f"\nProfiling on a {len(train_ds)}-record slice (full split skipped — only {args.steps} steps run)" + ) + model, tokenizer = build_model_and_tokenizer() + lora_cfg = build_lora_config() + with tempfile.TemporaryDirectory() as tmp: + sft_cfg = build_sft_config(output_dir=tmp, use_wandb=False) + + model = get_peft_model(model, lora_cfg) + model.print_trainable_parameters() + + class ProfStep(TrainerCallback): + def __init__(self, prof): + self.prof = prof + + def on_step_end(self, *a, **kw): + self.prof.step() + + trainer = SFTTrainer( + model=model, + args=sft_cfg, + train_dataset=train_ds, + processing_class=tokenizer, + ) + + activities = [ProfilerActivity.CPU] + if torch.cuda.is_available(): + activities.append(ProfilerActivity.CUDA) + + print(f"\nProfiling {args.steps} optimizer steps (schedule wait=1/warmup=2/active=3)...") + with profile( + activities=activities, + schedule=schedule(wait=1, warmup=2, active=3), + record_shapes=False, + profile_memory=False, + with_stack=False, + ) as prof: + trainer.add_callback(ProfStep(prof)) + trainer.train() + + report(prof, torch) + if args.trace: + trace_path = f"{out_dir}/trace.json" + print(f"\nExporting chrome trace to {trace_path} (slow — serializes every event)...") + prof.export_chrome_trace(trace_path) + print(f"Chrome trace: {trace_path} (open in chrome://tracing or perfetto.dev)") + + +if __name__ == "__main__": + main() diff --git a/scripts/replay-rocblas-bench.sh b/scripts/replay-rocblas-bench.sh new file mode 100755 index 00000000..c69f3729 --- /dev/null +++ b/scripts/replay-rocblas-bench.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Replay a captured rocblas-bench line and confirm it dispatches the faulting +# ISA1201 Tensile GEMM kernel. +# +# Success criterion: the replay crashes with a GPU page fault AND +# AMD_LOG_LEVEL=3 shows the ShaderName containing +# ...MT64x64x64...ISA1201...DTVB1... +# That proves the bug is inside rocBLAS/Tensile, independent of PyTorch and +# bitsandbytes — the form the upstream report needs. +# +# If the replay does NOT fault, note that finding in the upstream report and +# fall back to the HIP reproducer (rocblas_gemm_ex + bias, same entry point). +# +# Usage: +# bash scripts/replay-rocblas-bench.sh 'rocblas-bench -f gemm_ex ...' +# +# Or capture the line first: +# bash scripts/capture-rocblas-bench.sh +# then paste the printed line as the first argument here. +set -uo pipefail + +ROCM="${ROCM_PATH:-/opt/rocm}" +BENCH_BIN="$ROCM/bin/rocblas-bench" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUTPUT_DIR="$REPO_DIR/outputs/sft-stage2-gemma4-31b" +REPLAY_LOG="$OUTPUT_DIR/bench-replay.log" + +if [[ $# -lt 1 ]]; then + printf 'Usage: %s ""\n' "$0" >&2 + printf '\nExample:\n' >&2 + printf ' bash scripts/replay-rocblas-bench.sh \\\n' >&2 + printf " 'rocblas-bench -f gemm_ex --transposeA N --transposeB N -m 608 -n 5376 -k 21504 ...'\n" >&2 + exit 1 +fi + +bench_line="$1" +# Strip a leading "rocblas-bench" token if present so we can prepend the full path. +bench_args="${bench_line#rocblas-bench}" +bench_args="${bench_args#"$BENCH_BIN"}" +# An array is required to safely expand a constructed argument list. +read -ra bench_args_arr <<< "$bench_args" + +if [[ ! -x "$BENCH_BIN" ]]; then + printf 'rocblas-bench not found at %s\n' "$BENCH_BIN" >&2 + printf 'Install the rocblas-test or rocblas-benchmarks package.\n' >&2 + exit 1 +fi + +printf 'gfx1201 rocBLAS bench replay\n' +printf ' bin: %s\n' "$BENCH_BIN" +printf ' args: %s\n' "$bench_args" +printf ' replay log: %s\n' "$REPLAY_LOG" +printf '\n' + +# Run the replay under AMD_LOG_LEVEL=3 (names every kernel) + serialization. +# Level 3 is verbose but bench runs are short (microseconds) — log stays small. +AMD_LOG_LEVEL=3 \ + AMD_SERIALIZE_KERNEL=3 \ + TORCH_USE_HIPBLASLT=0 \ + "$BENCH_BIN" "${bench_args_arr[@]}" \ + > "$REPLAY_LOG" 2>&1 || true # non-zero expected if it faults + +printf 'Replay exited. Checking results...\n\n' + +# ── Fault? ────────────────────────────────────────────────────────────────── +if grep -qi 'page not present\|page fault\|memory access fault' "$REPLAY_LOG"; then + printf '[FAULT REPRODUCED] GPU page fault confirmed in standalone rocblas-bench run.\n' + grep -i 'page not present\|page fault\|memory access fault' "$REPLAY_LOG" | head -3 + fault_confirmed=1 +else + printf '[NO FAULT] Replay completed without a page fault.\n' + printf ' This means the bug requires the fused/UserArgs path (bias or other UserArgs).\n' + printf ' Next: try the HIP reproducer with rocblas_gemm_ex + bias extension.\n' + fault_confirmed=0 +fi + +printf '\n' + +# ── ShaderName match? ──────────────────────────────────────────────────────── +target_shader='MT64x64x64.*ISA1201.*DTVB1' +if grep -qE "$target_shader" "$REPLAY_LOG"; then + printf '[SHADER MATCH] Faulting kernel confirmed:\n' + grep -oE "Cijk[^ ]*MT64x64x64[^ ]*ISA1201[^ ]*" "$REPLAY_LOG" | head -1 +elif grep -qi 'Cijk\|MT64x64' "$REPLAY_LOG"; then + printf '[DIFFERENT SHADER] A Tensile GEMM was dispatched but not the expected tile:\n' + grep -oE 'Cijk[^ ]*' "$REPLAY_LOG" | head -3 + printf ' Check transB / ldb encoding — a wrong flag selects a different tile.\n' +else + printf '[NO SHADER NAME] AMD_LOG_LEVEL=3 did not log a kernel name.\n' + printf ' The kernel may have not been dispatched, or check AMD_LOG_LEVEL support.\n' +fi + +printf '\n' + +if [[ $fault_confirmed -eq 1 ]]; then + printf '=== READY FOR UPSTREAM ===\n' + printf 'Paste this rocblas-bench line into the ROCm/rocm-libraries report as the\n' + printf 'standalone reproducer — it faults without PyTorch or bitsandbytes.\n' +else + printf 'Replay did not fault — see notes above. Full log: %s\n' "$REPLAY_LOG" +fi diff --git a/scripts/repro_gfx1201_dense.py b/scripts/repro_gfx1201_dense.py new file mode 100644 index 00000000..7e8aea5d --- /dev/null +++ b/scripts/repro_gfx1201_dense.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Dense-VA reproducer for the gfx1201 ISA1201 Tensile GEMM page fault. + +Why the original repro_gfx1201_rocblas.py doesn't reliably crash: + The faulting kernel computes a wild host-VA roughly 6–7 GB beyond the B + operand end. In a small process (sparse VA) that address typically lands + in completely unregistered memory; the GPU page-walk may silently skip it + or raise a different error class. In production (19 GB model loaded) the + address lands in HMM-registered but page-not-present VA → observable + "Memory access fault … Page not present" every time. + +Fix: pre-allocate a ~20 GB "filler" on the GPU *before* building the layer. + The filler occupies a large VA band. The layer is then placed at a higher + VA; the wild address (B.ptr + ~6–7 GB) is more likely to fall inside a + filler chunk (or immediately above it), which the HMM page-walk resolves + as page-not-present → crash. This matches the production VA density + without loading a 31B model. + +Kernel selection invariant: + The MT64x64x64_ISA1201_DTVB1 kernel is selected by THREE co-conditions: + (1) compute_dtype=bfloat16, quant_type="nf4" + (2) bias=True — selects the BH_Bias_HA_S_SAV_UserArgs epilogue + (3) shape M=608, N=5376, K=21504 — Gemma-4-31B MLP gate/up projection + All three are fixed below; do not change them. + +Run (canonical): + AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1 AMD_LOG_LEVEL=3 \\ + TORCH_USE_HIPBLASLT=0 \\ + uv run --no-sync python scripts/repro_gfx1201_dense.py 2>&1 | tee repro_dense.log + +Confirm kernel dispatched (even if no crash): + grep -o 'MT64x64x64[^[:space:]]*ISA1201[^[:space:]]*' repro_dense.log | head -3 + +Expected if successful: + Process is killed with "Memory access fault by GPU node-1 … Page not present" + followed by "GPU coredump: handler exited with error". + +Expected if VA still misses: + 200 iterations complete, no crash. Check the grep above — if the ShaderName + appears the kernel was dispatched but the wild address landed in unmapped space + (no VMA at all), which the GPU handles differently. In that case the bug is + confirmed at the kernel level but cannot be triggered without the full model VA. + +Hardware: AMD Radeon AI PRO R9700, gfx1201 (RDNA4), 32 GB VRAM +Software: ROCm 7.2, torch 2.11.0+rocm7.2, bitsandbytes 0.49.2 +Upstream: github.com/ROCm/rocm-libraries/issues/7992 +""" + +import os +import sys + +os.environ.setdefault("AMD_SERIALIZE_KERNEL", "3") +os.environ.setdefault("HIP_LAUNCH_BLOCKING", "1") +os.environ.setdefault("AMD_LOG_LEVEL", "3") +os.environ.setdefault("TORCH_USE_HIPBLASLT", "0") + +import bitsandbytes as bnb # noqa: E402 +import torch # noqa: E402 + +FILLER_GB = 20 +FILLER_CHUNK_MB = 512 +N_ITERS = 200 + + +def print_env() -> None: + print("=== repro_gfx1201_dense.py ===") + print(f"torch: {torch.__version__}") + print(f"bitsandbytes: {bnb.__version__}") + print(f"CUDA available: {torch.cuda.is_available()}") + if torch.cuda.is_available(): + props = torch.cuda.get_device_properties(0) + total_gb = props.total_memory / 1024**3 + print(f"Device: {props.name} ({total_gb:.1f} GB)") + print(f"AMD_SERIALIZE_KERNEL: {os.environ.get('AMD_SERIALIZE_KERNEL')}") + print(f"HIP_LAUNCH_BLOCKING: {os.environ.get('HIP_LAUNCH_BLOCKING')}") + print(f"AMD_LOG_LEVEL: {os.environ.get('AMD_LOG_LEVEL')}") + print(f"TORCH_USE_HIPBLASLT: {os.environ.get('TORCH_USE_HIPBLASLT')}") + print() + + +def preallocate_filler(target_gb: int, chunk_mb: int) -> list[torch.Tensor]: + """Allocate GPU tensors to fill VA space before placing the layer.""" + chunk_floats = (chunk_mb * 1024 * 1024) // 2 # float16 = 2 bytes + n_target = (target_gb * 1024) // chunk_mb + chunks: list[torch.Tensor] = [] + print(f"Pre-allocating VA filler: target {target_gb} GB in {chunk_mb} MB chunks …") + for i in range(n_target): + try: + chunks.append(torch.empty(chunk_floats, dtype=torch.float16, device="cuda")) + except torch.cuda.OutOfMemoryError: + allocated_gb = i * chunk_mb / 1024 + print( + f" OOM at chunk {i} — {allocated_gb:.1f} GB allocated (headroom needed for layer)" + ) + break + allocated_gb = len(chunks) * chunk_mb / 1024 + print(f" Filler: {len(chunks)} chunks, {allocated_gb:.1f} GB allocated\n") + return chunks + + +def build_layer() -> bnb.nn.Linear4bit: + layer = bnb.nn.Linear4bit( + input_features=21504, + output_features=5376, + bias=True, + compute_dtype=torch.bfloat16, + quant_type="nf4", + ) + return layer.to("cuda") + + +def main() -> None: + print_env() + + if not torch.cuda.is_available(): + print("ERROR: no CUDA/ROCm device found", file=sys.stderr) + sys.exit(1) + + filler = preallocate_filler(FILLER_GB, FILLER_CHUNK_MB) + + print("Building Linear4bit layer (in=21504, out=5376, bias=True, nf4, bf16) …") + layer = build_layer() + + x = torch.randn(608, 21504, dtype=torch.bfloat16, device="cuda", requires_grad=True) + + with torch.no_grad(): + _ = layer(x.detach()) + torch.cuda.synchronize() + + allocated_mb = torch.cuda.memory_allocated() // (1024**2) + reserved_mb = torch.cuda.memory_reserved() // (1024**2) + print(f"After warmup: allocated={allocated_mb} MB reserved={reserved_mb} MB") + print(f"\nRunning {N_ITERS} forward+backward passes under SYNC …") + print("A GPU page fault kills the process — that is the success condition.\n") + + for i in range(1, N_ITERS + 1): + try: + out = layer(x) + loss = out.sum() + loss.backward() + torch.cuda.synchronize() + if x.grad is not None: + x.grad.zero_() + if i % 20 == 0: + print(f" iter {i:4d}/{N_ITERS} — OK") + except Exception as exc: + print(f" iter {i:4d}/{N_ITERS} — EXCEPTION: {exc}") + print("\nDevice is likely dirty — run make gpu-preflight before next launch.") + sys.exit(1) + + # Keep filler alive until the very end so it holds VA during all iterations. + del filler + + print(f"\nCompleted {N_ITERS} iterations without a catchable exception.") + print("Check the log for 'MT64x64x64' to confirm the kernel was dispatched.") + print("If the ShaderName appears but no crash: the wild address landed in") + print("fully unregistered VA (no VMA). The bug is confirmed at the kernel") + print("level but requires the full model VA footprint to observe the fault.") + + +if __name__ == "__main__": + main() diff --git a/scripts/repro_gfx1201_rocblas.py b/scripts/repro_gfx1201_rocblas.py new file mode 100644 index 00000000..8ba4ef4d --- /dev/null +++ b/scripts/repro_gfx1201_rocblas.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +""" +Minimal standalone reproducer for the gfx1201 ISA1201 Tensile GEMM page fault. + +Root cause (confirmed, all arms exhausted): + Tensile kernel Cijk_Ailk_Bjlk_BBS_BH_Bias_HA_S_SAV_UserArgs_MT64x64x64_...ISA1201..._DTVB1... + computes a wild host-VA from its column-major B descriptor. The bad address lands + ~1 GB outside both GEMM operand buffers — a Tensile tile/stride calculation bug. + +Operand shape: + A = (608, 21504) row-major bf16 (activation batch) + B = (21504, 5376) col-major NF4→bf16 (dequantized QLoRA weight, DTVB1 flag) + bias = True (required — BH_Bias_UserArgs epilogue selects this tile) + +Run command: + AMD_SERIALIZE_KERNEL=3 HIP_LAUNCH_BLOCKING=1 AMD_LOG_LEVEL=3 \\ + uv run --no-sync python scripts/repro_gfx1201_rocblas.py 2>&1 | tee repro_gfx1201.log + +Then grep the log for the ShaderName to confirm kernel dispatch: + grep -o 'MT64x64x64.*ISA1201[^ ]*' repro_gfx1201.log | head -3 + +Hardware: AMD Radeon AI PRO R9700, gfx1201 (RDNA4), 32 GB VRAM +Software: ROCm 7.2, torch 2.11.0+rocm7.2, bitsandbytes 0.49.2 +Upstream: ROCm/rocm-libraries — component: rocBLAS / Tensile + +NOTE on crash reproducibility: + This script dispatches the faulting kernel but may NOT crash in a small process. + The buggy kernel computes a wild host-VA ~1 GB from the B operand. Whether that + address faults depends on the process VA layout: in full Gemma 4 31B training + (~19 GB HMM-mapped model weights), the wild address reliably lands in a registered + but page-not-present region → GPU page fault every time (deterministic at fixed seed, + steps 22-24 from step 20). In this minimal script the VA map is sparse and the wild + address either falls outside GPU-accessible HMM memory or hits a valid page. + Use rocprof/rocm-compute-sanitizer to confirm ISA1201 kernel dispatch without needing a crash. +""" + +import os +import sys + +# These must be set before any HIP/torch import to take effect. +# Set here as a safety net; the run-command above is the canonical way. +os.environ.setdefault("AMD_SERIALIZE_KERNEL", "3") +os.environ.setdefault("HIP_LAUNCH_BLOCKING", "1") +os.environ.setdefault("AMD_LOG_LEVEL", "3") +# Keep hipBLASLt off — it has no kernel for col-major B at this tile on gfx1201 +# (confirmed run #11: hipBLASLt returns "Cannot find the function", falls back to Tensile) +os.environ.setdefault("TORCH_USE_HIPBLASLT", "0") + +import bitsandbytes as bnb +import torch + +N_ITERS = 200 # fault is probabilistic in async; SYNC makes it ~deterministic + + +def print_env() -> None: + + print("=== repro_gfx1201_rocblas.py ===") + print(f"torch: {torch.__version__}") + print(f"bitsandbytes: {bnb.__version__}") + print(f"CUDA available: {torch.cuda.is_available()}") + if torch.cuda.is_available(): + print(f"Device: {torch.cuda.get_device_name(0)}") + print(f"AMD_SERIALIZE_KERNEL: {os.environ.get('AMD_SERIALIZE_KERNEL')}") + print(f"HIP_LAUNCH_BLOCKING: {os.environ.get('HIP_LAUNCH_BLOCKING')}") + print(f"AMD_LOG_LEVEL: {os.environ.get('AMD_LOG_LEVEL')}") + print(f"TORCH_USE_HIPBLASLT: {os.environ.get('TORCH_USE_HIPBLASLT')}") + print() + + +def build_layer() -> bnb.nn.Linear4bit: + # bias=True is required — the BH_Bias_HA_S_SAV_UserArgs epilogue is what + # selects this Tensile tile over a bias-free alternative. + layer = bnb.nn.Linear4bit( + input_features=21504, + output_features=5376, + bias=True, + compute_dtype=torch.bfloat16, + quant_type="nf4", + ) + layer = layer.to("cuda") + return layer + + +def main() -> None: + print_env() + + if not torch.cuda.is_available(): + print("ERROR: no CUDA/ROCm device found", file=sys.stderr) + sys.exit(1) + + print("Building Linear4bit layer (in=21504, out=5376, bias=True, nf4, bf16)...") + layer = build_layer() + + # Warmup forward: triggers in-place NF4 quantization of the weight tensor. + x = torch.randn(608, 21504, dtype=torch.bfloat16, device="cuda") + with torch.no_grad(): + _ = layer(x) + torch.cuda.synchronize() + print("Warmup done — weight quantized to NF4. Starting fault probe loop.\n") + print(f"Running {N_ITERS} forward passes. Under SYNC the fault is near-deterministic;") + print("if this process is killed by a GPU page fault the reproducer succeeded.\n") + print("To confirm the ShaderName dispatched, grep the log for 'MT64x64x64'.\n") + + crashes = 0 + for i in range(1, N_ITERS + 1): + try: + with torch.no_grad(): + out = layer(x) + torch.cuda.synchronize() + if i % 20 == 0: + print(f" step {i:4d}/{N_ITERS} — OK (out shape {tuple(out.shape)})") + except Exception as exc: # noqa: BLE001 + crashes += 1 + print(f" step {i:4d}/{N_ITERS} — EXCEPTION: {exc}") + # After a HIP fault the device is unrecoverable without a driver reset; + # stop immediately rather than confounding further iterations. + print("\nFault caught as Python exception — device may be dirty.") + print("Run make gpu-preflight (or driver reset) before next launch.") + break + + print(f"\nDone: {i} iterations, {crashes} exception(s) caught.") + if crashes == 0: + print("No Python-catchable exception — either the fault didn't trigger,") + print("or it produced an unrecoverable hard kill (check process exit code).") + print("Check repro_gfx1201.log for 'MT64x64x64' kernel dispatch lines.") + + +if __name__ == "__main__": + main() diff --git a/scripts/serve_gemma4_12b.sh b/scripts/serve_gemma4_12b.sh new file mode 100755 index 00000000..e915587d --- /dev/null +++ b/scripts/serve_gemma4_12b.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Serve Gemma 4 12B-it (Unsloth UD-Q8_K_XL GGUF) as an OpenAI-compatible API on +# the NVIDIA RTX 4000 Ada (20 GB). Thin wrapper over the generic engine +# scripts/serve_gemma4_31b.sh (auto-selects DEV=CUDA0 when /dev/kfd is absent). +# +# Context is 32K (not the 31B's 150K): the 20 GB card holds the ~13.6 GB Q8 weights +# + KV + the in-process Qwen3.5 consultant, and KELE turns are <10K tokens anyway. +# +# MTP (multi-token prediction, llama.cpp PR #23398) — lossless speculative decode. +# Set MTP=1 to attach the base-derived drafter for a speed A/B. The drafter needs a +# llama.cpp build that includes PR #23398 (arch gemma4-assistant); stock builds +# cannot load it. MTP forces f16 KV cache: the PR's quantized-KV path (-ctk q8_0) +# initially showed 0% draft acceptance. +# +# Download once: +# hf download unsloth/gemma-4-12b-it-GGUF gemma-4-12b-it-UD-Q8_K_XL.gguf \ +# --local-dir ~/Documents/models/weights +# hf download unsloth/gemma-4-12b-it-GGUF MTP/gemma-4-12B-it-MTP-Q8_0.gguf \ +# --local-dir ~/Documents/models/weights +# +# Endpoint: http://localhost:8080/v1/chat/completions +# Override the weights dir with GEMMA4_12B_WEIGHTS_DIR=/some/path. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +GEMMA4_12B_WEIGHTS_DIR="${GEMMA4_12B_WEIGHTS_DIR:-$HOME/Documents/models/weights}" +WEIGHT_FILE="${GEMMA4_12B_WEIGHT_FILE:-gemma-4-12b-it-UD-Q8_K_XL.gguf}" +MTP_FILE="${GEMMA4_12B_MTP_FILE:-MTP/gemma-4-12B-it-MTP-Q8_0.gguf}" + +# GEMMA4_12B_KV=f16 forces the KV cache type without MTP — needed for 1:1 +# MTP on/off comparisons, since MTP forces f16 below while the engine default +# is q4_0 (a second variable otherwise). +KV_ARGS=() +if [[ -n "${GEMMA4_12B_KV:-}" ]]; then + KV_ARGS=(-ctk "$GEMMA4_12B_KV" -ctv "$GEMMA4_12B_KV") +fi + +MTP_ARGS=() +if [[ "${MTP:-0}" == "1" ]]; then + mtp_path="$GEMMA4_12B_WEIGHTS_DIR/$MTP_FILE" + if [[ ! -f "$mtp_path" ]]; then + printf 'error: MTP=1 but drafter not found at %s\n' "$mtp_path" >&2 + printf ' hf download unsloth/gemma-4-12b-it-GGUF %s --local-dir %s\n' \ + "$MTP_FILE" "$GEMMA4_12B_WEIGHTS_DIR" >&2 + exit 1 + fi + MTP_ARGS=( + --spec-type draft-mtp + --spec-draft-model "$mtp_path" + --spec-draft-n-max "${GEMMA4_12B_MTP_NMAX:-4}" + -ctk f16 -ctv f16 + ) +fi + +exec "$SCRIPT_DIR/serve_gemma4_31b.sh" \ + -m "$GEMMA4_12B_WEIGHTS_DIR/$WEIGHT_FILE" \ + -a "Gemma 4 12B" \ + -c "${GEMMA4_12B_CTX:-32768}" \ + "${KV_ARGS[@]}" \ + "${MTP_ARGS[@]}" \ + "$@" diff --git a/scripts/serve_gemma4_12b_sft.sh b/scripts/serve_gemma4_12b_sft.sh new file mode 100755 index 00000000..422235fb --- /dev/null +++ b/scripts/serve_gemma4_12b_sft.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Serve the merged Socratic-SFT Gemma 4 12B (Q8_0 GGUF) as an OpenAI-compatible API +# on the NVIDIA RTX 4000 Ada. Thin wrapper over scripts/serve_gemma4_31b.sh. +# +# The weight file is produced by the merge+convert pipeline (see the 12B PoC plan): +# scripts/merge_lora_gemma4_sft.py --base google/gemma-4-12b-it \ +# --adapter outputs/sft-gemma4-12b-qlora/final --out outputs/sft-gemma4-12b-qlora/merged +# bash scripts/convert_gemma4_12b_sft_to_gguf.sh +# The convert wrapper writes gemma-4-12B-kele-socratic-sft-Q8_0.gguf AND stages it +# into the weights dir below (matching GEMMA4_12B_SFT_WEIGHT_FILE) — no rename needed. +# +# Distinct alias "Gemma 4 12B SFT" so the eval orchestrator proves the SFT weights +# answered (gemma4-12b-sft-local.env). Stop the base server first — one model at a +# time on the 20 GB card. +# +# Endpoint: http://localhost:8080/v1/chat/completions +# Override the weights dir with GEMMA4_12B_WEIGHTS_DIR=/some/path. + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +GEMMA4_12B_WEIGHTS_DIR="${GEMMA4_12B_WEIGHTS_DIR:-$HOME/Documents/models/weights}" +WEIGHT_FILE="${GEMMA4_12B_SFT_WEIGHT_FILE:-gemma-4-12B-kele-socratic-sft-Q8_0.gguf}" +MTP_FILE="${GEMMA4_12B_MTP_FILE:-MTP/gemma-4-12B-it-MTP-Q8_0.gguf}" + +# MTP=1 attaches the same base-derived drafter as the base server. The drafter +# sees the SFT'd distribution → lower acceptance / smaller speedup than on base, +# but it stays lossless. Used by stage 3 when MTP wins the base on/off A/B. +MTP_ARGS=() +if [[ "${MTP:-0}" == "1" ]]; then + mtp_path="$GEMMA4_12B_WEIGHTS_DIR/$MTP_FILE" + if [[ ! -f "$mtp_path" ]]; then + printf 'error: MTP=1 but drafter not found at %s\n' "$mtp_path" >&2 + printf ' hf download unsloth/gemma-4-12b-it-GGUF %s --local-dir %s\n' \ + "$MTP_FILE" "$GEMMA4_12B_WEIGHTS_DIR" >&2 + exit 1 + fi + MTP_ARGS=( + --spec-type draft-mtp + --spec-draft-model "$mtp_path" + --spec-draft-n-max "${GEMMA4_12B_MTP_NMAX:-4}" + -ctk f16 -ctv f16 + ) +fi + +exec "$SCRIPT_DIR/serve_gemma4_31b.sh" \ + -m "$GEMMA4_12B_WEIGHTS_DIR/$WEIGHT_FILE" \ + -a "Gemma 4 12B SFT" \ + -c "${GEMMA4_12B_CTX:-32768}" \ + "${MTP_ARGS[@]}" \ + "$@" diff --git a/scripts/test_gpu_stack.sh b/scripts/test_gpu_stack.sh index 908cde6a..4a2f2280 100755 --- a/scripts/test_gpu_stack.sh +++ b/scripts/test_gpu_stack.sh @@ -14,13 +14,24 @@ # # No model weights are downloaded. Each step is independent. # +# This is the single unified GPU checker for the project — clean-state, driver, +# torch, bitsandbytes, transformers, PEFT, TRL, attention, llama.cpp. Do not add +# parallel one-off GPU check scripts; extend this one. +# # TODO: extend step 1 to also support NVIDIA/CUDA (nvidia-smi) for RTX 5090. # When nvidia-smi is found, skip ROCm-specific probes and use CUDA backend # checks instead. This allows the same script to run on both AMD and NVIDIA # machines without manual modification. # # Usage: -# bash scripts/test_gpu_stack.sh +# bash scripts/test_gpu_stack.sh # full 13-step stack test +# bash scripts/test_gpu_stack.sh --preflight # fast training gate: clean +# state + torch fwd/bwd (~5s) +# bash scripts/test_gpu_stack.sh --wait-clean 120 # poll up to 120s for a clean +# GPU (used by the resume monitor) +# +# Clean-state tunables (env): GPU_PROC_PATTERN (default train_sft\.py), +# GPU_VRAM_THRESHOLD_MB (default 1024). # # Exit code: 0 if all non-optional steps pass, 1 if any fail. @@ -33,8 +44,141 @@ fail() { echo -e " ${RED}FAIL${NC} $*"; } warn() { echo -e " ${YELLOW}WARN${NC} $*"; } step() { echo -e "\n${BOLD}[$1/13] $2${NC}"; } +# ── GPU clean-state + fwd/bwd helpers (shared by --preflight / --wait-clean and +# the resume monitor) ──────────────────────────────────────────────────────── +# A gfx1201 page fault leaves the amdkfd dirty: orphaned procs keep a HIP context +# and stale VRAM mapped, so the next launch faults early on stale page-table +# entries (docs/GFX1201_RDNA4_TRAINING.md §10). These confirm the GPU is actually +# free and functional before a (re)launch — the gate the monitor used to skip. +GPU_PROC_PATTERN="${GPU_PROC_PATTERN:-train_sft\\.py}" +GPU_VRAM_THRESHOLD_MB="${GPU_VRAM_THRESHOLD_MB:-1024}" + +gpu_vram_used_mb() { + rocm-smi --showmeminfo vram --json 2>/dev/null | .venv/bin/python -c ' +import json, sys +try: + d = json.load(sys.stdin) +except Exception: + print(-1); sys.exit(0) +best = 0 +for card in d.values(): + if isinstance(card, dict): + for k, v in card.items(): + if "used" in k.lower() and "memory" in k.lower(): + try: + best = max(best, int(str(v).strip())) + except ValueError: + pass +print(best // (1024 * 1024)) +' +} + +gpu_clean_state() { + local used pids is_dirty=0 + pids="$(rocm-smi --showpids 2>/dev/null | grep -iE "$GPU_PROC_PATTERN" || true)" + used="$(gpu_vram_used_mb)" + if [[ -n "$pids" ]]; then + fail "GPU busy — process matching /$GPU_PROC_PATTERN/ still resident:" + printf '%s\n' "$pids" | sed 's/^/ /' + is_dirty=1 + fi + if [[ "$used" -lt 0 ]]; then + warn "VRAM usage unreadable (rocm-smi JSON parse failed) — inconclusive" + elif [[ "$used" -gt "$GPU_VRAM_THRESHOLD_MB" ]]; then + fail "VRAM used ${used}MB > ${GPU_VRAM_THRESHOLD_MB}MB threshold (stale allocation?)" + is_dirty=1 + else + pass "GPU idle — VRAM used ${used}MB (≤ ${GPU_VRAM_THRESHOLD_MB}MB), no training proc" + fi + return "$is_dirty" +} + +gpu_fwd_bwd() { + local out + out="$(.venv/bin/python - 2>&1 <<'PY' +import sys +import torch +if not torch.cuda.is_available(): + print("FAIL torch sees no GPU"); sys.exit(1) +try: + torch.manual_seed(0) + a = torch.randn(2048, 2048, device="cuda", dtype=torch.bfloat16, requires_grad=True) + b = torch.randn(2048, 2048, device="cuda", dtype=torch.bfloat16) + ((a @ b).float().pow(2).mean()).backward() + torch.cuda.synchronize() + if a.grad is None or not torch.isfinite(a.grad).all(): + print("FAIL gradient missing/non-finite — GPU compute is wedged"); sys.exit(1) + free, total = torch.cuda.mem_get_info() + print(f"fwd+bwd OK free={free/1024**3:.1f}/{total/1024**3:.1f}GB") +except Exception as e: # noqa: BLE001 — any HIP fault here means the GPU is unusable + print(f"FAIL fwd+bwd raised {type(e).__name__}: {e}"); sys.exit(1) +PY +)" + if echo "$out" | grep -q "^FAIL"; then + fail "$(echo "$out" | grep '^FAIL' | sed 's/^FAIL //')" + return 1 + fi + pass "$out" + return 0 +} + +# ── Mode dispatch ────────────────────────────────────────────────────────────── +MODE=full +WAIT_SECONDS=0 +case "${1:-}" in + --preflight) MODE=preflight ;; + --wait-clean) MODE=wait-clean; WAIT_SECONDS="${2:-120}" ;; + -h|--help) + grep -E '^#( |$)' "$0" | sed -E 's/^# ?//' + exit 0 ;; + "") ;; + *) printf 'error: unknown arg %s (try --help)\n' "$1" >&2; exit 2 ;; +esac + +if ! command -v rocm-smi >/dev/null 2>&1 && [[ "$MODE" != full ]]; then + printf 'error: rocm-smi not found — not a ROCm host\n' >&2 + exit 2 +fi + +if [[ "$MODE" == "wait-clean" ]]; then + echo -e "${BOLD}Waiting up to ${WAIT_SECONDS}s for a clean GPU${NC}" + waited=0 + while true; do + if gpu_clean_state; then + pass "GPU clean — safe to (re)launch" + exit 0 + fi + if (( waited >= WAIT_SECONDS )); then + fail "GPU still dirty after ${waited}s — refusing to launch" + exit 1 + fi + sleep 3 + (( waited += 3 )) || true + done +fi + +if [[ "$MODE" == "preflight" ]]; then + echo -e "${BOLD}GPU pre-flight — clean state + torch fwd/bwd${NC}" + PF_FAIL=0 + gpu_clean_state || PF_FAIL=$((PF_FAIL + 1)) + gpu_fwd_bwd || PF_FAIL=$((PF_FAIL + 1)) + echo "" + if [[ "$PF_FAIL" -eq 0 ]]; then + echo -e "${GREEN}${BOLD}Pre-flight passed — GPU clean and computing.${NC}" + else + echo -e "${RED}${BOLD}Pre-flight failed ($PF_FAIL) — do NOT launch.${NC}" + echo " Clear it: pkill -9 -f '$GPU_PROC_PATTERN'; then re-run. A wedged" + echo " context that survives the kill needs a GPU/driver reset." + fi + exit "$PF_FAIL" +fi + +# ── Full stack test (MODE=full) ──────────────────────────────────────────────── FAILURES=0 +step 0 "GPU clean state (orphan procs / stale VRAM)" +gpu_clean_state || warn "GPU not idle — fine if another job is intentionally running" + # ── 1. ROCm driver ──────────────────────────────────────────────────────────── step 1 "ROCm driver + GPU visibility" if ! command -v rocm-smi &>/dev/null; then @@ -336,7 +480,7 @@ fi # ── 9. Efficient attention (PyTorch SDPA + optional flash-attn) ─────────────── step 9 "Efficient attention — PyTorch SDPA + flash-attn if installed" -SDPA_OUT=$(.venv/bin/python - 2>&1 <<'PY' +SDPA_OUT=$(FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE .venv/bin/python - 2>&1 <<'PY' import sys, warnings warnings.filterwarnings("ignore") try: @@ -544,10 +688,9 @@ if [[ "$FAILURES" -eq 0 ]]; then echo " LoRA fine-tuning: step 7 passing = PEFT/LoRA is ready." echo " SFT trainer: step 8 passing = TRL SFTTrainer is ready." echo " Efficient attention (torch): step 9 = PyTorch SDPA + flash-attn (if installed)" - echo " flash_attn install note: PyPI wheel fails on gfx1201 (CK-tile ISA bug)." - echo " Use ROCm fork with Triton backend:" - echo " git clone --recurse-submodules https://github.com/ROCm/flash-attention /tmp/fa" - echo " FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE uv pip install /tmp/fa --no-build-isolation" + echo " flash_attn install note: PyPI flash-attn 2.8.3+ works on gfx1201 via Triton JIT." + echo " Install (once): FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE uv pip install flash-attn --no-build-isolation" + echo " Set FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE at training time too (Triton JIT selects AMD path)." echo "" echo " llama-server build: step 10 = ROCm/HIP flags verified." echo " Flash attention (llama.cpp): step 11 = FA compiled in; check runtime logs for 'flash attn = 1'." diff --git a/scripts/train_sft.py b/scripts/train_sft.py index 867a26a9..2bced960 100644 --- a/scripts/train_sft.py +++ b/scripts/train_sft.py @@ -35,6 +35,15 @@ TRAIN_LOGGING_STEPS Log every N steps (default 10) TRAIN_SAVE_STEPS Save checkpoint every N steps (default 200) TRAIN_EVAL_STEPS Evaluate every N steps (default 200) + TRAIN_DATA_SEED Sampler-order seed, independent of TRAIN_SEED/seed. Unset → + falls back to args.seed. Vary per resume to break the + data-order-deterministic gfx1201 fault (PR #101 run #13). + TRAIN_LOG_DATA_ORDER Diagnostic: log token-length (M) + fingerprint of the first + N samples in consumption order, to verify a knob actually + reshuffles the post-resume order. 0/unset = off. + WANDB_PROJECT W&B project name (default: csen346-sft) + WANDB_RUN_NAME W&B run name (default: TRAIN_OUTPUT_DIR basename) + WANDB_API_KEY W&B API key (or run `wandb login` once) """ from __future__ import annotations @@ -105,21 +114,35 @@ def build_hf_datasets(): # Model + tokenizer # --------------------------------------------------------------------------- -# Gemma 4 training-compatible chat template. Renders byte-identical to the -# stock google/gemma-4-31b-it template (turn markers <|turn>..., native -# system role) but wraps assistant content in {% generation %} so TRL's +# Gemma 4 training-compatible chat template. For rendered message bodies it is +# byte-identical to the stock google/gemma-4-31b-it template (verified: turn +# markers <|turn>... — token id 105/106, NOT — and a +# native system role), but wraps assistant content in {% generation %} so TRL's # assistant_only_loss can find the assistant-token boundary. No tool-calling # macros — not needed for SFT on Socratic dialogues. +# +# Intentional divergence: the stock template's generation prompt appends a +# `<|channel>thought\n` reasoning primer (Gemma 4 is a thinking model); +# this template omits it, training the teacher to answer directly with no thought +# channel. Production serving must prime generation the same way (plain +# `<|turn>model\n`), or it reintroduces a train/serve mismatch this template avoids. +# +# The model-role terminator MUST sit INSIDE the {% generation %} block: +# assistant_only_loss masks everything outside it to -100, so a terminator left +# outside gets zero gradient and the model never learns to stop (non-terminating +# repetition collapse — see TRL chat_templates/README.md and gemma3_training.jinja, +# which keep inside the block for the same reason). The <|turn>model +# header stays outside (it is a prompt cue, not generated). _GEMMA4_TRAINING_CHAT_TEMPLATE = """{{- bos_token -}} {%- for message in messages -%} {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%} {{- '<|turn>' + role + '\n' -}} {%- if role == 'model' -%} -{% generation %}{{ message['content'] | trim }}{% endgeneration %} +{% generation %}{{ message['content'] | trim }}{{ '\n' }}{% endgeneration %} {%- else -%} {{- message['content'] | trim -}} -{%- endif -%} {{- '\n' -}} +{%- endif -%} {%- endfor -%} {%- if add_generation_prompt -%} {{- '<|turn>model\n' -}} @@ -178,17 +201,43 @@ def build_model_and_tokenizer(): _offload_dir = "offload_weights" _os.makedirs(_offload_dir, exist_ok=True) - model = AutoModelForCausalLM.from_pretrained( - base_model, - quantization_config=bnb_config, - torch_dtype=torch_dtype, - attn_implementation="sdpa", - device_map="auto", - low_cpu_mem_usage=True, - offload_folder=_offload_dir, - offload_state_dict=True, - trust_remote_code=True, - ) + # QLoRA: force all layers onto the single GPU. device_map="auto" reserves + # headroom and can dispatch modules to CPU, which bitsandbytes rejects — for + # live quant it offloads from the BF16 estimate, and for a pre-quantized + # *dynamic* 4-bit checkpoint (e.g. unsloth bnb, which keeps embeddings / + # lm_head / per-layer gates in 16-bit) it can offload one of those 16-bit + # modules. The NF4 model fits in 32 GB VRAM (~22-27 GB peak), so {"": 0} is + # safe and is the canonical single-GPU QLoRA placement. + _device_map = {"": 0} if method == "qlora" else "auto" + # caching_allocator_warmup is an NVIDIA-specific optimisation added in recent + # transformers. It calls torch.cuda.cudart() which has no ROCm equivalent and + # raises "Found no NVIDIA driver." Patch it out on HIP/ROCm — the warmup is + # not required for correctness, only for CUDA allocator pre-heating. + if torch.version.hip is not None: + import transformers.modeling_utils as _mu + + _mu.caching_allocator_warmup = lambda *_a, **_kw: None + # Only pass quantization_config when it is not None. When None is passed + # explicitly, auto_factory forwards it to AutoConfig.from_pretrained which + # calls from_dict(..., quantization_config=None), overwriting the + # checkpoint's valid config.json quantization_config with None — causing + # supports_quant_method to crash on the pre-quantized (TRAIN_PREQ) path. + # FA2 Triton backward OOMs on gfx1201: kernel needs 128 KB shared memory, + # hardware cap is 64 KB. SDPA is stable and uses PyTorch's built-in path. + _attn_impl = "sdpa" + _load_kwargs: dict = { + "attn_implementation": _attn_impl, + "device_map": _device_map, + "low_cpu_mem_usage": True, + "offload_folder": _offload_dir, + "offload_state_dict": True, + "trust_remote_code": True, + } + if bnb_config is not None: + _load_kwargs["quantization_config"] = bnb_config + if torch_dtype is not None: + _load_kwargs["dtype"] = torch_dtype + model = AutoModelForCausalLM.from_pretrained(base_model, **_load_kwargs) for _vision_attr in ("visual", "vision_tower"): if hasattr(model, _vision_attr): @@ -255,13 +304,14 @@ def build_lora_config(): # --------------------------------------------------------------------------- -def build_sft_config(output_dir: str | None = None) -> SFTConfig: +def build_sft_config(output_dir: str | None = None, use_wandb: bool = False) -> SFTConfig: """Build TRL SFTConfig from env vars. Args: output_dir: Override the output directory. Defaults to TRAIN_OUTPUT_DIR env var. Pass a temp dir in dry_run() to avoid creating the real output_dir as a side-effect of the SFTConfig (TrainingArguments) constructor. + use_wandb: Enable W&B logging via report_to=["wandb"]. """ from trl import SFTConfig @@ -277,6 +327,14 @@ def build_sft_config(output_dir: str | None = None) -> SFTConfig: logging_steps = int(_get("TRAIN_LOGGING_STEPS", "10")) save_steps = int(_get("TRAIN_SAVE_STEPS", "200")) eval_steps = int(_get("TRAIN_EVAL_STEPS", "200")) + max_steps = int(_get("TRAIN_MAX_STEPS", "-1")) + # CPU-side parallelism. dataset_num_proc fans the one-time chat-template + + # tokenization map across processes (it is single-process by default and + # pegs one core on the 77k-record split at startup). dataloader workers + # prefetch/collate batches off the main process during training. Neither + # touches the GPU-bound ~70s/step — they only cut startup + feed latency. + dataset_num_proc = int(_get("TRAIN_DATASET_NUM_PROC", "1")) + dataloader_workers = int(_get("TRAIN_DATALOADER_WORKERS", "0")) # In-training eval can OOM on memory-constrained setups: the eval forward # casts logits to fp32 for the loss (transformers/loss/loss_utils.py:58), # doubling the logits tensor briefly. On Gemma 4 31B QLoRA with 256K vocab @@ -285,13 +343,32 @@ def build_sft_config(output_dir: str | None = None) -> SFTConfig: # in-training eval entirely; the downstream eval pipeline produces # paper-grade numbers anyway. eval_strategy = _get("TRAIN_EVAL_STRATEGY", "steps").lower() + # Resume replays the exact sampler order (HF restores it from rng_state.pth + + # the seedable sampler), so a data-order-deterministic fault re-hits the same + # step on every resume (PR #101 run #13: stuck at step 22/24 from ckpt-20). + # data_seed drives the seedable sampler's permutation independently of args.seed; + # changing it per resume reshuffles which samples land at each step while + # ignore_data_skip stays False — so the skip still advances through fresh data + # and coverage is preserved. (ignore_data_skip=True would instead restart the + # epoch at batch 0 every resume and silently train on a tiny prefix — rejected.) + # Unset → None → HF falls back to args.seed (stock reproducible behaviour). + data_seed_raw = _get("TRAIN_DATA_SEED", "") + data_seed = int(data_seed_raw) if data_seed_raw else None + + run_name = (os.environ.get("WANDB_RUN_NAME") or Path(output_dir).name) if use_wandb else None effective_batch = batch * grad_accum print( f"\nSFT config output={output_dir}\n" f" epochs={epochs} lr={lr} batch={batch}×{grad_accum}={effective_batch}" f" max_length={max_seq_len} bf16={use_bf16} grad_ckpt={grad_ckpt}" - f" eval_strategy={eval_strategy}" + f" eval_strategy={eval_strategy} dataset_num_proc={dataset_num_proc}" + f" dataloader_workers={dataloader_workers}" + + ( + f" wandb_project={os.environ.get('WANDB_PROJECT')} run={run_name}" + if use_wandb + else "" + ) ) # load_best_model_at_end requires eval to function — auto-disable it when @@ -300,6 +377,7 @@ def build_sft_config(output_dir: str | None = None) -> SFTConfig: output_dir=output_dir, num_train_epochs=epochs, learning_rate=lr, + data_seed=data_seed, per_device_train_batch_size=batch, gradient_accumulation_steps=grad_accum, max_length=max_seq_len, @@ -307,12 +385,16 @@ def build_sft_config(output_dir: str | None = None) -> SFTConfig: gradient_checkpointing=grad_ckpt, gradient_checkpointing_kwargs={"use_reentrant": False} if grad_ckpt else None, logging_steps=logging_steps, + max_steps=max_steps, save_steps=save_steps, eval_steps=eval_steps, eval_strategy=eval_strategy, - save_total_limit=2, + save_total_limit=5, + dataset_num_proc=dataset_num_proc, + dataloader_num_workers=dataloader_workers, load_best_model_at_end=(eval_strategy != "no"), - report_to="none", + report_to=["wandb"] if use_wandb else "none", + run_name=run_name, assistant_only_loss=True, ) @@ -388,6 +470,11 @@ def main() -> None: ) args = parser.parse_args() + # Suppress verbose INFO/WARNING churn from transformers/trl during a full + # SFT run. ERROR-only (level 1) keeps logs readable; unset via env to see + # the original level-3 noise for debugging. + os.environ["TRANSFORMERS_VERBOSITY"] = "error" + if args.config: from src.project.config import load_env_file @@ -397,24 +484,69 @@ def main() -> None: dry_run() return + from src.project.wandb_tracking import SftTracker + + use_wandb = SftTracker().enabled + # ── Full training run ──────────────────────────────────────────────────── from peft import get_peft_model from trl import SFTTrainer train_ds, eval_ds = build_hf_datasets() + # With eval_strategy=no the eval split is never evaluated, but SFTTrainer still + # tokenizes whatever eval_dataset it's handed in __init__ — wasted work that also + # crashed the multiprocessing map on the RAM-constrained NVIDIA box. Drop it. + if _get("TRAIN_EVAL_STRATEGY", "steps").lower() == "no": + eval_ds = None model, tokenizer = build_model_and_tokenizer() lora_cfg = build_lora_config() - sft_cfg = build_sft_config() + sft_cfg = build_sft_config(use_wandb=use_wandb) model = get_peft_model(model, lora_cfg) model.print_trainable_parameters() + import torch + from transformers import TrainerCallback, TrainerControl, TrainerState, TrainingArguments + + from src.project.hf_callback import HFCheckpointCallback + + class VRAMLogCallback(TrainerCallback): + def on_log( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + logs=None, + **kwargs, + ): + if not torch.cuda.is_available(): + return + alloc = torch.cuda.memory_allocated() / 1024**3 + reserved = torch.cuda.memory_reserved() / 1024**3 + total = torch.cuda.get_device_properties(0).total_memory / 1024**3 + print( + f" VRAM alloc={alloc:.1f}GB reserved={reserved:.1f}GB total={total:.1f}GB", + flush=True, + ) + + _hf_repo = os.environ.get("TRAIN_HF_REPO", "").strip() + _hf_push_every = int(os.environ.get("TRAIN_HF_PUSH_EVERY", "50")) + + callbacks: list[TrainerCallback] = [VRAMLogCallback()] + if _hf_repo: + callbacks.append(HFCheckpointCallback(_hf_repo, _hf_push_every)) + print( + f"HF auto-push enabled → {_hf_repo} (every {_hf_push_every} steps)", + flush=True, + ) + trainer = SFTTrainer( model=model, args=sft_cfg, train_dataset=train_ds, eval_dataset=eval_ds, processing_class=tokenizer, + callbacks=callbacks, ) # Auto-resume from latest checkpoint if one exists in output_dir. @@ -425,6 +557,122 @@ def main() -> None: # To start over from scratch, rm -rf the checkpoint-* dirs first. has_checkpoint = any(Path(sft_cfg.output_dir).glob("checkpoint-*")) + # DIAGNOSTIC (gfx1201 page fault, PR #101) — BNB_DEQUANT_PROBE=1 logs the memory + # descriptor of all MatMul4Bit operands in both directions: + # [dequant_probe] — dequantized weight (B), via F.dequantize_4bit + # [gemm_operandA] — grad_output (backward A), via MatMul4Bit.backward + # [gemm_forward_input] — activation input (forward A), via MatMul4Bit.forward + # probe-2 revealed the faulting GEMM is a grad-checkpoint RECOMPUTED FORWARD pass + # (8 unpaired [dequant_probe] lines before the fault, no [gemm_operandA]) — the + # backward hook was blind to it. This extends coverage to MatMul4Bit.forward. + # Under HIP_LAUNCH_BLOCKING=1 the last pair before the fault ([gemm_forward_input] + # then [dequant_probe]) are the faulting GEMM's two operands. Compare both against + # the fault address to land in Bucket #1 (freed operand) or #2 (wild address). + # Remove once the fault is root-caused. + if os.environ.get("BNB_DEQUANT_PROBE"): + import bitsandbytes.functional as _bnbF + from bitsandbytes.autograd._functions import MatMul4Bit as _MatMul4Bit + + def _descr(t): + ptr = t.data_ptr() + end = ptr + t.numel() * t.element_size() + return ( + f"shape={tuple(t.shape)} stride={tuple(t.stride())} " + f"contig={t.is_contiguous()} off={t.storage_offset()} " + f"dtype={t.dtype} ptr=0x{ptr:x} end=0x{end:x}" + ) + + _orig_dequant = _bnbF.dequantize_4bit + + def _dequant_probe(*a, **k): + out = _orig_dequant(*a, **k) + print(f"[dequant_probe] {_descr(out)}", flush=True) + return out + + _bnbF.dequantize_4bit = _dequant_probe + + _orig_forward = _MatMul4Bit.forward + + def _forward_probe(ctx, A, B, out=None, bias=None, quant_state=None): + print(f"[gemm_forward_input] {_descr(A)}", flush=True) + return _orig_forward(ctx, A, B, out=out, bias=bias, quant_state=quant_state) + + _MatMul4Bit.forward = staticmethod(_forward_probe) + + _orig_backward = _MatMul4Bit.backward + + def _backward_probe(ctx, grad_output): + print(f"[gemm_operandA] {_descr(grad_output)}", flush=True) + return _orig_backward(ctx, grad_output) + + _MatMul4Bit.backward = staticmethod(_backward_probe) + print( + "BNB_DEQUANT_PROBE active — logging all MatMul4Bit operands " + "(forward input, backward grad_output, dequant weight).", + flush=True, + ) + + # DIAGNOSTIC (gfx1201 page fault, PR #101) — BNB_FORCE_B_CONTIGUOUS=1 forces the + # dequantized weight (B) row-major before it reaches the Tensile GEMM. The confirmed + # faulting kernel (MT64x64x64 ISA1201) is selected only when B is col-major (DTVB1, + # stride=(1,21504)). Forcing contiguous changes the descriptor to DTVB0 → Tensile + # should dispatch a different tile. Two outcomes are informative: + # - fault disappears → col-major B is the trigger; workaround exists (slower, extra copy) + # - fault persists with row-major B → stride hypothesis is incomplete; upstream filing + # needs revision before submission. + # Stacks cleanly on top of BNB_DEQUANT_PROBE if both are set. + if os.environ.get("BNB_FORCE_B_CONTIGUOUS"): + import bitsandbytes.functional as _bnbF # noqa: F811 (re-import is intentional) + + _prev_dequant = _bnbF.dequantize_4bit + + def _contiguous_dequant(*a, **k): + out = _prev_dequant(*a, **k) + return out.contiguous() if not out.is_contiguous() else out + + _bnbF.dequantize_4bit = _contiguous_dequant + print( + "BNB_FORCE_B_CONTIGUOUS active — forcing dequantized weight row-major before GEMM " + "(tests DTVB1 col-major B as fault trigger).", + flush=True, + ) + + # DIAGNOSTIC (gfx1201 page fault, PR #101) — TRAIN_LOG_DATA_ORDER=N logs the token + # length (M, the GEMM row dim that selects the faulting MT64x64x64 tile) and a + # deterministic fingerprint of the first N samples in CONSUMPTION order. Run #13 + # showed the fault is data-order sticky: every resume replays the same sequence and + # re-faults at the same step. This verifies whether TRAIN_DATA_SEED actually changes + # that order across resumes — run twice with different seeds and diff the [data_order] + # lines. Hooked at the collator (not the sampler): accelerate rewraps the sampler in a + # SeedableRandomSampler, so the collator is the one point that sees the true consumed + # order. The fingerprint folds token ids (int hashing is unsalted → stable across + # processes), so identical [data_order] sequences ⇒ identical data order. + if os.environ.get("TRAIN_LOG_DATA_ORDER"): + _order_n = int(os.environ["TRAIN_LOG_DATA_ORDER"]) + _orig_collator = trainer.data_collator + _order_seen = 0 + + def _order_collator(features, *a, **k): + nonlocal _order_seen + batch = _orig_collator(features, *a, **k) + for f in features: + if _order_seen >= _order_n: + break + ids = f.get("input_ids") or [] + fp = 0 + for v in ids[:64]: + fp = (fp * 1000003 + int(v)) & 0xFFFFFFFF + print(f"[data_order] #{_order_seen} M={len(ids)} fp={fp:08x}", flush=True) + _order_seen += 1 + return batch + + trainer.data_collator = _order_collator + print( + f"TRAIN_LOG_DATA_ORDER active — logging first {_order_n} samples' token length " + f"+ fingerprint in consumption order (data_seed={trainer.args.data_seed}).", + flush=True, + ) + print("\nStarting training...") if has_checkpoint: print(f" (resuming from latest checkpoint in {sft_cfg.output_dir})") diff --git a/src/project/dataset.py b/src/project/dataset.py index 0f7e3fdb..d74da057 100644 --- a/src/project/dataset.py +++ b/src/project/dataset.py @@ -355,15 +355,15 @@ def load_socrat_synthetic( seed: int = 42, hf_repo: str = "ulises-c/SocratDataset-SYNTHETIC", ) -> list[dict]: - """Load SocratDataset-SYNTHETIC from HuggingFace (both configs, combined). + """Load SocratDataset-SYNTHETIC from HuggingFace (75 records, single config). - Merges the `default` (37 records) and `n75_extension` (38 records) configs - into a single pool of 75 records before splitting. + The 38-record extension (ids 100038-100075) was merged into the default config + 2026-06-24, so the repo's single `train` split now holds all 75 ZH synthetic + dialogues. (Previously default held 37 + a separate n75_extension config of 38.) """ from datasets import load_dataset as hf_load - raw_default = [dict(r) for r in hf_load(hf_repo, split="train")] - raw_ext = [dict(r) for r in hf_load(hf_repo, name="n75_extension", split="train")] + raw = [dict(r) for r in hf_load(hf_repo, split="train")] def _convert(r: dict) -> dict: q = r["question"] @@ -386,7 +386,7 @@ def _convert(r: dict) -> dict: "ground_truth_states": states if states else None, } - converted = [_convert(r) for r in raw_default + raw_ext] + converted = [_convert(r) for r in raw] if split == "all": return converted return _split(converted, split, seed) @@ -591,6 +591,182 @@ def load_ultrachat( return _split(converted, split, seed) +# --------------------------------------------------------------------------- +# Inference-matching SFT sources (socrat-zh-sft, socrat-en-sft) +# --------------------------------------------------------------------------- +# These produce one record per dialogue *turn* in the exact single-message format +# that socratic_teaching_system.py:socrates_teacher sends at inference, fixing the +# train/serve schema drift that caused output collapse on the 5090-trained model. +# +# Differences from the multi-turn socrat-zh/en sources: +# - System prompt: the 7-line inference rules block, not _SOCRAT_ZH/EN_SYSTEM +# - Problem context (问题/选项/提示/知识点) NOT included — inference never sends it +# - Message structure: one (system, user, assistant) triple per turn +# - User message: 历史对话记录 blob + 当前学生输入 label, matching socrates_teacher() +# - History format: "学生: ...\n老师: ...\n" pairs, matching get_formatted_history() + +_TEACHER_INFERENCE_SYSTEM = """\ +你是一位使用苏格拉底教学法的小学科学教师，擅长启发式教学。 +接下来你会收到历史对话记录、当前学生输入和苏格拉底教学顾问对当前教学对话的评估及建议操作； +你的任务是遵循建议的操作并参考评估结果对学生提问以完成苏格拉底式教学。 +以下是你需要遵守的规则： +- 每次只能提出一个问题（输出时请检查问题数量，如超出请删去多余问题） +- 提出的问题必须与解题直接相关（输出时请检查问题是否偏离解题，如偏题请重新输出与解题直接相关的问题） +- 请确保提问符合小学阶段学生的知识水平，避免过于困难 +- 语气应该非常亲切并具有鼓励性 +- 除非苏格拉底教学顾问建议的操作要求，否则不能给出过于明显的提示 +- 如果接收到的建议操作为：对题目进行总结，则总结题目且不再提出问题""" + + +def _build_inference_user_message( + history_turns: list[tuple[str, str]], + student_input: str, + evaluation: str, + action: str, +) -> str: + """Build the user message exactly as socrates_teacher() does at inference. + + history_turns: list of (student, teacher) pairs for completed turns before this one. + evaluation/action are the consultant fields as inference sends them: the + free-form consultant prose and get_action_for_state(state) respectively. + """ + if history_turns: + history_lines = [] + for s, t in history_turns: + history_lines.append(f"学生: {s}") + history_lines.append(f"老师: {t}") + formatted_history = "\n".join(history_lines) + else: + formatted_history = "" + + return ( + f"\n历史对话记录:\n{formatted_history}\n\n" + f"当前学生输入: {student_input}\n\n" + f"苏格拉底教学顾问评估结果: {evaluation}\n" + f"苏格拉底教学顾问建议的操作: {action}\n" + ) + + +def _socrat_zh_to_sft_records(record: dict) -> list[dict]: + """One SocratDataset-ZH dialogue → N per-turn SFT records in inference format.""" + from src.project.socratic_teaching_system import get_action_for_state + + dialogue = record.get("dialogue", []) + records = [] + history: list[tuple[str, str]] = [] + + for i, turn in enumerate(dialogue): + state = _strip_quotes(turn.get("state", "")) + if not (state and _strip_quotes(turn.get("action", ""))): + history.append((turn["student"], turn["teacher"])) + continue + + action = get_action_for_state(state) + evaluation = _strip_quotes(turn.get("evaluation", "")) or f"学生处于 {state} 状态" + user_msg = _build_inference_user_message(history, turn["student"], evaluation, action) + records.append( + { + "id": f"{record['id']}_{i}", + "source": "socrat-zh-sft", + "messages": [ + {"role": "system", "content": _TEACHER_INFERENCE_SYSTEM}, + {"role": "user", "content": user_msg}, + {"role": "assistant", "content": turn["teacher"]}, + ], + "ground_truth_states": [state], + } + ) + history.append((turn["student"], turn["teacher"])) + + return records + + +def _socrat_en_to_sft_records(record: dict) -> list[dict]: + """One SocratDataset-EN dialogue → N per-turn SFT records in inference format.""" + from src.project.socratic_teaching_system import get_action_for_state + + dialogue = record.get("dialogue", []) + records = [] + history: list[tuple[str, str]] = [] + + for i, turn in enumerate(dialogue): + state = turn.get("state", "") + if not (state and turn.get("action", "")): + history.append((turn["student"], turn["teacher"])) + continue + + action = get_action_for_state(state) + evaluation = turn.get("evaluation", "") or f"学生处于 {state} 状态" + user_msg = _build_inference_user_message(history, turn["student"], evaluation, action) + records.append( + { + "id": f"{record['id']}_{i}", + "source": "socrat-en-sft", + "messages": [ + {"role": "system", "content": _TEACHER_INFERENCE_SYSTEM}, + {"role": "user", "content": user_msg}, + {"role": "assistant", "content": turn["teacher"]}, + ], + "ground_truth_states": [state], + } + ) + history.append((turn["student"], turn["teacher"])) + + return records + + +def load_socrat_zh_sft( + split: str = "train", + seed: int = 42, + hf_repo: str = "ulises-c/SocratDataset", +) -> list[dict]: + """Load SocratDataset-ZH as per-turn inference-matching SFT records. + + Splits at the dialogue level before expanding to per-turn records so that + all turns of a dialogue stay in the same partition. Splitting on the flat + per-turn list would scatter a dialogue's turns across train/test, causing + history blobs in test records to contain teacher responses seen in training. + """ + from datasets import load_dataset as hf_load + + raw = [dict(r) for r in hf_load(hf_repo, split="train")] + if split == "all": + converted: list[dict] = [] + for r in raw: + converted.extend(_socrat_zh_to_sft_records(r)) + return converted + split_raw = _split(raw, split, seed) + converted = [] + for r in split_raw: + converted.extend(_socrat_zh_to_sft_records(r)) + return converted + + +def load_socrat_en_sft( + split: str = "train", + seed: int = 42, + hf_repo: str = "ulises-c/SocratDataset-EN", +) -> list[dict]: + """Load SocratDataset-EN as per-turn inference-matching SFT records. + + Splits at the dialogue level before expanding to per-turn records — same + reasoning as load_socrat_zh_sft. + """ + from datasets import load_dataset as hf_load + + raw = [dict(r) for r in hf_load(hf_repo, split="train")] + if split == "all": + converted: list[dict] = [] + for r in raw: + converted.extend(_socrat_en_to_sft_records(r)) + return converted + split_raw = _split(raw, split, seed) + converted = [] + for r in split_raw: + converted.extend(_socrat_en_to_sft_records(r)) + return converted + + # --------------------------------------------------------------------------- # Unified entry points # --------------------------------------------------------------------------- @@ -600,9 +776,12 @@ def load_ultrachat( "openhermes": load_openhermes, "ultrachat": load_ultrachat, "slimorca": load_slimorca, - # Stage 2 — Socratic teaching + # Stage 2 — Socratic teaching (multi-turn, legacy format) "socrat-zh": load_socrat_zh, "socrat-en": load_socrat_en, + # Stage 2 — inference-matching per-turn SFT format (use these for SFT training) + "socrat-zh-sft": load_socrat_zh_sft, + "socrat-en-sft": load_socrat_en_sft, "socrat-synthetic": load_socrat_synthetic, "socrat-synthetic-en": load_socrat_synthetic_en, "socrateach-multi": load_socrateach_multi, diff --git a/src/project/hf_callback.py b/src/project/hf_callback.py new file mode 100644 index 00000000..ea0b6b33 --- /dev/null +++ b/src/project/hf_callback.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import threading +from pathlib import Path +from typing import TYPE_CHECKING, ClassVar + +from transformers import TrainerCallback + +if TYPE_CHECKING: + from transformers import TrainerControl, TrainerState, TrainingArguments + + +class HFCheckpointCallback(TrainerCallback): + """Push checkpoint-N to a HF model repo after every N-th save. + + Activated by TRAIN_HF_REPO=. Cadence controlled by + TRAIN_HF_PUSH_EVERY (default 50 steps). Runs in a daemon thread so it + never blocks the training loop; if the previous push is still in-flight + when the next one fires, the new push is skipped and logged. Also fires + unconditionally at on_train_end to capture the final state. + + Persists the last pushed step in ``.hf_last_push`` inside the output dir + so that resumed / crash-recovered runs do not re-push already-uploaded + checkpoints. On ``on_init`` the callback scans for the latest on-disk + checkpoint and pushes it synchronously if it was never uploaded. + """ + + def __init__(self, repo_id: str, push_every: int) -> None: + self._repo_id = repo_id + self._push_every = push_every + self._thread: threading.Thread | None = None + self._output_dir: str | None = None + + # ------------------------------------------------------------------ + # Persistence — track last pushed step so we never re-push on resume + # ------------------------------------------------------------------ + + def _last_push_file(self) -> Path | None: + if not self._output_dir: + return None + return Path(self._output_dir) / ".hf_last_push" + + def _read_last_pushed(self) -> int: + p = self._last_push_file() + if p and p.exists(): + try: + return int(p.read_text().strip()) + except (ValueError, OSError): + pass + return -1 + + def _write_last_pushed(self, step: int) -> None: + p = self._last_push_file() + if p: + p.write_text(str(step)) + + # ------------------------------------------------------------------ + # Checkpoint log — SFT_STAGE2B_CHECKPOINT_LOG.md on the HF repo + # ------------------------------------------------------------------ + + _LOG_HEADER: ClassVar[str] = """# Checkpoints in This Repo + +| Checkpoint | Step | Epoch | Loss | Token Accuracy | +|---|---|---|---|---| +""" + + @staticmethod + def _build_log_row(step: int, epoch: float, loss: float | None, acc: float | None) -> str: + loss_str = f"{loss:.4f}" if isinstance(loss, float) else "—" + acc_str = f"{acc:.4f}" if isinstance(acc, float) else "—" + return f"| checkpoint-{step}/ | {step} | {epoch:.3f} | {loss_str} | {acc_str} |" + + def _update_checkpoint_log( + self, + step: int, + epoch: float, + loss: float | None, + acc: float | None, + ) -> None: + from huggingface_hub import HfApi + + api = HfApi() + rows: dict[int, str] = {} + try: + path = api.hf_hub_download( + repo_id=self._repo_id, + filename="SFT_STAGE2B_CHECKPOINT_LOG.md", + repo_type="model", + ) + with open(path) as f: + for line in f: + if line.startswith("| checkpoint-"): + parts = [p.strip() for p in line.split("|")] + if len(parts) >= 3: + try: + # part[1] = "checkpoint-1230/" + ckpt_name = parts[1].rstrip("/") + s = int(ckpt_name.split("-")[-1]) + rows[s] = line.rstrip() + except (ValueError, IndexError): + pass + except Exception: + pass + + new_row = self._build_log_row(step, epoch, loss, acc) + rows[step] = new_row + + lines = [self._LOG_HEADER.rstrip("\n")] + for s in sorted(rows): + lines.append(rows[s]) + lines.append("") + content = "\n".join(lines) + + try: + api.upload_file( + path_or_fileobj=content.encode(), + path_in_repo="SFT_STAGE2B_CHECKPOINT_LOG.md", + repo_id=self._repo_id, + repo_type="model", + commit_message=f"checkpoint log: step {step}", + ) + except Exception as exc: + print(f" [HF] checkpoint log update failed (step {step}): {exc}", flush=True) + + # ------------------------------------------------------------------ + # Push logic + # ------------------------------------------------------------------ + + def _push( + self, + ckpt_dir: Path, + step: int, + commit_msg: str, + epoch: float = 0.0, + loss: float | None = None, + acc: float | None = None, + ) -> None: + from huggingface_hub import HfApi + + try: + HfApi().upload_folder( + folder_path=str(ckpt_dir), + path_in_repo=f"checkpoint-{step}", + repo_id=self._repo_id, + repo_type="model", + commit_message=commit_msg, + ) + print(f" [HF] pushed checkpoint-{step} → {self._repo_id}", flush=True) + self._write_last_pushed(step) + self._update_checkpoint_log(step, epoch, loss, acc) + except Exception as exc: + print(f" [HF] push failed (step {step}): {exc}", flush=True) + + def _maybe_push( + self, + args: TrainingArguments, + state: TrainerState, + force: bool = False, + ) -> None: + step = state.global_step + if not force and step % self._push_every != 0: + return + if not force and step <= self._read_last_pushed(): + return + output_dir = args.output_dir + if not output_dir: + return + ckpt_dir = Path(output_dir) / f"checkpoint-{step}" + if not ckpt_dir.exists(): + return + if self._thread and self._thread.is_alive(): + print( + f" [HF] previous push still running, skipping step {step}", + flush=True, + ) + return + log = state.log_history[-1] if state.log_history else {} + loss = log.get("loss", log.get("train_loss")) + acc = log.get("mean_token_accuracy") + epoch = state.epoch or 0.0 + if isinstance(loss, float) and isinstance(acc, float): + commit_msg = ( + f"checkpoint-{step} (step {step}, epoch {epoch:.3f}, " + f"loss {loss:.4f}, acc {acc:.4f})" + ) + else: + commit_msg = f"checkpoint-{step} (step {step}, epoch {epoch:.3f})" + self._thread = threading.Thread( + target=self._push, args=(ckpt_dir, step, commit_msg, epoch, loss, acc), daemon=True + ) + self._thread.start() + print(f" [HF] pushing checkpoint-{step} in background...", flush=True) + + # ------------------------------------------------------------------ + # Launch push — push latest checkpoint on init if not already pushed + # ------------------------------------------------------------------ + + def _push_latest_on_launch( + self, + args: TrainingArguments, + ) -> None: + output_dir = Path(args.output_dir) if args.output_dir else None + if not output_dir or not output_dir.exists(): + return + ckpt_dirs = list(output_dir.glob("checkpoint-*")) + if not ckpt_dirs: + return + latest = max(ckpt_dirs, key=lambda d: int(d.name.split("-")[-1])) + latest_step = int(latest.name.split("-")[-1]) + if latest_step <= self._read_last_pushed(): + return + if latest_step % self._push_every != 0: + return + commit_msg = f"checkpoint-{latest_step} (step {latest_step}, resume push)" + print(f" [HF] launch push: checkpoint-{latest_step}", flush=True) + self._push(latest, latest_step, commit_msg, epoch=0.0, loss=None, acc=None) + + # ------------------------------------------------------------------ + # Trainer callback hooks + # ------------------------------------------------------------------ + + def on_init_end( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ) -> None: + self._output_dir = args.output_dir + self._push_latest_on_launch(args) + + def on_save( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ) -> None: + self._maybe_push(args, state) + + def on_train_end( + self, + args: TrainingArguments, + state: TrainerState, + control: TrainerControl, + **kwargs, + ) -> None: + self._maybe_push(args, state, force=True) + if self._thread and self._thread.is_alive(): + print(" [HF] waiting for final push to complete...", flush=True) + self._thread.join() diff --git a/src/project/kele.py b/src/project/kele.py index 8c5d4301..525afb59 100644 --- a/src/project/kele.py +++ b/src/project/kele.py @@ -1,3 +1,5 @@ +# TODO(MELE rename): rename this module to MELE.py and change imports across the +# board — KELE→MELE is a major architectural change touching every importer. """ KELE Socratic Teaching System — working copy extended from the original. @@ -10,12 +12,17 @@ import random import threading import time +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from pathlib import Path from src.project.config import load_config -from src.project.socratic_teaching_system import SocraticTeachingSystem +from src.project.socratic_teaching_system import ( + SocraticTeachingSystem, + SocraticTeachingSystemOracle, +) +from src.project.wandb_tracking import EvalTracker RESOURCES_DIR = Path(__file__).resolve().parents[2] / "references" / "KELE" @@ -25,6 +32,7 @@ def create_system( experiment: str | None = None, unified: bool = False, bert_consultant: str | None = None, + oracle_consultant: bool = False, ) -> SocraticTeachingSystem: """Create a SocraticTeachingSystem from environment config. @@ -36,10 +44,17 @@ def create_system( use SocraticTeachingSystemBertConsultant: BERT predicts the state and the LLM only generates the teacher response (two-call style). Mutually exclusive with unified=True. + + If oracle_consultant=True, use SocraticTeachingSystemOracle: the eval loop + hands it the ground-truth state each turn (no prediction), isolating pure + teacher-turn quality. state_accuracy is ~perfect by construction — score on + ROUGE/BLEU. Mutually exclusive with unified and bert_consultant. """ cfg = load_config(experiment=experiment) - if bert_consultant and unified: - raise ValueError("--unified and --bert-consultant are mutually exclusive") + if sum([bool(bert_consultant), bool(unified), bool(oracle_consultant)]) > 1: + raise ValueError( + "--bert-consultant, --unified, and --oracle-consultant are mutually exclusive" + ) if bert_consultant: from src.project.socratic_teaching_bert_consultant import ( @@ -53,6 +68,9 @@ def create_system( cls = SocraticTeachingSystemUnified extra_kwargs = {} + elif oracle_consultant: + cls = SocraticTeachingSystemOracle + extra_kwargs = {} else: cls = SocraticTeachingSystem extra_kwargs = {} @@ -79,7 +97,7 @@ def load_dataset( seed: int = 42, source: str = "hf", hf_repo: str | list[str] = [ # noqa: B006 - "ulises-c/SocratDataset-EN", + # "ulises-c/SocratDataset-EN", # uncomment to eval the bilingual set (zh+en, ~1361 test) "ulises-c/SocratDataset", ], ) -> list[dict]: @@ -137,6 +155,9 @@ def run_single_dialogue(system: SocraticTeachingSystem, item: dict) -> dict: for turn in ground_truth: student_input = turn["student"] + # Oracle consultant: hand the GT state in before the turn (no-op otherwise). + if isinstance(system, SocraticTeachingSystemOracle): + system.prime_oracle_state(turn["state"]) teacher_response = system.process_student_input(student_input) turn_record: dict = { @@ -210,6 +231,28 @@ def _write_progress_log( ) +def _make_incremental_metrics_logger( + tracker: EvalTracker, dialogues_dir: Path +) -> Callable[[int], None] | None: + """Per-N-dialogues W&B logging (WANDB_EVAL_LOG_EVERY, default 10): recompute + metrics over everything on disk and log at step=completed, so eval metrics + form a convergence curve instead of a single end-of-run point.""" + every = int(os.environ.get("WANDB_EVAL_LOG_EVERY", "10")) + if every < 1: + return None + from src.project.metrics import compute_metrics_from_records, load_dialogue_records + + def on_result(completed: int) -> None: + if completed % every: + return + records = load_dialogue_records(dialogues_dir, skip_invalid=True) + metrics = compute_metrics_from_records(records) + if "error" not in metrics: + tracker.log_step(metrics, step=completed) + + return on_result + + def _run_sequential( pending: list[dict], dialogues_dir: Path, @@ -218,6 +261,7 @@ def _run_sequential( total_dataset: int, completed_initial: int, start_time: float, + on_result: Callable[[int], None] | None = None, ) -> int: completed = completed_initial for item in pending: @@ -240,6 +284,8 @@ def _run_sequential( flush=True, ) _write_progress_log(progress_log, completed, total_dataset, rate, remaining, elapsed) + if on_result is not None: + on_result(completed) return completed @@ -255,6 +301,8 @@ def _run_parallel( experiment: str | None, unified: bool, bert_consultant: str | None, + oracle_consultant: bool = False, + on_result: Callable[[int], None] | None = None, ) -> tuple[int, list[SocraticTeachingSystem]]: """Run pending dialogues concurrently using a ThreadPoolExecutor. @@ -280,6 +328,7 @@ def get_or_create_system() -> SocraticTeachingSystem: experiment=experiment, unified=unified, bert_consultant=bert_consultant, + oracle_consultant=oracle_consultant, ) thread_local.system = sys_ with worker_systems_lock: @@ -317,6 +366,8 @@ def process_one(item: dict) -> tuple[dict, dict]: _write_progress_log( progress_log, completed, total_dataset, rate, remaining, elapsed ) + if on_result is not None: + on_result(completed) return completed_box[0], worker_systems @@ -333,8 +384,10 @@ def run_batch_evaluation( worker_id: int = 0, num_workers: int = 1, bert_consultant: str | None = None, + oracle_consultant: bool = False, workers: int | None = None, sample_seed: int | None = None, + hf_repo: str | list[str] | None = None, ) -> None: """Run the full evaluation pipeline on the dataset. @@ -354,7 +407,10 @@ def run_batch_evaluation( if workers < 1: workers = 1 - dataset = load_dataset(dataset_path, split=split) + if hf_repo: + dataset = load_dataset(dataset_path, split=split, hf_repo=hf_repo) + else: + dataset = load_dataset(dataset_path, split=split) total = len(dataset) # Filter to start_id, optionally random-subsample, then apply limit. @@ -374,7 +430,11 @@ def run_batch_evaluation( # In sequential mode this is also the workhorse; in parallel mode each # worker thread creates its own via thread-local storage. system = create_system( - debug=False, experiment=experiment, unified=unified, bert_consultant=bert_consultant + debug=False, + experiment=experiment, + unified=unified, + bert_consultant=bert_consultant, + oracle_consultant=oracle_consultant, ) output_dir.mkdir(parents=True, exist_ok=True) @@ -450,6 +510,17 @@ def run_batch_evaluation( if completed and pending: print(f" (resuming: {completed} already on disk, {len(pending)} to run)") + # WANDB_EVAL_RUN_NAME lets a caller distinguish runs that share an + # --experiment but differ otherwise (e.g. the same teacher served with + # MTP off vs on → distinct output dirs, same experiment config). + run_name = os.environ.get("WANDB_EVAL_RUN_NAME") or experiment or output_dir.name + tracker = EvalTracker() + on_result = None + if num_workers == 1: + tracker.start(run_name) + if tracker.active: + on_result = _make_incremental_metrics_logger(tracker, dialogues_dir) + if workers == 1: completed = _run_sequential( pending, @@ -459,6 +530,7 @@ def run_batch_evaluation( len(dataset), completed, start_time, + on_result=on_result, ) else: completed, worker_systems = _run_parallel( @@ -472,6 +544,8 @@ def run_batch_evaluation( experiment=experiment, unified=unified, bert_consultant=bert_consultant, + oracle_consultant=oracle_consultant, + on_result=on_result, ) # Aggregate fallback counts across worker systems for unified mode. if unified: @@ -491,6 +565,10 @@ def run_batch_evaluation( "teacher_base_url": cfg.teacher.base_url, "consultant_model": cfg.consultant.model_name, "consultant_base_url": cfg.consultant.base_url, + # --bert-consultant replaces the LLM consultant entirely; without this + # field run_config misattributes state decisions to consultant_model. + "bert_consultant": bert_consultant, + "oracle_consultant": oracle_consultant, "thinking_budget": cfg.consultant.thinking_budget, "max_teaching_rounds": cfg.max_teaching_rounds, "unified": unified, @@ -521,6 +599,56 @@ def run_batch_evaluation( with open(output_dir / "metrics_summary.json", "w") as f: json.dump(metrics, f, indent=2) print(format_metrics_table(metrics)) + tracker.finish(metrics, step=completed) + + +def replay_wandb( + output_dir: Path, + every: int = 10, + run_name: str | None = None, + order: str = "completion", +) -> None: + """Re-log a finished eval to W&B as a per-dialogue metric curve. + + Recomputes metrics over growing prefixes of the saved dialogues and logs + one point per ``every`` dialogues, turning a run that produced a single + end-of-run data point into a convergence curve — no re-evaluation needed. + + order='completion' replays in the order dialogues actually finished + (file mtime); order='id' replays sorted by dialogue id. + """ + from src.project.metrics import compute_metrics_from_records + + dialogues_dir = output_dir / "dialogues" + files = sorted(dialogues_dir.glob("*.json")) + if order == "completion": + files.sort(key=lambda f: f.stat().st_mtime) + records = [] + for f in files: + data = json.loads(f.read_text()) + if "error" not in data: + records.append(data) + if not records: + print(f"No valid dialogues in {dialogues_dir}") + return + + # Explicit replay request implies wandb logging — bypass the WANDB_EVAL gate. + os.environ.setdefault("WANDB_EVAL", "1") + tracker = EvalTracker() + tracker.start(run_name or os.environ.get("WANDB_EVAL_RUN_NAME") or f"{output_dir.name}-curve") + if not tracker.active: + return + + print(f"Replaying {len(records)} dialogues from {dialogues_dir} (every {every}, {order} order)") + for n in [*range(every, len(records), every), len(records)]: + metrics = compute_metrics_from_records(records[:n]) + tracker.log_step(metrics, step=n) + print( + f" n={n:>4} rouge1={metrics['rouge1']:>6.2f} rougeL={metrics['rougeL']:>6.2f}" + f" bleu4={metrics['bleu4']:>6.2f}" + f" state_acc={metrics['state_accuracy']['overall']:>6.2f}" + ) + tracker.finish() def interactive( @@ -609,6 +737,14 @@ def main() -> None: "Replaces the LLM consultant with the BERT classifier; LLM only " "generates the teacher response (two-call style).", ) + eval_parser.add_argument( + "--oracle-consultant", + action="store_true", + help="Feed the ground-truth state each turn instead of predicting it " + "(no consultant call). Isolates teacher-turn quality given correct " + "state; state_accuracy is ~perfect by construction, so score on " + "ROUGE/BLEU. Mutually exclusive with --bert-consultant/--unified.", + ) eval_parser.add_argument( "--workers", type=int, @@ -618,6 +754,16 @@ def main() -> None: "KELE_PARALLEL_WORKERS). Recommended: 4. Server-side -np must be " "at least this value.", ) + eval_parser.add_argument( + "--hf-repo", + type=str, + nargs="+", + default=None, + help="One or more HuggingFace dataset repo IDs to evaluate (concatenated). " + "Default: ulises-c/SocratDataset (ZH). Use e.g. ulises-c/SocratDataset-EN " + "(held-out EN) or ulises-c/SocratDataset-SYNTHETIC{,-EN} with --split all " + "(synthetic sets are tiny, never-trained OOD probes).", + ) eval_parser.add_argument( "--dataset-path", type=Path, @@ -627,6 +773,27 @@ def main() -> None: "references/KELE-EN/SocratDataset.json for the English translation.", ) + # Replay a finished eval into W&B as a per-dialogue metric curve + replay_parser = sub.add_parser( + "wandb-replay", + help="Re-log a finished eval's results dir to W&B as a per-dialogue metric curve", + ) + replay_parser.add_argument( + "--output", type=Path, required=True, help="Results dir containing dialogues/" + ) + replay_parser.add_argument( + "--every", type=int, default=10, help="Log one point per N dialogues" + ) + replay_parser.add_argument( + "--run-name", type=str, default=None, help="W&B run name (default: -curve)" + ) + replay_parser.add_argument( + "--order", + choices=["completion", "id"], + default="completion", + help="Prefix order: completion (file mtime, the order dialogues finished) or id", + ) + # Quick test mode — run on a handful of dialogues test_parser = sub.add_parser("test", help="Quick test with a few dialogues") test_parser.add_argument("--n", type=int, default=3, help="Number of dialogues to test") @@ -643,6 +810,11 @@ def main() -> None: default=None, help="Path to a trained 34-state BERT classifier checkpoint dir.", ) + test_parser.add_argument( + "--oracle-consultant", + action="store_true", + help="Feed the ground-truth state each turn instead of predicting it.", + ) test_parser.add_argument( "--input", type=Path, @@ -653,6 +825,14 @@ def main() -> None: args = parser.parse_args() + # weave.init auto-patches the openai clients in socratic_teaching_system, so a + # single gated call here traces the whole consultant→teacher loop. Off unless set. + weave_project = os.getenv("WEAVE_PROJECT") + if weave_project: + import weave # pyright: ignore[reportMissingImports] + + weave.init(weave_project) + if args.command == "interactive": interactive( experiment=args.experiment, @@ -673,8 +853,17 @@ def main() -> None: worker_id=args.worker_id, num_workers=args.num_workers, bert_consultant=args.bert_consultant, + oracle_consultant=args.oracle_consultant, workers=args.workers, sample_seed=args.sample_seed, + hf_repo=args.hf_repo, + ) + elif args.command == "wandb-replay": + replay_wandb( + args.output, + every=args.every, + run_name=args.run_name, + order=args.order, ) elif args.command == "test": run_batch_evaluation( @@ -684,6 +873,7 @@ def main() -> None: experiment=args.experiment, unified=args.unified, bert_consultant=args.bert_consultant, + oracle_consultant=args.oracle_consultant, split="all" if args.dataset_path else "test", ) else: diff --git a/src/project/metrics.py b/src/project/metrics.py index f90c21db..378f1218 100644 --- a/src/project/metrics.py +++ b/src/project/metrics.py @@ -58,18 +58,37 @@ def compute_bleu(predictions: list[str], references: list[str]) -> float: return result.score +def load_dialogue_records(dialogues_dir: Path, *, skip_invalid: bool = False) -> list[dict]: + """Load dialogue JSONs sorted by filename (= dialogue id). + + skip_invalid tolerates partially-written files — needed when incremental + metrics logging reads the dir while parallel eval workers are still writing. + """ + records = [] + for f in sorted(dialogues_dir.glob("*.json")): + try: + records.append(json.loads(f.read_text())) + except (json.JSONDecodeError, OSError): + if not skip_invalid: + raise + return records + + def compute_state_accuracy(dialogues_dir: Path) -> StateAccuracy: """Compute how often our consultant picks the correct state vs ground truth. Returns overall accuracy and per-stage accuracy (a, b, c, d, e). """ + return _state_accuracy_from_records(load_dialogue_records(dialogues_dir)) + + +def _state_accuracy_from_records(records: list[dict]) -> StateAccuracy: correct = 0 total = 0 stage_correct: dict[str, int] = {} stage_total: dict[str, int] = {} - for f in sorted(dialogues_dir.glob("*.json")): - data = json.loads(f.read_text()) + for data in records: if "error" in data: continue for turn in data.get("dialogue", []): @@ -100,11 +119,14 @@ def compute_state_accuracy(dialogues_dir: Path) -> StateAccuracy: def extract_predictions_and_references(dialogues_dir: Path) -> tuple[list[str], list[str]]: """Extract teacher response predictions and ground truth references from dialogue outputs.""" + return _extract_from_records(load_dialogue_records(dialogues_dir)) + + +def _extract_from_records(records: list[dict]) -> tuple[list[str], list[str]]: predictions = [] references = [] - for f in sorted(dialogues_dir.glob("*.json")): - data = json.loads(f.read_text()) + for data in records: if "error" in data: continue for turn in data.get("dialogue", []): @@ -119,14 +141,23 @@ def extract_predictions_and_references(dialogues_dir: Path) -> tuple[list[str], def compute_all_metrics(dialogues_dir: Path) -> dict: """Compute all automated metrics for a completed evaluation run.""" - predictions, references = extract_predictions_and_references(dialogues_dir) + return compute_metrics_from_records(load_dialogue_records(dialogues_dir)) + + +def compute_metrics_from_records(records: list[dict]) -> dict: + """Compute all automated metrics over in-memory dialogue records. + + Record-based so partial-run metrics (per-N-dialogues W&B logging, replay + curves) can be computed over any prefix of completed dialogues. + """ + predictions, references = _extract_from_records(records) if not predictions: return {"error": "No dialogue data found"} rouge = compute_rouge(predictions, references) bleu = compute_bleu(predictions, references) - state_acc = compute_state_accuracy(dialogues_dir) + state_acc = _state_accuracy_from_records(records) return { "n_turns": len(predictions), diff --git a/src/project/socratic_teaching_system.py b/src/project/socratic_teaching_system.py index 0411aa65..c9a3114e 100644 --- a/src/project/socratic_teaching_system.py +++ b/src/project/socratic_teaching_system.py @@ -5,6 +5,54 @@ import openai +_DEFAULT_ACTION = "继续提问" + +# 状态到操作的映射表 — single source of truth shared with the SFT loader so +# training records carry the exact action string inference sends (PR #101/#94). +STATE_TO_ACTION = { + "a0": "引导学生提出问题", + "a1": "生成一个与解题相关的子问题", + "b2": "从不同角度生成问题", + "b3": "更改问题", + "b4": "从不同角度生成同一问题的相关子问题", + "b5": "提出可以检查学生概念的问题", + "b6": "复习学生已经学过的概念", + "b7": "复习这些概念并与错误进行比较", + "c8": "提供一个反例", + "c9": "帮助学生形成不完整规则并进一步研究它，或提出一个误导性的问题", + "c10": "询问原因", + "c11": "应指出并明确询问原因", + "c12": "帮助其形成不完整规则并进一步研究它，或提出一个误导性的问题，或提供一个反例", + "c13": "提供一个反例", + "c14": "鼓励学生做出预测并提出新原则", + "c15": "鼓励学生做出预测并提出新原则", + "c16": "鼓励学生做出预测并提出新原则", + "c17": "生成该子问题", + "c18": "要求学生重新考虑该点", + "c19": "生成具有诊断功能的问题", + "c20": "验证学生刚学习的概念", + "c21": "要求学生详细思考问题", + "c22": "问'为什么'", + "c23": "帮助学生重新形成完整假设", + "c24": "要求学生单独检验假设", + "c25": "提供一个验证方法", + "c26": "要求学生比较两个示例的差异", + "c27": "指导学生进行检验", + "c28": "告知学生错误情况并要求其提出其他可能的概念", + "c29": "提供正确概念并询问为什么之前没有想到", + "d30": "提出相关案例并要求预测，或问'为什么'", + "d31": "直接向学生展示正确概念和规则，并要求其重新思考这些概念和给出题目答案", + "d32": "提出相关案例并要求预测", + "d33": "建立一个普遍定义并要求学生给出题目答案", + "e34": "对题目进行总结", +} + + +def get_action_for_state(state: str) -> str: + """根据状态获取对应的提问操作 — module-level twin of the instance method, + importable by the SFT loader without constructing the full system.""" + return STATE_TO_ACTION.get(state, _DEFAULT_ACTION) + class SocraticTeachingSystem: def __init__( @@ -50,43 +98,7 @@ def __init__( ) # 状态到操作的映射表 - self.state_to_action = { - "a0": "引导学生提出问题", - "a1": "生成一个与解题相关的子问题", - "b2": "从不同角度生成问题", - "b3": "更改问题", - "b4": "从不同角度生成同一问题的相关子问题", - "b5": "提出可以检查学生概念的问题", - "b6": "复习学生已经学过的概念", - "b7": "复习这些概念并与错误进行比较", - "c8": "提供一个反例", - "c9": "帮助学生形成不完整规则并进一步研究它，或提出一个误导性的问题", - "c10": "询问原因", - "c11": "应指出并明确询问原因", - "c12": "帮助其形成不完整规则并进一步研究它，或提出一个误导性的问题，或提供一个反例", - "c13": "提供一个反例", - "c14": "鼓励学生做出预测并提出新原则", - "c15": "鼓励学生做出预测并提出新原则", - "c16": "鼓励学生做出预测并提出新原则", - "c17": "生成该子问题", - "c18": "要求学生重新考虑该点", - "c19": "生成具有诊断功能的问题", - "c20": "验证学生刚学习的概念", - "c21": "要求学生详细思考问题", - "c22": "问'为什么'", - "c23": "帮助学生重新形成完整假设", - "c24": "要求学生单独检验假设", - "c25": "提供一个验证方法", - "c26": "要求学生比较两个示例的差异", - "c27": "指导学生进行检验", - "c28": "告知学生错误情况并要求其提出其他可能的概念", - "c29": "提供正确概念并询问为什么之前没有想到", - "d30": "提出相关案例并要求预测，或问'为什么'", - "d31": "直接向学生展示正确概念和规则，并要求其重新思考这些概念和给出题目答案", - "d32": "提出相关案例并要求预测", - "d33": "建立一个普遍定义并要求学生给出题目答案", - "e34": "对题目进行总结", - } + self.state_to_action = STATE_TO_ACTION # 初始化系统状态 self.reset_session() @@ -641,3 +653,64 @@ def start_conversation(self) -> None: print( f"\n[Max teaching rounds reached ({self.max_teaching_rounds}). Please give your final answer to move to the summary.]" ) + + +_STAGE_NAMES = { + "a": "学生提问", + "b": "概念探查", + "c": "归纳推理", + "d": "规则建构", + "e": "教师总结", +} + + +class SocraticTeachingSystemOracle(SocraticTeachingSystem): + """Oracle consultant: instead of *predicting* the Socratic state, it is HANDED + the ground-truth state for each turn (primed by the eval loop via + ``prime_oracle_state``) and derives the action from the shared state→action + table. The teacher half is unchanged, so this isolates *pure teacher-turn + quality given the correct state* — removing classifier accuracy as a confound. + See docs/SFT_RESULTS_REPORT.md (consultant ablation, T1.1). + + Because the state is authoritative, ``process_student_input`` is overridden to + a streamlined path that sets ``current_state`` to the GT value directly and + **bypasses the anti-regression + max-round correction guards** of the base + class (those exist to fix a *predicting* consultant's mistakes; an oracle has + none, and the max-round guard would otherwise force ``d33`` on any dialogue + with more than ``max_teaching_rounds`` teaching turns, corrupting the GT + state). ``state_accuracy`` is therefore ~perfect by construction — this run is + scored on ROUGE/BLEU only. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._oracle_state: str | None = None + + def prime_oracle_state(self, state: str) -> None: + """Set the ground-truth state for the next ``process_student_input`` call.""" + self._oracle_state = state + + def process_student_input(self, student_input: str) -> str: + self.add_to_history("student", student_input) + + previous_state = self.current_state + state = self._oracle_state or self.current_state + action = self.get_action_for_state(state) + stage = state[0] if state else "a" + evaluation = ( + f"根据当前对话内容，学生处于 {stage}({_STAGE_NAMES.get(stage, '')}) 阶段的 " + f"{state} 状态（标准答案状态）。按照苏格拉底教学法，应采取的操作是：{action}。" + f"请基于该状态和操作，针对学生当前的表现，给出合适的教学回复。" + ) + + if previous_state == "a0" and state != "a0": + self.teaching_rounds = 1 + elif previous_state != "a0" and state != "a0": + self.teaching_rounds += 1 + + self.add_to_consultant_history(evaluation, state, action) + self.current_state = state + + socrates_response = self.socrates_teacher(student_input, evaluation, action) + self.add_to_history("teacher", socrates_response) + return socrates_response diff --git a/src/project/wandb_tracking.py b/src/project/wandb_tracking.py new file mode 100644 index 00000000..ce0bfb2d --- /dev/null +++ b/src/project/wandb_tracking.py @@ -0,0 +1,124 @@ +"""W&B (wandb.ai) tracking, detached from the eval/train entrypoints. + +Three small classes, split by side of the pipeline: + +- ``WandbAuth`` — shared availability + auth check, reused by both trackers. +- ``SftTracker`` — SFT side (scripts/train_sft.py). HF's SFTTrainer owns + ``wandb.init`` via ``report_to``; this only gates it and resolves + project/run-name. +- ``EvalTracker`` — eval side (src/project/kele.py). Logs eval metrics as a + W&B run so base↔SFT evals compare in one dashboard. + +Both trackers degrade to no-ops when wandb is unauthenticated, so neither the +training loop nor the eval pipeline depends on W&B being configured. +""" + +from __future__ import annotations + +import os +import sys + + +class WandbAuth: + """Shared wandb availability + authentication.""" + + @staticmethod + def authed() -> bool: + """True if wandb is importable and has a usable API key. + + ``relogin=False`` avoids interactive prompts in nohup runs; the + ``login`` callable check guards against the local ``wandb/`` + run-artifacts directory shadowing the installed package. + """ + try: + import wandb # pyright: ignore[reportMissingImports] + + if not callable(getattr(wandb, "login", None)): + raise ImportError("wandb package not importable (shadowed by local wandb/ dir?)") + return bool(wandb.login(relogin=False)) + except Exception: + return False + + +class SftTracker: + """SFT-side tracking. The SFTTrainer performs ``wandb.init`` itself via + ``report_to=["wandb"]``; this class only decides whether to enable it + (``.enabled``) and sets the default project as a side effect.""" + + DEFAULT_PROJECT = "csen346-sft" + + def __init__(self) -> None: + self.enabled = WandbAuth.authed() + if self.enabled: + os.environ.setdefault("WANDB_PROJECT", self.DEFAULT_PROJECT) + else: + print( + "WARNING: W&B not authenticated — tracking disabled. " + "Run `wandb login` or set WANDB_API_KEY to enable.", + file=sys.stderr, + ) + + +class EvalTracker: + """Eval-side tracking. Logs rouge/bleu + state-accuracy as a W&B run when + ``WANDB_EVAL`` is set and wandb is authed. Off by default so smoke/sanity + runs don't spawn runs. + + The run stays open across the eval so metrics can be logged incrementally + (``log_step`` per N completed dialogues, step = completed count). A + crash-resumed eval starts a fresh W&B run whose steps continue from where + the previous one stopped; same-named runs overlay into one curve in the UI. + """ + + DEFAULT_PROJECT = "csen346-eval" + + def __init__(self) -> None: + self._requested = bool(os.getenv("WANDB_EVAL")) + self.enabled = self._requested and WandbAuth.authed() + self._run = None + + @property + def active(self) -> bool: + return self._run is not None + + @staticmethod + def _flatten(metrics: dict) -> dict: + state_acc = metrics.get("state_accuracy", {}) + flat = { + "eval/n_turns": metrics.get("n_turns"), + "eval/rouge1": metrics.get("rouge1"), + "eval/rouge2": metrics.get("rouge2"), + "eval/rougeL": metrics.get("rougeL"), + "eval/bleu4": metrics.get("bleu4"), + "eval/state_accuracy_overall": state_acc.get("overall"), + } + for stage, acc in state_acc.get("per_stage", {}).items(): + flat[f"eval/state_acc/{stage}"] = acc + return flat + + def start(self, run_name: str) -> None: + if not self.enabled: + if self._requested: + print("WANDB_EVAL set but wandb is not authed — skipping. Run `wandb login`.") + return + import wandb # pyright: ignore[reportMissingImports] + + self._run = wandb.init( + project=os.getenv("WANDB_PROJECT", self.DEFAULT_PROJECT), + name=run_name, + job_type="eval", + config={"experiment": run_name}, + reinit=True, + ) + + def log_step(self, metrics: dict, step: int) -> None: + if self._run is not None: + self._run.log(self._flatten(metrics), step=step) + + def finish(self, metrics: dict | None = None, step: int | None = None) -> None: + if self._run is None: + return + if metrics is not None: + self._run.log(self._flatten(metrics), step=step) + self._run.finish() + self._run = None diff --git a/tests/test_hf_callback.py b/tests/test_hf_callback.py new file mode 100644 index 00000000..ce08cf5c --- /dev/null +++ b/tests/test_hf_callback.py @@ -0,0 +1,252 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from src.project.hf_callback import HFCheckpointCallback + + +def _state(step: int, epoch: float = 0.3, loss: float = 0.65, acc: float = 0.80): + log = {"loss": loss, "mean_token_accuracy": acc} + return SimpleNamespace(global_step=step, epoch=epoch, log_history=[log]) + + +def _args(tmp_path, step: int): + (tmp_path / f"checkpoint-{step}").mkdir() + return SimpleNamespace(output_dir=str(tmp_path)) + + +_ctrl = SimpleNamespace() + + +def test_skips_non_multiple_step(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + args = _args(tmp_path, 30) + cb.on_save(args, _state(30), _ctrl) + assert cb._thread is None + + +def test_pushes_at_multiple_step(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + args = _args(tmp_path, 50) + with patch.object(cb, "_push") as mock_push: + cb.on_save(args, _state(50), _ctrl) + cb._thread.join() + mock_push.assert_called_once() + ckpt_dir, step, commit_msg, epoch, loss, acc = mock_push.call_args[0] + assert step == 50 + assert "loss 0.6500" in commit_msg + assert "acc 0.8000" in commit_msg + + +def test_skips_when_thread_alive(tmp_path, capsys): + cb = HFCheckpointCallback("repo/id", push_every=50) + alive_thread = MagicMock() + alive_thread.is_alive.return_value = True + cb._thread = alive_thread + with patch.object(cb, "_push") as mock_push: + cb.on_save(_args(tmp_path, 50), _state(50), _ctrl) + mock_push.assert_not_called() + assert "skipping" in capsys.readouterr().out + + +def test_skips_when_checkpoint_dir_missing(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + args = SimpleNamespace(output_dir=str(tmp_path)) + cb.on_save(args, _state(50), _ctrl) + assert cb._thread is None + + +def test_commit_msg_without_log(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + empty_state = SimpleNamespace(global_step=100, epoch=0.5, log_history=[]) + with patch.object(cb, "_push") as mock_push: + cb.on_save(_args(tmp_path, 100), empty_state, _ctrl) + cb._thread.join() + _, step, commit_msg, *_ = mock_push.call_args[0] + assert "step 100" in commit_msg + assert "loss" not in commit_msg + + +def test_train_end_forces_push_at_non_multiple_step(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + with patch.object(cb, "_push") as mock_push: + cb.on_train_end(_args(tmp_path, 37), _state(37), _ctrl) + cb._thread.join() + mock_push.assert_called_once() + _, step, *_ = mock_push.call_args[0] + assert step == 37 + + +def test_train_end_joins_alive_thread(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + mock_thread = MagicMock() + mock_thread.is_alive.return_value = True + cb._thread = mock_thread + # no checkpoint dir → _maybe_push returns before spawning a new thread + # but on_train_end still joins the existing alive thread + args = SimpleNamespace(output_dir=str(tmp_path)) + cb.on_train_end(args, _state(37), _ctrl) + mock_thread.join.assert_called_once() + + +# --------------------------------------------------------------------------- +# Persistence — .hf_last_push file +# --------------------------------------------------------------------------- + + +def test_writes_and_reads_last_pushed(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + cb._output_dir = str(tmp_path) + cb._write_last_pushed(100) + assert cb._read_last_pushed() == 100 + cb._write_last_pushed(200) + assert cb._read_last_pushed() == 200 + + +def test_last_pushed_defaults_to_neg_one(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + assert cb._read_last_pushed() == -1 + + +def test_writes_last_pushed_on_successful_push(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + cb._output_dir = str(tmp_path) + with patch("huggingface_hub.HfApi.upload_folder"): + cb._push(tmp_path, 50, "test") + assert (tmp_path / ".hf_last_push").read_text() == "50" + + +# --------------------------------------------------------------------------- +# Step-skip guard — skip if already pushed +# --------------------------------------------------------------------------- + + +def test_skips_already_pushed_step(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + cb._output_dir = str(tmp_path) + (tmp_path / ".hf_last_push").write_text("50") + args = _args(tmp_path, 50) + with patch.object(cb, "_push") as mock_push: + cb.on_save(args, _state(50), _ctrl) + mock_push.assert_not_called() + + +def test_force_push_bypasses_last_pushed(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + cb._output_dir = str(tmp_path) + (tmp_path / ".hf_last_push").write_text("50") + with patch.object(cb, "_push") as mock_push: + cb.on_train_end(_args(tmp_path, 50), _state(50), _ctrl) + mock_push.assert_called_once() + + +# --------------------------------------------------------------------------- +# Launch check — on_init_end +# --------------------------------------------------------------------------- + + +def test_on_init_pushes_latest_checkpoint(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + (tmp_path / "checkpoint-1000").mkdir() + (tmp_path / "checkpoint-1200").mkdir() + args = SimpleNamespace(output_dir=str(tmp_path)) + with patch.object(cb, "_push") as mock_push: + cb.on_init_end(args, SimpleNamespace(), _ctrl) + mock_push.assert_called_once() + _, step, commit_msg, *_ = mock_push.call_args[0] + assert step == 1200 + assert "resume push" in commit_msg + + +def test_on_init_skips_if_already_pushed(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + (tmp_path / "checkpoint-1200").mkdir() + (tmp_path / ".hf_last_push").write_text("1200") + args = SimpleNamespace(output_dir=str(tmp_path)) + with patch.object(cb, "_push") as mock_push: + cb.on_init_end(args, SimpleNamespace(), _ctrl) + mock_push.assert_not_called() + + +def test_on_init_skips_non_multiple_step(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + (tmp_path / "checkpoint-1670").mkdir() # 1670 % 50 != 0 + args = SimpleNamespace(output_dir=str(tmp_path)) + with patch.object(cb, "_push") as mock_push: + cb.on_init_end(args, SimpleNamespace(), _ctrl) + mock_push.assert_not_called() + + +def test_on_init_skips_if_no_checkpoints(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + args = SimpleNamespace(output_dir=str(tmp_path)) + with patch.object(cb, "_push") as mock_push: + cb.on_init_end(args, SimpleNamespace(), _ctrl) + mock_push.assert_not_called() + + +def test_on_init_picks_highest_step_numerically(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + (tmp_path / "checkpoint-90").mkdir() + (tmp_path / "checkpoint-100").mkdir() + args = SimpleNamespace(output_dir=str(tmp_path)) + with patch.object(cb, "_push") as mock_push: + cb.on_init_end(args, SimpleNamespace(), _ctrl) + mock_push.assert_called_once() + _, step, *_ = mock_push.call_args[0] + assert step == 100 + + +# --------------------------------------------------------------------------- +# Checkpoint log — SFT_STAGE2B_CHECKPOINT_LOG.md +# --------------------------------------------------------------------------- + + +def test_build_log_row_with_metrics(): + row = HFCheckpointCallback._build_log_row(1230, 0.255, 0.6465, 0.798) + assert row == "| checkpoint-1230/ | 1230 | 0.255 | 0.6465 | 0.7980 |" + + +def test_build_log_row_without_metrics(): + row = HFCheckpointCallback._build_log_row(1220, 0.253, None, None) + assert row == "| checkpoint-1220/ | 1220 | 0.253 | — | — |" + + +def test_update_checkpoint_log_starts_fresh_when_no_existing(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + with patch("huggingface_hub.HfApi.hf_hub_download", side_effect=Exception("not found")): + with patch("huggingface_hub.HfApi.upload_file") as mock_upload: + cb._update_checkpoint_log(50, 0.3, 0.65, 0.80) + mock_upload.assert_called_once() + content = mock_upload.call_args[1]["path_or_fileobj"].decode() + assert "checkpoint-50/" in content + assert "0.6500" in content + + +def test_update_checkpoint_log_appends_new_row(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + existing_file = tmp_path / "existing_log.md" + existing_file.write_text("| checkpoint-50/ | 50 | 0.300 | 0.6500 | 0.8000 |\n") + with patch("huggingface_hub.HfApi.hf_hub_download", return_value=str(existing_file)): + with patch("huggingface_hub.HfApi.upload_file") as mock_upload: + cb._update_checkpoint_log(100, 0.5, 1.23, 0.90) + mock_upload.assert_called_once() + content = mock_upload.call_args[1]["path_or_fileobj"].decode() + assert "checkpoint-50/" in content + assert "checkpoint-100/" in content + assert "1.2300" in content + + +def test_update_checkpoint_log_replaces_existing_row(tmp_path): + cb = HFCheckpointCallback("repo/id", push_every=50) + existing_file = tmp_path / "existing_log.md" + existing_file.write_text("| checkpoint-50/ | 50 | 0.300 | 0.6500 | 0.8000 |\n") + with patch("huggingface_hub.HfApi.hf_hub_download", return_value=str(existing_file)): + with patch("huggingface_hub.HfApi.upload_file") as mock_upload: + cb._update_checkpoint_log(50, 0.4, 0.70, 0.82) + mock_upload.assert_called_once() + content = mock_upload.call_args[1]["path_or_fileobj"].decode() + assert "checkpoint-50/" in content + assert "0.400" in content + assert content.count("checkpoint-50/") == 1 diff --git a/tests/test_kele_batch_eval.py b/tests/test_kele_batch_eval.py index bd65bf1c..ba1df04c 100644 --- a/tests/test_kele_batch_eval.py +++ b/tests/test_kele_batch_eval.py @@ -60,6 +60,45 @@ def test_run_batch_evaluation_creates_expected_output_files(tmp_path, monkeypatc assert json.loads((output_dir / "metrics_summary.json").read_text()) == metrics +_PROBE_SYSTEM = SimpleNamespace( + teacher_model_name="teacher-model", consultant_model_name="consultant-model" +) + + +def test_run_batch_evaluation_forwards_hf_repo_and_split_to_load_dataset(tmp_path, monkeypatch): + captured = {} + + def fake_load_dataset(*args, **kwargs): + captured.update(kwargs) + return [] + + monkeypatch.setattr(kele, "load_dataset", fake_load_dataset) + monkeypatch.setattr(kele, "create_system", lambda *a, **kw: _PROBE_SYSTEM) + monkeypatch.setattr(kele, "load_config", lambda *a, **kw: fake_config()) + install_fake_metrics(monkeypatch, {"rouge1": 0}) + + kele.run_batch_evaluation( + tmp_path / "results", + split="all", + hf_repo=["ulises-c/SocratDataset-SYNTHETIC"], + ) + + assert captured["split"] == "all" + assert captured["hf_repo"] == ["ulises-c/SocratDataset-SYNTHETIC"] + + +def test_run_batch_evaluation_omits_hf_repo_kwarg_when_unset(tmp_path, monkeypatch): + captured = {} + monkeypatch.setattr(kele, "load_dataset", lambda *a, **kw: captured.update(kw) or []) + monkeypatch.setattr(kele, "create_system", lambda *a, **kw: _PROBE_SYSTEM) + monkeypatch.setattr(kele, "load_config", lambda *a, **kw: fake_config()) + install_fake_metrics(monkeypatch, {"rouge1": 0}) + + kele.run_batch_evaluation(tmp_path / "results") + + assert "hf_repo" not in captured # falls back to load_dataset's default ZH repo + + def test_run_batch_evaluation_skips_existing_dialogue_files(tmp_path, monkeypatch): dataset = [ {"id": 1, "question": "Q1", "answer": "A1", "dialogue": []}, diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py new file mode 100644 index 00000000..236640aa --- /dev/null +++ b/tests/test_llm_judge.py @@ -0,0 +1,94 @@ +"""Unit tests for the LLM-judge rubric parser (scripts/llm_judge_eval.py). + +Only _parse_rubric is exercised — it's the fragile seam that turns free-form +judge output into validated axis scores. The backends themselves (openai client / +`claude -p` subprocess) are integration-only. +""" + +import importlib.util +from pathlib import Path + +_spec = importlib.util.spec_from_file_location( + "llm_judge_eval", Path(__file__).resolve().parents[1] / "scripts" / "llm_judge_eval.py" +) +assert _spec and _spec.loader +llm_judge_eval = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(llm_judge_eval) +_parse_rubric = llm_judge_eval._parse_rubric + + +def test_parse_rubric_clean_json(): + scores = _parse_rubric( + '{"socratic_validity": 3, "advancement": 2, "age_appropriateness": 2, ' + '"question_form": 1, "comment": "ok"}' + ) + assert scores is not None + assert scores["total"] == 8 + assert scores["socratic_validity"] == 3 + + +def test_parse_rubric_strips_markdown_fence(): + scores = _parse_rubric( + '```json\n{"socratic_validity": 1, "advancement": 1, ' + '"age_appropriateness": 0, "question_form": 0}\n```' + ) + assert scores is not None + assert scores["total"] == 2 + + +def test_parse_rubric_missing_axis_returns_none(): + assert _parse_rubric('{"socratic_validity": 3, "advancement": 2}') is None + + +def test_parse_rubric_non_json_returns_none(): + assert _parse_rubric("I cannot score this.") is None + + +def test_parse_rubric_coerces_stringified_ints(): + scores = _parse_rubric( + '{"socratic_validity": "3", "advancement": "0", ' + '"age_appropriateness": "1", "question_form": "2"}' + ) + assert scores is not None + assert scores["total"] == 6 + + +import json as _json + + +def _make_dialogues(tmp_path): + ddir = tmp_path / "dialogues" + ddir.mkdir() + # 10 dialogues, each with one turn per stage a-e (state codes a1,b4,c16,d33,e34). + states = ["a1", "b4", "c16", "d33", "e34"] + for n in range(10): + turns = [ + {"student": "s", "teacher_response": "t", "ground_truth_state": st} for st in states + ] + (ddir / f"{n:04d}.json").write_text(_json.dumps({"id": n, "dialogue": turns})) + return ddir + + +def test_stratified_sample_counts_and_stage_filtering(tmp_path): + ddir = _make_dialogues(tmp_path) + sample = llm_judge_eval.build_stratified_sample(ddir, per_stage=3, stages="bcde", seed=42) + counts = {} + for _f, _i, s in sample: + counts[s] = counts.get(s, 0) + 1 + assert counts == {"b": 3, "c": 3, "d": 3, "e": 3} # 3 each, no 'a' + assert len(sample) == 12 + + +def test_stratified_sample_deterministic_under_seed(tmp_path): + ddir = _make_dialogues(tmp_path) + s1 = llm_judge_eval.build_stratified_sample(ddir, per_stage=3, stages="bcde", seed=42) + s2 = llm_judge_eval.build_stratified_sample(ddir, per_stage=3, stages="bcde", seed=42) + s3 = llm_judge_eval.build_stratified_sample(ddir, per_stage=3, stages="bcde", seed=7) + assert s1 == s2 + assert s1 != s3 + + +def test_stratified_sample_takes_all_when_pool_small(tmp_path): + ddir = _make_dialogues(tmp_path) # only 10 turns per stage available + sample = llm_judge_eval.build_stratified_sample(ddir, per_stage=50, stages="c", seed=42) + assert len(sample) == 10 # capped at pool size, no error diff --git a/tests/test_sft_inference_format.py b/tests/test_sft_inference_format.py new file mode 100644 index 00000000..27a22f8f --- /dev/null +++ b/tests/test_sft_inference_format.py @@ -0,0 +1,445 @@ +"""Train/serve format-parity tests for the inference-matching SFT sources. + +Background (PR #101, issue #94): the first Stage-2b SFT model collapsed into a +non-terminating repetition loop at eval. Root cause was a train/serve schema +mismatch — the multi-turn `socrat-zh/en` training format looked nothing like the +single-message prompt `SocraticTeachingSystem.socrates_teacher()` builds at +inference. The `socrat-zh-sft` / `socrat-en-sft` sources were added to emit one +`(system, user, assistant)` triple per dialogue turn in that exact inference +shape. + +These tests are the durable, faithful version of the ad-hoc "render-diff" check +quoted in the PR. They capture the *real* prompt the inference path constructs +(by intercepting the teacher call) and assert the SFT record renders identically +under the training chat template — so any future drift on either side breaks CI +instead of silently re-triggering the collapse. + +Chat-template note: the Gemma 4 training template applies `{{ content | trim }}` +to every message (scripts/train_sft.py), and the serving template trims too. +`jinja2.trim` == Python `str.strip()`, so two messages render identically iff +their `(role, content.strip())` match. We model render-equivalence with that +rule — no tokenizer / model download required. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Mock data builders (mirror tests/test_dataset.py conventions — no HF network) +# --------------------------------------------------------------------------- + + +def _mock_hf(records: list[dict]) -> MagicMock: + m = MagicMock() + m.__iter__ = MagicMock(side_effect=lambda: iter(records)) + return m + + +def _dialogue_record(id_: int, n_turns: int = 2) -> dict: + """SocratDataset-shaped record with `n_turns` fully-annotated turns.""" + return { + "id": id_, + "question": f"Q{id_}", + "options": ["opt1", "opt2"], + "newHint": "hint", + "newKnowledgePoint": "kp", + "dialogue": [ + { + "student": f"学生输入{t}", + "teacher": f"老师回复{t}", + "state": ("a1", "b2", "c8", "d33")[t % 4], + "action": f"action_{t}", + "evaluation": f"评估{t}", + } + for t in range(n_turns) + ], + } + + +# --------------------------------------------------------------------------- +# Faithful inference-prompt capture +# --------------------------------------------------------------------------- + + +def _capture_inference_prompt( + history_pairs: list[tuple[str, str]], + student_input: str, + evaluation: str, + state: str, + action: str, +) -> dict: + """Return the (system, user) prompt socrates_teacher() actually sends. + + Drives the real production code path: builds the same conversation history, + then intercepts `call_teacher_wrapped` (the seam right after apply_pre_call, + so the captured strings are exactly what hits the teacher model) instead of + making a network call. + """ + from src.project.socratic_teaching_system import SocraticTeachingSystem + + system = SocraticTeachingSystem( + consultant_api_key="test", + consultant_base_url="http://localhost:0", + consultant_model_name="m", + teacher_api_key="test", + teacher_base_url="http://localhost:0", + teacher_model_name="m", + ) + for s, t in history_pairs: + system.add_to_history("student", s) + system.add_to_history("teacher", t) + # The current student turn is added (unpaired) before the teacher call, + # exactly as process_student_input does in production. + system.add_to_history("student", student_input) + system.current_state = state + + captured: dict = {} + + def fake_call(client, model, system_prompt, user_input, predicted_state): # noqa: ANN001 + captured["system"] = system_prompt + captured["user"] = user_input + return "captured" + + with patch( + "src.project.tournament_utilizations.call_teacher_wrapped", + side_effect=fake_call, + ): + system.socrates_teacher(student_input, evaluation, action) + + return captured + + +def _render_eq(content_a: str, content_b: str) -> bool: + """True iff the two message contents render identically under the chat + template's per-message `| trim` (== Python str.strip()).""" + return content_a.strip() == content_b.strip() + + +# =========================================================================== +# Loader unit tests — socrat-zh-sft / socrat-en-sft +# =========================================================================== + + +@pytest.mark.parametrize( + ("loader_name", "source"), + [("load_socrat_zh_sft", "socrat-zh-sft"), ("load_socrat_en_sft", "socrat-en-sft")], +) +def test_sft_loader_one_record_per_turn(loader_name, source): + import src.project.dataset as ds + + raw = [_dialogue_record(1, n_turns=3)] + with patch("datasets.load_dataset", return_value=_mock_hf(raw)): + records = getattr(ds, loader_name)(split="all") + + # 3 fully-annotated turns -> 3 per-turn records + assert len(records) == 3 + for i, r in enumerate(records): + assert r["source"] == source + assert r["id"] == f"1_{i}" + roles = [m["role"] for m in r["messages"]] + assert roles == ["system", "user", "assistant"] + assert r["ground_truth_states"] == [("a1", "b2", "c8", "d33")[i % 4]] + + +def test_sft_system_is_the_inference_rules_block(): + import src.project.dataset as ds + + raw = [_dialogue_record(1)] + with patch("datasets.load_dataset", return_value=_mock_hf(raw)): + records = ds.load_socrat_zh_sft(split="all") + + system = records[0]["messages"][0]["content"] + assert system == ds._TEACHER_INFERENCE_SYSTEM + # Hallmarks of the 7-line inference block — NOT the legacy 1-line system. + assert system.startswith("你是一位使用苏格拉底教学法的小学科学教师") + assert "以下是你需要遵守的规则：" in system + # Problem context must NOT leak in — inference never sends it. + assert "问题：Q1" not in system + assert "选项：" not in system + + +def test_sft_user_message_structure_and_history_accumulation(): + import src.project.dataset as ds + + raw = [_dialogue_record(7, n_turns=3)] + with patch("datasets.load_dataset", return_value=_mock_hf(raw)): + records = ds.load_socrat_zh_sft(split="all") + + first_user = records[0]["messages"][1]["content"] + second_user = records[1]["messages"][1]["content"] + + # Required inference labels, in order. Post Phase-0 the eval line carries the + # dataset's free-form `evaluation` field and the action line carries the + # canonical map output get_action_for_state("a1") — NOT the raw dataset action. + assert "历史对话记录:" in first_user + assert "当前学生输入: 学生输入0" in first_user + assert "苏格拉底教学顾问评估结果: 评估0" in first_user + assert "苏格拉底教学顾问建议的操作: 生成一个与解题相关的子问题" in first_user + + # Turn 0 has empty history; turn 1 carries turn 0's student+teacher verbatim. + assert "学生: 学生输入0" not in first_user + assert "学生: 学生输入0" in second_user + assert "老师: 老师回复0" in second_user + assert "当前学生输入: 学生输入1" in second_user + + +def test_sft_assistant_target_is_clean_teacher_turn(): + import src.project.dataset as ds + + raw = [_dialogue_record(1)] + with patch("datasets.load_dataset", return_value=_mock_hf(raw)): + records = ds.load_socrat_zh_sft(split="all") + + for i, r in enumerate(records): + asst = r["messages"][2]["content"] + assert asst == f"老师回复{i}" + # No consultant markers must leak into the loss target. + assert "苏格拉底教学顾问" not in asst + assert not asst.startswith("[State:") + + +def test_sft_skips_turns_without_state_action_but_keeps_them_in_history(): + import src.project.dataset as ds + + record = { + "id": 1, + "dialogue": [ + {"student": "S0", "teacher": "T0", "state": "a1", "action": "act0"}, + {"student": "S1", "teacher": "T1"}, # no state/action -> not a target + {"student": "S2", "teacher": "T2", "state": "b2", "action": "act2"}, + ], + } + with patch("datasets.load_dataset", return_value=_mock_hf([record])): + records = ds.load_socrat_zh_sft(split="all") + + # Only turns 0 and 2 become training targets. + assert [r["id"] for r in records] == ["1_0", "1_2"] + # But the skipped turn 1 still appears in turn 2's history blob. + third_user = records[1]["messages"][1]["content"] + assert "学生: S1" in third_user and "老师: T1" in third_user + assert "当前学生输入: S2" in third_user + + +def test_sft_split_keeps_a_dialogue_whole_no_history_contamination(): + """Regression for the d04bc83 review fix: splitting the flat per-turn list + scattered a dialogue's turns across train/test, leaking turn-N teacher + targets into a test record's history blob. Split must be dialogue-level.""" + import src.project.dataset as ds + + raw = [_dialogue_record(i, n_turns=2) for i in range(40)] + with patch("datasets.load_dataset", side_effect=lambda *a, **k: _mock_hf(raw)): + train = ds.load_socrat_zh_sft(split="train", seed=42) + test = ds.load_socrat_zh_sft(split="test", seed=42) + + def dlg_ids(records): + return {r["id"].rsplit("_", 1)[0] for r in records} + + assert len(train) + len(test) == 80 # 40 dialogues x 2 turns + assert dlg_ids(train).isdisjoint(dlg_ids(test)), ( + "a dialogue's turns must not span train/test — would contaminate eval history" + ) + + +def test_sft_sources_registered_in_unified_entry_point(): + import src.project.dataset as ds + + assert "socrat-zh-sft" in ds._SOURCE_LOADERS + assert "socrat-en-sft" in ds._SOURCE_LOADERS + + +# =========================================================================== +# Render-diff gate — SFT record vs the REAL inference prompt +# =========================================================================== + + +def test_sft_record_renders_identically_to_inference_prompt(): + """The core fix: an SFT training record must render byte-for-byte identically + (under the chat template's per-message trim) to the prompt socrates_teacher() + sends at inference, when the consultant fields match. + + Post Phase-0 (PR #101) the action dimension is un-rigged: the loader sources + the action from get_action_for_state(state) — exactly what process_student_input + feeds socrates_teacher — instead of the raw dataset action. So when the dataset + `evaluation` matches the live consultant eval, parity is total: system AND user + render-equal, action line included, zero drift.""" + import src.project.dataset as ds + from src.project.socratic_teaching_system import get_action_for_state + + s0, t0 = "学生输入0", "老师回复0" + s1, state = "学生输入1", "b2" + eval1 = "学生展示了部分理解，但需要进一步引导其推理。" + + record = { + "id": 99, + "dialogue": [ + { + "student": s0, + "teacher": t0, + "state": "a1", + "action": "action_0", + "evaluation": "学生刚刚提出了问题。", + }, + { + "student": s1, + "teacher": "老师回复1", + "state": state, + "action": "action_1", + "evaluation": eval1, + }, + ], + } + with patch("datasets.load_dataset", return_value=_mock_hf([record])): + records = ds.load_socrat_zh_sft(split="all") + + sft_msgs = records[1]["messages"] # the turn-1 training example + sft_system = sft_msgs[0]["content"] + sft_user = sft_msgs[1]["content"] + + # Inference sends the canonical action for the state and the live consultant's + # free-form eval. Feed the same eval text the dataset carries so the only thing + # under test is structural parity — including the now-matched action line. + inf = _capture_inference_prompt( + history_pairs=[(s0, t0)], + student_input=s1, + evaluation=eval1, + state=state, + action=get_action_for_state(state), + ) + + assert _render_eq(sft_system, inf["system"]), ( + f"system drift:\n SFT: {sft_system!r}\n INF: {inf['system']!r}" + ) + assert _render_eq(sft_user, inf["user"]), ( + f"user drift:\n SFT: {sft_user!r}\n INF: {inf['user']!r}" + ) + + +def test_inference_system_block_is_trim_equivalent_despite_raw_whitespace(): + """The inference system string carries a leading newline + trailing indent + that the SFT constant omits. Document that this is immaterial: the chat + template trims it away, so the two are render-equivalent. (Guards against a + 'fix' that adds raw whitespace and introduces asymmetric drift.)""" + inf = _capture_inference_prompt( + history_pairs=[], + student_input="S", + evaluation="学生处于 a1 状态", + state="a1", + action="act", + ) + import src.project.dataset as ds + + raw_inf_system = inf["system"] + assert raw_inf_system != ds._TEACHER_INFERENCE_SYSTEM, "expected raw whitespace difference" + assert _render_eq(raw_inf_system, ds._TEACHER_INFERENCE_SYSTEM), "trim must reconcile them" + + +def test_model_turn_terminator_is_inside_the_loss_mask(): + """Regression for the EOS-collapse root cause (PR #101 / issue #94). + + `assistant_only_loss=True` masks every token *outside* the + `{% generation %}` block to -100. If the model-role terminator `` + renders outside that block it receives zero gradient and the model never + learns to stop — the non-terminating repetition collapse seen on both the + 5090 and R9700 runs. TRL's own gemma3_training.jinja keeps `` + inside the block for exactly this reason. + + Renders the real training template through transformers' jinja machinery + (no tokenizer/model download) and asserts the assistant loss span covers the + terminator. TRL's guard only checks that `{% generation %}` is *present*, not + that it is placed correctly, so this is the test that actually pins it. + """ + import sys + from pathlib import Path + + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + import train_sft + from transformers.utils.chat_template_utils import render_jinja_template + + conversation = [ + {"role": "system", "content": "SYS"}, + {"role": "user", "content": "U"}, + {"role": "assistant", "content": "老师回复"}, + ] + rendered, generation_indices = render_jinja_template( + conversations=[conversation], + chat_template=train_sft._GEMMA4_TRAINING_CHAT_TEMPLATE, + return_assistant_tokens_mask=True, + bos_token="", + ) + text, spans = rendered[0], generation_indices[0] + + assert len(spans) == 1, f"expected exactly one assistant loss span, got {spans}" + in_mask = text[spans[0][0] : spans[0][1]] + assert in_mask == "老师回复\n", ( + "the assistant loss span must be exactly content + terminator; " + f"got {in_mask!r} — terminator masking is broken" + ) + # The terminator must be trained... + assert "" in in_mask, ( + "model-turn terminator is OUTSIDE the loss mask — the model " + "will never learn to emit EOS (non-terminating repetition collapse)" + ) + # ...and the prompt-cue header must NOT be (it is not generated by the model). + assert "<|turn>model" not in in_mask + + +def test_evaluation_line_is_the_only_residual_drift_under_freeform_consultant(): + """The sole remaining residual after Phase 0 (PR #101): training carries the + dataset's *stored* `evaluation` annotation, but at inference the live consultant + regenerates free-form prose on that line, so the two seldom match token-for-token. + The action line no longer drifts (both sides use get_action_for_state). This test + pins the residual to *exactly one line* — if a future change reintroduces drift on + any OTHER line (e.g. the action line), this breaks, distinguishing a new regression + from the already-accepted, irreducible evaluation-line gap. + """ + import src.project.dataset as ds + from src.project.socratic_teaching_system import get_action_for_state + + s0, t0 = "学生输入0", "老师回复0" + s1, state = "学生输入1", "b2" + stored_eval = "数据集中标注的评估文本。" + + record = { + "id": 99, + "dialogue": [ + { + "student": s0, + "teacher": t0, + "state": "a1", + "action": "action_0", + "evaluation": "学生刚刚提出了问题。", + }, + { + "student": s1, + "teacher": "老师回复1", + "state": state, + "action": "action_1", + "evaluation": stored_eval, + }, + ], + } + with patch("datasets.load_dataset", return_value=_mock_hf([record])): + records = ds.load_socrat_zh_sft(split="all") + sft_user = records[1]["messages"][1]["content"] + + freeform_eval = "学生已理解基本概念，但在具体应用时仍有困难，建议通过追问巩固其推理过程。" + inf = _capture_inference_prompt( + history_pairs=[(s0, t0)], + student_input=s1, + evaluation=freeform_eval, + state=state, + action=get_action_for_state(state), + ) + + sft_lines = sft_user.strip().splitlines() + inf_lines = inf["user"].strip().splitlines() + assert len(sft_lines) == len(inf_lines), "line-count drift beyond the evaluation line" + + diffs = [(a, b) for a, b in zip(sft_lines, inf_lines, strict=True) if a != b] + assert len(diffs) == 1, f"expected exactly one drifting line, got {diffs}" + sft_line, inf_line = diffs[0] + assert sft_line == f"苏格拉底教学顾问评估结果: {stored_eval}" + assert inf_line == f"苏格拉底教学顾问评估结果: {freeform_eval}" diff --git a/tests/test_socratic_teaching_system.py b/tests/test_socratic_teaching_system.py index f7b3bb3c..0a5ed7b6 100644 --- a/tests/test_socratic_teaching_system.py +++ b/tests/test_socratic_teaching_system.py @@ -14,7 +14,10 @@ import src.project.socratic_teaching_system as stss import src.project.tournament_utilizations as tu -from src.project.socratic_teaching_system import SocraticTeachingSystem +from src.project.socratic_teaching_system import ( + SocraticTeachingSystem, + SocraticTeachingSystemOracle, +) def make_system(**overrides): @@ -30,6 +33,54 @@ def make_system(**overrides): return SocraticTeachingSystem(**kwargs) +def make_oracle(**overrides): + kwargs = dict( + consultant_api_key="x", + consultant_base_url="http://localhost:1/v1", + consultant_model_name="consultant", + teacher_api_key="x", + teacher_base_url="http://localhost:2/v1", + teacher_model_name="teacher", + ) + kwargs.update(overrides) + return SocraticTeachingSystemOracle(**kwargs) + + +def test_oracle_honors_primed_ground_truth_state_and_derives_action(monkeypatch): + sys = make_oracle() + monkeypatch.setattr(sys, "socrates_teacher", lambda si, ev, ac: f"[teacher:{ac}]") + + sys.prime_oracle_state("c10") + resp = sys.process_student_input("为什么？") + + assert sys.current_state == "c10" + last = sys.consultant_history[-1] + assert last["state"] == "c10" + assert last["action"] == sys.get_action_for_state("c10") + assert "标准答案状态" in last["evaluation"] + assert resp == f"[teacher:{sys.get_action_for_state('c10')}]" + + +def test_oracle_bypasses_max_round_guard(monkeypatch): + # More teaching turns than max_teaching_rounds must NOT force d33 — the GT + # state is authoritative (this is the whole point of the oracle). + sys = make_oracle(max_teaching_rounds=2) + monkeypatch.setattr(sys, "socrates_teacher", lambda si, ev, ac: "ok") + + for st in ["b2", "c8", "c9", "c10"]: # 4 teaching turns, max=2 + sys.prime_oracle_state(st) + sys.process_student_input("x") + + assert sys.current_state == "c10" # not clamped to d33 by the base guard + + +def test_oracle_falls_back_to_current_state_when_unprimed(monkeypatch): + sys = make_oracle() + monkeypatch.setattr(sys, "socrates_teacher", lambda si, ev, ac: "ok") + sys.process_student_input("x") # never primed + assert sys.current_state == "a0" + + def test_reset_session_initial_state(): sys = make_system() assert sys.current_state == "a0" diff --git a/uv.lock b/uv.lock index 163a5c2c..76c8f037 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,18 @@ resolution-markers = [ "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] +[[package]] +name = "abnf" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/f2/7b5fac50ee42e8b8d4a098d76743a394546f938c94125adbb93414e5ae7d/abnf-2.2.0.tar.gz", hash = "sha256:433380fd32855bbc60bc7b3d35d40616e21383a32ed1c9b8893d16d9f4a6c2f4", size = 197507, upload-time = "2023-03-17T18:26:24.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/95/f456ae7928a2f3a913f467d4fd9e662e295dd7349fc58b35f77f6c757a23/abnf-2.2.0-py3-none-any.whl", hash = "sha256:5dc2ae31a84ff454f7de46e08a2a21a442a0e21a092468420587a1590b490d1f", size = 39938, upload-time = "2023-03-17T18:26:22.608Z" }, +] + [[package]] name = "absl-py" version = "2.4.0" @@ -281,6 +293,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/22/91616fe707a5c5510de2cac9b046a30defe7007ba8a0c04f9c08f27df312/audioop_lts-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:b492c3b040153e68b9fdaff5913305aaaba5bb433d8a7f73d5cf6a64ed3cc1dd", size = 25206, upload-time = "2025-08-05T16:43:16.444Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "bidict" version = "0.23.1" @@ -306,6 +327,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/d4/501655842ad6771fb077f576d78cbedb5445d15b1c3c91343ed58ca46f0e/bitsandbytes-0.49.2-py3-none-win_amd64.whl", hash = "sha256:2e0ddd09cd778155388023cbe81f00afbb7c000c214caef3ce83386e7144df7d", size = 55372289, upload-time = "2026-02-16T21:26:16.267Z" }, ] +[[package]] +name = "cachetools" +version = "7.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f4/8b/0d3945a13955303b81272f759a0331e54c5c793da455e6f5706b89d2639c/cachetools-7.1.4.tar.gz", hash = "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6", size = 40085, upload-time = "2026-05-21T22:40:43.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/7b/1fc1c09cc0756cf25861a3be10565915953876da48bb228fb9a672b20a42/cachetools-7.1.4-py3-none-any.whl", hash = "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", size = 16761, upload-time = "2026-05-21T22:40:41.845Z" }, +] + [[package]] name = "certifi" version = "2026.4.22" @@ -407,6 +437,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/a2/3bbabfdb6bcbb9b0f952eac62ff277464d6359d126ce8f254ba513c1d4c8/chainlit-2.11.1-py3-none-any.whl", hash = "sha256:7cbd341a6cb2b7788845713dc41f86a44c6b36073dcd0fedb0fc7f230b542ec7", size = 11314614, upload-time = "2026-04-22T00:56:26.413Z" }, ] +[[package]] +name = "chardet" +version = "7.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/b6/9df434a8eeba2e6628c465a1dfa31034228ef79b26f76f46278f4ef7e49d/chardet-7.4.3.tar.gz", hash = "sha256:cc1d4eb92a4ec1c2df3b490836ffa46922e599d34ce0bb75cf41fd2bf6303d56", size = 784800, upload-time = "2026-04-13T21:33:39.803Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/33/29de185079e6675c3f375546e30a559b7ddc75ce972f18d6e566cd9ea4eb/chardet-7.4.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:75d3c65cc16bddf40b8da1fd25ba84fca5f8070f2b14e86083653c1c85aee971", size = 874870, upload-time = "2026-04-13T21:33:05.977Z" }, + { url = "https://files.pythonhosted.org/packages/9c/2f/4c5af01fd1a7506a1d5375403d68925eac70289229492db5aa68b58103d8/chardet-7.4.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:29af5999f654e8729d251f1724a62b538b1262d9292cccaefddf8a02aae1ef6a", size = 854859, upload-time = "2026-04-13T21:33:07.381Z" }, + { url = "https://files.pythonhosted.org/packages/36/21/edb36ad5dfa48d7f8eed97ab43931ecdaa8c15166c21b1d614967e49d681/chardet-7.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:626f00299ad62dfe937058a09572beed442ccc7b58f87aa667949b20fd3db235", size = 875032, upload-time = "2026-04-13T21:33:08.741Z" }, + { url = "https://files.pythonhosted.org/packages/e5/59/a32a241d861cf180853a11c8e5a67641cb1b2af13c3a5ccce83ec07e2c9f/chardet-7.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a4904dd5f071b7a7d7f50b4a67a86db3c902d243bf31708f1d5cde2f68239cb", size = 888283, upload-time = "2026-04-13T21:33:10.213Z" }, + { url = "https://files.pythonhosted.org/packages/87/2e/e1ee6a77abf3782c00e05b89c4d4328c8353bf9500661c4348df1dd68614/chardet-7.4.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d2879598bc220689e8ce509fe9c3f37ad2fca53a36be9c9bd91abdd91dd364f", size = 879974, upload-time = "2026-04-13T21:33:11.448Z" }, + { url = "https://files.pythonhosted.org/packages/32/60/fca69c534602a7ced04280c952a246ad1edde2a6ca3a164f65d32ac41fe7/chardet-7.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:4b2799bd58e7245cfa8d4ab2e8ad1d76a5c3a5b1f32318eb6acca4c69a3e7101", size = 943973, upload-time = "2026-04-13T21:33:12.756Z" }, + { url = "https://files.pythonhosted.org/packages/7c/43/79ac9b4db5bc87020c9dbc419125371d80882d1d197e9c4765ba8682b605/chardet-7.4.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9e4486df251b8962e86ea9f139ca235aa6e0542a00f7844c9a04160afb99aa9", size = 873769, upload-time = "2026-04-13T21:33:14.002Z" }, + { url = "https://files.pythonhosted.org/packages/55/5f/25bdec773905bff0ff6cf35ca73b17bd05593b4f87bd8c5fa43705f7167d/chardet-7.4.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fbff1907925b0c5a1064cffb5e040cd5e338585c9c552625f30de6bc2f3107a", size = 853991, upload-time = "2026-04-13T21:33:15.564Z" }, + { url = "https://files.pythonhosted.org/packages/b4/07/a29380ee0b215d23d77733b5ad60c5c0c7969650e080c667acdf9462040d/chardet-7.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:365135eaf37ba65a828f8e668eb0a8c38c479dcbec724dc25f4dfd781049c357", size = 874024, upload-time = "2026-04-13T21:33:16.915Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b1/3338e121cbd4c8a126b8ccb1061170c2ce51a53f678c502793ea49c6fd6d/chardet-7.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfc134b70c846c21ead8e43ada3ae1a805fff732f6922f8abcf2ff27b8f6493d", size = 887410, upload-time = "2026-04-13T21:33:18.368Z" }, + { url = "https://files.pythonhosted.org/packages/63/1c/44a9a9e0c59c185a5d307ceaeee8768afa1558f0a24f7a4b5fa11b67586b/chardet-7.4.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9acd9988a93e09390f3cd231201ea7166c415eb8da1b735928990ffc05cb9fbb", size = 879269, upload-time = "2026-04-13T21:33:20.377Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b3/5d0e77ea774bd3224321c248880ea0c0379000ac5c2bb6d77609549de247/chardet-7.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:e1b98790c284ff813f18f7cf7de5f05ea2435a080030c7f1a8318f3a4f80b131", size = 944155, upload-time = "2026-04-13T21:33:21.694Z" }, + { url = "https://files.pythonhosted.org/packages/70/a8/bf0811d859e13801279a2ae64f37a408027b282f2047bc0001c75dd356ad/chardet-7.4.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d892d3dcd652fdef53e3d6327d39b17c0df40a899dfc919abaeb64c974497531", size = 872887, upload-time = "2026-04-13T21:33:23.328Z" }, + { url = "https://files.pythonhosted.org/packages/51/ac/b9d68ebddfe1b02c77af5bf81120e12b036b4432dc6af7a303d90e2bc38b/chardet-7.4.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:acc46d1b8b7d5783216afe15db56d1c179b9a40e5a1558bc13164c4fd20674c4", size = 853964, upload-time = "2026-04-13T21:33:24.724Z" }, + { url = "https://files.pythonhosted.org/packages/2a/81/17fa103ea9caf5d325a5e4051ab2ba65996fd66baa60b81ee41af1f54e10/chardet-7.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ac3bf11c645734a1701a3804e43eabd98851838192267d08c353a834ab79fea", size = 876006, upload-time = "2026-04-13T21:33:26.098Z" }, + { url = "https://files.pythonhosted.org/packages/c2/20/193faab46a68ea550587331a698c3dca8099f8901d10937c4443135c7ed9/chardet-7.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e3bd9f936e04bae89c254262af08d9e5b98f805175ba1e29d454e6cba3107b7", size = 887680, upload-time = "2026-04-13T21:33:27.49Z" }, + { url = "https://files.pythonhosted.org/packages/40/c6/94a3c673327392652ee8bdea9a45bc8a5f5365197a7387d68f0eed007115/chardet-7.4.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:27cc23da03630cdecc9aa81a895aa86629c211f995cd57651f0fbc280717bf93", size = 879865, upload-time = "2026-04-13T21:33:29.052Z" }, + { url = "https://files.pythonhosted.org/packages/b1/2c/cad8b5e3623a987f3c930b68e2bdd06cfc388cd91cd42ed05f1227701b73/chardet-7.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:b95c934b9ad59e2ba8abb9be49df70d3ad1b0d95d864b9fdb7588d4fa8bd921c", size = 939594, upload-time = "2026-04-13T21:33:31.391Z" }, + { url = "https://files.pythonhosted.org/packages/33/e0/d06e42fd6f02a58e5e227e5106587751cb38adcff0aaf949add744b78b6e/chardet-7.4.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c77867f0c1cb8bd819502249fcdc500364aedb07881e11b743726fa2148e7b6e", size = 889714, upload-time = "2026-04-13T21:33:32.772Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ed/40d091954d48abea037baae6be8fb79905e5f78d34d12ea955132c7d8011/chardet-7.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cf1efeaf65a6ef2f5b9cc3a1df6f08ba2831b369ccaa4c7018eaf90aa757bb11", size = 872319, upload-time = "2026-04-13T21:33:34.427Z" }, + { url = "https://files.pythonhosted.org/packages/bb/77/82a46821dbfbdfe062710d2bf2ede13426304e3567a23c57d919c0c31630/chardet-7.4.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f3504c139a2ad544077dd2d9e412cd08b01786843d76997cd43bb6de311723c", size = 892021, upload-time = "2026-04-13T21:33:35.766Z" }, + { url = "https://files.pythonhosted.org/packages/49/57/42d30c562bda5b4a839766c1aad8d5856b798ad2a1c3247b72a679afec94/chardet-7.4.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457f619882ba66327d4d8d14c6c342269bdb1e4e1c38e8117df941d14d351b04", size = 902509, upload-time = "2026-04-13T21:33:37.096Z" }, + { url = "https://files.pythonhosted.org/packages/8c/6c/0a40afdb50a0fe041ab95553b835a8160b6cf0e81edf2ae2fe9f5224cbf9/chardet-7.4.3-py3-none-any.whl", hash = "sha256:1173b74051570cf08099d7429d92e4882d375ad4217f92a6e5240ccfb26f231e", size = 626562, upload-time = "2026-04-13T21:33:38.559Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.7" @@ -489,6 +550,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/93/342cc62a70ab727e093ed98e02a725d85b746345f05d2b5e5034649f4ec8/chevron-0.14.0-py3-none-any.whl", hash = "sha256:fbf996a709f8da2e745ef763f482ce2d311aa817d287593a5b990d6d6e4f0443", size = 11595, upload-time = "2021-01-02T22:47:57.847Z" }, ] +[[package]] +name = "cint" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/c8/3ae22fa142be0bf9eee856e90c314f4144dfae376cc5e3e55b9a169670fb/cint-1.0.0.tar.gz", hash = "sha256:66f026d28c46ef9ea9635be5cb342506c6a1af80d11cb1c881a8898ca429fc91", size = 4641, upload-time = "2019-03-19T01:07:48.723Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/c2/898e59963084e1e2cbd4aad1dee92c5bd7a79d121dcff1e659c2a0c2174e/cint-1.0.0-py3-none-any.whl", hash = "sha256:8aa33028e04015711c0305f918cb278f1dc8c5c9997acdc45efad2c7cb1abf50", size = 5573, upload-time = "2019-03-19T01:07:46.496Z" }, +] + [[package]] name = "click" version = "8.3.3" @@ -747,6 +817,7 @@ dependencies = [ { name = "transformers" }, { name = "trl" }, { name = "uvicorn" }, + { name = "wandb" }, ] [package.optional-dependencies] @@ -759,6 +830,9 @@ demo = [ wandb = [ { name = "wandb" }, ] +weave = [ + { name = "weave" }, +] [package.dev-dependencies] dev = [ @@ -794,9 +868,11 @@ requires-dist = [ { name = "transformers", specifier = ">=5.9.0" }, { name = "trl", specifier = ">=1.5.1" }, { name = "uvicorn", specifier = ">=0.49.0" }, + { name = "wandb", specifier = ">=0.18.0" }, { name = "wandb", marker = "extra == 'wandb'", specifier = ">=0.27.2" }, + { name = "weave", marker = "extra == 'weave'", specifier = ">=0.52" }, ] -provides-extras = ["wandb", "cuda", "demo"] +provides-extras = ["wandb", "cuda", "demo", "weave"] [package.metadata.requires-dev] dev = [ @@ -951,6 +1027,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] +[[package]] +name = "diskcache-weave" +version = "5.6.3.post1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/52/634e1f43486489fdaded1a7c9bd3524b7e0ca9bcc43af426afa511c541e2/diskcache_weave-5.6.3.post1.tar.gz", hash = "sha256:1fe7e648d1d85d517c05b296f1692e7c425a71714dc31a4b7a584a8f8f5604f2", size = 68297, upload-time = "2026-03-19T14:57:54.299Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/8d/92887441bc338fb8d0b8ea75eb0392c00e20a85ec0bbe02f273188849568/diskcache_weave-5.6.3.post1-py3-none-any.whl", hash = "sha256:b00e9842b74eeecf314456f9c833a6d4f7792ed12b20297b4d3b9df7859ee66f", size = 45905, upload-time = "2026-03-19T14:57:52.819Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -985,6 +1070,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, ] +[[package]] +name = "fickling" +version = "0.1.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/0b/0aea3be8edd5a8de3c1491a12f0149f6db5afd9467dfddaa5ed24a27bef9/fickling-0.1.11.tar.gz", hash = "sha256:3ca0dcc69967c53868b35787017d4d7d8943f096450431f7e3b3a9aadb02b0f5", size = 357476, upload-time = "2026-05-06T15:04:21.869Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/3b/45b8233feb53dd9da16208b039507604844a07c8b5bb3c5a4e39c520f32d/fickling-0.1.11-py3-none-any.whl", hash = "sha256:19ecb791d781d475e84ed951dc2c4a0c852108e237416d517ab0a8dd771d4098", size = 58549, upload-time = "2026-05-06T15:04:20.168Z" }, +] + [[package]] name = "filelock" version = "3.29.0" @@ -1248,6 +1342,44 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, ] +[[package]] +name = "gql" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "backoff" }, + { name = "graphql-core" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/9f/cf224a88ed71eb223b7aa0b9ff0aa10d7ecc9a4acdca2279eb046c26d5dc/gql-4.0.0.tar.gz", hash = "sha256:f22980844eb6a7c0266ffc70f111b9c7e7c7c13da38c3b439afc7eab3d7c9c8e", size = 215644, upload-time = "2025-08-17T14:32:35.397Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/94/30bbd09e8d45339fa77a48f5778d74d47e9242c11b3cd1093b3d994770a5/gql-4.0.0-py3-none-any.whl", hash = "sha256:f3beed7c531218eb24d97cb7df031b4a84fdb462f4a2beb86e2633d395937479", size = 89900, upload-time = "2025-08-17T14:32:34.029Z" }, +] + +[package.optional-dependencies] +httpx = [ + { name = "httpx" }, +] + +[[package]] +name = "graphql-core" +version = "3.2.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/90/f2aff026ab4aebd80eb71905106a0885f4cfde85dcf965543f45bed0d9ee/graphql_core-3.2.11.tar.gz", hash = "sha256:e7e156d10beb127cab5c89ff0da71416fc73d27c484a4757d3b2d35633774802", size = 528407, upload-time = "2026-06-05T13:45:22.915Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/15/b92b4e1d88d02c6eff9733c9eea21846ab435cc4d813d84ccc5d335955df/graphql_core-3.2.11-py3-none-any.whl", hash = "sha256:0b3e35ff41e9adba53021ab0cef475eb18f57c7f53f0f2ca55567fbf3c537ea0", size = 214879, upload-time = "2026-06-05T13:45:21.245Z" }, +] + +[[package]] +name = "graphviz" +version = "0.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/b3/3ac91e9be6b761a4b30d66ff165e54439dcd48b83f4e20d644867215f6ca/graphviz-0.21.tar.gz", hash = "sha256:20743e7183be82aaaa8ad6c93f8893c923bd6658a04c32ee115edb3c8a835f78", size = 200434, upload-time = "2025-06-15T09:35:05.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/4c/e0ce1ef95d4000ebc1c11801f9b944fa5910ecc15b5e351865763d8657f8/graphviz-0.21-py3-none-any.whl", hash = "sha256:54f33de9f4f911d7e84e4191749cac8cc5653f815b06738c54db9a15ab8b1e42", size = 47300, upload-time = "2025-06-15T09:35:04.433Z" }, +] + [[package]] name = "grpcio" version = "1.81.0" @@ -1414,6 +1546,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "intervaltree" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/c3/b2afa612aa0373f3e6bb190e6de35f293b307d1537f109e3e25dbfcdf212/intervaltree-3.2.1.tar.gz", hash = "sha256:f3f7e8baeb7dd75b9f7a6d33cf3ec10025984a8e66e3016d537e52130c73cfe2", size = 1231531, upload-time = "2025-12-24T04:25:06.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/7f/8a80a1c7c2ed05822b5a2b312d2995f30c533641f8198366ba2e26a7bb03/intervaltree-3.2.1-py2.py3-none-any.whl", hash = "sha256:a8a8381bbd35d48ceebee932c77ffc988492d22fb1d27d0ba1d74a7694eb8f0b", size = 25929, upload-time = "2025-12-24T04:25:05.298Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1534,6 +1678,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "kaitaistruct" +version = "0.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/b8/ca7319556912f68832daa4b81425314857ec08dfccd8dbc8c0f65c992108/kaitaistruct-0.11.tar.gz", hash = "sha256:053ee764288e78b8e53acf748e9733268acbd579b8d82a427b1805453625d74b", size = 11519, upload-time = "2025-09-08T15:46:25.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/4a/cf14bf3b1f5ffb13c69cf5f0ea78031247790558ee88984a8bdd22fae60d/kaitaistruct-0.11-py2.py3-none-any.whl", hash = "sha256:5c6ce79177b4e193a577ecd359e26516d1d6d000a0bffd6e1010f2a46a62a561", size = 11372, upload-time = "2025-09-08T15:46:23.635Z" }, +] + [[package]] name = "kiwisolver" version = "1.5.0" @@ -3055,6 +3208,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/2b/f8434233fab2bd66a02ec014febe4e5adced20e2693e0e90a07d118ed30e/pandas-3.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:5371b72c2d4d415d08765f32d689217a43227484e81b2305b52076e328f6f482", size = 9455341, upload-time = "2026-03-31T06:48:28.418Z" }, ] +[[package]] +name = "pdfminer-six" +version = "20260107" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "cryptography" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/a4/5cec1112009f0439a5ca6afa8ace321f0ab2f48da3255b7a1c8953014670/pdfminer_six-20260107.tar.gz", hash = "sha256:96bfd431e3577a55a0efd25676968ca4ce8fd5b53f14565f85716ff363889602", size = 8512094, upload-time = "2026-01-07T13:29:12.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/8b/28c4eaec9d6b036a52cb44720408f26b1a143ca9bce76cc19e8f5de00ab4/pdfminer_six-20260107-py3-none-any.whl", hash = "sha256:366585ba97e80dffa8f00cebe303d2f381884d8637af4ce422f1df3ef38111a9", size = 6592252, upload-time = "2026-01-07T13:29:10.742Z" }, +] + [[package]] name = "peft" version = "0.19.1" @@ -3163,6 +3329,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "polyfile-weave" +version = "0.5.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "abnf" }, + { name = "chardet" }, + { name = "cint" }, + { name = "fickling" }, + { name = "filelock" }, + { name = "graphviz" }, + { name = "intervaltree" }, + { name = "jinja2" }, + { name = "kaitaistruct" }, + { name = "networkx" }, + { name = "pdfminer-six" }, + { name = "pillow" }, + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/55/e5400762e3884f743d59291e71eaaa9c52dd7e144b75a11911e74ec1bac9/polyfile_weave-0.5.9.tar.gz", hash = "sha256:12341fab03e06ede1bfebbd3627dd24015fde5353ea74ece2da186321b818bdb", size = 6024974, upload-time = "2026-01-22T22:08:48.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/94/215005530a48c5f7d4ec4a31acdb5828f2bfb985cc6e577b0eaa5882c0e2/polyfile_weave-0.5.9-py3-none-any.whl", hash = "sha256:6ae4b1b5eeac9f5bfc862474484d6d3e33655fab31749d93af0b0a91fddabfc7", size = 1700174, upload-time = "2026-01-22T22:08:46.346Z" }, +] + [[package]] name = "portalocker" version = "3.2.0" @@ -3521,6 +3712,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, ] +[[package]] +name = "pyreadline3" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/6d/f94028646d7bbe6d9d873c47ee7c246f2d29129d253f0d96cb6fcab70733/pyreadline3-3.5.6.tar.gz", hash = "sha256:61e53218b99656091ddb077df9e71f25850e72e030b6183b39c9b7e6e4f4a9bf", size = 100368, upload-time = "2026-05-14T17:55:04.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/5e/35c856e186b74678c24927847ad9895a51f1bc02a0c6126477a6c6040064/pyreadline3-3.5.6-py3-none-any.whl", hash = "sha256:8449b734232e42a5dcd74048e39b60db2839a4c38cf3ae2bf7707d58b5389c0d", size = 85243, upload-time = "2026-05-14T17:55:03.262Z" }, +] + [[package]] name = "pyright" version = "1.1.409" @@ -4181,6 +4381,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sse-starlette" version = "3.4.4" @@ -4487,7 +4696,7 @@ wheels = [ [[package]] name = "transformers" -version = "5.9.0" +version = "5.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, @@ -4500,9 +4709,9 @@ dependencies = [ { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/58/7f843608f2e8421f86bb97060b54649be6239ec612b82bf9d41e65c26c00/transformers-5.9.0.tar.gz", hash = "sha256:25997cb8fa6053533171634b6162d7df54346530ec2aa9b42bb834e63668c842", size = 8642240, upload-time = "2026-05-20T14:50:49.278Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/38/d5f978bd5091019e89aef29b9a831f5cd70f2598963a3ead8b9570cab592/transformers-5.10.2.tar.gz", hash = "sha256:f9a44b9c8ca9ab1156b467f574d832ea066284299c2fd0ed84641ccb592751fc", size = 8799687, upload-time = "2026-06-04T18:43:49.119Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/ca/2eaa5359f2ccb8c2e1656bc26305ad0cf438aa392ce4b29ae67a315c186e/transformers-5.9.0-py3-none-any.whl", hash = "sha256:1d19509bcff7028ebc6b277d71caa712e8353778463d38764237d14b42b52788", size = 10787648, upload-time = "2026-05-20T14:50:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/73/6f/e1564b0cc182afa05e219a8e09a8e770ffaab879b6b824b56c819bd221da/transformers-5.10.2-py3-none-any.whl", hash = "sha256:8a669db546f82c7c3618cb46ceb0f0afd89292bc70f319c058f8332ec63e268d", size = 11003830, upload-time = "2026-06-04T18:43:45.303Z" }, ] [[package]] @@ -4733,6 +4942,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/4b/95ab2f256bb4af3cb2eb23b9317bda984ee6e0f11733a5c004a6c95b06e3/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377", size = 657684, upload-time = "2026-05-18T04:31:32.027Z" }, ] +[[package]] +name = "weave" +version = "0.52.42" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "click" }, + { name = "diskcache-weave" }, + { name = "gql", extra = ["httpx"] }, + { name = "jsonschema" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "packaging" }, + { name = "polyfile-weave" }, + { name = "pydantic" }, + { name = "sentry-sdk" }, + { name = "tenacity" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e6/50/f17558839bd9c387e6182149d0464c779e1db3be811444fcd195ddbd8f17/weave-0.52.42.tar.gz", hash = "sha256:900baa9cedc5bad960f68d8db525f51e04c5c4f0e6a3d9872b39d8187356611f", size = 985375, upload-time = "2026-06-02T03:31:28.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/cc/73624931672c9983191ad7e71e618f658629ccd350fd14a413520929d0e6/weave-0.52.42-py3-none-any.whl", hash = "sha256:6dc5fba5116c8d26e1aedce4f89662dc9f24100123da7265c5ab06df1a9f7fa8", size = 1206879, upload-time = "2026-06-02T03:31:26.344Z" }, +] + [[package]] name = "websockets" version = "16.0"