feat(release): distributed trainer, full SGLang pipeline migration, 5k unique-rollout cap

.cursor/rules/dependencies.mdc

@@ -6,7 +6,7 @@ alwaysApply: true
 # Dependencies (Concise)
 
 - Use `uv` only. Pin exact versions in `pyproject.toml`; run `uv lock` and `uv sync --all-extras`. Do not edit `uv.lock`.
-- Python 3.9–3.11.
+- Python 3.12+.
 - Lint/format: Ruff — `uv run ruff check .`, `uv run ruff format .`.
 - Defaults: full type hints; `logging.getLogger(__name__)` (no `print`); PEP 8, 100 cols; asyncio with timeouts; seed RNG; never log secrets; use dotenv; HTTPS.
 - Compatibility: keep Torch/Transformers/Tokenizers aligned; align `bittensor` with CLI/network.

.cursor/rules/project_context.mdc

@@ -18,7 +18,7 @@ GRAIL (Guaranteed Rollout Authenticity via Inference Ledger) enables cryptograph
 - Typer CLI: `grail mine`, `grail validate`, `grail train`
 
 **Non-Functional**:
-- Python 3.10 or later; seed `random`, `numpy`, `torch` for determinism
+- Python 3.12 or later; seed `random`, `numpy`, `torch` for determinism
 - Reproducibility: manage deps with `uv` (pin exact versions); use `safetensors`
 - Reliability: timeouts on all network calls; avoid blocking event loop; handle transient failures
 - Security: never log secrets; load config via environment; HTTPS with cert verification

.env.example

@@ -183,27 +183,34 @@ KERNEL_EVAL_BACKEND=persistent
 KERNEL_EVAL_TIMEOUT=60
 
 # ============================================================================
-# PIPELINED MINING CONFIGURATION (Recommended for 2+ GPUs)
+# PIPELINED MINING CONFIGURATION (mandatory; requires 2+ GPUs)
 # ============================================================================
-# Pipeline mode is the recommended way to run the miner with multiple GPUs.
-# It overlaps proof computation (GPU 1) with kernel eval after generation (GPU 0),
-# significantly improving per-window throughput.
+# The pipelined mining engine is the only generation path: the legacy
+# single-GPU fallback was removed in the Stage 1 refactor and the
+# GRAIL_PIPELINE_ENABLED flag was deleted (setting it has no effect).
+#
+# Pipeline mode overlaps proof computation (GPU 1) with kernel eval and
+# generation (GPU 0), significantly improving per-window throughput.
 #
 # GPU layout:
-#   GPU 0 = vLLM/SGLang generation server
+#   GPU 0 = SGLang/vLLM generation server
 #   GPU 1 = HuggingFace model for proof-of-work (logprobs + commitments)
 #   GPU 2 = Triton kernel evaluation (set via KERNEL_EVAL_GPU_IDS above)
 #
-# Requires at least 3 GPUs. On a 4-GPU machine, GPU 3 can be used for
-# the validator or as a second kernel eval GPU (KERNEL_EVAL_GPU_IDS=2,3).
+# Requires at least 2 GPUs (gen + proof). On a 4-GPU machine, GPU 3 can be
+# used for a second kernel eval GPU (KERNEL_EVAL_GPU_IDS=2,3).
 
-# Enable the 3-GPU pipelined mining engine
-GRAIL_PIPELINE_ENABLED=true
+# Generation backend: sglang (recommended, default) or vllm
+# SGLang is the validated production path: it uses the native /generate
+# endpoint with input_ids, avoiding text re-tokenization issues entirely.
+GRAIL_PIPELINE_BACKEND=sglang
 
-# Generation backend: vllm or sglang
-GRAIL_PIPELINE_BACKEND=vllm
+# Tensor-parallel size for the generation server (1 = single GPU, 4 = TP=4
+# across consecutive GPUs starting at GRAIL_PIPELINE_VLLM_GPU). Increase
+# only if you have spare GPUs and the model fits comfortably with TP=1.
+GRAIL_PIPELINE_VLLM_TP=1
 
-# GPU index for the vLLM/SGLang generation server (relative to CUDA_VISIBLE_DEVICES)
+# GPU index for the SGLang/vLLM generation server (relative to CUDA_VISIBLE_DEVICES)
 GRAIL_PIPELINE_VLLM_GPU=0
 
 # GPU index for the HuggingFace proof-of-work model (relative to CUDA_VISIBLE_DEVICES)
@@ -214,7 +221,7 @@ GRAIL_PIPELINE_PROOF_GPU=1
 # Setting this to true causes sketch divergence and proof failures.
 GRAIL_PIPELINE_PROOF_FLASH_ATTN=false
 
-# vLLM/SGLang server tuning (sensible defaults for A100 80GB)
+# Generation server tuning (sensible defaults for A100 80GB)
 GRAIL_PIPELINE_GPU_MEM_UTIL=0.90
 GRAIL_PIPELINE_MAX_MODEL_LEN=12288
 GRAIL_PIPELINE_MAX_NUM_SEQS=64
@@ -224,6 +231,7 @@ GRAIL_PIPELINE_SERVER_TIMEOUT=300
 # Symlink directory for vLLM weight reload (empty = auto, uses checkpoint parent dir)
 # Set to an ephemeral/tmpfs path if main disk is small:
 #   GRAIL_PIPELINE_SYMLINK_DIR=/dev/shm/grail
+# Only used by the vLLM backend; harmless when GRAIL_PIPELINE_BACKEND=sglang.
 GRAIL_PIPELINE_SYMLINK_DIR=
 
 # ============================================================================
@@ -363,6 +371,73 @@ GRAIL_CHECKPOINT_MILESTONE_INTERVAL=100
 # Options: sparse_codec_v2 (default), sparse_codec_v3, sparse_codec_v3.1
 GRAIL_DELTA_FORMAT=sparse_codec_v2
 
+# ============================================================================
+# DOCKER COMPOSE - VALIDATOR DEPLOYMENT
+# ============================================================================
+# These variables are consumed by docker/docker-compose.validator.yml. They
+# only matter when you run the validator via `docker compose` (the bare-metal
+# `grail validate` command ignores them). Operators running on bare metal can
+# leave them blank.
+
+# Override the validator image. Defaults to the public ghcr.io tag published
+# by .github/workflows/docker-publish.yml. Use a locally-built tag for
+# pre-release testing, or a pinned semver/sha256 tag in production:
+#   GRAIL_VALIDATOR_IMAGE=ghcr.io/one-covenant/grail:v0.5.10
+#   GRAIL_VALIDATOR_IMAGE=grail:base                # local build
+GRAIL_VALIDATOR_IMAGE=
+
+# Set to "true" to start the validator with --test-mode (it only validates
+# its own files and does NOT publish weights on-chain). Required for local
+# test deployments; MUST be unset/false in production.
+GRAIL_VALIDATOR_TEST_MODE=false
+
+# Comma-separated list of physical GPU indices to pin the validator to,
+# matching docker compose's deploy.resources.reservations.devices.device_ids
+# contract. Default "0" assigns the validator to GPU 0 alone (sufficient for
+# the validator path when GRAIL_GPU_EVAL=false). Bump to e.g. "0,1" if you
+# enable on-GPU kernel evaluation, or "2" to keep the validator off GPUs
+# 0+1 when a co-located miner uses them.
+GRAIL_VALIDATOR_GPU_IDS=0
+
+# Shared-memory size for the validator container. HF dataloaders and
+# multiprocessing IPC need more than the 64 MB Docker default.
+GRAIL_VALIDATOR_SHM_SIZE=2g
+
+# ─────────────────────────────────────────────────────────────────────────
+# Trainer-side protocol configuration (set on the TRAINER host)
+# ─────────────────────────────────────────────────────────────────────────
+# These values are baked into every published checkpoint's metadata. The
+# miner and validator override their own GRAIL_THINKING_MODE / env_id /
+# generation params from the loaded checkpoint at runtime, so the trainer
+# is the single source of truth for everything that affects prompt rendering
+# or generation. You only need to set these on the TRAINER host; the miner
+# and validator pick them up via R2 metadata. They are also passed through
+# the validator docker compose so any pre-checkpoint code path (boot
+# rendering, parser regex compile) sees the right value.
+#
+# GRAIL_THINKING_MODE controls the system prompt and chat template:
+#   "instructed" (DEFAULT) — custom <start_working_out>/</end_working_out>
+#                            for thinking + <SOLUTION>/</SOLUTION> for the
+#                            answer, injected via system prompt + a custom
+#                            ChatML template (mode grail builds itself)
+#   "native"               — model's built-in <think>/</think> tokens
+#                            (e.g. Qwen3) + <SOLUTION> for the answer
+# Miner and validator MUST agree, so the trainer publishes this and the
+# consumer sides override their local env var from the checkpoint.
+GRAIL_THINKING_MODE=instructed
+
+# Environment selection (trainer-side; passed via checkpoint metadata)
+GRAIL_ENV_ID=mbpp
+GRAIL_ENV_SPLIT=train
+
+# Generation parameters (trainer-side; passed via checkpoint metadata)
+# Defaults match grail/trainer/checkpoint_publisher.py::get_default_generation_params
+GRAIL_GEN_MAX_TOKENS=2048
+GRAIL_GEN_TEMPERATURE=0.7
+GRAIL_GEN_TOP_P=0.9
+GRAIL_GEN_TOP_K=50
+GRAIL_GEN_REPETITION_PENALTY=1.0
+
 # Host storage path for Docker volume mount (Docker Compose only)
 # If your machine has a large secondary disk (e.g., NVMe ephemeral storage),
 # set this to its mount point so checkpoints and caches are stored there.

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
           cache-dependency-glob: "uv.lock"
 
       - name: Set up Python
-        run: uv python install 3.11
+        run: uv python install 3.12
 
       - name: Install dependencies
         run: uv sync --extra dev
@@ -44,7 +44,7 @@ jobs:
           cache-dependency-glob: "uv.lock"
 
       - name: Set up Python
-        run: uv python install 3.11
+        run: uv python install 3.12
 
       - name: Install dependencies
         run: uv sync --extra dev
@@ -57,7 +57,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.12"]
     steps:
       - uses: actions/checkout@v4
 
@@ -73,7 +73,10 @@ jobs:
       - name: Install dependencies
         run: uv sync --extra dev
 
-      - name: Run tests with pytest
-        run: uv run pytest tests/ -v --tb=short
+      - name: Run tests (parallel)
+        run: uv run pytest tests/ -n auto --dist loadfile -m "not serial" -v --tb=short
+
+      - name: Run serial tests
+        run: uv run pytest tests/ -m "serial" -v --tb=short
         env:
           PYTHONPATH: ${{ github.workspace }}

.github/workflows/docker-publish.yml

@@ -38,7 +38,7 @@ jobs:
           cache-dependency-glob: "uv.lock"
 
       - name: Set up Python
-        run: uv python install 3.11
+        run: uv python install 3.12
 
       - name: Install dependencies
         run: uv sync --extra dev
@@ -130,6 +130,17 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      # Publishes the VALIDATOR-ONLY base image (docker/Dockerfile). The
+      # miner image (docker/Dockerfile.miner) adds SGLang on top of this
+      # base and is intentionally NOT published from the standard GitHub
+      # Actions runner: SGLang's full dependency closure (~3 GB extra +
+      # transformers downgrade) blows the runner's ~14 GB free disk
+      # budget. Miners build the miner image locally on their GPU host
+      # via:
+      #     docker build -f docker/Dockerfile.miner \
+      #         --build-arg GRAIL_BASE_IMAGE=ghcr.io/<owner>/grail:latest \
+      #         -t grail:miner .
+      # See docs/miner.md for the full operator workflow.
       - name: Build and push Docker image
         id: push
         uses: docker/build-push-action@v6

.gitignore

@@ -204,3 +204,6 @@ config.local.*
 
 # monitoring files
 wandb/
+
+# RFC drafts (tracked separately)
+docs/rfcs/

.python-version

@@ -1 +1 @@
-3.11
+3.12

CONTRIBUTING.md

@@ -17,7 +17,7 @@ Thanks for helping make this project better. Please keep changes focused, well-t
 - General bug fixes
 
 ### Getting started
-- **Environment**: Python 3.9–3.11. We use `uv` for dependency management.
+- **Environment**: Python 3.12+. We use `uv` for dependency management.
   - Install deps: `uv sync --all-extras`
   - Add deps: `uv add package==X.Y.Z` (pin exact versions) and then `uv lock`
   - Run tasks: `uv run <command>` (e.g., `uv run pytest -q`)

docker/Dockerfile

@@ -20,11 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     git lfs install && \
     rm -rf /var/lib/apt/lists/*
 
-# Install uv, then let uv install Python 3.11 (Ubuntu 22.04 only ships 3.10).
+# Install uv, then let uv install Python 3.12 (Ubuntu 22.04 only ships 3.10).
 # This matches the CI workflow and avoids needing the deadsnakes PPA.
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh && /root/.local/bin/uv --version
 ENV PATH="/root/.local/bin:${PATH}"
-RUN uv python install 3.11
+RUN uv python install 3.12
 
 WORKDIR /app
 
@@ -54,5 +54,47 @@ COPY README.md /app/README.md
 RUN uv sync --no-editable && \
     uv cache clean
 
-# Entrypoint delegates to uv
-ENTRYPOINT ["uv", "run", "grail"]
+# Flash Attention 2: required for consistent proof verification across
+# miner and validator. ATTN_IMPLEMENTATION is pinned to "flash_attention_2"
+# in grail/protocol/constants.py and provider.py raises at startup if the
+# import fails. Install the prebuilt cu12+torch2.9+cp312+cxx11abiTRUE wheel
+# from Dao-AILab releases (~254 MB, ~10 s install) instead of building from
+# source (~30-60 min per CI run).
+#
+# Compute-capability coverage of this single wheel (Dao-AILab cu12 standard):
+#   sm_80 (A100, A10G), sm_86 (A40, RTX 30xx), sm_89 (L40, L40S, RTX 40xx),
+#   sm_90 (H100, H200), sm_100 (B200 - empirically verified on
+#   basilica-grail-trainer at compute capability 10.0).
+#   Pre-Ampere GPUs (V100/sm_70, T4/sm_75, RTX 20xx/sm_75) are NOT covered;
+#   miners on those cards must build flash-attn from source.
+#
+# The cxx11abi must match torch's; PyTorch >= 2.5 Linux wheels are
+# cxx11abi=TRUE by default. Pinned by SHA256 for supply-chain safety. Bump
+# FLASH_ATTN_VERSION (and the wheel filename + SHA256) when upgrading
+# torch, since the wheel name encodes the matching torch minor.
+ARG FLASH_ATTN_VERSION=2.8.3
+ARG FLASH_ATTN_WHEEL=flash_attn-2.8.3+cu12torch2.9cxx11abiTRUE-cp312-cp312-linux_x86_64.whl
+ARG FLASH_ATTN_SHA256=4e2f9e39313266b1544b68138b15b91ee6221eccf14f7902b7c6620351340810
+RUN mkdir -p /tmp/wheels && \
+    curl -LsSf -o "/tmp/wheels/${FLASH_ATTN_WHEEL}" \
+        "https://github.com/Dao-AILab/flash-attention/releases/download/v${FLASH_ATTN_VERSION}/${FLASH_ATTN_WHEEL}" && \
+    echo "${FLASH_ATTN_SHA256}  /tmp/wheels/${FLASH_ATTN_WHEEL}" | sha256sum -c - && \
+    uv pip install "/tmp/wheels/${FLASH_ATTN_WHEEL}" && \
+    rm -rf /tmp/wheels && \
+    uv cache clean
+
+# Verify grail's import surface in the base image. This catches any
+# silent dep downgrade introduced by future flash-attn/torch pins.
+RUN /app/.venv/bin/python -c "\
+import torch, transformers, grail; \
+assert torch.__version__.startswith('2.9.1'), f'torch={torch.__version__}'; \
+assert transformers.__version__ == '4.57.1', f'transformers={transformers.__version__}'; \
+print('grail base image OK: torch=', torch.__version__, 'transformers=', transformers.__version__)"
+
+# Validator-only base image. The pipelined miner needs SGLang too;
+# build the miner image on top of this one via docker/Dockerfile.miner.
+#
+# Entrypoint: --no-sync so uv does NOT re-resolve the venv on every
+# container start (which would silently undo any post-uv-sync installs
+# such as the flash-attn wheel and the future SGLang layer).
+ENTRYPOINT ["uv", "run", "--no-sync", "grail"]

docker/Dockerfile.miner

@@ -0,0 +1,48 @@
+ARG GRAIL_BASE_IMAGE=grail:base
+FROM ${GRAIL_BASE_IMAGE} AS miner
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Pipelined miner image. Builds on the validator-only base
+# (docker/Dockerfile) and adds SGLang for the pipeline generation server.
+#
+# Why a separate image:
+#   - SGLang is large (~3 GB extra over the base) and pulls a long
+#     dependency chain (sgl-kernel, flashinfer, xgrammar, ...). Validators
+#     do NOT need any of it; bundling it would balloon the public ghcr.io
+#     image and break the GitHub Actions runner disk budget.
+#   - SGLang's pip metadata hard-pins transformers==5.3.0, which conflicts
+#     with grail's transformers==4.57.1. We work around it with a two-pass
+#     uv pip install: install sglang first, then force-reinstall the
+#     conflicting deps back to grail's pinned versions. This is the same
+#     install pattern that works on basilica-grail-tester and is verified
+#     end-to-end against the live validator.
+#
+# This image is intended to be built locally on a GPU host (or by a
+# self-hosted GitHub Actions runner with extra disk). The base image
+# stays on the standard ghcr.io publish path.
+# ─────────────────────────────────────────────────────────────────────────────
+
+ARG SGLANG_VERSION=0.5.10
+# Pinned to 0.5.10: matches the version validated end-to-end on
+# basilica-grail-tester with torch 2.9.1 / transformers 4.57.1. Older
+# SGLang releases (0.4.x) silently downgrade torch and transformers,
+# which breaks grail's imports. Bump only after a working e2e on a
+# new SGLang minor + the corresponding grail torch pin.
+RUN uv pip install "sglang[all]==${SGLANG_VERSION}" httpx --prerelease=allow && \
+    uv pip install --reinstall \
+        "transformers==4.57.1" \
+        "tokenizers==0.22.1" \
+        "torch==2.9.1" && \
+    uv cache clean
+
+# Verify grail's import surface still works after the sglang install. Fail
+# the build loudly if a transitive dep silently downgraded torch or
+# transformers (the historical failure mode for SGLang in this image).
+RUN /app/.venv/bin/python -c "\
+import torch, transformers, sglang, grail; \
+assert torch.__version__.startswith('2.9.1'), f'torch={torch.__version__}'; \
+assert transformers.__version__ == '4.57.1', f'transformers={transformers.__version__}'; \
+assert sglang.__version__ == '${SGLANG_VERSION}', f'sglang={sglang.__version__}'; \
+print('grail miner image OK: torch=', torch.__version__, 'transformers=', transformers.__version__, 'sglang=', sglang.__version__)"
+
+# Inherits ENTRYPOINT ["uv", "run", "--no-sync", "grail"] from the base.