Harden the king-of-the-hill subnet: build, state integrity, liveness, weight correctness, and the short-generation exploit

.github/workflows/docker-publish.yml

@@ -19,13 +19,16 @@ on:
     branches:
       - main
       - master
-    # Only rebuild when relevant files change
+    # Only rebuild when relevant files change.
+    # NOTE: this used to watch 'leoma.py' (deleted) and NOT the package, so
+    # changes to the actual code never triggered a rebuild.
     paths:
-      - 'leoma.py'
+      - 'leoma/**'
       - 'pyproject.toml'
       - 'Dockerfile'
+      - 'Dockerfile.eval'
       - '.github/workflows/docker-publish.yml'
-  
+
   # Allow manual trigger for testing
   workflow_dispatch:
 
@@ -38,18 +41,26 @@ env:
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
-    
+
     permissions:
       contents: read
       packages: write
 
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Slim CPU validator image.
+          - dockerfile: Dockerfile
+            suffix: ""
+          # CUDA eval-server image (installs the [eval] extra).
+          - dockerfile: Dockerfile.eval
+            suffix: "-eval"
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -64,6 +75,8 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          flavor: |
+            suffix=${{ matrix.suffix }},onlatest=true
           tags: |
             # Tag as 'latest' for default branch
             type=raw,value=latest,enable={{is_default_branch}}
@@ -79,14 +92,15 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
+          file: ${{ matrix.dockerfile }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           # Cache layers for faster builds
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          # Build for both AMD64 and ARM64
-          platforms: linux/amd64,linux/arm64
+          # amd64 only: the eval image is CUDA, and no validator runs on arm64.
+          platforms: linux/amd64
 
       - name: Generate build summary
         run: |

.github/workflows/tests.yml

@@ -0,0 +1,101 @@
+# Leoma — tests + build safety.
+#
+# The repo previously had NO test workflow at all, which is how three bugs
+# shipped: a Dockerfile COPY of a deleted file, a chain.toml that was read at
+# import but never packaged, and numpy declared only in the [eval] extra while
+# being imported on the validator path.
+#
+# Each job below exists to catch exactly one of those classes.
+name: Tests
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  # 1. The unit suite (pure logic: verdict, king chain, seeds, metrics, state, HTTP contract).
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      # ffprobe/ffmpeg are declared system deps (video utilities).
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg
+
+      - name: Install package + test extra
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e '.[test]'
+
+      - name: Run tests
+        run: pytest -q
+
+  # 2. Import-safety on a BASE (non-eval) install.
+  #    Catches: numpy-only-in-[eval], and chain.toml not shipping in the wheel.
+  #    Deliberately installs from a BUILT WHEEL, not the source tree, so a missing
+  #    package-data entry fails here rather than in production.
+  import-safety:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build wheel and install it (no [eval] extra)
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build --wheel
+          pip install dist/*.whl
+
+      - name: Import the validator path from OUTSIDE the source tree
+        # cd /tmp so python cannot accidentally import the repo checkout.
+        run: |
+          cd /tmp
+          python - <<'PY'
+          import leoma.infra.chain_config as c   # reads chain.toml at import -> must be packaged
+          import leoma.app.validator.main        # the validator entrypoint
+          import leoma.eval.metrics              # imports numpy at module scope
+          import leoma.eval_server               # FastAPI app factory
+          print("import-safety OK:", c.NAME, c.ARCH_PIPELINE)
+          PY
+
+      - name: CLI entrypoint resolves
+        run: |
+          cd /tmp
+          leoma --help > /dev/null && echo "CLI OK"
+
+  # 3. Docker build smoke. Catches a broken COPY / a missing file in the image.
+  #    Only the slim validator image is built here (the CUDA eval image is large;
+  #    it is built on publish). Both Dockerfiles run an in-image import check.
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Build validator image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile
+          push: false
+          load: true
+          tags: leoma:ci
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Smoke-run the image
+        run: |
+          docker run --rm leoma:ci leoma --help > /dev/null
+          docker run --rm leoma:ci python -c "import leoma.infra.chain_config as c; print('chain.toml in image:', c.NAME)"

Dockerfile

@@ -1,6 +1,12 @@
+# Leoma VALIDATOR image (slim, CPU).
+#
+# The validator scans on-chain reveals, dispatches duels to the GPU eval server,
+# crowns winners and sets weights. It never loads a model, so it needs no torch.
+# The GPU eval server uses Dockerfile.eval instead.
 FROM python:3.12-slim
 
-# ffprobe/ffmpeg for video processing (eval server); curl for healthchecks; build-essential for compiling Python deps if needed.
+# ffprobe/ffmpeg for the video utilities; curl for healthchecks;
+# build-essential for any deps that need compiling.
 RUN apt-get update && apt-get install -y --no-install-recommends \
     ffmpeg curl build-essential \
     && rm -rf /var/lib/apt/lists/*
@@ -9,11 +15,16 @@ WORKDIR /app
 RUN pip install --no-cache-dir uv
 
 COPY pyproject.toml README.md ./
-COPY leoma.py ./
 COPY leoma ./leoma
 
-# Install package (non-editable for production image)
+# Install the package (non-editable for a production image). chain.toml ships
+# INSIDE the package via [tool.setuptools.package-data] — it is consensus-critical
+# and is read at import time, so it must exist in site-packages, not just in git.
 RUN uv pip install --system --no-cache .
 
-# Override in compose: leoma serve (validator) or leoma api (API service)
-CMD ["leoma"]
+# Fail at BUILD time if the package cannot import. This is exactly the bug class
+# that shipped before (a missing chain.toml / missing numpy only surfaced when the
+# container started and crash-looped).
+RUN python -c "import leoma.infra.chain_config as c, leoma.app.validator.main; print('validator image OK, chain =', c.NAME)"
+
+CMD ["leoma", "serve"]

Dockerfile.eval

@@ -0,0 +1,43 @@
+# Leoma EVAL SERVER image (CUDA).
+#
+# This is the GPU box: it downloads king + challenger weights from Hippius Hub,
+# loads the pinned diffusers I2V pipeline, generates video on the held-out clips
+# and scores each generation against the real continuation.
+#
+# It needs the [eval] extra (torch / diffusers / lpips / opencv / open_clip).
+# The validator image (Dockerfile) deliberately does NOT.
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+# python3.12 + ffmpeg (ground-truth decode) + curl (healthcheck).
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      software-properties-common \
+ && add-apt-repository -y ppa:deadsnakes/ppa \
+ && apt-get update && apt-get install -y --no-install-recommends \
+      python3.12 python3.12-venv python3.12-dev \
+      ffmpeg curl build-essential \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN python3.12 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:${PATH}"
+
+WORKDIR /app
+RUN pip install --no-cache-dir uv
+
+COPY pyproject.toml README.md ./
+COPY leoma ./leoma
+
+# The [eval] extra is the whole point of this image.
+RUN uv pip install --no-cache '.[eval]'
+
+# Fail at BUILD time if the eval stack can't import (torch/diffusers/chain.toml).
+RUN python -c "import leoma.eval_server, leoma.eval.metrics, leoma.infra.chain_config as c; print('eval image OK, pipeline =', c.ARCH_PIPELINE)"
+
+EXPOSE 9000
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+  CMD curl -fsS http://127.0.0.1:9000/health || exit 1
+
+CMD ["leoma", "servers", "eval-server"]

docker-compose.yml

@@ -35,27 +35,39 @@ services:
       R2_OWN_REGION: ${R2_OWN_REGION:-auto}
 
   # Eval server: GPU box that downloads king + challenger and runs the duel.
-  # Requires an NVIDIA runtime; install leoma with the [eval] extra in the image.
+  # Built from Dockerfile.eval (CUDA base + the [eval] extra). The slim validator
+  # image has no torch, so it CANNOT run this service.
   eval-server:
-    build: .
-    image: rendixnetwork/leoma:latest
+    build:
+      context: .
+      dockerfile: Dockerfile.eval
+    image: rendixnetwork/leoma:latest-eval
     container_name: leoma-eval-server
     restart: unless-stopped
     command: leoma servers eval-server
     env_file:
       - .env
     environment:
       PYTHONUNBUFFERED: "1"
+      # 0.0.0.0 is required for cross-container networking; the port is NOT
+      # published to the host. See LEOMA_EVAL_TOKEN in the hardening plan.
       EVAL_SERVER_HOST: "0.0.0.0"
       EVAL_SERVER_PORT: "9000"
       HIPPIUS_HUB_TOKEN: ${HIPPIUS_HUB_TOKEN:-}
-      LEOMA_MODEL_CACHE_DIR: ${LEOMA_MODEL_CACHE_DIR:-/tmp/leoma/hippius_models}
+      LEOMA_MODEL_CACHE_DIR: ${LEOMA_MODEL_CACHE_DIR:-/var/lib/leoma/models}
       R2_VIDEOS_READ_ACCESS_KEY: ${R2_VIDEOS_READ_ACCESS_KEY:-}
       R2_VIDEOS_READ_SECRET_KEY: ${R2_VIDEOS_READ_SECRET_KEY:-}
+    volumes:
+      # Model snapshots are multi-GB; keep them off the container layer so a
+      # restart doesn't re-download the king.
+      - leoma-models:${LEOMA_MODEL_CACHE_DIR:-/var/lib/leoma/models}
     # deploy:  # uncomment on a GPU host
     #   resources:
     #     reservations:
     #       devices:
     #         - driver: nvidia
     #           count: 1
     #           capabilities: [gpu]
+
+volumes:
+  leoma-models:

leoma/app/validator/__init__.py

@@ -1,6 +1,9 @@
-"""Validator service entry points (king-of-the-hill duel + weight-setter)."""
+"""Validator service (king-of-the-hill duel + weight-setter).
 
-from leoma.app.validator.main import main, main_sync
+Deliberately re-exports nothing. Importing the ``main`` *function* here shadowed
+the ``main`` *module*, so ``leoma.app.validator.main`` resolved to a function —
+which made the module unreachable by attribute access (and unpatchable in tests).
+Import the entry point from its module instead:
 
-
-__all__ = ["main", "main_sync"]
+    from leoma.app.validator.main import main
+"""

leoma/app/validator/dashboard.py

@@ -0,0 +1,80 @@
+"""Build + publish the public ``dashboard.json`` (Teutonic-style).
+
+The validator has no API, so — like Teutonic — it publishes a single JSON
+snapshot to its own bucket that the website polls. This module builds that
+payload from the in-memory king state (pure, testable) and writes it to the
+bucket. The bucket object must be public-read for the site to fetch it.
+
+Shape (consumed by the leoma-app dashboard):
+  updated_at, chain{name,seed_repo,seed_digest,netuid}, duel_params,
+  king{hotkey,uid,model_repo,model_digest,reign_number,crowned_at,crowned_block,...},
+  king_chain[{...,uid,weight}]   (current king first; weight = equal share, 1/n),
+  stats{accepted,rejected,failed},
+  queue[{hotkey,uid,model_repo,model_digest,block,status}],
+  history[{hotkey,uid,model_repo,verdict,accepted,mu_hat,lcb,...}]  (newest first)
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from leoma.app.validator import king as K
+from leoma.app.validator.state_store import JsonBucketStore, KingState
+
+KEY_DASHBOARD = "dashboard.json"
+
+
+def _king_entry(entry: dict, uid_map: dict[str, int]) -> dict:
+    hk = entry.get("hotkey", "")
+    return {
+        "hotkey": hk,
+        "uid": uid_map.get(hk),
+        "model_repo": entry.get("model_repo", ""),
+        "model_digest": entry.get("model_digest", ""),
+        "reign_number": entry.get("reign_number"),
+        "crowned_at": entry.get("crowned_at"),
+        "crowned_block": entry.get("crowned_block"),
+        "challenge_id": entry.get("challenge_id"),
+        "previous_repo": entry.get("previous_repo", ""),
+    }
+
+
+def build_dashboard(
+    state: KingState,
+    uid_map: dict[str, int],
+    *,
+    chain_meta: dict,
+    duel_params: dict,
+    updated_at: str,
+    queue: Optional[list[dict]] = None,
+) -> dict:
+    """Assemble the dashboard payload (pure — no I/O, no wall clock)."""
+    # Distinct king hotkeys sharing emission (current king first), and the equal
+    # share among those actually registered on the metagraph.
+    hks = K.king_hotkeys(state.king, state.king_chain)
+    registered = [hk for hk in hks if hk in uid_map]
+    weight = round(1.0 / len(registered), 9) if registered else None
+
+    chain = ([state.king] if state.king else []) + list(state.king_chain or [])
+    king_chain = []
+    for entry in chain:
+        row = _king_entry(entry, uid_map)
+        row["weight"] = weight if entry.get("hotkey", "") in uid_map else None
+        king_chain.append(row)
+
+    king = _king_entry(state.king, uid_map) if state.king else {}
+
+    return {
+        "updated_at": updated_at,
+        "chain": chain_meta,
+        "duel_params": duel_params,
+        "king": king,
+        "king_chain": king_chain,
+        "stats": dict(state.stats),
+        "queue": list(queue or []),
+        "history": list(state.history),
+    }
+
+
+async def publish_dashboard(store: JsonBucketStore, payload: dict) -> None:
+    """Write the dashboard snapshot to the bucket (public-read object)."""
+    await store.put(KEY_DASHBOARD, payload)