serving: attest image split from the runtime; per-card attestation pays N cards per hotkey; sampled audit

- docker/attest is its own image (entrius/gt-attest, attest-image.yml): one long-running container per box beside the
  runtime, on every GPU; runtime images no longer carry the sidecar. GET /info self-report (telemetry only).
- attest.py judges every card the sidecar reports: capacity = passing cards, score = speed credit x cards, UUID dedupe
  across all cards; a card without the model resident (free VRAM above total - 0.8 x reservation) does not count.
- Verification is sampled per (hotkey, round): every baseline prompt and failed request, plus max(10, 20%) of the
  completed gateway requests.
- Loadout carries runtime_image (digest, written by the conformance job) and attest.image; both persisted per round
  for the /compute release card. SERVING_ATTEST_REFERENCE_URL + compose `attest` service for the reference.
- Conformance on Lium rents a second pod for the attest image; check_serving_runtime --attest-url.

Claude-Session: https://claude.ai/code/session_01FJtQK6NCjUgYkGTopczEf4

Author: anderdc

.env.example

@@ -60,5 +60,9 @@ SERVING_ENABLED=false
 # Overrides the primary release's reference_url. Unset = reference_url from the loadout, else its audit_bank snapshot.
 # SERVING_REFERENCE_URL=http://reference:8080
 # SERVING_REFERENCE_API_KEY=<bearer for a remote reference>
+# The attest container beside the reference (compose service `attest`, image entrius/gt-attest:GT_ATTEST_TAG).
+# Default: the reference host on port 8081 — set it when the sidecar is a separate service or host.
+# SERVING_ATTEST_REFERENCE_URL=http://attest:8081
+# GT_ATTEST_TAG=v1
 # SPARKINFER_TAG=<commit>   # image tag for the reference sidecar = runtime_pin commit in the loadout
 # SPARKINFER_MODEL_SHA256=<model_sha256 from the loadout release>

.github/workflows/attest-image.yml

@@ -0,0 +1,74 @@
+name: attest image
+
+# Builds the attestation sidecar (docker/attest) and publishes it as entrius/gt-attest:<tag>. One image for every
+# blessed release: miners run it beside their runtime container, validators beside their reference. Bump the tag
+# (v1, v2, ...) when docker/attest changes and point `attest.image` in serving_loadout.json at it. No GPU in CI: the
+# build proves it compiles; the sparkinfer conformance job exercises it on a rented 5090.
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: "Tag to push (e.g. v1). Empty = the tag in attest.image of serving_loadout.json."
+        required: false
+        default: ""
+      cuda_archs:
+        description: "CMAKE_CUDA_ARCHITECTURES"
+        required: false
+        default: "120"
+  push:
+    paths:
+      - docker/attest/**
+      - .github/workflows/attest-image.yml
+
+jobs:
+  image:
+    name: ${{ github.event_name == 'workflow_dispatch' && 'build and push' || 'build only (dry run)' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Resolve the tag (input, else attest.image in serving_loadout.json)
+        id: tag
+        run: |
+          TAG="${{ inputs.image_tag }}"
+          if [ -z "$TAG" ]; then
+            TAG=$(jq -r '.releases[0].attest.image // "entrius/gt-attest:dev"' gittensor/validator/weights/serving_loadout.json | sed 's/.*://')
+          fi
+          echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "building docker/attest as entrius/gt-attest:$TAG"
+
+      - name: Free disk space (CUDA devel image is large)
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+          docker system prune -af
+
+      - name: Log in to Docker Hub
+        if: github.event_name == 'workflow_dispatch'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+
+      - name: Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/attest/Dockerfile
+          push: ${{ github.event_name == 'workflow_dispatch' }}
+          build-args: |
+            CUDA_ARCHS=${{ inputs.cuda_archs || '120' }}
+            GT_ATTEST_VERSION=${{ steps.tag.outputs.tag }}
+          tags: |
+            entrius/gt-attest:${{ steps.tag.outputs.tag }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Digest
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "entrius/gt-attest:${{ steps.tag.outputs.tag }}@${{ steps.build.outputs.digest }}"

.github/workflows/sparkinfer-image.yml

@@ -38,6 +38,7 @@ jobs:
     outputs:
       ref: ${{ steps.ref.outputs.ref }}
       tag: ${{ steps.ref.outputs.tag }}
+      digest: ${{ steps.build.outputs.digest }}
     # push-triggered runs (Dockerfile/workflow changed) only BUILD, to catch Dockerfile breakage in CI;
     # the image is pushed to Docker Hub only from the Run workflow button (workflow_dispatch).
     name: ${{ github.event_name == 'workflow_dispatch' && 'build and push' || 'build only (dry run)' }}
@@ -73,6 +74,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Build and push
+        id: build
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -96,6 +98,7 @@ jobs:
       LIUM_API_KEY: ${{ secrets.LIUM_API_KEY }}
       REF: ${{ needs.image.outputs.ref }}
       TAG: ${{ needs.image.outputs.tag }}
+      DIGEST: ${{ needs.image.outputs.digest }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
@@ -123,11 +126,12 @@ jobs:
           path: conformance-${{ env.REF }}/
           if-no-files-found: warn
 
-      - name: Bump runtime_pin and the release's speed facts
+      - name: Bump runtime_pin, runtime_image and the release's speed facts
         run: |
-          jq --arg pin "gittensor-ai-lab/sparkinfer@$REF" --arg run "$GITHUB_RUN_ID" \
+          jq --arg pin "gittensor-ai-lab/sparkinfer@$REF" --arg image "entrius/sparkinfer:$TAG@$DIGEST" --arg run "$GITHUB_RUN_ID" \
             --slurpfile speed "conformance-$REF/speed.json" \
             '.releases[0].runtime_pin = $pin
+             | .releases[0].runtime_image = $image
              | .releases[0].speed = ((.releases[0].speed // {}) + {
                  single_stream_decode_tps: $speed[0].single_stream_decode_tps,
                  decode_per_request: $speed[0].decode_per_request,
@@ -153,7 +157,8 @@ jobs:
           body: |
             `entrius/sparkinfer:${{ env.REF }}` passed `scripts/check_serving_runtime.py` on a rented RTX 5090
             (run ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} — report in the
-            `serving-conformance-${{ env.REF }}` artifact). This bumps `runtime_pin` and writes the release's
+            `serving-conformance-${{ env.REF }}` artifact). This bumps `runtime_pin`, pins `runtime_image` to the pushed
+            image's digest, and writes the release's
             blessing-time speed facts (`speed.decode_per_request`, the per-request decode curve served traffic is
             priced against) and attestation facts (`attest.*`). Update the sha named in the comments at
             `gittensor/constants.py` and `gittensor/serving/audit.py` if the blessing rules changed.

docker-compose.vali.yml

@@ -35,7 +35,8 @@ services:
   # sparkinfer_server image miners run (built by us from a pinned upstream commit, docker/sparkinfer.Dockerfile),
   # on the validator's own GPU, so audits compare against a live honest copy instead of a bank snapshot.
   #   SPARKINFER_TAG=<runtime_pin commit> docker compose -f docker-compose.vali.yml --profile reference up -d
-  # then set SERVING_REFERENCE_URL=http://reference:8080 in .env. See the Serving Runtime Contract in the miner docs.
+  # then set SERVING_REFERENCE_URL=http://reference:8080 and SERVING_ATTEST_REFERENCE_URL=http://attest:8081 in .env.
+  # See the Serving Runtime Contract in the miner docs.
   reference:
     image: entrius/sparkinfer:${SPARKINFER_TAG:-latest}
     container_name: gt-vali-reference
@@ -56,6 +57,21 @@ services:
     volumes:
       - ./data/models:/opt/sparkinfer/models
 
+  # Attestation sidecar beside the reference (profile "reference"): the validator asks it for the expected digest
+  # and reference wall time of each round's challenge. Same image every miner box runs (attest.image in the loadout).
+  attest:
+    image: entrius/gt-attest:${GT_ATTEST_TAG:-v1}
+    container_name: gt-vali-attest
+    restart: unless-stopped
+    profiles: ["reference"]
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+
   watchtower:
     image: nickfedor/watchtower
     container_name: gittensor-watchtower

docker/attest/Dockerfile

@@ -0,0 +1,39 @@
+# syntax=docker/dockerfile:1.7
+# Gittensor attestation sidecar: one long-running container per miner box (and beside a validator's reference), on
+# every GPU the box has. Answers the validator's hardware challenge (gt_attest: seeded VRAM fill + deterministic
+# GEMM chain, docker/attest/gt_attest.cu) and a self-report (GET /info). Runtime-agnostic — the same image serves
+# every blessed release, so blessing a new runtime never touches it. Pinned by `attest.image` in the serving loadout.
+#
+#   docker build -f docker/attest/Dockerfile -t entrius/gt-attest:<tag> .
+#   docker run -d --name gt-attest --gpus all -p 8081:8081 entrius/gt-attest:<tag>
+
+ARG CUDA_VERSION=12.8.1
+ARG UBUNTU_VERSION=24.04
+
+FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build
+# sm_120 = RTX 5090. The digest is only comparable across cards of one architecture; add more only if the fleet
+# blesses other hardware.
+ARG CUDA_ARCHS=120
+COPY docker/attest/gt_attest.cu /src/gt_attest.cu
+# -fmad=false: the summation is bit-identical on every card of the architecture (the validator's reference recomputes
+# the digest).
+RUN nvcc -O3 -fmad=false -gencode arch=compute_${CUDA_ARCHS},code=sm_${CUDA_ARCHS} \
+        -o /src/gt_attest /src/gt_attest.cu -lnvidia-ml
+
+FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends python3 curl \
+    && rm -rf /var/lib/apt/lists/*
+WORKDIR /opt/gt-attest
+COPY --from=build /src/gt_attest          bin/gt_attest
+COPY docker/attest/attest_server.py       bin/attest_server.py
+ARG GT_ATTEST_VERSION=dev
+ENV GT_ATTEST_BIN=/opt/gt-attest/bin/gt_attest \
+    GT_ATTEST_VERSION=${GT_ATTEST_VERSION} \
+    ATTEST_PORT=8081
+EXPOSE 8081
+LABEL io.gittensor.serving.attest="gt-attest" \
+      io.gittensor.serving.attest_version="${GT_ATTEST_VERSION}"
+HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
+    CMD curl -fsS http://127.0.0.1:8081/info || exit 1
+ENTRYPOINT ["python3", "bin/attest_server.py"]

docker/attest/attest_server.py

@@ -1,9 +1,14 @@
 #!/usr/bin/env python3
-"""Attestation sidecar for the serving runtime image: runs gt_attest on request (stdlib only).
+"""Attestation sidecar (image entrius/gt-attest, docker/attest/Dockerfile): runs gt_attest on request (stdlib only).
 
 POST /v1/attest  {"seed": <u64>, "iters": <n>, "fill": true}  -> gt_attest's JSON plus "queued_ms"
 GET  /v1/attest/info                                          -> devices without running a challenge (iters 1, no fill,
                                                                  small working set)
+GET  /info                                                    -> self-report: GPUs as nvidia-smi sees them, driver,
+                                                                 host CPU/RAM, sidecar version. Telemetry for the
+                                                                 operator and the /compute page — a host can say
+                                                                 anything here, so nothing is paid on it; the
+                                                                 challenge above is the proof.
 Challenges run one at a time; concurrent requests queue and report their own wall including the wait, which is what
 lets a validator see two hotkeys sharing one card. Optional bearer via ATTEST_API_KEY.
 """
@@ -15,7 +20,8 @@
 import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
-BIN = os.environ.get('GT_ATTEST_BIN', '/opt/sparkinfer/bin/gt_attest')
+BIN = os.environ.get('GT_ATTEST_BIN', '/opt/gt-attest/bin/gt_attest')
+VERSION = os.environ.get('GT_ATTEST_VERSION', 'dev')
 PORT = int(os.environ.get('ATTEST_PORT', '8081'))
 API_KEY = os.environ.get('ATTEST_API_KEY')
 LOCK = threading.Lock()
@@ -37,6 +43,55 @@ def run(args, timeout=120.0):
     return 200, out
 
 
+def self_report() -> dict:
+    query = 'uuid,name,driver_version,memory.total,memory.used,pci.bus_id,temperature.gpu,utilization.gpu'
+    gpus, error = [], None
+    try:
+        out = subprocess.run(
+            ['nvidia-smi', f'--query-gpu={query}', '--format=csv,noheader,nounits'],
+            capture_output=True,
+            text=True,
+            timeout=10.0,
+        )
+        if out.returncode != 0:
+            error = (out.stderr or out.stdout).strip()[:300]
+        for line in out.stdout.splitlines():
+            cols = [c.strip() for c in line.split(',')]
+            if len(cols) == 8:
+                gpus.append(
+                    {
+                        'uuid': cols[0],
+                        'name': cols[1],
+                        'driver': cols[2],
+                        'memory_total_mib': int(cols[3]),
+                        'memory_used_mib': int(cols[4]),
+                        'pci_bus_id': cols[5],
+                        'temperature_c': int(cols[6]),
+                        'utilization_pct': int(cols[7]),
+                    }
+                )
+    except Exception as e:  # no nvidia-smi in the container, or a hung driver
+        error = repr(e)[:300]
+    mem_total_kib = None
+    try:
+        with open('/proc/meminfo') as f:
+            for line in f:
+                if line.startswith('MemTotal:'):
+                    mem_total_kib = int(line.split()[1])
+                    break
+    except OSError:
+        pass
+    return {
+        'version': VERSION,
+        'hostname': os.uname().nodename,
+        'cpus': os.cpu_count(),
+        'mem_total_gib': round(mem_total_kib / 1048576, 1) if mem_total_kib else None,
+        'gpus': gpus,
+        'error': error,
+        'ts': time.time(),
+    }
+
+
 class Handler(BaseHTTPRequestHandler):
     def _send(self, status, payload):
         body = json.dumps(payload).encode()
@@ -53,7 +108,11 @@ def _auth(self):
         return True
 
     def do_GET(self):
-        if self.path != '/v1/attest/info' or not self._auth():
+        if not self._auth():
+            return
+        if self.path == '/info':
+            return self._send(200, self_report())
+        if self.path != '/v1/attest/info':
             return self._send(404, {'error': 'not found'})
         self._send(
             *run(['--seed', '1', '--iters', '1', '--dim', '256', '--matrices', '2', '--device', 'all'], timeout=30.0)

docker/attest/entrypoint.sh

@@ -9,6 +9,7 @@
 #
 #   docker build -f docker/sparkinfer.Dockerfile --build-arg SPARKINFER_REF=<commit> -t entrius/sparkinfer:<commit> .
 #   docker run --gpus all -p 8080:8080 -v $PWD/data/models:/opt/sparkinfer/models entrius/sparkinfer:<commit>
+# A miner box also runs the attestation container (docker/attest, entrius/gt-attest) beside it.
 
 ARG CUDA_VERSION=12.8.1
 ARG UBUNTU_VERSION=24.04
@@ -38,11 +39,6 @@ RUN git clone ${SPARKINFER_REPO} sparkinfer \
 RUN cmake -S sparkinfer -B sparkinfer/build -G Ninja \
         -DCMAKE_BUILD_TYPE=Release -DBUILD_SERVER=ON -DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHS} \
     && cmake --build sparkinfer/build
-# Hardware attestation challenge (docker/attest): deterministic fp32 GEMM chain + VRAM fill. -fmad=false so the
-# summation is bit-identical on every card of the architecture (the validator's reference recomputes the digest).
-COPY docker/attest/gt_attest.cu /src/attest/gt_attest.cu
-RUN nvcc -O3 -fmad=false -gencode arch=compute_${CUDA_ARCHS},code=sm_${CUDA_ARCHS} \
-        -o /src/attest/gt_attest /src/attest/gt_attest.cu -lnvidia-ml
 
 # ---------- runtime ----------
 FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
@@ -60,9 +56,6 @@ COPY --from=build /src/sparkinfer/build/runtime       build/runtime
 COPY --from=build /src/sparkinfer/build/moe           build/moe
 COPY --from=build /src/sparkinfer/build/server        build/server
 COPY --from=build /src/SPARKINFER_COMMIT              SPARKINFER_COMMIT
-COPY --from=build /src/attest/gt_attest               bin/gt_attest
-COPY docker/attest/attest_server.py                   bin/attest_server.py
-COPY docker/attest/entrypoint.sh                      bin/entrypoint.sh
 # sparkinfer_server links libsparkinfer_runtime.so + libsparkinfer_moe.so (ldd-verified on 1b8b962, 9e43bfa).
 ENV LD_LIBRARY_PATH=/opt/sparkinfer/build/runtime:/opt/sparkinfer/build/moe:${LD_LIBRARY_PATH}
 
@@ -81,15 +74,15 @@ ENV SPARKINFER_REF=${SPARKINFER_REF} \
     HOST=0.0.0.0 \
     PORT=8080
 VOLUME ["/opt/sparkinfer/models"]
-# 8080 inference, 8081 attestation sidecar (docker/attest/attest_server.py)
-EXPOSE 8080 8081
+# 8080 inference. Hardware attestation is the separate entrius/gt-attest container (docker/attest), not this image.
+EXPOSE 8080
 LABEL org.opencontainers.image.source="https://github.com/gittensor-ai-lab/sparkinfer" \
       io.gittensor.serving.runtime="sparkinfer" \
       io.gittensor.serving.runtime_ref="${SPARKINFER_REF}"
 
 HEALTHCHECK --interval=30s --timeout=5s --start-period=600s --retries=5 \
     CMD curl -fsS http://127.0.0.1:8080/v1/models || exit 1
 
-# bin/entrypoint.sh starts the attestation sidecar (:8081), then run.sh downloads the blessed model + tokenizer into
-# MODELS_DIR on first start (--download) and execs the prebuilt server binary.
-ENTRYPOINT ["bash", "bin/entrypoint.sh"]
+# run.sh downloads the blessed model + tokenizer into MODELS_DIR on first start (--download) and execs the prebuilt
+# server binary.
+ENTRYPOINT ["bash", "server/run.sh", "--download"]

docker/sparkinfer.Dockerfile

@@ -188,6 +188,10 @@
 SERVING_ATTEST_MIN_FILL_RATIO = 0.6
 SERVING_ATTEST_TIMEOUT = 45.0  # seconds: fill + chain on a 5090 is ~1.5 s; queued challenges show up as slow
 SERVING_ATTEST_UUID_MEMORY_ROUNDS = 12
+# A card counts only with the model resident: free VRAM before the fill must be at most total minus this fraction of
+# the reservation (a bare 5090 shows ~32 GB free, one holding the model ~8 GB). Every passing card is a card-hour, so
+# a multi-GPU hotkey earns per card it actually serves from — a second card with nothing loaded earns nothing.
+SERVING_ATTEST_MODEL_RESIDENT_RATIO = 0.8
 SERVING_VRAM_MODEL_RESERVED_BYTES = 24e9  # what sparkinfer holds with the model loaded (7498736 on a 5090: 23.7 GB)
 SERVING_VERIFY_WORKERS = 8  # concurrent /v1/score calls to the reference per round
 # Latency credit on the validator-observed time to first streamed token (network + queue + prefill). An honest
@@ -231,6 +235,12 @@
 #   python scripts/serving_cheat_experiment.py analyze --honest <honest rows> --cheaters <cheater rows>
 SERVING_AUDIT_WINDOW = 10
 SERVING_AUDIT_WINDOW_THRESHOLDS = ((1, 0.8),)
+# Verification is one reference prefill per request, so it is sampled: per (hotkey, round) every baseline prompt and
+# every failed request is judged, and of the completed gateway requests a random max(SAMPLE_MIN, SAMPLE_FRACTION x n)
+# — the floor fills a window in one round, the fraction bounds reference load under real traffic. Nothing on the
+# wire says which requests were drawn.
+SERVING_AUDIT_SAMPLE_FRACTION = 0.2
+SERVING_AUDIT_SAMPLE_MIN = 10
 SERVING_QUARANTINE_S = 3600.0
 # Baseline traffic: every round the validator sends each serving axon this many baseline prompts of its own, at
 # random moments spread over the round (so they do not mark the round boundary), over the same path as user