conformance: fill speed facts only where the release has none (#1741) #58
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: sparkinfer image | |
| # Builds the blessed serving runtime (see the Serving Runtime Contract in the miner docs) from a pinned upstream commit and | |
| # publishes it as entrius/sparkinfer:<ref>. Run manually with the commit you intend to bless; the | |
| # resulting tag is what goes into `runtime_pin` in gittensor/validator/weights/serving_loadout.json. | |
| # No GPU in CI: the build job only proves the release builds. The `conformance` job then rents one RTX 5090 | |
| # on Lium (scripts/serving_conformance_on_lium.sh), runs scripts/check_serving_runtime.py against the pushed | |
| # image, uploads the report, and on a clean pass opens the pin-bump PR. Needs the LIUM_API_KEY secret. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| sparkinfer_ref: | |
| description: "sparkinfer commit to build (becomes the image tag). Empty = the runtime_pin in serving_loadout.json." | |
| required: false | |
| default: "" | |
| cuda_archs: | |
| description: "CMAKE_CUDA_ARCHITECTURES" | |
| required: false | |
| default: "120" | |
| image_tag: | |
| description: "Image tag to push (default: the sparkinfer ref). Use e.g. <ref>-attest1 for a gittensor-side image change so the blessed tag is not overwritten." | |
| required: false | |
| default: "" | |
| conformance: | |
| description: "After the push, rent a 5090 on Lium, run the conformance checker and open the pin-bump PR on a pass" | |
| type: boolean | |
| required: false | |
| default: true | |
| build: | |
| description: "Build and push the image. Uncheck to conformance-check a tag already on Docker Hub without republishing it (the digest is read from the registry)." | |
| type: boolean | |
| required: false | |
| default: true | |
| # Build-only check (never pushes) whenever the Dockerfile or this workflow changes. | |
| push: | |
| paths: | |
| - docker/sparkinfer.Dockerfile | |
| - .github/workflows/sparkinfer-image.yml | |
| jobs: | |
| image: | |
| if: github.event_name != 'workflow_dispatch' || inputs.build | |
| outputs: | |
| ref: ${{ steps.ref.outputs.ref }} | |
| tag: ${{ steps.ref.outputs.tag }} | |
| digest: ${{ steps.build.outputs.digest }} | |
| # push-triggered runs (Dockerfile/workflow changed) only BUILD, to catch Dockerfile breakage in CI; | |
| # the image is pushed to Docker Hub only from the Run workflow button (workflow_dispatch). | |
| name: ${{ github.event_name == 'workflow_dispatch' && 'build and push' || 'build only (dry run)' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Resolve the sparkinfer ref (input, else runtime_pin in serving_loadout.json) | |
| id: ref | |
| run: | | |
| REF="${{ inputs.sparkinfer_ref }}" | |
| if [ -z "$REF" ]; then | |
| REF=$(jq -r '.releases[0].runtime_pin' gittensor/validator/weights/serving_loadout.json | sed 's/.*@//') | |
| fi | |
| echo "ref=$REF" >> "$GITHUB_OUTPUT" | |
| TAG="${{ inputs.image_tag }}"; echo "tag=${TAG:-$REF}" >> "$GITHUB_OUTPUT" | |
| echo "building sparkinfer@$REF as entrius/sparkinfer:${TAG:-$REF}" | |
| - name: Free disk space (CUDA devel image is large) | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| docker system prune -af | |
| - name: Log in to Docker Hub | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Set up Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push | |
| id: build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/sparkinfer.Dockerfile | |
| push: ${{ github.event_name == 'workflow_dispatch' }} | |
| build-args: | | |
| SPARKINFER_REF=${{ steps.ref.outputs.ref }} | |
| CUDA_ARCHS=${{ inputs.cuda_archs || '120' }} | |
| tags: | | |
| entrius/sparkinfer:${{ steps.ref.outputs.tag }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| conformance: | |
| name: conformance on a rented 5090 | |
| needs: image | |
| # Runs whether or not the image job ran: with `build` unchecked it checks a tag already on Docker Hub, so a | |
| # blessed tag is never republished just to re-measure it. | |
| if: | | |
| always() && github.event_name == 'workflow_dispatch' && inputs.conformance | |
| && (needs.image.result == 'success' || needs.image.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| env: | |
| LIUM_API_KEY: ${{ secrets.LIUM_API_KEY }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Resolve ref, tag and digest (from the build job, else from the registry) | |
| id: img | |
| run: | | |
| REF="${{ needs.image.outputs.ref }}" | |
| TAG="${{ needs.image.outputs.tag }}" | |
| DIGEST="${{ needs.image.outputs.digest }}" | |
| if [ -z "$REF" ]; then | |
| REF="${{ inputs.sparkinfer_ref }}" | |
| [ -n "$REF" ] || REF=$(jq -r '.releases[0].runtime_pin' gittensor/validator/weights/serving_loadout.json | sed 's/.*@//') | |
| fi | |
| if [ -z "$TAG" ]; then | |
| TAG="${{ inputs.image_tag }}"; TAG="${TAG:-$REF}" | |
| fi | |
| if [ -z "$DIGEST" ]; then | |
| DIGEST=$(curl -fsS "https://hub.docker.com/v2/repositories/entrius/sparkinfer/tags/$TAG" | jq -r '.digest // empty') | |
| [ -n "$DIGEST" ] || { echo "entrius/sparkinfer:$TAG is not on Docker Hub; build it first"; exit 1; } | |
| echo "checking the published entrius/sparkinfer:$TAG ($DIGEST) — not rebuilt" | |
| fi | |
| echo "REF=$REF" >> "$GITHUB_ENV" | |
| echo "TAG=$TAG" >> "$GITHUB_ENV" | |
| echo "DIGEST=$DIGEST" >> "$GITHUB_ENV" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| - name: Install Python + deps + lium | |
| run: | | |
| uv python install 3.12 | |
| uv sync --extra dev | |
| uv tool install lium.io | |
| sudo apt-get install -y -qq jq | |
| - name: Rent, boot, check, tear down | |
| env: | |
| CONFORMANCE_RENT_WAIT_MIN: "20" | |
| # lium registers ~/.ssh/id_ed25519 with every pod it rents (the SDK reads only that path); the checker's | |
| # ssh tunnel into the runtime pod uses the same key. | |
| run: | | |
| mkdir -p -m 700 ~/.ssh | |
| [ -f ~/.ssh/id_ed25519 ] || ssh-keygen -t ed25519 -N '' -q -f ~/.ssh/id_ed25519 | |
| scripts/serving_conformance_on_lium.sh "$TAG" "conformance-$REF" | |
| - name: Upload the report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: serving-conformance-${{ env.REF }} | |
| path: conformance-${{ env.REF }}/ | |
| if-no-files-found: warn | |
| - name: Bump runtime_pin and runtime_image; fill speed facts only where the release has none | |
| run: | | |
| jq --arg pin "gittensor-ai-lab/sparkinfer@$REF" --arg image "entrius/sparkinfer:$TAG@$DIGEST" --arg run "$GITHUB_RUN_ID" \ | |
| --slurpfile speed "conformance-$REF/speed.json" \ | |
| '.releases[0].runtime_pin = $pin | |
| | .releases[0].runtime_image = $image | |
| | (.releases[0].speed // {}) as $cur | |
| | .releases[0].speed = ($cur + { | |
| single_stream_decode_tps: ($cur.single_stream_decode_tps // $speed[0].single_stream_decode_tps), | |
| aggregate_decode_tps: ($cur.aggregate_decode_tps // $speed[0].aggregate_decode_tps), | |
| decode_per_request: ($cur.decode_per_request // $speed[0].decode_per_request), | |
| measured_on: ($cur.measured_on // ($pin + " on a rented RTX 5090, actions run " + $run)) | |
| }) | |
| | .releases[0].attest = ((.releases[0].attest // {}) + { | |
| iters: ($speed[0].attest.attest_iters // 3), | |
| vram_model_reserved_bytes: ($speed[0].attest.vram_model_reserved_bytes // .releases[0].attest.vram_model_reserved_bytes // 24000000000), | |
| reference_wall_ms: $speed[0].attest.attest_ref_wall_ms | |
| })' \ | |
| gittensor/validator/weights/serving_loadout.json > /tmp/loadout.json | |
| mv /tmp/loadout.json gittensor/validator/weights/serving_loadout.json | |
| git diff --stat | |
| - name: Open the pin-bump PR | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.RELEASE_TOKEN }} | |
| base: test | |
| branch: serving/pin-${{ env.REF }} | |
| commit-message: "serving: bless sparkinfer ${{ env.REF }}" | |
| title: "serving: bless sparkinfer ${{ env.REF }}" | |
| body: | | |
| `entrius/sparkinfer:${{ env.REF }}` passed `scripts/check_serving_runtime.py` on a rented RTX 5090 | |
| (run ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} — report in the | |
| `serving-conformance-${{ env.REF }}` artifact). This bumps `runtime_pin`, pins `runtime_image` to the pushed | |
| image's digest, and writes the attestation facts (`attest.reference_wall_ms`). Speed facts | |
| (`speed.aggregate_decode_tps` sets the per-token pay rate; `speed.decode_per_request` is the curve served | |
| traffic is priced against) are filled only where the release has none: a CI host measured over the | |
| network reads 10-25% under an on-box blessing, so re-blessing throughput is a deliberate human edit — the | |
| run's measurement is in the artifact's `speed.json`. Update the sha named in the comments at | |
| `gittensor/constants.py` and `gittensor/serving/audit.py` if the blessing rules changed. | |
| add-paths: gittensor/validator/weights/serving_loadout.json |