ci(sparkinfer): conformance-check a published tag without republishin… #38
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: sparkinfer image | |
| # Builds the blessed serving runtime (see the Serving Runtime Contract in the miner docs) from a pinned upstream commit and | |
| # publishes it as entrius/sparkinfer:<ref>. Run manually with the commit you intend to bless; the | |
| # resulting tag is what goes into `runtime_pin` in gittensor/validator/weights/serving_loadout.json. | |
| # No GPU in CI: the build job only proves the release builds. The `conformance` job then rents one RTX 5090 | |
| # on Lium (scripts/serving_conformance_on_lium.sh), runs scripts/check_serving_runtime.py against the pushed | |
| # image, uploads the report, and on a clean pass opens the pin-bump PR. Needs the LIUM_API_KEY secret. | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| sparkinfer_ref: | |
| description: "sparkinfer commit to build (becomes the image tag). Empty = the runtime_pin in serving_loadout.json." | |
| required: false | |
| default: "" | |
| cuda_archs: | |
| description: "CMAKE_CUDA_ARCHITECTURES" | |
| required: false | |
| default: "120" | |
| image_tag: | |
| description: "Image tag to push (default: the sparkinfer ref). Use e.g. <ref>-attest1 for a gittensor-side image change so the blessed tag is not overwritten." | |
| required: false | |
| default: "" | |
| conformance: | |
| description: "After the push, rent a 5090 on Lium, run the conformance checker and open the pin-bump PR on a pass" | |
| type: boolean | |
| required: false | |
| default: true | |
| build: | |
| description: "Build and push the image. Uncheck to conformance-check a tag already on Docker Hub without republishing it (the digest is read from the registry)." | |
| type: boolean | |
| required: false | |
| default: true | |
| # Build-only check (never pushes) whenever the Dockerfile or this workflow changes. | |
| push: | |
| paths: | |
| - docker/sparkinfer.Dockerfile | |
| - .github/workflows/sparkinfer-image.yml | |
| jobs: | |
| image: | |
| if: github.event_name != 'workflow_dispatch' || inputs.build | |
| outputs: | |
| ref: ${{ steps.ref.outputs.ref }} | |
| tag: ${{ steps.ref.outputs.tag }} | |
| digest: ${{ steps.build.outputs.digest }} | |
| # push-triggered runs (Dockerfile/workflow changed) only BUILD, to catch Dockerfile breakage in CI; | |
| # the image is pushed to Docker Hub only from the Run workflow button (workflow_dispatch). | |
| name: ${{ github.event_name == 'workflow_dispatch' && 'build and push' || 'build only (dry run)' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Resolve the sparkinfer ref (input, else runtime_pin in serving_loadout.json) | |
| id: ref | |
| run: | | |
| REF="${{ inputs.sparkinfer_ref }}" | |
| if [ -z "$REF" ]; then | |
| REF=$(jq -r '.releases[0].runtime_pin' gittensor/validator/weights/serving_loadout.json | sed 's/.*@//') | |
| fi | |
| echo "ref=$REF" >> "$GITHUB_OUTPUT" | |
| TAG="${{ inputs.image_tag }}"; echo "tag=${TAG:-$REF}" >> "$GITHUB_OUTPUT" | |
| echo "building sparkinfer@$REF as entrius/sparkinfer:${TAG:-$REF}" | |
| - name: Free disk space (CUDA devel image is large) | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| docker system prune -af | |
| - name: Log in to Docker Hub | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Set up Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push | |
| id: build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: docker/sparkinfer.Dockerfile | |
| push: ${{ github.event_name == 'workflow_dispatch' }} | |
| build-args: | | |
| SPARKINFER_REF=${{ steps.ref.outputs.ref }} | |
| CUDA_ARCHS=${{ inputs.cuda_archs || '120' }} | |
| tags: | | |
| entrius/sparkinfer:${{ steps.ref.outputs.tag }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| conformance: | |
| name: conformance on a rented 5090 | |
| needs: image | |
| # Runs whether or not the image job ran: with `build` unchecked it checks a tag already on Docker Hub, so a | |
| # blessed tag is never republished just to re-measure it. | |
| if: | | |
| always() && github.event_name == 'workflow_dispatch' && inputs.conformance | |
| && (needs.image.result == 'success' || needs.image.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| env: | |
| LIUM_API_KEY: ${{ secrets.LIUM_API_KEY }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Resolve ref, tag and digest (from the build job, else from the registry) | |
| id: img | |
| run: | | |
| REF="${{ needs.image.outputs.ref }}" | |
| TAG="${{ needs.image.outputs.tag }}" | |
| DIGEST="${{ needs.image.outputs.digest }}" | |
| if [ -z "$REF" ]; then | |
| REF="${{ inputs.sparkinfer_ref }}" | |
| [ -n "$REF" ] || REF=$(jq -r '.releases[0].runtime_pin' gittensor/validator/weights/serving_loadout.json | sed 's/.*@//') | |
| fi | |
| if [ -z "$TAG" ]; then | |
| TAG="${{ inputs.image_tag }}"; TAG="${TAG:-$REF}" | |
| fi | |
| if [ -z "$DIGEST" ]; then | |
| DIGEST=$(curl -fsS "https://hub.docker.com/v2/repositories/entrius/sparkinfer/tags/$TAG" | jq -r '.digest // empty') | |
| [ -n "$DIGEST" ] || { echo "entrius/sparkinfer:$TAG is not on Docker Hub; build it first"; exit 1; } | |
| echo "checking the published entrius/sparkinfer:$TAG ($DIGEST) — not rebuilt" | |
| fi | |
| echo "REF=$REF" >> "$GITHUB_ENV" | |
| echo "TAG=$TAG" >> "$GITHUB_ENV" | |
| echo "DIGEST=$DIGEST" >> "$GITHUB_ENV" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v4 | |
| - name: Install Python + deps + lium | |
| run: | | |
| uv python install 3.12 | |
| uv sync --extra dev | |
| uv tool install lium.io | |
| sudo apt-get install -y -qq jq | |
| - name: Rent, boot, check, tear down | |
| env: | |
| CONFORMANCE_RENT_WAIT_MIN: "20" | |
| run: scripts/serving_conformance_on_lium.sh "$TAG" "conformance-$REF" | |
| - name: Upload the report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: serving-conformance-${{ env.REF }} | |
| path: conformance-${{ env.REF }}/ | |
| if-no-files-found: warn | |
| - name: Bump runtime_pin, runtime_image and the release's speed facts | |
| run: | | |
| jq --arg pin "gittensor-ai-lab/sparkinfer@$REF" --arg image "entrius/sparkinfer:$TAG@$DIGEST" --arg run "$GITHUB_RUN_ID" \ | |
| --slurpfile speed "conformance-$REF/speed.json" \ | |
| '.releases[0].runtime_pin = $pin | |
| | .releases[0].runtime_image = $image | |
| | .releases[0].speed = ((.releases[0].speed // {}) + { | |
| single_stream_decode_tps: $speed[0].single_stream_decode_tps, | |
| decode_per_request: $speed[0].decode_per_request, | |
| measured_on: ($pin + " on a rented RTX 5090, actions run " + $run) | |
| }) | |
| | .releases[0].attest = ((.releases[0].attest // {}) + { | |
| iters: ($speed[0].attest.attest_iters // 3), | |
| vram_model_reserved_bytes: ($speed[0].attest.vram_model_reserved_bytes // 24000000000), | |
| reference_wall_ms: $speed[0].attest.attest_ref_wall_ms | |
| })' \ | |
| gittensor/validator/weights/serving_loadout.json > /tmp/loadout.json | |
| mv /tmp/loadout.json gittensor/validator/weights/serving_loadout.json | |
| git diff --stat | |
| - name: Open the pin-bump PR | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| token: ${{ secrets.RELEASE_TOKEN }} | |
| base: test | |
| branch: serving/pin-${{ env.REF }} | |
| commit-message: "serving: bless sparkinfer ${{ env.REF }}" | |
| title: "serving: bless sparkinfer ${{ env.REF }}" | |
| body: | | |
| `entrius/sparkinfer:${{ env.REF }}` passed `scripts/check_serving_runtime.py` on a rented RTX 5090 | |
| (run ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} — report in the | |
| `serving-conformance-${{ env.REF }}` artifact). This bumps `runtime_pin`, pins `runtime_image` to the pushed | |
| image's digest, and writes the release's | |
| blessing-time speed facts (`speed.decode_per_request`, the per-request decode curve served traffic is | |
| priced against) and attestation facts (`attest.*`). Update the sha named in the comments at | |
| `gittensor/constants.py` and `gittensor/serving/audit.py` if the blessing rules changed. | |
| add-paths: gittensor/validator/weights/serving_loadout.json |