diff --git a/.github/scripts/gpu-smoke/remote-smoke.sh b/.github/scripts/gpu-smoke/remote-smoke.sh new file mode 100755 index 000000000000..5050a70ac92a --- /dev/null +++ b/.github/scripts/gpu-smoke/remote-smoke.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Runs ON the rented GPU box. Smokes one released backend archive end-to-end: +# download release asset -> quantize f16 -> NVFP4 with the SHIPPED +# llama-quantize -> llama-server on GPU -> coherence assert -> llama-bench +# -> assert the GPU backend actually did the work (no silent CPU fallback). +# +# args: $1 = backend id (linux-x64-vulkan | linux-x64-cuda-13.3) +# $2 = release tag (e.g. dev-latest or b10018-1.0.0) +# Writes /root/smoke.log (progress) and /root/smoke.status (OK/FAIL last line). +set -uo pipefail # NOT -e: we handle failures explicitly to always write status + +BACKEND="${1:?backend id required}" +TAG="${2:-dev-latest}" +REPO="AtomicBot-ai/atomic-llama-cpp-turboquant" +WORK=/root/smoke +BIN="$WORK/bin/build/bin" + +fail() { echo "FAIL: $*"; echo "FAIL" > /root/smoke.status; exit 1; } + +mkdir -p "$WORK" && cd "$WORK" +export DEBIAN_FRONTEND=noninteractive + +echo "== [1/6] runtime deps for $BACKEND ==" +apt-get update -q >/dev/null 2>&1 || true +apt-get install -yq curl jq >/dev/null 2>&1 || fail "apt basic deps" +if [ "$BACKEND" = "linux-x64-vulkan" ]; then + # Lessons from manual runs on vast boxes: + # - libGLX_nvidia (the vulkan ICD) silently needs X11 client libs + # - the stock jammy vulkan loader (1.3.204) cannot negotiate with the ICD + # of current NVIDIA drivers -> take the loader from LunarG + apt-get install -yq libvulkan1 libxext6 libx11-6 wget gnupg >/dev/null 2>&1 || fail "apt vulkan deps" + wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - >/dev/null 2>&1 + wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list \ + https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list + apt-get update -q >/dev/null 2>&1 + apt-get install -yq vulkan-sdk >/dev/null 2>&1 || fail "apt vulkan-sdk (LunarG)" +fi +# CUDA backend: driver comes from the host, cudart/cublas are bundled in the archive. + +echo "== [2/6] release asset ==" +curl -sfLo bin.tar.gz \ + "https://github.com/$REPO/releases/download/$TAG/llama-turboquant-$BACKEND.tar.gz" \ + || fail "asset download llama-turboquant-$BACKEND.tar.gz @ $TAG" +mkdir -p bin && tar xzf bin.tar.gz -C bin || fail "unpack" +VERSION_LINE=$("$BIN/llama-server" --version 2>&1 | head -1) +echo "version: $VERSION_LINE" +echo "$VERSION_LINE" | grep -q "version:" || fail "llama-server --version" + +echo "== [3/6] f16 -> NVFP4 with shipped llama-quantize ==" +curl -sfLo m-f16.gguf \ + "https://huggingface.co/bartowski/SmolLM2-135M-Instruct-GGUF/resolve/main/SmolLM2-135M-Instruct-f16.gguf" \ + || fail "model download" +"$BIN/llama-quantize" m-f16.gguf m-nvfp4.gguf NVFP4 >quant.log 2>&1 || fail "llama-quantize NVFP4" +[ -s m-nvfp4.gguf ] || fail "nvfp4 gguf empty" +ls -lh m-*.gguf + +echo "== [4/6] llama-server on GPU ==" +"$BIN/llama-server" -m m-nvfp4.gguf --port 8099 --no-webui -ngl 99 >server.log 2>&1 & +SRV=$! +UP="" +for i in $(seq 1 90); do + if curl -sf http://127.0.0.1:8099/health >/dev/null 2>&1; then UP=1; break; fi + kill -0 $SRV 2>/dev/null || break + sleep 2 +done +[ -n "$UP" ] || { tail -20 server.log; fail "server did not become healthy"; } +echo "health: ok" + +echo "== [5/6] generation + coherence assert ==" +ANSWER=$(curl -sf http://127.0.0.1:8099/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"What is the capital of France? Reply with just the city name."}],"max_tokens":20,"temperature":0}' \ + | jq -r '.choices[0].message.content // empty') +echo "answer: $ANSWER" +echo "$ANSWER" | grep -qi paris || { kill $SRV 2>/dev/null; fail "answer lacks 'Paris'"; } +kill $SRV 2>/dev/null; wait $SRV 2>/dev/null + +echo "== [6/6] llama-bench + GPU-actually-used assert ==" +"$BIN/llama-bench" -m m-nvfp4.gguf -ngl 99 -p 512 -n 128 >bench.log 2>&1 || fail "llama-bench" +sed -n '/| model/,$p' bench.log +case "$BACKEND" in + linux-x64-vulkan) + grep -q "load_backend: loaded Vulkan backend" bench.log || fail "Vulkan backend not loaded" + grep -Eq "ggml_vulkan: 0 = NVIDIA" bench.log || fail "Vulkan device is not the NVIDIA GPU" + ;; + linux-x64-cuda-13.3) + grep -q "load_backend: loaded CUDA backend" bench.log || fail "CUDA backend not loaded" + grep -Eiq "cuda.*(NVIDIA|GeForce|RTX|H[0-9]+)" bench.log || fail "CUDA device is not an NVIDIA GPU" + ;; + *) fail "unknown backend $BACKEND" ;; +esac +TG=$(awk -F'|' '/tg128/ {gsub(/ /,"",$7); print $7}' bench.log | head -1) +echo "tg128: ${TG:-?} t/s" + +echo "SMOKE OK: $BACKEND @ $TAG ($VERSION_LINE)" +echo "OK" > /root/smoke.status diff --git a/.github/scripts/gpu-smoke/rent.sh b/.github/scripts/gpu-smoke/rent.sh new file mode 100755 index 000000000000..bde32b4d7725 --- /dev/null +++ b/.github/scripts/gpu-smoke/rent.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Rent ONE vast.ai GPU box for the smoke test: walk the cheapest offers, +# create sequentially, first to reach `running` + answer ssh wins. +# Losers/failures are destroyed. Simpler sibling of atomic-forge's +# rent_race.sh (we need one short-lived box, not a race). +# +# env in : VAST_API_KEY, GPU_QUERY, SSH_KEY_FILE, [DISK_GB=40] [MAX_OFFERS=5] +# out : iid/host/port appended to $GITHUB_OUTPUT +set -euo pipefail + +DISK_GB="${DISK_GB:-40}" +MAX_OFFERS="${MAX_OFFERS:-5}" +IMAGE="nvidia/cuda:12.8.0-runtime-ubuntu22.04" + +vastai set api-key "$VAST_API_KEY" >/dev/null + +vastai search offers \ + "$GPU_QUERY disk_space>=$DISK_GB inet_down>=500 reliability>0.98 rentable=true" \ + -o dph --raw > offers.json +N=$(jq 'length' offers.json) +[ "$N" -gt 0 ] || { echo "::error::no vast offers match: $GPU_QUERY"; exit 1; } +echo "offers found: $N, trying up to $MAX_OFFERS cheapest" + +IID="" +cleanup() { [ -n "$IID" ] && vastai destroy instance "$IID" >/dev/null 2>&1 || true; } + +for i in $(seq 0 $((MAX_OFFERS - 1))); do + [ "$i" -lt "$N" ] || break + OFFER=$(jq -r ".[$i].id" offers.json) + DPH=$(jq -r ".[$i].dph_total" offers.json) + echo "--- offer $OFFER (\$${DPH}/h)" + if ! vastai create instance "$OFFER" --image "$IMAGE" \ + --disk "$DISK_GB" --ssh --direct --raw > create.json 2>&1; then + echo "create failed, next offer"; continue + fi + IID=$(jq -r '.new_contract // empty' create.json) + [ -n "$IID" ] || { echo "no contract id, next offer"; continue; } + + # created -> loading (image pull) -> running; give it 10 minutes + for tick in $(seq 1 40); do + ST=$(vastai show instance "$IID" --raw 2>/dev/null | jq -r '.actual_status // "?"') + [ "$ST" = "running" ] && break + sleep 15 + done + if [ "$ST" != "running" ]; then + echo "offer $OFFER never reached running ($ST), destroying" + cleanup; IID=""; continue + fi + + URL=$(vastai ssh-url "$IID") + HOST=$(echo "$URL" | sed -E 's|ssh://root@([^:]+):.*|\1|') + PORT=$(echo "$URL" | sed -E 's|.*:([0-9]+)$|\1|') + + # ssh может подняться на минуту позже статуса running — пробуем с ретраями + OK="" + for try in $(seq 1 8); do + if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=15 \ + -i "$SSH_KEY_FILE" -p "$PORT" root@"$HOST" 'echo ssh-ok' 2>/dev/null | grep -q ssh-ok; then + OK=1; break + fi + sleep 15 + done + if [ -z "$OK" ]; then + echo "offer $OFFER: ssh unreachable, destroying" + cleanup; IID=""; continue + fi + + echo "rented: iid=$IID $HOST:$PORT" + { echo "iid=$IID"; echo "host=$HOST"; echo "port=$PORT"; } >> "$GITHUB_OUTPUT" + exit 0 +done + +echo "::error::all offers failed" +exit 1 diff --git a/.github/workflows/backend-smoke.yml b/.github/workflows/backend-smoke.yml new file mode 100644 index 000000000000..4b8927eb1967 --- /dev/null +++ b/.github/workflows/backend-smoke.yml @@ -0,0 +1,128 @@ +# GPU smoke test of RELEASED backend archives on a rented vast.ai box. +# +# Not a required check by design: spot GPU rental is nondeterministic and +# costs money, so it runs on demand (button) and nightly — never as a PR +# gate. The result is posted as a NON-required commit status +# (gpu-smoke/) on the commit the tested release points at, so the +# dev -> master promotion PR shows the badge. +# +# What one run does (see .github/scripts/gpu-smoke/): +# rent cheapest matching GPU -> download the released archive -> quantize +# a tiny f16 model to NVFP4 with the SHIPPED llama-quantize -> serve it +# with -ngl 99 -> assert a coherent answer -> llama-bench -> assert the +# GPU backend actually did the work (a silent CPU fallback must FAIL). +# +# Secrets: VAST_API_KEY, VAST_SSH_KEY (private key registered with vast). + +name: GPU smoke (vast.ai) + +on: + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to smoke' + default: 'dev-latest' + backends: + description: 'Backends to test (space-separated)' + default: 'linux-x64-vulkan linux-x64-cuda-13.3' + gpu_query: + description: 'vast.ai offer filter' + default: 'gpu_name=RTX_5090 num_gpus=1' + schedule: + # nightly against dev-latest; ~$1/night at current spot prices + - cron: '0 3 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.release_tag || 'dev-latest' }} + cancel-in-progress: false + +jobs: + smoke: + runs-on: ubuntu-22.04 + permissions: + contents: read + statuses: write + strategy: + fail-fast: false + matrix: + backend: ['linux-x64-vulkan', 'linux-x64-cuda-13.3'] + env: + RELEASE_TAG: ${{ github.event.inputs.release_tag || 'dev-latest' }} + BACKENDS: ${{ github.event.inputs.backends || 'linux-x64-vulkan linux-x64-cuda-13.3' }} + GPU_QUERY: ${{ github.event.inputs.gpu_query || 'gpu_name=RTX_5090 num_gpus=1' }} + + steps: + - name: Skip if backend not selected + id: gate + run: | + if echo "$BACKENDS" | grep -qw "${{ matrix.backend }}"; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "backend ${{ matrix.backend }} not in '$BACKENDS' — skipping" + fi + + - name: Clone + if: steps.gate.outputs.run == 'true' + uses: actions/checkout@v6 + + - name: Install vast CLI + ssh key + if: steps.gate.outputs.run == 'true' + run: | + pip install -q vastai + install -m 600 /dev/null vast_key + printf '%s\n' "${{ secrets.VAST_SSH_KEY }}" > vast_key + + - name: Rent GPU box + if: steps.gate.outputs.run == 'true' + id: rent + env: + VAST_API_KEY: ${{ secrets.VAST_API_KEY }} + SSH_KEY_FILE: vast_key + run: bash .github/scripts/gpu-smoke/rent.sh + + - name: Run smoke on box + if: steps.gate.outputs.run == 'true' + id: run_smoke + env: + HOST: ${{ steps.rent.outputs.host }} + PORT: ${{ steps.rent.outputs.port }} + run: | + SSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=20 -o ServerAliveInterval=30 -i vast_key -p $PORT root@$HOST" + scp -o StrictHostKeyChecking=no -i vast_key -P "$PORT" \ + .github/scripts/gpu-smoke/remote-smoke.sh root@"$HOST":/root/remote-smoke.sh + # vast hosts are known to drop long ssh sessions -> nohup + poll + $SSH "nohup bash /root/remote-smoke.sh '${{ matrix.backend }}' '$RELEASE_TAG' >/root/smoke.log 2>&1 &" + for i in $(seq 1 60); do + STATUS=$($SSH 'cat /root/smoke.status 2>/dev/null' 2>/dev/null || true) + [ -n "$STATUS" ] && break + sleep 20 + done + echo "===== smoke.log =====" + $SSH 'cat /root/smoke.log' 2>/dev/null || true + echo "=====================" + [ "$STATUS" = "OK" ] || { echo "::error::smoke failed for ${{ matrix.backend }}"; exit 1; } + + - name: Post commit status + if: always() && steps.gate.outputs.run == 'true' && steps.rent.outcome == 'success' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + SHA=$(gh api "repos/${{ github.repository }}/commits/$RELEASE_TAG" --jq .sha 2>/dev/null || true) + [ -n "$SHA" ] || { echo "cannot resolve $RELEASE_TAG to a commit, skipping status"; exit 0; } + STATE=failure + [ "${{ steps.run_smoke.outcome }}" = "success" ] && STATE=success + gh api "repos/${{ github.repository }}/statuses/$SHA" \ + -f state="$STATE" \ + -f context="gpu-smoke/${{ matrix.backend }}" \ + -f description="NVFP4 smoke on rented GPU ($RELEASE_TAG)" \ + -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + - name: Destroy GPU box + if: always() && steps.rent.outputs.iid != '' + env: + VAST_API_KEY: ${{ secrets.VAST_API_KEY }} + run: | + vastai set api-key "$VAST_API_KEY" >/dev/null + vastai destroy instance "${{ steps.rent.outputs.iid }}" || true + echo "destroyed instance ${{ steps.rent.outputs.iid }}"