Promote dev: NVFP4 quantize target, linux-x64-cuda-13.3 build, GPU smoke CI, laguna test fix

.github/scripts/gpu-smoke/remote-smoke.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# Runs ON the rented GPU box. Smokes one released backend archive end-to-end:
+#   download release asset -> quantize f16 -> NVFP4 with the SHIPPED
+#   llama-quantize -> llama-server on GPU -> coherence assert -> llama-bench
+#   -> assert the GPU backend actually did the work (no silent CPU fallback).
+#
+# args: $1 = backend id (linux-x64-vulkan | linux-x64-cuda-13.3)
+#       $2 = release tag (e.g. dev-latest or b10018-1.0.0)
+# Writes /root/smoke.log (progress) and /root/smoke.status (OK/FAIL last line).
+set -uo pipefail   # NOT -e: we handle failures explicitly to always write status
+
+BACKEND="${1:?backend id required}"
+TAG="${2:-dev-latest}"
+REPO="AtomicBot-ai/atomic-llama-cpp-turboquant"
+WORK=/root/smoke
+BIN="$WORK/bin/build/bin"
+
+fail() { echo "FAIL: $*"; echo "FAIL" > /root/smoke.status; exit 1; }
+
+mkdir -p "$WORK" && cd "$WORK"
+export DEBIAN_FRONTEND=noninteractive
+
+echo "== [1/6] runtime deps for $BACKEND =="
+apt-get update -q >/dev/null 2>&1 || true
+apt-get install -yq curl jq >/dev/null 2>&1 || fail "apt basic deps"
+if [ "$BACKEND" = "linux-x64-vulkan" ]; then
+  # Lessons from manual runs on vast boxes:
+  #  - libGLX_nvidia (the vulkan ICD) silently needs X11 client libs
+  #  - the stock jammy vulkan loader (1.3.204) cannot negotiate with the ICD
+  #    of current NVIDIA drivers -> take the loader from LunarG
+  apt-get install -yq libvulkan1 libxext6 libx11-6 wget gnupg >/dev/null 2>&1 || fail "apt vulkan deps"
+  wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add - >/dev/null 2>&1
+  wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list \
+    https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list
+  apt-get update -q >/dev/null 2>&1
+  apt-get install -yq vulkan-sdk >/dev/null 2>&1 || fail "apt vulkan-sdk (LunarG)"
+fi
+# CUDA backend: driver comes from the host, cudart/cublas are bundled in the archive.
+
+echo "== [2/6] release asset =="
+curl -sfLo bin.tar.gz \
+  "https://github.com/$REPO/releases/download/$TAG/llama-turboquant-$BACKEND.tar.gz" \
+  || fail "asset download llama-turboquant-$BACKEND.tar.gz @ $TAG"
+mkdir -p bin && tar xzf bin.tar.gz -C bin || fail "unpack"
+VERSION_LINE=$("$BIN/llama-server" --version 2>&1 | head -1)
+echo "version: $VERSION_LINE"
+echo "$VERSION_LINE" | grep -q "version:" || fail "llama-server --version"
+
+echo "== [3/6] f16 -> NVFP4 with shipped llama-quantize =="
+curl -sfLo m-f16.gguf \
+  "https://huggingface.co/bartowski/SmolLM2-135M-Instruct-GGUF/resolve/main/SmolLM2-135M-Instruct-f16.gguf" \
+  || fail "model download"
+"$BIN/llama-quantize" m-f16.gguf m-nvfp4.gguf NVFP4 >quant.log 2>&1 || fail "llama-quantize NVFP4"
+[ -s m-nvfp4.gguf ] || fail "nvfp4 gguf empty"
+ls -lh m-*.gguf
+
+echo "== [4/6] llama-server on GPU =="
+"$BIN/llama-server" -m m-nvfp4.gguf --port 8099 --no-webui -ngl 99 >server.log 2>&1 &
+SRV=$!
+UP=""
+for i in $(seq 1 90); do
+  if curl -sf http://127.0.0.1:8099/health >/dev/null 2>&1; then UP=1; break; fi
+  kill -0 $SRV 2>/dev/null || break
+  sleep 2
+done
+[ -n "$UP" ] || { tail -20 server.log; fail "server did not become healthy"; }
+echo "health: ok"
+
+echo "== [5/6] generation + coherence assert =="
+ANSWER=$(curl -sf http://127.0.0.1:8099/v1/chat/completions \
+  -H 'Content-Type: application/json' \
+  -d '{"messages":[{"role":"user","content":"What is the capital of France? Reply with just the city name."}],"max_tokens":20,"temperature":0}' \
+  | jq -r '.choices[0].message.content // empty')
+echo "answer: $ANSWER"
+echo "$ANSWER" | grep -qi paris || { kill $SRV 2>/dev/null; fail "answer lacks 'Paris'"; }
+kill $SRV 2>/dev/null; wait $SRV 2>/dev/null
+
+echo "== [6/6] llama-bench + GPU-actually-used assert =="
+"$BIN/llama-bench" -m m-nvfp4.gguf -ngl 99 -p 512 -n 128 >bench.log 2>&1 || fail "llama-bench"
+sed -n '/| model/,$p' bench.log
+case "$BACKEND" in
+  linux-x64-vulkan)
+    grep -q "load_backend: loaded Vulkan backend" bench.log || fail "Vulkan backend not loaded"
+    grep -Eq "ggml_vulkan: 0 = NVIDIA" bench.log || fail "Vulkan device is not the NVIDIA GPU"
+    ;;
+  linux-x64-cuda-13.3)
+    grep -q "load_backend: loaded CUDA backend" bench.log || fail "CUDA backend not loaded"
+    # bench log format: "  Device 0: NVIDIA GeForce RTX 5090, compute capability 12.0"
+    grep -Eq "Device [0-9]+: NVIDIA" bench.log || fail "CUDA device is not an NVIDIA GPU"
+    ;;
+  *) fail "unknown backend $BACKEND" ;;
+esac
+TG=$(awk -F'|' '/tg128/ {gsub(/^ +| +$/,"",$8); split($8,a," "); print a[1]; exit}' bench.log)
+echo "tg128: ${TG:-?} t/s"
+
+echo "SMOKE OK: $BACKEND @ $TAG ($VERSION_LINE)"
+echo "OK" > /root/smoke.status

.github/scripts/gpu-smoke/rent.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Rent ONE vast.ai GPU box for the smoke test: walk the cheapest offers,
+# create sequentially, first to reach `running` + answer ssh wins.
+# Losers/failures are destroyed. Simpler sibling of atomic-forge's
+# rent_race.sh (we need one short-lived box, not a race).
+#
+# env in : VAST_API_KEY, GPU_QUERY, SSH_KEY_FILE, [DISK_GB=40] [MAX_OFFERS=5]
+# out    : iid/host/port appended to $GITHUB_OUTPUT
+set -euo pipefail
+
+DISK_GB="${DISK_GB:-40}"
+MAX_OFFERS="${MAX_OFFERS:-5}"
+IMAGE="nvidia/cuda:12.8.0-runtime-ubuntu22.04"
+
+vastai set api-key "$VAST_API_KEY" >/dev/null
+
+vastai search offers \
+  "$GPU_QUERY disk_space>=$DISK_GB inet_down>=500 reliability>0.98 rentable=true" \
+  -o dph --raw > offers.json
+N=$(jq 'length' offers.json)
+[ "$N" -gt 0 ] || { echo "::error::no vast offers match: $GPU_QUERY"; exit 1; }
+echo "offers found: $N, trying up to $MAX_OFFERS cheapest"
+
+IID=""
+cleanup() { [ -n "$IID" ] && vastai destroy instance "$IID" >/dev/null 2>&1 || true; }
+
+for i in $(seq 0 $((MAX_OFFERS - 1))); do
+  [ "$i" -lt "$N" ] || break
+  OFFER=$(jq -r ".[$i].id" offers.json)
+  DPH=$(jq -r ".[$i].dph_total" offers.json)
+  echo "--- offer $OFFER (\$${DPH}/h)"
+  if ! vastai create instance "$OFFER" --image "$IMAGE" \
+       --disk "$DISK_GB" --ssh --direct --raw > create.json 2>&1; then
+    echo "create failed, next offer"; continue
+  fi
+  IID=$(jq -r '.new_contract // empty' create.json)
+  [ -n "$IID" ] || { echo "no contract id, next offer"; continue; }
+
+  # created -> loading (image pull) -> running; give it 10 minutes
+  for tick in $(seq 1 40); do
+    ST=$(vastai show instance "$IID" --raw 2>/dev/null | jq -r '.actual_status // "?"')
+    [ "$ST" = "running" ] && break
+    sleep 15
+  done
+  if [ "$ST" != "running" ]; then
+    echo "offer $OFFER never reached running ($ST), destroying"
+    cleanup; IID=""; continue
+  fi
+
+  URL=$(vastai ssh-url "$IID")
+  HOST=$(echo "$URL" | sed -E 's|ssh://root@([^:]+):.*|\1|')
+  PORT=$(echo "$URL" | sed -E 's|.*:([0-9]+)$|\1|')
+
+  # ssh может подняться на минуту позже статуса running — пробуем с ретраями
+  OK=""
+  for try in $(seq 1 8); do
+    if ssh -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
+           -i "$SSH_KEY_FILE" -p "$PORT" root@"$HOST" 'echo ssh-ok' 2>/dev/null | grep -q ssh-ok; then
+      OK=1; break
+    fi
+    sleep 15
+  done
+  if [ -z "$OK" ]; then
+    echo "offer $OFFER: ssh unreachable, destroying"
+    cleanup; IID=""; continue
+  fi
+
+  echo "rented: iid=$IID $HOST:$PORT"
+  { echo "iid=$IID"; echo "host=$HOST"; echo "port=$PORT"; } >> "$GITHUB_OUTPUT"
+  exit 0
+done
+
+echo "::error::all offers failed"
+exit 1

.github/workflows/backend-smoke.yml

@@ -0,0 +1,128 @@
+# GPU smoke test of RELEASED backend archives on a rented vast.ai box.
+#
+# Not a required check by design: spot GPU rental is nondeterministic and
+# costs money, so it runs on demand (button) and nightly — never as a PR
+# gate. The result is posted as a NON-required commit status
+# (gpu-smoke/<backend>) on the commit the tested release points at, so the
+# dev -> master promotion PR shows the badge.
+#
+# What one run does (see .github/scripts/gpu-smoke/):
+#   rent cheapest matching GPU -> download the released archive -> quantize
+#   a tiny f16 model to NVFP4 with the SHIPPED llama-quantize -> serve it
+#   with -ngl 99 -> assert a coherent answer -> llama-bench -> assert the
+#   GPU backend actually did the work (a silent CPU fallback must FAIL).
+#
+# Secrets: VAST_API_KEY, VAST_SSH_KEY (private key registered with vast).
+
+name: GPU smoke (vast.ai)
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: 'Release tag to smoke'
+        default: 'dev-latest'
+      backends:
+        description: 'Backends to test (space-separated)'
+        default: 'linux-x64-vulkan linux-x64-cuda-13.3'
+      gpu_query:
+        description: 'vast.ai offer filter'
+        default: 'gpu_name=RTX_5090 num_gpus=1'
+  schedule:
+    # nightly against dev-latest; ~$1/night at current spot prices
+    - cron: '0 3 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.inputs.release_tag || 'dev-latest' }}
+  cancel-in-progress: false
+
+jobs:
+  smoke:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      statuses: write
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ['linux-x64-vulkan', 'linux-x64-cuda-13.3']
+    env:
+      RELEASE_TAG: ${{ github.event.inputs.release_tag || 'dev-latest' }}
+      BACKENDS: ${{ github.event.inputs.backends || 'linux-x64-vulkan linux-x64-cuda-13.3' }}
+      GPU_QUERY: ${{ github.event.inputs.gpu_query || 'gpu_name=RTX_5090 num_gpus=1' }}
+
+    steps:
+      - name: Skip if backend not selected
+        id: gate
+        run: |
+          if echo "$BACKENDS" | grep -qw "${{ matrix.backend }}"; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=false" >> "$GITHUB_OUTPUT"
+            echo "backend ${{ matrix.backend }} not in '$BACKENDS' — skipping"
+          fi
+
+      - name: Clone
+        if: steps.gate.outputs.run == 'true'
+        uses: actions/checkout@v6
+
+      - name: Install vast CLI + ssh key
+        if: steps.gate.outputs.run == 'true'
+        run: |
+          pip install -q vastai
+          install -m 600 /dev/null vast_key
+          printf '%s\n' "${{ secrets.VAST_SSH_KEY }}" > vast_key
+
+      - name: Rent GPU box
+        if: steps.gate.outputs.run == 'true'
+        id: rent
+        env:
+          VAST_API_KEY: ${{ secrets.VAST_API_KEY }}
+          SSH_KEY_FILE: vast_key
+        run: bash .github/scripts/gpu-smoke/rent.sh
+
+      - name: Run smoke on box
+        if: steps.gate.outputs.run == 'true'
+        id: run_smoke
+        env:
+          HOST: ${{ steps.rent.outputs.host }}
+          PORT: ${{ steps.rent.outputs.port }}
+        run: |
+          SSH="ssh -o StrictHostKeyChecking=no -o ConnectTimeout=20 -o ServerAliveInterval=30 -i vast_key -p $PORT root@$HOST"
+          scp -o StrictHostKeyChecking=no -i vast_key -P "$PORT" \
+            .github/scripts/gpu-smoke/remote-smoke.sh root@"$HOST":/root/remote-smoke.sh
+          # vast hosts are known to drop long ssh sessions -> nohup + poll
+          $SSH "nohup bash /root/remote-smoke.sh '${{ matrix.backend }}' '$RELEASE_TAG' >/root/smoke.log 2>&1 &"
+          for i in $(seq 1 60); do
+            STATUS=$($SSH 'cat /root/smoke.status 2>/dev/null' 2>/dev/null || true)
+            [ -n "$STATUS" ] && break
+            sleep 20
+          done
+          echo "===== smoke.log ====="
+          $SSH 'cat /root/smoke.log' 2>/dev/null || true
+          echo "====================="
+          [ "$STATUS" = "OK" ] || { echo "::error::smoke failed for ${{ matrix.backend }}"; exit 1; }
+
+      - name: Post commit status
+        if: always() && steps.gate.outputs.run == 'true' && steps.rent.outcome == 'success'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          SHA=$(gh api "repos/${{ github.repository }}/commits/$RELEASE_TAG" --jq .sha 2>/dev/null || true)
+          [ -n "$SHA" ] || { echo "cannot resolve $RELEASE_TAG to a commit, skipping status"; exit 0; }
+          STATE=failure
+          [ "${{ steps.run_smoke.outcome }}" = "success" ] && STATE=success
+          gh api "repos/${{ github.repository }}/statuses/$SHA" \
+            -f state="$STATE" \
+            -f context="gpu-smoke/${{ matrix.backend }}" \
+            -f description="NVFP4 smoke on rented GPU ($RELEASE_TAG)" \
+            -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+      - name: Destroy GPU box
+        if: always() && steps.rent.outputs.iid != ''
+        env:
+          VAST_API_KEY: ${{ secrets.VAST_API_KEY }}
+        run: |
+          vastai set api-key "$VAST_API_KEY" >/dev/null
+          vastai destroy instance "${{ steps.rent.outputs.iid }}" || true
+          echo "destroyed instance ${{ steps.rent.outputs.iid }}"

.github/workflows/build-vulkan.yml

@@ -17,12 +17,12 @@ on:
       '**/*.glsl'
     ]
 
+  # No paths filter: ubuntu-llvmpipe/ubuntu-arm64 are required status checks
+  # on dev, and a required check that never reports leaves the PR stuck in
+  # "Expected" forever (e.g. a docs-only PR). This is also the only PR job
+  # that runs the full ctest suite, so running it on every PR is intended.
   pull_request:
     types: [opened, synchronize, reopened]
-    paths: [
-      '.github/workflows/build-vulkan.yml',
-      'ggml/src/ggml-vulkan/**'
-    ]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}