wheels-released #574
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Receiver workflow for `wheels-dev/apt-wheels` (R2-backed architecture). | |
| # | |
| # Listens for `repository_dispatch` from `wheels-dev/wheels`'s release workflow, | |
| # downloads the new `.deb` asset from the upstream GitHub Release, syncs the | |
| # existing apt repo state from R2 (so apt-ftparchive can see prior versions), | |
| # regenerates `Packages.gz` / `Release` / `InRelease` via `apt-ftparchive`, | |
| # signs with GPG, and uploads the changed tree to the `wheels-apt` R2 bucket | |
| # (which is served at https://apt.wheels.dev via R2 custom-domain). | |
| # | |
| # This repo no longer commits pool/ or dists/ — those are R2-resident only. | |
| # The repo's job is to hold the WORKFLOW + REGEN SCRIPT + LANDING + GPG KEY. | |
| # | |
| # Trigger payload contract (sender: wheels-dev/wheels release.yml): | |
| # event_type: "wheels-released" | |
| # client_payload: | |
| # version: "<x.y.z>" or "<x.y.z>-snapshot.<n>" | |
| # channel: "stable" | "bleeding-edge" | |
| # | |
| # Manual dispatch is supported via `workflow_dispatch` for backfill / | |
| # disaster-recovery. | |
| name: Publish to apt.wheels.dev | |
| on: | |
| repository_dispatch: | |
| types: [wheels-released] | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Wheels version (e.g. 4.0.1 or 4.0.1-snapshot.1700)' | |
| required: true | |
| type: string | |
| channel: | |
| description: 'Release channel' | |
| required: true | |
| default: stable | |
| type: choice | |
| options: | |
| - stable | |
| - bleeding-edge | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # Serialize across channels — apt-ftparchive scans the whole pool, so | |
| # parallel runs would race on the regenerated Packages.gz / Release uploads. | |
| group: publish-apt | |
| cancel-in-progress: false | |
| env: | |
| R2_BUCKET: wheels-apt | |
| CLOUDFLARE_ACCOUNT_ID: "511d04f367103ec276d875ab41a24dea" | |
| jobs: | |
| publish: | |
| name: Publish ${{ github.event.client_payload.version || inputs.version }} (${{ github.event.client_payload.channel || inputs.channel }}) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Resolve inputs | |
| id: inputs | |
| env: | |
| CLIENT_VERSION: ${{ github.event.client_payload.version }} | |
| CLIENT_CHANNEL: ${{ github.event.client_payload.channel }} | |
| INPUT_VERSION: ${{ inputs.version }} | |
| INPUT_CHANNEL: ${{ inputs.channel }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ github.event_name }}" = "repository_dispatch" ]; then | |
| VERSION="$CLIENT_VERSION" | |
| CHANNEL="$CLIENT_CHANNEL" | |
| else | |
| VERSION="$INPUT_VERSION" | |
| CHANNEL="$INPUT_CHANNEL" | |
| fi | |
| if [ -z "$VERSION" ] || [ -z "$CHANNEL" ]; then | |
| echo "::error::Both version and channel are required." | |
| exit 1 | |
| fi | |
| case "$CHANNEL" in | |
| stable|bleeding-edge) ;; | |
| *) echo "::error::Unsupported channel: $CHANNEL"; exit 1 ;; | |
| esac | |
| case "$CHANNEL" in | |
| stable) PKG="wheels"; UPSTREAM_REPO="wheels-dev/wheels" ;; | |
| bleeding-edge) PKG="wheels-be"; UPSTREAM_REPO="wheels-dev/wheels-snapshots" ;; | |
| esac | |
| { | |
| echo "version=$VERSION" | |
| echo "channel=$CHANNEL" | |
| echo "pkg=$PKG" | |
| echo "upstream_repo=$UPSTREAM_REPO" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Checkout bucket repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Install apt-ftparchive + gpg + wrangler | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends apt-utils gnupg jq | |
| apt-ftparchive --version | |
| gpg --version | head -1 | |
| # wrangler ships with Node — runner has it. Pin to a known-good version. | |
| sudo npm install -g wrangler@4.95.0 | |
| wrangler --version | |
| - name: Import GPG signing key | |
| env: | |
| GPG_PRIVATE_KEY: ${{ secrets.WHEELS_REPO_GPG_PRIVATE_KEY }} | |
| GPG_PASSPHRASE: ${{ secrets.WHEELS_REPO_GPG_PASSPHRASE }} | |
| run: | | |
| set -euo pipefail | |
| if [ -z "${GPG_PRIVATE_KEY:-}" ]; then | |
| echo "::error::WHEELS_REPO_GPG_PRIVATE_KEY is unset; cannot sign Release." | |
| exit 1 | |
| fi | |
| echo "$GPG_PRIVATE_KEY" | gpg --batch --yes --import | |
| # import-ownertrust needs the 40-char fingerprint (fpr: field 10), | |
| # NOT the 16-char key_id (sec: field 5). Using key_id emits a | |
| # non-fatal "invalid fingerprint" warning that's noisy in CI logs. | |
| FINGERPRINT=$(gpg --list-secret-keys --with-colons \ | |
| | awk -F: '/^fpr:/ { print $10; exit }') | |
| KEY_ID=$(gpg --list-secret-keys --keyid-format=long --with-colons \ | |
| | awk -F: '/^sec:/ { print $5; exit }') | |
| echo "Imported signing key: $KEY_ID (fpr $FINGERPRINT)" | |
| echo "$FINGERPRINT:6:" | gpg --batch --yes --import-ownertrust | |
| echo "GPG_KEY_ID=$KEY_ID" >> "$GITHUB_ENV" | |
| - name: Pull existing Packages index from R2 | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CHANNEL: ${{ steps.inputs.outputs.channel }} | |
| run: | | |
| set -euo pipefail | |
| # Incremental regen: download the channel's current Packages index | |
| # (~KBs) instead of the whole pool (~18 GB). regenerate-apt-metadata.sh | |
| # appends the new package's stanza and rebuilds Release, so the | |
| # historical .deb files never need to be pulled again. | |
| pull_index() { | |
| local key="$1" attempt out | |
| for attempt in 1 2 3; do | |
| if out="$(wrangler r2 object get "${R2_BUCKET}/${key}" --file="$key" --remote 2>&1)"; then | |
| echo " pulled ${key} ($(wc -c < "$key") bytes)" | |
| return 0 | |
| fi | |
| # Missing index → first publish on this channel: start empty. | |
| if echo "$out" | grep -qiE "does not exist|NoSuchKey|NoSuchObject"; then | |
| : > "$key" | |
| echo " ${key} absent — starting from an empty index" | |
| return 0 | |
| fi | |
| echo "::warning::index pull failed for ${key} (attempt ${attempt}/3): ${out}" >&2 | |
| sleep $((attempt * 5)) | |
| done | |
| echo "::error::index pull failed permanently for ${key}" >&2 | |
| return 1 | |
| } | |
| for arch in amd64 arm64; do | |
| key="dists/${CHANNEL}/main/binary-${arch}/Packages" | |
| mkdir -p "$(dirname "$key")" | |
| pull_index "$key" | |
| done | |
| - name: Download new .deb from upstream Release | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| VERSION: ${{ steps.inputs.outputs.version }} | |
| PKG: ${{ steps.inputs.outputs.pkg }} | |
| UPSTREAM_REPO: ${{ steps.inputs.outputs.upstream_repo }} | |
| run: | | |
| set -euo pipefail | |
| mkdir -p incoming | |
| # GitHub Release URLs rewrite `~` to `.` at upload time. The version | |
| # in client_payload uses SemVer hyphen (-snapshot.N); the on-URL form | |
| # uses dot (.snapshot.N). Translate before fetch. | |
| URL_VERSION="${VERSION/-snapshot./.snapshot.}" | |
| ASSET="${PKG}_${URL_VERSION}_all.deb" | |
| TAG="v${VERSION}" | |
| echo "Fetching ${ASSET} from ${UPSTREAM_REPO}@${TAG}" | |
| gh release download "$TAG" \ | |
| --repo "$UPSTREAM_REPO" \ | |
| --pattern "$ASSET" \ | |
| --dir incoming/ | |
| ls -lh incoming/ | |
| - name: Slot new .deb into local pool | |
| env: | |
| VERSION: ${{ steps.inputs.outputs.version }} | |
| CHANNEL: ${{ steps.inputs.outputs.channel }} | |
| PKG: ${{ steps.inputs.outputs.pkg }} | |
| run: | | |
| set -euo pipefail | |
| # Canonical pool path uses ~-form (SemVer pre-release separator). | |
| POOL_DIR="pool/${CHANNEL}/${PKG:0:1}/${PKG}" | |
| POOL_FILE="${POOL_DIR}/${PKG}_${VERSION}_all.deb" | |
| mkdir -p "$POOL_DIR" | |
| URL_VERSION="${VERSION/-snapshot./.snapshot.}" | |
| mv "incoming/${PKG}_${URL_VERSION}_all.deb" "$POOL_FILE" | |
| echo "Placed: $POOL_FILE ($(du -h "$POOL_FILE" | cut -f1))" | |
| - name: Regenerate apt metadata + sign | |
| env: | |
| GPG_PASSPHRASE: ${{ secrets.WHEELS_REPO_GPG_PASSPHRASE }} | |
| GPG_KEY_ID: ${{ env.GPG_KEY_ID }} | |
| # Scope regen to the dispatched channel only. The workflow pulls only | |
| # that channel's Packages index and slots only its new .deb, so | |
| # regenerating the OTHER channel would append against a missing index | |
| # and clobber its R2 dists on upload (#3218 / #2838). The other | |
| # channel's dists are left untouched. | |
| CHANNELS: ${{ steps.inputs.outputs.channel }} | |
| run: | | |
| set -euo pipefail | |
| chmod +x scripts/regenerate-apt-metadata.sh | |
| ./scripts/regenerate-apt-metadata.sh | |
| - name: Upload pool + dists to R2 | |
| id: upload | |
| env: | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CHANNEL: ${{ steps.inputs.outputs.channel }} | |
| run: | | |
| set -euo pipefail | |
| # Cache-Control is the load-bearing fix for the stale-index failure | |
| # (#3218 follow-up): Cloudflare auto-caches by file extension, so a | |
| # published `Packages.gz` got edge-cached and kept being served AFTER | |
| # the next publish rewrote it — its hash no longer matched the fresh | |
| # Release, and `apt-get update` rejected it ("File has unexpected | |
| # size"). apt METADATA changes every publish, so it must never be | |
| # edge-cached: tag it `no-store`. The .deb pool files are immutable | |
| # (version-stamped filenames) so they get a long immutable cache. | |
| DEB_CC="public, max-age=31536000, immutable" | |
| META_CC="no-store, max-age=0" | |
| upload_one() { | |
| local local_path="$1" ct="$2" cc="$3" key="$1" attempt out | |
| echo " uploading $key (ct=${ct}; cache-control=${cc})" | |
| for attempt in 1 2 3; do | |
| if out="$(wrangler r2 object put "${R2_BUCKET}/${key}" \ | |
| --file="$local_path" --content-type="$ct" --cache-control="$cc" --remote 2>&1)"; then | |
| return 0 | |
| fi | |
| echo "::warning::r2 object put failed for ${key} (attempt ${attempt}/3): ${out}" >&2 | |
| sleep $((attempt * 5)) | |
| done | |
| echo "::error::r2 object put failed permanently for ${key}" >&2 | |
| return 1 | |
| } | |
| # Collect the absolute URLs we touch so the purge step can evict any | |
| # legacy cached copies (objects cached BEFORE no-store was set linger | |
| # at the edge until their old TTL — a one-time purge unsticks them). | |
| : > /tmp/purge-urls.txt | |
| # Upload pool files (.deb). Only the one we just added is new, but | |
| # uploading all is idempotent (overwrites with same content). | |
| find pool/${CHANNEL} -type f -name '*.deb' | while read -r f; do | |
| upload_one "$f" "application/vnd.debian.binary-package" "$DEB_CC" | |
| echo "https://apt.wheels.dev/$f" >> /tmp/purge-urls.txt | |
| done | |
| # Upload regenerated dists tree. apt-ftparchive rewrites these on | |
| # every run; upload all files in dists/ (regen is now per-channel). | |
| find dists -type f | while read -r f; do | |
| # Pick a sensible content-type per file | |
| case "$f" in | |
| *.gz) ct="application/gzip" ;; | |
| *.gpg) ct="application/pgp-signature" ;; | |
| *InRelease|*Release) ct="text/plain" ;; | |
| *Packages) ct="text/plain" ;; | |
| *) ct="application/octet-stream" ;; | |
| esac | |
| upload_one "$f" "$ct" "$META_CC" | |
| echo "https://apt.wheels.dev/$f" >> /tmp/purge-urls.txt | |
| done | |
| echo "R2 upload complete for channel=${CHANNEL}." | |
| - name: Purge Cloudflare edge cache for the published files | |
| env: | |
| # Cache purge is a SEPARATE Cloudflare permission (Zone.Cache-Purge) | |
| # from R2 (account-scoped). The shared CLOUDFLARE_API_TOKEN only has | |
| # R2 + Zone:Read, so purge with it returns 401. Use a dedicated | |
| # CF_PURGE_TOKEN (Zone.Cache-Purge on the wheels.dev zone) if one is | |
| # configured; otherwise skip — `no-store` on the metadata (set in the | |
| # upload step) already guarantees the edge never serves a stale index, | |
| # so the purge is an optional latency optimization, not correctness. | |
| CF_PURGE_TOKEN: ${{ secrets.CF_PURGE_TOKEN }} | |
| run: | | |
| set -uo pipefail | |
| [ -s /tmp/purge-urls.txt ] || { echo "No URLs to purge."; exit 0; } | |
| if [ -z "${CF_PURGE_TOKEN:-}" ]; then | |
| echo "No CF_PURGE_TOKEN configured — skipping purge. (no-store keeps every publish fresh at the edge; add a Zone.Cache-Purge token as CF_PURGE_TOKEN to evict legacy entries immediately.)" | |
| exit 0 | |
| fi | |
| ZONE_ID="$(curl -fsS "https://api.cloudflare.com/client/v4/zones?name=wheels.dev" \ | |
| -H "Authorization: Bearer ${CF_PURGE_TOKEN}" | jq -r '.result[0].id // empty')" | |
| if [ -z "$ZONE_ID" ]; then | |
| echo "::warning::CF_PURGE_TOKEN is set but the wheels.dev zone could not be resolved (token scope?). Relying on no-store." | |
| exit 0 | |
| fi | |
| # Purge in batches of 30 URLs (Cloudflare per-request cap). | |
| mapfile -t urls < /tmp/purge-urls.txt | |
| i=0 | |
| while [ "$i" -lt "${#urls[@]}" ]; do | |
| batch=("${urls[@]:i:30}") | |
| payload="$(printf '%s\n' "${batch[@]}" | jq -R . | jq -s '{files: .}')" | |
| resp="$(curl -fsS -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/purge_cache" \ | |
| -H "Authorization: Bearer ${CF_PURGE_TOKEN}" \ | |
| -H "Content-Type: application/json" --data "$payload" || true)" | |
| if [ "$(jq -r '.success // false' <<<"$resp")" = "true" ]; then | |
| echo "Purged ${#batch[@]} url(s)." | |
| else | |
| echo "::warning::CF_PURGE_TOKEN purge did not report success: $(jq -rc '.errors // .' <<<"$resp" 2>/dev/null || echo "$resp")" | |
| fi | |
| i=$((i + 30)) | |
| done |