akshaydeo · Oct 29, 2026
@@ -2680,12 +2680,11 @@ jobs:
             api.tokenfactory.nebius.com:443
             api.x.ai:443
             auth.docker.io:443
-            azure.ports.ubuntu.com:80
             bedrock-mantle.us-east-1.api.aws:443
             bedrock.us-east-1.amazonaws.com:443
             bifrost-gateway-resource.openai.azure.com:443
             dl-cdn.alpinelinux.org:443
-            esm.ubuntu.com:443
+            download.docker.com:443
             fonts.googleapis.com:443
             fonts.gstatic.com:443
             generativelanguage.googleapis.com:443
@@ -2716,20 +2715,16 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      # The arm-64-2cpu-8gb image ships without Docker, libatomic (needed by the
+      # linux-arm64 Node binary) or build tools; see the script for details.
+      - name: Install arm64 runner dependencies
+        run: bash ./.github/workflows/scripts/install-arm64-runner-deps.sh node docker jq
+
       - name: Set up Go
         uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
           go-version: "1.27.3"
 
-      # The arm64 base image does not ship libatomic1, which the official
-      # linux-arm64 Node binary links against; without it node exits 127.
-      - name: Install libatomic1 for Node.js
-        run: |
-          if ! ldconfig -p | grep -q 'libatomic\.so\.1'; then
-            sudo apt-get update
-            sudo apt-get install -y --no-install-recommends libatomic1
-          fi
-
       - name: Set up Node.js
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
@@ -3359,8 +3354,6 @@ jobs:
           allowed-endpoints: >
             7defe2860d5ee49a1e667e1eeea34b25.r2.cloudflarestorage.com:443
             api.github.com:443
-            azure.ports.ubuntu.com:80
-            esm.ubuntu.com:443
             files.pythonhosted.org:443
             fonts.googleapis.com:443
             fonts.gstatic.com:443
@@ -3383,20 +3376,16 @@ jobs:
       - name: Pull latest changes
         run: git pull origin ${{ github.ref_name }}
 
+      # The arm-64-2cpu-8gb image ships without Docker, libatomic (needed by the
+      # linux-arm64 Node binary) or build tools; see the script for details.
+      - name: Install arm64 runner dependencies
+        run: bash ./.github/workflows/scripts/install-arm64-runner-deps.sh node build
+
       - name: Set up Go
         uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
           go-version: "1.27.3"
 
-      # The arm64 base image does not ship libatomic1, which the official
-      # linux-arm64 Node binary links against; without it node exits 127.
-      - name: Install libatomic1 for Node.js
-        run: |
-          if ! ldconfig -p | grep -q 'libatomic\.so\.1'; then
-            sudo apt-get update
-            sudo apt-get install -y --no-install-recommends libatomic1
-          fi
-
       - name: Set up Node.js
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
@@ -3639,11 +3628,13 @@ jobs:
           allowed-endpoints: >
             auth.docker.io:443
             dl-cdn.alpinelinux.org:443
+            download.docker.com:443
             fonts.googleapis.com:443
             fonts.gstatic.com:443
             ghcr.io:443
             github.com:443
             pkg-containers.githubusercontent.com:443
+            ports.ubuntu.com:80
             production.cloudflare.docker.com:443
             production.cloudfront.docker.com:443
             proxy.golang.org:443
@@ -3657,6 +3648,11 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      # The arm-64-2cpu-8gb image ships without Docker, libatomic (needed by the
+      # linux-arm64 Node binary) or build tools; see the script for details.
+      - name: Install arm64 runner dependencies
+        run: bash ./.github/workflows/scripts/install-arm64-runner-deps.sh docker gh
+
       - name: Verify bifrost-http release
         id: verify
         continue-on-error: true
@@ -3935,11 +3931,13 @@ jobs:
           allowed-endpoints: >
             auth.docker.io:443
             dl-cdn.alpinelinux.org:443
+            download.docker.com:443
             fonts.googleapis.com:443
             fonts.gstatic.com:443
             ghcr.io:443
             github.com:443
             pkg-containers.githubusercontent.com:443
+            ports.ubuntu.com:80
             production.cloudflare.docker.com:443
             production.cloudfront.docker.com:443
             proxy.golang.org:443
@@ -3955,6 +3953,11 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
+      # The arm-64-2cpu-8gb image ships without Docker, libatomic (needed by the
+      # linux-arm64 Node binary) or build tools; see the script for details.
+      - name: Install arm64 runner dependencies
+        run: bash ./.github/workflows/scripts/install-arm64-runner-deps.sh docker gh
+
       - name: Verify bifrost-http release
         id: verify
         continue-on-error: true

@@ -44,6 +44,13 @@ import yaml
 # esm.ubuntu.com serves the Expanded Security Maintenance lists apt refreshes
 # alongside it.
 REQUIRED_APT_HOSTS = ("azure.archive.ubuntu.com:80", "esm.ubuntu.com:443")
+# arm64 Ubuntu pulls from the ports mirror (ubuntu-ports), never the archive one;
+# the arm-64-2cpu-8gb image's apt sources point straight at ports.ubuntu.com.
+REQUIRED_ARM64_APT_HOSTS = ("ports.ubuntu.com:80",)
+# install-arm64-runner-deps.sh runs apt-get itself, so a step calling it never
+# shows "apt-get" here; its `docker` component also adds Docker's apt repo.
+ARM64_INSTALLER = "install-arm64-runner-deps.sh"
+DOCKER_APT_HOST = "download.docker.com:443"
 
 failures = []
 checked = 0
@@ -57,13 +64,15 @@ for path in sys.argv[1:]:
             continue
         steps = job.get("steps") or []
 
-        uses_apt = any(
-            "apt-get" in (step.get("run") or "")
-            for step in steps
-            if isinstance(step, dict)
-        )
+        runs = [step.get("run") or "" for step in steps if isinstance(step, dict)]
+        uses_apt = any("apt-get" in run or ARM64_INSTALLER in run for run in runs)
         if not uses_apt:
             continue
+        installs_docker = any(
+            ARM64_INSTALLER in line and "docker" in line.split(ARM64_INSTALLER, 1)[1].split()
+            for run in runs
+            for line in run.splitlines()
+        )
 
         harden = next(
             (
@@ -83,7 +92,11 @@ for path in sys.argv[1:]:
 
         checked += 1
         allowed = set((with_block.get("allowed-endpoints") or "").split())
-        missing = [host for host in REQUIRED_APT_HOSTS if host not in allowed]
+        is_arm = "arm" in str(job.get("runs-on") or "").lower()
+        required = REQUIRED_ARM64_APT_HOSTS if is_arm else REQUIRED_APT_HOSTS
+        if installs_docker:
+            required += (DOCKER_APT_HOST,)
+        missing = [host for host in required if host not in allowed]
         if missing:
             failures.append((path, job_name, missing))
 

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Installs the tools the arm-64-2cpu-8gb runner image no longer ships.
+#
+# Since late 2026-10 that label boots a bare "Base Image for Ubuntu Server 22.04"
+# instead of GitHub's ubuntu-24.04-arm runner image, so Docker (with buildx and
+# compose), jq, gh, a C toolchain, and libatomic (which the official linux-arm64
+# Node binary links against) are all missing. Each component is skipped when it
+# is already present, so on a full runner image this is a no-op.
+#
+# Runs after harden-runner like every other step. Callers allowlist
+# ports.ubuntu.com:80, plus download.docker.com:443 when they ask for `docker`;
+# check-egress-allowlist.sh enforces both. Docker's apt repo is trusted only
+# through a key whose primary fingerprint matches DOCKER_KEY_FINGERPRINT, and
+# apt then verifies every package against that key.
+#
+# Usage: ./install-arm64-runner-deps.sh <component>...
+#   node    libatomic1, for the official linux-arm64 Node binary
+#   docker  Docker CE, containerd, buildx and compose plugins
+#   build   make, gcc, g++ and libc headers for native CGO builds
+#   jq, gh  the tools of the same name
+
+DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
+OS_RELEASE="${OS_RELEASE:-/etc/os-release}"
+
+if [[ $# -eq 0 ]]; then
+  echo "Usage: $0 <node|docker|build|jq|gh>..." >&2
+  exit 1
+fi
+
+has() { command -v "$1" >/dev/null 2>&1; }
+
+packages=()
+want_docker=0
+for component in "$@"; do
+  case "$component" in
+    # grep without -q: an early exit would SIGPIPE ldconfig and fail the pipeline.
+    node) ldconfig -p | grep 'libatomic\.so\.1' >/dev/null || packages+=(libatomic1) ;;
+    build) { has make && has gcc && has g++; } || packages+=(make gcc g++ libc6-dev) ;;
+    jq | gh) has "$component" || packages+=("$component") ;;
+    docker)
+      if ! { has docker && docker buildx version >/dev/null 2>&1 && docker compose version >/dev/null 2>&1; }; then
+        want_docker=1
+        packages+=(ca-certificates)
+        has curl || packages+=(curl)
+        has gpg || packages+=(gnupg)
+      fi
+      ;;
+    *) echo "::error::unknown component: $component" >&2; exit 1 ;;
+  esac
+done
+
+if [[ ${#packages[@]} -gt 0 ]]; then
+  echo "Installing from Ubuntu: ${packages[*]}"
+  sudo apt-get update
+  sudo apt-get install -y --no-install-recommends "${packages[@]}"
+fi
+
+if [[ $want_docker -eq 1 ]]; then
+  KEY="$(mktemp)"
+  trap 'rm -f "$KEY"' EXIT
+  curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o "$KEY"
+
+  # Primary keys only: subkeys carry their own fpr lines, and a served file with
+  # an extra primary key must fail rather than ride along with the pinned one.
+  primaries="$(gpg --with-colons --show-keys "$KEY" | awk -F: '$1 == "pub" { p = 1; next } $1 == "fpr" && p { print $10; p = 0 }')"
+  if [[ "$primaries" != "$DOCKER_KEY_FINGERPRINT" ]]; then
+    echo "::error::Docker apt key primary fingerprint(s) [${primaries//$'\n'/ }] do not match ${DOCKER_KEY_FINGERPRINT}" >&2
+    exit 1
+  fi
+
+  # shellcheck source=/dev/null
+  codename="$(. "$OS_RELEASE" && echo "$VERSION_CODENAME")"
+  sudo install -D -m 0644 "$KEY" /etc/apt/keyrings/docker.asc
+  echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${codename} stable" \
+    | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
+  sudo apt-get update
+  sudo apt-get install -y --no-install-recommends \
+    docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+  # The runner process started before the docker group existed, so later steps
+  # never pick up that membership; open the socket on this throwaway VM instead.
+  sudo chmod 666 /var/run/docker.sock
+  docker version
+fi
@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Exercise the real installer offline: per-component skip-when-present, the
+# packages each component asks apt for, and the Docker key pin failing closed
+# before Docker's repo is ever added. sudo is a stub that records each command
+# instead of running it, so nothing here touches the host.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+mkdir -p "$WORK/bin" "$WORK/base"
+
+# Only these real tools are visible, so the host's own jq/gcc/docker cannot leak in.
+for tool in awk bash cat grep mktemp rm seq; do ln -s "$(command -v "$tool")" "$WORK/base/$tool"; done
+
+printf 'VERSION_CODENAME=jammy\n' > "$WORK/os-release"
+GOOD_FPR=9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+
+cat > "$WORK/base/sudo" <<'EOF'
+#!/usr/bin/env bash
+echo "sudo $*" >> "$TEST_LOG"
+[[ "$1" == tee ]] && cat >> "$TEST_LOG"
+exit 0
+EOF
+# Like the real cache, the match is followed by a long tail, so a reader that
+# stops at the first match (grep -q) SIGPIPEs this writer under pipefail.
+cat > "$WORK/base/ldconfig" <<'EOF'
+#!/usr/bin/env bash
+[[ "${TEST_HAS_LIBATOMIC:-0}" == 1 ]] && echo "libatomic.so.1 (libc6,AArch64) => /lib/aarch64-linux-gnu/libatomic.so.1"
+for i in $(seq 1 20000); do echo "libstub$i.so.1 (libc6,AArch64) => /lib/aarch64-linux-gnu/libstub$i.so.1"; done
+EOF
+cat > "$WORK/base/curl" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "$2" == "https://download.docker.com/linux/ubuntu/gpg" ]]
+printf 'key fixture\n' > "$4"
+EOF
+cat > "$WORK/base/gpg" <<'EOF'
+#!/usr/bin/env bash
+for fpr in $TEST_KEY_PRIMARIES; do
+  echo "pub:-:4096:1:8D81803C0EBFCD88:1487788586:::-:::scESA::::::23::0:"
+  echo "fpr:::::::::${fpr}:"
+  echo "sub:-:4096:1:7EA0A9C3F273FCD8:1487792064:::::s::::::23:"
+  echo "fpr:::::::::D3306A018370199E527AE7997EA0A9C3F273FCD8:"
+done
+EOF
+cat > "$WORK/base/dpkg" <<'EOF'
+#!/usr/bin/env bash
+echo arm64
+EOF
+# A docker stub whose buildx/compose subcommands answer only when the full set is installed.
+cat > "$WORK/docker-full" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+cat > "$WORK/docker-bare" <<'EOF'
+#!/usr/bin/env bash
+[[ "$1" == version ]]
+EOF
+cat > "$WORK/present" <<'EOF'
+#!/usr/bin/env bash
+exit 0
+EOF
+chmod +x "$WORK/base/"* "$WORK/docker-full" "$WORK/docker-bare" "$WORK/present"
+
+passed=0
+# run <case-name> <tools on PATH> -- <components>...
+run() {
+  local tools="$1"; shift
+  rm -rf "${WORK:?}/bin" && mkdir -p "$WORK/bin"
+  for tool in $tools; do
+    case "$tool" in
+      docker) ln -s "$WORK/docker-full" "$WORK/bin/docker" ;;
+      docker-bare) ln -s "$WORK/docker-bare" "$WORK/bin/docker" ;;
+      *) ln -s "$WORK/present" "$WORK/bin/$tool" ;;
+    esac
+  done
+  export TEST_LOG="$WORK/log"
+  : > "$TEST_LOG"
+  PATH="$WORK/bin:$WORK/base" OS_RELEASE="$WORK/os-release" \
+    "$BASH" "$SCRIPT_DIR/install-arm64-runner-deps.sh" "$@" > "$WORK/out" 2>&1
+}
+fail() { cat "$WORK/out" "$TEST_LOG" >&2; echo "FAIL: $1" >&2; exit 1; }
+ok() { echo "ok - $1"; passed=$((passed + 1)); }
+
+export TEST_HAS_LIBATOMIC=1 TEST_KEY_PRIMARIES="$GOOD_FPR"
+run "docker jq gh make gcc g++" node docker build jq gh || fail "full image errored"
+[[ ! -s "$TEST_LOG" ]] || fail "full image still called sudo"
+ok "a full runner image installs nothing"
+
+export TEST_HAS_LIBATOMIC=0
+run "" node || fail "node errored"
+grep -qx "sudo apt-get install -y --no-install-recommends libatomic1" "$TEST_LOG" || fail "node did not install exactly libatomic1"
+ok "node installs libatomic1 when ldconfig lacks it"
+
+run "make gcc" build jq gh || fail "build errored"
+grep -qx "sudo apt-get install -y --no-install-recommends make gcc g++ libc6-dev jq gh" "$TEST_LOG" || fail "build/jq/gh package list wrong"
+ok "build installs the toolchain when any of make/gcc/g++ is missing, plus jq and gh"
+
+run "docker-bare" docker || fail "docker errored"
+grep -qx "sudo install -D -m 0644 .* /etc/apt/keyrings/docker.asc" "$TEST_LOG" || fail "key not installed"
+grep -qx "deb \[arch=arm64 signed-by=/etc/apt/keyrings/docker.asc\] https://download.docker.com/linux/ubuntu jammy stable" "$TEST_LOG" || fail "repo line wrong"
+grep -qx "sudo apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin" "$TEST_LOG" || fail "docker packages wrong"
+grep -qx "sudo chmod 666 /var/run/docker.sock" "$TEST_LOG" || fail "socket not opened"
+ok "docker without buildx/compose adds the pinned repo and installs the full set"
+
+for bad in "0000000000000000000000000000000000000000" "$GOOD_FPR 1111111111111111111111111111111111111111"; do
+  export TEST_KEY_PRIMARIES="$bad"
+  if run "docker-bare" docker; then fail "accepted key with primaries [$bad]"; fi
+  ! grep -q "docker.list\|docker-ce" "$TEST_LOG" || fail "repo added despite bad key [$bad]"
+  grep -q "do not match $GOOD_FPR" "$WORK/out" || fail "no fingerprint error for [$bad]"
+done
+ok "a wrong or extra primary key fails before Docker's repo is added"
+
+export TEST_KEY_PRIMARIES="$GOOD_FPR"
+if run "" nodejs; then fail "accepted unknown component"; fi
+[[ ! -s "$TEST_LOG" ]] || fail "unknown component still called sudo"
+ok "an unknown component fails without installing anything"
+
+echo "$passed passed"
@@ -65,3 +65,6 @@ jobs:
 
       - name: Check Docker Compose installer
         run: bash ./.github/workflows/scripts/install-docker-compose.test.sh
+
+      - name: Check arm64 runner dependency installer
+        run: bash ./.github/workflows/scripts/install-arm64-runner-deps.test.sh