akshaydeo · Oct 29, 2026 · Oct 29, 2026
@@ -95,7 +95,7 @@ jobs:
             api.perplexity.ai:443
             api.replicate.com:443
             api.x.ai:443
-            azure.archive.ubuntu.com:80
+            archive.ubuntu.com:443
             esm.ubuntu.com:443
             bifrost-gateway-resource.openai.azure.com:443
             bedrock-mantle.us-east-1.api.aws:443

@@ -122,7 +122,7 @@ jobs:
             api.perplexity.ai:443
             api.replicate.com:443
             api.x.ai:443
-            azure.archive.ubuntu.com:80
+            archive.ubuntu.com:443
             esm.ubuntu.com:443
             bifrost-gateway-resource.openai.azure.com:443
             bedrock-mantle.us-east-1.api.aws:443
@@ -208,6 +208,7 @@ jobs:
 
       - name: Install test dependencies
         run: |
+          ./.github/workflows/scripts/apt-sources-https.sh
           sudo apt-get update
           sudo apt-get install -y jq
 

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Points the runner's Ubuntu apt sources at HTTPS. Run it before every
+# `apt-get update` in CI.
+#
+# Runner images ship apt sources on plain HTTP. apt's GPG-signed InRelease and
+# per-file SHA256 already stop a tampered package, but HTTP still shows anyone on
+# the path what we install, lets them replay a stale (validly signed) InRelease
+# to hide security updates, and forces a port-80 entry into every harden-runner
+# allowlist. After this runs, apt needs only :443.
+#
+#   http://azure.archive.ubuntu.com  -> https://archive.ubuntu.com
+#       The Azure-local mirror the amd64 images use serves no HTTPS at all (443
+#       is closed), so it is swapped for Canonical's primary archive, which
+#       carries the same pockets, -security included.
+#   http://[azure.]ports.ubuntu.com  -> https://ports.ubuntu.com   (arm64)
+#   http://archive|security.ubuntu.com -> https:// on the same host
+#
+# Any other http:// source on an ubuntu.com host is an error rather than a
+# silent pass: it would reach for :80, which the allowlists no longer carry, and
+# fail as an opaque mirror timeout. check-egress-allowlist.sh requires the
+# matching :443 hosts and that every apt-using step runs this script first.
+#
+# APT_ROOT (default /etc/apt) and APT_SUDO (default sudo; empty runs unprivileged)
+# exist for apt-sources-https.test.sh.
+
+APT_ROOT="${APT_ROOT:-/etc/apt}"
+APT_SUDO="${APT_SUDO-sudo}"
+
+files=()
+for f in "$APT_ROOT/sources.list" "$APT_ROOT"/sources.list.d/*.list "$APT_ROOT"/sources.list.d/*.sources; do
+  [[ -f "$f" ]] && files+=("$f")
+done
+if [[ ${#files[@]} -eq 0 ]]; then
+  echo "apt-sources-https: no apt sources under $APT_ROOT"
+  exit 0
+fi
+
+# Written through a temp copy instead of sed -i so it behaves the same under GNU
+# and BSD sed, and `cp` onto the existing file keeps its owner and mode.
+tmp="$(mktemp)"
+trap 'rm -f "$tmp"' EXIT
+for f in "${files[@]}"; do
+  sed -E \
+    -e 's#http://azure\.archive\.ubuntu\.com([/[:space:]]|$)#https://archive.ubuntu.com\1#g' \
+    -e 's#http://(azure\.)?ports\.ubuntu\.com([/[:space:]]|$)#https://ports.ubuntu.com\2#g' \
+    -e 's#http://(archive|security)\.ubuntu\.com([/[:space:]]|$)#https://\1.ubuntu.com\2#g' \
+    "$f" > "$tmp"
+  if ! cmp -s "$tmp" "$f"; then
+    ${APT_SUDO:+$APT_SUDO} cp "$tmp" "$f"
+    echo "apt-sources-https: switched $f to https"
+  fi
+done
+
+# Comment lines are skipped: stock sources.list carries "http://help.ubuntu.com"
+# links in comments that apt never fetches.
+if leftover="$(grep -HnE '^[^#]*http://[A-Za-z0-9.-]*ubuntu\.com' "${files[@]}")"; then
+  echo "::error::apt sources still use http:// on an ubuntu.com host this script does not map to https:" >&2
+  echo "$leftover" >&2
+  exit 1
+fi
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Exercise apt-sources-https.sh against fixture apt trees shaped like the two
+# runner images: the arm64 Ubuntu 22.04 base image (one-line sources.list on
+# ports.ubuntu.com) and the amd64 Ubuntu 24.04 image (deb822 ubuntu.sources on
+# the Azure-local mirror, plus third-party repos). Runs unprivileged.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+passed=0
+fail() { cat "$WORK/out" >&2; echo "FAIL: $1" >&2; exit 1; }
+ok() { echo "ok - $1"; passed=$((passed + 1)); }
+run() { APT_ROOT="$WORK/apt" APT_SUDO="" "$BASH" "$SCRIPT_DIR/apt-sources-https.sh" > "$WORK/out" 2>&1; }
+reset() { rm -rf "$WORK/apt" && mkdir -p "$WORK/apt/sources.list.d"; }
+
+reset
+cat > "$WORK/apt/sources.list" <<'EOF'
+## See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade
+deb http://ports.ubuntu.com/ubuntu-ports jammy main restricted
+deb http://azure.ports.ubuntu.com/ubuntu-ports jammy-updates main
+deb http://ports.ubuntu.com/ubuntu-ports jammy-security main
+EOF
+run || fail "arm64 sources errored"
+grep -qx "deb https://ports.ubuntu.com/ubuntu-ports jammy main restricted" "$WORK/apt/sources.list" || fail "ports not switched"
+grep -qx "deb https://ports.ubuntu.com/ubuntu-ports jammy-updates main" "$WORK/apt/sources.list" || fail "azure.ports not mapped to ports over https"
+grep -qx "## See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade" "$WORK/apt/sources.list" || fail "comment rewritten"
+ok "arm64 one-line sources move to https://ports.ubuntu.com and comments are left alone"
+
+reset
+cat > "$WORK/apt/sources.list.d/ubuntu.sources" <<'EOF'
+Types: deb
+URIs: http://azure.archive.ubuntu.com/ubuntu/
+Suites: noble noble-updates noble-backports noble-security
+Components: main universe restricted multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+EOF
+cat > "$WORK/apt/sources.list.d/microsoft-prod.list" <<'EOF'
+deb [arch=amd64,arm64,armhf] https://packages.microsoft.com/ubuntu/24.04/prod noble main
+EOF
+cp "$WORK/apt/sources.list.d/microsoft-prod.list" "$WORK/ms.before"
+run || fail "amd64 sources errored"
+grep -qx "URIs: https://archive.ubuntu.com/ubuntu/" "$WORK/apt/sources.list.d/ubuntu.sources" || fail "azure.archive not mapped to https://archive.ubuntu.com"
+cmp -s "$WORK/ms.before" "$WORK/apt/sources.list.d/microsoft-prod.list" || fail "third-party repo touched"
+ok "amd64 deb822 Azure mirror moves to https://archive.ubuntu.com and other repos are untouched"
+
+cp "$WORK/apt/sources.list.d/ubuntu.sources" "$WORK/ubuntu.before"
+run || fail "second run errored"
+cmp -s "$WORK/ubuntu.before" "$WORK/apt/sources.list.d/ubuntu.sources" || fail "second run changed the file"
+! grep -q "switched" "$WORK/out" || fail "second run reported a switch"
+ok "running twice is a no-op"
+
+reset
+printf 'deb http://security.ubuntu.com/ubuntu noble-security main\n' > "$WORK/apt/sources.list"
+run || fail "security sources errored"
+grep -qx "deb https://security.ubuntu.com/ubuntu noble-security main" "$WORK/apt/sources.list" || fail "security not switched"
+ok "security.ubuntu.com stays on its host over https"
+
+reset
+printf 'deb http://us.archive.ubuntu.com/ubuntu noble main\n' > "$WORK/apt/sources.list"
+if run; then fail "accepted an unmapped http ubuntu mirror"; fi
+grep -q "us.archive.ubuntu.com" "$WORK/out" || fail "error does not name the leftover source"
+ok "an unmapped http:// ubuntu.com mirror fails loudly"
+
+rm -rf "$WORK/apt" && mkdir -p "$WORK/apt"
+run || fail "empty apt root errored"
+ok "no apt sources is a no-op"
+
+echo "$passed passed"
@@ -39,18 +39,28 @@ python3 - "${FILES[@]}" <<'PY'
 import sys
 import yaml
 
-# Hosts apt-get needs on a GitHub-hosted ubuntu runner. azure.archive.ubuntu.com
-# is the Azure-local mirror the runner images point at (plain HTTP), and
+# Hosts apt-get needs on a GitHub-hosted ubuntu runner once apt-sources-https.sh
+# has moved the sources to HTTPS. The images point at the Azure-local mirror over
+# plain HTTP, which serves no HTTPS, so the script swaps it for archive.ubuntu.com;
 # esm.ubuntu.com serves the Expanded Security Maintenance lists apt refreshes
 # alongside it.
-REQUIRED_APT_HOSTS = ("azure.archive.ubuntu.com:80", "esm.ubuntu.com:443")
+REQUIRED_APT_HOSTS = ("archive.ubuntu.com:443", "esm.ubuntu.com:443")
 # arm64 Ubuntu pulls from the ports mirror (ubuntu-ports), never the archive one;
 # the arm-64-2cpu-8gb image's apt sources point straight at ports.ubuntu.com.
-REQUIRED_ARM64_APT_HOSTS = ("ports.ubuntu.com:80",)
+REQUIRED_ARM64_APT_HOSTS = ("ports.ubuntu.com:443",)
 # install-arm64-runner-deps.sh runs apt-get itself, so a step calling it never
 # shows "apt-get" here; its `docker` component also adds Docker's apt repo.
 ARM64_INSTALLER = "install-arm64-runner-deps.sh"
-DOCKER_APT_HOST = "download.docker.com:443"
+# Installers that run apt-get themselves, and switch the sources to HTTPS first.
+APT_INSTALLERS = (ARM64_INSTALLER, "install-cross-compilers.sh")
+APT_SOURCES_HTTPS = "apt-sources-https.sh"
+# Docker's apt repo, plus apt's SRV probe for it, and the sigstore TUF root that
+# the freshly installed dockerd fetches when buildx bootstraps its builder.
+DOCKER_HOSTS = (
+    "download.docker.com:443",
+    "_https._tcp.download.docker.com:443",
+    "tuf-repo-cdn.sigstore.dev:443",
+)
 
 failures = []
 checked = 0
@@ -65,7 +75,7 @@ for path in sys.argv[1:]:
         steps = job.get("steps") or []
 
         runs = [step.get("run") or "" for step in steps if isinstance(step, dict)]
-        uses_apt = any("apt-get" in run or ARM64_INSTALLER in run for run in runs)
+        uses_apt = any("apt-get" in run or any(i in run for i in APT_INSTALLERS) for run in runs)
         if not uses_apt:
             continue
         installs_docker = any(
@@ -92,17 +102,27 @@ for path in sys.argv[1:]:
 
         checked += 1
         allowed = set((with_block.get("allowed-endpoints") or "").split())
+        # An inline apt-get update must switch the sources to HTTPS first, in the
+        # same step: otherwise apt reaches for the plain-HTTP mirror, which no
+        # allowlist carries any more, and fails as an opaque fetch timeout.
+        for run in runs:
+            update = run.find("apt-get update")
+            if update != -1 and run.find(APT_SOURCES_HTTPS) not in range(0, update):
+                failures.append((path, job_name, [f"{APT_SOURCES_HTTPS} before apt-get update"]))
+        plain = sorted(e for e in allowed if e.endswith("ubuntu.com:80"))
+        if plain:
+            failures.append((path, job_name, [f"no plain-HTTP apt host ({', '.join(plain)})"]))
         is_arm = "arm" in str(job.get("runs-on") or "").lower()
         required = REQUIRED_ARM64_APT_HOSTS if is_arm else REQUIRED_APT_HOSTS
         if installs_docker:
-            required += (DOCKER_APT_HOST,)
+            required += DOCKER_HOSTS
         missing = [host for host in required if host not in allowed]
         if missing:
             failures.append((path, job_name, missing))
 
 for path, job_name, missing in failures:
     print(f"FAIL {path} :: job '{job_name}' runs apt-get under egress-policy: block "
-          f"but omits {', '.join(missing)}", file=sys.stderr)
+          f"but needs {', '.join(missing)}", file=sys.stderr)
 
 if failures:
     print(f"\n{len(failures)} of {checked} apt-using blocked-egress job(s) have an "

@@ -9,8 +9,9 @@ set -euo pipefail
 # Node binary links against) are all missing. Each component is skipped when it
 # is already present, so on a full runner image this is a no-op.
 #
-# Runs after harden-runner like every other step. Callers allowlist
-# ports.ubuntu.com:80, plus download.docker.com:443 when they ask for `docker`;
+# Runs after harden-runner like every other step. apt-sources-https.sh moves the
+# Ubuntu sources to HTTPS before the first apt-get update, so callers allowlist
+# ports.ubuntu.com:443, plus download.docker.com:443 when they ask for `docker`;
 # check-egress-allowlist.sh enforces both. Docker's apt repo is trusted only
 # through a key whose primary fingerprint matches DOCKER_KEY_FINGERPRINT, and
 # apt then verifies every package against that key.
@@ -23,6 +24,8 @@ set -euo pipefail
 
 DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
 OS_RELEASE="${OS_RELEASE:-/etc/os-release}"
+# ${BASH_SOURCE%/*}, not dirname: the installer test runs with a minimal PATH.
+APT_SOURCES_HTTPS="${BASH_SOURCE%/*}/apt-sources-https.sh"
 
 if [[ $# -eq 0 ]]; then
   echo "Usage: $0 <node|docker|build|jq|gh>..." >&2
@@ -31,6 +34,16 @@ fi
 
 has() { command -v "$1" >/dev/null 2>&1; }
 
+# Both install paths call apt-get update; the sources switch to HTTPS once, before the first.
+sources_https=0
+apt_update() {
+  if [[ $sources_https -eq 0 ]]; then
+    "$BASH" "$APT_SOURCES_HTTPS"
+    sources_https=1
+  fi
+  sudo apt-get update
+}
+
 packages=()
 want_docker=0
 for component in "$@"; do
@@ -53,7 +66,7 @@ done
 
 if [[ ${#packages[@]} -gt 0 ]]; then
   echo "Installing from Ubuntu: ${packages[*]}"
-  sudo apt-get update
+  apt_update
   sudo apt-get install -y --no-install-recommends "${packages[@]}"
 fi
 
@@ -75,7 +88,7 @@ if [[ $want_docker -eq 1 ]]; then
   sudo install -D -m 0644 "$KEY" /etc/apt/keyrings/docker.asc
   echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu ${codename} stable" \
     | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
-  sudo apt-get update
+  apt_update
   sudo apt-get install -y --no-install-recommends \
     docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 

@@ -11,9 +11,13 @@ trap 'rm -rf "$WORK"' EXIT
 mkdir -p "$WORK/bin" "$WORK/base"
 
 # Only these real tools are visible, so the host's own jq/gcc/docker cannot leak in.
-for tool in awk bash cat grep mktemp rm seq; do ln -s "$(command -v "$tool")" "$WORK/base/$tool"; done
+for tool in awk bash cat cmp cp grep mktemp rm sed seq; do ln -s "$(command -v "$tool")" "$WORK/base/$tool"; done
 
 printf 'VERSION_CODENAME=jammy\n' > "$WORK/os-release"
+# apt-sources-https.sh rewrites this fixture instead of /etc/apt, unprivileged so
+# the sudo stub does not swallow the copy.
+mkdir -p "$WORK/apt"
+reset_apt() { printf 'deb http://ports.ubuntu.com/ubuntu-ports jammy main\n' > "$WORK/apt/sources.list"; }
 GOOD_FPR=9DC858229FC7DD38854AE2D88D81803C0EBFCD88
 
 cat > "$WORK/base/sudo" <<'EOF'
@@ -61,7 +65,9 @@ cat > "$WORK/present" <<'EOF'
 #!/usr/bin/env bash
 exit 0
 EOF
-chmod +x "$WORK/base/"* "$WORK/docker-full" "$WORK/docker-bare" "$WORK/present"
+# Only the stubs: $WORK/base also links to real tools, and GNU chmod follows a
+# link into a root-owned binary, so a glob fails for a non-root runner user.
+chmod +x "$WORK/base/"{sudo,ldconfig,curl,gpg,dpkg} "$WORK/docker-full" "$WORK/docker-bare" "$WORK/present"
 
 passed=0
 # run <case-name> <tools on PATH> -- <components>...
@@ -77,7 +83,8 @@ run() {
   done
   export TEST_LOG="$WORK/log"
   : > "$TEST_LOG"
-  PATH="$WORK/bin:$WORK/base" OS_RELEASE="$WORK/os-release" \
+  reset_apt
+  PATH="$WORK/bin:$WORK/base" OS_RELEASE="$WORK/os-release" APT_ROOT="$WORK/apt" APT_SUDO="" \
     "$BASH" "$SCRIPT_DIR/install-arm64-runner-deps.sh" "$@" > "$WORK/out" 2>&1
 }
 fail() { cat "$WORK/out" "$TEST_LOG" >&2; echo "FAIL: $1" >&2; exit 1; }
@@ -86,12 +93,14 @@ ok() { echo "ok - $1"; passed=$((passed + 1)); }
 export TEST_HAS_LIBATOMIC=1 TEST_KEY_PRIMARIES="$GOOD_FPR"
 run "docker jq gh make gcc g++" node docker build jq gh || fail "full image errored"
 [[ ! -s "$TEST_LOG" ]] || fail "full image still called sudo"
+grep -qx "deb http://ports.ubuntu.com/ubuntu-ports jammy main" "$WORK/apt/sources.list" || fail "full image still rewrote apt sources"
 ok "a full runner image installs nothing"
 
 export TEST_HAS_LIBATOMIC=0
 run "" node || fail "node errored"
 grep -qx "sudo apt-get install -y --no-install-recommends libatomic1" "$TEST_LOG" || fail "node did not install exactly libatomic1"
-ok "node installs libatomic1 when ldconfig lacks it"
+grep -qx "deb https://ports.ubuntu.com/ubuntu-ports jammy main" "$WORK/apt/sources.list" || fail "apt sources not switched to https before apt-get update"
+ok "node installs libatomic1 when ldconfig lacks it, after moving apt to https"
 
 run "make gcc" build jq gh || fail "build errored"
 grep -qx "sudo apt-get install -y --no-install-recommends make gcc g++ libc6-dev jq gh" "$TEST_LOG" || fail "build/jq/gh package list wrong"

@@ -7,6 +7,7 @@ set -euo pipefail
 echo "📦 Installing cross-compilation toolchains for Go + CGO..."
 
 # Install all required packages
+"$(dirname "${BASH_SOURCE[0]}")/apt-sources-https.sh"
 sudo apt-get update
 sudo apt-get install -y \
   gcc-x86-64-linux-gnu \

@@ -4,7 +4,7 @@ set -euo pipefail
 # API integrations test: compiles bifrost-http, runs it against PostgreSQL using
 # tests/config.json (with a runtime config_store/logs_store overlay), then runs
 # the api-management newman collection via tests/e2e/api/runners/run-newman-api-tests.sh.
-# The Warp, routing-wiring and model-catalog-wiring collections follow, each on a
+# The Warp (unless SKIP_WARP=1), routing-wiring and model-catalog-wiring collections follow, each on a
 # server of its own.
 
 if command -v readlink >/dev/null 2>&1 && readlink -f "$0" >/dev/null 2>&1; then
@@ -219,10 +219,13 @@ echo "🧪 Running api-management newman collection..."
 # Warp boots its own server on its own database (bifrost_warp_e2e on the same
 # Postgres), because its answers cover every row in the logs table and the
 # api-management run above has just written to this server's. It reuses the
-# compose stack's Weaviate and the job's OPENAI_API_KEY.
-echo ""
-echo "🧪 Running Warp newman collection..."
-"$REPO_ROOT/tests/e2e/api/runners/individual/run-newman-warp-tests.sh" --binary "$BIFROST_BINARY" --port "$((PORT + 8))" $REPORT_ARGS
+# compose stack's Weaviate and the job's OPENAI_API_KEY. CI sets SKIP_WARP=1
+# and runs it as the parallel test-warp job instead (test-warp.sh).
+if [ "${SKIP_WARP:-0}" != "1" ]; then
+  echo ""
+  echo "🧪 Running Warp newman collection..."
+  "$REPO_ROOT/tests/e2e/api/runners/individual/run-newman-warp-tests.sh" --binary "$BIFROST_BINARY" --port "$((PORT + 8))" $REPORT_ARGS
+fi
 
 # The wiring collections each boot a clean server (no providers, sqlite stores):
 # routing-wiring creates the global standard providers (openai, azure, bedrock,