chore: reproducible build

.github/workflows/build.yml

@@ -0,0 +1,87 @@
+name: Build & Deploy
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - chore/reproducible-build
+    tags:
+      - 'v*'
+
+env:
+  REGISTRY: docker.io
+  IMAGE_REPOSITORY: ${{ vars.DOCKER_REGISTRY_USER }}/${{ github.event.repository.name }}
+
+jobs:
+  reproducible-docker-image:
+    name: Reproducible Docker Image
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ vars.DOCKER_REGISTRY_USER }}
+          password: ${{ secrets.DOCKER_REGISTRY_TOKEN }}
+
+      - name: Extract image tag from GitHub ref (branch or tag)
+        run: |
+          if [[ "${GITHUB_REF_TYPE}" == 'tag' ]]; then
+            TAG=${GITHUB_REF_NAME#v}
+            echo "Using '${TAG}' image tag for ${GITHUB_REF_NAME} tag"
+          elif [[ "${GITHUB_REF_TYPE}" == 'branch' ]]; then
+            TAG=$(if [[ "${GITHUB_REF_NAME}" == 'main' ]]; then echo 'latest'; else echo 'dev'; fi)
+            echo "Using '${TAG}' image tag for ${GITHUB_REF_NAME} branch"
+          else
+            echo "Unsupported ref type: ${GITHUB_REF_TYPE}" >&2
+            exit 1
+          fi
+
+          if [ -z "${TAG}" ]; then
+            echo "Unable to parse image tag from ${GITHUB_REF_TYPE}: ${GITHUB_REF_NAME}" >&2
+            exit 1
+          fi
+          echo "IMAGE_REFERENCE=${{ env.REGISTRY }}/${{ env.IMAGE_REPOSITORY }}:${TAG}" >> "$GITHUB_ENV"
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y skopeo jq
+
+      - name: Build and push reproducible image
+        run: |
+          ./build-image.sh --push "${{ env.IMAGE_REFERENCE }}"
+
+      - name: Get image digest
+        run: |
+          DIGEST=$(skopeo inspect oci-archive:./oci.tar | jq -r '.Digest')
+          if [ -z "${DIGEST}" ]; then
+            echo "Failed to get image digest from OCI archive" >&2
+            exit 1
+          fi
+          echo "IMAGE_DIGEST=${DIGEST}" >> "$GITHUB_ENV"
+
+      - name: Generate artifact attestation
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_REPOSITORY }}
+          subject-digest: ${{ env.IMAGE_DIGEST }}
+          push-to-registry: true
+
+      - name: Generate build summary
+        run: |
+          {
+            echo "## ${{ env.IMAGE_REPOSITORY }} docker image"
+            echo ""
+            echo "- tag: \`${{ env.IMAGE_REFERENCE }}\`"
+            echo "- digest: \`${{ env.IMAGE_DIGEST }}\`"
+            echo "- sigstore: https://search.sigstore.dev/?hash=${{ env.IMAGE_DIGEST }}"
+          } >> "$GITHUB_STEP_SUMMARY"

Dockerfile

@@ -1,68 +1,113 @@
-FROM rust:1.86-alpine AS rust-builder
+FROM rust:1.86-alpine@sha256:661d708cc863ce32007cf46807a72062a80d2944a6fae9e0d83742d2e04d5375 AS rust-builder
 RUN apk add --no-cache musl-dev
 RUN rustup target add x86_64-unknown-linux-musl
 WORKDIR /build
 COPY service-mesh/ /build/service-mesh/
 WORKDIR /build/service-mesh
 RUN cargo build --target x86_64-unknown-linux-musl
 
-FROM golang:1.23-alpine AS go-builder
+
+FROM golang:1.23-alpine@sha256:383395b794dffa5b53012a212365d40c8e37109a626ca30d6151c8348d380b5f AS go-builder
 WORKDIR /build
 COPY vpc-api-server/ /build/
 RUN go mod init vpc-api-server || true
 RUN go get github.com/gin-gonic/gin
 RUN CGO_ENABLED=0 GOOS=linux go build -a -o vpc-api-server main.go
 
-FROM alpine AS ko-builder
+
+FROM alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 AS ko-builder
 RUN apk add --no-cache wget jq bash squashfs-tools
 WORKDIR /build
 COPY ./extract-modules.sh /build/
 RUN ./extract-modules.sh
 
-FROM ubuntu:24.04
-RUN apt-get update && apt-get install -y \
-    ca-certificates \
-    wget \
-    curl \
-    jq \
-    nginx \
-    supervisor \
-    gettext-base \
-    socat \
-    kmod \
-    etcd-server \
-    etcd-client
 
-RUN curl -fsSL https://get.docker.com | sh
-RUN usermod -aG docker root
+FROM debian:bookworm-slim@sha256:78d2f66e0fec9e5a39fb2c72ea5e052b548df75602b5215ed01a17171529f706 AS runtime
+
+# Bootstrap by installing ca-certificates which will be overridden by the pinned packages.
+# Otherwise the source list cannot be fetched from the debian snapshot.
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache
 
+# Install Docker before package pinning (Docker installation conflicts with pinned packages)
+RUN curl -fsSL https://get.docker.com | sh && \
+    usermod -aG docker root
+
+# Install pinned apt dependencies
+RUN --mount=type=bind,source=pinned-packages.txt,target=/tmp/pinned-packages.txt,ro \
+    set -e; \
+    # Create a sources.list file pointing to a specific snapshot
+    echo 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian/20250411T024939Z bookworm main' > /etc/apt/sources.list && \
+    echo 'deb [check-valid-until=no] https://snapshot.debian.org/archive/debian-security/20250411T024939Z bookworm-security main' >> /etc/apt/sources.list && \
+    echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/10no-check-valid-until && \
+    # Create preferences file to pin all packages
+    rm -rf /etc/apt/sources.list.d/* && \
+    mkdir -p /etc/apt/preferences.d && \
+    cat /tmp/pinned-packages.txt | while read line; do \
+        pkg=$(echo $line | cut -d= -f1); \
+        ver=$(echo $line | cut -d= -f2); \
+        if [ ! -z "$pkg" ] && [ ! -z "$ver" ]; then \
+            printf "Package: %s\nPin: version %s\nPin-Priority: 1001\n\n" "$pkg" "$ver" >> /etc/apt/preferences.d/pinned-packages; \
+        fi; \
+    done && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        ca-certificates \
+        wget \
+        curl \
+        jq \
+        nginx \
+        supervisor \
+        gettext-base \
+        socat \
+        kmod \
+        etcd-server \
+        etcd-client \
+        && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache
 
 RUN mkdir -p /var/run/dstack \
     /etc/dstack \
     /etc/ssl/certs \
     /etc/ssl/private \
     /var/log/supervisor \
     /var/log/nginx \
-    /scripts \
     /lib/extra-modules \
     /var/lib/etcd \
     /etc/etcd
 
-COPY --from=rust-builder /build/service-mesh/target/x86_64-unknown-linux-musl/debug/dstack-mesh /usr/local/bin/dstack-mesh
-RUN chmod +x /usr/local/bin/dstack-mesh
+# Copy binaries (executable)
+COPY --chmod=755 --from=rust-builder /build/service-mesh/target/x86_64-unknown-linux-musl/debug/dstack-mesh /usr/local/bin/dstack-mesh
+COPY --chmod=755 --from=go-builder /build/vpc-api-server /usr/local/bin/vpc-api-server
+COPY --chmod=644 --from=ko-builder /build/netfilter-modules/*.ko /lib/extra-modules/
+
+# Copy configs (read-only)
+COPY --chmod=644 configs/nginx.conf /etc/nginx/nginx.conf
+COPY --chmod=644 configs/nginx-client-proxy.conf /etc/nginx/conf.d/client-proxy.conf
+COPY --chmod=644 configs/nginx-server-proxy.conf.template /etc/nginx/templates/server-proxy.conf.template
+COPY --chmod=644 configs/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY --chmod=644 configs/headscale_config.yaml /etc/headscale/config.yaml
 
-COPY --from=go-builder /build/vpc-api-server /usr/local/bin/vpc-api-server
-RUN chmod +x /usr/local/bin/vpc-api-server
+# Copy scripts (executable)
+COPY --chmod=755 scripts/ /scripts/
 
-COPY --from=ko-builder /build/netfilter-modules/*.ko /lib/extra-modules/
+COPY --chmod=644 .GIT_REV /etc/
 
-COPY configs/nginx.conf /etc/nginx/nginx.conf
-COPY configs/nginx-client-proxy.conf /etc/nginx/conf.d/client-proxy.conf
-COPY configs/nginx-server-proxy.conf.template /etc/nginx/templates/server-proxy.conf.template
-COPY configs/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-COPY configs/headscale_config.yaml /etc/headscale/config.yaml
-COPY scripts /scripts
-RUN chmod +x /scripts/*.sh
+# Remove all non-deterministic files
+RUN rm -rf \
+    /var/log/dpkg.log \
+    /var/log/apt/*.log \
+    /var/log/alternatives.log \
+    /var/cache/ldconfig/aux-cache \
+    /etc/machine-id \
+    /var/lib/dbus/machine-id \
+    /var/log/*.log \
+    /tmp/* \
+    /var/tmp/* && \
+    # Create empty machine-id files for deterministic builds
+    touch /etc/machine-id /var/lib/dbus/machine-id && \
+    # Ensure log directories exist but are empty
+    mkdir -p /var/log/apt /var/log/supervisor /var/log/nginx
 
 EXPOSE 80 443 8091 8092 2379 2380
 

build-image.sh

@@ -1,35 +1,85 @@
 #!/bin/bash
 
-IMAGE_NAME="nearaidev/dstack-service"
-PUSH_IMAGE=false
+# Parse command line arguments
+PUSH=false
+REPO=""
 
 while [[ $# -gt 0 ]]; do
-    case "$1" in
+    case $1 in
         --push)
-            PUSH_IMAGE=true
-            shift
-            ;;
-        -t)
-            IMAGE_NAME="$2"
+            PUSH=true
+            REPO="$2"
+            if [ -z "$REPO" ]; then
+                echo "Error: --push requires a repository argument"
+                echo "Usage: $0 [--push <repo>[:<tag>]]"
+                exit 1
+            fi
             shift 2
             ;;
         *)
-            echo "Unknown argument: $1"
+            echo "Usage: $0 [--push <repo>[:<tag>]]"
             exit 1
             ;;
     esac
 done
 
-THIS_DIR=$(cd "$(dirname "$0")" && pwd)
+require_command() {
+    local cmd="$1"
+    if ! command -v "$cmd" >/dev/null 2>&1; then
+        echo "Error: required command '$cmd' not found in PATH" >&2
+        exit 1
+    fi
+}
+
+for required in docker skopeo jq git; do
+    require_command "$required"
+done
+
+# Check if buildkit_20 already exists before creating it
+if ! docker buildx inspect buildkit_20 &>/dev/null; then
+    docker buildx create --use --driver-opt image=moby/buildkit:v0.20.2 --name buildkit_20
+fi
+touch pinned-packages.txt
+git rev-parse HEAD > .GIT_REV
+TEMP_TAG="dstack-vpc-temp:$(date +%s)"
+docker buildx build --builder buildkit_20 --no-cache --build-arg SOURCE_DATE_EPOCH="0" \
+    --output type=oci,dest=./oci.tar,rewrite-timestamp=true \
+    --output type=docker,name="$TEMP_TAG",rewrite-timestamp=true .
 
-docker build "$THIS_DIR" -t "$IMAGE_NAME"
+if [ "$?" -ne 0 ]; then
+    echo "Build failed"
+    rm .GIT_REV
+    exit 1
+fi
 
-if [ "$PUSH_IMAGE" = true ]; then
-    echo "Pushing image to Docker Hub..."
-    docker push "$IMAGE_NAME"
-    echo "Image pushed successfully!"
+echo "Build completed, manifest digest:"
+echo ""
+skopeo inspect oci-archive:./oci.tar | jq .Digest
+echo ""
+
+if [ "$PUSH" = true ]; then
+    echo "Pushing image to $REPO..."
+    skopeo copy --insecure-policy oci-archive:./oci.tar docker://"$REPO"
+    echo "Image pushed successfully to $REPO"
 else
-    echo "Image built locally. To push to Docker Hub, use:"
-    echo "  docker push $IMAGE_NAME"
-    echo "Or run this script with --push flag"
+    echo "To push the image to a registry, run:"
+    echo ""
+    echo " $0 --push <repo>[:<tag>]"
+    echo ""
+    echo "Or use skopeo directly:"
+    echo ""
+    echo " skopeo copy --insecure-policy oci-archive:./oci.tar docker://<repo>[:<tag>]"
+    echo ""
 fi
+echo ""
+
+# Extract package information from the built image
+echo "Extracting package information from built image: $TEMP_TAG"
+docker run --rm --entrypoint bash "$TEMP_TAG" -c "dpkg -l | grep '^ii' | awk '{print \$2\"=\"\$3}' | sort" > pinned-packages.txt
+
+echo "Package information extracted to pinned-packages.txt ($(wc -l < pinned-packages.txt) packages)"
+
+# Clean up the temporary image from Docker daemon
+docker rmi "$TEMP_TAG" 2>/dev/null || true
+
+rm .GIT_REV