CI enhancement: docker workflow speedup (#2232)

* bump actions

show sccache stats

update cache from/to config

try to optimize dockerfile

fix dockerfile

fix chef command

try to use sccache directory of the gh runner

enable debug logs for sccache

use build context to mount runner sccache to container sccache

fix typo

update sccache dir

update target destination

test

test

test

test

test

test

debug

revert dockerfile changes

try to cache manually

change cache dir

Trigger pipeline: test caching

remove sccache dir

Trigger pipeline: test caching

remove uneeded args

remove mozilla-actions/sccache-action, it doesn't work anyway

Trigger pipeline: test caching docker layers manually

set a better cache invalidation policy

use cargo.lock for cache key better than toml files

Trigger pipeline: local cache should work

fix growing cache size issue

Trigger pipeline: test after fixing growing cache size

try to inject sccache into docker builds

change key for docker caching

debug sccache

add sccache just command

debug sccache env vars

update rust wrapper env var

remove sccache debug logs

Trigger pipeline: test latest state

include dev-tools dockerfile in creating cache key

use cache map

Trigger pipeline: test latest state

try gha cache after letting gha set env vars automatically

Trigger pipeline: test gha cache

clean up after gha worked and try warp build cache

Trigger pipeline: test gha warpbuild cache

* debug sccache

* remove cache mount from dockerfile

* clean up commented actions

* restore sccache mounting

* bump mozilla-actions/sccache-action

* comply to the convention of dockerfile ending with a newline

* remove excessive build arg

Author: aradwann

.github/workflows/binaries.yml

@@ -76,7 +76,7 @@ jobs:
         run: rustup target add aarch64-apple-darwin
 
       - name: Run sccache-cache
-        uses: mozilla-actions/[email protected].4
+        uses: mozilla-actions/[email protected].6
 
       - name: Cache Rust
         uses: Swatinem/rust-cache@v2
@@ -194,7 +194,7 @@ jobs:
           EOF
 
       - name: Build and push multiplatform image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true

.github/workflows/docker.yml

@@ -67,10 +67,22 @@ jobs:
           # https://docs.warpbuild.com/cache/docker-layer-caching#step-1-set-up-docker-buildx-action
           driver-opts: |
             network=host
+      
+      - name: Cache sccache
+        uses: actions/cache@v4
+        with:
+          path: sccache-cache
+          key: ${{ runner.os }}-sccache-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: Run sccache-cache
-        uses: mozilla-actions/[email protected]
-
+      - name: Inject sccache-cache into Docker
+        uses: reproducible-containers/buildkit-cache-dance@v3
+        with:
+          cache-map: |
+            {
+              "sccache-cache": "/var/cache/sccache"
+            }
+          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
+          
       - name: Log into GitHub container registry
         uses: docker/login-action@v3
         with:
@@ -107,7 +119,7 @@ jobs:
 
       - name: Build${{inputs.uploadImageAsTarball != true && ' and push ' || ' '}}Docker image
         id: build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: "docker/Dockerfile"
@@ -119,13 +131,10 @@ jobs:
           network: host
           build-args: |
             CARGO_PROFILE_RELEASE_DEBUG=${{ inputs.debug }}
-            ACTIONS_CACHE_URL=http://127.0.0.1:49160/
-            ACTIONS_RUNTIME_TOKEN
-            SCCACHE_GHA_ENABLED=true
             RESTATE_FEATURES=${{ inputs.features || '' }}
-          cache-from: type=gha,scope=${{ github.workflow }},url=http://127.0.0.1:49160/
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }},url=http://127.0.0.1:49160/
-
+          cache-from: type=gha,url=http://127.0.0.1:49160/
+          cache-to: type=gha,url=http://127.0.0.1:49160/,mode=max
+    
       - name: Save docker image as tar
         if: ${{ inputs.uploadImageAsTarball }}
         run: |
@@ -144,7 +153,7 @@ jobs:
       # This won't actually build again, it'll just use cache
       - name: Push Docker image
         if: ${{ inputs.uploadImageAsTarball && github.ref == 'refs/heads/main' }}
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: "docker/Dockerfile"
@@ -157,9 +166,7 @@ jobs:
           network: host
           build-args: |
             CARGO_PROFILE_RELEASE_DEBUG=${{ inputs.debug }}
-            ACTIONS_CACHE_URL=http://127.0.0.1:49160/
-            ACTIONS_RUNTIME_TOKEN
-            SCCACHE_GHA_ENABLED=true
             RESTATE_FEATURES=${{ inputs.features || '' }}
-          cache-from: type=gha,scope=${{ github.workflow }},url=http://127.0.0.1:49160/
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }},url=http://127.0.0.1:49160/
+          cache-from: type=gha,url=http://127.0.0.1:49160/
+          cache-to: type=gha,url=http://127.0.0.1:49160/,mode=max
+    

docker/Dockerfile

@@ -26,19 +26,24 @@ ARG SCCACHE_SERVER_PORT=4227
 FROM base-$TARGETARCH AS builder
 ARG SCCACHE_SERVER_PORT
 ARG TARGETARCH
-# https://github.com/mozilla/sccache/blob/main/docs/GHA.md
-ARG ACTIONS_CACHE_URL=''
-ARG ACTIONS_RUNTIME_TOKEN=''
-ARG SCCACHE_GHA_ENABLED=''
+
+ENV RUSTC_WRAPPER=/usr/bin/sccache
+ENV SCCACHE_DIR=/var/cache/sccache
+
 # Overrides the behaviour of the release profile re including debug symbols, which in our repo is not to include them.
 # Should be set to 'false' or 'true'. See https://doc.rust-lang.org/cargo/reference/environment-variables.html
 ARG CARGO_PROFILE_RELEASE_DEBUG=false
 ARG RESTATE_FEATURES=''
-# Caching layer if nothing has changed
-# Only build restate binary to avoid compiling unneeded crates
+
 RUN just arch=$TARGETARCH libc=gnu features=$RESTATE_FEATURES chef-cook --release --bin restate-server
+
 COPY . .
-RUN just arch=$TARGETARCH libc=gnu features=$RESTATE_FEATURES build --release --bin restate-server && \
+
+# Mount the sccache directory as a cache to leverage sccache during build
+# Caching layer if nothing has changed
+# Use sccache during the main build
+RUN --mount=type=cache,target=/var/cache/sccache \
+    just arch=$TARGETARCH libc=gnu features=$RESTATE_FEATURES build --release --bin restate-server && \
     just notice-file && \
     mv target/$(just arch=$TARGETARCH libc=gnu print-target)/release/restate-server target/restate-server