feat(rechunk): Add the ability to rechunk an image

blue-build · Dec 2, 2024 · b4fbac2 · b4fbac2
1 parent ffa1789
commit b4fbac2
Show file tree

Hide file tree

Showing 22 changed files with 996 additions and 184 deletions.
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
@@ -82,7 +82,7 @@ jobs:
           earthly --ci --push -P +prebuild
 
   build-images:
-    timeout-minutes: 30
+    timeout-minutes: 60
     runs-on: ubuntu-latest
     if: github.repository == 'blue-build/cli'
     needs:
@@ -253,9 +253,9 @@ jobs:
           BB_BUILDKIT_CACHE_GHA: true
         run: just test-docker-build
 
-  arm64-build:
-    timeout-minutes: 40
-    runs-on: ubuntu-latest
+  rechunk-build:
+    timeout-minutes: 20
+    runs-on: ubuntu-24.04
     permissions:
       contents: read
       packages: write
@@ -266,11 +266,9 @@ jobs:
         uses: ublue-os/remove-unwanted-software@v6
 
       - uses: sigstore/[email protected]
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
         with:
-          install: true
+          install-dir: /usr/bin
+          use-sudo: true
 
       - uses: actions-rust-lang/setup-rust-toolchain@v1
 
@@ -290,11 +288,14 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           GH_PR_EVENT_NUMBER: ${{ github.event.number }}
           COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }}
-          BB_BUILDKIT_CACHE_GHA: true
-        run: just test-arm64-build
+        run: |
+          just install-debug-all-features
+          cd integration-tests/test-repo
+          export CARGO_HOME=$HOME/.cargo
+          sudo -E $CARGO_HOME/bin/bluebuild build --push -vv --rechunk recipes/recipe-rechunk.yml
 
-  docker-build-external-login:
-    timeout-minutes: 20
+  arm64-build:
+    timeout-minutes: 40
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -312,13 +313,6 @@ jobs:
         with:
           install: true
 
-      - name: Docker Login
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-
       - uses: actions-rust-lang/setup-rust-toolchain@v1
 
       - uses: actions/checkout@v4
@@ -334,12 +328,13 @@ jobs:
 
       - name: Run Build
         env:
+          GH_TOKEN: ${{ github.token }}
           GH_PR_EVENT_NUMBER: ${{ github.event.number }}
           COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }}
           BB_BUILDKIT_CACHE_GHA: true
-        run: just test-docker-build-external-login
+        run: just test-arm64-build
 
-  docker-build-oauth-login:
+  docker-build-external-login:
     timeout-minutes: 20
     runs-on: ubuntu-latest
     permissions:
@@ -348,16 +343,6 @@ jobs:
       id-token: write
 
     steps:
-      - name: Google Auth
-        id: auth
-        uses: "google-github-actions/auth@v2"
-        with:
-          token_format: "access_token"
-          service_account: ${{ secrets.SERVICE_ACCOUNT }}
-          project_id: bluebuild-oidc
-          create_credentials_file: false
-          workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY }}
-
       - name: Maximize build space
         uses: ublue-os/remove-unwanted-software@v6
 
@@ -368,15 +353,14 @@ jobs:
         with:
           install: true
 
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
-
-      - name: Docker Auth
-        id: docker-auth
-        uses: "docker/login-action@v3"
+      - name: Docker Login
+        uses: docker/login-action@v3
         with:
-          username: "oauth2accesstoken"
-          password: "${{ steps.auth.outputs.access_token }}"
-          registry: us-east1-docker.pkg.dev
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
 
       - uses: actions/checkout@v4
         with:
@@ -394,7 +378,65 @@ jobs:
           GH_PR_EVENT_NUMBER: ${{ github.event.number }}
           COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }}
           BB_BUILDKIT_CACHE_GHA: true
-        run: just test-docker-build-oauth-login
+        run: just test-docker-build-external-login
+
+  # Free trial is over
+  # docker-build-oauth-login:
+  #   timeout-minutes: 20
+  #   runs-on: ubuntu-latest
+  #   permissions:
+  #     contents: read
+  #     packages: write
+  #     id-token: write
+
+  #   steps:
+  #     - name: Google Auth
+  #       id: auth
+  #       uses: "google-github-actions/auth@v2"
+  #       with:
+  #         token_format: "access_token"
+  #         service_account: ${{ secrets.SERVICE_ACCOUNT }}
+  #         project_id: bluebuild-oidc
+  #         create_credentials_file: false
+  #         workload_identity_provider: ${{ secrets.WORKLOAD_IDENTITY }}
+
+  #     - name: Maximize build space
+  #       uses: ublue-os/remove-unwanted-software@v6
+
+  #     - uses: sigstore/[email protected]
+
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v3
+  #       with:
+  #         install: true
+
+  #     - uses: actions-rust-lang/setup-rust-toolchain@v1
+
+  #     - name: Docker Auth
+  #       id: docker-auth
+  #       uses: "docker/login-action@v3"
+  #       with:
+  #         username: "oauth2accesstoken"
+  #         password: "${{ steps.auth.outputs.access_token }}"
+  #         registry: us-east1-docker.pkg.dev
+
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         fetch-depth: 0
+  #         ref: ${{github.event.pull_request.head.ref}}
+  #         repository: ${{github.event.pull_request.head.repo.full_name}}
+
+  #     - name: Expose GitHub Runtime
+  #       uses: crazy-max/ghaction-github-runtime@v3
+
+  #     - uses: extractions/setup-just@v1
+
+  #     - name: Run Build
+  #       env:
+  #         GH_PR_EVENT_NUMBER: ${{ github.event.number }}
+  #         COSIGN_PRIVATE_KEY: ${{ secrets.TEST_SIGNING_SECRET }}
+  #         BB_BUILDKIT_CACHE_GHA: true
+  #       run: just test-docker-build-oauth-login
 
   podman-build:
     timeout-minutes: 20