Merge pull request #8243 from OpenMined/rasswanth/fix-multi-arch-builds

Fix Multi Arch Builds
OpenMined · Nov 11, 2023 · 9b19791 · 9b19791
2 parents 96fa7f9 + 56a163c
commit 9b19791
Show file tree

Hide file tree

Showing 18 changed files with 147 additions and 89 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.2-beta.55
+current_version = 0.8.2-beta.56
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -43,6 +43,11 @@ jobs:
       matrix:
         runner: [sh-arc-linux-x64, sh-arc-linux-arm64]
     runs-on: ${{ matrix.runner }}
+
+    outputs:
+      release_tag: ${{ steps.release_metadata.outputs.release_tag }}
+      grid_version: ${{ steps.release_metadata.outputs.grid_version }}
+
     steps:
       - uses: actions/checkout@v3
 
@@ -98,7 +103,7 @@ jobs:
           else
             echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT
           fi
-          echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
+          echo "grid_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
 
       - name: Set up Docker Buildx
         id: buildx
@@ -110,79 +115,120 @@ jobs:
           username: ${{ secrets.DOCKER_LOGIN }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Docker meta for Grid-Backend
-        id: meta_grid_backend
-        uses: docker/metadata-action@v5
-        with:
-          images: openmined/grid-backend
-          tags: |
-            type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }}
-            type=raw,value=${{ steps.release_metadata.outputs.release_tag }}
-
       - name: Build and push `grid-backend` image to DockerHub
+        id: grid-backend-build
         uses: docker/build-push-action@v5
         with:
           context: ./packages
           file: ./packages/grid/backend/backend.dockerfile
-          push: true
           platforms: ${{ steps.release_metadata.outputs.release_platform }}
-          tags: ${{ steps.meta_grid_backend.outputs.tags }}
           target: backend
+          outputs: type=image,name=openmined/grid-backend,push-by-digest=true,name-canonical=true,push=true
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-      - name: Docker meta for Grid-Frontend
-        id: meta_grid_frontend
-        uses: docker/metadata-action@v5
-        with:
-          images: openmined/grid-frontend
-          tags: |
-            type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }}
-            type=raw,value=${{ steps.release_metadata.outputs.release_tag }}
+      - name: Export digest for grid-backend
+        run: |
+          mkdir -p /tmp/digests/grid-backend
+          digest="${{ steps.grid-backend-build.outputs.digest }}"
+          touch "/tmp/digests/grid-backend/${digest#sha256:}"
 
       - name: Build and push `grid-frontend` image to DockerHub
+        id: grid-frontend-build
         uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/frontend
           file: ./packages/grid/frontend/frontend.dockerfile
-          push: true
           platforms: ${{ steps.release_metadata.outputs.release_platform }}
-          tags: ${{ steps.meta_grid_frontend.outputs.tags }}
+          outputs: type=image,name=openmined/grid-frontend,push-by-digest=true,name-canonical=true,push=true
           target: grid-ui-development
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-      - name: Docker meta for Grid-Enclave
-        id: meta_grid_enclave
-        uses: docker/metadata-action@v5
-        with:
-          images: openmined/grid-enclave
-          tags: |
-            type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }}
-            type=raw,value=${{ steps.release_metadata.outputs.release_tag }}
+      - name: Export digest for grid-frontend
+        run: |
+          mkdir -p /tmp/digests/grid-frontend
+          digest="${{ steps.grid-frontend-build.outputs.digest }}"
+          touch "/tmp/digests/grid-frontend/${digest#sha256:}"
 
       - name: Build and push `grid-enclave` image to DockerHub
+        id: grid-enclave-build
         uses: docker/build-push-action@v5
         with:
           context: ./packages
           file: ./packages/grid/worker/worker.dockerfile
-          push: true
           platforms: ${{ steps.release_metadata.outputs.release_platform }}
-          tags: ${{ steps.meta_grid_enclave.outputs.tags }}
+          outputs: type=image,name=openmined/grid-enclave,push-by-digest=true,name-canonical=true,push=true
           target: worker
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Export digest for grid-enclave
+        run: |
+          mkdir -p /tmp/digests/grid-enclave
+          digest="${{ steps.grid-enclave-build.outputs.digest }}"
+          touch "/tmp/digests/grid-enclave/${digest#sha256:}"
+
+      - name: Upload digest for grid-backend, grid-frontend and grid-enclave
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests-${{ steps.release_metadata.outputs.grid_version }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  #Used to merge x64 and arm64 into one docker image
+  merge-docker-images:
+    needs: [build-and-push-docker-images]
+    if: always() && (needs.build-and-push-docker-images.result == 'success')
+
+    runs-on: sh-arc-linux-x64
+
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests-${{ needs.build-and-push-docker-images.outputs.grid_version }}
+          path: /tmp/digests
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_LOGIN }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Create manifest list and push for grid-backend
+        working-directory: /tmp/digests/grid-backend
+        run: |
+          docker buildx imagetools create  \
+            -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
+            -t openmined/grid-backend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
+            $(printf 'openmined/grid-backend@sha256:%s ' *)
+
+      - name: Create manifest list and push for grid-frontend
+        working-directory: /tmp/digests/grid-frontend
+        run: |
+          docker buildx imagetools create  \
+            -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
+            -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
+            $(printf 'openmined/grid-frontend@sha256:%s ' *)
+
+      - name: Create manifest list and push for grid-enclave
+        working-directory: /tmp/digests/grid-enclave
+        run: |
+          docker buildx imagetools create  \
+            -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
+            -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
+            $(printf 'openmined/grid-enclave@sha256:%s ' *)
+
   deploy-syft:
-    needs:
-      [
-        call-pr-tests-linting,
-        call-pr-tests-syft,
-        call-pr-tests-stack,
-        build-and-push-docker-images,
-      ]
-    if: always() && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true') && needs.build-and-push-docker-images.result == 'success'
-    # runs-on: [self-hosted, Linux]
+    needs: [merge-docker-images]
+    if: always() &&  needs.merge-docker-images.result == 'success'
+
     runs-on: ubuntu-latest
     permissions:
       contents: write # For tag and release notes.
@@ -305,11 +351,6 @@ jobs:
             twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
           fi
 
-      - name: Set Grid package version
-        id: grid-version
-        shell: bash
-        run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
-
       # Checkout Infra repo (nested)
       - name: Checkout Infra Repo
         uses: actions/checkout@v3

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.2-beta.55"
+__version__ = "0.8.2-beta.56"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/packages/grid/VERSION b/packages/grid/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.2-beta.55"
+__version__ = "0.8.2-beta.56"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml
@@ -15,13 +15,13 @@ pipelines:
     run: |-
       run_dependencies --all
       ensure_pull_secrets --all
-      build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.2-beta.55 -t dev-latest
+      build_images --all -t $(git rev-parse --short=6 HEAD) -t 0.8.2-beta.56 -t dev-latest
       create_deployments --all
 
 vars:
   DEVSPACE_ENV_FILE: "default.env"
   CONTAINER_REGISTRY: "docker.io"
-  VERSION: "0.8.2-beta.55"
+  VERSION: "0.8.2-beta.56"
 
 # This is a list of `images` that DevSpace can build for this project
 # We recommend to skip image building during development (devspace dev) as much as possible

diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "pygrid-ui",
-  "version": "0.8.2-beta.55",
+  "version": "0.8.2-beta.56",
   "private": true,
   "scripts": {
     "dev": "pnpm i && vite dev --host --port 80",