-
-
Notifications
You must be signed in to change notification settings - Fork 2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #8239 from OpenMined/rasswanth/switch-cd-to-self-h…
…osted [WIP] Switch CD to Self hosted runners.
- Loading branch information
Showing
17 changed files
with
201 additions
and
142 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -35,9 +35,153 @@ jobs: | |
| uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@dev | ||
| secrets: inherit | ||
|
|
||
| deploy-syft: | ||
| build-and-push-docker-images: | ||
| needs: [call-pr-tests-linting, call-pr-tests-syft, call-pr-tests-stack] | ||
| if: always() && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true') | ||
|
|
||
| strategy: | ||
| matrix: | ||
| runner: [sh-arc-linux-x64, sh-arc-linux-arm64] | ||
| runs-on: ${{ matrix.runner }} | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
|
|
||
| # actions/setup-python doesn't yet support ARM | ||
| - name: Setup Python on x64 | ||
| if: ${{ !endsWith(matrix.runner, '-arm64') }} | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: "3.11" | ||
|
|
||
| - name: Install Metadata packages for arm64 | ||
| if: ${{ endsWith(matrix.runner, '-arm64') }} | ||
| run: | | ||
| sudo apt update -y | ||
| sudo apt install software-properties-common -y | ||
| - name: Setup Python on arm64 | ||
| if: ${{ endsWith(matrix.runner, '-arm64') }} | ||
| uses: deadsnakes/[email protected] | ||
| with: | ||
| python-version: "3.11" | ||
|
|
||
| - name: Check python version | ||
| run: | | ||
| python --version | ||
| python3 --version | ||
| which python | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install --upgrade bump2version | ||
| - name: Bump the Version | ||
| id: bump-version | ||
| run: | | ||
| ls **/VERSION | xargs -I {} python {} | ||
| cat packages/grid/devspace.yaml | grep '0\.' | ||
| bump2version prenum --allow-dirty --no-commit | ||
| ls **/VERSION | xargs -I {} python {} | ||
| cat packages/grid/devspace.yaml | grep '0\.' | ||
| - name: Generate Release Metadata | ||
| id: release_metadata | ||
| run: | | ||
| if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then | ||
| echo "release_tag=beta" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "release_tag=latest" >> $GITHUB_OUTPUT | ||
| fi | ||
| if [[ ${{matrix.runner}} == *"x64"* ]]; then | ||
| echo "release_platform=linux/amd64" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "release_platform=linux/arm64" >> $GITHUB_OUTPUT | ||
| fi | ||
| echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT | ||
| - name: Set up Docker Buildx | ||
| id: buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
|
|
||
| - name: Login to Docker | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKER_LOGIN }} | ||
| password: ${{ secrets.DOCKER_PASSWORD }} | ||
|
|
||
| - name: Docker meta for Grid-Backend | ||
| id: meta_grid_backend | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: openmined/grid-backend | ||
| tags: | | ||
| type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }} | ||
| type=raw,value=${{ steps.release_metadata.outputs.release_tag }} | ||
| - name: Build and push `grid-backend` image to DockerHub | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ./packages | ||
| file: ./packages/grid/backend/backend.dockerfile | ||
| push: true | ||
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | ||
| tags: ${{ steps.meta_grid_backend.outputs.tags }} | ||
| target: backend | ||
| cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} | ||
| cache-to: type=inline | ||
|
|
||
| - name: Docker meta for Grid-Frontend | ||
| id: meta_grid_frontend | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: openmined/grid-frontend | ||
| tags: | | ||
| type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }} | ||
| type=raw,value=${{ steps.release_metadata.outputs.release_tag }} | ||
| - name: Build and push `grid-frontend` image to DockerHub | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ./packages/grid/frontend | ||
| file: ./packages/grid/frontend/frontend.dockerfile | ||
| push: true | ||
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | ||
| tags: ${{ steps.meta_grid_frontend.outputs.tags }} | ||
| target: grid-ui-development | ||
| cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }} | ||
| cache-to: type=inline | ||
|
|
||
| - name: Docker meta for Grid-Enclave | ||
| id: meta_grid_enclave | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: openmined/grid-enclave | ||
| tags: | | ||
| type=raw,value=${{ steps.release_metadata.outputs.GRID_VERSION }} | ||
| type=raw,value=${{ steps.release_metadata.outputs.release_tag }} | ||
| - name: Build and push `grid-enclave` image to DockerHub | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ./packages | ||
| file: ./packages/grid/worker/worker.dockerfile | ||
| push: true | ||
| platforms: ${{ steps.release_metadata.outputs.release_platform }} | ||
| tags: ${{ steps.meta_grid_enclave.outputs.tags }} | ||
| target: worker | ||
| cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} | ||
| cache-to: type=inline | ||
|
|
||
| deploy-syft: | ||
| needs: | ||
| [ | ||
| call-pr-tests-linting, | ||
| call-pr-tests-syft, | ||
| call-pr-tests-stack, | ||
| build-and-push-docker-images, | ||
| ] | ||
| if: always() && (needs.call-pr-tests-linting.result == 'success' && needs.call-pr-tests-syft.result == 'success' && needs.call-pr-tests-stack.result == 'success' || github.event.inputs.skip_tests == 'true') && needs.build-and-push-docker-images.result == 'success' | ||
| # runs-on: [self-hosted, Linux] | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
|
|
@@ -115,11 +259,9 @@ jobs: | |
| run: | | ||
| if [[ $(python packages/grid/VERSION) == *"beta"* ]]; then | ||
| echo "is_pre_release=true" >> $GITHUB_OUTPUT | ||
| echo "release_tag=beta" >> $GITHUB_OUTPUT | ||
| echo "github_release_version=$(python packages/grid/VERSION | sed 's/-beta./b/')" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "is_pre_release=false" >> $GITHUB_OUTPUT | ||
| echo "release_tag=latest" >> $GITHUB_OUTPUT | ||
| echo "github_release_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT | ||
| fi | ||
|
|
@@ -215,105 +357,6 @@ jobs: | |
| ./packages/hagrid/hagrid/manifest_template.yml | ||
| tag_name: v${{ steps.release_checks.outputs.github_release_version }} | ||
|
|
||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
|
|
||
| - name: Set up Docker Buildx | ||
| id: buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
|
|
||
| - name: Login to Docker | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.DOCKER_LOGIN }} | ||
| password: ${{ secrets.DOCKER_PASSWORD }} | ||
|
|
||
| - name: Docker meta for Grid-Backend | ||
| id: meta_grid_backend | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: openmined/grid-backend | ||
| tags: | | ||
| type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} | ||
| type=raw,value=${{ steps.release_checks.outputs.release_tag }} | ||
| - name: Build and push `grid-backend` image to DockerHub | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ./packages | ||
| file: ./packages/grid/backend/backend.dockerfile | ||
| push: true | ||
| platforms: linux/amd64,linux/arm64 | ||
| tags: ${{ steps.meta_grid_backend.outputs.tags }} | ||
| target: backend | ||
| cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }} | ||
| cache-to: type=inline | ||
|
|
||
| - name: Docker meta for Grid-Frontend | ||
| id: meta_grid_frontend | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: openmined/grid-frontend | ||
| tags: | | ||
| type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} | ||
| type=raw,value=${{ steps.release_checks.outputs.release_tag }} | ||
| - name: Build and push `grid-frontend` image to DockerHub | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ./packages/grid/frontend | ||
| file: ./packages/grid/frontend/frontend.dockerfile | ||
| push: true | ||
| platforms: linux/amd64,linux/arm64 | ||
| tags: ${{ steps.meta_grid_frontend.outputs.tags }} | ||
| target: grid-ui-development | ||
| cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }} | ||
| cache-to: type=inline | ||
|
|
||
| # free 10GB of space | ||
| - name: Remove unnecessary files | ||
| run: | | ||
| sudo rm -rf /usr/share/dotnet | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| docker image prune --all --force | ||
| docker builder prune --all --force | ||
| docker system prune --all --force | ||
| - name: Docker meta for Grid-Enclave | ||
| id: meta_grid_enclave | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: openmined/grid-enclave | ||
| tags: | | ||
| type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }} | ||
| type=raw,value=${{ steps.release_checks.outputs.release_tag }} | ||
| - name: Build and push `grid-enclave` image to DockerHub | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ./packages | ||
| file: ./packages/grid/worker/worker.dockerfile | ||
| push: true | ||
| platforms: linux/amd64,linux/arm64 | ||
| tags: ${{ steps.meta_grid_enclave.outputs.tags }} | ||
| target: worker | ||
| cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }} | ||
| cache-to: type=inline | ||
|
|
||
| # free 10GB of space | ||
| - name: Remove unnecessary files | ||
| run: | | ||
| sudo rm -rf /usr/share/dotnet | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| docker image prune --all --force | ||
| docker builder prune --all --force | ||
| docker system prune --all --force | ||
| - name: Login to Docker-Helm | ||
| run: | | ||
| helm registry login registry-1.docker.io -u ${{ secrets.DOCKER_LOGIN }} --password ${{secrets.DOCKER_PASSWORD}} | ||
| cd packages/grid/helm/repo && helm push syft-${{ steps.grid-version.outputs.GRID_VERSION }}.tgz oci://registry-1.docker.io/openmined | ||
| # Checkout to gh-pages and update helm repo | ||
| - name: Checkout to gh-pages | ||
| uses: actions/checkout@v3 | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.