Skip to content

Commit

Permalink
Merge pull request #8213 from OpenMined/rasswanth/revert_cd
Browse files Browse the repository at this point in the history 
revert cd syft to original state
  • Loading branch information
@rasswanth-s
rasswanth-s authored Nov 1, 2023
2 parents 19ce9cb + 0fb5330 commit 8841973
Show file tree
Hide file tree
Showing 2 changed files with 187 additions and 169 deletions.
354 changes: 186 additions & 168 deletions .github/workflows/cd-syft.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,19 @@ jobs:
python -m pip install --upgrade pip
pip install --upgrade tox setuptools wheel twine bump2version PyYAML
# - name: Bump the Version
# run: |
# ls **/VERSION | xargs -I {} python {}
# cat packages/grid/devspace.yaml | grep '0\.'
# bump2version prenum --allow-dirty --no-commit
# tox -e lint || true
# ls **/VERSION | xargs -I {} python {}
# cat packages/grid/devspace.yaml | grep '0\.'
# python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION)

# - name: Check Protocol Version
# run: |
# tox -e syft.protocol.check
- name: Bump the Version
run: |
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
bump2version prenum --allow-dirty --no-commit
tox -e lint || true
ls **/VERSION | xargs -I {} python {}
cat packages/grid/devspace.yaml | grep '0\.'
python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION)
- name: Check Protocol Version
run: |
tox -e syft.protocol.check
- name: Build Helm Chart
shell: bash
Expand All @@ -98,39 +98,39 @@ jobs:
tox -e syft.build.helm
tox -e syft.package.helm
# - name: Check and Bump Protocol Version
# run: |
# export BUMP=True
# tox -e syft.protocol.check

# - name: Commit changes to Syft
# uses: EndBug/add-and-commit@v9
# with:
# author_name: ${{ secrets.OM_BOT_NAME }}
# author_email: ${{ secrets.OM_BOT_EMAIL }}
# message: "[syft]bump version"
# add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']"

# - name: Scheduled Build and Publish
# if: github.event_name == 'schedule'
# run: |
# tox -e syft.publish
# twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*

# - name: Manual Build and Publish
# if: github.event_name != 'schedule'
# run: |
# tox -e syft.publish
# if [ "${{ github.event.inputs.release_platform }}" = "REAL_PYPI" ]; then
# twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
# fi
# if [ "${{ github.event.inputs.release_platform }}" = "TEST_PYPI" ]; then
# twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
# fi
# if [ "${{ github.event.inputs.release_platform }}" = "REAL_AND_TEST_PYPI" ]; then
# twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
# twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
# fi
- name: Check and Bump Protocol Version
run: |
export BUMP=True
tox -e syft.protocol.check
- name: Commit changes to Syft
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "[syft]bump version"
add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' , 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json']"

- name: Scheduled Build and Publish
if: github.event_name == 'schedule'
run: |
tox -e syft.publish
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
- name: Manual Build and Publish
if: github.event_name != 'schedule'
run: |
tox -e syft.publish
if [ "${{ github.event.inputs.release_platform }}" = "REAL_PYPI" ]; then
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
fi
if [ "${{ github.event.inputs.release_platform }}" = "TEST_PYPI" ]; then
twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
fi
if [ "${{ github.event.inputs.release_platform }}" = "REAL_AND_TEST_PYPI" ]; then
twine upload -u __token__ -p ${{ secrets.OM_SYFT_PYPI_TOKEN }} packages/syft/dist/*
twine upload -r testpypi -u __token__ -p ${{ secrets.OM_SYFT_TEST_PYPI_TOKEN }} packages/syft/dist/*
fi
- name: Set Grid package version
id: grid-version
Expand All @@ -149,128 +149,146 @@ jobs:
echo "github_release_version=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
fi
# # Checkout Infra repo (nested)
# - name: Checkout Infra Repo
# uses: actions/checkout@v3
# with:
# repository: ${{ secrets.INFRA_REPO }}
# ref: "main"
# token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
# path: infrastructure

# # This step will copy the generated K8s manifest files to the correct directory in Infra repo
# - name: Copy files to Infra Repo
# run: |
# if $is_pre_release; then
# rm -rf infrastructure/gitops/environments/pre-release/.
# cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/
# else
# rm -rf infrastructure/gitops/environments/stable/.
# cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/
# fi

# - name: Commit changes to Infra Repo
# uses: EndBug/add-and-commit@v9
# with:
# author_name: ${{ secrets.OM_BOT_NAME }}
# author_email: ${{ secrets.OM_BOT_EMAIL }}
# message: "Update K8s Manifests from Syft Repo"
# add: "."
# push: "origin main"
# cwd: "./infrastructure/"

# - name: Create SyftCLI Config assets
# run: |
# pip install pyyaml
# python scripts/create_syftcli_config.py

# - name: GitHub Release
# uses: softprops/action-gh-release@v1
# with:
# name: v${{ steps.release_checks.outputs.github_release_version }}
# generate_release_notes: true
# prerelease: ${{ steps.release_checks.outputs.is_pre_release }}
# files: |
# ./packages/syftcli/manifest.yml
# ./build/syftcli-config/*
# ./packages/hagrid/hagrid/manifest_template.yml
# tag_name: v${{ steps.release_checks.outputs.github_release_version }}

# - name: Set up QEMU
# uses: docker/setup-qemu-action@v2

# - name: Set up Docker Buildx
# id: buildx
# uses: docker/setup-buildx-action@v2

# - name: Login to Docker
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKER_LOGIN }}
# password: ${{ secrets.DOCKER_PASSWORD }}

# - name: Docker meta for Grid-Backend
# id: meta_grid_backend
# uses: docker/metadata-action@v3
# with:
# images: openmined/grid-backend
# tags: |
# type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
# type=raw,value=beta

# - name: Build and push `grid-backend` image to DockerHub
# uses: docker/build-push-action@v4
# with:
# context: ./packages
# file: ./packages/grid/backend/backend.dockerfile
# push: true
# platforms: linux/amd64,linux/arm64
# tags: ${{ steps.meta_grid_backend.outputs.tags }}
# target: backend
# cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }}
# cache-to: type=inline

# - name: Docker meta for Grid-Frontend
# id: meta_grid_frontend
# uses: docker/metadata-action@v3
# with:
# images: openmined/grid-frontend
# tags: |
# type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
# type=raw,value=beta

# - name: Build and push `grid-frontend` image to DockerHub
# uses: docker/build-push-action@v4
# with:
# context: ./packages/grid/frontend
# file: ./packages/grid/frontend/frontend.dockerfile
# push: true
# platforms: linux/amd64,linux/arm64
# tags: ${{ steps.meta_grid_frontend.outputs.tags }}
# target: grid-ui-development
# cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }}
# cache-to: type=inline

# - name: Docker meta for Grid-Enclave
# id: meta_grid_enclave
# uses: docker/metadata-action@v3
# with:
# images: openmined/grid-enclave
# tags: |
# type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
# type=raw,value=beta

# - name: Build and push `grid-enclave` image to DockerHub
# uses: docker/build-push-action@v4
# with:
# context: ./packages
# file: ./packages/grid/worker/worker.dockerfile
# push: true
# platforms: linux/amd64,linux/arm64
# tags: ${{ steps.meta_grid_enclave.outputs.tags }}
# target: worker
# cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }}
# cache-to: type=inline
# Checkout Infra repo (nested)
- name: Checkout Infra Repo
uses: actions/checkout@v3
with:
repository: ${{ secrets.INFRA_REPO }}
ref: "main"
token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
path: infrastructure

# This step will copy the generated K8s manifest files to the correct directory in Infra repo
- name: Copy files to Infra Repo
run: |
if $is_pre_release; then
rm -rf infrastructure/gitops/environments/pre-release/.
cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/pre_release/
else
rm -rf infrastructure/gitops/environments/stable/.
cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/stable/
fi
- name: Commit changes to Infra Repo
uses: EndBug/add-and-commit@v9
with:
author_name: ${{ secrets.OM_BOT_NAME }}
author_email: ${{ secrets.OM_BOT_EMAIL }}
message: "Update K8s Manifests from Syft Repo"
add: "."
push: "origin main"
cwd: "./infrastructure/"

- name: Create SyftCLI Config assets
run: |
pip install pyyaml
python scripts/create_syftcli_config.py
- name: GitHub Release
uses: softprops/action-gh-release@v1
with:
name: v${{ steps.release_checks.outputs.github_release_version }}
generate_release_notes: true
prerelease: ${{ steps.release_checks.outputs.is_pre_release }}
files: |
./packages/syftcli/manifest.yml
./build/syftcli-config/*
./packages/hagrid/hagrid/manifest_template.yml
tag_name: v${{ steps.release_checks.outputs.github_release_version }}

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2

- name: Login to Docker
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_LOGIN }}
password: ${{ secrets.DOCKER_PASSWORD }}

- name: Docker meta for Grid-Backend
id: meta_grid_backend
uses: docker/metadata-action@v3
with:
images: openmined/grid-backend
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-backend` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages
file: ./packages/grid/backend/backend.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_backend.outputs.tags }}
target: backend
cache-from: type=registry,ref=${{ steps.meta_grid_backend.outputs.tags }}
cache-to: type=inline

- name: Docker meta for Grid-Frontend
id: meta_grid_frontend
uses: docker/metadata-action@v3
with:
images: openmined/grid-frontend
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-frontend` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages/grid/frontend
file: ./packages/grid/frontend/frontend.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_frontend.outputs.tags }}
target: grid-ui-development
cache-from: type=registry,ref= ${{ steps.meta_grid_frontend.outputs.tags }}
cache-to: type=inline

# free 10GB of space
- name: Remove unnecessary files
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
docker image prune --all --force
docker builder prune --all --force
docker system prune --all --force
- name: Docker meta for Grid-Enclave
id: meta_grid_enclave
uses: docker/metadata-action@v3
with:
images: openmined/grid-enclave
tags: |
type=raw,value=${{ steps.grid-version.outputs.GRID_VERSION }}
type=raw,value=beta
- name: Build and push `grid-enclave` image to DockerHub
uses: docker/build-push-action@v4
with:
context: ./packages
file: ./packages/grid/worker/worker.dockerfile
push: true
platforms: linux/amd64,linux/arm64
tags: ${{ steps.meta_grid_enclave.outputs.tags }}
target: worker
cache-from: type=registry,ref=${{ steps.meta_grid_enclave.outputs.tags }}
cache-to: type=inline

# free 10GB of space
- name: Remove unnecessary files
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
docker image prune --all --force
docker builder prune --all --force
docker system prune --all --force
- name: Login to Docker-Helm
run: |
Expand Down
2 changes: 1 addition & 1 deletion packages/grid/helm/helm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def get_protocol_changes() -> str:

protocol_changes = ""
if protocol_path.exists():
dev_protocol_changes = json.loads(protocol_path.read_text())["dev"]
dev_protocol_changes = json.loads(protocol_path.read_text()).get("dev", {})
protocol_changes = json.dumps(
dev_protocol_changes.get("object_versions", {}), indent=4
)
Expand Down

0 comments on commit 8841973

Please sign in to comment.