merge

OpenMined · Mar 18, 2024 · 5ad3368 · 5ad3368
2 parents c51e144 + 2933921
commit 5ad3368
Show file tree

Hide file tree

Showing 378 changed files with 13,453 additions and 6,820 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.5-beta.1
+current_version = 0.8.5-beta.9
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/cd-docs.yml b/.github/workflows/cd-docs.yml
@@ -16,7 +16,7 @@ jobs:
   cd-docs:
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -27,7 +27,8 @@ jobs:
 
       - name: Install tox
         run: |
-          pip install -U tox
+          pip install --upgrade pip uv==0.1.18 tox tox-uv==1.5.1
+          uv --version
 
       - name: Build the docs
         run: |

diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml
@@ -67,7 +67,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install dependencies
         if: ${{needs.hagrid-deploy.outputs.current_hash}} != ${{needs.hagrid-deploy.outputs.previous_hash}}

diff --git a/packages/grid/veilid/veilid.py → .github/workflows/cd-post-release-tests.yml b/packages/grid/veilid/veilid.py → .github/workflows/cd-post-release-tests.yml
diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml
@@ -64,7 +64,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Set up Docker Buildx
         id: buildx
@@ -121,6 +121,17 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:dev-${{ github.sha }}
             ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }}
 
+      - name: Build and push `grid-veilid` image to registry
+        uses: docker/build-push-action@v5
+        with:
+          context: ./packages/grid/veilid
+          file: ./packages/grid/veilid/veilid.dockerfile
+          push: true
+          tags: |
+            ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev
+            ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-veilid:${{ steps.grid.outputs.GRID_VERSION }}
+
       - name: Build Helm Chart & Copy to infra
         if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         shell: bash

diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -40,7 +40,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       # The steps ensure that the cron job is able to run only for
       # for beta releases and not for stable releases
@@ -100,7 +100,7 @@ jobs:
         if: ${{ !endsWith(matrix.runner, '-arm64') }}
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       # Currently psutil package requires gcc to be installed on arm
       # for building psutil from source
@@ -118,7 +118,12 @@ jobs:
         if: ${{ endsWith(matrix.runner, '-arm64') }}
         uses: deadsnakes/[email protected]
         with:
-          python-version: "3.11"
+          python-version: "3.12"
+
+      - name: Install Git
+        run: |
+          sudo apt-get update
+          sudo apt-get install git -y
 
       - name: Check python version
         run: |
@@ -128,8 +133,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade  bump2version tox
+          pip install --upgrade pip uv==0.1.18 bump2version tox tox-uv==1.5.1
+          uv --version
 
       - name: Get Release tag
         id: get_release_tag
@@ -246,7 +251,24 @@ jobs:
           digest="${{ steps.grid-seaweedfs-build.outputs.digest }}"
           touch "/tmp/digests/grid-seaweedfs/${digest#sha256:}"
 
-      - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs
+      - name: Build and push `grid-veilid` image to DockerHub
+        id: grid-veilid-build
+        uses: docker/build-push-action@v5
+        with:
+          context: ./packages/grid/veilid
+          file: ./packages/grid/veilid/veilid.dockerfile
+          platforms: ${{ steps.release_metadata.outputs.release_platform }}
+          outputs: type=image,name=openmined/grid-veilid,push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform }}
+          cache-to: type=registry,ref=openmined/grid-veilid:cache-${{ steps.release_metadata.outputs.short_release_platform}},mode=max
+
+      - name: Export digest for grid-veilid
+        run: |
+          mkdir -p /tmp/digests/grid-veilid
+          digest="${{ steps.grid-veilid-build.outputs.digest }}"
+          touch "/tmp/digests/grid-veilid/${digest#sha256:}"
+
+      - name: Upload digest for grid-backend, grid-frontend and grid-seaweedfs, grid-veilid
         uses: actions/upload-artifact@v4
         with:
           name: digests-${{ steps.release_metadata.outputs.grid_version }}-${{ steps.release_metadata.outputs.short_release_platform }}
@@ -305,6 +327,14 @@ jobs:
             -t openmined/grid-seaweedfs:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
             $(printf 'openmined/grid-seaweedfs@sha256:%s ' *)
 
+      - name: Create manifest list and push for grid-veilid
+        working-directory: /tmp/digests/grid-veilid
+        run: |
+          docker buildx imagetools create  \
+            -t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
+            -t openmined/grid-veilid:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
+            $(printf 'openmined/grid-veilid@sha256:%s ' *)
+
   deploy-syft:
     needs: [merge-docker-images]
     if: always() &&  needs.merge-docker-images.result == 'success'
@@ -337,11 +367,11 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install --upgrade tox setuptools wheel twine bump2version PyYAML
+          pip install --upgrade pip uv==0.1.18 tox tox-uv==1.5.1 setuptools wheel twine bump2version PyYAML
+          uv --version
 
       - name: Bump the Version
         if: needs.merge-docker-images.outputs.release_tag == 'beta'
@@ -486,7 +516,7 @@ jobs:
 
       - name: GitHub Release
         if: github.event.inputs.release_platform  != 'TEST_PYPI'
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           name: v${{ steps.release_checks.outputs.github_release_version }}
           generate_release_notes: true

diff --git a/.github/workflows/cd-syftcli.yml b/.github/workflows/cd-syftcli.yml
@@ -60,7 +60,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install dependencies
         if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
@@ -117,7 +117,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Install build dependencies for syftcli
         run: |
@@ -181,7 +181,7 @@ jobs:
 
       - name: GitHub Release
         if: steps.check-pypi.outputs.released == 'true'
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           name: syftcli-v${{ needs.deploy-syft-cli.outputs.deployed_version }}
           generate_release_notes: false

diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml
@@ -189,7 +189,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       #Generate SBOM
       - name: Generate SBOM

diff --git a/.github/workflows/e2e-tests-notebook.yml b/.github/workflows/e2e-tests-notebook.yml
@@ -0,0 +1,90 @@
+name: E2E - Notebook Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      syft_version:
+        description: "Syft version to test"
+        required: true
+        type: string
+      node_url:
+        description: "Node URL to use"
+        required: true
+        type: string
+      node_port:
+        description: "Node port"
+        required: true
+        type: number
+      exclude_notebooks:
+        description: "Notebooks to exclude ex: not 11-container-images-k8s.ipynb"
+        required: false
+        type: string
+
+  workflow_call:
+    inputs:
+      syft_version:
+        description: "Syft version to test"
+        required: true
+        type: string
+      node_url:
+        description: "Node URL to use"
+        required: true
+        type: string
+      node_port:
+        description: "Node port"
+        required: true
+        type: number
+      exclude_notebooks:
+        description: "Notebooks to exclude ex: not 11-container-images-k8s.ipynb"
+        required: false
+        type: string
+
+jobs:
+  notebook-test-hagrid:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Upgrade pip
+        run: |
+          python -m pip install --upgrade --user pip
+
+      - name: Get pip cache dir
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: pip cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+
+      - name: Install tox
+        run: |
+          pip install tox
+
+      - name: Run Notebook tests
+        env:
+          SYFT_VERSION: ${{ inputs.syft_version }}
+          NODE_URL: ${{ inputs.node_url }}
+          NODE_PORT: ${{ inputs.node_port }}
+          EXCLUDE_NOTEBOOKS: ${{ inputs.exclude_notebooks }}
+        run: |
+          tox -e e2e.test.notebook
+
diff --git a/.github/workflows/manual-delete-buildjet-cache.yml b/.github/workflows/manual-delete-buildjet-cache.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/.github/workflows/pr-tests-enclave.yml b/.github/workflows/pr-tests-enclave.yml
@@ -1,13 +1,14 @@
 name: PR Tests - Enclave
 
 on:
-  workflow_call:
+  # Temporarily disabled oblv tests
+  # workflow_call:
 
-  pull_request:
-    branches:
-      - dev
-      - main
-      - "0.8"
+  # pull_request:
+  #   branches:
+  #     - dev
+  #     - main
+  #     - "0.8"
 
   workflow_dispatch:
     inputs:
@@ -25,7 +26,7 @@ jobs:
       max-parallel: 4
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.11"]
+        python-version: ["3.12"]
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -58,31 +59,31 @@ jobs:
       - name: Upgrade pip
         if: steps.changes.outputs.syft == 'true'
         run: |
-          python -m pip install --upgrade --user pip
+          pip install --upgrade pip uv==0.1.18
+          uv --version
 
       - name: Get pip cache dir
         id: pip-cache
         if: steps.changes.outputs.syft == 'true'
         shell: bash
         run: |
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
 
       - name: pip cache
         uses: actions/cache@v4
         if: steps.changes.outputs.syft == 'true'
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
           restore-keys: |
-            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}-
 
       - name: Install Dependencies
         if: steps.changes.outputs.syft == 'true'
         run: |
-          pip install --upgrade tox packaging wheel --default-timeout=60
+          pip install --upgrade tox tox-uv==1.5.1
 
-      # Temporarily disabled oblv tests
-      # - name: Run Enclave tests
-      #   if: steps.changes.outputs.syft == 'true'
-      #   run: |
-      #     tox -e stack.test.integration.enclave.oblv
+      - name: Run Enclave tests
+        if: steps.changes.outputs.syft == 'true'
+        run: |
+          tox -e stack.test.integration.enclave.oblv