finish docker setup

OpenMined · Oct 16, 2023 · 0350660 · 0350660
2 parents 9be1bd7 + 24740da
commit 0350660
Show file tree

Hide file tree

Showing 103 changed files with 4,818 additions and 1,708 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.2-beta.33
+current_version = 0.8.2-beta.35
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml
@@ -1,19 +1,48 @@
-name: CD - Syft - Deploy `dev` to K8s
+name: CD - Kubernetes - Dev
 
 on:
-  # TODO: Re-enable once merged & tested
-  # schedule:
-  #   - cron: "0 */3 * * *"
+  schedule:
+    - cron: "0 */3 * * *"
 
   workflow_dispatch:
     inputs:
-      none:
-        description: "Run Tests Manually"
-        required: false
+      check-cache:
+        type: boolean
+        description: "Check workflow cache"
+        default: false
+      deploy-helm:
+        type: boolean
+        description: "Deploy Helm Charts"
+        default: false
 
 jobs:
+  check-last-run:
+    runs-on: om-ci-16vcpu-ubuntu2204
+
+    outputs:
+      same-commit: ${{ steps.cache.outputs.cache-hit }}
+
+    steps:
+      - name: Check for new changes
+        if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true'
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: scripts/k8s_dev_hash # we don't care about the file, just the key
+          key: k8s-dev-${{ github.sha }}
+          lookup-only: true
+
+      - name: Save Commit SHA
+        if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true'
+        # only for making the cache action happy
+        run: echo "${{ github.sha }}" > scripts/k8s_dev_hash
+
   deploy-syft-dev:
+    needs: check-last-run
+    if: needs.check-last-run.outputs.same-commit != 'true'
+
     runs-on: om-ci-16vcpu-ubuntu2204
+
     steps:
       - name: Permission to home directory
         run: |
@@ -39,7 +68,17 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to Azure container registry
+      - name: Install Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az version
+
+      - name: Login to Azure CLI
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS_GITHUB_CI }}
+
+      - name: Login to Azure Container Registry
         uses: azure/docker-login@v1
         with:
           login-server: ${{ secrets.ACR_SERVER }}
@@ -51,8 +90,8 @@ jobs:
         shell: bash
         run: echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT
 
-      - name: Build and push `grid-backend` image to DockerHub
-        uses: docker/build-push-action@v4
+      - name: Build and push `grid-backend` image to registry
+        uses: docker/build-push-action@v5
         with:
           context: ./packages
           file: ./packages/grid/backend/backend.dockerfile
@@ -61,21 +100,23 @@ jobs:
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }}
 
-      - name: Build and push `grid-frontend` image to DockerHub
-        uses: docker/build-push-action@v4
+      - name: Build and push `grid-frontend` image to registry
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/frontend
           file: ./packages/grid/frontend/frontend.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }}
           target: grid-ui-development
 
       # TODO: Re-enable once we have Enclave up and running
-      # - name: Build and push `grid-enclave` image to DockerHub
-      #   uses: docker/build-push-action@v4
+      # - name: Build and push `grid-enclave` image to registry
+      #   uses: docker/build-push-action@v5
       #   with:
       #     context: ./packages
       #     file: ./packages/grid/worker/worker.dockerfile
@@ -84,38 +125,43 @@ jobs:
       #     tags: |
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
       #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
+      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }}
 
-      - name: Build and push `grid-headscale` image to DockerHub
-        uses: docker/build-push-action@v4
+      - name: Build and push `grid-headscale` image to registry
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/vpn
           file: ./packages/grid/vpn/headscale.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-headscale:${{ steps.grid.outputs.GRID_VERSION }}
 
-      - name: Build and push `grid-tailscale` image to DockerHub
-        uses: docker/build-push-action@v4
+      - name: Build and push `grid-tailscale` image to registry
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/vpn
           file: ./packages/grid/vpn/tailscale.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-tailscale:${{ steps.grid.outputs.GRID_VERSION }}
 
-      - name: Build and push `grid-vpn-iptables` image to DockerHub
-        uses: docker/build-push-action@v2
+      - name: Build and push `grid-vpn-iptables` image to registry
+        uses: docker/build-push-action@v5
         with:
           context: ./packages/grid/vpn
           file: ./packages/grid/vpn/iptables.dockerfile
           push: true
           tags: |
             ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev
             ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:dev-${{ github.sha }}
+            ${{ secrets.ACR_SERVER }}/openmined/grid-vpn-iptables:${{ steps.grid.outputs.GRID_VERSION }}
 
       - name: Build Helm Chart & Copy to infra
+        if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         shell: bash
         run: |
           K3D_VERSION=v5.6.0
@@ -142,6 +188,7 @@ jobs:
           cp -R packages/grid/helm/syft/. packages/grid/helm/manifests.yaml infrastructure/gitops/environments/dev/
 
       - name: Commit & push changes to Infra Repo
+        if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         uses: EndBug/add-and-commit@v9
         with:
           author_name: ${{ secrets.OM_BOT_NAME }}
@@ -150,3 +197,29 @@ jobs:
           add: "."
           push: "origin main"
           cwd: "./infrastructure/"
+
+      - name: Cleanup Azure Container Registry
+        run: |
+          ACR_REGISTRY_NAME=${{ secrets.ACR_REGISTRY_NAME }}
+
+          echo ">> Fetching repo list.."
+          REPO_LIST=$(az acr repository list -n $ACR_REGISTRY_NAME -o tsv)
+
+          KEEP_PREV_VERSIONS=5
+          TAIL_FROM_LINE=$(($KEEP_PREV_VERSIONS + 1))
+
+          for repo in $REPO_LIST
+          do
+              echo "Cleaning up '$repo'"
+              az acr repository show-tags --name $ACR_REGISTRY_NAME --repository $repo --orderby time_desc --output tsv \
+                | grep dev- \
+                | tail -n +$TAIL_FROM_LINE \
+                | xargs -r -I% az acr repository untag --name $ACR_REGISTRY_NAME --image $repo:%
+          done
+
+      - name: Logout and cleanup Azure account
+        if: always()
+        run: |
+          az logout
+          az cache purge
+          az account clear
diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -73,7 +73,8 @@ jobs:
         shell: bash
         run: |
           # install k3d
-          wget https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64
+          K3D_VERSION=v5.6.0
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
           mv k3d-linux-amd64 k3d
           chmod +x k3d
           export PATH=`pwd`:$PATH
@@ -175,6 +176,7 @@ jobs:
           files: |
             ./packages/syftcli/manifest.yml
             ./build/syftcli-config/*
+            ./packages/hagrid/hagrid/manifest_template.yml
           tag_name: v${{ steps.release_checks.outputs.github_release_version }}
 
       - name: Set up QEMU

diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -538,7 +538,7 @@ jobs:
           chmod 700 get_helm.sh
           ./get_helm.sh
 
-      - name: Run integration tests
+      - name: Run K8s & Helm integration tests
         if: steps.changes.outputs.stack == 'true'
         timeout-minutes: 60
         env:
@@ -547,7 +547,8 @@ jobs:
         shell: bash
         run: |
           # install k3d
-          wget https://github.com/k3d-io/k3d/releases/download/v5.5.1/k3d-linux-amd64
+          K3D_VERSION=v5.6.0
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
           mv k3d-linux-amd64 k3d
           chmod +x k3d
           export PATH=`pwd`:$PATH
@@ -556,7 +557,7 @@ jobs:
           curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
           chmod +x devspace
           devspace version
-          # tox -e stack.test.integration.k8s
+          tox -e stack.test.integration.k8s
           tox -e syft.build.helm
           tox -e syft.package.helm
           # tox -e syft.test.helm
diff --git a/.gitignore b/.gitignore
@@ -73,3 +73,5 @@ nohup.out
 
 # notebook data
 notebooks/helm/scenario_data.jsonl
+# tox syft.build.helm generated file
+out.txt
diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.2-beta.33"
+__version__ = "0.8.2-beta.35"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/notebooks/api/0.8/06-multiple-code-requests.ipynb b/notebooks/api/0.8/06-multiple-code-requests.ipynb
@@ -103,12 +103,14 @@
    },
    "outputs": [],
    "source": [
-    "root_client.register(name=\"Sheldon Cooper\",\n",
-    "                             email=\"[email protected]\",\n",
-    "                             password=\"abc123\",\n",
-    "                             password_verify=\"abc123\",\n",
-    "                             institution=\"Caltech\",\n",
-    "                             website=\"https://www.caltech.edu/\")"
+    "root_client.register(\n",
+    "    name=\"Sheldon Cooper\",\n",
+    "    email=\"[email protected]\",\n",
+    "    password=\"abc123\",\n",
+    "    password_verify=\"abc123\",\n",
+    "    institution=\"Caltech\",\n",
+    "    website=\"https://www.caltech.edu/\"\n",
+    ")"
    ]
   },
   {
@@ -167,13 +169,15 @@
    "outputs": [],
    "source": [
     "# The Data Scientist does some basic statistics function on the object\n",
-    "@sy.syft_function(input_policy=sy.ExactMatch(data=dataset_ptr.assets[0]),\n",
-    "                  output_policy=sy.SingleExecutionExactOutput())\n",
+    "@sy.syft_function(\n",
+    "    input_policy=sy.ExactMatch(data=dataset_ptr.assets[0]),\n",
+    "    output_policy=sy.SingleExecutionExactOutput()\n",
+    ")\n",
     "def calculate_sum(data):\n",
     "    import opendp.prelude as dp\n",
     "    dp.enable_features(\"contrib\")\n",
     "    from opendp.measurements import make_laplace\n",
-    "    \n",
+    "\n",
     "    #compute sum\n",
     "    res = data.sum()\n",
     "    base_lap = dp.m.make_base_laplace(\n",
@@ -225,7 +229,7 @@
    "source": [
     "project = new_project.start()\n",
     "\n",
-    "project "
+    "project"
    ]
   },
   {
@@ -249,7 +253,7 @@
    },
    "outputs": [],
    "source": [
-    "assert len(project.events) ==1\n",
+    "assert len(project.events) == 1\n",
     "assert isinstance(project.events[0], sy.service.project.project.ProjectRequest)"
    ]
   },
@@ -263,13 +267,15 @@
    "outputs": [],
    "source": [
     "# The Data Scientist would like to submit another code request after the start of the project\n",
-    "@sy.syft_function(input_policy=sy.ExactMatch(data=dataset_ptr.assets[0]),\n",
-    "                  output_policy=sy.SingleExecutionExactOutput())\n",
+    "@sy.syft_function(\n",
+    "    input_policy=sy.ExactMatch(data=dataset_ptr.assets[0]),\n",
+    "    output_policy=sy.SingleExecutionExactOutput()\n",
+    ")\n",
     "def calculate_mean(data):\n",
     "    import opendp.prelude as dp\n",
     "    dp.enable_features(\"contrib\")\n",
     "    from opendp.measurements import make_laplace\n",
-    "    \n",
+    "\n",
     "    #compute mean\n",
     "    mean = data.mean()\n",
     "    base_lap = dp.m.make_base_laplace(\n",
@@ -290,7 +296,7 @@
    },
    "outputs": [],
    "source": [
-    "res = project.create_code_request(calculate_mean ,ds_client)\n",
+    "res = project.create_code_request(calculate_mean, ds_client)\n",
     "res"
    ]
   },
@@ -482,7 +488,7 @@
    },
    "outputs": [],
    "source": [
-    "assert isinstance(mean_result,float)"
+    "assert isinstance(mean_result, float)"
    ]
   },
   {
@@ -515,7 +521,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.11.5"
   },
   "toc": {
    "base_numbering": 1,