Merge branch 'dev' into freeze-protocol-version

OpenMined · Feb 8, 2024 · 3d3e017 · 3d3e017
2 parents cb0b949 + a7ae983
commit 3d3e017
Show file tree

Hide file tree

Showing 63 changed files with 1,221 additions and 546 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.4-beta.19
+current_version = 0.8.4-beta.21
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/README.md b/README.md
@@ -81,14 +81,27 @@ SYFT_VERSION="<paste the chart version number>"
 #### 4. Provisioning Helm Charts
 
 ```sh
-helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik
+helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className="traefik"
 ```
 
-### Azure or GCP Ingress
+### Ingress Controllers
 
+For Azure AKS
+
+```sh
+helm install ... --set ingress.className="azure-application-gateway"
 ```
-helm install ... --set ingress.ingressClass="azure/application-gateway"
-helm install ... --set ingress.ingressClass="gce"
+
+For AWS EKS
+
+```sh
+helm install ... --set ingress.className="alb"
+```
+
+For Google GKE we need the [`gce` annotation](https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress) annotation.
+
+```sh
+helm install ... --set ingress.class="gce"
 ```
 
 ## Deploy to a Container Engine or Cloud

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.4-beta.19"
+__version__ = "0.8.4-beta.21"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb
@@ -133,6 +133,28 @@
     "docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62762ceb-38da-46f1-acac-cdf5bbf29513",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test image build locally\n",
+    "test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n",
+    "test_build_res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0235e567-c65c-48fe-825d-79ea3e219166",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -1406,7 +1428,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.9.7"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb
@@ -74,6 +74,14 @@
     "domain_client"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "fe3d0aa7",
+   "metadata": {},
+   "source": [
+    "### Scaling Default Worker Pool"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "55439eb5-1e92-46a6-a45a-471917a86265",
@@ -92,6 +100,101 @@
     "domain_client.worker_pools"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "0ff8e268",
+   "metadata": {},
+   "source": [
+    "Scale up to 3 workers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de9872be",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = domain_client.api.services.worker_pool.scale(\n",
+    "    number=3, pool_name=\"default-pool\"\n",
+    ")\n",
+    "assert not isinstance(result, sy.SyftError), str(result)\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da6a499b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = domain_client.api.services.worker_pool.get_by_name(pool_name=\"default-pool\")\n",
+    "assert len(result.workers) == 3, str(result.to_dict())\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27761f0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# stdlib\n",
+    "# wait for some time for scale up to be ready\n",
+    "from time import sleep\n",
+    "\n",
+    "sleep(5)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c1276b5c",
+   "metadata": {},
+   "source": [
+    "Scale down to 1 worker"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f0aa94c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "default_worker_pool = domain_client.api.services.worker_pool.scale(\n",
+    "    number=1, pool_name=\"default-pool\"\n",
+    ")\n",
+    "assert not isinstance(result, sy.SyftError), str(result)\n",
+    "default_worker_pool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52acc6f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = domain_client.api.services.worker_pool.get_by_name(pool_name=\"default-pool\")\n",
+    "assert len(result.workers) == 1, str(result.to_dict())\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9a7b40a3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "default_worker_pool = domain_client.api.services.worker_pool.get_by_name(\n",
+    "    pool_name=\"default-pool\"\n",
+    ")\n",
+    "default_worker_pool"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "3c7a124a",
@@ -1153,7 +1256,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.2"
+   "version": "3.11.7"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb b/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb
@@ -142,7 +142,7 @@
    "metadata": {},
    "source": [
     "```bash\n",
-    "helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik\n",
+    "helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=traefik\n",
     "```"
    ]
   },

diff --git a/packages/grid/VERSION b/packages/grid/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.8.4-beta.19"
+__version__ = "0.8.4-beta.21"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/packages/grid/backend/worker_cpu.dockerfile b/packages/grid/backend/worker_cpu.dockerfile
@@ -9,7 +9,7 @@
 # Later we'd want to uninstall old python, and then install a new python runtime...
 # ... but pre-built syft deps may break!
 
-ARG SYFT_VERSION_TAG="0.8.4-beta.19"
+ARG SYFT_VERSION_TAG="0.8.4-beta.21"
 FROM openmined/grid-backend:${SYFT_VERSION_TAG}
 
 ARG PYTHON_VERSION="3.11"

diff --git a/packages/grid/default.env b/packages/grid/default.env
@@ -26,7 +26,7 @@ DOCKER_IMAGE_TRAEFIK=traefik
 TRAEFIK_VERSION=v2.10
 REDIS_VERSION=6.2
 RABBITMQ_VERSION=3
-SEAWEEDFS_VERSION=3.59
+SEAWEEDFS_VERSION=3.62
 DOCKER_IMAGE_SEAWEEDFS=openmined/grid-seaweedfs
 VERSION=latest
 VERSION_HASH=unknown
@@ -73,6 +73,7 @@ S3_REGION="us-east-1"
 S3_PRESIGNED_TIMEOUT_SECS=1800
 S3_VOLUME_SIZE_MB=1024
 
+
 # Jax
 JAX_ENABLE_X64=True
 

diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml
@@ -25,7 +25,7 @@ vars:
   DEVSPACE_ENV_FILE: "default.env"
   CONTAINER_REGISTRY: "docker.io"
   NODE_NAME: "mynode"
-  VERSION: "0.8.4-beta.19"
+  VERSION: "0.8.4-beta.21"
 
 # This is a list of `images` that DevSpace can build for this project
 # We recommend to skip image building during development (devspace dev) as much as possible
@@ -66,8 +66,8 @@ deployments:
         syft:
           registry: ${CONTAINER_REGISTRY}
           version: dev-${DEVSPACE_TIMESTAMP}
-        workerBuilds:
-          mountInBackend: true
+        registry:
+          maxStorage: "5Gi"
         node:
           settings:
             nodeName: ${NODE_NAME}

diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json
@@ -1,6 +1,6 @@
 {
   "name": "pygrid-ui",
-  "version": "0.8.4-beta.19",
+  "version": "0.8.4-beta.21",
   "private": true,
   "scripts": {
     "dev": "pnpm i && vite dev --host --port 80",