diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57e08dc1658..f2d6045e2a9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.4-beta.19 +current_version = 0.8.4-beta.21 tag = False tag_name = {new_version} commit = True diff --git a/README.md b/README.md index 6e402cc6342..735eee063d3 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,27 @@ SYFT_VERSION="" #### 4. Provisioning Helm Charts ```sh -helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik +helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className="traefik" ``` -### Azure or GCP Ingress +### Ingress Controllers +For Azure AKS + +```sh +helm install ... --set ingress.className="azure-application-gateway" ``` -helm install ... --set ingress.ingressClass="azure/application-gateway" -helm install ... --set ingress.ingressClass="gce" + +For AWS EKS + +```sh +helm install ... --set ingress.className="alb" +``` + +For Google GKE we need the [`gce` annotation](https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress) annotation. + +```sh +helm install ... --set ingress.class="gce" ``` ## Deploy to a Container Engine or Cloud diff --git a/VERSION b/VERSION index d8f01b13857..1f184fb3f81 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.21" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb index 81ba6a05d13..35f26e791ff 100644 --- a/notebooks/api/0.8/10-container-images.ipynb +++ b/notebooks/api/0.8/10-container-images.ipynb @@ -133,6 +133,28 @@ "docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "62762ceb-38da-46f1-acac-cdf5bbf29513", + "metadata": {}, + "outputs": [], + "source": [ + "# test image build locally\n", + "test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n", + "test_build_res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0235e567-c65c-48fe-825d-79ea3e219166", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1406,7 +1428,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index 07c26a78b6a..c28b0187567 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -74,6 +74,14 @@ "domain_client" ] }, + { + "cell_type": "markdown", + "id": "fe3d0aa7", + "metadata": {}, + "source": [ + "### Scaling Default Worker Pool" + ] + }, { "cell_type": "markdown", "id": "55439eb5-1e92-46a6-a45a-471917a86265", @@ -92,6 +100,101 @@ "domain_client.worker_pools" ] }, + { + "cell_type": "markdown", + "id": "0ff8e268", + "metadata": {}, + "source": [ + "Scale up to 3 workers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de9872be", + "metadata": {}, + "outputs": [], + "source": [ + "result = domain_client.api.services.worker_pool.scale(\n", + " number=3, pool_name=\"default-pool\"\n", + ")\n", + "assert not isinstance(result, sy.SyftError), str(result)\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da6a499b", + "metadata": {}, + "outputs": [], + "source": [ + "result = domain_client.api.services.worker_pool.get_by_name(pool_name=\"default-pool\")\n", + "assert len(result.workers) == 3, str(result.to_dict())\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27761f0c", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "# wait for some time for scale up to be ready\n", + "from time import sleep\n", + "\n", + "sleep(5)" + ] + }, + { + "cell_type": "markdown", + "id": "c1276b5c", + "metadata": {}, + "source": [ + "Scale down to 1 worker" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f0aa94c", + "metadata": {}, + "outputs": [], + "source": [ + "default_worker_pool = domain_client.api.services.worker_pool.scale(\n", + " number=1, pool_name=\"default-pool\"\n", + ")\n", + "assert not isinstance(result, sy.SyftError), str(result)\n", + "default_worker_pool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52acc6f6", + "metadata": {}, + "outputs": [], + "source": [ + "result = domain_client.api.services.worker_pool.get_by_name(pool_name=\"default-pool\")\n", + "assert len(result.workers) == 1, str(result.to_dict())\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a7b40a3", + "metadata": {}, + "outputs": [], + "source": [ + "default_worker_pool = domain_client.api.services.worker_pool.get_by_name(\n", + " pool_name=\"default-pool\"\n", + ")\n", + "default_worker_pool" + ] + }, { "cell_type": "markdown", "id": "3c7a124a", @@ -1153,7 +1256,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb b/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb index ed1537ef73a..729b5751c2f 100644 --- a/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb +++ b/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb @@ -142,7 +142,7 @@ "metadata": {}, "source": [ "```bash\n", - "helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik\n", + "helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=traefik\n", "```" ] }, diff --git a/packages/grid/VERSION b/packages/grid/VERSION index d8f01b13857..1f184fb3f81 100644 --- a/packages/grid/VERSION +++ b/packages/grid/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.21" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/backend/worker_cpu.dockerfile b/packages/grid/backend/worker_cpu.dockerfile index cccf70f7cb6..d2971433e5d 100644 --- a/packages/grid/backend/worker_cpu.dockerfile +++ b/packages/grid/backend/worker_cpu.dockerfile @@ -9,7 +9,7 @@ # Later we'd want to uninstall old python, and then install a new python runtime... # ... but pre-built syft deps may break! -ARG SYFT_VERSION_TAG="0.8.4-beta.19" +ARG SYFT_VERSION_TAG="0.8.4-beta.21" FROM openmined/grid-backend:${SYFT_VERSION_TAG} ARG PYTHON_VERSION="3.11" diff --git a/packages/grid/default.env b/packages/grid/default.env index 5e69aca2580..d599e47cf4e 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -26,7 +26,7 @@ DOCKER_IMAGE_TRAEFIK=traefik TRAEFIK_VERSION=v2.10 REDIS_VERSION=6.2 RABBITMQ_VERSION=3 -SEAWEEDFS_VERSION=3.59 +SEAWEEDFS_VERSION=3.62 DOCKER_IMAGE_SEAWEEDFS=openmined/grid-seaweedfs VERSION=latest VERSION_HASH=unknown @@ -73,6 +73,7 @@ S3_REGION="us-east-1" S3_PRESIGNED_TIMEOUT_SECS=1800 S3_VOLUME_SIZE_MB=1024 + # Jax JAX_ENABLE_X64=True diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 9accc527d5d..1f10b1170ad 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -25,7 +25,7 @@ vars: DEVSPACE_ENV_FILE: "default.env" CONTAINER_REGISTRY: "docker.io" NODE_NAME: "mynode" - VERSION: "0.8.4-beta.19" + VERSION: "0.8.4-beta.21" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible @@ -66,8 +66,8 @@ deployments: syft: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} - workerBuilds: - mountInBackend: true + registry: + maxStorage: "5Gi" node: settings: nodeName: ${NODE_NAME} diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json index 93458c129d8..d6a2110587b 100644 --- a/packages/grid/frontend/package.json +++ b/packages/grid/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pygrid-ui", - "version": "0.8.4-beta.19", + "version": "0.8.4-beta.21", "private": true, "scripts": { "dev": "pnpm i && vite dev --host --port 80", diff --git a/packages/grid/helm/repo/index.yaml b/packages/grid/helm/repo/index.yaml index db8ca70cd90..61fb7eb380f 100644 --- a/packages/grid/helm/repo/index.yaml +++ b/packages/grid/helm/repo/index.yaml @@ -1,12 +1,36 @@ apiVersion: v1 entries: syft: + - apiVersion: v2 + appVersion: 0.8.4-beta.21 + created: "2024-02-08T12:28:05.631588027Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: 7dce153d2fcae7513e9c132e139b2721fd975ea3cc43a370e34dbeb2a1b7f683 + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.4-beta.21.tgz + version: 0.8.4-beta.21 + - apiVersion: v2 + appVersion: 0.8.4-beta.20 + created: "2024-02-08T12:28:05.63107618Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: c51189a187bbf24135382e25cb00964e0330dfcd3b2f0c884581a6686f05dd28 + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.4-beta.20.tgz + version: 0.8.4-beta.20 - apiVersion: v2 appVersion: 0.8.4-beta.19 - created: "2024-02-05T14:11:25.654223+05:30" + created: "2024-02-08T12:28:05.629522532Z" description: Perform numpy-like analysis on data that remains in someone elses server - digest: c2868a765415fe17b1bdad35254fcb7f355960bbb2b1f04e17e46cd1c4bf9ab7 + digest: 8219575dedb42fa2ddbf2768a4e9afbfacbc2dff7e953d77c7b10a41b78dc687 icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png name: syft type: application @@ -15,7 +39,7 @@ entries: version: 0.8.4-beta.19 - apiVersion: v2 appVersion: 0.8.4-beta.18 - created: "2024-02-05T14:11:25.653769+05:30" + created: "2024-02-08T12:28:05.629121832Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6418cde559cf12f1f7fea5a2b123bba950e50eeb3be002441827d2ab7f9e4ef7 @@ -27,7 +51,7 @@ entries: version: 0.8.4-beta.18 - apiVersion: v2 appVersion: 0.8.4-beta.17 - created: "2024-02-05T14:11:25.653372+05:30" + created: "2024-02-08T12:28:05.628684253Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 71b39c5a4c64037eadbb154f7029282ba90d9a0d703f8d4c7dfc1ba2f5d81498 @@ -39,7 +63,7 @@ entries: version: 0.8.4-beta.17 - apiVersion: v2 appVersion: 0.8.4-beta.16 - created: "2024-02-05T14:11:25.653094+05:30" + created: "2024-02-08T12:28:05.628283122Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9c9840a7c9476dbb08e0ac83926330718fe50c89879752dd8f92712b036109c0 @@ -51,7 +75,7 @@ entries: version: 0.8.4-beta.16 - apiVersion: v2 appVersion: 0.8.4-beta.15 - created: "2024-02-05T14:11:25.652801+05:30" + created: "2024-02-08T12:28:05.627874367Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0955fd22da028315e30c68132cbfa4bdc82bae622039bcfce0de339707bb82eb @@ -63,7 +87,7 @@ entries: version: 0.8.4-beta.15 - apiVersion: v2 appVersion: 0.8.4-beta.14 - created: "2024-02-05T14:11:25.65251+05:30" + created: "2024-02-08T12:28:05.627474338Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 56208571956abe20ed7a5cc1867cab2667ed792c63e53d0e8bb70a9b438b7bf6 @@ -75,7 +99,7 @@ entries: version: 0.8.4-beta.14 - apiVersion: v2 appVersion: 0.8.4-beta.13 - created: "2024-02-05T14:11:25.652256+05:30" + created: "2024-02-08T12:28:05.627121317Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: d7222c72412b6ee5833fbb07d2549be179cdfc7ccd89e0ad947d112fce799b83 @@ -87,7 +111,7 @@ entries: version: 0.8.4-beta.13 - apiVersion: v2 appVersion: 0.8.4-beta.12 - created: "2024-02-05T14:11:25.652008+05:30" + created: "2024-02-08T12:28:05.626772324Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: af08c723756e397962b2d5190dedfd50797b771c5caf58b93a6f65d8fa24785c @@ -99,7 +123,7 @@ entries: version: 0.8.4-beta.12 - apiVersion: v2 appVersion: 0.8.4-beta.11 - created: "2024-02-05T14:11:25.651755+05:30" + created: "2024-02-08T12:28:05.626422319Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a0235835ba57d185a83dd8a26281fa37b2077c3a37fe3a1c50585005695927e3 @@ -111,7 +135,7 @@ entries: version: 0.8.4-beta.11 - apiVersion: v2 appVersion: 0.8.4-beta.10 - created: "2024-02-05T14:11:25.651506+05:30" + created: "2024-02-08T12:28:05.62607571Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 910ddfeba0c5e66651500dd11404afff092adc0f768ed68e0d93b04b83aa4388 @@ -123,7 +147,7 @@ entries: version: 0.8.4-beta.10 - apiVersion: v2 appVersion: 0.8.4-beta.9 - created: "2024-02-05T14:11:25.656422+05:30" + created: "2024-02-08T12:28:05.634222038Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c25ca8a9f072d6a5d02232448deaef5668aca05f24dfffbba3ebe30a4f75bb26 @@ -135,7 +159,7 @@ entries: version: 0.8.4-beta.9 - apiVersion: v2 appVersion: 0.8.4-beta.8 - created: "2024-02-05T14:11:25.656176+05:30" + created: "2024-02-08T12:28:05.633825235Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7249a39d4137e457b369384ba0a365c271c780d93a8327ce25083df763c39999 @@ -147,7 +171,7 @@ entries: version: 0.8.4-beta.8 - apiVersion: v2 appVersion: 0.8.4-beta.7 - created: "2024-02-05T14:11:25.655931+05:30" + created: "2024-02-08T12:28:05.633394809Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ee750c7c8d6ea05bd447375e624fdd7f66dd87680ab81f7b7e73df7379a9024a @@ -159,7 +183,7 @@ entries: version: 0.8.4-beta.7 - apiVersion: v2 appVersion: 0.8.4-beta.6 - created: "2024-02-05T14:11:25.655685+05:30" + created: "2024-02-08T12:28:05.633059241Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0e046be9f73df7444a995608c59af16fab9030b139b2acb4d6db6185b8eb5337 @@ -171,7 +195,7 @@ entries: version: 0.8.4-beta.6 - apiVersion: v2 appVersion: 0.8.4-beta.5 - created: "2024-02-05T14:11:25.655439+05:30" + created: "2024-02-08T12:28:05.632680322Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b56e9a23d46810eccdb4cf5272cc05126da3f6db314e541959c3efb5f260620b @@ -183,7 +207,7 @@ entries: version: 0.8.4-beta.5 - apiVersion: v2 appVersion: 0.8.4-beta.4 - created: "2024-02-05T14:11:25.655187+05:30" + created: "2024-02-08T12:28:05.632316361Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 1d5808ecaf55391f3b27ae6236400066508acbd242e33db24a1ab4bffa77409e @@ -195,7 +219,7 @@ entries: version: 0.8.4-beta.4 - apiVersion: v2 appVersion: 0.8.4-beta.3 - created: "2024-02-05T14:11:25.65492+05:30" + created: "2024-02-08T12:28:05.631968049Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b64efa8529d82be56c6ab60487ed24420a5614d96d2509c1f93c1003eda71a54 @@ -207,7 +231,7 @@ entries: version: 0.8.4-beta.3 - apiVersion: v2 appVersion: 0.8.4-beta.2 - created: "2024-02-05T14:11:25.654654+05:30" + created: "2024-02-08T12:28:05.630078702Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -223,7 +247,7 @@ entries: version: 0.8.4-beta.2 - apiVersion: v2 appVersion: 0.8.4-beta.1 - created: "2024-02-05T14:11:25.65124+05:30" + created: "2024-02-08T12:28:05.625699115Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -239,7 +263,7 @@ entries: version: 0.8.4-beta.1 - apiVersion: v2 appVersion: 0.8.3 - created: "2024-02-05T14:11:25.650813+05:30" + created: "2024-02-08T12:28:05.624975934Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -255,7 +279,7 @@ entries: version: 0.8.3 - apiVersion: v2 appVersion: 0.8.3-beta.6 - created: "2024-02-05T14:11:25.650294+05:30" + created: "2024-02-08T12:28:05.623739249Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -271,7 +295,7 @@ entries: version: 0.8.3-beta.6 - apiVersion: v2 appVersion: 0.8.3-beta.5 - created: "2024-02-05T14:11:25.649854+05:30" + created: "2024-02-08T12:28:05.62316792Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -287,7 +311,7 @@ entries: version: 0.8.3-beta.5 - apiVersion: v2 appVersion: 0.8.3-beta.4 - created: "2024-02-05T14:11:25.649015+05:30" + created: "2024-02-08T12:28:05.622575381Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -303,7 +327,7 @@ entries: version: 0.8.3-beta.4 - apiVersion: v2 appVersion: 0.8.3-beta.3 - created: "2024-02-05T14:11:25.648526+05:30" + created: "2024-02-08T12:28:05.621895208Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -319,7 +343,7 @@ entries: version: 0.8.3-beta.3 - apiVersion: v2 appVersion: 0.8.3-beta.2 - created: "2024-02-05T14:11:25.648132+05:30" + created: "2024-02-08T12:28:05.621344558Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -335,7 +359,7 @@ entries: version: 0.8.3-beta.2 - apiVersion: v2 appVersion: 0.8.3-beta.1 - created: "2024-02-05T14:11:25.647739+05:30" + created: "2024-02-08T12:28:05.620789068Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -351,7 +375,7 @@ entries: version: 0.8.3-beta.1 - apiVersion: v2 appVersion: 0.8.2 - created: "2024-02-05T14:11:25.64733+05:30" + created: "2024-02-08T12:28:05.620194615Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -367,7 +391,7 @@ entries: version: 0.8.2 - apiVersion: v2 appVersion: 0.8.2-beta.60 - created: "2024-02-05T14:11:25.646837+05:30" + created: "2024-02-08T12:28:05.619520473Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -383,7 +407,7 @@ entries: version: 0.8.2-beta.60 - apiVersion: v2 appVersion: 0.8.2-beta.59 - created: "2024-02-05T14:11:25.646354+05:30" + created: "2024-02-08T12:28:05.618093883Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -399,7 +423,7 @@ entries: version: 0.8.2-beta.59 - apiVersion: v2 appVersion: 0.8.2-beta.58 - created: "2024-02-05T14:11:25.645853+05:30" + created: "2024-02-08T12:28:05.617464305Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -415,7 +439,7 @@ entries: version: 0.8.2-beta.58 - apiVersion: v2 appVersion: 0.8.2-beta.57 - created: "2024-02-05T14:11:25.645306+05:30" + created: "2024-02-08T12:28:05.6168203Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -431,7 +455,7 @@ entries: version: 0.8.2-beta.57 - apiVersion: v2 appVersion: 0.8.2-beta.56 - created: "2024-02-05T14:11:25.644509+05:30" + created: "2024-02-08T12:28:05.616154193Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -447,7 +471,7 @@ entries: version: 0.8.2-beta.56 - apiVersion: v2 appVersion: 0.8.2-beta.53 - created: "2024-02-05T14:11:25.644028+05:30" + created: "2024-02-08T12:28:05.615507422Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -463,7 +487,7 @@ entries: version: 0.8.2-beta.53 - apiVersion: v2 appVersion: 0.8.2-beta.52 - created: "2024-02-05T14:11:25.643545+05:30" + created: "2024-02-08T12:28:05.61487075Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -479,7 +503,7 @@ entries: version: 0.8.2-beta.52 - apiVersion: v2 appVersion: 0.8.2-beta.51 - created: "2024-02-05T14:11:25.643061+05:30" + created: "2024-02-08T12:28:05.614198733Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -495,7 +519,7 @@ entries: version: 0.8.2-beta.51 - apiVersion: v2 appVersion: 0.8.2-beta.50 - created: "2024-02-05T14:11:25.642566+05:30" + created: "2024-02-08T12:28:05.61341177Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -511,7 +535,7 @@ entries: version: 0.8.2-beta.50 - apiVersion: v2 appVersion: 0.8.2-beta.49 - created: "2024-02-05T14:11:25.642086+05:30" + created: "2024-02-08T12:28:05.611892844Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -527,7 +551,7 @@ entries: version: 0.8.2-beta.49 - apiVersion: v2 appVersion: 0.8.2-beta.48 - created: "2024-02-05T14:11:25.641572+05:30" + created: "2024-02-08T12:28:05.611252335Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -543,7 +567,7 @@ entries: version: 0.8.2-beta.48 - apiVersion: v2 appVersion: 0.8.2-beta.47 - created: "2024-02-05T14:11:25.641031+05:30" + created: "2024-02-08T12:28:05.610587792Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -559,7 +583,7 @@ entries: version: 0.8.2-beta.47 - apiVersion: v2 appVersion: 0.8.2-beta.46 - created: "2024-02-05T14:11:25.640572+05:30" + created: "2024-02-08T12:28:05.6100102Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -575,7 +599,7 @@ entries: version: 0.8.2-beta.46 - apiVersion: v2 appVersion: 0.8.2-beta.45 - created: "2024-02-05T14:11:25.63981+05:30" + created: "2024-02-08T12:28:05.609428862Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -591,7 +615,7 @@ entries: version: 0.8.2-beta.45 - apiVersion: v2 appVersion: 0.8.2-beta.44 - created: "2024-02-05T14:11:25.639409+05:30" + created: "2024-02-08T12:28:05.608863474Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -607,7 +631,7 @@ entries: version: 0.8.2-beta.44 - apiVersion: v2 appVersion: 0.8.2-beta.43 - created: "2024-02-05T14:11:25.639009+05:30" + created: "2024-02-08T12:28:05.608249264Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -623,7 +647,7 @@ entries: version: 0.8.2-beta.43 - apiVersion: v2 appVersion: 0.8.2-beta.41 - created: "2024-02-05T14:11:25.638518+05:30" + created: "2024-02-08T12:28:05.607577507Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -639,7 +663,7 @@ entries: version: 0.8.2-beta.41 - apiVersion: v2 appVersion: 0.8.2-beta.40 - created: "2024-02-05T14:11:25.638016+05:30" + created: "2024-02-08T12:28:05.606466964Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -655,7 +679,7 @@ entries: version: 0.8.2-beta.40 - apiVersion: v2 appVersion: 0.8.2-beta.39 - created: "2024-02-05T14:11:25.6376+05:30" + created: "2024-02-08T12:28:05.605294589Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -671,7 +695,7 @@ entries: version: 0.8.2-beta.39 - apiVersion: v2 appVersion: 0.8.2-beta.38 - created: "2024-02-05T14:11:25.637211+05:30" + created: "2024-02-08T12:28:05.6046963Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -687,7 +711,7 @@ entries: version: 0.8.2-beta.38 - apiVersion: v2 appVersion: 0.8.2-beta.37 - created: "2024-02-05T14:11:25.636805+05:30" + created: "2024-02-08T12:28:05.604110794Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -703,7 +727,7 @@ entries: version: 0.8.2-beta.37 - apiVersion: v2 appVersion: 0.8.1 - created: "2024-02-05T14:11:25.63636+05:30" + created: "2024-02-08T12:28:05.603486115Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -717,4 +741,4 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.1.tgz version: 0.8.1 -generated: "2024-02-05T14:11:25.635473+05:30" +generated: "2024-02-08T12:28:05.602790282Z" diff --git a/packages/grid/helm/repo/syft-0.8.4-beta.20.tgz b/packages/grid/helm/repo/syft-0.8.4-beta.20.tgz new file mode 100644 index 00000000000..32faf0de4c2 Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.4-beta.20.tgz differ diff --git a/packages/grid/helm/repo/syft-0.8.4-beta.21.tgz b/packages/grid/helm/repo/syft-0.8.4-beta.21.tgz new file mode 100644 index 00000000000..04e87789baf Binary files /dev/null and b/packages/grid/helm/repo/syft-0.8.4-beta.21.tgz differ diff --git a/packages/grid/helm/syft/Chart.yaml b/packages/grid/helm/syft/Chart.yaml index 4007f542f39..0bc2815ea67 100644 --- a/packages/grid/helm/syft/Chart.yaml +++ b/packages/grid/helm/syft/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: syft description: Perform numpy-like analysis on data that remains in someone elses server type: application -version: "0.8.4-beta.19" -appVersion: "0.8.4-beta.19" +version: "0.8.4-beta.21" +appVersion: "0.8.4-beta.21" icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/NOTES.txt b/packages/grid/helm/syft/templates/NOTES.txt index fd95e4181d4..e3d0595cf02 100644 --- a/packages/grid/helm/syft/templates/NOTES.txt +++ b/packages/grid/helm/syft/templates/NOTES.txt @@ -238,6 +238,13 @@ "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", "action": "remove" } + }, + "SeaweedFSBlobDeposit": { + "2": { + "version": 2, + "hash": "07d84a95324d95d9c868cd7d1c33c908f77aa468671d76c144586aab672bcbb5", + "action": "add" + } } } diff --git a/packages/grid/helm/syft/templates/backend-service-account.yaml b/packages/grid/helm/syft/templates/backend-service-account.yaml index 97608b4fd4e..56e552b2b7f 100644 --- a/packages/grid/helm/syft/templates/backend-service-account.yaml +++ b/packages/grid/helm/syft/templates/backend-service-account.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: backend-service-account - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} @@ -14,7 +13,6 @@ apiVersion: v1 kind: Secret metadata: name: backend-service-secret - namespace: {{ .Release.Namespace }} annotations: kubernetes.io/service-account.name: "backend-service-account" labels: @@ -29,7 +27,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: backend-service-role - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} @@ -53,7 +50,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: backend-service-role-binding - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} @@ -61,7 +57,6 @@ metadata: subjects: - kind: ServiceAccount name: backend-service-account - namespace: {{ .Release.Namespace }} roleRef: kind: Role name: backend-service-role diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 1bd3ab2e898..ebebae80cc2 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -136,12 +136,6 @@ spec: name: credentials-data readOnly: false subPath: credentials-data - {{- if .Values.workerBuilds.mountInBackend }} - # mount for debugging and inspection of worker-build volume - - mountPath: /root/data/images/ - name: worker-builds - readOnly: true - {{- end }} dnsConfig: null ephemeralContainers: null hostAliases: null @@ -155,12 +149,6 @@ spec: terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null - {{- if .Values.workerBuilds.mountInBackend }} - volumes: - - name: worker-builds - persistentVolumeClaim: - claimName: worker-builds - {{- end }} volumeClaimTemplates: - metadata: labels: diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index dfc5d39549a..f43fd0018dc 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -29,7 +29,7 @@ spec: command: null env: - name: VERSION - value: {{ .Values.syft.version }} + value: "{{ .Values.syft.version }}" - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE diff --git a/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml b/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml index 2eef50b54c6..6aed72bd414 100644 --- a/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml +++ b/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml @@ -8,8 +8,14 @@ metadata: app.kubernetes.io/component: ingress app.kubernetes.io/managed-by: Helm name: grid-stack-ingress + {{- if .Values.ingress.class }} + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.class }} + {{- end }} spec: - ingressClassName: {{ .Values.ingress.ingressClass }} + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} defaultBackend: service: name: proxy diff --git a/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml b/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml index a263c910156..58db0a03e29 100644 --- a/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml +++ b/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml @@ -8,8 +8,14 @@ metadata: app.kubernetes.io/component: ingress app.kubernetes.io/managed-by: Helm name: grid-stack-ingress-tls + {{- if .Values.ingress.class }} + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.class }} + {{- end }} spec: - ingressClassName: {{ .Values.ingress.ingressClass }} + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} defaultBackend: service: name: proxy diff --git a/packages/grid/helm/syft/templates/registry-service.yaml b/packages/grid/helm/syft/templates/registry-service.yaml new file mode 100644 index 00000000000..f96060e3a4d --- /dev/null +++ b/packages/grid/helm/syft/templates/registry-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: registry + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 5000 + selector: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry diff --git a/packages/grid/helm/syft/templates/registry-statefulset.yaml b/packages/grid/helm/syft/templates/registry-statefulset.yaml new file mode 100644 index 00000000000..c4fb60d474d --- /dev/null +++ b/packages/grid/helm/syft/templates/registry-statefulset.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: registry + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: registry:2 + name: registry + env: + - name: REGISTRY_STORAGE_DELETE_ENABLED + value: "true" + ports: + - containerPort: 5000 + volumeMounts: + - mountPath: /var/lib/registry + name: registry-data + volumeClaimTemplates: + - metadata: + name: registry-data + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.registry.maxStorage }} diff --git a/packages/grid/helm/syft/templates/worker-builds-pvc.yaml b/packages/grid/helm/syft/templates/worker-builds-pvc.yaml deleted file mode 100644 index 54eb4f7acc6..00000000000 --- a/packages/grid/helm/syft/templates/worker-builds-pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: worker-builds - labels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: worker-builds - app.kubernetes.io/managed-by: Helm -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.workerBuilds.maxStorage }} diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 43514759766..ace63b50481 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -1,7 +1,3 @@ -# Default values for syft. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - secrets: syft: syft-default-secret mongo: mongo-default-secret @@ -28,13 +24,12 @@ seaweedfs: queue: port: 5556 -workerBuilds: +registry: maxStorage: "10Gi" - mountInBackend: false syft: registry: "docker.io" - version: 0.8.4-beta.19 + version: 0.8.4-beta.21 node: settings: @@ -49,7 +44,16 @@ node: inMemoryWorkers: false defaultWorkerPoolCount: 1 +# ---------------------------------------- +# For Azure +# className: "azure-application-gateway" +# ---------------------------------------- +# For AWS +# className: "alb" +# ---------------------------------------- +# For GCE, https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress +# class: "gce" +# ---------------------------------------- ingress: - ingressClass: "" - # ingressClass: "azure/application-gateway" - # ingressClass: "gce" + class: null + className: null diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index 7082eb46295..7b3a4a1191d 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -31,7 +31,7 @@ data: RABBITMQ_VERSION: 3 SEAWEEDFS_VERSION: 3.59 DOCKER_IMAGE_SEAWEEDFS: chrislusf/seaweedfs:3.55 - VERSION: 0.8.4-beta.19 + VERSION: 0.8.4-beta.21 VERSION_HASH: unknown STACK_API_KEY: "" diff --git a/packages/grid/podman/podman-kube/podman-syft-kube.yaml b/packages/grid/podman/podman-kube/podman-syft-kube.yaml index 7ff11fc4494..2f1c09ad3fc 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube.yaml @@ -41,7 +41,7 @@ spec: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-backend:0.8.4-beta.19 + image: docker.io/openmined/grid-backend:0.8.4-beta.21 imagePullPolicy: IfNotPresent resources: {} tty: true @@ -57,7 +57,7 @@ spec: envFrom: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-frontend:0.8.4-beta.19 + image: docker.io/openmined/grid-frontend:0.8.4-beta.21 imagePullPolicy: IfNotPresent resources: {} tty: true diff --git a/packages/grid/seaweedfs/seaweedfs.dockerfile b/packages/grid/seaweedfs/seaweedfs.dockerfile index 3982e621c3b..5b53d14c0bc 100644 --- a/packages/grid/seaweedfs/seaweedfs.dockerfile +++ b/packages/grid/seaweedfs/seaweedfs.dockerfile @@ -1,6 +1,6 @@ ARG SEAWEEDFS_VERSION -FROM chrislusf/seaweedfs:${SEAWEEDFS_VERSION} +FROM chrislusf/seaweedfs:${SEAWEEDFS_VERSION}_large_disk WORKDIR / @@ -8,7 +8,8 @@ RUN apk update && \ apk add --no-cache python3 py3-pip ca-certificates bash COPY requirements.txt app.py / -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --break-system-packages -r requirements.txt + COPY --chmod=755 start.sh mount_command.sh / diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg index d1128b5281a..5e8d9c0fbed 100644 --- a/packages/hagrid/.bumpversion.cfg +++ b/packages/hagrid/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.106 +current_version = 0.3.107 tag = False tag_name = {new_version} commit = True diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 6fea1d62ae0..133b8837071 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -460,6 +460,13 @@ def clean(location: str) -> None: type=str, help="Set root password for s3 blob storage", ) +@click.option( + "--set-volume-size-limit-mb", + default=1024, + required=False, + type=click.IntRange(1024, 50000), + help="Set the volume size limit (in MBs)", +) def launch(args: TypeTuple[str], **kwargs: Any) -> None: verb = get_launch_verb() try: @@ -1258,6 +1265,7 @@ def create_launch_cmd( if parsed_kwargs["use_blob_storage"]: parsed_kwargs["set_s3_username"] = kwargs["set_s3_username"] parsed_kwargs["set_s3_password"] = kwargs["set_s3_password"] + parsed_kwargs["set_volume_size_limit_mb"] = kwargs["set_volume_size_limit_mb"] parsed_kwargs["node_count"] = ( int(kwargs["node_count"]) if "node_count" in kwargs else 1 @@ -2262,6 +2270,12 @@ def create_launch_docker_cmd( if "set_s3_password" in kwargs and kwargs["set_s3_password"] is not None: envs["S3_ROOT_PWD"] = kwargs["set_s3_password"] + if ( + "set_volume_size_limit_mb" in kwargs + and kwargs["set_volume_size_limit_mb"] is not None + ): + envs["S3_VOLUME_SIZE_MB"] = kwargs["set_volume_size_limit_mb"] + if "release" in kwargs: envs["RELEASE"] = kwargs["release"] diff --git a/packages/hagrid/hagrid/deps.py b/packages/hagrid/hagrid/deps.py index 8a74cc92291..c68cab019d2 100644 --- a/packages/hagrid/hagrid/deps.py +++ b/packages/hagrid/hagrid/deps.py @@ -42,7 +42,7 @@ from .version import __version__ LATEST_STABLE_SYFT = "0.8.3" -LATEST_BETA_SYFT = "0.8.4-beta.19" +LATEST_BETA_SYFT = "0.8.4-beta.21" DOCKER_ERROR = """ You are running an old version of docker, possibly on Linux. You need to install v2. diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index 69361b9bc05..0572eacecf5 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 -hagrid_version: 0.3.106 -syft_version: 0.8.4-beta.19 -dockerTag: 0.8.4-beta.19 +hagrid_version: 0.3.107 +syft_version: 0.8.4-beta.21 +dockerTag: 0.8.4-beta.21 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: cfea9e04c882a71797f2c7bf16e8deebb2295f23 +hash: 73aa361520477bfb282bf7517f66b4867ce7c838 target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py index 1a90352e02a..37151a1b796 100644 --- a/packages/hagrid/hagrid/version.py +++ b/packages/hagrid/hagrid/version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # HAGrid Version -__version__ = "0.3.106" +__version__ = "0.3.107" if __name__ == "__main__": print(__version__) diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py index e8c7cbf999c..377da3b1768 100644 --- a/packages/hagrid/setup.py +++ b/packages/hagrid/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup -__version__ = "0.3.106" +__version__ = "0.3.107" DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]} diff --git a/packages/syft/PYPI.md b/packages/syft/PYPI.md index 2265c5057ef..bee711a131a 100644 --- a/packages/syft/PYPI.md +++ b/packages/syft/PYPI.md @@ -78,14 +78,27 @@ SYFT_VERSION="" #### 4. Provisioning Helm Charts ```sh -helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik +helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className="traefik" ``` -### Azure or GCP Ingress +### Ingress Controllers +For Azure AKS + +```sh +helm install ... --set ingress.className="azure-application-gateway" ``` -helm install ... --set ingress.ingressClass="azure/application-gateway" -helm install ... --set ingress.ingressClass="gce" + +For AWS EKS + +```sh +helm install ... --set ingress.className="alb" +``` + +For Google GKE we need the [`gce` annotation](https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress) annotation. + +```sh +helm install ... --set ingress.class="gce" ``` ## Deploy to a Container Engine or Cloud diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index e098c8c9f4e..fb7c56a6e53 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = syft -version = attr: "0.8.4-beta.19" +version = attr: "0.8.4-beta.21" description = Perform numpy-like analysis on data that remains in someone elses server author = OpenMined author_email = info@openmined.org @@ -62,7 +62,7 @@ syft = numpy>=1.23.5,<=1.24.4 pandas==1.5.3 docker==6.1.3 - kr8s==0.13.0 + kr8s==0.13.1 PyYAML==6.0.1 azure-storage-blob==12.19 diff --git a/packages/syft/src/syft/VERSION b/packages/syft/src/syft/VERSION index d8f01b13857..1f184fb3f81 100644 --- a/packages/syft/src/syft/VERSION +++ b/packages/syft/src/syft/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.21" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index f83cc10e6d9..67b416ccb96 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.21" # stdlib import pathlib diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index eacc52533cd..d9d8cc3ae4e 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -191,9 +191,10 @@ def upload_files( try: result = [] - for file in tqdm(expanded_file_list): + for file in expanded_file_list: if not isinstance(file, BlobFile): file = BlobFile(path=file, file_name=file.name) + print("Uploading", file.file_name) if not file.uploaded: file.upload_to_blobstorage(self) result.append(file) diff --git a/packages/syft/src/syft/custom_worker/builder_docker.py b/packages/syft/src/syft/custom_worker/builder_docker.py index e1f24520c25..3f7f16cf185 100644 --- a/packages/syft/src/syft/custom_worker/builder_docker.py +++ b/packages/syft/src/syft/custom_worker/builder_docker.py @@ -1,7 +1,6 @@ # stdlib import contextlib import io -import json from pathlib import Path from typing import Iterable from typing import Optional @@ -10,16 +9,16 @@ import docker # relative +from .builder_types import BUILD_IMAGE_TIMEOUT_SEC from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult +from .utils import iterator_to_string __all__ = ["DockerBuilder"] class DockerBuilder(BuilderBase): - BUILD_MAX_WAIT = 30 * 60 - def build_image( self, tag: str, @@ -40,7 +39,7 @@ def build_image( with contextlib.closing(docker.from_env()) as client: image_result, logs = client.images.build( tag=tag, - timeout=self.BUILD_MAX_WAIT, + timeout=BUILD_IMAGE_TIMEOUT_SEC, buildargs=buildargs, **kwargs, ) @@ -70,13 +69,4 @@ def push_image( return ImagePushResult(logs=result, exit_code=0) def _parse_output(self, log_iterator: Iterable) -> str: - log = "" - for line in log_iterator: - for item in line.values(): - if isinstance(item, str): - log += item - elif isinstance(item, dict): - log += json.dumps(item) + "\n" - else: - log += str(item) - return log + return iterator_to_string(iterator=log_iterator) diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py index a80df6470cd..1be16d3c0ac 100644 --- a/packages/syft/src/syft/custom_worker/builder_k8s.py +++ b/packages/syft/src/syft/custom_worker/builder_k8s.py @@ -1,24 +1,27 @@ # stdlib from hashlib import sha256 -import os from pathlib import Path from typing import Dict from typing import List from typing import Optional # third party -import kr8s -from kr8s.objects import APIObject from kr8s.objects import ConfigMap from kr8s.objects import Job +from kr8s.objects import Secret # relative +from .builder_types import BUILD_IMAGE_TIMEOUT_SEC from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult -from .k8s import BUILD_OUTPUT_PVC +from .builder_types import PUSH_IMAGE_TIMEOUT_SEC +from .k8s import INTERNAL_REGISTRY_HOST from .k8s import JOB_COMPLETION_TTL from .k8s import KUBERNETES_NAMESPACE +from .k8s import KubeUtils +from .k8s import get_kr8s_client +from .utils import ImageUtils __all__ = ["KubernetesBuilder"] @@ -28,8 +31,10 @@ class BuildFailed(Exception): class KubernetesBuilder(BuilderBase): + COMPONENT = "builder" + def __init__(self): - self.client = kr8s.api(namespace=KUBERNETES_NAMESPACE) + self.client = get_kr8s_client() def build_image( self, @@ -39,6 +44,9 @@ def build_image( buildargs: Optional[dict] = None, **kwargs, ) -> ImageBuildResult: + image_digest = None + logs = None + config = None job_id = self._new_job_id(tag) if dockerfile: @@ -46,28 +54,31 @@ def build_image( elif dockerfile_path: dockerfile = dockerfile_path.read_text() - # Create a ConfigMap with the Dockerfile - config = self._create_build_config(job_id, dockerfile) - config.refresh() - - # Create and start the job - job = self._create_kaniko_build_job( - job_id=job_id, - tag=tag, - job_config=config, - build_args=buildargs, - ) - try: + # Create a ConfigMap with the Dockerfile + config = self._create_build_config(job_id, dockerfile) + config.refresh() + + # Create and start the job + job = self._create_kaniko_build_job( + job_id=job_id, + tag=tag, + job_config=config, + build_args=buildargs, + ) + # wait for job to complete/fail - job.wait(["condition=Complete", "condition=Failed"]) + job.wait( + ["condition=Complete", "condition=Failed"], + timeout=BUILD_IMAGE_TIMEOUT_SEC, + ) # get logs logs = self._get_logs(job) image_digest = self._get_image_digest(job) if not image_digest: - exit_code = self._get_container_exit_code(job) + exit_code = self._get_exit_code(job) raise BuildFailed( "Failed to build the image. " f"Kaniko exit code={exit_code}. " @@ -78,7 +89,7 @@ def build_image( raise finally: # don't delete the job, kubernetes will gracefully do that for us - config.delete() + config and config.delete(propagation_policy="Foreground") return ImageBuildResult( image_hash=image_digest, @@ -93,18 +104,38 @@ def push_image( registry_url: str, **kwargs, ) -> ImagePushResult: - # Create and start the job + exit_code = 1 + logs = None job_id = self._new_job_id(tag) - job = self._create_push_job( - job_id=job_id, - tag=tag, - username=username, - password=password, - registry_url=registry_url, - ) - job.wait(["condition=Complete", "condition=Failed"]) - exit_code = self._get_container_exit_code(job)[0] - return ImagePushResult(logs=self._get_logs(job), exit_code=exit_code) + push_secret = None + + try: + push_secret = self._create_push_secret( + id=job_id, + url=registry_url, + username=username, + password=password, + ) + push_secret.refresh() + + job = self._create_push_job( + job_id=job_id, + tag=tag, + push_secret=push_secret, + ) + + job.wait( + ["condition=Complete", "condition=Failed"], + timeout=PUSH_IMAGE_TIMEOUT_SEC, + ) + exit_code = self._get_exit_code(job)[0] + logs = self._get_logs(job) + except Exception: + raise + finally: + push_secret and push_secret.delete(propagation_policy="Foreground") + + return ImagePushResult(logs=logs, exit_code=exit_code) def _new_job_id(self, tag: str) -> str: return self._get_tag_hash(tag)[:16] @@ -115,49 +146,35 @@ def _get_tag_hash(self, tag: str) -> str: def _get_image_digest(self, job: Job) -> Optional[str]: selector = {"batch.kubernetes.io/job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) - for pod in pods: - for container_status in pod.status.containerStatuses: - if container_status.state.terminated.exitCode != 0: - continue - return container_status.state.terminated.message - return None - - def _get_container_exit_code(self, job: Job) -> List[int]: + return KubeUtils.get_container_exit_message(pods) + + def _get_exit_code(self, job: Job) -> List[int]: selector = {"batch.kubernetes.io/job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) - exit_codes = [] - for pod in pods: - for container_status in pod.status.containerStatuses: - exit_codes.append(container_status.state.terminated.exitCode) - return exit_codes + return KubeUtils.get_container_exit_code(pods) def _get_logs(self, job: Job) -> str: selector = {"batch.kubernetes.io/job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) - logs = [] - for pod in pods: - logs.append(f"----------Logs for pod={pod.metadata.name}----------") - for log in pod.logs(): - logs.append(log) - - return "\n".join(logs) - - def _check_success(self, job: Job) -> bool: - # TODO - return True + return KubeUtils.get_logs(pods) def _create_build_config(self, job_id: str, dockerfile: str) -> ConfigMap: config_map = ConfigMap( { "metadata": { "name": f"build-{job_id}", + "labels": { + "app.kubernetes.io/name": KUBERNETES_NAMESPACE, + "app.kubernetes.io/component": KubernetesBuilder.COMPONENT, + "app.kubernetes.io/managed-by": "kr8s", + }, }, "data": { "Dockerfile": dockerfile, }, } ) - return self._create_or_get(config_map) + return KubeUtils.create_or_get(config_map) def _create_kaniko_build_job( self, @@ -168,14 +185,20 @@ def _create_kaniko_build_job( ) -> Job: # for push build_args = build_args or {} - tag_hash = self._get_tag_hash(tag) + build_args_list = [] + + internal_tag = ImageUtils.change_registry(tag, registry=INTERNAL_REGISTRY_HOST) + + for k, v in build_args.items(): + build_args_list.append(f'--build-arg="{k}={v}"') + job = Job( { "metadata": { "name": f"build-{job_id}", "labels": { "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": "builder", + "app.kubernetes.io/component": KubernetesBuilder.COMPONENT, "app.kubernetes.io/managed-by": "kr8s", }, }, @@ -190,32 +213,32 @@ def _create_kaniko_build_job( "name": "kaniko", "image": "gcr.io/kaniko-project/executor:latest", "args": [ - # build_args "--dockerfile=Dockerfile", "--context=dir:///workspace", - f"--destination={tag}", + f"--destination={internal_tag}", # Disabling --reproducible because it eats up a lot of CPU+RAM # https://github.com/GoogleContainerTools/kaniko/issues/1960 # https://github.com/GoogleContainerTools/kaniko/pull/2477 # "--reproducible", - # Build outputs - f"--tar-path=/output/{tag_hash}.tar", + # cache args + "--cache=true", + "--cache-copy-layers", + "--cache-run-layers", + f"--cache-repo={INTERNAL_REGISTRY_HOST}/builder-cache", + # outputs args "--digest-file=/dev/termination-log", - "--no-push", # other kaniko conf + f"--insecure-registry={INTERNAL_REGISTRY_HOST}", + f"--skip-tls-verify-registry={INTERNAL_REGISTRY_HOST}", "--log-format=text", "--verbosity=info", - ], + ] + + build_args_list, "volumeMounts": [ { "name": "build-input", "mountPath": "/workspace", }, - { - "name": "build-output", - "mountPath": "/output", - "readOnly": False, - }, ], "resources": { "requests": { @@ -237,12 +260,6 @@ def _create_kaniko_build_job( "name": job_config.metadata.name, }, }, - { - "name": "build-output", - "persistentVolumeClaim": { - "claimName": BUILD_OUTPUT_PVC, - }, - }, ], } }, @@ -250,33 +267,25 @@ def _create_kaniko_build_job( } ) - return self._create_or_get(job) + return KubeUtils.create_or_get(job) def _create_push_job( self, job_id: str, tag: str, - username: str, - password: str, - registry_url: Optional[str] = None, + push_secret: Secret, ) -> Job: - tag_hash = self._get_tag_hash(tag) - registry_url = registry_url or tag.split("/")[0] - - extra_flags = "" - if os.getenv("DEV_MODE") == "True": - extra_flags = "--insecure" + internal_tag = ImageUtils.change_registry(tag, registry=INTERNAL_REGISTRY_HOST) + internal_reg, internal_repo, _ = ImageUtils.parse_tag(internal_tag) run_cmds = [ - "echo Logging in to $REG_URL with user $REG_USERNAME...", - # login to registry - "crane auth login $REG_URL -u $REG_USERNAME -p $REG_PASSWORD", # push with credentials - "echo Pushing image....", - f"crane push --image-refs /dev/termination-log {extra_flags} /output/{tag_hash}.tar {tag}", - # cleanup built tarfile - "echo Cleaning up tar....", - f"rm /output/{tag_hash}.tar", + "echo Pushing image...", + f"crane copy {internal_tag} {tag}", + # cleanup image from internal registry + "echo Cleaning up...", + f"IMG_DIGEST=$(crane digest {internal_tag})", + f"crane delete {internal_reg}/{internal_repo}@$IMG_DIGEST; echo Done", ] job = Job( @@ -286,7 +295,7 @@ def _create_push_job( "name": f"push-{job_id}", "labels": { "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": "builder", + "app.kubernetes.io/component": KubernetesBuilder.COMPONENT, "app.kubernetes.io/managed-by": "kr8s", }, }, @@ -301,27 +310,14 @@ def _create_push_job( "name": "crane", # debug is needed for "sh" to be available "image": "gcr.io/go-containerregistry/crane:debug", - "env": [ - { - "name": "REG_URL", - "value": registry_url, - }, - { - "name": "REG_USERNAME", - "value": username, - }, - { - "name": "REG_PASSWORD", - "value": password, - }, - ], "command": ["sh"], "args": ["-c", " && ".join(run_cmds)], "volumeMounts": [ { - "name": "build-output", - "mountPath": "/output", - "readOnly": False, + "name": "push-secret", + "mountPath": "/root/.docker/config.json", + "subPath": "config.json", + "readOnly": True, }, ], "resources": { @@ -339,9 +335,15 @@ def _create_push_job( ], "volumes": [ { - "name": "build-output", - "persistentVolumeClaim": { - "claimName": BUILD_OUTPUT_PVC, + "name": "push-secret", + "secret": { + "secretName": push_secret.metadata.name, + "items": [ + { + "key": ".dockerconfigjson", + "path": "config.json", + }, + ], }, }, ], @@ -350,11 +352,15 @@ def _create_push_job( }, } ) - return self._create_or_get(job) - - def _create_or_get(self, obj: APIObject) -> APIObject: - if not obj.exists(): - obj.create() - else: - obj.refresh() - return obj + return KubeUtils.create_or_get(job) + + def _create_push_secret(self, id: str, url: str, username: str, password: str): + return KubeUtils.create_dockerconfig_secret( + secret_name=f"push-secret-{id}", + component=KubernetesBuilder.COMPONENT, + registries=[ + # TODO: authorize internal registry? + (INTERNAL_REGISTRY_HOST, "username", id), + (url, username, password), + ], + ) diff --git a/packages/syft/src/syft/custom_worker/builder_types.py b/packages/syft/src/syft/custom_worker/builder_types.py index 53c27788791..8007bf476e9 100644 --- a/packages/syft/src/syft/custom_worker/builder_types.py +++ b/packages/syft/src/syft/custom_worker/builder_types.py @@ -7,7 +7,17 @@ # third party from pydantic import BaseModel -__all__ = ["BuilderBase", "ImageBuildResult", "ImagePushResult"] +__all__ = [ + "BuilderBase", + "ImageBuildResult", + "ImagePushResult", + "BUILD_IMAGE_TIMEOUT_SEC", + "PUSH_IMAGE_TIMEOUT_SEC", +] + + +BUILD_IMAGE_TIMEOUT_SEC = 30 * 60 +PUSH_IMAGE_TIMEOUT_SEC = 10 * 60 class ImageBuildResult(BaseModel): diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 7c3867a5392..c54d4f77c40 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -1,5 +1,7 @@ # stdlib +import contextlib from hashlib import sha256 +import io from pathlib import Path from typing import Any from typing import Dict @@ -8,6 +10,7 @@ from typing import Union # third party +import docker from packaging import version from pydantic import validator from typing_extensions import Self @@ -15,7 +18,10 @@ # relative from ..serde.serializable import serializable +from ..service.response import SyftError +from ..service.response import SyftSuccess from ..types.base import SyftBaseModel +from .utils import iterator_to_string PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -159,3 +165,20 @@ def __str__(self) -> str: def set_description(self, description_text: str) -> None: self.description = description_text + + def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: + try: + with contextlib.closing(docker.from_env()) as client: + if not client.ping(): + return SyftError( + "Cannot reach docker server. Please check if docker is running." + ) + + kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8")) + _, logs = client.images.build( + tag=tag, + **kwargs, + ) + return SyftSuccess(message=iterator_to_string(iterator=logs)) + except Exception as e: + return SyftError(message=f"Failed to build: {e}") diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 491d7333b38..fb777f6ec6c 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -1,24 +1,36 @@ # stdlib +import base64 from enum import Enum +from functools import cache +import json import os +from typing import Dict +from typing import Iterable +from typing import List from typing import Optional +from typing import Tuple +from typing import Union # third party -from kr8s._data_utils import list_dict_unpack +import kr8s +from kr8s.objects import APIObject +from kr8s.objects import Pod +from kr8s.objects import Secret from pydantic import BaseModel # Time after which Job will be deleted JOB_COMPLETION_TTL = 60 -# Persistent volume claim for storing build output -BUILD_OUTPUT_PVC = "worker-builds" - # Kubernetes namespace KUBERNETES_NAMESPACE = os.getenv("K8S_NAMESPACE", "syft") # Kubernetes runtime flag IN_KUBERNETES = os.getenv("CONTAINER_HOST") == "k8s" +# Internal registry URL +DEFAULT_INTERNAL_REGISTRY = f"registry.{KUBERNETES_NAMESPACE}.svc.cluster.local" +INTERNAL_REGISTRY_HOST = os.getenv("INTERNAL_REGISTRY_HOST", DEFAULT_INTERNAL_REGISTRY) + class PodPhase(Enum): Pending = "Pending" @@ -36,7 +48,7 @@ class PodCondition(BaseModel): @classmethod def from_conditions(cls, conditions: list): - pod_cond = list_dict_unpack(conditions, key="type", value="status") + pod_cond = KubeUtils.list_dict_unpack(conditions, key="type", value="status") pod_cond_flags = {k: v == "True" for k, v in pod_cond.items()} return cls( pod_scheduled=pod_cond_flags.get("PodScheduled", False), @@ -82,3 +94,167 @@ def from_status_dict(cls: "PodStatus", status: dict): status.get("containerStatuses", {})[0] ), ) + + +@cache +def get_kr8s_client() -> kr8s.Api: + if not IN_KUBERNETES: + raise RuntimeError("Not inside a kubernetes environment") + return kr8s.api(namespace=KUBERNETES_NAMESPACE) + + +class KubeUtils: + """ + This class contains utility functions for interacting with kubernetes objects. + + DO NOT call `get_kr8s_client()` inside this class, instead pass it as an argument to the functions. + This is to avoid calling these functions on resources across namespaces! + """ + + @staticmethod + def resolve_pod(client: kr8s.Api, pod: Union[str, Pod]) -> Optional[Pod]: + """Return the first pod that matches the given name""" + if isinstance(pod, Pod): + return pod + + for _pod in client.get("pods", pod): + return _pod + + @staticmethod + def get_logs(pods: List[Pod]): + """Combine and return logs for all the pods as string""" + logs = [] + for pod in pods: + logs.append(f"----------Logs for pod={pod.metadata.name}----------") + for log in pod.logs(): + logs.append(log) + + return "\n".join(logs) + + @staticmethod + def get_pod_status(pod: Pod) -> Optional[PodStatus]: + """Map the status of the given pod to PodStatuss.""" + if not pod: + return None + return PodStatus.from_status_dict(pod.status) + + @staticmethod + def get_pod_env(pod: Pod) -> Optional[List[Dict]]: + """Return the environment variables of the first container in the pod.""" + if not pod: + return + + for container in pod.spec.containers: + return container.env.to_list() + + @staticmethod + def get_container_exit_code(pods: List[Pod]) -> List[int]: + """Return the exit codes of all the containers in the given pods.""" + exit_codes = [] + for pod in pods: + for container_status in pod.status.containerStatuses: + exit_codes.append(container_status.state.terminated.exitCode) + return exit_codes + + @staticmethod + def get_container_exit_message(pods: List[Pod]) -> Optional[str]: + """Return the exit message of the first container that exited with non-zero code.""" + for pod in pods: + for container_status in pod.status.containerStatuses: + if container_status.state.terminated.exitCode != 0: + continue + return container_status.state.terminated.message + return None + + @staticmethod + def b64encode_secret(data: str) -> str: + """Convert the data to base64 encoded string for Secret.""" + return base64.b64encode(data.encode()).decode() + + @staticmethod + def create_dockerconfig_secret( + secret_name: str, + component: str, + registries: Iterable[Tuple[str, str, str]], + ) -> Secret: + auths = {} + + for url, uname, passwd in registries: + auths[url] = { + "username": uname, + "password": passwd, + "auth": KubeUtils.b64encode_secret(f"{uname}:{passwd}"), + } + + config_str = json.dumps({"auths": auths}) + + return KubeUtils.create_secret( + secret_name=secret_name, + type="kubernetes.io/dockerconfigjson", + component=component, + data={ + ".dockerconfigjson": KubeUtils.b64encode_secret(config_str), + }, + ) + + @staticmethod + def create_secret( + secret_name: str, + type: str, + component: str, + data: str, + encoded=True, + ) -> Secret: + if not encoded: + for k, v in data.items(): + data[k] = KubeUtils.b64encode_secret(v) + + secret = Secret( + { + "metadata": { + "name": secret_name, + "labels": { + "app.kubernetes.io/name": KUBERNETES_NAMESPACE, + "app.kubernetes.io/component": component, + "app.kubernetes.io/managed-by": "kr8s", + }, + }, + "type": type, + "data": data, + } + ) + return KubeUtils.create_or_get(secret) + + @staticmethod + def create_or_get(obj: APIObject) -> APIObject: + if obj.exists(): + obj.refresh() + else: + obj.create() + return obj + + @staticmethod + def patch_env_vars(env_list: List[Dict], env_dict: Dict) -> List[Dict]: + """Patch kubernetes pod environment variables in the list with the provided dictionary.""" + + # update existing + for item in env_list: + k = item["name"] + if k in env_dict: + v = env_dict.pop(k) + item["value"] = v + + # append remaining + for k, v in env_dict.items(): + env_list.append({"name": k, "value": v}) + + return env_list + + @staticmethod + def list_dict_unpack( + input_list: List[Dict], + key: str = "key", + value: str = "value", + ) -> Dict: + # Snapshot from kr8s._data_utils + return {i[key]: i[value] for i in input_list} diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 0838836989b..3b35830c0f4 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -1,62 +1,73 @@ # stdlib -import base64 -import copy -import json -import os -from time import sleep +from typing import Dict from typing import List from typing import Optional from typing import Union # third party -import kr8s -from kr8s.objects import APIObject from kr8s.objects import Pod from kr8s.objects import Secret from kr8s.objects import StatefulSet # relative from .k8s import KUBERNETES_NAMESPACE +from .k8s import KubeUtils from .k8s import PodStatus +from .k8s import get_kr8s_client + +JSONPATH_AVAILABLE_REPLICAS = "{.status.availableReplicas}" +CREATE_POOL_TIMEOUT_SEC = 60 +SCALE_POOL_TIMEOUT_SEC = 60 class KubernetesRunner: def __init__(self): - self.client = kr8s.api(namespace=KUBERNETES_NAMESPACE) + self.client = get_kr8s_client() def create_pool( self, pool_name: str, tag: str, replicas: int = 1, - env_vars: Optional[dict] = None, + env_vars: Optional[List[Dict]] = None, + mount_secrets: Optional[Dict] = None, reg_username: Optional[str] = None, reg_password: Optional[str] = None, reg_url: Optional[str] = None, **kwargs, ) -> StatefulSet: - # create pull secret if registry credentials are passed - pull_secret = None - if reg_username and reg_password and reg_url: - pull_secret = self._create_image_pull_secret( - pool_name, - reg_username, - reg_password, - reg_url, + try: + # create pull secret if registry credentials are passed + pull_secret = None + if reg_username and reg_password and reg_url: + pull_secret = self._create_image_pull_secret( + pool_name, + reg_username, + reg_password, + reg_url, + ) + + # create a stateful set deployment + deployment = self._create_stateful_set( + pool_name=pool_name, + tag=tag, + replicas=replicas, + env_vars=env_vars, + mount_secrets=mount_secrets, + pull_secret=pull_secret, + **kwargs, ) - # create a stateful set deployment - deployment = self._create_stateful_set( - pool_name, - tag, - replicas, - env_vars, - pull_secret=pull_secret, - **kwargs, - ) - - # wait for replicas to be available and ready - self.wait(deployment, available_replicas=replicas) + # wait for replicas to be available and ready + deployment.wait( + f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}", + timeout=CREATE_POOL_TIMEOUT_SEC, + ) + except Exception: + raise + finally: + if pull_secret: + pull_secret.delete(propagation_policy="Foreground") # return return deployment @@ -66,9 +77,15 @@ def scale_pool(self, pool_name: str, replicas: int) -> Optional[StatefulSet]: if not deployment: return None deployment.scale(replicas) - self.wait(deployment, available_replicas=replicas) + deployment.wait( + f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}", + timeout=SCALE_POOL_TIMEOUT_SEC, + ) return deployment + def exists(self, pool_name: str) -> bool: + return bool(self.get_pool(pool_name)) + def get_pool(self, pool_name: str) -> Optional[StatefulSet]: selector = {"app.kubernetes.io/component": pool_name} for _set in self.client.get("statefulsets", label_selector=selector): @@ -78,21 +95,21 @@ def get_pool(self, pool_name: str) -> Optional[StatefulSet]: def delete_pool(self, pool_name: str) -> bool: selector = {"app.kubernetes.io/component": pool_name} for _set in self.client.get("statefulsets", label_selector=selector): - _set.delete() + _set.delete(propagation_policy="Foreground") for _secret in self.client.get("secrets", label_selector=selector): - _secret.delete() + _secret.delete(propagation_policy="Foreground") return True def delete_pod(self, pod_name: str) -> bool: pods = self.client.get("pods", pod_name) for pod in pods: - pod.delete() + pod.delete(propagation_policy="Foreground") return True return False - def get_pods(self, pool_name: str) -> List[Pod]: + def get_pool_pods(self, pool_name: str) -> List[Pod]: selector = {"app.kubernetes.io/component": pool_name} pods = self.client.get("pods", label_selector=selector) if len(pods) > 0: @@ -101,62 +118,15 @@ def get_pods(self, pool_name: str) -> List[Pod]: def get_pod_logs(self, pod_name: str) -> str: pods = self.client.get("pods", pod_name) - logs = [] - for pod in pods: - logs.append(f"----------Logs for pod={pod.metadata.name}----------") - for log in pod.logs(): - logs.append(log) - - return "\n".join(logs) + return KubeUtils.get_logs(pods) def get_pod_status(self, pod: Union[str, Pod]) -> Optional[PodStatus]: - if isinstance(pod, str): - pods = self.client.get("pods", pod) - if len(pods) == 0: - return None - pod = pods[0] - else: - pod.refresh() - - return PodStatus.from_status_dict(pod.status) - - def wait( - self, - deployment: StatefulSet, - available_replicas: int, - timeout: int = 300, - ) -> None: - # TODO: Report wait('jsonpath=') bug to kr8s - # Until then this is the substitute implementation - - if available_replicas <= 0: - return + pod = KubeUtils.resolve_pod(self.client, pod) + return KubeUtils.get_pod_status(pod) - while True: - if timeout == 0: - raise TimeoutError("Timeout waiting for replicas") - - deployment.refresh() - if deployment.status.availableReplicas == available_replicas: - break - - timeout -= 1 - sleep(1) - - def _current_pod_name(self) -> str: - env_val = os.getenv("K8S_POD_NAME") - if env_val: - return env_val - - selector = {"app.kubernetes.io/component": "backend"} - for pod in self.client.get("pods", label_selector=selector): - return pod.name - - def _get_obj_from_list(self, objs: List[dict], name: str) -> dict: - """Helper function extract kubernetes object from list by name""" - for obj in objs: - if obj.name == name: - return obj + def get_pod_env_vars(self, pod: Union[str, Pod]) -> Optional[List[Dict]]: + pod = KubeUtils.resolve_pod(self.client, pod) + return KubeUtils.get_pod_env(pod) def _create_image_pull_secret( self, @@ -166,67 +136,49 @@ def _create_image_pull_secret( reg_url: str, **kwargs, ): - _secret = Secret( - { - "metadata": { - "name": f"pull-secret-{pool_name}", - "labels": { - "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": pool_name, - "app.kubernetes.io/managed-by": "kr8s", - }, - }, - "type": "kubernetes.io/dockerconfigjson", - "data": { - ".dockerconfigjson": self._create_dockerconfig_json( - reg_username, - reg_password, - reg_url, - ) - }, - } + return KubeUtils.create_dockerconfig_secret( + secret_name=f"pull-secret-{pool_name}", + component=pool_name, + registries=[ + (reg_url, reg_username, reg_password), + ], ) - return self._create_or_get(_secret) - def _create_stateful_set( self, pool_name: str, tag: str, replicas=1, - env_vars: Optional[dict] = None, + env_vars: Optional[List[Dict]] = None, + mount_secrets: Optional[Dict] = None, pull_secret: Optional[Secret] = None, **kwargs, ) -> StatefulSet: """Create a stateful set for a pool""" - env_vars = env_vars or {} + volumes = [] + volume_mounts = [] pull_secret_obj = None - - _pod = Pod.get(self._current_pod_name()) - - creds_volume = self._get_obj_from_list( - objs=_pod.spec.volumes, - name="credentials-data", - ) - creds_volume_mount = self._get_obj_from_list( - objs=_pod.spec.containers[0].volumeMounts, - name="credentials-data", - ) - - env = _pod.spec.containers[0].env.to_list() - env_clone = copy.deepcopy(env) - - # update existing - for item in env_clone: - k = item["name"] - if k in env_vars: - v = env_vars.pop(k) - item["value"] = v - - # append remaining - for k, v in env_vars.items(): - env_clone.append({"name": k, "value": v}) + env_vars = env_vars or [] + + if mount_secrets: + for secret_name, mount_opts in mount_secrets.items(): + volumes.append( + { + "name": secret_name, + "secret": { + "secretName": secret_name, + }, + } + ) + volume_mounts.append( + { + "name": secret_name, + "mountPath": mount_opts.get("mountPath"), + "subPath": mount_opts.get("subPath"), + "readOnly": True, + } + ) if pull_secret: pull_secret_obj = [ @@ -265,41 +217,15 @@ def _create_stateful_set( "name": pool_name, "imagePullPolicy": "IfNotPresent", "image": tag, - "env": env_clone, - "volumeMounts": [creds_volume_mount], + "env": env_vars, + "volumeMounts": volume_mounts, } ], - "volumes": [creds_volume], + "volumes": volumes, "imagePullSecrets": pull_secret_obj, }, }, }, } ) - return self._create_or_get(stateful_set) - - def _create_or_get(self, obj: APIObject) -> APIObject: - if not obj.exists(): - obj.create() - else: - obj.refresh() - return obj - - def _create_dockerconfig_json( - self, - reg_username: str, - reg_password: str, - reg_url: str, - ): - config = { - "auths": { - reg_url: { - "username": reg_username, - "password": reg_password, - "auth": base64.b64encode( - f"{reg_username}:{reg_password}".encode() - ).decode(), - } - } - } - return base64.b64encode(json.dumps(config).encode()).decode() + return KubeUtils.create_or_get(stateful_set) diff --git a/packages/syft/src/syft/custom_worker/utils.py b/packages/syft/src/syft/custom_worker/utils.py new file mode 100644 index 00000000000..597e4bb6aff --- /dev/null +++ b/packages/syft/src/syft/custom_worker/utils.py @@ -0,0 +1,39 @@ +# stdlib +import json +from typing import Iterable +from typing import Optional +from typing import Tuple + + +def iterator_to_string(iterator: Iterable) -> str: + log = "" + for line in iterator: + for item in line.values(): + if isinstance(item, str): + log += item + elif isinstance(item, dict): + log += json.dumps(item) + "\n" + else: + log += str(item) + return log + + +class ImageUtils: + @staticmethod + def parse_tag(tag: str) -> Tuple[Optional[str], str, str]: + url, tag = tag.rsplit(":", 1) + args = url.rsplit("/", 2) + + if len(args) == 3: + registry = args[0] + repo = "/".join(args[1:]) + else: + registry = None + repo = "/".join(args) + + return registry, repo, tag + + @staticmethod + def change_registry(tag: str, registry: str) -> str: + _, repo, tag = ImageUtils.parse_tag(tag) + return f"{registry}/{repo}:{tag}" diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 2bdb32445b1..681033fc46d 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -296,6 +296,10 @@ def __init__( sqlite_path: Optional[str] = None, blob_storage_config: Optional[BlobStorageConfig] = None, queue_config: Optional[QueueConfig] = None, + queue_port: Optional[int] = None, + n_consumers: int = 0, + create_producer: bool = False, + thread_workers: bool = False, node_side_type: Union[str, NodeSideType] = NodeSideType.HIGH_SIDE, enable_warnings: bool = False, dev_mode: bool = False, @@ -398,7 +402,15 @@ def __init__( self.post_init() self.create_initial_settings(admin_email=root_email) - self.init_queue_manager(queue_config=queue_config) + self.queue_config = self.create_queue_config( + n_consumers=n_consumers, + create_producer=create_producer, + thread_workers=thread_workers, + queue_port=queue_port, + queue_config=queue_config, + ) + + self.init_queue_manager(queue_config=self.queue_config) self.init_blob_storage(config=blob_storage_config) @@ -451,20 +463,40 @@ def stop(self): def close(self): self.stop() - def init_queue_manager(self, queue_config: Optional[QueueConfig]): - queue_config_ = ZMQQueueConfig() if queue_config is None else queue_config - self.queue_config = queue_config_ + def create_queue_config( + self, + n_consumers: int, + create_producer: bool, + thread_workers: bool, + queue_port: Optional[int], + queue_config: Optional[QueueConfig], + ) -> QueueConfig: + if queue_config: + queue_config_ = queue_config + elif queue_port is not None or n_consumers > 0 or create_producer: + queue_config_ = ZMQQueueConfig( + client_config=ZMQClientConfig( + create_producer=create_producer, + queue_port=queue_port, + n_consumers=n_consumers, + ), + thread_workers=thread_workers, + ) + else: + queue_config_ = ZMQQueueConfig() + + return queue_config_ + def init_queue_manager(self, queue_config: QueueConfig): MessageHandlers = [APICallMessageHandler] - if self.is_subprocess: return - self.queue_manager = QueueManager(config=queue_config_) + self.queue_manager = QueueManager(config=queue_config) for message_handler in MessageHandlers: queue_name = message_handler.queue_name # client config - if getattr(queue_config_.client_config, "create_producer", True): + if getattr(queue_config.client_config, "create_producer", True): context = AuthedServiceContext( node=self, credentials=self.verify_key, @@ -479,16 +511,16 @@ def init_queue_manager(self, queue_config: Optional[QueueConfig]): producer.run() address = producer.address else: - port = queue_config_.client_config.queue_port + port = queue_config.client_config.queue_port if port is not None: address = get_queue_address(port) else: address = None - if address is None and queue_config_.client_config.n_consumers > 0: + if address is None and queue_config.client_config.n_consumers > 0: raise ValueError("address unknown for consumers") - service_name = queue_config_.client_config.consumer_service + service_name = queue_config.client_config.consumer_service if not service_name: # Create consumers for default worker pool @@ -537,7 +569,6 @@ def named( node_side_type: Union[str, NodeSideType] = NodeSideType.HIGH_SIDE, enable_warnings: bool = False, n_consumers: int = 0, - consumer_service: Optional[str] = None, thread_workers: bool = False, create_producer: bool = False, queue_port: Optional[int] = None, @@ -600,19 +631,6 @@ def named( client_config=blob_client_config ) - if queue_port is not None or n_consumers > 0 or create_producer: - queue_config = ZMQQueueConfig( - client_config=ZMQClientConfig( - create_producer=create_producer, - queue_port=queue_port, - n_consumers=n_consumers, - consumer_service=consumer_service, - ), - thread_workers=thread_workers, - ) - else: - queue_config = None - return cls( name=name, id=uid, @@ -624,7 +642,10 @@ def named( node_side_type=node_side_type, enable_warnings=enable_warnings, blob_storage_config=blob_storage_config, - queue_config=queue_config, + queue_port=queue_port, + n_consumers=n_consumers, + thread_workers=thread_workers, + create_producer=create_producer, dev_mode=dev_mode, migrate=migrate, in_memory_workers=in_memory_workers, diff --git a/packages/syft/src/syft/node/server.py b/packages/syft/src/syft/node/server.py index e49c21b9740..fc7bbb2bc8d 100644 --- a/packages/syft/src/syft/node/server.py +++ b/packages/syft/src/syft/node/server.py @@ -125,6 +125,8 @@ async def _run_uvicorn( migrate=True, in_memory_workers=in_memory_workers, queue_port=queue_port, + create_producer=create_producer, + n_consumers=n_consumers, ) router = make_routes(worker=worker) app = make_app(worker.name, router=router) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 5c307fe3246..cfb8c1d575e 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -235,6 +235,13 @@ "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", "action": "remove" } + }, + "SeaweedFSBlobDeposit": { + "2": { + "version": 2, + "hash": "07d84a95324d95d9c868cd7d1c33c908f77aa468671d76c144586aab672bcbb5", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 020830d2508..a449845115d 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -462,6 +462,10 @@ def input_policy(self) -> Optional[InputPolicy]: print(f"Failed to deserialize custom input policy state. {e}") return None + @property + def output_policy_approved(self): + return self.status.approved + @property def output_policy(self) -> Optional[OutputPolicy]: if not self.status.approved: diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 20c8630c231..da409b1dac0 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -311,7 +311,7 @@ def is_execution_allowed(self, code, context, output_policy): # Check if the user has permission to execute the code. elif not (has_code_permission := self.has_code_permission(code, context)): return has_code_permission - elif code.output_policy is None: + elif not code.output_policy_approved: return SyftError("Output policy not approved", code) elif not output_policy.valid: return output_policy.valid @@ -399,9 +399,9 @@ def _call( code=code, context=context, output_policy=output_policy ) if not can_execute: - if output_policy is None: + if not code.output_policy_approved: return Err( - "UserCodeStatus.DENIED: Function has no output policy" + "Execution denied: Your code is waiting for approval" ) if not (is_valid := output_policy.valid): if ( diff --git a/packages/syft/src/syft/service/worker/image_identifier.py b/packages/syft/src/syft/service/worker/image_identifier.py index 531b5d3b726..43623f44d36 100644 --- a/packages/syft/src/syft/service/worker/image_identifier.py +++ b/packages/syft/src/syft/service/worker/image_identifier.py @@ -1,12 +1,12 @@ # stdlib from typing import Optional -from typing import Tuple from typing import Union # third party from typing_extensions import Self # relative +from ...custom_worker.utils import ImageUtils from ...serde.serializable import serializable from ...types.base import SyftBaseModel from .image_registry import SyftImageRegistry @@ -38,7 +38,7 @@ class SyftWorkerImageIdentifier(SyftBaseModel): @classmethod def with_registry(cls, tag: str, registry: SyftImageRegistry) -> Self: """Build a SyftWorkerImageTag from Docker tag & a previously created SyftImageRegistry object.""" - registry_str, repo, tag = SyftWorkerImageIdentifier.parse_str(tag) + registry_str, repo, tag = ImageUtils.parse_tag(tag) # if we parsed a registry string, make sure it matches the registry object if registry_str and registry_str != registry.url: @@ -49,23 +49,9 @@ def with_registry(cls, tag: str, registry: SyftImageRegistry) -> Self: @classmethod def from_str(cls, tag: str) -> Self: """Build a SyftWorkerImageTag from a pure-string standard Docker tag.""" - registry, repo, tag = SyftWorkerImageIdentifier.parse_str(tag) + registry, repo, tag = ImageUtils.parse_tag(tag) return cls(repo=repo, registry=registry, tag=tag) - @staticmethod - def parse_str(tag: str) -> Tuple[Optional[str], str, str]: - url, tag = tag.rsplit(":", 1) - args = url.rsplit("/", 2) - - if len(args) == 3: - registry = args[0] - repo = "/".join(args[1:]) - else: - registry = None - repo = "/".join(args) - - return registry, repo, tag - @property def repo_with_tag(self) -> str: if self.repo or self.tag: diff --git a/packages/syft/src/syft/service/worker/image_registry.py b/packages/syft/src/syft/service/worker/image_registry.py index cf3c36c0e0d..806a0946d2b 100644 --- a/packages/syft/src/syft/service/worker/image_registry.py +++ b/packages/syft/src/syft/service/worker/image_registry.py @@ -1,4 +1,5 @@ # stdlib +import re from urllib.parse import urlparse # third party @@ -10,6 +11,8 @@ from ...types.syft_object import SyftObject from ...types.uid import UID +REGX_DOMAIN = re.compile(r"^(localhost|([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*))(\:\d{1,5})?$") + @serializable() class SyftImageRegistry(SyftObject): @@ -26,15 +29,19 @@ class SyftImageRegistry(SyftObject): @validator("url") def validate_url(cls, val: str): - if val.startswith("http") or "://" in val: - raise ValueError("Registry URL must be a valid RFC 3986 URI") + if not val: + raise ValueError("Invalid Registry URL. Must not be empty") + + if not bool(re.match(REGX_DOMAIN, val)): + raise ValueError("Invalid Registry URL. Must be a valid domain.") + return val @classmethod def from_url(cls, full_str: str): + # this is only for urlparse if "://" not in full_str: full_str = f"http://{full_str}" - parsed = urlparse(full_str) # netloc includes the host & port, so local dev should work as expected diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index 9f677eb7f98..fa216e4e6f7 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -1,6 +1,7 @@ # stdlib import contextlib import os +from pathlib import Path import socket import socketserver import sys @@ -19,6 +20,7 @@ from ...custom_worker.builder_types import ImagePushResult from ...custom_worker.config import DockerWorkerConfig from ...custom_worker.config import PrebuiltWorkerConfig +from ...custom_worker.k8s import KubeUtils from ...custom_worker.k8s import PodStatus from ...custom_worker.runner_k8s import KubernetesRunner from ...node.credentials import SyftVerifyKey @@ -36,6 +38,7 @@ DEFAULT_WORKER_IMAGE_TAG = "openmined/default-worker-image-cpu:0.0.1" DEFAULT_WORKER_POOL_NAME = "default-pool" +K8S_NODE_CREDS_NAME = "node-creds" def backend_container_name() -> str: @@ -254,9 +257,46 @@ def run_workers_in_threads( return results +def prepare_kubernetes_pool_env(runner: KubernetesRunner, env_vars: dict): + # get current backend pod name + backend_pod_name = os.getenv("K8S_POD_NAME") + if not backend_pod_name: + raise ValueError(message="Pod name not provided in environment variable") + + # get current backend's credentials path + creds_path = os.getenv("CREDENTIALS_PATH") + if not creds_path: + raise ValueError(message="Credentials path not provided") + + creds_path = Path(creds_path) + if not creds_path.exists(): + raise ValueError(message="Credentials file does not exist") + + # create a secret for the node credentials owned by the backend, not the pool. + node_secret = KubeUtils.create_secret( + secret_name=K8S_NODE_CREDS_NAME, + type="Opaque", + component=backend_pod_name, + data={creds_path.name: creds_path.read_text()}, + encoded=False, + ) + + # clone and patch backend environment variables + backend_env = runner.get_pod_env_vars(backend_pod_name) or [] + env_vars = KubeUtils.patch_env_vars(backend_env, env_vars) + mount_secrets = { + node_secret.metadata.name: { + "mountPath": str(creds_path), + "subPath": creds_path.name, + }, + } + + return env_vars, mount_secrets + + def create_kubernetes_pool( runner: KubernetesRunner, - worker_image: SyftWorker, + tag: str, pool_name: str, replicas: int, queue_port: int, @@ -273,14 +313,13 @@ def create_kubernetes_pool( print( "Creating new pool " f"name={pool_name} " - f"tag={worker_image.image_identifier.full_name_with_tag} " + f"tag={tag} " f"replicas={replicas}" ) - pool = runner.create_pool( - pool_name=pool_name, - tag=worker_image.image_identifier.full_name_with_tag, - replicas=replicas, - env_vars={ + + env_vars, mount_secrets = prepare_kubernetes_pool_env( + runner, + { "SYFT_WORKER": "True", "DEV_MODE": f"{debug}", "QUEUE_PORT": f"{queue_port}", @@ -289,6 +328,15 @@ def create_kubernetes_pool( "CREATE_PRODUCER": "False", "INMEMORY_WORKERS": "False", }, + ) + + # run the pool with args + secret + pool = runner.create_pool( + pool_name=pool_name, + tag=tag, + replicas=replicas, + env_vars=env_vars, + mount_secrets=mount_secrets, reg_username=reg_username, reg_password=reg_password, reg_url=reg_url, @@ -300,7 +348,7 @@ def create_kubernetes_pool( if error and pool: pool.delete() - return runner.get_pods(pool_name=pool_name) + return runner.get_pool_pods(pool_name=pool_name) def scale_kubernetes_pool( @@ -318,7 +366,7 @@ def scale_kubernetes_pool( except Exception as e: return SyftError(message=f"Failed to scale workers {e}") - return runner.get_pods(pool_name=pool_name) + return runner.get_pool_pods(pool_name=pool_name) def run_workers_in_kubernetes( @@ -336,10 +384,10 @@ def run_workers_in_kubernetes( spawn_status = [] runner = KubernetesRunner() - if start_idx == 0: + if not runner.exists(pool_name=pool_name): pool_pods = create_kubernetes_pool( runner=runner, - worker_image=worker_image, + tag=worker_image.image_identifier.full_name_with_tag, pool_name=pool_name, replicas=worker_count, queue_port=queue_port, diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 806881547da..11e83b01112 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -11,6 +11,7 @@ from ...custom_worker.config import CustomWorkerConfig from ...custom_worker.config import WorkerConfig from ...custom_worker.k8s import IN_KUBERNETES +from ...custom_worker.runner_k8s import KubernetesRunner from ...serde.serializable import serializable from ...store.document_store import DocumentStore from ...store.linked_obj import LinkedObject @@ -36,6 +37,7 @@ from .utils import get_orchestration_type from .utils import run_containers from .utils import run_workers_in_threads +from .utils import scale_kubernetes_pool from .worker_image import SyftWorkerImage from .worker_image_stash import SyftWorkerImageStash from .worker_pool import ContainerSpawnStatus @@ -430,6 +432,94 @@ def add_workers( return container_statuses + @service_method( + path="worker_pool.scale", + name="scale", + roles=DATA_OWNER_ROLE_LEVEL, + ) + def scale( + self, + context: AuthedServiceContext, + number: int, + pool_id: Optional[UID] = None, + pool_name: Optional[str] = None, + ): + """ + Scale the worker pool to the given number of workers in Kubernetes. + Allows both scaling up and down the worker pool. + """ + + if not IN_KUBERNETES: + return SyftError(message="Scaling is only supported in Kubernetes mode") + elif number < 0: + # zero is a valid scale down + return SyftError(message=f"Invalid number of workers: {number}") + + result = self._get_worker_pool(context, pool_id, pool_name) + if isinstance(result, SyftError): + return result + + worker_pool = result + current_worker_count = len(worker_pool.worker_list) + + if current_worker_count == number: + return SyftSuccess(message=f"Worker pool already has {number} workers") + elif number > current_worker_count: + workers_to_add = number - current_worker_count + return self.add_workers( + context=context, + number=workers_to_add, + pool_id=pool_id, + pool_name=pool_name, + # kube scaling doesn't require password as it replicates an existing deployment + reg_username=None, + reg_password=None, + ) + else: + # scale down at kubernetes control plane + runner = KubernetesRunner() + result = scale_kubernetes_pool( + runner, + pool_name=worker_pool.name, + replicas=number, + ) + if isinstance(result, SyftError): + return result + + # scale down removes the last "n" workers + # workers to delete = len(workers) - number + workers_to_delete = worker_pool.worker_list[ + -(current_worker_count - number) : + ] + + worker_stash = context.node.get_service("WorkerService").stash + # delete linkedobj workers + for worker in workers_to_delete: + delete_result = worker_stash.delete_by_uid( + credentials=context.credentials, + uid=worker.object_uid, + ) + if delete_result.is_err(): + print(f"Failed to delete worker: {worker.object_uid}") + + # update worker_pool + worker_pool.max_count = number + worker_pool.worker_list = worker_pool.worker_list[:number] + update_result = self.stash.update( + credentials=context.credentials, + obj=worker_pool, + ) + + if update_result.is_err(): + return SyftError( + message=( + f"Pool {worker_pool.name} was scaled down, " + f"but failed update the stash with err: {result.err()}" + ) + ) + + return SyftSuccess(message=f"Worker pool scaled to {number} workers") + @service_method( path="worker_pool.filter_by_image_id", name="filter_by_image_id", diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 2a27fc2518a..85524236a37 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -1,8 +1,9 @@ # stdlib from io import BytesIO import math +from queue import Queue +import threading from typing import Dict -from typing import Generator from typing import List from typing import Optional from typing import Type @@ -14,6 +15,7 @@ from botocore.client import ClientError as BotoClientError from botocore.client import Config import requests +from tqdm import tqdm from typing_extensions import Self # relative @@ -33,27 +35,33 @@ from ...types.blob_storage import SeaweedSecureFilePathLocation from ...types.blob_storage import SecureFilePathLocation from ...types.grid_url import GridURL +from ...types.syft_migration import migrate from ...types.syft_object import SYFT_OBJECT_VERSION_1 +from ...types.syft_object import SYFT_OBJECT_VERSION_2 +from ...types.transforms import drop +from ...types.transforms import make_set_default from ...util.constants import DEFAULT_TIMEOUT WRITE_EXPIRATION_TIME = 900 # seconds -DEFAULT_CHUNK_SIZE = 1024**3 # 1 GB +DEFAULT_FILE_PART_SIZE = (1024**3) * 5 # 5GB +DEFAULT_UPLOAD_CHUNK_SIZE = 819200 -def _byte_chunks(bytes: BytesIO, size: int) -> Generator[bytes, None, None]: - while True: - try: - yield bytes.read(size) - except BlockingIOError: - return +@serializable() +class SeaweedFSBlobDepositV1(BlobDeposit): + __canonical_name__ = "SeaweedFSBlobDeposit" + __version__ = SYFT_OBJECT_VERSION_1 + + urls: List[GridURL] @serializable() class SeaweedFSBlobDeposit(BlobDeposit): __canonical_name__ = "SeaweedFSBlobDeposit" - __version__ = SYFT_OBJECT_VERSION_1 + __version__ = SYFT_OBJECT_VERSION_2 urls: List[GridURL] + size: int def write(self, data: BytesIO) -> Union[SyftSuccess, SyftError]: # relative @@ -68,24 +76,83 @@ def write(self, data: BytesIO) -> Union[SyftSuccess, SyftError]: try: no_lines = 0 - for part_no, (byte_chunk, url) in enumerate( - zip(_byte_chunks(data, DEFAULT_CHUNK_SIZE), self.urls), - start=1, - ): - no_lines += byte_chunk.count(b"\n") - if api is not None: - blob_url = api.connection.to_blob_route( - url.url_path, host=url.host_or_ip + # this loops over the parts, we have multiple parts to allow for + # concurrent uploads of a single file. (We are currently not using that) + # a part may for instance be 5GB + # parts are then splitted into chunks which are MBs (order of magnitude) + part_size = math.ceil(self.size / len(self.urls)) + chunk_size = DEFAULT_UPLOAD_CHUNK_SIZE + + # this is the total nr of chunks in all parts + total_iterations = math.ceil(part_size / chunk_size) * len(self.urls) + + with tqdm( + total=total_iterations, + desc=f"Uploading progress", # noqa + ) as pbar: + for part_no, url in enumerate( + self.urls, + start=1, + ): + if api is not None: + blob_url = api.connection.to_blob_route( + url.url_path, host=url.host_or_ip + ) + else: + blob_url = url + + # read a chunk untill we have read part_size + class PartGenerator: + def __init__(self): + self.no_lines = 0 + + def async_generator(self, chunk_size=DEFAULT_UPLOAD_CHUNK_SIZE): + item_queue: Queue = Queue() + threading.Thread( + target=self.add_chunks_to_queue, + kwargs={"queue": item_queue, "chunk_size": chunk_size}, + daemon=True, + ).start() + item = item_queue.get() + while item != 0: + yield item + pbar.update(1) + item = item_queue.get() + + def add_chunks_to_queue( + self, queue, chunk_size=DEFAULT_UPLOAD_CHUNK_SIZE + ): + """Creates a data geneator for the part""" + n = 0 + + while n * chunk_size <= part_size: + try: + chunk = data.read(chunk_size) + self.no_lines += chunk.count(b"\n") + n += 1 + queue.put(chunk) + except BlockingIOError: + # if end of file, stop + queue.put(0) + # if end of part, stop + queue.put(0) + + gen = PartGenerator() + + response = requests.put( + url=str(blob_url), + data=gen.async_generator(chunk_size), + timeout=DEFAULT_TIMEOUT, + stream=True, ) - else: - blob_url = url - response = requests.put( - url=str(blob_url), data=byte_chunk, timeout=DEFAULT_TIMEOUT - ) - response.raise_for_status() - etag = response.headers["ETag"] - etags.append({"ETag": etag, "PartNumber": part_no}) + + response.raise_for_status() + no_lines += gen.no_lines + etag = response.headers["ETag"] + etags.append({"ETag": etag, "PartNumber": part_no}) + except requests.RequestException as e: + print(e) return SyftError(message=str(e)) mark_write_complete_method = from_api_or_context( @@ -98,6 +165,20 @@ def write(self, data: BytesIO) -> Union[SyftSuccess, SyftError]: ) +@migrate(SeaweedFSBlobDeposit, SeaweedFSBlobDepositV1) +def downgrade_seaweedblobdeposit_v2_to_v1(): + return [ + drop(["size"]), + ] + + +@migrate(SeaweedFSBlobDepositV1, SeaweedFSBlobDeposit) +def upgrade_seaweedblobdeposit_v1_to_v2(): + return [ + make_set_default("size", 1), + ] + + @serializable() class SeaweedFSClientConfig(BlobStorageClientConfig): host: str @@ -188,7 +269,7 @@ def allocate( ) def write(self, obj: BlobStorageEntry) -> BlobDeposit: - total_parts = math.ceil(obj.file_size / DEFAULT_CHUNK_SIZE) + total_parts = math.ceil(obj.file_size / DEFAULT_FILE_PART_SIZE) urls = [ GridURL.from_url( @@ -205,8 +286,9 @@ def write(self, obj: BlobStorageEntry) -> BlobDeposit: ) for i in range(total_parts) ] - - return SeaweedFSBlobDeposit(blob_storage_entry_id=obj.id, urls=urls) + return SeaweedFSBlobDeposit( + blob_storage_entry_id=obj.id, urls=urls, size=obj.file_size + ) def complete_multipart_upload( self, diff --git a/packages/syft/tests/conftest.py b/packages/syft/tests/conftest.py index abe1fef2bd0..0cdbb2f1d04 100644 --- a/packages/syft/tests/conftest.py +++ b/packages/syft/tests/conftest.py @@ -1,5 +1,6 @@ # stdlib import json +import os from pathlib import Path from unittest import mock @@ -46,6 +47,14 @@ def remove_file(filepath: Path): filepath.unlink(missing_ok=True) +# Pytest hook to set the number of workers for xdist +def pytest_xdist_auto_num_workers(config): + num = config.option.numprocesses + if num == "auto" or num == "logical": + return os.cpu_count() + return None + + @pytest.fixture(autouse=True) def protocol_file(): random_name = sy.UID().to_string() diff --git a/packages/syft/tests/syft/request/request_code_accept_deny_test.py b/packages/syft/tests/syft/request/request_code_accept_deny_test.py index 242de50ff85..7451dd26d91 100644 --- a/packages/syft/tests/syft/request/request_code_accept_deny_test.py +++ b/packages/syft/tests/syft/request/request_code_accept_deny_test.py @@ -205,4 +205,4 @@ def simple_function(data): result = ds_client.code.simple_function(data=action_obj) assert isinstance(result, SyftError) - assert "UserCodeStatus.DENIED" in result.message + assert "Execution denied" in result.message diff --git a/packages/syft/tests/syft/request/request_multiple_nodes_test.py b/packages/syft/tests/syft/request/request_multiple_nodes_test.py index 4beb780cc31..9ec214ea7fe 100644 --- a/packages/syft/tests/syft/request/request_multiple_nodes_test.py +++ b/packages/syft/tests/syft/request/request_multiple_nodes_test.py @@ -110,6 +110,7 @@ def dataset_2(client_do_2): return client_do_2.datasets[0].assets[0] +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_blocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): @@ -147,6 +148,7 @@ def compute_sum(data) -> float: assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean() +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_nonblocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): diff --git a/packages/syftcli/manifest.yml b/packages/syftcli/manifest.yml index ccc9180f1da..d509f916827 100644 --- a/packages/syftcli/manifest.yml +++ b/packages/syftcli/manifest.yml @@ -1,11 +1,11 @@ manifestVersion: 1.0 -syftVersion: 0.8.4-beta.19 -dockerTag: 0.8.4-beta.19 +syftVersion: 0.8.4-beta.21 +dockerTag: 0.8.4-beta.21 images: - - docker.io/openmined/grid-frontend:0.8.4-beta.19 - - docker.io/openmined/grid-backend:0.8.4-beta.19 + - docker.io/openmined/grid-frontend:0.8.4-beta.21 + - docker.io/openmined/grid-backend:0.8.4-beta.21 - docker.io/library/mongo:7.0.4 - docker.io/traefik:v2.10 diff --git a/scripts/build_images.sh b/scripts/build_images.sh new file mode 100644 index 00000000000..280ca544620 --- /dev/null +++ b/scripts/build_images.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +REGISTRY=${1:-"k3d-registry.localhost:5000"} +TAG=${2:-"latest"} + +docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t $REGISTRY/openmined/grid-backend:$TAG ./packages +docker image build -f ./packages/grid/frontend/frontend.dockerfile --target grid-ui-development -t $REGISTRY/openmined/grid-frontend:$TAG ./packages/grid/frontend +docker image build -f ./packages/grid/seaweedfs/seaweedfs.dockerfile --build-arg SEAWEEDFS_VERSION=3.59 -t $REGISTRY/openmined/grid-seaweedfs:$TAG ./packages/grid/seaweedfs diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash index b0293a9d097..3c519269255 100644 --- a/scripts/hagrid_hash +++ b/scripts/hagrid_hash @@ -1 +1 @@ -55fa40104063afa8e52213b1c8d29634 +37c1c6fabbb3c234ec5c7a10af136585 diff --git a/tox.ini b/tox.ini index 838d8fe35e5..0362e12bfcb 100644 --- a/tox.ini +++ b/tox.ini @@ -783,19 +783,9 @@ commands = # ignore 06 because of opendp on arm64 # Run 0.8 notebooks - bash -c 'echo Gateway Cluster Info; kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Gateway Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Domain Cluster Info; kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1' - bash -c 'echo Domain Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1' - bash -c " source ./scripts/get_k8s_secret_ci.sh; \ pytest --nbmake notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv --nbmake-timeout=1000" - bash -c 'echo Gateway Cluster Info; kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Gateway Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Domain Cluster Info; kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1' - bash -c 'echo Domain Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1' - #Integration + Gateway Connection Tests # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure # high/low side warning flag. @@ -1014,10 +1004,12 @@ allowlist_externals = commands = bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft; sleep 3' bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 || true' + bash -c 'kubectl config use-context k3d-syft-dev' bash -c 'kubectl delete all --all --namespace syft || true' bash -c 'kubectl delete pvc --all --namespace syft || true' bash -c 'kubectl delete secret --all --namespace syft || true' bash -c 'kubectl delete configmap --all --namespace syft || true' + bash -c 'kubectl delete serviceaccount --all --namespace syft || true' [testenv:dev.k8s.destroy] description = Destroy local Kubernetes cluster