From cdd41cc9b889854a2e0e7e6ecfff989195b4b328 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 30 Jan 2024 22:34:03 +0100 Subject: [PATCH 01/46] add tested changes for uploads --- packages/grid/default.env | 5 +- packages/grid/seaweedfs/seaweedfs.dockerfile | 4 +- .../syft/src/syft/client/domain_client.py | 3 +- .../src/syft/protocol/protocol_version.json | 7 + .../src/syft/store/blob_storage/seaweedfs.py | 138 ++++++++++++++---- 5 files changed, 125 insertions(+), 32 deletions(-) diff --git a/packages/grid/default.env b/packages/grid/default.env index 5e69aca2580..f6be744d49c 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -26,7 +26,7 @@ DOCKER_IMAGE_TRAEFIK=traefik TRAEFIK_VERSION=v2.10 REDIS_VERSION=6.2 RABBITMQ_VERSION=3 -SEAWEEDFS_VERSION=3.59 +SEAWEEDFS_VERSION=3.62 DOCKER_IMAGE_SEAWEEDFS=openmined/grid-seaweedfs VERSION=latest VERSION_HASH=unknown @@ -71,7 +71,8 @@ S3_ROOT_PWD="admin" # needs randomizing S3_REGION="us-east-1" #not-using S3_PRESIGNED_TIMEOUT_SECS=1800 -S3_VOLUME_SIZE_MB=1024 +S3_VOLUME_SIZE_MB=40000 + # Jax JAX_ENABLE_X64=True diff --git a/packages/grid/seaweedfs/seaweedfs.dockerfile b/packages/grid/seaweedfs/seaweedfs.dockerfile index 3982e621c3b..cb9c1afbf2b 100644 --- a/packages/grid/seaweedfs/seaweedfs.dockerfile +++ b/packages/grid/seaweedfs/seaweedfs.dockerfile @@ -1,6 +1,6 @@ ARG SEAWEEDFS_VERSION -FROM chrislusf/seaweedfs:${SEAWEEDFS_VERSION} +FROM chrislusf/seaweedfs:${SEAWEEDFS_VERSION}_large_disk WORKDIR / @@ -9,6 +9,8 @@ RUN apk update && \ COPY requirements.txt app.py / RUN pip install --no-cache-dir -r requirements.txt +# RUN pip install --no-cache-dir --break-system-packages -r requirements.txt + COPY --chmod=755 start.sh mount_command.sh / diff --git a/packages/syft/src/syft/client/domain_client.py b/packages/syft/src/syft/client/domain_client.py index eacc52533cd..d9d8cc3ae4e 100644 --- a/packages/syft/src/syft/client/domain_client.py +++ b/packages/syft/src/syft/client/domain_client.py @@ -191,9 +191,10 @@ def upload_files( try: result = [] - for file in tqdm(expanded_file_list): + for file in expanded_file_list: if not isinstance(file, BlobFile): file = BlobFile(path=file, file_name=file.name) + print("Uploading", file.file_name) if not file.uploaded: file.upload_to_blobstorage(self) result.append(file) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index d394d8f29bd..67047c5b269 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -1182,6 +1182,13 @@ "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", "action": "remove" } + }, + "SeaweedFSBlobDeposit": { + "2": { + "version": 2, + "hash": "07d84a95324d95d9c868cd7d1c33c908f77aa468671d76c144586aab672bcbb5", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index 2a27fc2518a..85524236a37 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -1,8 +1,9 @@ # stdlib from io import BytesIO import math +from queue import Queue +import threading from typing import Dict -from typing import Generator from typing import List from typing import Optional from typing import Type @@ -14,6 +15,7 @@ from botocore.client import ClientError as BotoClientError from botocore.client import Config import requests +from tqdm import tqdm from typing_extensions import Self # relative @@ -33,27 +35,33 @@ from ...types.blob_storage import SeaweedSecureFilePathLocation from ...types.blob_storage import SecureFilePathLocation from ...types.grid_url import GridURL +from ...types.syft_migration import migrate from ...types.syft_object import SYFT_OBJECT_VERSION_1 +from ...types.syft_object import SYFT_OBJECT_VERSION_2 +from ...types.transforms import drop +from ...types.transforms import make_set_default from ...util.constants import DEFAULT_TIMEOUT WRITE_EXPIRATION_TIME = 900 # seconds -DEFAULT_CHUNK_SIZE = 1024**3 # 1 GB +DEFAULT_FILE_PART_SIZE = (1024**3) * 5 # 5GB +DEFAULT_UPLOAD_CHUNK_SIZE = 819200 -def _byte_chunks(bytes: BytesIO, size: int) -> Generator[bytes, None, None]: - while True: - try: - yield bytes.read(size) - except BlockingIOError: - return +@serializable() +class SeaweedFSBlobDepositV1(BlobDeposit): + __canonical_name__ = "SeaweedFSBlobDeposit" + __version__ = SYFT_OBJECT_VERSION_1 + + urls: List[GridURL] @serializable() class SeaweedFSBlobDeposit(BlobDeposit): __canonical_name__ = "SeaweedFSBlobDeposit" - __version__ = SYFT_OBJECT_VERSION_1 + __version__ = SYFT_OBJECT_VERSION_2 urls: List[GridURL] + size: int def write(self, data: BytesIO) -> Union[SyftSuccess, SyftError]: # relative @@ -68,24 +76,83 @@ def write(self, data: BytesIO) -> Union[SyftSuccess, SyftError]: try: no_lines = 0 - for part_no, (byte_chunk, url) in enumerate( - zip(_byte_chunks(data, DEFAULT_CHUNK_SIZE), self.urls), - start=1, - ): - no_lines += byte_chunk.count(b"\n") - if api is not None: - blob_url = api.connection.to_blob_route( - url.url_path, host=url.host_or_ip + # this loops over the parts, we have multiple parts to allow for + # concurrent uploads of a single file. (We are currently not using that) + # a part may for instance be 5GB + # parts are then splitted into chunks which are MBs (order of magnitude) + part_size = math.ceil(self.size / len(self.urls)) + chunk_size = DEFAULT_UPLOAD_CHUNK_SIZE + + # this is the total nr of chunks in all parts + total_iterations = math.ceil(part_size / chunk_size) * len(self.urls) + + with tqdm( + total=total_iterations, + desc=f"Uploading progress", # noqa + ) as pbar: + for part_no, url in enumerate( + self.urls, + start=1, + ): + if api is not None: + blob_url = api.connection.to_blob_route( + url.url_path, host=url.host_or_ip + ) + else: + blob_url = url + + # read a chunk untill we have read part_size + class PartGenerator: + def __init__(self): + self.no_lines = 0 + + def async_generator(self, chunk_size=DEFAULT_UPLOAD_CHUNK_SIZE): + item_queue: Queue = Queue() + threading.Thread( + target=self.add_chunks_to_queue, + kwargs={"queue": item_queue, "chunk_size": chunk_size}, + daemon=True, + ).start() + item = item_queue.get() + while item != 0: + yield item + pbar.update(1) + item = item_queue.get() + + def add_chunks_to_queue( + self, queue, chunk_size=DEFAULT_UPLOAD_CHUNK_SIZE + ): + """Creates a data geneator for the part""" + n = 0 + + while n * chunk_size <= part_size: + try: + chunk = data.read(chunk_size) + self.no_lines += chunk.count(b"\n") + n += 1 + queue.put(chunk) + except BlockingIOError: + # if end of file, stop + queue.put(0) + # if end of part, stop + queue.put(0) + + gen = PartGenerator() + + response = requests.put( + url=str(blob_url), + data=gen.async_generator(chunk_size), + timeout=DEFAULT_TIMEOUT, + stream=True, ) - else: - blob_url = url - response = requests.put( - url=str(blob_url), data=byte_chunk, timeout=DEFAULT_TIMEOUT - ) - response.raise_for_status() - etag = response.headers["ETag"] - etags.append({"ETag": etag, "PartNumber": part_no}) + + response.raise_for_status() + no_lines += gen.no_lines + etag = response.headers["ETag"] + etags.append({"ETag": etag, "PartNumber": part_no}) + except requests.RequestException as e: + print(e) return SyftError(message=str(e)) mark_write_complete_method = from_api_or_context( @@ -98,6 +165,20 @@ def write(self, data: BytesIO) -> Union[SyftSuccess, SyftError]: ) +@migrate(SeaweedFSBlobDeposit, SeaweedFSBlobDepositV1) +def downgrade_seaweedblobdeposit_v2_to_v1(): + return [ + drop(["size"]), + ] + + +@migrate(SeaweedFSBlobDepositV1, SeaweedFSBlobDeposit) +def upgrade_seaweedblobdeposit_v1_to_v2(): + return [ + make_set_default("size", 1), + ] + + @serializable() class SeaweedFSClientConfig(BlobStorageClientConfig): host: str @@ -188,7 +269,7 @@ def allocate( ) def write(self, obj: BlobStorageEntry) -> BlobDeposit: - total_parts = math.ceil(obj.file_size / DEFAULT_CHUNK_SIZE) + total_parts = math.ceil(obj.file_size / DEFAULT_FILE_PART_SIZE) urls = [ GridURL.from_url( @@ -205,8 +286,9 @@ def write(self, obj: BlobStorageEntry) -> BlobDeposit: ) for i in range(total_parts) ] - - return SeaweedFSBlobDeposit(blob_storage_entry_id=obj.id, urls=urls) + return SeaweedFSBlobDeposit( + blob_storage_entry_id=obj.id, urls=urls, size=obj.file_size + ) def complete_multipart_upload( self, From e6b4caee8d0e9dceeef6725895cb5e4d62956e7a Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 30 Jan 2024 22:35:12 +0100 Subject: [PATCH 02/46] remove comment --- packages/grid/seaweedfs/seaweedfs.dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/grid/seaweedfs/seaweedfs.dockerfile b/packages/grid/seaweedfs/seaweedfs.dockerfile index cb9c1afbf2b..9f74b706206 100644 --- a/packages/grid/seaweedfs/seaweedfs.dockerfile +++ b/packages/grid/seaweedfs/seaweedfs.dockerfile @@ -9,7 +9,6 @@ RUN apk update && \ COPY requirements.txt app.py / RUN pip install --no-cache-dir -r requirements.txt -# RUN pip install --no-cache-dir --break-system-packages -r requirements.txt COPY --chmod=755 start.sh mount_command.sh / From afc8bb2a42847d624bec3511dbe994b405928bfd Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 31 Jan 2024 11:42:41 +0100 Subject: [PATCH 03/46] fix seaweed image --- packages/grid/seaweedfs/seaweedfs.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/seaweedfs/seaweedfs.dockerfile b/packages/grid/seaweedfs/seaweedfs.dockerfile index 9f74b706206..5b53d14c0bc 100644 --- a/packages/grid/seaweedfs/seaweedfs.dockerfile +++ b/packages/grid/seaweedfs/seaweedfs.dockerfile @@ -8,7 +8,7 @@ RUN apk update && \ apk add --no-cache python3 py3-pip ca-certificates bash COPY requirements.txt app.py / -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --break-system-packages -r requirements.txt COPY --chmod=755 start.sh mount_command.sh / From 771aede8ae6a8f129caffbac1abd420f55574ba8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 31 Jan 2024 17:43:39 +0530 Subject: [PATCH 04/46] add a test image build method to DockerWorkerConfig --- .../syft/src/syft/custom_worker/config.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 319273cc1a4..4493261e37f 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -4,6 +4,7 @@ from typing import Any from typing import Dict from typing import List +from typing import Literal from typing import Optional from typing import Union @@ -15,7 +16,9 @@ # relative from ..serde.serializable import serializable +from ..service.response import SyftError from ..types.base import SyftBaseModel +from .builder_types import ImageBuildResult PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -136,3 +139,29 @@ def __str__(self) -> str: def set_description(self, description_text: str) -> None: self.description = description_text + + def test_image_build( + self, orchestration_type: Literal["docker", "k8s"], tag: str, **build_args + ) -> Union[ImageBuildResult, SyftError]: + # relative + from .builder_docker import DockerBuilder + from .builder_k8s import KubernetesBuilder + + builder = ( + KubernetesBuilder() if orchestration_type == "k8s" else DockerBuilder() + ) + + # TODO: Remove this check once we know how test with k8s + if orchestration_type == "k8s": + return SyftError( + message="We currently support test builds using `docker` only." + ) + + try: + return builder.build_image( + tag=tag, + dockerfile=self.dockerfile, + buildargs=build_args, + ) + except Exception as e: + return SyftError(message=f"Failed to build: {e}") From ccfdbb813429acbca5346d8a0858f26089d16d03 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 2 Feb 2024 14:44:45 +0530 Subject: [PATCH 05/46] [helm] fix MultiAttachError for PVCs --- packages/grid/helm/syft/templates/backend-statefulset.yaml | 5 +++++ packages/grid/helm/syft/templates/worker-builds-pvc.yaml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 1bd3ab2e898..943ebbd9e89 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -170,7 +170,12 @@ spec: name: credentials-data spec: accessModes: + # k3d does not support ReadWriteMany + {{- if .Values.configuration.devmode }} - ReadWriteOnce + {{- else }} + - ReadWriteMany + {{- end }} resources: requests: storage: 100Mi diff --git a/packages/grid/helm/syft/templates/worker-builds-pvc.yaml b/packages/grid/helm/syft/templates/worker-builds-pvc.yaml index 54eb4f7acc6..57533018a35 100644 --- a/packages/grid/helm/syft/templates/worker-builds-pvc.yaml +++ b/packages/grid/helm/syft/templates/worker-builds-pvc.yaml @@ -9,7 +9,12 @@ metadata: app.kubernetes.io/managed-by: Helm spec: accessModes: + # k3d does not support ReadWriteMany + {{- if .Values.configuration.devmode }} - ReadWriteOnce + {{- else}} + - ReadWriteMany + {{- end}} resources: requests: storage: {{ .Values.workerBuilds.maxStorage }} From 78ef9373e51f842989b97ea01abd510e6a9ebb74 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 2 Feb 2024 15:17:47 +0530 Subject: [PATCH 06/46] [helm] fix ingressClassName + ingress annotation --- README.md | 21 +++++++++++++++---- ...11-installing-and-upgrading-via-helm.ipynb | 2 +- .../templates/grid-stack-ingress-ingress.yaml | 8 ++++++- .../grid-stack-ingress-tls-ingress.yaml | 8 ++++++- packages/grid/helm/syft/values.yaml | 19 ++++++++++------- packages/syft/PYPI.md | 21 +++++++++++++++---- 6 files changed, 61 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6e402cc6342..735eee063d3 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,27 @@ SYFT_VERSION="" #### 4. Provisioning Helm Charts ```sh -helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik +helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className="traefik" ``` -### Azure or GCP Ingress +### Ingress Controllers +For Azure AKS + +```sh +helm install ... --set ingress.className="azure-application-gateway" ``` -helm install ... --set ingress.ingressClass="azure/application-gateway" -helm install ... --set ingress.ingressClass="gce" + +For AWS EKS + +```sh +helm install ... --set ingress.className="alb" +``` + +For Google GKE we need the [`gce` annotation](https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress) annotation. + +```sh +helm install ... --set ingress.class="gce" ``` ## Deploy to a Container Engine or Cloud diff --git a/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb b/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb index ed1537ef73a..729b5751c2f 100644 --- a/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb +++ b/notebooks/tutorials/data-engineer/11-installing-and-upgrading-via-helm.ipynb @@ -142,7 +142,7 @@ "metadata": {}, "source": [ "```bash\n", - "helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik\n", + "helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=traefik\n", "```" ] }, diff --git a/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml b/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml index 2eef50b54c6..6aed72bd414 100644 --- a/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml +++ b/packages/grid/helm/syft/templates/grid-stack-ingress-ingress.yaml @@ -8,8 +8,14 @@ metadata: app.kubernetes.io/component: ingress app.kubernetes.io/managed-by: Helm name: grid-stack-ingress + {{- if .Values.ingress.class }} + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.class }} + {{- end }} spec: - ingressClassName: {{ .Values.ingress.ingressClass }} + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} defaultBackend: service: name: proxy diff --git a/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml b/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml index a263c910156..58db0a03e29 100644 --- a/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml +++ b/packages/grid/helm/syft/templates/grid-stack-ingress-tls-ingress.yaml @@ -8,8 +8,14 @@ metadata: app.kubernetes.io/component: ingress app.kubernetes.io/managed-by: Helm name: grid-stack-ingress-tls + {{- if .Values.ingress.class }} + annotations: + kubernetes.io/ingress.class: {{ .Values.ingress.class }} + {{- end }} spec: - ingressClassName: {{ .Values.ingress.ingressClass }} + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} defaultBackend: service: name: proxy diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 7690e7f82f8..88bf8e9361d 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -1,7 +1,3 @@ -# Default values for syft. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - secrets: syft: syft-default-secret mongo: mongo-default-secret @@ -49,7 +45,16 @@ node: inMemoryWorkers: false defaultWorkerPoolCount: 1 +# ---------------------------------------- +# For Azure +# className: "azure-application-gateway" +# ---------------------------------------- +# For AWS +# className: "alb" +# ---------------------------------------- +# For GCE, https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress +# class: "gce" +# ---------------------------------------- ingress: - ingressClass: "" - # ingressClass: "azure/application-gateway" - # ingressClass: "gce" + class: null + className: null diff --git a/packages/syft/PYPI.md b/packages/syft/PYPI.md index 2265c5057ef..bee711a131a 100644 --- a/packages/syft/PYPI.md +++ b/packages/syft/PYPI.md @@ -78,14 +78,27 @@ SYFT_VERSION="" #### 4. Provisioning Helm Charts ```sh -helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.ingressClass=traefik +helm install my-domain openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className="traefik" ``` -### Azure or GCP Ingress +### Ingress Controllers +For Azure AKS + +```sh +helm install ... --set ingress.className="azure-application-gateway" ``` -helm install ... --set ingress.ingressClass="azure/application-gateway" -helm install ... --set ingress.ingressClass="gce" + +For AWS EKS + +```sh +helm install ... --set ingress.className="alb" +``` + +For Google GKE we need the [`gce` annotation](https://cloud.google.com/kubernetes-engine/docs/how-to/load-balance-ingress#create-ingress) annotation. + +```sh +helm install ... --set ingress.class="gce" ``` ## Deploy to a Container Engine or Cloud From cbc034055fce0f02732ca5b90534cb49fb0461c4 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 2 Feb 2024 19:19:06 +0530 Subject: [PATCH 07/46] [k8s] remove build volume and replace with internal registry --- packages/grid/devspace.yaml | 4 +- .../templates/backend-service-account.yaml | 5 - .../syft/templates/backend-statefulset.yaml | 12 - .../helm/syft/templates/registry-service.yaml | 17 ++ .../syft/templates/registry-statefulset.yaml | 47 ++++ .../syft/templates/worker-builds-pvc.yaml | 20 -- packages/grid/helm/syft/values.yaml | 3 +- .../src/syft/custom_worker/builder_k8s.py | 205 ++++++++++-------- packages/syft/src/syft/custom_worker/k8s.py | 7 +- packages/syft/src/syft/custom_worker/utils.py | 33 +++ .../syft/service/worker/image_identifier.py | 20 +- tox.ini | 1 + 12 files changed, 228 insertions(+), 146 deletions(-) create mode 100644 packages/grid/helm/syft/templates/registry-service.yaml create mode 100644 packages/grid/helm/syft/templates/registry-statefulset.yaml delete mode 100644 packages/grid/helm/syft/templates/worker-builds-pvc.yaml create mode 100644 packages/syft/src/syft/custom_worker/utils.py diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 6224606d0c2..e678b68bc31 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -66,8 +66,8 @@ deployments: syft: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} - workerBuilds: - mountInBackend: true + registry: + maxStorage: "5Gi" node: settings: nodeName: ${NODE_NAME} diff --git a/packages/grid/helm/syft/templates/backend-service-account.yaml b/packages/grid/helm/syft/templates/backend-service-account.yaml index 97608b4fd4e..56e552b2b7f 100644 --- a/packages/grid/helm/syft/templates/backend-service-account.yaml +++ b/packages/grid/helm/syft/templates/backend-service-account.yaml @@ -2,7 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: backend-service-account - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} @@ -14,7 +13,6 @@ apiVersion: v1 kind: Secret metadata: name: backend-service-secret - namespace: {{ .Release.Namespace }} annotations: kubernetes.io/service-account.name: "backend-service-account" labels: @@ -29,7 +27,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: backend-service-role - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} @@ -53,7 +50,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: backend-service-role-binding - namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/version: {{ .Chart.AppVersion }} @@ -61,7 +57,6 @@ metadata: subjects: - kind: ServiceAccount name: backend-service-account - namespace: {{ .Release.Namespace }} roleRef: kind: Role name: backend-service-role diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 943ebbd9e89..cafe0851aa0 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -136,12 +136,6 @@ spec: name: credentials-data readOnly: false subPath: credentials-data - {{- if .Values.workerBuilds.mountInBackend }} - # mount for debugging and inspection of worker-build volume - - mountPath: /root/data/images/ - name: worker-builds - readOnly: true - {{- end }} dnsConfig: null ephemeralContainers: null hostAliases: null @@ -155,12 +149,6 @@ spec: terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null - {{- if .Values.workerBuilds.mountInBackend }} - volumes: - - name: worker-builds - persistentVolumeClaim: - claimName: worker-builds - {{- end }} volumeClaimTemplates: - metadata: labels: diff --git a/packages/grid/helm/syft/templates/registry-service.yaml b/packages/grid/helm/syft/templates/registry-service.yaml new file mode 100644 index 00000000000..f96060e3a4d --- /dev/null +++ b/packages/grid/helm/syft/templates/registry-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: registry + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - protocol: TCP + port: 80 + targetPort: 5000 + selector: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry diff --git a/packages/grid/helm/syft/templates/registry-statefulset.yaml b/packages/grid/helm/syft/templates/registry-statefulset.yaml new file mode 100644 index 00000000000..c4fb60d474d --- /dev/null +++ b/packages/grid/helm/syft/templates/registry-statefulset.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: registry + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm + spec: + containers: + - image: registry:2 + name: registry + env: + - name: REGISTRY_STORAGE_DELETE_ENABLED + value: "true" + ports: + - containerPort: 5000 + volumeMounts: + - mountPath: /var/lib/registry + name: registry-data + volumeClaimTemplates: + - metadata: + name: registry-data + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/component: registry + app.kubernetes.io/managed-by: Helm + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.registry.maxStorage }} diff --git a/packages/grid/helm/syft/templates/worker-builds-pvc.yaml b/packages/grid/helm/syft/templates/worker-builds-pvc.yaml deleted file mode 100644 index 57533018a35..00000000000 --- a/packages/grid/helm/syft/templates/worker-builds-pvc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: worker-builds - labels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/version: {{ .Chart.AppVersion }} - app.kubernetes.io/component: worker-builds - app.kubernetes.io/managed-by: Helm -spec: - accessModes: - # k3d does not support ReadWriteMany - {{- if .Values.configuration.devmode }} - - ReadWriteOnce - {{- else}} - - ReadWriteMany - {{- end}} - resources: - requests: - storage: {{ .Values.workerBuilds.maxStorage }} diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 88bf8e9361d..5a1af1f20d9 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -24,9 +24,8 @@ seaweedfs: queue: port: 5556 -workerBuilds: +registry: maxStorage: "10Gi" - mountInBackend: false syft: registry: "docker.io" diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py index a80df6470cd..0464b3d9862 100644 --- a/packages/syft/src/syft/custom_worker/builder_k8s.py +++ b/packages/syft/src/syft/custom_worker/builder_k8s.py @@ -1,6 +1,5 @@ # stdlib from hashlib import sha256 -import os from pathlib import Path from typing import Dict from typing import List @@ -11,14 +10,17 @@ from kr8s.objects import APIObject from kr8s.objects import ConfigMap from kr8s.objects import Job +from kr8s.objects import Secret # relative from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult -from .k8s import BUILD_OUTPUT_PVC +from .k8s import INTERNAL_REGISTRY_HOST from .k8s import JOB_COMPLETION_TTL from .k8s import KUBERNETES_NAMESPACE +from .utils import create_dockerconfig_json +from .utils import parse_tag __all__ = ["KubernetesBuilder"] @@ -39,6 +41,9 @@ def build_image( buildargs: Optional[dict] = None, **kwargs, ) -> ImageBuildResult: + image_digest = None + logs = None + config = None job_id = self._new_job_id(tag) if dockerfile: @@ -46,19 +51,19 @@ def build_image( elif dockerfile_path: dockerfile = dockerfile_path.read_text() - # Create a ConfigMap with the Dockerfile - config = self._create_build_config(job_id, dockerfile) - config.refresh() - - # Create and start the job - job = self._create_kaniko_build_job( - job_id=job_id, - tag=tag, - job_config=config, - build_args=buildargs, - ) - try: + # Create a ConfigMap with the Dockerfile + config = self._create_build_config(job_id, dockerfile) + config.refresh() + + # Create and start the job + job = self._create_kaniko_build_job( + job_id=job_id, + tag=tag, + job_config=config, + build_args=buildargs, + ) + # wait for job to complete/fail job.wait(["condition=Complete", "condition=Failed"]) @@ -78,7 +83,7 @@ def build_image( raise finally: # don't delete the job, kubernetes will gracefully do that for us - config.delete() + config and config.delete(propagation_policy="Background") return ImageBuildResult( image_hash=image_digest, @@ -93,18 +98,35 @@ def push_image( registry_url: str, **kwargs, ) -> ImagePushResult: - # Create and start the job + exit_code = 1 + logs = None job_id = self._new_job_id(tag) - job = self._create_push_job( - job_id=job_id, - tag=tag, - username=username, - password=password, - registry_url=registry_url, - ) - job.wait(["condition=Complete", "condition=Failed"]) - exit_code = self._get_container_exit_code(job)[0] - return ImagePushResult(logs=self._get_logs(job), exit_code=exit_code) + push_secret = None + + try: + push_secret = self._create_push_secret( + id=job_id, + url=registry_url, + username=username, + password=password, + ) + push_secret.refresh() + + job = self._create_push_job( + job_id=job_id, + tag=tag, + push_secret=push_secret, + ) + + job.wait(["condition=Complete", "condition=Failed"]) + exit_code = self._get_container_exit_code(job)[0] + logs = self._get_logs(job) + except Exception: + raise + finally: + push_secret and push_secret.delete(propagation_policy="Background") + + return ImagePushResult(logs=logs, exit_code=exit_code) def _new_job_id(self, tag: str) -> str: return self._get_tag_hash(tag)[:16] @@ -142,10 +164,6 @@ def _get_logs(self, job: Job) -> str: return "\n".join(logs) - def _check_success(self, job: Job) -> bool: - # TODO - return True - def _create_build_config(self, job_id: str, dockerfile: str) -> ConfigMap: config_map = ConfigMap( { @@ -159,6 +177,12 @@ def _create_build_config(self, job_id: str, dockerfile: str) -> ConfigMap: ) return self._create_or_get(config_map) + def _change_registry(self, old_tag: str, registry: str): + _, repo, tag = parse_tag(old_tag) + new_tag = f"{registry}/{repo}:{tag}" + print(f"changed from {old_tag} to {new_tag}") + return new_tag + def _create_kaniko_build_job( self, job_id: str, @@ -168,7 +192,13 @@ def _create_kaniko_build_job( ) -> Job: # for push build_args = build_args or {} - tag_hash = self._get_tag_hash(tag) + build_args_list = [] + + internal_tag = self._change_registry(tag, registry=INTERNAL_REGISTRY_HOST) + + for k, v in build_args.items(): + build_args_list.append(f'--build-arg="{k}={v}"') + job = Job( { "metadata": { @@ -190,32 +220,32 @@ def _create_kaniko_build_job( "name": "kaniko", "image": "gcr.io/kaniko-project/executor:latest", "args": [ - # build_args "--dockerfile=Dockerfile", "--context=dir:///workspace", - f"--destination={tag}", + f"--destination={internal_tag}", # Disabling --reproducible because it eats up a lot of CPU+RAM # https://github.com/GoogleContainerTools/kaniko/issues/1960 # https://github.com/GoogleContainerTools/kaniko/pull/2477 # "--reproducible", - # Build outputs - f"--tar-path=/output/{tag_hash}.tar", + # cache args + "--cache=true", + "--cache-copy-layers", + "--cache-run-layers", + f"--cache-repo={INTERNAL_REGISTRY_HOST}/builder-cache", + # outputs args "--digest-file=/dev/termination-log", - "--no-push", # other kaniko conf + f"--insecure-registry={INTERNAL_REGISTRY_HOST}", + f"--skip-tls-verify-registry={INTERNAL_REGISTRY_HOST}", "--log-format=text", "--verbosity=info", - ], + ] + + build_args_list, "volumeMounts": [ { "name": "build-input", "mountPath": "/workspace", }, - { - "name": "build-output", - "mountPath": "/output", - "readOnly": False, - }, ], "resources": { "requests": { @@ -237,12 +267,6 @@ def _create_kaniko_build_job( "name": job_config.metadata.name, }, }, - { - "name": "build-output", - "persistentVolumeClaim": { - "claimName": BUILD_OUTPUT_PVC, - }, - }, ], } }, @@ -256,27 +280,19 @@ def _create_push_job( self, job_id: str, tag: str, - username: str, - password: str, - registry_url: Optional[str] = None, + push_secret: Secret, ) -> Job: - tag_hash = self._get_tag_hash(tag) - registry_url = registry_url or tag.split("/")[0] - - extra_flags = "" - if os.getenv("DEV_MODE") == "True": - extra_flags = "--insecure" + internal_tag = self._change_registry(tag, registry=INTERNAL_REGISTRY_HOST) + internal_reg, internal_repo, _ = parse_tag(internal_tag) run_cmds = [ - "echo Logging in to $REG_URL with user $REG_USERNAME...", - # login to registry - "crane auth login $REG_URL -u $REG_USERNAME -p $REG_PASSWORD", # push with credentials - "echo Pushing image....", - f"crane push --image-refs /dev/termination-log {extra_flags} /output/{tag_hash}.tar {tag}", - # cleanup built tarfile - "echo Cleaning up tar....", - f"rm /output/{tag_hash}.tar", + "echo Pushing image...", + f"crane copy {internal_tag} {tag}", + # cleanup image from internal registry + "echo Cleaning up...", + f"IMG_DIGEST=$(crane digest {internal_tag})", + f"crane delete {internal_reg}/{internal_repo}@$IMG_DIGEST; echo Done", ] job = Job( @@ -301,27 +317,14 @@ def _create_push_job( "name": "crane", # debug is needed for "sh" to be available "image": "gcr.io/go-containerregistry/crane:debug", - "env": [ - { - "name": "REG_URL", - "value": registry_url, - }, - { - "name": "REG_USERNAME", - "value": username, - }, - { - "name": "REG_PASSWORD", - "value": password, - }, - ], "command": ["sh"], "args": ["-c", " && ".join(run_cmds)], "volumeMounts": [ { - "name": "build-output", - "mountPath": "/output", - "readOnly": False, + "name": "push-secret", + "mountPath": "/root/.docker/config.json", + "subPath": "config.json", + "readOnly": True, }, ], "resources": { @@ -339,9 +342,15 @@ def _create_push_job( ], "volumes": [ { - "name": "build-output", - "persistentVolumeClaim": { - "claimName": BUILD_OUTPUT_PVC, + "name": "push-secret", + "secret": { + "secretName": push_secret.metadata.name, + "items": [ + { + "key": ".dockerconfigjson", + "path": "config.json", + }, + ], }, }, ], @@ -352,6 +361,32 @@ def _create_push_job( ) return self._create_or_get(job) + def _create_push_secret(self, id: str, url: str, username: str, password: str): + _secret = Secret( + { + "metadata": { + "name": f"push-secret-{id}", + "labels": { + "app.kubernetes.io/name": KUBERNETES_NAMESPACE, + "app.kubernetes.io/component": "builder", + "app.kubernetes.io/managed-by": "kr8s", + }, + }, + "type": "kubernetes.io/dockerconfigjson", + "data": { + ".dockerconfigjson": create_dockerconfig_json( + registries=[ + # authorize internal registry? + (INTERNAL_REGISTRY_HOST, "username", id), + (url, username, password), + ] + ) + }, + } + ) + + return self._create_or_get(_secret) + def _create_or_get(self, obj: APIObject) -> APIObject: if not obj.exists(): obj.create() diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 491d7333b38..be64ff051e7 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -10,15 +10,16 @@ # Time after which Job will be deleted JOB_COMPLETION_TTL = 60 -# Persistent volume claim for storing build output -BUILD_OUTPUT_PVC = "worker-builds" - # Kubernetes namespace KUBERNETES_NAMESPACE = os.getenv("K8S_NAMESPACE", "syft") # Kubernetes runtime flag IN_KUBERNETES = os.getenv("CONTAINER_HOST") == "k8s" +# Internal registry URL +DEFAULT_INTERNAL_REGISTRY = f"registry.{KUBERNETES_NAMESPACE}.svc.cluster.local" +INTERNAL_REGISTRY_HOST = os.getenv("INTERNAL_REGISTRY_HOST", DEFAULT_INTERNAL_REGISTRY) + class PodPhase(Enum): Pending = "Pending" diff --git a/packages/syft/src/syft/custom_worker/utils.py b/packages/syft/src/syft/custom_worker/utils.py new file mode 100644 index 00000000000..6a19908d5d7 --- /dev/null +++ b/packages/syft/src/syft/custom_worker/utils.py @@ -0,0 +1,33 @@ +# stdlib +import base64 +import json +from typing import Iterable +from typing import Optional +from typing import Tuple + + +def parse_tag(tag: str) -> Tuple[Optional[str], str, str]: + url, tag = tag.rsplit(":", 1) + args = url.rsplit("/", 2) + + if len(args) == 3: + registry = args[0] + repo = "/".join(args[1:]) + else: + registry = None + repo = "/".join(args) + + return registry, repo, tag + + +def create_dockerconfig_json(registries: Iterable[Tuple[str, str, str]]): + config = {"auths": {}} + + for url, uname, passwd in registries: + config["auths"][url] = { + "username": uname, + "password": passwd, + "auth": base64.b64encode(f"{uname}:{passwd}".encode()).decode(), + } + + return base64.b64encode(json.dumps(config).encode()).decode() diff --git a/packages/syft/src/syft/service/worker/image_identifier.py b/packages/syft/src/syft/service/worker/image_identifier.py index 531b5d3b726..616288e209a 100644 --- a/packages/syft/src/syft/service/worker/image_identifier.py +++ b/packages/syft/src/syft/service/worker/image_identifier.py @@ -1,12 +1,12 @@ # stdlib from typing import Optional -from typing import Tuple from typing import Union # third party from typing_extensions import Self # relative +from ...custom_worker.utils import parse_tag from ...serde.serializable import serializable from ...types.base import SyftBaseModel from .image_registry import SyftImageRegistry @@ -38,7 +38,7 @@ class SyftWorkerImageIdentifier(SyftBaseModel): @classmethod def with_registry(cls, tag: str, registry: SyftImageRegistry) -> Self: """Build a SyftWorkerImageTag from Docker tag & a previously created SyftImageRegistry object.""" - registry_str, repo, tag = SyftWorkerImageIdentifier.parse_str(tag) + registry_str, repo, tag = parse_tag(tag) # if we parsed a registry string, make sure it matches the registry object if registry_str and registry_str != registry.url: @@ -49,23 +49,9 @@ def with_registry(cls, tag: str, registry: SyftImageRegistry) -> Self: @classmethod def from_str(cls, tag: str) -> Self: """Build a SyftWorkerImageTag from a pure-string standard Docker tag.""" - registry, repo, tag = SyftWorkerImageIdentifier.parse_str(tag) + registry, repo, tag = parse_tag(tag) return cls(repo=repo, registry=registry, tag=tag) - @staticmethod - def parse_str(tag: str) -> Tuple[Optional[str], str, str]: - url, tag = tag.rsplit(":", 1) - args = url.rsplit("/", 2) - - if len(args) == 3: - registry = args[0] - repo = "/".join(args[1:]) - else: - registry = None - repo = "/".join(args) - - return registry, repo, tag - @property def repo_with_tag(self) -> str: if self.repo or self.tag: diff --git a/tox.ini b/tox.ini index 838d8fe35e5..5bb05684027 100644 --- a/tox.ini +++ b/tox.ini @@ -1018,6 +1018,7 @@ commands = bash -c 'kubectl delete pvc --all --namespace syft || true' bash -c 'kubectl delete secret --all --namespace syft || true' bash -c 'kubectl delete configmap --all --namespace syft || true' + bash -c 'kubectl delete serviceaccount --all --namespace syft || true' [testenv:dev.k8s.destroy] description = Destroy local Kubernetes cluster From 3d25170303017b4e8ced4731194eadc4f7312c39 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sat, 3 Feb 2024 01:56:55 +0530 Subject: [PATCH 08/46] [k8s] mount node creds as config instead of volume --- .../src/syft/custom_worker/builder_k8s.py | 109 +++------ packages/syft/src/syft/custom_worker/k8s.py | 182 ++++++++++++++- .../syft/src/syft/custom_worker/runner_k8s.py | 217 +++++++----------- packages/syft/src/syft/custom_worker/utils.py | 41 ++-- .../syft/service/worker/image_identifier.py | 6 +- .../syft/src/syft/service/worker/utils.py | 68 +++++- 6 files changed, 370 insertions(+), 253 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py index 0464b3d9862..395028d69b4 100644 --- a/packages/syft/src/syft/custom_worker/builder_k8s.py +++ b/packages/syft/src/syft/custom_worker/builder_k8s.py @@ -6,8 +6,6 @@ from typing import Optional # third party -import kr8s -from kr8s.objects import APIObject from kr8s.objects import ConfigMap from kr8s.objects import Job from kr8s.objects import Secret @@ -19,8 +17,9 @@ from .k8s import INTERNAL_REGISTRY_HOST from .k8s import JOB_COMPLETION_TTL from .k8s import KUBERNETES_NAMESPACE -from .utils import create_dockerconfig_json -from .utils import parse_tag +from .k8s import KubeUtils +from .k8s import get_kr8s_client +from .utils import ImageUtils __all__ = ["KubernetesBuilder"] @@ -30,8 +29,10 @@ class BuildFailed(Exception): class KubernetesBuilder(BuilderBase): + COMPONENT = "builder" + def __init__(self): - self.client = kr8s.api(namespace=KUBERNETES_NAMESPACE) + self.client = get_kr8s_client() def build_image( self, @@ -72,7 +73,7 @@ def build_image( image_digest = self._get_image_digest(job) if not image_digest: - exit_code = self._get_container_exit_code(job) + exit_code = self._get_exit_code(job) raise BuildFailed( "Failed to build the image. " f"Kaniko exit code={exit_code}. " @@ -83,7 +84,7 @@ def build_image( raise finally: # don't delete the job, kubernetes will gracefully do that for us - config and config.delete(propagation_policy="Background") + config and config.delete(propagation_policy="Foreground") return ImageBuildResult( image_hash=image_digest, @@ -119,12 +120,12 @@ def push_image( ) job.wait(["condition=Complete", "condition=Failed"]) - exit_code = self._get_container_exit_code(job)[0] + exit_code = self._get_exit_code(job)[0] logs = self._get_logs(job) except Exception: raise finally: - push_secret and push_secret.delete(propagation_policy="Background") + push_secret and push_secret.delete(propagation_policy="Foreground") return ImagePushResult(logs=logs, exit_code=exit_code) @@ -137,51 +138,35 @@ def _get_tag_hash(self, tag: str) -> str: def _get_image_digest(self, job: Job) -> Optional[str]: selector = {"batch.kubernetes.io/job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) - for pod in pods: - for container_status in pod.status.containerStatuses: - if container_status.state.terminated.exitCode != 0: - continue - return container_status.state.terminated.message - return None - - def _get_container_exit_code(self, job: Job) -> List[int]: + return KubeUtils.get_container_exit_message(pods) + + def _get_exit_code(self, job: Job) -> List[int]: selector = {"batch.kubernetes.io/job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) - exit_codes = [] - for pod in pods: - for container_status in pod.status.containerStatuses: - exit_codes.append(container_status.state.terminated.exitCode) - return exit_codes + return KubeUtils.get_container_exit_code(pods) def _get_logs(self, job: Job) -> str: selector = {"batch.kubernetes.io/job-name": job.metadata.name} pods = self.client.get("pods", label_selector=selector) - logs = [] - for pod in pods: - logs.append(f"----------Logs for pod={pod.metadata.name}----------") - for log in pod.logs(): - logs.append(log) - - return "\n".join(logs) + return KubeUtils.get_logs(pods) def _create_build_config(self, job_id: str, dockerfile: str) -> ConfigMap: config_map = ConfigMap( { "metadata": { "name": f"build-{job_id}", + "labels": { + "app.kubernetes.io/name": KUBERNETES_NAMESPACE, + "app.kubernetes.io/component": KubernetesBuilder.COMPONENT, + "app.kubernetes.io/managed-by": "kr8s", + }, }, "data": { "Dockerfile": dockerfile, }, } ) - return self._create_or_get(config_map) - - def _change_registry(self, old_tag: str, registry: str): - _, repo, tag = parse_tag(old_tag) - new_tag = f"{registry}/{repo}:{tag}" - print(f"changed from {old_tag} to {new_tag}") - return new_tag + return KubeUtils.create_or_get(config_map) def _create_kaniko_build_job( self, @@ -194,7 +179,7 @@ def _create_kaniko_build_job( build_args = build_args or {} build_args_list = [] - internal_tag = self._change_registry(tag, registry=INTERNAL_REGISTRY_HOST) + internal_tag = ImageUtils.change_registry(tag, registry=INTERNAL_REGISTRY_HOST) for k, v in build_args.items(): build_args_list.append(f'--build-arg="{k}={v}"') @@ -205,7 +190,7 @@ def _create_kaniko_build_job( "name": f"build-{job_id}", "labels": { "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": "builder", + "app.kubernetes.io/component": KubernetesBuilder.COMPONENT, "app.kubernetes.io/managed-by": "kr8s", }, }, @@ -274,7 +259,7 @@ def _create_kaniko_build_job( } ) - return self._create_or_get(job) + return KubeUtils.create_or_get(job) def _create_push_job( self, @@ -282,8 +267,8 @@ def _create_push_job( tag: str, push_secret: Secret, ) -> Job: - internal_tag = self._change_registry(tag, registry=INTERNAL_REGISTRY_HOST) - internal_reg, internal_repo, _ = parse_tag(internal_tag) + internal_tag = ImageUtils.change_registry(tag, registry=INTERNAL_REGISTRY_HOST) + internal_reg, internal_repo, _ = ImageUtils.parse_tag(internal_tag) run_cmds = [ # push with credentials @@ -302,7 +287,7 @@ def _create_push_job( "name": f"push-{job_id}", "labels": { "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": "builder", + "app.kubernetes.io/component": KubernetesBuilder.COMPONENT, "app.kubernetes.io/managed-by": "kr8s", }, }, @@ -359,37 +344,15 @@ def _create_push_job( }, } ) - return self._create_or_get(job) + return KubeUtils.create_or_get(job) def _create_push_secret(self, id: str, url: str, username: str, password: str): - _secret = Secret( - { - "metadata": { - "name": f"push-secret-{id}", - "labels": { - "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": "builder", - "app.kubernetes.io/managed-by": "kr8s", - }, - }, - "type": "kubernetes.io/dockerconfigjson", - "data": { - ".dockerconfigjson": create_dockerconfig_json( - registries=[ - # authorize internal registry? - (INTERNAL_REGISTRY_HOST, "username", id), - (url, username, password), - ] - ) - }, - } + return KubeUtils.create_dockerconfig_secret( + secret_name=f"push-secret-{id}", + component=KubernetesBuilder.COMPONENT, + registries=[ + # TODO: authorize internal registry? + (INTERNAL_REGISTRY_HOST, "username", id), + (url, username, password), + ], ) - - return self._create_or_get(_secret) - - def _create_or_get(self, obj: APIObject) -> APIObject: - if not obj.exists(): - obj.create() - else: - obj.refresh() - return obj diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index be64ff051e7..f3c99934b91 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -1,10 +1,21 @@ # stdlib +import base64 from enum import Enum +from functools import cache +import json import os +from typing import Dict +from typing import Iterable +from typing import List from typing import Optional +from typing import Tuple +from typing import Union # third party -from kr8s._data_utils import list_dict_unpack +import kr8s +from kr8s.objects import APIObject +from kr8s.objects import Pod +from kr8s.objects import Secret from pydantic import BaseModel # Time after which Job will be deleted @@ -37,7 +48,7 @@ class PodCondition(BaseModel): @classmethod def from_conditions(cls, conditions: list): - pod_cond = list_dict_unpack(conditions, key="type", value="status") + pod_cond = KubeUtils.list_dict_unpack(conditions, key="type", value="status") pod_cond_flags = {k: v == "True" for k, v in pod_cond.items()} return cls( pod_scheduled=pod_cond_flags.get("PodScheduled", False), @@ -83,3 +94,170 @@ def from_status_dict(cls: "PodStatus", status: dict): status.get("containerStatuses", {})[0] ), ) + + +@cache +def get_kr8s_client() -> kr8s.Api: + IN_KUBERNETES = True + if not IN_KUBERNETES: + raise RuntimeError("Not inside a kubernetes environment") + return kr8s.api(namespace=KUBERNETES_NAMESPACE) + + +class KubeUtils: + """ + This class contains utility functions for interacting with kubernetes objects. + + DO NOT call `get_kr8s_client()` inside this class, instead pass it as an argument to the functions. + This is to avoid calling these functions on resources across namespaces! + """ + + OPAQUE_SECRET = "Opaque" + + @staticmethod + def resolve_pod(client: kr8s.Api, pod: Union[str, Pod]) -> Optional[Pod]: + """Return the first pod that matches the given name""" + if isinstance(pod, Pod): + return pod + + for _pod in client.get("pods", pod): + return _pod + + @staticmethod + def get_logs(pods: List[Pod]): + """Combine and return logs for all the pods as string""" + logs = [] + for pod in pods: + logs.append(f"----------Logs for pod={pod.metadata.name}----------") + for log in pod.logs(): + logs.append(log) + + return "\n".join(logs) + + @staticmethod + def get_pod_status(pod: Pod) -> Optional[PodStatus]: + """Map the status of the given pod to PodStatuss.""" + if not pod: + return None + return PodStatus.from_status_dict(pod.status) + + @staticmethod + def get_pod_env(pod: Pod) -> Optional[List[Dict]]: + """Return the environment variables of the first container in the pod.""" + if not pod: + return + + for container in pod.spec.containers: + return container.env.to_list() + + @staticmethod + def get_container_exit_code(pods: List[Pod]) -> List[int]: + """Return the exit codes of all the containers in the given pods.""" + exit_codes = [] + for pod in pods: + for container_status in pod.status.containerStatuses: + exit_codes.append(container_status.state.terminated.exitCode) + return exit_codes + + @staticmethod + def get_container_exit_message(pods: List[Pod]) -> Optional[str]: + """Return the exit message of the first container that exited with non-zero code.""" + for pod in pods: + for container_status in pod.status.containerStatuses: + if container_status.state.terminated.exitCode != 0: + continue + return container_status.state.terminated.message + return None + + @staticmethod + def b64encode_secret(data: str) -> str: + """Convert the data to base64 encoded string for Secret.""" + return base64.b64encode(data.encode()).decode() + + @staticmethod + def create_dockerconfig_secret( + secret_name: str, + component: str, + registries: Iterable[Tuple[str, str, str]], + ) -> Secret: + auths = {} + + for url, uname, passwd in registries: + auths[url] = { + "username": uname, + "password": passwd, + "auth": base64.b64encode(f"{uname}:{passwd}".encode()).decode(), + } + + config_str = json.dumps({"auths": auths}) + + return KubeUtils.create_secret( + secret_name=secret_name, + type="kubernetes.io/dockerconfigjson", + component=component, + data={ + ".dockerconfigjson": KubeUtils.b64encode_secret(config_str), + }, + ) + + @staticmethod + def create_secret( + secret_name: str, + type: str, + component: str, + data: str, + encoded=True, + ) -> Secret: + if not encoded: + for k, v in data.items(): + data[k] = KubeUtils.b64encode_secret(v) + + secret = Secret( + { + "metadata": { + "name": secret_name, + "labels": { + "app.kubernetes.io/name": KUBERNETES_NAMESPACE, + "app.kubernetes.io/component": component, + "app.kubernetes.io/managed-by": "kr8s", + }, + }, + "type": type, + "data": data, + } + ) + return KubeUtils.create_or_get(secret) + + @staticmethod + def create_or_get(obj: APIObject) -> APIObject: + if obj.exists(): + obj.refresh() + else: + obj.create() + return obj + + @staticmethod + def patch_env_vars(env_list: List[Dict], env_dict: Dict) -> List[Dict]: + """Patch kubernetes pod environment variables in the list with the provided dictionary.""" + + # update existing + for item in env_list: + k = item["name"] + if k in env_dict: + v = env_dict.pop(k) + item["value"] = v + + # append remaining + for k, v in env_dict.items(): + env_list.append({"name": k, "value": v}) + + return env_list + + @staticmethod + def list_dict_unpack( + input_list: List[Dict], + key: str = "key", + value: str = "value", + ) -> Dict: + # Snapshot from kr8s._data_utils + return {i[key]: i[value] for i in input_list} diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 0838836989b..dd1266b98ed 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -1,62 +1,67 @@ # stdlib -import base64 -import copy -import json -import os from time import sleep +from typing import Dict from typing import List from typing import Optional from typing import Union # third party -import kr8s -from kr8s.objects import APIObject from kr8s.objects import Pod from kr8s.objects import Secret from kr8s.objects import StatefulSet # relative from .k8s import KUBERNETES_NAMESPACE +from .k8s import KubeUtils from .k8s import PodStatus +from .k8s import get_kr8s_client class KubernetesRunner: def __init__(self): - self.client = kr8s.api(namespace=KUBERNETES_NAMESPACE) + self.client = get_kr8s_client() def create_pool( self, pool_name: str, tag: str, replicas: int = 1, - env_vars: Optional[dict] = None, + env_vars: Optional[List[Dict]] = None, + mount_secrets: Optional[Dict] = None, reg_username: Optional[str] = None, reg_password: Optional[str] = None, reg_url: Optional[str] = None, **kwargs, ) -> StatefulSet: - # create pull secret if registry credentials are passed - pull_secret = None - if reg_username and reg_password and reg_url: - pull_secret = self._create_image_pull_secret( - pool_name, - reg_username, - reg_password, - reg_url, + try: + # create pull secret if registry credentials are passed + pull_secret = None + if reg_username and reg_password and reg_url: + pull_secret = self._create_image_pull_secret( + pool_name, + reg_username, + reg_password, + reg_url, + ) + + # create a stateful set deployment + deployment = self._create_stateful_set( + pool_name=pool_name, + tag=tag, + replicas=replicas, + env_vars=env_vars, + mount_secrets=mount_secrets, + pull_secret=pull_secret, + **kwargs, ) - # create a stateful set deployment - deployment = self._create_stateful_set( - pool_name, - tag, - replicas, - env_vars, - pull_secret=pull_secret, - **kwargs, - ) - - # wait for replicas to be available and ready - self.wait(deployment, available_replicas=replicas) + # wait for replicas to be available and ready + self.wait(deployment, available_replicas=replicas) + except Exception: + raise + finally: + if pull_secret: + pull_secret.delete(propagation_policy="Foreground") # return return deployment @@ -78,21 +83,21 @@ def get_pool(self, pool_name: str) -> Optional[StatefulSet]: def delete_pool(self, pool_name: str) -> bool: selector = {"app.kubernetes.io/component": pool_name} for _set in self.client.get("statefulsets", label_selector=selector): - _set.delete() + _set.delete(propagation_policy="Foreground") for _secret in self.client.get("secrets", label_selector=selector): - _secret.delete() + _secret.delete(propagation_policy="Foreground") return True def delete_pod(self, pod_name: str) -> bool: pods = self.client.get("pods", pod_name) for pod in pods: - pod.delete() + pod.delete(propagation_policy="Foreground") return True return False - def get_pods(self, pool_name: str) -> List[Pod]: + def get_pool_pods(self, pool_name: str) -> List[Pod]: selector = {"app.kubernetes.io/component": pool_name} pods = self.client.get("pods", label_selector=selector) if len(pods) > 0: @@ -101,24 +106,15 @@ def get_pods(self, pool_name: str) -> List[Pod]: def get_pod_logs(self, pod_name: str) -> str: pods = self.client.get("pods", pod_name) - logs = [] - for pod in pods: - logs.append(f"----------Logs for pod={pod.metadata.name}----------") - for log in pod.logs(): - logs.append(log) - - return "\n".join(logs) + return KubeUtils.get_logs(pods) def get_pod_status(self, pod: Union[str, Pod]) -> Optional[PodStatus]: - if isinstance(pod, str): - pods = self.client.get("pods", pod) - if len(pods) == 0: - return None - pod = pods[0] - else: - pod.refresh() + pod = KubeUtils.resolve_pod(self.client, pod) + return KubeUtils.get_pod_status(pod) - return PodStatus.from_status_dict(pod.status) + def get_pod_env_vars(self, pod: Union[str, Pod]) -> Optional[List[Dict]]: + pod = KubeUtils.resolve_pod(self.client, pod) + return KubeUtils.get_pod_env(pod) def wait( self, @@ -143,21 +139,6 @@ def wait( timeout -= 1 sleep(1) - def _current_pod_name(self) -> str: - env_val = os.getenv("K8S_POD_NAME") - if env_val: - return env_val - - selector = {"app.kubernetes.io/component": "backend"} - for pod in self.client.get("pods", label_selector=selector): - return pod.name - - def _get_obj_from_list(self, objs: List[dict], name: str) -> dict: - """Helper function extract kubernetes object from list by name""" - for obj in objs: - if obj.name == name: - return obj - def _create_image_pull_secret( self, pool_name: str, @@ -166,67 +147,49 @@ def _create_image_pull_secret( reg_url: str, **kwargs, ): - _secret = Secret( - { - "metadata": { - "name": f"pull-secret-{pool_name}", - "labels": { - "app.kubernetes.io/name": KUBERNETES_NAMESPACE, - "app.kubernetes.io/component": pool_name, - "app.kubernetes.io/managed-by": "kr8s", - }, - }, - "type": "kubernetes.io/dockerconfigjson", - "data": { - ".dockerconfigjson": self._create_dockerconfig_json( - reg_username, - reg_password, - reg_url, - ) - }, - } + return KubeUtils.create_dockerconfig_secret( + secret_name=f"pull-secret-{pool_name}", + component=pool_name, + registries=[ + (reg_url, reg_username, reg_password), + ], ) - return self._create_or_get(_secret) - def _create_stateful_set( self, pool_name: str, tag: str, replicas=1, - env_vars: Optional[dict] = None, + env_vars: Optional[List[Dict]] = None, + mount_secrets: Optional[Dict] = None, pull_secret: Optional[Secret] = None, **kwargs, ) -> StatefulSet: """Create a stateful set for a pool""" - env_vars = env_vars or {} + volumes = [] + volume_mounts = [] pull_secret_obj = None - - _pod = Pod.get(self._current_pod_name()) - - creds_volume = self._get_obj_from_list( - objs=_pod.spec.volumes, - name="credentials-data", - ) - creds_volume_mount = self._get_obj_from_list( - objs=_pod.spec.containers[0].volumeMounts, - name="credentials-data", - ) - - env = _pod.spec.containers[0].env.to_list() - env_clone = copy.deepcopy(env) - - # update existing - for item in env_clone: - k = item["name"] - if k in env_vars: - v = env_vars.pop(k) - item["value"] = v - - # append remaining - for k, v in env_vars.items(): - env_clone.append({"name": k, "value": v}) + env_vars = env_vars or [] + + if mount_secrets: + for secret_name, mount_opts in mount_secrets.items(): + volumes.append( + { + "name": secret_name, + "secret": { + "secretName": secret_name, + }, + } + ) + volume_mounts.append( + { + "name": secret_name, + "mountPath": mount_opts.get("mountPath"), + "subPath": mount_opts.get("subPath"), + "readOnly": True, + } + ) if pull_secret: pull_secret_obj = [ @@ -265,41 +228,15 @@ def _create_stateful_set( "name": pool_name, "imagePullPolicy": "IfNotPresent", "image": tag, - "env": env_clone, - "volumeMounts": [creds_volume_mount], + "env": env_vars, + "volumeMounts": volume_mounts, } ], - "volumes": [creds_volume], + "volumes": volumes, "imagePullSecrets": pull_secret_obj, }, }, }, } ) - return self._create_or_get(stateful_set) - - def _create_or_get(self, obj: APIObject) -> APIObject: - if not obj.exists(): - obj.create() - else: - obj.refresh() - return obj - - def _create_dockerconfig_json( - self, - reg_username: str, - reg_password: str, - reg_url: str, - ): - config = { - "auths": { - reg_url: { - "username": reg_username, - "password": reg_password, - "auth": base64.b64encode( - f"{reg_username}:{reg_password}".encode() - ).decode(), - } - } - } - return base64.b64encode(json.dumps(config).encode()).decode() + return KubeUtils.create_or_get(stateful_set) diff --git a/packages/syft/src/syft/custom_worker/utils.py b/packages/syft/src/syft/custom_worker/utils.py index 6a19908d5d7..4ec45cc1358 100644 --- a/packages/syft/src/syft/custom_worker/utils.py +++ b/packages/syft/src/syft/custom_worker/utils.py @@ -1,33 +1,24 @@ # stdlib -import base64 -import json -from typing import Iterable from typing import Optional from typing import Tuple -def parse_tag(tag: str) -> Tuple[Optional[str], str, str]: - url, tag = tag.rsplit(":", 1) - args = url.rsplit("/", 2) +class ImageUtils: + @staticmethod + def parse_tag(tag: str) -> Tuple[Optional[str], str, str]: + url, tag = tag.rsplit(":", 1) + args = url.rsplit("/", 2) - if len(args) == 3: - registry = args[0] - repo = "/".join(args[1:]) - else: - registry = None - repo = "/".join(args) + if len(args) == 3: + registry = args[0] + repo = "/".join(args[1:]) + else: + registry = None + repo = "/".join(args) - return registry, repo, tag + return registry, repo, tag - -def create_dockerconfig_json(registries: Iterable[Tuple[str, str, str]]): - config = {"auths": {}} - - for url, uname, passwd in registries: - config["auths"][url] = { - "username": uname, - "password": passwd, - "auth": base64.b64encode(f"{uname}:{passwd}".encode()).decode(), - } - - return base64.b64encode(json.dumps(config).encode()).decode() + @staticmethod + def change_registry(tag: str, registry: str) -> str: + _, repo, tag = ImageUtils.parse_tag(tag) + return f"{registry}/{repo}:{tag}" diff --git a/packages/syft/src/syft/service/worker/image_identifier.py b/packages/syft/src/syft/service/worker/image_identifier.py index 616288e209a..43623f44d36 100644 --- a/packages/syft/src/syft/service/worker/image_identifier.py +++ b/packages/syft/src/syft/service/worker/image_identifier.py @@ -6,7 +6,7 @@ from typing_extensions import Self # relative -from ...custom_worker.utils import parse_tag +from ...custom_worker.utils import ImageUtils from ...serde.serializable import serializable from ...types.base import SyftBaseModel from .image_registry import SyftImageRegistry @@ -38,7 +38,7 @@ class SyftWorkerImageIdentifier(SyftBaseModel): @classmethod def with_registry(cls, tag: str, registry: SyftImageRegistry) -> Self: """Build a SyftWorkerImageTag from Docker tag & a previously created SyftImageRegistry object.""" - registry_str, repo, tag = parse_tag(tag) + registry_str, repo, tag = ImageUtils.parse_tag(tag) # if we parsed a registry string, make sure it matches the registry object if registry_str and registry_str != registry.url: @@ -49,7 +49,7 @@ def with_registry(cls, tag: str, registry: SyftImageRegistry) -> Self: @classmethod def from_str(cls, tag: str) -> Self: """Build a SyftWorkerImageTag from a pure-string standard Docker tag.""" - registry, repo, tag = parse_tag(tag) + registry, repo, tag = ImageUtils.parse_tag(tag) return cls(repo=repo, registry=registry, tag=tag) @property diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index 9f677eb7f98..e422f081b4e 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -1,6 +1,7 @@ # stdlib import contextlib import os +from pathlib import Path import socket import socketserver import sys @@ -19,6 +20,7 @@ from ...custom_worker.builder_types import ImagePushResult from ...custom_worker.config import DockerWorkerConfig from ...custom_worker.config import PrebuiltWorkerConfig +from ...custom_worker.k8s import KubeUtils from ...custom_worker.k8s import PodStatus from ...custom_worker.runner_k8s import KubernetesRunner from ...node.credentials import SyftVerifyKey @@ -36,6 +38,7 @@ DEFAULT_WORKER_IMAGE_TAG = "openmined/default-worker-image-cpu:0.0.1" DEFAULT_WORKER_POOL_NAME = "default-pool" +K8S_SECRET_NODE_CREDS = "node-creds" def backend_container_name() -> str: @@ -254,9 +257,46 @@ def run_workers_in_threads( return results +def prepare_worker_pool_env(runner: KubernetesRunner, env_vars: dict): + # get current backend pod name + backend_pod_name = os.getenv("K8S_POD_NAME") + if not backend_pod_name: + raise ValueError(message="Pod name not provided in environment variable") + + # get current backend's credentials path + creds_path = os.getenv("CREDENTIALS_PATH") + if not creds_path: + raise ValueError(message="Credentials path not provided") + + creds_path = Path(creds_path) + if not creds_path.exists(): + raise ValueError(message="Credentials file does not exist") + + # create a secret for the node credentials owned by the backend, not the pool. + node_secret = KubeUtils.create_secret( + secret_name=K8S_SECRET_NODE_CREDS, + type=KubeUtils.OPAQUE_SECRET, + component=backend_pod_name, + data={creds_path.name: creds_path.read_text()}, + encoded=False, + ) + + # clone and patch backend environment variables + backend_env = runner.get_pod_env_vars(backend_pod_name) or [] + env_vars = KubeUtils.patch_env_vars(backend_env, env_vars) + mount_secrets = { + node_secret.metadata.name: { + "mountPath": str(creds_path), + "subPath": creds_path.name, + }, + } + + return env_vars, mount_secrets + + def create_kubernetes_pool( runner: KubernetesRunner, - worker_image: SyftWorker, + tag: str, pool_name: str, replicas: int, queue_port: int, @@ -273,14 +313,13 @@ def create_kubernetes_pool( print( "Creating new pool " f"name={pool_name} " - f"tag={worker_image.image_identifier.full_name_with_tag} " + f"tag={tag} " f"replicas={replicas}" ) - pool = runner.create_pool( - pool_name=pool_name, - tag=worker_image.image_identifier.full_name_with_tag, - replicas=replicas, - env_vars={ + + env_vars, mount_secrets = prepare_worker_pool_env( + runner, + { "SYFT_WORKER": "True", "DEV_MODE": f"{debug}", "QUEUE_PORT": f"{queue_port}", @@ -289,6 +328,15 @@ def create_kubernetes_pool( "CREATE_PRODUCER": "False", "INMEMORY_WORKERS": "False", }, + ) + + # run the pool with args + secret + pool = runner.create_pool( + pool_name=pool_name, + tag=tag, + replicas=replicas, + env_vars=env_vars, + mount_secrets=mount_secrets, reg_username=reg_username, reg_password=reg_password, reg_url=reg_url, @@ -300,7 +348,7 @@ def create_kubernetes_pool( if error and pool: pool.delete() - return runner.get_pods(pool_name=pool_name) + return runner.get_pool_pods(pool_name=pool_name) def scale_kubernetes_pool( @@ -318,7 +366,7 @@ def scale_kubernetes_pool( except Exception as e: return SyftError(message=f"Failed to scale workers {e}") - return runner.get_pods(pool_name=pool_name) + return runner.get_pool_pods(pool_name=pool_name) def run_workers_in_kubernetes( @@ -339,7 +387,7 @@ def run_workers_in_kubernetes( if start_idx == 0: pool_pods = create_kubernetes_pool( runner=runner, - worker_image=worker_image, + tag=worker_image.image_identifier.full_name_with_tag, pool_name=pool_name, replicas=worker_count, queue_port=queue_port, From 3a8b84967b6d4cdd0a5cf55ea30af42e1c06b706 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sat, 3 Feb 2024 02:40:59 +0530 Subject: [PATCH 09/46] [k8s] update kr8s with latest bugfix --- packages/syft/setup.cfg | 2 +- .../syft/src/syft/custom_worker/runner_k8s.py | 27 ++----------------- 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index e098c8c9f4e..a740dc3c79b 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -62,7 +62,7 @@ syft = numpy>=1.23.5,<=1.24.4 pandas==1.5.3 docker==6.1.3 - kr8s==0.13.0 + kr8s==0.13.1 PyYAML==6.0.1 azure-storage-blob==12.19 diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index dd1266b98ed..9e7d01e9e26 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -1,5 +1,4 @@ # stdlib -from time import sleep from typing import Dict from typing import List from typing import Optional @@ -56,7 +55,8 @@ def create_pool( ) # wait for replicas to be available and ready - self.wait(deployment, available_replicas=replicas) + status_path = "{.status.availableReplicas}" + deployment.wait(f"jsonpath='{status_path}'={replicas}") except Exception: raise finally: @@ -116,29 +116,6 @@ def get_pod_env_vars(self, pod: Union[str, Pod]) -> Optional[List[Dict]]: pod = KubeUtils.resolve_pod(self.client, pod) return KubeUtils.get_pod_env(pod) - def wait( - self, - deployment: StatefulSet, - available_replicas: int, - timeout: int = 300, - ) -> None: - # TODO: Report wait('jsonpath=') bug to kr8s - # Until then this is the substitute implementation - - if available_replicas <= 0: - return - - while True: - if timeout == 0: - raise TimeoutError("Timeout waiting for replicas") - - deployment.refresh() - if deployment.status.availableReplicas == available_replicas: - break - - timeout -= 1 - sleep(1) - def _create_image_pull_secret( self, pool_name: str, From 892dd4ca90d48a6ae7bb21b56d81ef3a2223d13a Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sat, 3 Feb 2024 02:43:12 +0530 Subject: [PATCH 10/46] [k8s] fix security lint warning --- packages/syft/src/syft/service/worker/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index e422f081b4e..a5a486a5f85 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -38,7 +38,7 @@ DEFAULT_WORKER_IMAGE_TAG = "openmined/default-worker-image-cpu:0.0.1" DEFAULT_WORKER_POOL_NAME = "default-pool" -K8S_SECRET_NODE_CREDS = "node-creds" +K8S_NODE_CREDS_NAME = "node-creds" def backend_container_name() -> str: @@ -274,7 +274,7 @@ def prepare_worker_pool_env(runner: KubernetesRunner, env_vars: dict): # create a secret for the node credentials owned by the backend, not the pool. node_secret = KubeUtils.create_secret( - secret_name=K8S_SECRET_NODE_CREDS, + secret_name=K8S_NODE_CREDS_NAME, type=KubeUtils.OPAQUE_SECRET, component=backend_pod_name, data={creds_path.name: creds_path.read_text()}, From 9efdec9d999e09c41198989cd82f1a9aff3c8aa6 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sat, 3 Feb 2024 02:55:47 +0530 Subject: [PATCH 11/46] [k8s] fix another security lint warning --- packages/syft/src/syft/custom_worker/k8s.py | 2 +- packages/syft/src/syft/service/worker/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index f3c99934b91..0aba3a0a38d 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -112,7 +112,7 @@ class KubeUtils: This is to avoid calling these functions on resources across namespaces! """ - OPAQUE_SECRET = "Opaque" + OPAQUE_SECRET_TYPE = "Opaque" @staticmethod def resolve_pod(client: kr8s.Api, pod: Union[str, Pod]) -> Optional[Pod]: diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index a5a486a5f85..c4bac6fb8c0 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -275,7 +275,7 @@ def prepare_worker_pool_env(runner: KubernetesRunner, env_vars: dict): # create a secret for the node credentials owned by the backend, not the pool. node_secret = KubeUtils.create_secret( secret_name=K8S_NODE_CREDS_NAME, - type=KubeUtils.OPAQUE_SECRET, + type=KubeUtils.OPAQUE_SECRET_TYPE, component=backend_pod_name, data={creds_path.name: creds_path.read_text()}, encoded=False, From aad976d8fa10c32dda802d3ee4d3701a59f2f3c2 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sat, 3 Feb 2024 03:03:17 +0530 Subject: [PATCH 12/46] [k8s] fix manually placed flag --- packages/syft/src/syft/custom_worker/k8s.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 0aba3a0a38d..59e5253c2d0 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -98,7 +98,6 @@ def from_status_dict(cls: "PodStatus", status: dict): @cache def get_kr8s_client() -> kr8s.Api: - IN_KUBERNETES = True if not IN_KUBERNETES: raise RuntimeError("Not inside a kubernetes environment") return kr8s.api(namespace=KUBERNETES_NAMESPACE) From f3205c9d775fccd7efc3f7f842236e340b6046ae Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Sat, 3 Feb 2024 03:16:42 +0530 Subject: [PATCH 13/46] [k8s] remove OPAQUE_SECRET_TYPE --- packages/syft/src/syft/custom_worker/k8s.py | 2 -- packages/syft/src/syft/service/worker/utils.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 59e5253c2d0..427a12862ff 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -111,8 +111,6 @@ class KubeUtils: This is to avoid calling these functions on resources across namespaces! """ - OPAQUE_SECRET_TYPE = "Opaque" - @staticmethod def resolve_pod(client: kr8s.Api, pod: Union[str, Pod]) -> Optional[Pod]: """Return the first pod that matches the given name""" diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index c4bac6fb8c0..ca472683702 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -275,7 +275,7 @@ def prepare_worker_pool_env(runner: KubernetesRunner, env_vars: dict): # create a secret for the node credentials owned by the backend, not the pool. node_secret = KubeUtils.create_secret( secret_name=K8S_NODE_CREDS_NAME, - type=KubeUtils.OPAQUE_SECRET_TYPE, + type="Opaque", component=backend_pod_name, data={creds_path.name: creds_path.read_text()}, encoded=False, From fc0002dd37268a997bfc43d807d2012e57d060d3 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 5 Feb 2024 11:05:09 +0530 Subject: [PATCH 14/46] [helm] remove ReadWriteMany --- packages/grid/helm/syft/templates/backend-statefulset.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index cafe0851aa0..ebebae80cc2 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -158,12 +158,7 @@ spec: name: credentials-data spec: accessModes: - # k3d does not support ReadWriteMany - {{- if .Values.configuration.devmode }} - ReadWriteOnce - {{- else }} - - ReadWriteMany - {{- end }} resources: requests: storage: 100Mi From b48390ec46f7a6872ce11769540f24debbf9b5c4 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 5 Feb 2024 12:15:50 +0530 Subject: [PATCH 15/46] [k8s] fix wait attribute exception --- packages/syft/src/syft/custom_worker/runner_k8s.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 9e7d01e9e26..ff2c3120ebb 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -15,6 +15,8 @@ from .k8s import PodStatus from .k8s import get_kr8s_client +JSONPATH_AVAILABLE_REPLICAS = "{.status.availableReplicas}" + class KubernetesRunner: def __init__(self): @@ -55,8 +57,7 @@ def create_pool( ) # wait for replicas to be available and ready - status_path = "{.status.availableReplicas}" - deployment.wait(f"jsonpath='{status_path}'={replicas}") + deployment.wait(f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}") except Exception: raise finally: @@ -71,7 +72,7 @@ def scale_pool(self, pool_name: str, replicas: int) -> Optional[StatefulSet]: if not deployment: return None deployment.scale(replicas) - self.wait(deployment, available_replicas=replicas) + deployment.wait(f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}") return deployment def get_pool(self, pool_name: str) -> Optional[StatefulSet]: From ac77fcf518d6f775ad88b177354a1047c026e206 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 5 Feb 2024 12:16:03 +0530 Subject: [PATCH 16/46] [helm] fix VERSION string error --- packages/grid/helm/syft/templates/frontend-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index dfc5d39549a..f43fd0018dc 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -29,7 +29,7 @@ spec: command: null env: - name: VERSION - value: {{ .Values.syft.version }} + value: "{{ .Values.syft.version }}" - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE From 4bb02b803e57c14cf7b544e9ba59c514a91c0012 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 5 Feb 2024 12:16:45 +0530 Subject: [PATCH 17/46] [scripts] add script to build images locally --- scripts/build_images.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 scripts/build_images.sh diff --git a/scripts/build_images.sh b/scripts/build_images.sh new file mode 100644 index 00000000000..78f3a0255bf --- /dev/null +++ b/scripts/build_images.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t syftpilot.azurecr.io/openmined/grid-backend:$1 ./packages +docker image build -f ./packages/grid/frontend/frontend.dockerfile --target grid-ui-development -t syftpilot.azurecr.io/openmined/grid-frontend:$1 ./packages/grid/frontend +docker image build -f ./packages/grid/seaweedfs/seaweedfs.dockerfile --build-arg SEAWEEDFS_VERSION=3.59 -t syftpilot.azurecr.io/openmined/grid-seaweedfs:$1 ./packages/grid/seaweedfs From 9bfd515b192bb3d54224b71ece52db72fbd658d2 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 5 Feb 2024 14:11:20 +0530 Subject: [PATCH 18/46] [worker] fix registry domain validation --- .../syft/src/syft/service/worker/image_registry.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/worker/image_registry.py b/packages/syft/src/syft/service/worker/image_registry.py index cf3c36c0e0d..806a0946d2b 100644 --- a/packages/syft/src/syft/service/worker/image_registry.py +++ b/packages/syft/src/syft/service/worker/image_registry.py @@ -1,4 +1,5 @@ # stdlib +import re from urllib.parse import urlparse # third party @@ -10,6 +11,8 @@ from ...types.syft_object import SyftObject from ...types.uid import UID +REGX_DOMAIN = re.compile(r"^(localhost|([a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*))(\:\d{1,5})?$") + @serializable() class SyftImageRegistry(SyftObject): @@ -26,15 +29,19 @@ class SyftImageRegistry(SyftObject): @validator("url") def validate_url(cls, val: str): - if val.startswith("http") or "://" in val: - raise ValueError("Registry URL must be a valid RFC 3986 URI") + if not val: + raise ValueError("Invalid Registry URL. Must not be empty") + + if not bool(re.match(REGX_DOMAIN, val)): + raise ValueError("Invalid Registry URL. Must be a valid domain.") + return val @classmethod def from_url(cls, full_str: str): + # this is only for urlparse if "://" not in full_str: full_str = f"http://{full_str}" - parsed = urlparse(full_str) # netloc includes the host & port, so local dev should work as expected From fcf0112776d3ccbc0ad639f72989e60f12737225 Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 5 Feb 2024 12:06:07 +0200 Subject: [PATCH 19/46] revert to the initial default value for S3_VOLUME_SIZE_MB --- packages/grid/default.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/default.env b/packages/grid/default.env index f6be744d49c..d599e47cf4e 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -71,7 +71,7 @@ S3_ROOT_PWD="admin" # needs randomizing S3_REGION="us-east-1" #not-using S3_PRESIGNED_TIMEOUT_SECS=1800 -S3_VOLUME_SIZE_MB=40000 +S3_VOLUME_SIZE_MB=1024 # Jax From 6147a2cbb53b236824fb45fd33648311f01b5303 Mon Sep 17 00:00:00 2001 From: teo Date: Mon, 5 Feb 2024 12:06:34 +0200 Subject: [PATCH 20/46] added hagrid option for S3_VOLUME_SIZE_MB --- packages/hagrid/hagrid/cli.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 6fea1d62ae0..8a6bd035fe4 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -460,6 +460,13 @@ def clean(location: str) -> None: type=str, help="Set root password for s3 blob storage", ) +@click.option( + '--set-s3-volume-size-mb', + default=1024, + required=False, + type=click.IntRange(1024, 50000), + help="Set the volume size limit (in MBs)" +) def launch(args: TypeTuple[str], **kwargs: Any) -> None: verb = get_launch_verb() try: @@ -1258,6 +1265,7 @@ def create_launch_cmd( if parsed_kwargs["use_blob_storage"]: parsed_kwargs["set_s3_username"] = kwargs["set_s3_username"] parsed_kwargs["set_s3_password"] = kwargs["set_s3_password"] + parsed_kwargs['set_s3_volume_size_mb'] = kwargs["set_s3_volume_size_mb"] parsed_kwargs["node_count"] = ( int(kwargs["node_count"]) if "node_count" in kwargs else 1 @@ -2262,6 +2270,9 @@ def create_launch_docker_cmd( if "set_s3_password" in kwargs and kwargs["set_s3_password"] is not None: envs["S3_ROOT_PWD"] = kwargs["set_s3_password"] + if "set_s3_volume_size_mb" in kwargs and kwargs["set_s3_volume_size_mb"] is not None: + envs["S3_VOLUME_SIZE_MB"] = kwargs['set_s3_volume_size_mb'] + if "release" in kwargs: envs["RELEASE"] = kwargs["release"] From cd7a9b692544aebe98aaa07aa61e055ddf5c72fc Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 5 Feb 2024 18:55:14 +0530 Subject: [PATCH 21/46] update test build config to work with only docker --- .../syft/src/syft/custom_worker/config.py | 57 +++++++++++-------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index ccb516e5c59..fa49d5b1c8b 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -1,14 +1,18 @@ # stdlib +import contextlib from hashlib import sha256 +import io +import json from pathlib import Path from typing import Any from typing import Dict +from typing import Iterable from typing import List -from typing import Literal from typing import Optional from typing import Union # third party +import docker from packaging import version from pydantic import validator from typing_extensions import Self @@ -17,8 +21,8 @@ # relative from ..serde.serializable import serializable from ..service.response import SyftError +from ..service.response import SyftSuccess from ..types.base import SyftBaseModel -from .builder_types import ImageBuildResult PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -163,28 +167,33 @@ def __str__(self) -> str: def set_description(self, description_text: str) -> None: self.description = description_text - def test_image_build( - self, orchestration_type: Literal["docker", "k8s"], tag: str, **build_args - ) -> Union[ImageBuildResult, SyftError]: - # relative - from .builder_docker import DockerBuilder - from .builder_k8s import KubernetesBuilder - - builder = ( - KubernetesBuilder() if orchestration_type == "k8s" else DockerBuilder() - ) - - # TODO: Remove this check once we know how test with k8s - if orchestration_type == "k8s": - return SyftError( - message="We currently support test builds using `docker` only." - ) - + def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: try: - return builder.build_image( - tag=tag, - dockerfile=self.dockerfile, - buildargs=build_args, - ) + with contextlib.closing(docker.from_env()) as client: + if not client.ping(): + return SyftError( + "Cannot reach docker server. Please check if docker is running." + ) + + kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8")) + _, logs = client.images.build( + tag=tag, + timeout=self.BUILD_MAX_WAIT, + **kwargs, + ) + return SyftSuccess(message=self._parse_output(logs)) except Exception as e: return SyftError(message=f"Failed to build: {e}") + + @staticmethod + def _parse_output(log_iterator: Iterable) -> str: + log = "" + for line in log_iterator: + for item in line.values(): + if isinstance(item, str): + log += item + elif isinstance(item, dict): + log += json.dumps(item) + "\n" + else: + log += str(item) + return log From cb79f7cfedbd0f01528c5276fa2c553a4179b759 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Mon, 5 Feb 2024 19:08:19 +0530 Subject: [PATCH 22/46] [custom_worker] make a reusable function to convert iteratable to string [custom_worker] reuse the function to parse docker logs --- .../src/syft/custom_worker/builder_docker.py | 13 ++----------- packages/syft/src/syft/custom_worker/config.py | 18 ++---------------- packages/syft/src/syft/custom_worker/utils.py | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 27 deletions(-) create mode 100644 packages/syft/src/syft/custom_worker/utils.py diff --git a/packages/syft/src/syft/custom_worker/builder_docker.py b/packages/syft/src/syft/custom_worker/builder_docker.py index e1f24520c25..9446a5530a4 100644 --- a/packages/syft/src/syft/custom_worker/builder_docker.py +++ b/packages/syft/src/syft/custom_worker/builder_docker.py @@ -1,7 +1,6 @@ # stdlib import contextlib import io -import json from pathlib import Path from typing import Iterable from typing import Optional @@ -13,6 +12,7 @@ from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult +from .utils import iterator_to_string __all__ = ["DockerBuilder"] @@ -70,13 +70,4 @@ def push_image( return ImagePushResult(logs=result, exit_code=0) def _parse_output(self, log_iterator: Iterable) -> str: - log = "" - for line in log_iterator: - for item in line.values(): - if isinstance(item, str): - log += item - elif isinstance(item, dict): - log += json.dumps(item) + "\n" - else: - log += str(item) - return log + return iterator_to_string(iterator=log_iterator) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index fa49d5b1c8b..1b55e34e021 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -2,11 +2,9 @@ import contextlib from hashlib import sha256 import io -import json from pathlib import Path from typing import Any from typing import Dict -from typing import Iterable from typing import List from typing import Optional from typing import Union @@ -23,6 +21,7 @@ from ..service.response import SyftError from ..service.response import SyftSuccess from ..types.base import SyftBaseModel +from .utils import iterator_to_string PYTHON_DEFAULT_VER = "3.11" PYTHON_MIN_VER = version.parse("3.10") @@ -181,19 +180,6 @@ def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: timeout=self.BUILD_MAX_WAIT, **kwargs, ) - return SyftSuccess(message=self._parse_output(logs)) + return SyftSuccess(message=iterator_to_string(iterator=logs)) except Exception as e: return SyftError(message=f"Failed to build: {e}") - - @staticmethod - def _parse_output(log_iterator: Iterable) -> str: - log = "" - for line in log_iterator: - for item in line.values(): - if isinstance(item, str): - log += item - elif isinstance(item, dict): - log += json.dumps(item) + "\n" - else: - log += str(item) - return log diff --git a/packages/syft/src/syft/custom_worker/utils.py b/packages/syft/src/syft/custom_worker/utils.py new file mode 100644 index 00000000000..be6d3cb5915 --- /dev/null +++ b/packages/syft/src/syft/custom_worker/utils.py @@ -0,0 +1,16 @@ +# stdlib +import json +from typing import Iterable + + +def iterator_to_string(iterator: Iterable) -> str: + log = "" + for line in iterator: + for item in line.values(): + if isinstance(item, str): + log += item + elif isinstance(item, dict): + log += json.dumps(item) + "\n" + else: + log += str(item) + return log From efc847d680526c4587d531fb4ca613eecbf5e139 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Mon, 5 Feb 2024 15:59:15 +0000 Subject: [PATCH 23/46] rename hagrid arg for volume size limit --- packages/hagrid/hagrid/cli.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 8a6bd035fe4..133b8837071 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -461,11 +461,11 @@ def clean(location: str) -> None: help="Set root password for s3 blob storage", ) @click.option( - '--set-s3-volume-size-mb', + "--set-volume-size-limit-mb", default=1024, required=False, type=click.IntRange(1024, 50000), - help="Set the volume size limit (in MBs)" + help="Set the volume size limit (in MBs)", ) def launch(args: TypeTuple[str], **kwargs: Any) -> None: verb = get_launch_verb() @@ -1265,7 +1265,7 @@ def create_launch_cmd( if parsed_kwargs["use_blob_storage"]: parsed_kwargs["set_s3_username"] = kwargs["set_s3_username"] parsed_kwargs["set_s3_password"] = kwargs["set_s3_password"] - parsed_kwargs['set_s3_volume_size_mb'] = kwargs["set_s3_volume_size_mb"] + parsed_kwargs["set_volume_size_limit_mb"] = kwargs["set_volume_size_limit_mb"] parsed_kwargs["node_count"] = ( int(kwargs["node_count"]) if "node_count" in kwargs else 1 @@ -2270,8 +2270,11 @@ def create_launch_docker_cmd( if "set_s3_password" in kwargs and kwargs["set_s3_password"] is not None: envs["S3_ROOT_PWD"] = kwargs["set_s3_password"] - if "set_s3_volume_size_mb" in kwargs and kwargs["set_s3_volume_size_mb"] is not None: - envs["S3_VOLUME_SIZE_MB"] = kwargs['set_s3_volume_size_mb'] + if ( + "set_volume_size_limit_mb" in kwargs + and kwargs["set_volume_size_limit_mb"] is not None + ): + envs["S3_VOLUME_SIZE_MB"] = kwargs["set_volume_size_limit_mb"] if "release" in kwargs: envs["RELEASE"] = kwargs["release"] From 1a06b6303ae4bafff7c7692b8f11f23d2a60819b Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 6 Feb 2024 14:38:00 +0530 Subject: [PATCH 24/46] [tox] fix non-dev resources being deleted --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 5bb05684027..570bad4c0e4 100644 --- a/tox.ini +++ b/tox.ini @@ -1014,6 +1014,7 @@ allowlist_externals = commands = bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft; sleep 3' bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5000 || true' + bash -c 'kubectl config use-context k3d-syft-dev' bash -c 'kubectl delete all --all --namespace syft || true' bash -c 'kubectl delete pvc --all --namespace syft || true' bash -c 'kubectl delete secret --all --namespace syft || true' From 0f11380f89271e15b27df78fa7f2410d0a1a4652 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 6 Feb 2024 14:41:28 +0530 Subject: [PATCH 25/46] address review comments --- packages/syft/src/syft/custom_worker/k8s.py | 2 +- scripts/build_images.sh | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index 427a12862ff..fb777f6ec6c 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -183,7 +183,7 @@ def create_dockerconfig_secret( auths[url] = { "username": uname, "password": passwd, - "auth": base64.b64encode(f"{uname}:{passwd}".encode()).decode(), + "auth": KubeUtils.b64encode_secret(f"{uname}:{passwd}"), } config_str = json.dumps({"auths": auths}) diff --git a/scripts/build_images.sh b/scripts/build_images.sh index 78f3a0255bf..280ca544620 100644 --- a/scripts/build_images.sh +++ b/scripts/build_images.sh @@ -1,5 +1,8 @@ #!/bin/bash -docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t syftpilot.azurecr.io/openmined/grid-backend:$1 ./packages -docker image build -f ./packages/grid/frontend/frontend.dockerfile --target grid-ui-development -t syftpilot.azurecr.io/openmined/grid-frontend:$1 ./packages/grid/frontend -docker image build -f ./packages/grid/seaweedfs/seaweedfs.dockerfile --build-arg SEAWEEDFS_VERSION=3.59 -t syftpilot.azurecr.io/openmined/grid-seaweedfs:$1 ./packages/grid/seaweedfs +REGISTRY=${1:-"k3d-registry.localhost:5000"} +TAG=${2:-"latest"} + +docker image build -f ./packages/grid/backend/backend.dockerfile --target backend -t $REGISTRY/openmined/grid-backend:$TAG ./packages +docker image build -f ./packages/grid/frontend/frontend.dockerfile --target grid-ui-development -t $REGISTRY/openmined/grid-frontend:$TAG ./packages/grid/frontend +docker image build -f ./packages/grid/seaweedfs/seaweedfs.dockerfile --build-arg SEAWEEDFS_VERSION=3.59 -t $REGISTRY/openmined/grid-seaweedfs:$TAG ./packages/grid/seaweedfs From 1f3a242c6c233da8bc3d871bc42a51f3aec549f4 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 6 Feb 2024 14:49:56 +0530 Subject: [PATCH 26/46] address review comments --- packages/syft/src/syft/service/worker/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index ca472683702..14b5799d825 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -257,7 +257,7 @@ def run_workers_in_threads( return results -def prepare_worker_pool_env(runner: KubernetesRunner, env_vars: dict): +def prepare_kubernetes_pool_env(runner: KubernetesRunner, env_vars: dict): # get current backend pod name backend_pod_name = os.getenv("K8S_POD_NAME") if not backend_pod_name: @@ -317,7 +317,7 @@ def create_kubernetes_pool( f"replicas={replicas}" ) - env_vars, mount_secrets = prepare_worker_pool_env( + env_vars, mount_secrets = prepare_kubernetes_pool_env( runner, { "SYFT_WORKER": "True", From 903df9cd53cac11cd729bfb4dbcde8ef7db6f85a Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 6 Feb 2024 15:48:07 +0530 Subject: [PATCH 27/46] [k8s] scale worker pool in kubernetes --- .../service/worker/worker_pool_service.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/packages/syft/src/syft/service/worker/worker_pool_service.py b/packages/syft/src/syft/service/worker/worker_pool_service.py index 806881547da..11e83b01112 100644 --- a/packages/syft/src/syft/service/worker/worker_pool_service.py +++ b/packages/syft/src/syft/service/worker/worker_pool_service.py @@ -11,6 +11,7 @@ from ...custom_worker.config import CustomWorkerConfig from ...custom_worker.config import WorkerConfig from ...custom_worker.k8s import IN_KUBERNETES +from ...custom_worker.runner_k8s import KubernetesRunner from ...serde.serializable import serializable from ...store.document_store import DocumentStore from ...store.linked_obj import LinkedObject @@ -36,6 +37,7 @@ from .utils import get_orchestration_type from .utils import run_containers from .utils import run_workers_in_threads +from .utils import scale_kubernetes_pool from .worker_image import SyftWorkerImage from .worker_image_stash import SyftWorkerImageStash from .worker_pool import ContainerSpawnStatus @@ -430,6 +432,94 @@ def add_workers( return container_statuses + @service_method( + path="worker_pool.scale", + name="scale", + roles=DATA_OWNER_ROLE_LEVEL, + ) + def scale( + self, + context: AuthedServiceContext, + number: int, + pool_id: Optional[UID] = None, + pool_name: Optional[str] = None, + ): + """ + Scale the worker pool to the given number of workers in Kubernetes. + Allows both scaling up and down the worker pool. + """ + + if not IN_KUBERNETES: + return SyftError(message="Scaling is only supported in Kubernetes mode") + elif number < 0: + # zero is a valid scale down + return SyftError(message=f"Invalid number of workers: {number}") + + result = self._get_worker_pool(context, pool_id, pool_name) + if isinstance(result, SyftError): + return result + + worker_pool = result + current_worker_count = len(worker_pool.worker_list) + + if current_worker_count == number: + return SyftSuccess(message=f"Worker pool already has {number} workers") + elif number > current_worker_count: + workers_to_add = number - current_worker_count + return self.add_workers( + context=context, + number=workers_to_add, + pool_id=pool_id, + pool_name=pool_name, + # kube scaling doesn't require password as it replicates an existing deployment + reg_username=None, + reg_password=None, + ) + else: + # scale down at kubernetes control plane + runner = KubernetesRunner() + result = scale_kubernetes_pool( + runner, + pool_name=worker_pool.name, + replicas=number, + ) + if isinstance(result, SyftError): + return result + + # scale down removes the last "n" workers + # workers to delete = len(workers) - number + workers_to_delete = worker_pool.worker_list[ + -(current_worker_count - number) : + ] + + worker_stash = context.node.get_service("WorkerService").stash + # delete linkedobj workers + for worker in workers_to_delete: + delete_result = worker_stash.delete_by_uid( + credentials=context.credentials, + uid=worker.object_uid, + ) + if delete_result.is_err(): + print(f"Failed to delete worker: {worker.object_uid}") + + # update worker_pool + worker_pool.max_count = number + worker_pool.worker_list = worker_pool.worker_list[:number] + update_result = self.stash.update( + credentials=context.credentials, + obj=worker_pool, + ) + + if update_result.is_err(): + return SyftError( + message=( + f"Pool {worker_pool.name} was scaled down, " + f"but failed update the stash with err: {result.err()}" + ) + ) + + return SyftSuccess(message=f"Worker pool scaled to {number} workers") + @service_method( path="worker_pool.filter_by_image_id", name="filter_by_image_id", From 895a208d8516d5bff6ce2e58a9aa474d1f7042e0 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 6 Feb 2024 13:12:41 +0000 Subject: [PATCH 28/46] remove queue constructor from named method - pass queue arguments to node - add method to create queue config in node constructor --- packages/syft/src/syft/node/node.py | 72 +++++++++++++++++---------- packages/syft/src/syft/node/server.py | 2 + 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 2bdb32445b1..96ebff3b3a9 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -11,7 +11,7 @@ from multiprocessing import current_process import os from pathlib import Path -import subprocess # nosec +import subprocess import traceback from typing import Any from typing import Callable @@ -296,6 +296,10 @@ def __init__( sqlite_path: Optional[str] = None, blob_storage_config: Optional[BlobStorageConfig] = None, queue_config: Optional[QueueConfig] = None, + queue_port: Optional[int] = None, + n_consumers: int = 0, + create_producer: bool = False, + thread_workers: bool = False, node_side_type: Union[str, NodeSideType] = NodeSideType.HIGH_SIDE, enable_warnings: bool = False, dev_mode: bool = False, @@ -398,7 +402,15 @@ def __init__( self.post_init() self.create_initial_settings(admin_email=root_email) - self.init_queue_manager(queue_config=queue_config) + queue_config_ = self.init_queue_config( + n_consumers=n_consumers, + create_producer=create_producer, + thread_workers=thread_workers, + queue_port=queue_port, + queue_config=queue_config, + ) + + self.init_queue_manager(queue_config=queue_config_) self.init_blob_storage(config=blob_storage_config) @@ -451,20 +463,39 @@ def stop(self): def close(self): self.stop() - def init_queue_manager(self, queue_config: Optional[QueueConfig]): - queue_config_ = ZMQQueueConfig() if queue_config is None else queue_config - self.queue_config = queue_config_ + def init_queue_config( + n_consumers: int, + create_producer: bool, + thread_workers: bool, + queue_port: Optional[int], + queue_config: Optional[QueueConfig], + ) -> QueueConfig: + if queue_config: + queue_config_ = queue_config + elif queue_port is not None or n_consumers > 0 or create_producer: + queue_config_ = ZMQQueueConfig( + client_config=ZMQClientConfig( + create_producer=create_producer, + queue_port=queue_port, + n_consumers=n_consumers, + ), + thread_workers=thread_workers, + ) + else: + queue_config_ = ZMQQueueConfig() - MessageHandlers = [APICallMessageHandler] + return queue_config_ + def init_queue_manager(self, queue_config: QueueConfig): + MessageHandlers = [APICallMessageHandler] if self.is_subprocess: return - self.queue_manager = QueueManager(config=queue_config_) + self.queue_manager = QueueManager(config=queue_config) for message_handler in MessageHandlers: queue_name = message_handler.queue_name # client config - if getattr(queue_config_.client_config, "create_producer", True): + if getattr(queue_config.client_config, "create_producer", True): context = AuthedServiceContext( node=self, credentials=self.verify_key, @@ -479,16 +510,16 @@ def init_queue_manager(self, queue_config: Optional[QueueConfig]): producer.run() address = producer.address else: - port = queue_config_.client_config.queue_port + port = queue_config.client_config.queue_port if port is not None: address = get_queue_address(port) else: address = None - if address is None and queue_config_.client_config.n_consumers > 0: + if address is None and queue_config.client_config.n_consumers > 0: raise ValueError("address unknown for consumers") - service_name = queue_config_.client_config.consumer_service + service_name = queue_config.client_config.consumer_service if not service_name: # Create consumers for default worker pool @@ -537,7 +568,6 @@ def named( node_side_type: Union[str, NodeSideType] = NodeSideType.HIGH_SIDE, enable_warnings: bool = False, n_consumers: int = 0, - consumer_service: Optional[str] = None, thread_workers: bool = False, create_producer: bool = False, queue_port: Optional[int] = None, @@ -600,19 +630,6 @@ def named( client_config=blob_client_config ) - if queue_port is not None or n_consumers > 0 or create_producer: - queue_config = ZMQQueueConfig( - client_config=ZMQClientConfig( - create_producer=create_producer, - queue_port=queue_port, - n_consumers=n_consumers, - consumer_service=consumer_service, - ), - thread_workers=thread_workers, - ) - else: - queue_config = None - return cls( name=name, id=uid, @@ -624,7 +641,10 @@ def named( node_side_type=node_side_type, enable_warnings=enable_warnings, blob_storage_config=blob_storage_config, - queue_config=queue_config, + queue_port=queue_port, + n_consumers=n_consumers, + thread_workers=thread_workers, + create_producer=create_producer, dev_mode=dev_mode, migrate=migrate, in_memory_workers=in_memory_workers, diff --git a/packages/syft/src/syft/node/server.py b/packages/syft/src/syft/node/server.py index e49c21b9740..fc7bbb2bc8d 100644 --- a/packages/syft/src/syft/node/server.py +++ b/packages/syft/src/syft/node/server.py @@ -125,6 +125,8 @@ async def _run_uvicorn( migrate=True, in_memory_workers=in_memory_workers, queue_port=queue_port, + create_producer=create_producer, + n_consumers=n_consumers, ) router = make_routes(worker=worker) app = make_app(worker.name, router=router) From b801e41a930429aebad4c9d934360a105e6aa6c8 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 6 Feb 2024 13:47:38 +0000 Subject: [PATCH 29/46] fix missing self in create_queue_config method --- packages/syft/src/syft/node/node.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 96ebff3b3a9..10a727755c2 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -402,7 +402,7 @@ def __init__( self.post_init() self.create_initial_settings(admin_email=root_email) - queue_config_ = self.init_queue_config( + self.queue_config = self.create_queue_config( n_consumers=n_consumers, create_producer=create_producer, thread_workers=thread_workers, @@ -410,7 +410,7 @@ def __init__( queue_config=queue_config, ) - self.init_queue_manager(queue_config=queue_config_) + self.init_queue_manager(queue_config=self.queue_config) self.init_blob_storage(config=blob_storage_config) @@ -463,7 +463,8 @@ def stop(self): def close(self): self.stop() - def init_queue_config( + def create_queue_config( + self, n_consumers: int, create_producer: bool, thread_workers: bool, From 878cbe41a484874ba8c378e16ea982af5485fba5 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 6 Feb 2024 22:12:49 +0530 Subject: [PATCH 30/46] add nosec to subprocess - find available queue port if queue port is None --- packages/hagrid/hagrid/orchestra.py | 3 +++ packages/syft/src/syft/node/node.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py index eecb5ca51bd..db9fe3d4843 100644 --- a/packages/hagrid/hagrid/orchestra.py +++ b/packages/hagrid/hagrid/orchestra.py @@ -263,6 +263,9 @@ def deploy_to_python( print("Staging Protocol Changes...") stage_protocol_changes() + if queue_port is None: + queue_port = find_available_port(host="localhost", port=None, search=True) + kwargs = { "name": name, "host": host, diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 10a727755c2..681033fc46d 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -11,7 +11,7 @@ from multiprocessing import current_process import os from pathlib import Path -import subprocess +import subprocess # nosec import traceback from typing import Any from typing import Callable From 40bf2ab167e73aad9acad061206c2b110dc48fae Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 6 Feb 2024 17:39:02 +0000 Subject: [PATCH 31/46] add new property to usercode, change error handling --- packages/syft/src/syft/service/code/user_code.py | 4 ++++ packages/syft/src/syft/service/code/user_code_service.py | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 020830d2508..a449845115d 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -462,6 +462,10 @@ def input_policy(self) -> Optional[InputPolicy]: print(f"Failed to deserialize custom input policy state. {e}") return None + @property + def output_policy_approved(self): + return self.status.approved + @property def output_policy(self) -> Optional[OutputPolicy]: if not self.status.approved: diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index 20c8630c231..da409b1dac0 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -311,7 +311,7 @@ def is_execution_allowed(self, code, context, output_policy): # Check if the user has permission to execute the code. elif not (has_code_permission := self.has_code_permission(code, context)): return has_code_permission - elif code.output_policy is None: + elif not code.output_policy_approved: return SyftError("Output policy not approved", code) elif not output_policy.valid: return output_policy.valid @@ -399,9 +399,9 @@ def _call( code=code, context=context, output_policy=output_policy ) if not can_execute: - if output_policy is None: + if not code.output_policy_approved: return Err( - "UserCodeStatus.DENIED: Function has no output policy" + "Execution denied: Your code is waiting for approval" ) if not (is_valid := output_policy.valid): if ( From b181ffd9b07f0bee3d4132884db7024a1677ea20 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 6 Feb 2024 17:41:59 +0000 Subject: [PATCH 32/46] fix test --- .../syft/tests/syft/request/request_code_accept_deny_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/tests/syft/request/request_code_accept_deny_test.py b/packages/syft/tests/syft/request/request_code_accept_deny_test.py index 242de50ff85..7451dd26d91 100644 --- a/packages/syft/tests/syft/request/request_code_accept_deny_test.py +++ b/packages/syft/tests/syft/request/request_code_accept_deny_test.py @@ -205,4 +205,4 @@ def simple_function(data): result = ds_client.code.simple_function(data=action_obj) assert isinstance(result, SyftError) - assert "UserCodeStatus.DENIED" in result.message + assert "Execution denied" in result.message From eeb11796596877d671353f82377cba20289719b6 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 7 Feb 2024 09:47:30 +0530 Subject: [PATCH 33/46] [k8s] add kr8s wait timeouts --- .../syft/src/syft/custom_worker/builder_docker.py | 5 ++--- packages/syft/src/syft/custom_worker/builder_k8s.py | 12 ++++++++++-- .../syft/src/syft/custom_worker/builder_types.py | 12 +++++++++++- packages/syft/src/syft/custom_worker/runner_k8s.py | 12 ++++++++++-- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/custom_worker/builder_docker.py b/packages/syft/src/syft/custom_worker/builder_docker.py index e1f24520c25..9eda180e08b 100644 --- a/packages/syft/src/syft/custom_worker/builder_docker.py +++ b/packages/syft/src/syft/custom_worker/builder_docker.py @@ -10,6 +10,7 @@ import docker # relative +from .builder_types import BUILD_IMAGE_TIMEOUT_SEC from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult @@ -18,8 +19,6 @@ class DockerBuilder(BuilderBase): - BUILD_MAX_WAIT = 30 * 60 - def build_image( self, tag: str, @@ -40,7 +39,7 @@ def build_image( with contextlib.closing(docker.from_env()) as client: image_result, logs = client.images.build( tag=tag, - timeout=self.BUILD_MAX_WAIT, + timeout=BUILD_IMAGE_TIMEOUT_SEC, buildargs=buildargs, **kwargs, ) diff --git a/packages/syft/src/syft/custom_worker/builder_k8s.py b/packages/syft/src/syft/custom_worker/builder_k8s.py index 395028d69b4..1be16d3c0ac 100644 --- a/packages/syft/src/syft/custom_worker/builder_k8s.py +++ b/packages/syft/src/syft/custom_worker/builder_k8s.py @@ -11,9 +11,11 @@ from kr8s.objects import Secret # relative +from .builder_types import BUILD_IMAGE_TIMEOUT_SEC from .builder_types import BuilderBase from .builder_types import ImageBuildResult from .builder_types import ImagePushResult +from .builder_types import PUSH_IMAGE_TIMEOUT_SEC from .k8s import INTERNAL_REGISTRY_HOST from .k8s import JOB_COMPLETION_TTL from .k8s import KUBERNETES_NAMESPACE @@ -66,7 +68,10 @@ def build_image( ) # wait for job to complete/fail - job.wait(["condition=Complete", "condition=Failed"]) + job.wait( + ["condition=Complete", "condition=Failed"], + timeout=BUILD_IMAGE_TIMEOUT_SEC, + ) # get logs logs = self._get_logs(job) @@ -119,7 +124,10 @@ def push_image( push_secret=push_secret, ) - job.wait(["condition=Complete", "condition=Failed"]) + job.wait( + ["condition=Complete", "condition=Failed"], + timeout=PUSH_IMAGE_TIMEOUT_SEC, + ) exit_code = self._get_exit_code(job)[0] logs = self._get_logs(job) except Exception: diff --git a/packages/syft/src/syft/custom_worker/builder_types.py b/packages/syft/src/syft/custom_worker/builder_types.py index 53c27788791..8007bf476e9 100644 --- a/packages/syft/src/syft/custom_worker/builder_types.py +++ b/packages/syft/src/syft/custom_worker/builder_types.py @@ -7,7 +7,17 @@ # third party from pydantic import BaseModel -__all__ = ["BuilderBase", "ImageBuildResult", "ImagePushResult"] +__all__ = [ + "BuilderBase", + "ImageBuildResult", + "ImagePushResult", + "BUILD_IMAGE_TIMEOUT_SEC", + "PUSH_IMAGE_TIMEOUT_SEC", +] + + +BUILD_IMAGE_TIMEOUT_SEC = 30 * 60 +PUSH_IMAGE_TIMEOUT_SEC = 10 * 60 class ImageBuildResult(BaseModel): diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index ff2c3120ebb..9f21102221f 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -16,6 +16,8 @@ from .k8s import get_kr8s_client JSONPATH_AVAILABLE_REPLICAS = "{.status.availableReplicas}" +CREATE_POOL_TIMEOUT_SEC = 60 +SCALE_POOL_TIMEOUT_SEC = 60 class KubernetesRunner: @@ -57,7 +59,10 @@ def create_pool( ) # wait for replicas to be available and ready - deployment.wait(f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}") + deployment.wait( + f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}", + timeout=CREATE_POOL_TIMEOUT_SEC, + ) except Exception: raise finally: @@ -72,7 +77,10 @@ def scale_pool(self, pool_name: str, replicas: int) -> Optional[StatefulSet]: if not deployment: return None deployment.scale(replicas) - deployment.wait(f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}") + deployment.wait( + f"jsonpath='{JSONPATH_AVAILABLE_REPLICAS}'={replicas}", + timeout=SCALE_POOL_TIMEOUT_SEC, + ) return deployment def get_pool(self, pool_name: str) -> Optional[StatefulSet]: From 5900ad992b415b139d2a215333485898ef6bbda0 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 7 Feb 2024 10:23:36 +0530 Subject: [PATCH 34/46] [k8s] fix bug whehn scaling up from 0 workers --- notebooks/api/0.8/11-container-images-k8s.ipynb | 16 +++++++++++++--- .../syft/src/syft/custom_worker/runner_k8s.py | 3 +++ packages/syft/src/syft/service/worker/utils.py | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index 07c26a78b6a..2916c692858 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -46,8 +46,8 @@ "os.environ[\"DEV_MODE\"] = \"True\"\n", "\n", "# Uncomment this to add custom values\n", - "# os.environ[\"NODE_URL\"] = \"http://localhost\"\n", - "# os.environ[\"NODE_PORT\"] = \"8080\"" + "os.environ[\"NODE_URL\"] = \"http://localhost\"\n", + "os.environ[\"NODE_PORT\"] = \"8080\"" ] }, { @@ -92,6 +92,16 @@ "domain_client.worker_pools" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "de9872be", + "metadata": {}, + "outputs": [], + "source": [ + "domain_client.api.services.worker_pool.scale(number=2, pool_name=\"default-pool\")" + ] + }, { "cell_type": "markdown", "id": "3c7a124a", @@ -1153,7 +1163,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.2" + "version": "3.11.7" } }, "nbformat": 4, diff --git a/packages/syft/src/syft/custom_worker/runner_k8s.py b/packages/syft/src/syft/custom_worker/runner_k8s.py index 9f21102221f..3b35830c0f4 100644 --- a/packages/syft/src/syft/custom_worker/runner_k8s.py +++ b/packages/syft/src/syft/custom_worker/runner_k8s.py @@ -83,6 +83,9 @@ def scale_pool(self, pool_name: str, replicas: int) -> Optional[StatefulSet]: ) return deployment + def exists(self, pool_name: str) -> bool: + return bool(self.get_pool(pool_name)) + def get_pool(self, pool_name: str) -> Optional[StatefulSet]: selector = {"app.kubernetes.io/component": pool_name} for _set in self.client.get("statefulsets", label_selector=selector): diff --git a/packages/syft/src/syft/service/worker/utils.py b/packages/syft/src/syft/service/worker/utils.py index 14b5799d825..fa216e4e6f7 100644 --- a/packages/syft/src/syft/service/worker/utils.py +++ b/packages/syft/src/syft/service/worker/utils.py @@ -384,7 +384,7 @@ def run_workers_in_kubernetes( spawn_status = [] runner = KubernetesRunner() - if start_idx == 0: + if not runner.exists(pool_name=pool_name): pool_pods = create_kubernetes_pool( runner=runner, tag=worker_image.image_identifier.full_name_with_tag, From 8f191d25c3833852b9aeb17120b4384639e27220 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 7 Feb 2024 11:28:24 +0530 Subject: [PATCH 35/46] [notebooks] comment back dev env vars --- notebooks/api/0.8/11-container-images-k8s.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index 2916c692858..a8855d00886 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -46,8 +46,8 @@ "os.environ[\"DEV_MODE\"] = \"True\"\n", "\n", "# Uncomment this to add custom values\n", - "os.environ[\"NODE_URL\"] = \"http://localhost\"\n", - "os.environ[\"NODE_PORT\"] = \"8080\"" + "# os.environ[\"NODE_URL\"] = \"http://localhost\"\n", + "# os.environ[\"NODE_PORT\"] = \"8080\"" ] }, { From cd722774779734b0c78a70b2dac4cd2834fbc0ee Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 7 Feb 2024 11:38:44 +0530 Subject: [PATCH 36/46] remove debug commands --- tox.ini | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tox.ini b/tox.ini index 570bad4c0e4..0362e12bfcb 100644 --- a/tox.ini +++ b/tox.ini @@ -783,19 +783,9 @@ commands = # ignore 06 because of opendp on arm64 # Run 0.8 notebooks - bash -c 'echo Gateway Cluster Info; kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Gateway Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Domain Cluster Info; kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1' - bash -c 'echo Domain Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1' - bash -c " source ./scripts/get_k8s_secret_ci.sh; \ pytest --nbmake notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv --nbmake-timeout=1000" - bash -c 'echo Gateway Cluster Info; kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Gateway Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1' - bash -c 'echo Domain Cluster Info; kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1' - bash -c 'echo Domain Logs; kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1' - #Integration + Gateway Connection Tests # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure # high/low side warning flag. From 7b23702d753a221dc00dffe857d97e2c2f4a306d Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 7 Feb 2024 15:11:07 +0530 Subject: [PATCH 37/46] remove selecting queue port if not provided --- packages/hagrid/hagrid/orchestra.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py index db9fe3d4843..eecb5ca51bd 100644 --- a/packages/hagrid/hagrid/orchestra.py +++ b/packages/hagrid/hagrid/orchestra.py @@ -263,9 +263,6 @@ def deploy_to_python( print("Staging Protocol Changes...") stage_protocol_changes() - if queue_port is None: - queue_port = find_available_port(host="localhost", port=None, search=True) - kwargs = { "name": name, "host": host, From d38797182cc358ad24bc442b0eb97313681d6ee6 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 7 Feb 2024 16:44:50 +0530 Subject: [PATCH 38/46] add retry to tests in request_multiple_nodes_test.py --- packages/syft/tests/syft/request/request_multiple_nodes_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/syft/tests/syft/request/request_multiple_nodes_test.py b/packages/syft/tests/syft/request/request_multiple_nodes_test.py index 4beb780cc31..9ec214ea7fe 100644 --- a/packages/syft/tests/syft/request/request_multiple_nodes_test.py +++ b/packages/syft/tests/syft/request/request_multiple_nodes_test.py @@ -110,6 +110,7 @@ def dataset_2(client_do_2): return client_do_2.datasets[0].assets[0] +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_blocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): @@ -147,6 +148,7 @@ def compute_sum(data) -> float: assert result_ds_blocking == result_ds_nonblocking == dataset_2.data.mean() +@pytest.mark.flaky(reruns=2, reruns_delay=1) def test_transfer_request_nonblocking( client_ds_1, client_do_1, client_do_2, dataset_1, dataset_2 ): From 1a7714ee59862289c422a4033cdde7a3c2815df4 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Wed, 7 Feb 2024 17:15:02 +0530 Subject: [PATCH 39/46] add image test build method in notebooks --- notebooks/api/0.8/10-container-images.ipynb | 24 ++++++++++++++++++- .../syft/src/syft/custom_worker/config.py | 1 - 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/notebooks/api/0.8/10-container-images.ipynb b/notebooks/api/0.8/10-container-images.ipynb index 81ba6a05d13..35f26e791ff 100644 --- a/notebooks/api/0.8/10-container-images.ipynb +++ b/notebooks/api/0.8/10-container-images.ipynb @@ -133,6 +133,28 @@ "docker_config = sy.DockerWorkerConfig(dockerfile=custom_dockerfile_str)" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "62762ceb-38da-46f1-acac-cdf5bbf29513", + "metadata": {}, + "outputs": [], + "source": [ + "# test image build locally\n", + "test_build_res = docker_config.test_image_build(tag=\"openmined/custom-worker:0.7.8\")\n", + "test_build_res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0235e567-c65c-48fe-825d-79ea3e219166", + "metadata": {}, + "outputs": [], + "source": [ + "assert isinstance(test_build_res, sy.SyftSuccess), str(test_build_res)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -1406,7 +1428,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.9.7" } }, "nbformat": 4, diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 1b55e34e021..c54d4f77c40 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -177,7 +177,6 @@ def test_image_build(self, tag: str, **kwargs) -> Union[SyftSuccess, SyftError]: kwargs["fileobj"] = io.BytesIO(self.dockerfile.encode("utf-8")) _, logs = client.images.build( tag=tag, - timeout=self.BUILD_MAX_WAIT, **kwargs, ) return SyftSuccess(message=iterator_to_string(iterator=logs)) From 7793d4b11dde11a986a9cb81c9fbb4b5054e78ba Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 7 Feb 2024 19:15:39 +0530 Subject: [PATCH 40/46] [notebooks] add scale up+down tests --- .../api/0.8/11-container-images-k8s.ipynb | 95 ++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/notebooks/api/0.8/11-container-images-k8s.ipynb b/notebooks/api/0.8/11-container-images-k8s.ipynb index a8855d00886..c28b0187567 100644 --- a/notebooks/api/0.8/11-container-images-k8s.ipynb +++ b/notebooks/api/0.8/11-container-images-k8s.ipynb @@ -74,6 +74,14 @@ "domain_client" ] }, + { + "cell_type": "markdown", + "id": "fe3d0aa7", + "metadata": {}, + "source": [ + "### Scaling Default Worker Pool" + ] + }, { "cell_type": "markdown", "id": "55439eb5-1e92-46a6-a45a-471917a86265", @@ -92,6 +100,14 @@ "domain_client.worker_pools" ] }, + { + "cell_type": "markdown", + "id": "0ff8e268", + "metadata": {}, + "source": [ + "Scale up to 3 workers" + ] + }, { "cell_type": "code", "execution_count": null, @@ -99,7 +115,84 @@ "metadata": {}, "outputs": [], "source": [ - "domain_client.api.services.worker_pool.scale(number=2, pool_name=\"default-pool\")" + "result = domain_client.api.services.worker_pool.scale(\n", + " number=3, pool_name=\"default-pool\"\n", + ")\n", + "assert not isinstance(result, sy.SyftError), str(result)\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da6a499b", + "metadata": {}, + "outputs": [], + "source": [ + "result = domain_client.api.services.worker_pool.get_by_name(pool_name=\"default-pool\")\n", + "assert len(result.workers) == 3, str(result.to_dict())\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27761f0c", + "metadata": {}, + "outputs": [], + "source": [ + "# stdlib\n", + "# wait for some time for scale up to be ready\n", + "from time import sleep\n", + "\n", + "sleep(5)" + ] + }, + { + "cell_type": "markdown", + "id": "c1276b5c", + "metadata": {}, + "source": [ + "Scale down to 1 worker" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f0aa94c", + "metadata": {}, + "outputs": [], + "source": [ + "default_worker_pool = domain_client.api.services.worker_pool.scale(\n", + " number=1, pool_name=\"default-pool\"\n", + ")\n", + "assert not isinstance(result, sy.SyftError), str(result)\n", + "default_worker_pool" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52acc6f6", + "metadata": {}, + "outputs": [], + "source": [ + "result = domain_client.api.services.worker_pool.get_by_name(pool_name=\"default-pool\")\n", + "assert len(result.workers) == 1, str(result.to_dict())\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a7b40a3", + "metadata": {}, + "outputs": [], + "source": [ + "default_worker_pool = domain_client.api.services.worker_pool.get_by_name(\n", + " pool_name=\"default-pool\"\n", + ")\n", + "default_worker_pool" ] }, { From e0e69219308573712332eb679a99bf1eaf40b24d Mon Sep 17 00:00:00 2001 From: alfred-openmined-bot <145415986+alfred-openmined-bot@users.noreply.github.com> Date: Thu, 8 Feb 2024 06:23:11 +0000 Subject: [PATCH 41/46] [syft]bump version --- .bumpversion.cfg | 2 +- VERSION | 2 +- packages/grid/VERSION | 2 +- packages/grid/backend/worker_cpu.dockerfile | 2 +- packages/grid/devspace.yaml | 2 +- packages/grid/frontend/package.json | 2 +- packages/grid/helm/repo/index.yaml | 114 ++++++++++-------- .../grid/helm/repo/syft-0.8.4-beta.20.tgz | Bin 0 -> 7645 bytes packages/grid/helm/syft/Chart.yaml | 4 +- packages/grid/helm/syft/templates/NOTES.txt | 7 ++ packages/grid/helm/syft/values.yaml | 2 +- .../podman-kube/podman-syft-kube-config.yaml | 2 +- .../podman/podman-kube/podman-syft-kube.yaml | 4 +- packages/hagrid/hagrid/deps.py | 2 +- packages/hagrid/hagrid/manifest_template.yml | 6 +- packages/syft/setup.cfg | 2 +- packages/syft/src/syft/VERSION | 2 +- packages/syft/src/syft/__init__.py | 2 +- packages/syftcli/manifest.yml | 8 +- 19 files changed, 93 insertions(+), 74 deletions(-) create mode 100644 packages/grid/helm/repo/syft-0.8.4-beta.20.tgz diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 57e08dc1658..7312b9aaa58 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.4-beta.19 +current_version = 0.8.4-beta.20 tag = False tag_name = {new_version} commit = True diff --git a/VERSION b/VERSION index d8f01b13857..4a8c672fcaa 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.20" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/VERSION b/packages/grid/VERSION index d8f01b13857..4a8c672fcaa 100644 --- a/packages/grid/VERSION +++ b/packages/grid/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.20" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/backend/worker_cpu.dockerfile b/packages/grid/backend/worker_cpu.dockerfile index cccf70f7cb6..90bbe3feb6d 100644 --- a/packages/grid/backend/worker_cpu.dockerfile +++ b/packages/grid/backend/worker_cpu.dockerfile @@ -9,7 +9,7 @@ # Later we'd want to uninstall old python, and then install a new python runtime... # ... but pre-built syft deps may break! -ARG SYFT_VERSION_TAG="0.8.4-beta.19" +ARG SYFT_VERSION_TAG="0.8.4-beta.20" FROM openmined/grid-backend:${SYFT_VERSION_TAG} ARG PYTHON_VERSION="3.11" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 126064c486c..40fb030b15d 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -25,7 +25,7 @@ vars: DEVSPACE_ENV_FILE: "default.env" CONTAINER_REGISTRY: "docker.io" NODE_NAME: "mynode" - VERSION: "0.8.4-beta.19" + VERSION: "0.8.4-beta.20" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json index 93458c129d8..ed667bd1b45 100644 --- a/packages/grid/frontend/package.json +++ b/packages/grid/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pygrid-ui", - "version": "0.8.4-beta.19", + "version": "0.8.4-beta.20", "private": true, "scripts": { "dev": "pnpm i && vite dev --host --port 80", diff --git a/packages/grid/helm/repo/index.yaml b/packages/grid/helm/repo/index.yaml index db8ca70cd90..b71a5cc664b 100644 --- a/packages/grid/helm/repo/index.yaml +++ b/packages/grid/helm/repo/index.yaml @@ -1,12 +1,24 @@ apiVersion: v1 entries: syft: + - apiVersion: v2 + appVersion: 0.8.4-beta.20 + created: "2024-02-08T06:21:21.698140004Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: c51189a187bbf24135382e25cb00964e0330dfcd3b2f0c884581a6686f05dd28 + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.4-beta.20.tgz + version: 0.8.4-beta.20 - apiVersion: v2 appVersion: 0.8.4-beta.19 - created: "2024-02-05T14:11:25.654223+05:30" + created: "2024-02-08T06:21:21.696826113Z" description: Perform numpy-like analysis on data that remains in someone elses server - digest: c2868a765415fe17b1bdad35254fcb7f355960bbb2b1f04e17e46cd1c4bf9ab7 + digest: 8219575dedb42fa2ddbf2768a4e9afbfacbc2dff7e953d77c7b10a41b78dc687 icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png name: syft type: application @@ -15,7 +27,7 @@ entries: version: 0.8.4-beta.19 - apiVersion: v2 appVersion: 0.8.4-beta.18 - created: "2024-02-05T14:11:25.653769+05:30" + created: "2024-02-08T06:21:21.696046201Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6418cde559cf12f1f7fea5a2b123bba950e50eeb3be002441827d2ab7f9e4ef7 @@ -27,7 +39,7 @@ entries: version: 0.8.4-beta.18 - apiVersion: v2 appVersion: 0.8.4-beta.17 - created: "2024-02-05T14:11:25.653372+05:30" + created: "2024-02-08T06:21:21.6956452Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 71b39c5a4c64037eadbb154f7029282ba90d9a0d703f8d4c7dfc1ba2f5d81498 @@ -39,7 +51,7 @@ entries: version: 0.8.4-beta.17 - apiVersion: v2 appVersion: 0.8.4-beta.16 - created: "2024-02-05T14:11:25.653094+05:30" + created: "2024-02-08T06:21:21.695242956Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9c9840a7c9476dbb08e0ac83926330718fe50c89879752dd8f92712b036109c0 @@ -51,7 +63,7 @@ entries: version: 0.8.4-beta.16 - apiVersion: v2 appVersion: 0.8.4-beta.15 - created: "2024-02-05T14:11:25.652801+05:30" + created: "2024-02-08T06:21:21.694833999Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0955fd22da028315e30c68132cbfa4bdc82bae622039bcfce0de339707bb82eb @@ -63,7 +75,7 @@ entries: version: 0.8.4-beta.15 - apiVersion: v2 appVersion: 0.8.4-beta.14 - created: "2024-02-05T14:11:25.65251+05:30" + created: "2024-02-08T06:21:21.694429541Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 56208571956abe20ed7a5cc1867cab2667ed792c63e53d0e8bb70a9b438b7bf6 @@ -75,7 +87,7 @@ entries: version: 0.8.4-beta.14 - apiVersion: v2 appVersion: 0.8.4-beta.13 - created: "2024-02-05T14:11:25.652256+05:30" + created: "2024-02-08T06:21:21.694042436Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: d7222c72412b6ee5833fbb07d2549be179cdfc7ccd89e0ad947d112fce799b83 @@ -87,7 +99,7 @@ entries: version: 0.8.4-beta.13 - apiVersion: v2 appVersion: 0.8.4-beta.12 - created: "2024-02-05T14:11:25.652008+05:30" + created: "2024-02-08T06:21:21.69355409Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: af08c723756e397962b2d5190dedfd50797b771c5caf58b93a6f65d8fa24785c @@ -99,7 +111,7 @@ entries: version: 0.8.4-beta.12 - apiVersion: v2 appVersion: 0.8.4-beta.11 - created: "2024-02-05T14:11:25.651755+05:30" + created: "2024-02-08T06:21:21.693092135Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a0235835ba57d185a83dd8a26281fa37b2077c3a37fe3a1c50585005695927e3 @@ -111,7 +123,7 @@ entries: version: 0.8.4-beta.11 - apiVersion: v2 appVersion: 0.8.4-beta.10 - created: "2024-02-05T14:11:25.651506+05:30" + created: "2024-02-08T06:21:21.692743201Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 910ddfeba0c5e66651500dd11404afff092adc0f768ed68e0d93b04b83aa4388 @@ -123,7 +135,7 @@ entries: version: 0.8.4-beta.10 - apiVersion: v2 appVersion: 0.8.4-beta.9 - created: "2024-02-05T14:11:25.656422+05:30" + created: "2024-02-08T06:21:21.700594634Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c25ca8a9f072d6a5d02232448deaef5668aca05f24dfffbba3ebe30a4f75bb26 @@ -135,7 +147,7 @@ entries: version: 0.8.4-beta.9 - apiVersion: v2 appVersion: 0.8.4-beta.8 - created: "2024-02-05T14:11:25.656176+05:30" + created: "2024-02-08T06:21:21.700258685Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7249a39d4137e457b369384ba0a365c271c780d93a8327ce25083df763c39999 @@ -147,7 +159,7 @@ entries: version: 0.8.4-beta.8 - apiVersion: v2 appVersion: 0.8.4-beta.7 - created: "2024-02-05T14:11:25.655931+05:30" + created: "2024-02-08T06:21:21.699923176Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ee750c7c8d6ea05bd447375e624fdd7f66dd87680ab81f7b7e73df7379a9024a @@ -159,7 +171,7 @@ entries: version: 0.8.4-beta.7 - apiVersion: v2 appVersion: 0.8.4-beta.6 - created: "2024-02-05T14:11:25.655685+05:30" + created: "2024-02-08T06:21:21.69958385Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0e046be9f73df7444a995608c59af16fab9030b139b2acb4d6db6185b8eb5337 @@ -171,7 +183,7 @@ entries: version: 0.8.4-beta.6 - apiVersion: v2 appVersion: 0.8.4-beta.5 - created: "2024-02-05T14:11:25.655439+05:30" + created: "2024-02-08T06:21:21.699246127Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b56e9a23d46810eccdb4cf5272cc05126da3f6db314e541959c3efb5f260620b @@ -183,7 +195,7 @@ entries: version: 0.8.4-beta.5 - apiVersion: v2 appVersion: 0.8.4-beta.4 - created: "2024-02-05T14:11:25.655187+05:30" + created: "2024-02-08T06:21:21.698902854Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 1d5808ecaf55391f3b27ae6236400066508acbd242e33db24a1ab4bffa77409e @@ -195,7 +207,7 @@ entries: version: 0.8.4-beta.4 - apiVersion: v2 appVersion: 0.8.4-beta.3 - created: "2024-02-05T14:11:25.65492+05:30" + created: "2024-02-08T06:21:21.698547317Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b64efa8529d82be56c6ab60487ed24420a5614d96d2509c1f93c1003eda71a54 @@ -207,7 +219,7 @@ entries: version: 0.8.4-beta.3 - apiVersion: v2 appVersion: 0.8.4-beta.2 - created: "2024-02-05T14:11:25.654654+05:30" + created: "2024-02-08T06:21:21.697693897Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -223,7 +235,7 @@ entries: version: 0.8.4-beta.2 - apiVersion: v2 appVersion: 0.8.4-beta.1 - created: "2024-02-05T14:11:25.65124+05:30" + created: "2024-02-08T06:21:21.692234307Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -239,7 +251,7 @@ entries: version: 0.8.4-beta.1 - apiVersion: v2 appVersion: 0.8.3 - created: "2024-02-05T14:11:25.650813+05:30" + created: "2024-02-08T06:21:21.691282723Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -255,7 +267,7 @@ entries: version: 0.8.3 - apiVersion: v2 appVersion: 0.8.3-beta.6 - created: "2024-02-05T14:11:25.650294+05:30" + created: "2024-02-08T06:21:21.690281683Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -271,7 +283,7 @@ entries: version: 0.8.3-beta.6 - apiVersion: v2 appVersion: 0.8.3-beta.5 - created: "2024-02-05T14:11:25.649854+05:30" + created: "2024-02-08T06:21:21.688841323Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -287,7 +299,7 @@ entries: version: 0.8.3-beta.5 - apiVersion: v2 appVersion: 0.8.3-beta.4 - created: "2024-02-05T14:11:25.649015+05:30" + created: "2024-02-08T06:21:21.68767687Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -303,7 +315,7 @@ entries: version: 0.8.3-beta.4 - apiVersion: v2 appVersion: 0.8.3-beta.3 - created: "2024-02-05T14:11:25.648526+05:30" + created: "2024-02-08T06:21:21.686397443Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -319,7 +331,7 @@ entries: version: 0.8.3-beta.3 - apiVersion: v2 appVersion: 0.8.3-beta.2 - created: "2024-02-05T14:11:25.648132+05:30" + created: "2024-02-08T06:21:21.685378332Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -335,7 +347,7 @@ entries: version: 0.8.3-beta.2 - apiVersion: v2 appVersion: 0.8.3-beta.1 - created: "2024-02-05T14:11:25.647739+05:30" + created: "2024-02-08T06:21:21.684302185Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -351,7 +363,7 @@ entries: version: 0.8.3-beta.1 - apiVersion: v2 appVersion: 0.8.2 - created: "2024-02-05T14:11:25.64733+05:30" + created: "2024-02-08T06:21:21.683179261Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -367,7 +379,7 @@ entries: version: 0.8.2 - apiVersion: v2 appVersion: 0.8.2-beta.60 - created: "2024-02-05T14:11:25.646837+05:30" + created: "2024-02-08T06:21:21.681954926Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -383,7 +395,7 @@ entries: version: 0.8.2-beta.60 - apiVersion: v2 appVersion: 0.8.2-beta.59 - created: "2024-02-05T14:11:25.646354+05:30" + created: "2024-02-08T06:21:21.680549752Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -399,7 +411,7 @@ entries: version: 0.8.2-beta.59 - apiVersion: v2 appVersion: 0.8.2-beta.58 - created: "2024-02-05T14:11:25.645853+05:30" + created: "2024-02-08T06:21:21.679228709Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -415,7 +427,7 @@ entries: version: 0.8.2-beta.58 - apiVersion: v2 appVersion: 0.8.2-beta.57 - created: "2024-02-05T14:11:25.645306+05:30" + created: "2024-02-08T06:21:21.678499322Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -431,7 +443,7 @@ entries: version: 0.8.2-beta.57 - apiVersion: v2 appVersion: 0.8.2-beta.56 - created: "2024-02-05T14:11:25.644509+05:30" + created: "2024-02-08T06:21:21.677685106Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -447,7 +459,7 @@ entries: version: 0.8.2-beta.56 - apiVersion: v2 appVersion: 0.8.2-beta.53 - created: "2024-02-05T14:11:25.644028+05:30" + created: "2024-02-08T06:21:21.676619549Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -463,7 +475,7 @@ entries: version: 0.8.2-beta.53 - apiVersion: v2 appVersion: 0.8.2-beta.52 - created: "2024-02-05T14:11:25.643545+05:30" + created: "2024-02-08T06:21:21.675975351Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -479,7 +491,7 @@ entries: version: 0.8.2-beta.52 - apiVersion: v2 appVersion: 0.8.2-beta.51 - created: "2024-02-05T14:11:25.643061+05:30" + created: "2024-02-08T06:21:21.675241746Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -495,7 +507,7 @@ entries: version: 0.8.2-beta.51 - apiVersion: v2 appVersion: 0.8.2-beta.50 - created: "2024-02-05T14:11:25.642566+05:30" + created: "2024-02-08T06:21:21.674390341Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -511,7 +523,7 @@ entries: version: 0.8.2-beta.50 - apiVersion: v2 appVersion: 0.8.2-beta.49 - created: "2024-02-05T14:11:25.642086+05:30" + created: "2024-02-08T06:21:21.673145227Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -527,7 +539,7 @@ entries: version: 0.8.2-beta.49 - apiVersion: v2 appVersion: 0.8.2-beta.48 - created: "2024-02-05T14:11:25.641572+05:30" + created: "2024-02-08T06:21:21.671957191Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -543,7 +555,7 @@ entries: version: 0.8.2-beta.48 - apiVersion: v2 appVersion: 0.8.2-beta.47 - created: "2024-02-05T14:11:25.641031+05:30" + created: "2024-02-08T06:21:21.671281194Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -559,7 +571,7 @@ entries: version: 0.8.2-beta.47 - apiVersion: v2 appVersion: 0.8.2-beta.46 - created: "2024-02-05T14:11:25.640572+05:30" + created: "2024-02-08T06:21:21.670630594Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -575,7 +587,7 @@ entries: version: 0.8.2-beta.46 - apiVersion: v2 appVersion: 0.8.2-beta.45 - created: "2024-02-05T14:11:25.63981+05:30" + created: "2024-02-08T06:21:21.669960068Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -591,7 +603,7 @@ entries: version: 0.8.2-beta.45 - apiVersion: v2 appVersion: 0.8.2-beta.44 - created: "2024-02-05T14:11:25.639409+05:30" + created: "2024-02-08T06:21:21.669387535Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -607,7 +619,7 @@ entries: version: 0.8.2-beta.44 - apiVersion: v2 appVersion: 0.8.2-beta.43 - created: "2024-02-05T14:11:25.639009+05:30" + created: "2024-02-08T06:21:21.668791447Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -623,7 +635,7 @@ entries: version: 0.8.2-beta.43 - apiVersion: v2 appVersion: 0.8.2-beta.41 - created: "2024-02-05T14:11:25.638518+05:30" + created: "2024-02-08T06:21:21.667891681Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -639,7 +651,7 @@ entries: version: 0.8.2-beta.41 - apiVersion: v2 appVersion: 0.8.2-beta.40 - created: "2024-02-05T14:11:25.638016+05:30" + created: "2024-02-08T06:21:21.667163155Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -655,7 +667,7 @@ entries: version: 0.8.2-beta.40 - apiVersion: v2 appVersion: 0.8.2-beta.39 - created: "2024-02-05T14:11:25.6376+05:30" + created: "2024-02-08T06:21:21.665109924Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -671,7 +683,7 @@ entries: version: 0.8.2-beta.39 - apiVersion: v2 appVersion: 0.8.2-beta.38 - created: "2024-02-05T14:11:25.637211+05:30" + created: "2024-02-08T06:21:21.664547109Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -687,7 +699,7 @@ entries: version: 0.8.2-beta.38 - apiVersion: v2 appVersion: 0.8.2-beta.37 - created: "2024-02-05T14:11:25.636805+05:30" + created: "2024-02-08T06:21:21.663964237Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -703,7 +715,7 @@ entries: version: 0.8.2-beta.37 - apiVersion: v2 appVersion: 0.8.1 - created: "2024-02-05T14:11:25.63636+05:30" + created: "2024-02-08T06:21:21.663345798Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -717,4 +729,4 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.1.tgz version: 0.8.1 -generated: "2024-02-05T14:11:25.635473+05:30" +generated: "2024-02-08T06:21:21.662588138Z" diff --git a/packages/grid/helm/repo/syft-0.8.4-beta.20.tgz b/packages/grid/helm/repo/syft-0.8.4-beta.20.tgz new file mode 100644 index 0000000000000000000000000000000000000000..32faf0de4c22174b975936b5fc36e6d55678e65d GIT binary patch literal 7645 zcmV<39U|f%iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhbK5ww=zP|%z)`mL>~2Le-UL;(r>^5@GCmVOj^$+To^x+` z@n}jMQ>2!p?D5Rx{`OlW^?v!KC{9MGns`Ju8fXyR=m#3e?t=W_sHfAsd8hl)cb}tR zjIrdr{EIPO{EIn}-w`e(!5(3peuoJql75HqXX(vC8D_aozr)XNYmvCGr07BT5z;J7 z;ugB0H9Z(ie&VKT@`iz2Hz}?;$ZQ%8^77R=q(PGQQ9SGq?ix{e4M@j2y34{0B{6b3 z*C_Al9Hr3LVVt2bMp@E_BnAXg1{umAy@9kA+5%lK&j(rS;2_nv&2E_YhGv*SYLhsJ zIB(jde{eQ{_#}+M9h~2FiYe>h=D;O3I|%#TgFMV5xGJXdRTOr6d2f$fTFM;=2_9O{-skrYoWS$7uIXFe+^*>Q!<4RvQ|r>8)kWW*IKMY za9OGRHZ_DMaH#GQdktwbEC*$cJgddU0GD4V&%?M|ju1k5l#NHd_*snbXhiGi6>xhY*!Kj^Gi43)7{=uSAP z(F?o1>cwc*6vNx@>o97edKd@E|4IfB_lpIfnWWtcG)lVf;RePa)x$78fqs(S{gkBF zkj{}gLS4N%Pm<`Um=!HVYPDCW@xW4hg}zTxbolEqh1x4*Bb{ZV(a^=S#>^_y=;|D9 z^oR)xQyjVnU8k#KR~W|Eh>CpL}Tv`E8eS z{m-F4h;$Cw!Rgs$yVJ~n$+y|Y=JlUQM#RGU&!k88>;EoFDH$(&I=)7C$q)reio!U{ zbrgkh7yb4dYF5T!!I0?Fr)CNNk7Ssl6e7?WpfE$l;^9i*i|XM#_+^43&}oeNNeW1t zWSFD82V)GcQMvGyphfu;>Os^;S+4V8whAk1#7mE}fwslFMJ!#&k&dEOXiCtk+3%Am zN^T2^OX)LSwKAkrKoI6VNRf69+(8QcMc~K+80U~si~4cc|f!uu)yccVM3M1unm-8;H=<2QQTRguI-`R|9sVVpj8=pC8SOC%qj(w}K4J?0B_eaWg2kbC~@52brb>4fQ zRC>`oQFIegbRAKFlPmy7HHW}xWvP+Y)1DFF8{spbDB>w%86%Wo7RXA3lF=NZdAg|*U7P$1lbR1y;#VInPe9tVz<)6N&v2jzKzPZaf~M*&;QsZ%xx0^cei z91EqLT~A516wVOy`k9gn8!0)#7&KO#FkA38=28l)eS$5ufiIX4%$LLi&5dSED$ikI zT>u{Uo;2wHRhZ*j619x`BH&m$YJCq2Y(Yr_3R}u>zBr20zUVxwnX~W_cdo0IF^D0z6lf-te8ng`kX4k0;L13wDg%_ z?nz|)_sM@9MIprbDjt3k894BbRmurzodj?)u%6(;cO>u$*WeV#R!Shm7sdxptQD9T zu2cX4g$-nEedhbbg75{i7MxH(s1=^nS`d#rQLMv0@f9(c*nm@TPEf@huQ0q)?p*H6GU5vw^Ru;6RLF-iE$!TAQDwMmTW3 zhpkToim68=IL0(3#Iu3m#8ZC2M4+9N#&JWOVZPG>Q)!6viuLWu!~9|3biS2&&V1*5 ztieb^fH=!NVpZUz(*b1|e8RXC7E3Cu)YSK|=Ud|iSW~Z1j&?ZMj9Q~rV4SC<6`lzE zLV8+8nBtBq3Q_?B6M(SXV2#1^nWESbN>yP*K7H*fMa?!=Uu`^9NIcFp_{3sA@D*p4 zQDzNgp2b=QHUQ%hOSBC{;4uX>uz|4D^ETK3TOVvHI3`M2p*3hFrBArCHV6tGIgo$@ z?op3mr3Gf3aASy&UJyv?dydOz4EDzmZnrShl2PR|#RBla6N<~ikm6KxCaDDpG{D+{ zpn>E>Y9k5v4Wm*EEDe_4F6T9Hf*lCOia|71=s=-z1zRehhWd`G02I-fn?QiirB%eo1XIOyG0U~@Ie`W7nNS9_^E{@c z&=l|nf7{k=DHRZp;KI~1h8gAuf=lCjfs?-Fic0De<|{+F=X=r^%SoV>$8^9MF;+j# zHCX8JmE(Ehz}dnC_pG&&I$=2DjyleiVJ?u8816j547Q$CPWa4mL7Xy5IwrZ(iut~> zYL}z#3+f96z78~C>=_7{HOkkT2m`)w&RK35!N9b#z==`{YF7$#8KG>xJpPxSC zy(k6CzJ)=Oh50r-Af8iP`+`yKd{JoYQd#FYVi~hOR)Obf&83naah|k6aODpAAe2^%ro4=G$csNwn3aETb#8mb{|qa);A>PLj2}6OAcqpw z1N(CjCEdlQS2nl;rTxJJ~usjFvwOlo#i@*U>IdvAJV7J?>AxW!npguY2Gj+ z@RvE=vtjAm7|r;Vi$sBG^Wsp@c>zIzV@3?z!~El~zhT_>JH($+HtT=eNeR2p{`2_a z{%=Xde*gDQ%7g8{IWB#vVrpde!7n+av5t<q)bas4unYX(En_m5Q~W%;UEJRWy`Gu{nV@3Tc$T9=jjb~HAAHwnmf{6Ae!pDzRM z$^W33F4=#W@9n>xlv(~Sl&h%{(-}>H&VqgP0vP}4E|o<&HkKHJ#GRC|eTeh(Bns_a z3!SA7QgCG+L7975X7*(*M3xw{U+9hRXGZ3+WAg|yA4uqE2|r!!Awy`)^dkahzHm{Z zIa(f1&7HJ)Vogm;^XmAd*#$xX1Z~$i}w5W zVP{2!L7KQ>8S`7J^Ra!=IX*kx{B;ZEqIols=F8uAgyrgo!_E&IKTMoBXpWQEn=?t} ztG?0c+1vKj|+#qqn-vy1lC<@?T6`{Ls4qO+o2nbp5s z^@q#L^L0{ZRZ|6gXI%x(*>a7ev(wANyl-Ef zy?Ouf5pA}=Km73i^6KL3?DDF8a(Mjy5p1(~ro<}NnQP!Pi}wvIG}G1fLY`HTze5NgvA(b|2mMD-^RNpRZFZI_<-s+U>VjCubi{ zFCT?;a*)hQy#Mu~{h|E;d~-(r3K?_JKD=ySonM^2{cyDM8|Ue81(TFK+4*qNUMVtH zN!rts_Q~1BKdyc{yZCW+t+hC?gFESZCD8oI_vEo!Y-TprYsqhc~WW&~HCM>98&5Z{#ix5Dc9 zTqH(e0QSyCu+V4}-arglcAh3?@m)126HBUrCvQ^FE?ieF13OH^{O%};bND4+e4Fbu z9}ZSkyeW@pdJBa)w`izU?@o%L%w}z`igQc*<)F}^4+;zBz^34`gPE^3A|(X}aUSX@ zYZP!Z00l*7adbC(ve2l*LsQ{jQ`^N^mD@PiV;J_C<07X%F~qzVK@uh1yUw6!|0s#GJk?=5LwWV-k0KrRFUKCu%H#WW zcsDlAzB$*2wUA{eiGvxZ#RX{hQyS)Q7EcWFX^eD+BmXsoY?e3B_}H=*A{d{9`*_+P zs4V#ZgS0$!(={9nqU5d*&)xg=-9z?&32)K=Pbk~_|94T=`~TlgX21(}|BvVE8pyT& z|4F^ESpAaD{W&5Y>fO&%okRC-bf9Ay@VkI&tIL;m?k~`@J@5XreDy1R@qZ{cX8V6% zUY&n<^Zxj#^bvlzSYem_>9-YNpSEt0jHk#S&LWw8wKoVpvq3N&{cL}j=<2dg|J&vU zuzT}=#Nzqil6i8k|LvlD6Zt<&+T4RSfE5o$3&UggfPJ(#@(fz z@5iHa!C#oom}G0TimLufR3TFd zZR)^uAV-gnrU$F9YiL?$rm@M%`EN#1gvwZ})`-@e=W>uHzl>x5rA(bF?96O)V{LM-(REF)>cEO_Oj~WCgSp?;wFKuW1mQ2W-+mj9#Fu8) z&rt4{|9SM>HQ+w^FBw^o{{+kZ{VzKy3ld~6`yV9x3#Qo1|E_v z6)YFk^*R&cMm=mijK-a>H4E~;y5MQ6r++-;{13bs|KSqv<^L|qH(~$J&)S!I`bXKz zhmy9d@4u4&t1Ww5yASX_{f~N!{4WIE^Z!oD<4(n>uet#6WglX3HqieXLTyaP3V+Ak z#e~h#t%OB8Bk(^}WS*IyQS0yMFHw?Q4+p4zL+YqrjZ}TrT|}$L>aXbcu{woZX!EJI zHEQOq&UM*N7xLMr>jedys?R2dwBwsa93|fREYKXXzoa^LhtZ&?PltWPDEb{4$sx9X z9gW0n&GOSMHTXOF*D%RvM-HM4)>W^=FScV&xZ0dn z`lb2iTPbVg{}!|VpA!EOav%TMP5E~8zh?%2#@Ef>7ylWsk1_nnllJpX4J z#(jkPUsZYunLo2p=&wHQXFjC3Q-MXPog8xwJFy!`NME#OV zJbK=+NgUj(6z|pF9jd<*{;48fCwkCY`}O)dj2jhPZNvBd(#zMrJhQCS|8^b!;fwxX z;qiU^e<$Uei2uwhbPtRFOqTf{6h0={(KJS}|iUS6!W36=sd=&F+#LogPG?Iauoc zB3k+Q@a?%z9%yul@}k472pWgz*8yH^}I$S6w!q zMF0cDF#7YKt0rtOLG_GHcJ%{lRAT1p8t%}aew#f#`x6>%%U^hR{?pq}s4>hSMWOB& z2ujy$DVkoFvg!5vFw0+G>TWlTyRQ#jKa5dB4+ha)9sQrOBfq19kxUzQM`IW73f*my$-nIk_kGz^*2@1a=78Sk|MA$8{AXk@ z|94WhA^*Qlr}6Ssf2oP1lb$}5#7D^Uwfw)u9MF69Ke_1tk&N&6|LmkZGXHZrKNfR9 zzjU_c+Ma#BjLZ4^O4jCF&VxNJ>vFzsm3O(^?8eN?6)5VuP+0HHysV)1d6(aE*&zS7 z7680Y{u6KM{%1k=`JX!}d;Kqa?x~?lsfX+xsKDmXkXba0v4^0kz~<1<88jSY@&Gg= zu$`!^i^15vx8Vhq_40pv0pNS&zYuuo{%^jI|L&$dTmFBINyCpesMoIw1()JAb(QQP%9B zfZR@XlY?oXacL*TQfzAq@PM@J6^J+WV;rX3axa$9uE<3zkKH~%1XQY2KDLHE= z)7sZBCcpj3EA@sMz3UA`Gctc}2hq1-#xB}_cR{}G<3I29{|ULc|6ed~AOGD+d2IZr zm>>J&KOYp3S}|0gF&33;dwnMnePq~c+S5_YYo5HB zfY;=7`sF~`nl5jSmMyegw>=#f*8Nqbp!djs z>IuHM|Hs>(|F@g6W-F(=i*-M=&AaJd&e^LMMh>DG-Dz^6=ZrQLCEC5wk&Z2-i=Kn@ zX2!`fE)6tHqZX={O&v@wBb?gTEADuBZX+X<^%)y?Yk-_zp)b_;!&e8bt-{m{7xZgmRFm}dLf7^YEu zBS2w6k0DlKU4ou%u&{oYrlB2XdD0&lIx8U%`k}mlWqy9HZnBurbGA|%4hH2I&*Q9~ z6&ClRmja<6DU7g4XT8>V5BO-wnH4qNC^34{nC*eRK84$foSOAvWDvWfMS-a0_O}7IrR$ENh|Gm;aN+HsDcmw6lNHfoM>}s!24BgVuLVp>F35RaOyodhnuT69o zM|UWR%bG=(&;U{tg)ua1&9|MaPM)MtdxegYem{xP$DEpkI4IdJtNpE+-41GhYZ`rB`&%>b7yqB6VYl|T|5tm3 zKI$|~h8a43+s9xT>gIm00960 LyPwRu0NwxqT$*+o literal 0 HcmV?d00001 diff --git a/packages/grid/helm/syft/Chart.yaml b/packages/grid/helm/syft/Chart.yaml index 4007f542f39..78826d48c67 100644 --- a/packages/grid/helm/syft/Chart.yaml +++ b/packages/grid/helm/syft/Chart.yaml @@ -2,6 +2,6 @@ apiVersion: v2 name: syft description: Perform numpy-like analysis on data that remains in someone elses server type: application -version: "0.8.4-beta.19" -appVersion: "0.8.4-beta.19" +version: "0.8.4-beta.20" +appVersion: "0.8.4-beta.20" icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png \ No newline at end of file diff --git a/packages/grid/helm/syft/templates/NOTES.txt b/packages/grid/helm/syft/templates/NOTES.txt index fd95e4181d4..e3d0595cf02 100644 --- a/packages/grid/helm/syft/templates/NOTES.txt +++ b/packages/grid/helm/syft/templates/NOTES.txt @@ -238,6 +238,13 @@ "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", "action": "remove" } + }, + "SeaweedFSBlobDeposit": { + "2": { + "version": 2, + "hash": "07d84a95324d95d9c868cd7d1c33c908f77aa468671d76c144586aab672bcbb5", + "action": "add" + } } } diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 57d8ea8ff03..2abc00f629b 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -29,7 +29,7 @@ registry: syft: registry: "docker.io" - version: 0.8.4-beta.19 + version: 0.8.4-beta.20 node: settings: diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index 7082eb46295..e873c7d7168 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -31,7 +31,7 @@ data: RABBITMQ_VERSION: 3 SEAWEEDFS_VERSION: 3.59 DOCKER_IMAGE_SEAWEEDFS: chrislusf/seaweedfs:3.55 - VERSION: 0.8.4-beta.19 + VERSION: 0.8.4-beta.20 VERSION_HASH: unknown STACK_API_KEY: "" diff --git a/packages/grid/podman/podman-kube/podman-syft-kube.yaml b/packages/grid/podman/podman-kube/podman-syft-kube.yaml index 7ff11fc4494..3594dc84756 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube.yaml @@ -41,7 +41,7 @@ spec: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-backend:0.8.4-beta.19 + image: docker.io/openmined/grid-backend:0.8.4-beta.20 imagePullPolicy: IfNotPresent resources: {} tty: true @@ -57,7 +57,7 @@ spec: envFrom: - configMapRef: name: podman-syft-config - image: docker.io/openmined/grid-frontend:0.8.4-beta.19 + image: docker.io/openmined/grid-frontend:0.8.4-beta.20 imagePullPolicy: IfNotPresent resources: {} tty: true diff --git a/packages/hagrid/hagrid/deps.py b/packages/hagrid/hagrid/deps.py index 8a74cc92291..d2eb8c4e7e3 100644 --- a/packages/hagrid/hagrid/deps.py +++ b/packages/hagrid/hagrid/deps.py @@ -42,7 +42,7 @@ from .version import __version__ LATEST_STABLE_SYFT = "0.8.3" -LATEST_BETA_SYFT = "0.8.4-beta.19" +LATEST_BETA_SYFT = "0.8.4-beta.20" DOCKER_ERROR = """ You are running an old version of docker, possibly on Linux. You need to install v2. diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index 69361b9bc05..ae9c6d03fa5 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 hagrid_version: 0.3.106 -syft_version: 0.8.4-beta.19 -dockerTag: 0.8.4-beta.19 +syft_version: 0.8.4-beta.20 +dockerTag: 0.8.4-beta.20 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: cfea9e04c882a71797f2c7bf16e8deebb2295f23 +hash: dc0a24932b8a9b1041e5ae2cac5ce60ac4f761f0 target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index a740dc3c79b..da1c7e8b178 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = syft -version = attr: "0.8.4-beta.19" +version = attr: "0.8.4-beta.20" description = Perform numpy-like analysis on data that remains in someone elses server author = OpenMined author_email = info@openmined.org diff --git a/packages/syft/src/syft/VERSION b/packages/syft/src/syft/VERSION index d8f01b13857..4a8c672fcaa 100644 --- a/packages/syft/src/syft/VERSION +++ b/packages/syft/src/syft/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.20" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index f83cc10e6d9..cf3f61cbf09 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8.4-beta.19" +__version__ = "0.8.4-beta.20" # stdlib import pathlib diff --git a/packages/syftcli/manifest.yml b/packages/syftcli/manifest.yml index ccc9180f1da..fd91e93a25d 100644 --- a/packages/syftcli/manifest.yml +++ b/packages/syftcli/manifest.yml @@ -1,11 +1,11 @@ manifestVersion: 1.0 -syftVersion: 0.8.4-beta.19 -dockerTag: 0.8.4-beta.19 +syftVersion: 0.8.4-beta.20 +dockerTag: 0.8.4-beta.20 images: - - docker.io/openmined/grid-frontend:0.8.4-beta.19 - - docker.io/openmined/grid-backend:0.8.4-beta.19 + - docker.io/openmined/grid-frontend:0.8.4-beta.20 + - docker.io/openmined/grid-backend:0.8.4-beta.20 - docker.io/library/mongo:7.0.4 - docker.io/traefik:v2.10 From c90480ded31c481164dd956f905cb1a90ebdf08d Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:15:50 +0530 Subject: [PATCH 42/46] Added tunshell debugging --- .github/workflows/pr-tests-syft.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 0852db9269b..2d04826abdb 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -24,13 +24,13 @@ jobs: strategy: max-parallel: 99 matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest] python-version: ["3.11"] - include: - - python-version: "3.9" - os: "ubuntu-latest" - - python-version: "3.10" - os: "ubuntu-latest" + # include: + # - python-version: "3.9" + # os: "ubuntu-latest" + # - python-version: "3.10" + # os: "ubuntu-latest" runs-on: ${{ matrix.os }} steps: @@ -92,6 +92,10 @@ jobs: if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest' uses: crazy-max/ghaction-setup-docker@v3.0.0 + - name: Run Tunshell + run: | + curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T jTpo0lxxboRspUT5IzBhPD lQaLkeg8Q2sInsYPH2PVdS eu.relay.tunshell.com + - name: Run unit tests if: steps.changes.outputs.syft == 'true' run: | From af6f0f431a3dc15f51199aa86d29caa456498d6d Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:51:26 +0530 Subject: [PATCH 43/46] Revert "Added tunshell debugging" This reverts commit c90480ded31c481164dd956f905cb1a90ebdf08d. --- .github/workflows/pr-tests-syft.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml index 2d04826abdb..0852db9269b 100644 --- a/.github/workflows/pr-tests-syft.yml +++ b/.github/workflows/pr-tests-syft.yml @@ -24,13 +24,13 @@ jobs: strategy: max-parallel: 99 matrix: - os: [ubuntu-latest] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.11"] - # include: - # - python-version: "3.9" - # os: "ubuntu-latest" - # - python-version: "3.10" - # os: "ubuntu-latest" + include: + - python-version: "3.9" + os: "ubuntu-latest" + - python-version: "3.10" + os: "ubuntu-latest" runs-on: ${{ matrix.os }} steps: @@ -92,10 +92,6 @@ jobs: if: steps.changes.outputs.syft == 'true' && matrix.os == 'macos-latest' uses: crazy-max/ghaction-setup-docker@v3.0.0 - - name: Run Tunshell - run: | - curl -sSf https://lets.tunshell.com/init.sh | sh -s -- T jTpo0lxxboRspUT5IzBhPD lQaLkeg8Q2sInsYPH2PVdS eu.relay.tunshell.com - - name: Run unit tests if: steps.changes.outputs.syft == 'true' run: | From 331395d85b8afbbca9fb92e1df19564c1988e023 Mon Sep 17 00:00:00 2001 From: rasswanth-s <43314053+rasswanth-s@users.noreply.github.com> Date: Thu, 8 Feb 2024 13:59:13 +0530 Subject: [PATCH 44/46] Added pytest hook to pass cpu count by os.cpu_count() --- packages/syft/tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/syft/tests/conftest.py b/packages/syft/tests/conftest.py index 0ce969a38ea..9d9dd60e0d6 100644 --- a/packages/syft/tests/conftest.py +++ b/packages/syft/tests/conftest.py @@ -1,5 +1,6 @@ # stdlib import json +import os from pathlib import Path from unittest import mock @@ -46,6 +47,14 @@ def remove_file(filepath: Path): filepath.unlink(missing_ok=True) +# Pytest hook to set the number of workers for xdist +def pytest_xdist_auto_num_workers(config): + num = config.option.numprocesses + if num == "auto" or num == "logical": + return os.cpu_count() + return None + + @pytest.fixture(autouse=True) def protocol_file(): random_name = sy.UID().to_string() From 73aa361520477bfb282bf7517f66b4867ce7c838 Mon Sep 17 00:00:00 2001 From: alfred-openmined-bot <145415986+alfred-openmined-bot@users.noreply.github.com> Date: Thu, 8 Feb 2024 12:29:45 +0000 Subject: [PATCH 45/46] [syft]bump version --- .bumpversion.cfg | 2 +- VERSION | 2 +- packages/grid/VERSION | 2 +- packages/grid/backend/worker_cpu.dockerfile | 2 +- packages/grid/devspace.yaml | 2 +- packages/grid/frontend/package.json | 2 +- packages/grid/helm/repo/index.yaml | 114 ++++++++++-------- .../grid/helm/repo/syft-0.8.4-beta.21.tgz | Bin 0 -> 7644 bytes packages/grid/helm/syft/Chart.yaml | 4 +- packages/grid/helm/syft/values.yaml | 2 +- .../podman-kube/podman-syft-kube-config.yaml | 2 +- .../podman/podman-kube/podman-syft-kube.yaml | 4 +- packages/hagrid/hagrid/deps.py | 2 +- packages/hagrid/hagrid/manifest_template.yml | 6 +- packages/syft/setup.cfg | 2 +- packages/syft/src/syft/VERSION | 2 +- packages/syft/src/syft/__init__.py | 2 +- packages/syftcli/manifest.yml | 8 +- 18 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 packages/grid/helm/repo/syft-0.8.4-beta.21.tgz diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7312b9aaa58..f2d6045e2a9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.4-beta.20 +current_version = 0.8.4-beta.21 tag = False tag_name = {new_version} commit = True diff --git a/VERSION b/VERSION index 4a8c672fcaa..1f184fb3f81 100644 --- a/VERSION +++ b/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.20" +__version__ = "0.8.4-beta.21" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/VERSION b/packages/grid/VERSION index 4a8c672fcaa..1f184fb3f81 100644 --- a/packages/grid/VERSION +++ b/packages/grid/VERSION @@ -1,5 +1,5 @@ # Mono Repo Global Version -__version__ = "0.8.4-beta.20" +__version__ = "0.8.4-beta.21" # elsewhere we can call this file: `python VERSION` and simply take the stdout # stdlib diff --git a/packages/grid/backend/worker_cpu.dockerfile b/packages/grid/backend/worker_cpu.dockerfile index 90bbe3feb6d..d2971433e5d 100644 --- a/packages/grid/backend/worker_cpu.dockerfile +++ b/packages/grid/backend/worker_cpu.dockerfile @@ -9,7 +9,7 @@ # Later we'd want to uninstall old python, and then install a new python runtime... # ... but pre-built syft deps may break! -ARG SYFT_VERSION_TAG="0.8.4-beta.20" +ARG SYFT_VERSION_TAG="0.8.4-beta.21" FROM openmined/grid-backend:${SYFT_VERSION_TAG} ARG PYTHON_VERSION="3.11" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 40fb030b15d..1f10b1170ad 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -25,7 +25,7 @@ vars: DEVSPACE_ENV_FILE: "default.env" CONTAINER_REGISTRY: "docker.io" NODE_NAME: "mynode" - VERSION: "0.8.4-beta.20" + VERSION: "0.8.4-beta.21" # This is a list of `images` that DevSpace can build for this project # We recommend to skip image building during development (devspace dev) as much as possible diff --git a/packages/grid/frontend/package.json b/packages/grid/frontend/package.json index ed667bd1b45..d6a2110587b 100644 --- a/packages/grid/frontend/package.json +++ b/packages/grid/frontend/package.json @@ -1,6 +1,6 @@ { "name": "pygrid-ui", - "version": "0.8.4-beta.20", + "version": "0.8.4-beta.21", "private": true, "scripts": { "dev": "pnpm i && vite dev --host --port 80", diff --git a/packages/grid/helm/repo/index.yaml b/packages/grid/helm/repo/index.yaml index b71a5cc664b..61fb7eb380f 100644 --- a/packages/grid/helm/repo/index.yaml +++ b/packages/grid/helm/repo/index.yaml @@ -1,9 +1,21 @@ apiVersion: v1 entries: syft: + - apiVersion: v2 + appVersion: 0.8.4-beta.21 + created: "2024-02-08T12:28:05.631588027Z" + description: Perform numpy-like analysis on data that remains in someone elses + server + digest: 7dce153d2fcae7513e9c132e139b2721fd975ea3cc43a370e34dbeb2a1b7f683 + icon: https://raw.githubusercontent.com/OpenMined/PySyft/dev/docs/img/title_syft_light.png + name: syft + type: application + urls: + - https://openmined.github.io/PySyft/helm/syft-0.8.4-beta.21.tgz + version: 0.8.4-beta.21 - apiVersion: v2 appVersion: 0.8.4-beta.20 - created: "2024-02-08T06:21:21.698140004Z" + created: "2024-02-08T12:28:05.63107618Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c51189a187bbf24135382e25cb00964e0330dfcd3b2f0c884581a6686f05dd28 @@ -15,7 +27,7 @@ entries: version: 0.8.4-beta.20 - apiVersion: v2 appVersion: 0.8.4-beta.19 - created: "2024-02-08T06:21:21.696826113Z" + created: "2024-02-08T12:28:05.629522532Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 8219575dedb42fa2ddbf2768a4e9afbfacbc2dff7e953d77c7b10a41b78dc687 @@ -27,7 +39,7 @@ entries: version: 0.8.4-beta.19 - apiVersion: v2 appVersion: 0.8.4-beta.18 - created: "2024-02-08T06:21:21.696046201Z" + created: "2024-02-08T12:28:05.629121832Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 6418cde559cf12f1f7fea5a2b123bba950e50eeb3be002441827d2ab7f9e4ef7 @@ -39,7 +51,7 @@ entries: version: 0.8.4-beta.18 - apiVersion: v2 appVersion: 0.8.4-beta.17 - created: "2024-02-08T06:21:21.6956452Z" + created: "2024-02-08T12:28:05.628684253Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 71b39c5a4c64037eadbb154f7029282ba90d9a0d703f8d4c7dfc1ba2f5d81498 @@ -51,7 +63,7 @@ entries: version: 0.8.4-beta.17 - apiVersion: v2 appVersion: 0.8.4-beta.16 - created: "2024-02-08T06:21:21.695242956Z" + created: "2024-02-08T12:28:05.628283122Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 9c9840a7c9476dbb08e0ac83926330718fe50c89879752dd8f92712b036109c0 @@ -63,7 +75,7 @@ entries: version: 0.8.4-beta.16 - apiVersion: v2 appVersion: 0.8.4-beta.15 - created: "2024-02-08T06:21:21.694833999Z" + created: "2024-02-08T12:28:05.627874367Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0955fd22da028315e30c68132cbfa4bdc82bae622039bcfce0de339707bb82eb @@ -75,7 +87,7 @@ entries: version: 0.8.4-beta.15 - apiVersion: v2 appVersion: 0.8.4-beta.14 - created: "2024-02-08T06:21:21.694429541Z" + created: "2024-02-08T12:28:05.627474338Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 56208571956abe20ed7a5cc1867cab2667ed792c63e53d0e8bb70a9b438b7bf6 @@ -87,7 +99,7 @@ entries: version: 0.8.4-beta.14 - apiVersion: v2 appVersion: 0.8.4-beta.13 - created: "2024-02-08T06:21:21.694042436Z" + created: "2024-02-08T12:28:05.627121317Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: d7222c72412b6ee5833fbb07d2549be179cdfc7ccd89e0ad947d112fce799b83 @@ -99,7 +111,7 @@ entries: version: 0.8.4-beta.13 - apiVersion: v2 appVersion: 0.8.4-beta.12 - created: "2024-02-08T06:21:21.69355409Z" + created: "2024-02-08T12:28:05.626772324Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: af08c723756e397962b2d5190dedfd50797b771c5caf58b93a6f65d8fa24785c @@ -111,7 +123,7 @@ entries: version: 0.8.4-beta.12 - apiVersion: v2 appVersion: 0.8.4-beta.11 - created: "2024-02-08T06:21:21.693092135Z" + created: "2024-02-08T12:28:05.626422319Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: a0235835ba57d185a83dd8a26281fa37b2077c3a37fe3a1c50585005695927e3 @@ -123,7 +135,7 @@ entries: version: 0.8.4-beta.11 - apiVersion: v2 appVersion: 0.8.4-beta.10 - created: "2024-02-08T06:21:21.692743201Z" + created: "2024-02-08T12:28:05.62607571Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 910ddfeba0c5e66651500dd11404afff092adc0f768ed68e0d93b04b83aa4388 @@ -135,7 +147,7 @@ entries: version: 0.8.4-beta.10 - apiVersion: v2 appVersion: 0.8.4-beta.9 - created: "2024-02-08T06:21:21.700594634Z" + created: "2024-02-08T12:28:05.634222038Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: c25ca8a9f072d6a5d02232448deaef5668aca05f24dfffbba3ebe30a4f75bb26 @@ -147,7 +159,7 @@ entries: version: 0.8.4-beta.9 - apiVersion: v2 appVersion: 0.8.4-beta.8 - created: "2024-02-08T06:21:21.700258685Z" + created: "2024-02-08T12:28:05.633825235Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 7249a39d4137e457b369384ba0a365c271c780d93a8327ce25083df763c39999 @@ -159,7 +171,7 @@ entries: version: 0.8.4-beta.8 - apiVersion: v2 appVersion: 0.8.4-beta.7 - created: "2024-02-08T06:21:21.699923176Z" + created: "2024-02-08T12:28:05.633394809Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: ee750c7c8d6ea05bd447375e624fdd7f66dd87680ab81f7b7e73df7379a9024a @@ -171,7 +183,7 @@ entries: version: 0.8.4-beta.7 - apiVersion: v2 appVersion: 0.8.4-beta.6 - created: "2024-02-08T06:21:21.69958385Z" + created: "2024-02-08T12:28:05.633059241Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 0e046be9f73df7444a995608c59af16fab9030b139b2acb4d6db6185b8eb5337 @@ -183,7 +195,7 @@ entries: version: 0.8.4-beta.6 - apiVersion: v2 appVersion: 0.8.4-beta.5 - created: "2024-02-08T06:21:21.699246127Z" + created: "2024-02-08T12:28:05.632680322Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b56e9a23d46810eccdb4cf5272cc05126da3f6db314e541959c3efb5f260620b @@ -195,7 +207,7 @@ entries: version: 0.8.4-beta.5 - apiVersion: v2 appVersion: 0.8.4-beta.4 - created: "2024-02-08T06:21:21.698902854Z" + created: "2024-02-08T12:28:05.632316361Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: 1d5808ecaf55391f3b27ae6236400066508acbd242e33db24a1ab4bffa77409e @@ -207,7 +219,7 @@ entries: version: 0.8.4-beta.4 - apiVersion: v2 appVersion: 0.8.4-beta.3 - created: "2024-02-08T06:21:21.698547317Z" + created: "2024-02-08T12:28:05.631968049Z" description: Perform numpy-like analysis on data that remains in someone elses server digest: b64efa8529d82be56c6ab60487ed24420a5614d96d2509c1f93c1003eda71a54 @@ -219,7 +231,7 @@ entries: version: 0.8.4-beta.3 - apiVersion: v2 appVersion: 0.8.4-beta.2 - created: "2024-02-08T06:21:21.697693897Z" + created: "2024-02-08T12:28:05.630078702Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -235,7 +247,7 @@ entries: version: 0.8.4-beta.2 - apiVersion: v2 appVersion: 0.8.4-beta.1 - created: "2024-02-08T06:21:21.692234307Z" + created: "2024-02-08T12:28:05.625699115Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -251,7 +263,7 @@ entries: version: 0.8.4-beta.1 - apiVersion: v2 appVersion: 0.8.3 - created: "2024-02-08T06:21:21.691282723Z" + created: "2024-02-08T12:28:05.624975934Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -267,7 +279,7 @@ entries: version: 0.8.3 - apiVersion: v2 appVersion: 0.8.3-beta.6 - created: "2024-02-08T06:21:21.690281683Z" + created: "2024-02-08T12:28:05.623739249Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -283,7 +295,7 @@ entries: version: 0.8.3-beta.6 - apiVersion: v2 appVersion: 0.8.3-beta.5 - created: "2024-02-08T06:21:21.688841323Z" + created: "2024-02-08T12:28:05.62316792Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -299,7 +311,7 @@ entries: version: 0.8.3-beta.5 - apiVersion: v2 appVersion: 0.8.3-beta.4 - created: "2024-02-08T06:21:21.68767687Z" + created: "2024-02-08T12:28:05.622575381Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -315,7 +327,7 @@ entries: version: 0.8.3-beta.4 - apiVersion: v2 appVersion: 0.8.3-beta.3 - created: "2024-02-08T06:21:21.686397443Z" + created: "2024-02-08T12:28:05.621895208Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -331,7 +343,7 @@ entries: version: 0.8.3-beta.3 - apiVersion: v2 appVersion: 0.8.3-beta.2 - created: "2024-02-08T06:21:21.685378332Z" + created: "2024-02-08T12:28:05.621344558Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -347,7 +359,7 @@ entries: version: 0.8.3-beta.2 - apiVersion: v2 appVersion: 0.8.3-beta.1 - created: "2024-02-08T06:21:21.684302185Z" + created: "2024-02-08T12:28:05.620789068Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -363,7 +375,7 @@ entries: version: 0.8.3-beta.1 - apiVersion: v2 appVersion: 0.8.2 - created: "2024-02-08T06:21:21.683179261Z" + created: "2024-02-08T12:28:05.620194615Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -379,7 +391,7 @@ entries: version: 0.8.2 - apiVersion: v2 appVersion: 0.8.2-beta.60 - created: "2024-02-08T06:21:21.681954926Z" + created: "2024-02-08T12:28:05.619520473Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -395,7 +407,7 @@ entries: version: 0.8.2-beta.60 - apiVersion: v2 appVersion: 0.8.2-beta.59 - created: "2024-02-08T06:21:21.680549752Z" + created: "2024-02-08T12:28:05.618093883Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -411,7 +423,7 @@ entries: version: 0.8.2-beta.59 - apiVersion: v2 appVersion: 0.8.2-beta.58 - created: "2024-02-08T06:21:21.679228709Z" + created: "2024-02-08T12:28:05.617464305Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -427,7 +439,7 @@ entries: version: 0.8.2-beta.58 - apiVersion: v2 appVersion: 0.8.2-beta.57 - created: "2024-02-08T06:21:21.678499322Z" + created: "2024-02-08T12:28:05.6168203Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -443,7 +455,7 @@ entries: version: 0.8.2-beta.57 - apiVersion: v2 appVersion: 0.8.2-beta.56 - created: "2024-02-08T06:21:21.677685106Z" + created: "2024-02-08T12:28:05.616154193Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -459,7 +471,7 @@ entries: version: 0.8.2-beta.56 - apiVersion: v2 appVersion: 0.8.2-beta.53 - created: "2024-02-08T06:21:21.676619549Z" + created: "2024-02-08T12:28:05.615507422Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -475,7 +487,7 @@ entries: version: 0.8.2-beta.53 - apiVersion: v2 appVersion: 0.8.2-beta.52 - created: "2024-02-08T06:21:21.675975351Z" + created: "2024-02-08T12:28:05.61487075Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -491,7 +503,7 @@ entries: version: 0.8.2-beta.52 - apiVersion: v2 appVersion: 0.8.2-beta.51 - created: "2024-02-08T06:21:21.675241746Z" + created: "2024-02-08T12:28:05.614198733Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -507,7 +519,7 @@ entries: version: 0.8.2-beta.51 - apiVersion: v2 appVersion: 0.8.2-beta.50 - created: "2024-02-08T06:21:21.674390341Z" + created: "2024-02-08T12:28:05.61341177Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -523,7 +535,7 @@ entries: version: 0.8.2-beta.50 - apiVersion: v2 appVersion: 0.8.2-beta.49 - created: "2024-02-08T06:21:21.673145227Z" + created: "2024-02-08T12:28:05.611892844Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -539,7 +551,7 @@ entries: version: 0.8.2-beta.49 - apiVersion: v2 appVersion: 0.8.2-beta.48 - created: "2024-02-08T06:21:21.671957191Z" + created: "2024-02-08T12:28:05.611252335Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -555,7 +567,7 @@ entries: version: 0.8.2-beta.48 - apiVersion: v2 appVersion: 0.8.2-beta.47 - created: "2024-02-08T06:21:21.671281194Z" + created: "2024-02-08T12:28:05.610587792Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -571,7 +583,7 @@ entries: version: 0.8.2-beta.47 - apiVersion: v2 appVersion: 0.8.2-beta.46 - created: "2024-02-08T06:21:21.670630594Z" + created: "2024-02-08T12:28:05.6100102Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -587,7 +599,7 @@ entries: version: 0.8.2-beta.46 - apiVersion: v2 appVersion: 0.8.2-beta.45 - created: "2024-02-08T06:21:21.669960068Z" + created: "2024-02-08T12:28:05.609428862Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -603,7 +615,7 @@ entries: version: 0.8.2-beta.45 - apiVersion: v2 appVersion: 0.8.2-beta.44 - created: "2024-02-08T06:21:21.669387535Z" + created: "2024-02-08T12:28:05.608863474Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -619,7 +631,7 @@ entries: version: 0.8.2-beta.44 - apiVersion: v2 appVersion: 0.8.2-beta.43 - created: "2024-02-08T06:21:21.668791447Z" + created: "2024-02-08T12:28:05.608249264Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -635,7 +647,7 @@ entries: version: 0.8.2-beta.43 - apiVersion: v2 appVersion: 0.8.2-beta.41 - created: "2024-02-08T06:21:21.667891681Z" + created: "2024-02-08T12:28:05.607577507Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -651,7 +663,7 @@ entries: version: 0.8.2-beta.41 - apiVersion: v2 appVersion: 0.8.2-beta.40 - created: "2024-02-08T06:21:21.667163155Z" + created: "2024-02-08T12:28:05.606466964Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -667,7 +679,7 @@ entries: version: 0.8.2-beta.40 - apiVersion: v2 appVersion: 0.8.2-beta.39 - created: "2024-02-08T06:21:21.665109924Z" + created: "2024-02-08T12:28:05.605294589Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -683,7 +695,7 @@ entries: version: 0.8.2-beta.39 - apiVersion: v2 appVersion: 0.8.2-beta.38 - created: "2024-02-08T06:21:21.664547109Z" + created: "2024-02-08T12:28:05.6046963Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -699,7 +711,7 @@ entries: version: 0.8.2-beta.38 - apiVersion: v2 appVersion: 0.8.2-beta.37 - created: "2024-02-08T06:21:21.663964237Z" + created: "2024-02-08T12:28:05.604110794Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -715,7 +727,7 @@ entries: version: 0.8.2-beta.37 - apiVersion: v2 appVersion: 0.8.1 - created: "2024-02-08T06:21:21.663345798Z" + created: "2024-02-08T12:28:05.603486115Z" dependencies: - name: component-chart repository: https://charts.devspace.sh @@ -729,4 +741,4 @@ entries: urls: - https://openmined.github.io/PySyft/helm/syft-0.8.1.tgz version: 0.8.1 -generated: "2024-02-08T06:21:21.662588138Z" +generated: "2024-02-08T12:28:05.602790282Z" diff --git a/packages/grid/helm/repo/syft-0.8.4-beta.21.tgz b/packages/grid/helm/repo/syft-0.8.4-beta.21.tgz new file mode 100644 index 0000000000000000000000000000000000000000..04e87789baf7299845ffffec8e72793abfaa5a84 GIT binary patch literal 7644 zcmV<29V6l&iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PKBhbK5ww=zP|%z)`mL>~2Le-UL;(r>^5@GCmVOj^$+To^x+` z@n}jMQ>2!p?D5Rx{`OlW^?v!KC{9MGns`Ju8fXyR=m#3e?t=W_sHfAsd8hl)cb}tR zjIrdr{EIPO{EIn}-w`e(A)GTRzr%zQDZWGav-D=646|IP-{EJswMg7oQuHAF2x%53 zaSPqhnjQ=$KXFqvdBZ@io0QZXWHt>4dHL!b(jZCuC?56)ca12#2Bc#h-DP2hk{CIi zYn1nNj#B9BFwRgIqb%t|5(9!LgA8Sm-auLlZGo`EwMm>q zoHuRKKR6ped=kds4$kj7#guh$bKnx29fbYvK_2E2ToqIKDhj*3yg7)wwOIF|g^Gu@ z{B8g(qz8j2w7Q_6cC(tuFa0%kUrPSJ(a{jH&n5uw;eSdwUF3gG>7M_0Q8KV8-^Drr{va!zgc5Z)*tklen9-sz1wbGtjpHF34Kb|6PQt*(46a?l3JGuxQW0O+Rta zLV=DlsMV@AY6uOIG;bm5VdB*g>Lpp;LiO@(y@pUB8YRWhVvLnPFOnqxkU`o) zb?y3LTrY8Ed6McbpeXEzc~&&6f9X`uTBt7Gh4otPUqcweluRLntkqKJhFPB8wHE6T zTvjT-O%0(59ICs-UPIap%RyNq&uVcoz~vXp^Dyp~BZN>MW#dsVeikD<8qqp>gNCU(EYpOM}6}6vXrxRmlQ(3dOYgb4?61=LuKp~x)V-n z^ulhhdNGziPzmfsO{bB)VCTX_91)WcRj{Dd`^{$0S^>2isn2;kB@i2K z9YtZ>MZf)qnw4=_FeLi)saeARBN?VBg$Q&8D9li?c(@YyqIx(Fewm;MbQ+_6k^<5u z8RjVO!5G79R4#laXi@%zdJy$dmg{_&t-^{L@zUdLpl$JP5la_xq@!pRni8~X_WLA? zlG}pfQu>TntqkcD5QKRTQly;&caTCqxdGR#>it&!TQ4kn_w=qZj2%GWWB5 zG(ul*A@X|uO;2aNdJEO5_JtBgYN{kq9uVyZEbuvVm{8?0Yy%}2I4ihM6nB;=>^tHE z0kDFb`T|~U^QytRcCJ2oSB=4^3b=k)0=?C_ejmoyTcCnq6Zjl_UoubnA|OUn#R6;y z1`m6jX%5C3AHWkD0z)(wAe{u_DA68y5*26WXR!Hcl3##!cUz!BDT+N`Dam|82xzON zl12jdd`YD9tp;nEGnQ&a2&F;+4ww%hECWK=lc@O5#2n|)pYJ}~)P1(5`@;B43m>Qw z9+y60%y*P3DxHu5`_gAjY6?EX#%E3%7JzhuW8Z3Q1IwS({gHCr0Xt0L`!IrYo%h}+ zm0mPY6x~D=T}M>lBn!Y%%^@&aS!$&9v}XkPM)=Gpig=1x#t3Ct`BH1bu#tfiLV3)8 z8h;cD-a|-w8nYNfM8V|F!-{&I9}w#YLOJj`CWI@`r`#H)mGvzJOH|-IOC+(>3mk?g zk#qt5B!|lS*#ZsPdB$>2VXd?i6bQE@RW3vOX7*A z&9(BgY?%;?g_Txo>N&%KDPbk2f=R~>2CI$5)+68(O0lCLEl{GVCrKe2wDg|F^KYWW zTtJ?NaHFF)cONd^Z-G-37Cr!Gm`KZ=$AM$zwDSe^L3v)_6GeULQNY%6>XZ$Fz_$tr z$3kgm*Hcn0g)_vwex{_tMoLaF28|Ub%oebE6rP%5zv) z7l6mTCk^_473TPsL@ndK2soCGTHnJ0TTs$~!j>`|aNkIyxU`ZPU;C6{Yn+sf_<{!1 zXorJmtlDeHc+!-SoG@^hfX{uaEs;L;ea)E#j-?=hZvuq{D`t|BKBr2sK1`d26I}V0$4vI4Z+F9xO#$hP5K<(itQ80Do zY(wQ1So%H~Zn>a=<<#>Fal@RjLOV&h2n+*1aM~(Gc%T?pp5lr~jXlPm{tZf-;bP?2 z+X5lZDptr#pZMGZ=6ukafee)JDPq0+@lnprgeYFAU#SP zY(a|pIItRUAcPE5VgA#oI4Zq!N5d>n z`sK7b>gjkZ2C1>wcUBiX;WHI5!!$L{x5_X}0|)}ExZ*D0QWDNJr@|UpXn2NdBbj(M zIWunCZPBEp@_C@FwU~JhBndbVeCeye5yFhd)G`||3}i}K4Dx6izO9SYU+E~^R4j$tf^NhM>`yBMy=5*FwRrb3Qq)n zAw4Z4OmRmQ1*w362|!qGu*Tr|Oi^qIrK&I@pT2gLqGlVbuQr}4Bp&A)d}6U5_=+>j zD6@t#&tfeD8-VeMCE5le@R$M`*g#n7c^hnitq(R8922Fi&>FOo(kI+m8w7=p97w;lB7^|P= z8Z7kq%JDpL;A~-nd)8V>oiLnnM;&L%Fc(Nk40oPj23yZ6Cw%6(AWj)29h2N?#e82` zwaZcW1@(mjUk4g6_6!8f8s%$EgaKbT=Pb93U|?EV;KV1wlYx)5m4P)@J;U`;O!2om z*W2*H28=qN8&4RlrQx8Y1?w~CbFFqtf|eRL%|40%3uDIOwL@K^18H(!tcPuq?#?jkebiyG9SRiy+Gy8KgI% zg=g&zyl?-nWPBn1?|IDI$NzUx=Jx-R+I1Ma7V1( z&1+TlZanH2NdylV^RxhanFEv#BN#0sdJw)#li?t1q5rAZ|8I#Qg)AAS7RqOX#AWr@ zs9x; zf&Dp%lI~*DD;r#a(@b70ApSWqpPL>%7-XxO&T^eYFpRRT59w3q_nR5Qg8XTd&t0gV52m&&3X8%vBq;!aA~KE(NX5{34z zh0an3DY!C^pv=82GyAd@B1??fFZ9OuGb8iZv3UfU4iq2O)#>3$dnUe8CgFddCjHi2 z&?N{VawD~Q>G`?Nd#%a0E1eB1n=6{?oF5*2hW5(DtEEOy&Q9N*U7ep@TrRfQLiL&D zuFUElFL(7dXnr{BtU$AbrI%#+>;#-g@}bkdST*7+5REg;*CRVW>~wxQyLfy5a8_pi zk8rnQNT(b$YJTJssJb!RdR4CBu7%Ph$RqRO@$vYmy>eMf-jG zu(P7VAWht`jQK6q`Pjbb9G{(T{R^X2b5!gBS)Vdsa9A0|#5G{;Ho&6%X~ zRp03J>}~t%@*n4$zx6!Iar*jNeD}_sWqgYgaSmVS_-UOSXU(rd+_^kF`tj=U{P^m} z_CG$;+$_H7nyH!P+FlFjZ~^%t8BT}<;4owyPk~yJZ!&P+rs>5 z3@fUwA%}1()pA>J+aIq^&fY$vcW=*54v$Z-j?PX`+eeoL`>tt`VYXt-+oy+b-nXyL z-n{?#h&J2bAAWd$d3AAic6rr4IXr&<2)0=~Q(_hC%r)?t#rp;pn(69#AtqXyN~Ys6^dDs&)2CHo%Z2R?e^QNld}(} zmybd^IY?$D-v9d0{?L8^zBwa*g^amqA6~Yv&M(g1emGkBjq`N4f=NoA?0h(BuN0Z9 zB<<-*`{eB6A6Gw}UHrJZ)>@`oIA~=hA@9%LUA=FAY(Ius$EUh%V$!O+SLbJE@2`$e z4&QBFLrX1Wd~go>^yy%7mdrsn4P9fT7PZzZE)VL+Q8AewGlHwnqnR5>h;K^%TVZv4 zE)t_K0DEU6SZFi~Zy<&&J5Lj{_^uk1i6zy*lQ$`77p|+8fgPq{es`3_IsB3@zRh)- z4+pC%-jqi)y@kS@TQtOhU1w0Vf0V>op6W23p}hL^N0AQumt&7+ zug*Wbd4GIV`UpQ;2!=bTrBMWq?mep`)?=Zx8E8l49WxFS8J4cR0eEjp@E0aIW;kqoa`_`g%raxt5E>2Mc;LpIQe`Wbow+AlXs*Vb0=)uP(Hg|A_%W7q zn>sKZ$kF4Y>A~vj8k*LbX>4+G{+m%0p)%I0HKH}=xg4a)FXPyMDO0Bk`LJ&w9f{AG z?3rGk55`afo%d!eJ2TtdSeu+{bY0V{IGp45%`}fGSAG#En`7u_F3P@lAfK7B&y6?&sHkh}r0O&qu>Ty0J( z{nC8%t&}zLe~a1wPl^8sZy*2JP5E~8zh?%2#@Ef>7ylWsk1_nnllJpX4J z#(jkPUsZYunLo2p=&wHQXFjC3Q-MXPog8xwJFy!`NME#OV zJbK=+NgUj(6z|pF9jd<*{;48fCwkCY`}O)dj2jhPZNvBd(#zMrJhQCS|8^b!;fwxX z;qiU^e<$Uei2uwhbPtRFOqTf{6h0={(KJS}|iUS6!W36=sd=&F+#LogPG?Iauoc zB3Hga7zv)&lbtA9fW!z?{8VSE7b4Kh0GRhNxt z5x@X3jQ;%RstMamP(5RlUHyO>m6*A@hCB49-)2wG{)9%`@)zEn|Md0~Y78?-QK4`bBOgF$pxNB^hn$nU6NB$V|aiqLbFpGMl-(b&biLU&ta@^3rCeP1?}weo+9IiUCXe>}D%{~6uO z|DBX=$p5d?X}mnuUuxp$q^A!h@e%TTE&p#Z2lQV3PcHg@B;))2KRYRp%>SIukHs9& zFP&|Z6xMq)FDs~h-sQJkHpu_2 z1px1p|HNCm|5>no{^w50UjNIUdupgs>LGgvDzG^;WEKr$>>+3>usJkz1`WrUJOIrI zY$qz~VlZ~^ZFoUtz5L%^0QesHF9cq?|C{gQzq={Vmj7R4((t1V>h){JOHUr2C-o@_ z#4G$HUpV@_NYjf>Hdu7fuZTXcxxVt_#NU7NX5+1$`-?zd%0-~tx*lcH&L8f6lr=jj zAh%Q9#Np7>6mh+>0f&D{|3Ilq_eX&-lRh8R_3lO3vEJ zwD$Fj$!~x1O1)u5?|K8#jLe_gLG*2yv5WTKU6614_|JR&e?l(q{}&wZ|k8rm>5H*E)L=Y~^%!vF?Yqc{kn5IeYcO$U!utJ54V1oYAJDM7uXS(y@hf(Q}aA z%s5%brGbWN)I#;Lse{R7gj4%^#T^gNZDfS9K4asKZSO6%GDWHwW8=osWSGy@T{Vj7 zu(0gwg_Ty_F!7gv9{lsK^%)3=^YrdK3FCaB8x2&y1v3jez7Ye}A2hhD+SY^cpMOQa z|BmLKy2NHH9;e_EyM^8^eWk+wZH(3RdwN{JZlMp8Z#et9A37JotxjPX(`+9b!!)XI z1Sl-%F~mx&OVG0o7S`|5G_=DkPx>Q6XC(weKa>}+%+Jr&O%@Y+&Q?mp!Jr)Dd7Ra= z!s1@^QXmv0g%K9%tk)Xv0Us?nv!bROB}Pvgvpular*J!wQ?ovd3}SZ}C-GfB8D<}{ z(s--hmjF@HotV0C15pbdpMHNml%7E*fZit-FvlYp>8{FU(Lj z9F*H4M#&8Yw^nhiFz&uaDMUICZ=k#xY38|(UF{W$p<5bS=r1EN;m~cE_t3xnwTaH+ z=nf@uS+nR88bFGoFotHW`L=V_$&(anuh3D_? Date: Thu, 8 Feb 2024 12:45:20 +0000 Subject: [PATCH 46/46] [hagrid] bump version --- packages/hagrid/.bumpversion.cfg | 2 +- packages/hagrid/hagrid/manifest_template.yml | 4 ++-- packages/hagrid/hagrid/version.py | 2 +- packages/hagrid/setup.py | 2 +- scripts/hagrid_hash | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/hagrid/.bumpversion.cfg b/packages/hagrid/.bumpversion.cfg index d1128b5281a..5e8d9c0fbed 100644 --- a/packages/hagrid/.bumpversion.cfg +++ b/packages/hagrid/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.106 +current_version = 0.3.107 tag = False tag_name = {new_version} commit = True diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index 231319d361d..0572eacecf5 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -1,9 +1,9 @@ manifestVersion: 0.1 -hagrid_version: 0.3.106 +hagrid_version: 0.3.107 syft_version: 0.8.4-beta.21 dockerTag: 0.8.4-beta.21 baseUrl: https://raw.githubusercontent.com/OpenMined/PySyft/ -hash: 639535349895459c986d633a29a01bad7013ef63 +hash: 73aa361520477bfb282bf7517f66b4867ce7c838 target_dir: ~/.hagrid/PySyft/ files: grid: diff --git a/packages/hagrid/hagrid/version.py b/packages/hagrid/hagrid/version.py index 1a90352e02a..37151a1b796 100644 --- a/packages/hagrid/hagrid/version.py +++ b/packages/hagrid/hagrid/version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # HAGrid Version -__version__ = "0.3.106" +__version__ = "0.3.107" if __name__ == "__main__": print(__version__) diff --git a/packages/hagrid/setup.py b/packages/hagrid/setup.py index e8c7cbf999c..377da3b1768 100644 --- a/packages/hagrid/setup.py +++ b/packages/hagrid/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages from setuptools import setup -__version__ = "0.3.106" +__version__ = "0.3.107" DATA_FILES = {"img": ["hagrid/img/*.png"], "hagrid": ["*.yml"]} diff --git a/scripts/hagrid_hash b/scripts/hagrid_hash index b0293a9d097..3c519269255 100644 --- a/scripts/hagrid_hash +++ b/scripts/hagrid_hash @@ -1 +1 @@ -55fa40104063afa8e52213b1c8d29634 +37c1c6fabbb3c234ec5c7a10af136585