diff --git a/.github/workflows/pr-tests-enclave.yml b/.github/workflows/pr-tests-enclave.yml deleted file mode 100644 index 48a59f789de..00000000000 --- a/.github/workflows/pr-tests-enclave.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: PR Tests - Enclave - -on: - # Temporarily disabled oblv tests - # workflow_call: - - # pull_request: - # branches: - # - dev - # - main - # - "0.8" - - workflow_dispatch: - inputs: - none: - description: "Run Tests Manually" - required: false - -concurrency: - group: enclave-${{ github.event_name == 'pull_request' && format('{0}-{1}', github.workflow, github.event.pull_request.number) || github.workflow_ref }} - cancel-in-progress: true - -jobs: - pr-tests-enclave-oblv: - strategy: - max-parallel: 4 - matrix: - os: [ubuntu-latest] - python-version: ["3.12"] - - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - # free 10GB of space - - name: Remove unnecessary files - if: matrix.os == 'ubuntu-latest' - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - docker image prune --all --force - docker builder prune --all --force - docker system prune --all --force - - - name: Check for file changes - uses: dorny/paths-filter@v3 - id: changes - with: - base: ${{ github.ref }} - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - if: steps.changes.outputs.syft == 'true' - with: - python-version: ${{ matrix.python-version }} - - - name: Upgrade pip - if: steps.changes.outputs.syft == 'true' - run: | - pip install --upgrade pip uv==0.1.18 - uv --version - - - name: Get pip cache dir - id: pip-cache - if: steps.changes.outputs.syft == 'true' - shell: bash - run: | - echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - - - name: pip cache - uses: actions/cache@v4 - if: steps.changes.outputs.syft == 'true' - with: - path: ${{ steps.pip-cache.outputs.dir }} - key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }} - restore-keys: | - ${{ runner.os }}-uv-py${{ matrix.python-version }}- - - - name: Install Dependencies - if: steps.changes.outputs.syft == 'true' - run: | - pip install --upgrade tox tox-uv==1.5.1 - - - name: Run Enclave tests - if: steps.changes.outputs.syft == 'true' - run: | - tox -e stack.test.integration.enclave.oblv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e1ead0e3f0..cc221c69c61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -173,7 +173,7 @@ repos: name: "mypy: syft" always_run: true files: "^packages/syft/src/syft/" - exclude: "packages/syft/src/syft/types/dicttuple.py|^packages/syft/src/syft/service/action/action_graph.py|^packages/syft/src/syft/external/oblv/" + exclude: "packages/syft/src/syft/types/dicttuple.py|^packages/syft/src/syft/service/action/action_graph.py" args: [ "--follow-imports=skip", "--ignore-missing-imports", diff --git a/docs/source/api_reference/syft.external.oblv.rst b/docs/source/api_reference/syft.external.oblv.rst deleted file mode 100644 index 18eeb313704..00000000000 --- a/docs/source/api_reference/syft.external.oblv.rst +++ /dev/null @@ -1,82 +0,0 @@ -syft.external.oblv package -========================== - -.. automodule:: syft.external.oblv - :members: - :undoc-members: - :show-inheritance: - -Submodules ----------- - -syft.external.oblv.auth module ------------------------------- - -.. automodule:: syft.external.oblv.auth - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.constants module ------------------------------------ - -.. automodule:: syft.external.oblv.constants - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.deployment module ------------------------------------- - -.. automodule:: syft.external.oblv.deployment - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.deployment\_client module --------------------------------------------- - -.. automodule:: syft.external.oblv.deployment_client - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.exceptions module ------------------------------------- - -.. automodule:: syft.external.oblv.exceptions - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.oblv\_keys module ------------------------------------- - -.. automodule:: syft.external.oblv.oblv_keys - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.oblv\_keys\_stash module -------------------------------------------- - -.. automodule:: syft.external.oblv.oblv_keys_stash - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.oblv\_proxy module -------------------------------------- - -.. automodule:: syft.external.oblv.oblv_proxy - :members: - :undoc-members: - :show-inheritance: - -syft.external.oblv.oblv\_service module ---------------------------------------- - -.. automodule:: syft.external.oblv.oblv_service - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api_reference/syft.external.rst b/docs/source/api_reference/syft.external.rst index 0a59c9187b3..00110dba1d7 100644 --- a/docs/source/api_reference/syft.external.rst +++ b/docs/source/api_reference/syft.external.rst @@ -25,14 +25,3 @@ - - - -.. rubric:: Modules - -.. autosummary:: - :toctree: - :recursive: - - syft.external.oblv - diff --git a/docs/source/api_reference/syft.node.node.rst b/docs/source/api_reference/syft.node.node.rst index fe87e6e5fa6..1c71dd53574 100644 --- a/docs/source/api_reference/syft.node.node.rst +++ b/docs/source/api_reference/syft.node.node.rst @@ -14,7 +14,6 @@ syft.node.node .. autosummary:: create_admin_new - create_oblv_key_pair create_worker_metadata get_default_root_email get_default_root_password diff --git a/notebooks/tutorials/data-engineer/01-setting-up-dev-mode.ipynb b/notebooks/tutorials/data-engineer/01-setting-up-dev-mode.ipynb index c0805affa3b..ed84817235f 100644 --- a/notebooks/tutorials/data-engineer/01-setting-up-dev-mode.ipynb +++ b/notebooks/tutorials/data-engineer/01-setting-up-dev-mode.ipynb @@ -120,7 +120,6 @@ "syft.test.unit\n", "syft.test.notebook\n", "stack.test.notebook\n", - "stack.test.integration.enclave.oblv\n", "stack.test.vm\n", "frontend.test.unit\n", "frontend.test.e2e\n", diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 1520190f0e1..e8d049dc6e5 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -96,8 +96,6 @@ ENV PATH=$PATH:$HOME/.local/bin \ DEV_MODE="False" \ DEBUGGER_ENABLED="False" \ CONTAINER_HOST="docker" \ - OBLV_ENABLED="False" \ - OBLV_LOCALHOST_PORT=3030 \ DEFAULT_ROOT_EMAIL="info@openmined.org" \ DEFAULT_ROOT_PASSWORD="changethis" \ STACK_API_KEY="changeme" \ diff --git a/packages/grid/backend/install_oblivious.sh b/packages/grid/backend/install_oblivious.sh deleted file mode 100755 index 486b812ac9e..00000000000 --- a/packages/grid/backend/install_oblivious.sh +++ /dev/null @@ -1,15 +0,0 @@ -#! /usr/bin/env bash - -echo "Running install_oblivious.sh with RELEASE=${RELEASE}" - -if [[ ("${OBLV_ENABLED}" == "true") && ("${SERVICE_NAME}" == "backend" || "${SERVICE_NAME}" == "celeryworker" ) ]]; then - echo "Allowed to install Oblv CLI" - # Oblivious Proxy Client Installation - mkdir -p oblv-ccli-0.4.0-x86_64-unknown-linux-musl - tar -xf /app/wheels/oblv-ccli-0.4.0-x86_64-unknown-linux-musl.tar.gz -C oblv-ccli-0.4.0-x86_64-unknown-linux-musl - chmod +x $(pwd)/oblv-ccli-0.4.0-x86_64-unknown-linux-musl/oblv - ln -sf $(pwd)/oblv-ccli-0.4.0-x86_64-unknown-linux-musl/oblv /usr/local/bin/oblv #-f is for force - echo "Installed Oblivious CLI: $(/usr/local/bin/oblv --version)" -else - echo "Oblivious CLI not installed OBLV_ENABLED:${OBLV_ENABLED} , SERVICE_NAME:${SERVICE_NAME} " -fi diff --git a/packages/grid/backend/wheels/oblv-ccli-0.4.0-x86_64-unknown-linux-musl.tar.gz b/packages/grid/backend/wheels/oblv-ccli-0.4.0-x86_64-unknown-linux-musl.tar.gz deleted file mode 100644 index 622ee32898c..00000000000 Binary files a/packages/grid/backend/wheels/oblv-ccli-0.4.0-x86_64-unknown-linux-musl.tar.gz and /dev/null differ diff --git a/packages/grid/default.env b/packages/grid/default.env index f5b42ff6323..1874f29031b 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -108,11 +108,6 @@ NODE_SIDE_TYPE=high # Worker USE_BLOB_STORAGE=False -#Oblivious -OBLV_ENABLED=false -OBLV_KEY_PATH="~/.oblv" -OBLV_LOCALHOST_PORT=3030 - # Registation ENABLE_SIGNUP=False diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 4108d23f634..432eef13d66 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -149,8 +149,6 @@ services: - JAEGER_PORT=${JAEGER_PORT} - ASSOCIATION_TIMEOUT=${ASSOCIATION_TIMEOUT} - DEV_MODE=${DEV_MODE} - - OBLV_LOCALHOST_PORT=${OBLV_LOCALHOST_PORT} - - OBLV_ENABLED=${OBLV_ENABLED} - DEFAULT_ROOT_EMAIL=${DEFAULT_ROOT_EMAIL} - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} - BACKEND_STORAGE_PATH=${BACKEND_STORAGE_PATH} @@ -196,8 +194,6 @@ services: # - JAEGER_HOST=${JAEGER_HOST} # - JAEGER_PORT=${JAEGER_PORT} # - DEV_MODE=${DEV_MODE} - # - OBLV_LOCALHOST_PORT=${OBLV_LOCALHOST_PORT} - # - OBLV_ENABLED=${OBLV_ENABLED} # network_mode: service:proxy # volumes: # - credentials-data:/root/data/creds/ @@ -230,8 +226,6 @@ services: # - JAEGER_HOST=${JAEGER_HOST} # - JAEGER_PORT=${JAEGER_PORT} # - DEV_MODE=${DEV_MODE} - # - OBLV_LOCALHOST_PORT=${OBLV_LOCALHOST_PORT} - # - OBLV_ENABLED=${OBLV_ENABLED} # command: "/app/grid/worker-start.sh" # network_mode: service:proxy # volumes: diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index 3ee246adbdd..2374f2dc1cd 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -114,13 +114,6 @@ spec: value: "localhost" - name: JAEGER_PORT value: "14268" - # Oblivious - {{- if .Values.node.oblv.enabled }} - - name: OBLV_LOCALHOST_PORT - value: {{ .Values.node.oblv.port | quote }} - - name: OBLV_ENABLED - value: {{ .Values.node.oblv.enabled | quote }} - {{- end }} # Veilid {{- if .Values.veilid.enabled }} - name: VEILID_ENABLED diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 29936296481..ef26c8ccdf9 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -105,11 +105,6 @@ node: username: apikey password: password - # Oblivious settings - oblv: - enabled: false - port: 3030 - # Extra environment vars env: null diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index 5c7356e6b8d..8868c322cd4 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -103,10 +103,5 @@ data: # Worker USE_BLOB_STORAGE: False - #Oblivious - OBLV_ENABLED: false - OBLV_KEY_PATH: "~/.oblv" - OBLV_LOCALHOST_PORT: 3030 - # Registation ENABLE_SIGNUP: False diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 6a0059c9744..cd6f642013f 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -392,11 +392,6 @@ def clean(location: str) -> None: is_flag=True, help="Turn off auto health checks post node launch", ) -@click.option( - "--oblv", - is_flag=True, - help="Installs Oblivious CLI tool", -) @click.option( "--set-root-email", default=None, @@ -1309,8 +1304,6 @@ def create_launch_cmd( headless = bool(kwargs["headless"]) parsed_kwargs["headless"] = headless - parsed_kwargs["oblv"] = bool(kwargs["oblv"]) - parsed_kwargs["tls"] = bool(kwargs["tls"]) parsed_kwargs["test"] = bool(kwargs["test"]) parsed_kwargs["dev"] = bool(kwargs["dev"]) @@ -2197,7 +2190,6 @@ def create_launch_docker_cmd( smtp_port = kwargs.get("smtp_port") smtp_host = kwargs.get("smtp_host") - enable_oblv = bool(kwargs["oblv"]) print(" - NAME: " + str(snake_name)) print(" - TEMPLATE DIR: " + template_grid_dir) if compose_src_path: @@ -2215,9 +2207,6 @@ def create_launch_docker_cmd( print(" - PORT: " + str(host_term.free_port)) print(" - DOCKER COMPOSE: " + docker_version) print(" - IN-MEMORY WORKERS: " + str(in_mem_workers)) - if enable_oblv: - print(" - OBLV: ", enable_oblv) - print("\n") use_blob_storage = ( @@ -2255,7 +2244,6 @@ def create_launch_docker_cmd( "STACK_API_KEY": str( generate_sec_random_password(length=48, special_chars=False) ), - "OBLV_ENABLED": str(enable_oblv).lower(), "CREDENTIALS_VOLUME": host_path, "NODE_SIDE_TYPE": kwargs["node_side_type"], "SINGLE_CONTAINER_MODE": single_container_mode, diff --git a/packages/syft/src/syft/__init__.py b/packages/syft/src/syft/__init__.py index 8801eb8c987..7feb69fca8c 100644 --- a/packages/syft/src/syft/__init__.py +++ b/packages/syft/src/syft/__init__.py @@ -26,8 +26,6 @@ from .client.user_settings import UserSettings # noqa: F401 from .client.user_settings import settings # noqa: F401 from .custom_worker.config import DockerWorkerConfig # noqa: F401 -from .external import OBLV_ENABLED # noqa: F401 -from .external import enable_external_lib # noqa: F401 from .node.credentials import SyftSigningKey # noqa: F401 from .node.domain import Domain # noqa: F401 from .node.enclave import Enclave # noqa: F401 @@ -107,10 +105,6 @@ except: # noqa: E722 pass # nosec -# For server-side, to enable by environment variable -if OBLV_ENABLED: - enable_external_lib("oblv") - def module_property(func: Any) -> Callable: """Decorator to turn module functions into properties. diff --git a/packages/syft/src/syft/external/__init__.py b/packages/syft/src/syft/external/__init__.py deleted file mode 100644 index b03c6594322..00000000000 --- a/packages/syft/src/syft/external/__init__.py +++ /dev/null @@ -1,64 +0,0 @@ -"""This module contains all the external libraries that Syft supports. -We lazy load the external libraries when they are needed. -""" - -# stdlib -import importlib -import os -from typing import Any - -# relative -from ..service.response import SyftError -from ..service.response import SyftSuccess -from ..service.service import AbstractService -from ..util.util import str_to_bool - -# Contains all the external libraries that Syft supports. -# Used to check if a library is supported -# if the external library is not installed, we prompt the user -# to install it with the pip package name. - -OBLV_ENABLED = str_to_bool(os.getenv("OBLV_ENABLED", "false")) - -EXTERNAL_LIBS = { - "oblv": { - "pip_package_name": "oblv-ctl", - "module_name": "oblv_ctl", - } -} - - -def OblvServiceProvider(*args: Any, **kwargs: Any) -> type[AbstractService] | None: - if OBLV_ENABLED: - # relative - from .oblv.oblv_service import OblvService - - return OblvService(*args, **kwargs) - return None - - -def package_exists(package_name: str) -> bool: - try: - importlib.import_module(package_name) - return True - except ImportError: - return False - - -def enable_external_lib(lib_name: str) -> SyftSuccess | SyftError: - if lib_name in EXTERNAL_LIBS: - syft_module_name = f"syft.external.{lib_name}" - pip_package_name = EXTERNAL_LIBS[lib_name]["pip_package_name"] - if not package_exists(EXTERNAL_LIBS[lib_name]["module_name"]): - return SyftError( - message=f"Package: {pip_package_name} for library: {lib_name} not installed.\n" - + f"Kindly install it with 'pip install {pip_package_name}'" - ) - - importlib.import_module(syft_module_name) - return SyftSuccess(message=f"Successfully enabled external library: {lib_name}") - else: - return SyftError( - message=f"External library {lib_name} not supported. \n" - + f"Supported external libraries are: {list(EXTERNAL_LIBS.keys())}" - ) diff --git a/packages/syft/src/syft/external/oblv/__init__.py b/packages/syft/src/syft/external/oblv/__init__.py deleted file mode 100644 index 1fa96bd120e..00000000000 --- a/packages/syft/src/syft/external/oblv/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# relative -from ...serde.deserialize import _deserialize -from ...serde.serializable import recursive_serde_register -from ...serde.serialize import _serialize -from .auth import login # noqa: F401 -from .deployment import create_deployment # noqa: F401 -from .oblv_proxy import check_oblv_proxy_installation_status # noqa: F401 -from .oblv_proxy import create_oblv_key_pair # noqa: F401 -from .oblv_proxy import get_oblv_public_key # noqa: F401 -from .oblv_proxy import install_oblv_proxy # noqa: F401 - -try: - # third party - from oblv_ctl.oblv_client import OblvClient - - # Oblivious Client serde - recursive_serde_register( - OblvClient, - serialize=lambda x: _serialize([x.token, x.oblivious_user_id], to_bytes=True), - deserialize=lambda x: OblvClient(*_deserialize(x, from_bytes=True)), - ) - -except Exception: # nosec - pass diff --git a/packages/syft/src/syft/external/oblv/auth.py b/packages/syft/src/syft/external/oblv/auth.py deleted file mode 100644 index 0bb6b9aec78..00000000000 --- a/packages/syft/src/syft/external/oblv/auth.py +++ /dev/null @@ -1,13 +0,0 @@ -# stdlib -from getpass import getpass -from typing import Any - -# third party -from oblv_ctl import authenticate - - -def login(apikey: str | None = None) -> Any: - if apikey is None: - apikey = getpass("Please provide your oblv API_KEY to login:") - - return authenticate(apikey) diff --git a/packages/syft/src/syft/external/oblv/constants.py b/packages/syft/src/syft/external/oblv/constants.py deleted file mode 100644 index 444c915f986..00000000000 --- a/packages/syft/src/syft/external/oblv/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -INFRA = "m5.4xlarge" -REPO_OWNER = "OpenMined" -REPO_NAME = "syft-enclave" -REF = "dev" -REGION = "us-west-2" -VCS = "github" -VISIBILITY = "private" -REF_TYPE = "branch" -LOCAL_MODE = True -OBLV_LOCALHOST_PORT = 3030 diff --git a/packages/syft/src/syft/external/oblv/deployment.py b/packages/syft/src/syft/external/oblv/deployment.py deleted file mode 100644 index 23750e28577..00000000000 --- a/packages/syft/src/syft/external/oblv/deployment.py +++ /dev/null @@ -1,140 +0,0 @@ -# stdlib -from typing import Any - -# third party -from oblv_ctl import OblvClient -from oblv_ctl.models import CreateDeploymentInput -import yaml - -# relative -from ...util.util import bcolors -from .auth import login -from .constants import INFRA -from .constants import REF -from .constants import REF_TYPE -from .constants import REGION -from .constants import REPO_NAME -from .constants import REPO_OWNER -from .constants import VCS -from .constants import VISIBILITY -from .deployment_client import DeploymentClient -from .exceptions import OblvKeyNotFoundError -from .oblv_proxy import create_oblv_key_pair -from .oblv_proxy import get_oblv_public_key - -SUPPORTED_REGION_LIST = ["us-east-1", "us-west-2", "eu-central-1", "eu-west-2"] - -SUPPORTED_INFRA = [ - "c5.xlarge", - "m5.xlarge", - "r5.xlarge", - "c5.2xlarge", - "m5.2xlarge", - "m5.4xlarge", -] - - -def create_deployment( - domain_clients: list, - deployment_name: str | None = None, - key_name: str | None = None, - oblv_client: OblvClient | None = None, - infra: str = INFRA, - region: str = REGION, -) -> DeploymentClient: - """Creates a new deployment with predefined codebase - Args: - client : Oblivious Client. - domain_clients: List of domain_clients. - deployment_name: Unique name for the deployment. - key_name: User's key to be used for deployment creation. - infra: Represent the AWS infrastructure to be used. Default is "m5.2xlarge". The available options are\n - - "c5.xlarge" {'CPU':4, 'RAM':8, 'Total/hr':0.68}\n - - "m5.xlarge" {'CPU':4, 'RAM':16, 'Total/hr':0.768}\n - - "r5.xlarge" {'CPU':4, 'RAM':32, 'Total/hr':1.008}\n - - "c5.2xlarge" {'CPU':8, 'RAM':16, 'Total/hr':1.36}\n - - "m5.2xlarge" {'CPU':8, 'RAM':32, 'Total/hr':1.536}\n - As of now, PySyft only works with RAM >= 32 - region: AWS Region to be deployed in. Default is "us-east-1". The available options are \n - - "us-east-1" : "US East (N. Virginia)",\n - - "us-west-2" : "US West (Oregon)",\n - - "eu-central-1" : "Europe (Frankfurt)",\n - - "eu-west-2" : "Europe (London)" - - Returns: - resp: Deployment Client Object - """ - if not oblv_client: - oblv_client = login() - if deployment_name is None: - deployment_name = input("Kindly provide deployment name") - if key_name is None: - key_name = input("Please provide your key name") - - while not SUPPORTED_INFRA.__contains__(infra): - infra = input(f"Provide infra from one of the following - {SUPPORTED_INFRA}") - - while not SUPPORTED_REGION_LIST.__contains__(region): - region = input( - f"Provide region from one of the following - {SUPPORTED_REGION_LIST}" - ) - - try: - user_public_key = get_oblv_public_key(key_name) - except FileNotFoundError: - user_public_key = create_oblv_key_pair(key_name) - print( - bcolors.green(bcolors.bold("Created")) - + f" a new public/private key pair with key_name: {key_name}" - ) - except Exception as e: - raise Exception(e) - build_args: dict[str, Any] = { - "auth": {}, - "users": {"domain": [], "user": []}, - "additional_args": {}, - "infra_reqs": infra, - "runtime_args": "", - } - users = [] - runtime_args: list[str] = [] - for domain_client in domain_clients: - try: - users.append( - { - "user_name": domain_client.name, - "public key": domain_client.api.services.oblv.get_public_key(), - } - ) - except OblvKeyNotFoundError: - raise OblvKeyNotFoundError( - f"Oblv Public Key not found for {domain_client.name}" - ) - - build_args["runtime_args"] = yaml.dump({"outbound": runtime_args}) - build_args["users"]["domain"] = users - profile = oblv_client.user_profile() - users = [{"user_name": profile.oblivious_login, "public key": user_public_key}] - build_args["users"]["user"] = users - depl_input = CreateDeploymentInput( - owner=REPO_OWNER, - repo=REPO_NAME, - account_type=VCS, - ref=REF, - ref_type=REF_TYPE, - region_name=region, - deployment_name=deployment_name, - visibility=VISIBILITY, - is_dev_env=True, - tags=[], - build_args=build_args, - ) - # By default the deployment is in PROD mode - res = oblv_client.create_deployment(depl_input) - result = DeploymentClient( - deployment_id=res.deployment_id, - oblv_client=oblv_client, - domain_clients=domain_clients, - key_name=key_name, - ) - return result diff --git a/packages/syft/src/syft/external/oblv/deployment_client.py b/packages/syft/src/syft/external/oblv/deployment_client.py deleted file mode 100644 index 4ea10db2602..00000000000 --- a/packages/syft/src/syft/external/oblv/deployment_client.py +++ /dev/null @@ -1,374 +0,0 @@ -# future -from __future__ import annotations - -# stdlib -from collections.abc import Callable -from datetime import datetime -import os -from signal import SIGTERM -import subprocess # nosec -import sys -import time -from typing import Any -from typing import TYPE_CHECKING - -# third party -from oblv_ctl import OblvClient -from pydantic import field_validator -import requests - -# relative -from ...client.api import SyftAPI -from ...client.client import SyftClient -from ...client.client import login -from ...client.client import login_as_guest -from ...client.enclave_client import EnclaveMetadata -from ...node.credentials import SyftSigningKey -from ...serde.serializable import serializable -from ...service.response import SyftError -from ...types.uid import UID -from ...util.util import bcolors -from .constants import LOCAL_MODE -from .exceptions import OblvEnclaveError -from .exceptions import OblvUnAuthorizedError -from .oblv_proxy import check_oblv_proxy_installation_status - -if TYPE_CHECKING: - # relative - from ...service.code.user_code import SubmitUserCode - - -@serializable() -class OblvMetadata(EnclaveMetadata): - """Contains Metadata to connect to Oblivious Enclave""" - - deployment_id: str | None = None - oblv_client: OblvClient | None = None - - @field_validator("deployment_id") - @classmethod - def check_valid_deployment_id(cls, deployment_id: str) -> str: - if not deployment_id and not LOCAL_MODE: - raise ValueError( - f"Deployment ID should be a valid string: {deployment_id}" - + "in cloud deployment of enclave" - + "For testing set the LOCAL_MODE variable in constants.py" - ) - return deployment_id - - @field_validator("oblv_client") - @classmethod - def check_valid_oblv_client(cls, oblv_client: OblvClient) -> OblvClient: - if not oblv_client and not LOCAL_MODE: - raise ValueError( - f"Oblivious Client should be a valid client: {oblv_client}" - + "in cloud deployment of enclave" - + "For testing set the LOCAL_MODE variable in constants.py" - ) - return oblv_client - - -class DeploymentClient: - deployment_id: str - key_name: str - domain_clients: list[SyftClient] # List of domain client objects - oblv_client: OblvClient = None - __conn_string: str - __logs: Any - __process: Any - __enclave_client: SyftClient | None - - def __init__( - self, - domain_clients: list[SyftClient], - deployment_id: str, - oblv_client: OblvClient | None = None, - key_name: str | None = None, - api: SyftAPI | None = None, - ): - if not domain_clients: - raise Exception( - "domain_clients should be populated with valid domain nodes" - ) - self.deployment_id = deployment_id - self.key_name: str | None = key_name - self.oblv_client = oblv_client - self.domain_clients = domain_clients - self.__conn_string = "" - self.__process = None - self.__logs = None - self._api = api - self.__enclave_client: SyftClient | None = None - - def make_request_to_enclave( - self, - request_method: Callable, - connection_string: str, - params: dict | None = None, - files: dict | None = None, - data: dict | None = None, - json: dict | None = None, - ) -> Any: - header = {} - if LOCAL_MODE: - header["x-oblv-user-name"] = "enclave_test" - header["x-oblv-user-role"] = "user" - else: - depl = self.oblv_client.deployment_info(self.deployment_id) - if depl.is_deleted: - raise Exception( - "User cannot connect to this deployment, as it is no longer available." - ) - return request_method( - connection_string, - headers=header, - params=params, - files=files, - data=data, - json=json, - ) - - def set_conn_string(self, url: str) -> None: - self.__conn_string = url - - def initiate_connection(self, connection_port: int = 3030) -> None: - if LOCAL_MODE: - self.__conn_string = f"http://127.0.0.1:{connection_port}" - return - check_oblv_proxy_installation_status() - self.close_connection() # To close any existing connections - public_file_name = os.path.join( - os.path.expanduser("~"), - ".ssh", - self.key_name, - self.key_name + "_public.der", - ) - private_file_name = os.path.join( - os.path.expanduser("~"), - ".ssh", - self.key_name, - self.key_name + "_private.der", - ) - log_file_name = os.path.join( - os.path.expanduser("~"), - ".oblv_syft_logs", - "proxy_logs_" + datetime.now().strftime("%d_%m_%Y_%H_%M_%S") + ".log", - ) - # Creating directory if not exist - os.makedirs(os.path.dirname(log_file_name), exist_ok=True) - log_file = open(log_file_name, "wb") - depl = self.oblv_client.deployment_info(self.deployment_id) - if depl.is_deleted: - raise Exception( - "User cannot connect to this deployment, as it is no longer available." - ) - try: - if depl.is_dev_env: - process = subprocess.Popen( # nosec - [ - "oblv", - "connect", - "--private-key", - private_file_name, - "--public-key", - public_file_name, - "--url", - depl.instance.service_url, - "--pcr0", - depl.pcr_codes[0], - "--pcr1", - depl.pcr_codes[1], - "--pcr2", - depl.pcr_codes[2], - "--port", - "443", - "--lport", - str(connection_port), - "--disable-pcr-check", - ], - stdout=log_file, - stderr=log_file, - ) - else: - process = subprocess.Popen( # nosec - [ - "oblv", - "connect", - "--private-key", - private_file_name, - "--public-key", - public_file_name, - "--url", - depl.instance.service_url, - "--pcr0", - depl.pcr_codes[0], - "--pcr1", - depl.pcr_codes[1], - "--pcr2", - depl.pcr_codes[2], - "--port", - "443", - "--lport", - str(connection_port), - ], - stdout=log_file, - stderr=log_file, - ) - with open(log_file_name) as log_file_read: - while True: - log_line = log_file_read.readline() - if "Error: Invalid PCR Values" in log_line: - raise Exception("PCR Validation Failed") - if "Only one usage of each socket address" in log_line: - raise Exception( - "Another oblv proxy instance running. Either close that connection" - + "or change the *connection_port*" - ) - elif "error" in log_line.lower(): - raise Exception(log_line) - elif "listening on" in log_line: - break - except Exception as e: - raise e - else: - print( - f"Successfully connected to proxy on port {connection_port}. The logs can be found at {log_file_name}" - ) - self.__conn_string = f"http://127.0.0.1:{connection_port}" - self.__logs = log_file_name - self.__process = process - return - - def register( - self, - name: str, - email: str, - password: str, - institution: str | None = None, - website: str | None = None, - ) -> SyftError | SyftSigningKey | None: - self.check_connection_string() - guest_client = login_as_guest(url=self.__conn_string) - return guest_client.register( - name=name, - email=email, - password=password, - institution=institution, - website=website, - ) - - def login( - self, - email: str, - password: str, - ) -> None: - self.check_connection_string() - self.__enclave_client = login( - url=self.__conn_string, email=email, password=password - ) - - def check_connection_string(self) -> None: - if not self.__conn_string: - raise Exception( - "Either proxy not running or not initiated using syft." - + " Run the method initiate_connection to initiate the proxy connection" - ) - - def sanity_check_oblv_response(self, req: requests.Response) -> str: - if req.status_code == 401: - raise OblvUnAuthorizedError() - elif req.status_code == 400: - raise OblvEnclaveError(req.json()["detail"]) - elif req.status_code == 422: - print(req.text) - # ToDo - Update here - elif req.status_code != 200: - raise OblvEnclaveError( - f"Failed to perform the operation with status {req.status_code}, {req.content!r}" - ) - return "Failed" - - def request_code_execution(self, code: SubmitUserCode) -> Any: - # relative - from ...service.code.user_code import SubmitUserCode - - if not isinstance(code, SubmitUserCode): - raise Exception( - f"The input code should be of type: {SubmitUserCode} got:{type(code)}" - ) - - enclave_metadata = OblvMetadata( - deployment_id=self.deployment_id, oblv_client=self.oblv_client - ) - - code_id = UID() - code.id = code_id - code.enclave_metadata = enclave_metadata - - for domain_client in self.domain_clients: - domain_client.code.request_code_execution(code=code) - print(f"Sent code execution request to {domain_client.name}") - - res = self.api.services.code.request_code_execution(code=code) - print(f"Execution will be done on {self.__enclave_client.name}") - - return res - - @property - def api(self) -> SyftAPI: - if not self.__enclave_client: - raise Exception("Kindly login or register with the enclave") - - return self.__enclave_client.api - - def close_connection(self) -> str | None: - if self.check_proxy_running(): - os.kill(self.__process.pid, SIGTERM) - return None - else: - return "No Proxy Connection Running" - - def check_proxy_running(self) -> bool: - if self.__process is not None: - if self.__process.poll() is not None: - return False - else: - return True - return False - - def fetch_current_proxy_logs( - self, follow: bool = False, tail: bool = False - ) -> None: - """Returns the logs of the running enclave instance - - Args: - follow (bool, optional): To follow the logs as they grow. Defaults to False. - tail (bool, optional): Only show the new generated logs. - To be used only when follow is True. Defaults to False. - """ - if self.__logs is None: - print( - bcolors.RED - + bcolors.BOLD - + "Exception" - + bcolors.BLACK - + bcolors.ENDC - + ": Logs not initiated", - file=sys.stderr, - ) - log_file = open(self.__logs) - if not follow: - print(log_file.read()) - else: - if tail: - log_file.seek(0, 2) - while True: - line = log_file.readline() - if not line: - time.sleep(0.1) - continue - print(line) - - -# Todo - Method to check if proxy is running -# Todo diff --git a/packages/syft/src/syft/external/oblv/exceptions.py b/packages/syft/src/syft/external/oblv/exceptions.py deleted file mode 100644 index 05acc2e47cf..00000000000 --- a/packages/syft/src/syft/external/oblv/exceptions.py +++ /dev/null @@ -1,54 +0,0 @@ -class OblvProxyConnectPCRError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = "Failed to connect to enclave. Unauthorized deployment provided." - super().__init__(message) - - -class OblvEnclaveUnAuthorizedError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = "Domain unauthorized to perform this action in enclave" - super().__init__(message) - - -class OblvEnclaveError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = "Failed to connect to the enclave" - super().__init__(message) - - -class OblvError(Exception): - def __init__(self, message: str = "") -> None: - super().__init__(message) - - -class OblvUnAuthorizedError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = "User unauthorized to perform this action in enclave" - super().__init__(message) - - -class OblvKeyAlreadyExistsError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = "Currently each domain node could have only one oblv public/private key pair" - super().__init__(message) - - -class OblvLocalEnclaveError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = ( - "Failed to connect to locally deployed FastAPI based enclave services." - ) - super().__init__(message) - - -class OblvKeyNotFoundError(Exception): - def __init__(self, message: str = "") -> None: - if not message: - message = "Oblivious public key not found. Kindly request admin to create a new one" - super().__init__(message) diff --git a/packages/syft/src/syft/external/oblv/oblv_keys.py b/packages/syft/src/syft/external/oblv/oblv_keys.py deleted file mode 100644 index 040d41e1824..00000000000 --- a/packages/syft/src/syft/external/oblv/oblv_keys.py +++ /dev/null @@ -1,19 +0,0 @@ -# relative -from ...serde.serializable import serializable -from ...types.syft_object import SYFT_OBJECT_VERSION_2 -from ...types.syft_object import SyftObject - - -@serializable() -class OblvKeys(SyftObject): - # version - __canonical_name__ = "OblvKeys" - __version__ = SYFT_OBJECT_VERSION_2 - - # fields - public_key: bytes - private_key: bytes - - # serde / storage rules - __attr_searchable__ = ["private_key", "public_key"] - __attr_unique__ = ["private_key", "public_key"] diff --git a/packages/syft/src/syft/external/oblv/oblv_keys_stash.py b/packages/syft/src/syft/external/oblv/oblv_keys_stash.py deleted file mode 100644 index 8d4ba434418..00000000000 --- a/packages/syft/src/syft/external/oblv/oblv_keys_stash.py +++ /dev/null @@ -1,68 +0,0 @@ -# stdlib -from typing import Any - -# third party -from result import Err -from result import Ok -from result import Result - -# relative -from ...node.credentials import SyftVerifyKey -from ...serde.serializable import serializable -from ...service.response import SyftError -from ...store.document_store import BaseStash -from ...store.document_store import DocumentStore -from ...store.document_store import PartitionSettings -from ...store.document_store import QueryKeys -from ...store.document_store import UIDPartitionKey -from ...types.uid import UID -from .oblv_keys import OblvKeys - - -@serializable() -class OblvKeysStash(BaseStash): - object_type = OblvKeys - settings: PartitionSettings = PartitionSettings( - name=OblvKeys.__canonical_name__, object_type=OblvKeys, db_name="app" - ) - - def __init__(self, store: DocumentStore) -> None: - super().__init__(store=store) - - def check_type(self, obj: Any, type_: type) -> Result[Any, str]: - return ( - Ok(obj) - if isinstance(obj, type_) - else Err(f"{type(obj)} does not match required type: {type_}") - ) - - def set( - self, credentials: SyftVerifyKey, oblv_keys: OblvKeys - ) -> Result[OblvKeys, Err]: - if not len(self): - valid = self.check_type(oblv_keys, self.object_type) - if valid.is_err(): - return SyftError(message=valid.err()) - - return super().set(credentials, oblv_keys) - else: - return Err("Domain Node already has an existing public/private key pair") - - def get_by_uid( - self, credentials: SyftVerifyKey, uid: UID - ) -> Result[OblvKeys | None, str]: - qks = QueryKeys(qks=[UIDPartitionKey.with_obj(uid)]) - return Ok(self.query_one(credentials=credentials, qks=qks)) - - def delete_by_uid(self, credentials: SyftVerifyKey, uid: UID) -> Result[bool, str]: - qk = UIDPartitionKey.with_obj(uid) - return super().delete(qk=qk) - - def update( - self, credentials: SyftVerifyKey, task: OblvKeys - ) -> Result[OblvKeys, str]: - valid = self.check_type(task, self.object_type) - if valid.is_err(): - return SyftError(message=valid.err()) - - return super().update(credentials, task) diff --git a/packages/syft/src/syft/external/oblv/oblv_proxy.py b/packages/syft/src/syft/external/oblv/oblv_proxy.py deleted file mode 100644 index c8f6ff9d7b1..00000000000 --- a/packages/syft/src/syft/external/oblv/oblv_proxy.py +++ /dev/null @@ -1,214 +0,0 @@ -# stdlib -import base64 -import os -import platform -import subprocess # nosec -import sys -import tarfile -import zipfile - -# third party -import requests - -# relative -from ...util.util import bcolors - - -def check_oblv_proxy_installation_status(): - try: - result = subprocess.run(["oblv", "-V"], capture_output=True, text=True) # nosec - if result.stderr: - raise subprocess.CalledProcessError( # nosec - returncode=result.returncode, cmd=result.args, stderr=result.stderr - ) - result = result.stdout.strip() - return result - except Exception as e: - if e.__class__ == FileNotFoundError: - system_name = platform.system() - result = "Oblv Proxy Not Installed. Call the method install_oblv_proxy " - if system_name == "Windows": - result += ( - "to install the proxy for this session. If you already have the proxy installed," - " add it to your PATH." - ) - elif system_name == "Linux": - result += ( - "to install the proxy globally. If you already have the proxy installed," - " create a link to the installation as /usr/local/bin/oblv" - ) - - print( - bcolors.RED - + bcolors.BOLD - + "Exception" - + bcolors.BLACK - + bcolors.ENDC - + ": " - + result, - file=sys.stderr, - ) - else: - raise Exception(e) - - -def install_oblv_proxy(with_package: bool = False): - """Oblivious Proxy Installation - - Args: - with_package (bool, optional): Only available for .msi, .deb, .rpm. Defaults to False. - """ - system_name = platform.system() - if system_name == "Windows": - windows_proxy_installation(with_package) - elif system_name == "Linux": - linux_proxy_installation(with_package) - elif system_name == "Darwin": - darwin_proxy_installation() - - -def windows_proxy_installation(with_package: bool = False): - try: - if with_package: - url = "https://api.oblivious.ai/oblv-ccli/0.4.0/packages/oblv-0.4.0-x86_64.msi" - res = requests.get(url) # nosec - path = os.path.join(os.path.expanduser("~"), "oblv-0.4.0-x86_64.msi") - with open(path, "wb") as f: - f.write(res.content) - os.system(f"msiexec /I {path} /quiet /QB-!") # nosec - else: - url = "https://api.oblivious.ai/oblv-ccli/0.4.0/oblv-ccli-0.4.0-x86_64-pc-windows-msvc.zip" - res = requests.get(url) # nosec - path = ( - os.getcwd().replace("\\", "/") - + "/oblv-ccli-0.4.0-x86_64-pc-windows-msvc.zip" - ) - with open(path, "wb") as f: - f.write(res.content) - with zipfile.ZipFile(path, "r") as zipObj: # nosec - zipObj.extractall() # nosec - os.environ["PATH"] += ( - ";" + os.getcwd() + "\\oblv-ccli-0.4.0-x86_64-pc-windows-msvc;" - ) - except Exception as e: - print( - bcolors.RED - + bcolors.BOLD - + "Exception" - + bcolors.BLACK - + bcolors.ENDC - + ": " - + e.__cause__, - file=sys.stderr, - ) - - -def linux_proxy_installation(with_package: bool = False): - try: - if with_package: - try: - os.system("dpkg") # nosec - except Exception: - url = "https://api.oblivious.ai/oblv-ccli/0.4.0/packages/oblv-0.4.0-1.x86_64.rpm" - res = requests.get(url) # nosec - path = os.path.join(os.path.expanduser("~"), "oblv-0.4.0-1.x86_64.rpm") - with open(path, "wb") as f: - f.write(res.content) - os.system(f"rpm -i {path}") # nosec - else: - url = "https://api.oblivious.ai/oblv-ccli/0.4.0/packages/oblv_0.4.0_amd64.deb" - res = requests.get(url) # nosec - path = os.path.join(os.path.expanduser("~"), "oblv_0.4.0_amd64.deb") - with open(path, "wb") as f: - f.write(res.content) - os.system(f"dpkg -i {path}") # nosec - else: - url = "https://api.oblivious.ai/oblv-ccli/0.4.0/oblv-ccli-0.4.0-x86_64-unknown-linux-musl.tar.gz" - file_name = "oblv-ccli-0.4.0-x86_64-unknown-linux-musl.tar.gz" - res = requests.get(url, stream=True) # nosec - if res.status_code == 200: - with open(file_name, "wb") as f: - f.write(res.raw.read()) - path = os.getcwd() + "/oblv-ccli-0.4.0-x86_64-unknown-linux-musl" - file = tarfile.open(file_name) # nosec - file.extractall(path) # nosec - - os.symlink( - "/usr/local/bin/oblv", - os.getcwd() + "/oblv-ccli-0.4.0-x86_64-unknown-linux-musl/oblv", - ) - print( - bcolors.green(bcolors.bold("Successfully")) + " installed Oblivous CLI" - ) - except Exception as e: - print( - bcolors.RED - + bcolors.BOLD - + "Exception" - + bcolors.BLACK - + bcolors.ENDC - + ": " - + e.__cause__, - file=sys.stderr, - ) - - -def darwin_proxy_installation(): - url = "https://api.oblivious.ai/oblv-ccli/0.4.0/oblv-ccli-0.4.0-x86_64-apple-darwin.tar.gz" - file_name = "oblv-ccli-0.4.0-x86_64-apple-darwin.tar.gz" - res = requests.get(url, stream=True) # nosec - if res.status_code == 200: - with open(file_name, "wb") as f: - f.write(res.raw.read()) - path = os.getcwd() + "/oblv-ccli-0.4.0-x86_64-apple-darwin" - file = tarfile.open(file_name) - file.extractall(path) # nosec - - os.symlink( - "/usr/local/bin/oblv", os.getcwd() + "/oblv-ccli-0.4.0-x86_64-apple-darwin/oblv" - ) - print(bcolors.green(bcolors.bold("Successfully")) + " installed Oblivous CLI") - - -def create_oblv_key_pair(key_name): - if check_oblv_proxy_installation_status() is None: - return - try: - file_path = os.path.join(os.path.expanduser("~"), ".ssh", key_name) - result = subprocess.run( # nosec - ["oblv", "keygen", "--key-name", key_name, "--output", file_path], - capture_output=True, - ) - if result.stderr: - raise subprocess.CalledProcessError( # nosec - returncode=result.returncode, cmd=result.args, stderr=result.stderr - ) - result = result.stdout.strip() - return get_oblv_public_key(key_name) - except Exception as e: - raise Exception(e) - - -def get_oblv_public_key(key_name): - try: - filepath = os.path.join( - os.path.expanduser("~"), ".ssh", key_name, key_name + "_public.der" - ) - with open(filepath, "rb") as f: - public_key = f.read() - public_key = base64.encodebytes(public_key).decode("UTF-8").replace("\n", "") - return public_key - except FileNotFoundError: - print( - bcolors.RED - + bcolors.BOLD - + "Exception" - + bcolors.BLACK - + bcolors.ENDC - + ": " - + "No key found with given name", - file=sys.stderr, - ) - raise FileNotFoundError - except Exception as e: - raise Exception(e) diff --git a/packages/syft/src/syft/external/oblv/oblv_service.py b/packages/syft/src/syft/external/oblv/oblv_service.py deleted file mode 100644 index f72efb532df..00000000000 --- a/packages/syft/src/syft/external/oblv/oblv_service.py +++ /dev/null @@ -1,405 +0,0 @@ -# stdlib -from base64 import encodebytes -from collections.abc import Callable -import os -import random -import subprocess # nosec -from typing import Any -from typing import cast - -# third party -from oblv_ctl import OblvClient -import requests -from result import Err -from result import Ok -from result import Result - -# relative -from ...client.api import SyftAPI -from ...client.client import HTTPConnection -from ...client.client import Routes -from ...node.credentials import SyftSigningKey -from ...node.credentials import SyftVerifyKey -from ...serde.deserialize import _deserialize as deserialize -from ...serde.serializable import serializable -from ...service.action.action_object import ActionObject -from ...service.code.user_code import UserCodeStatus -from ...service.context import AuthedServiceContext -from ...service.response import SyftError -from ...service.service import AbstractService -from ...service.service import service_method -from ...service.user.user_roles import GUEST_ROLE_LEVEL -from ...store.document_store import DocumentStore -from ...types.uid import UID -from ...util.util import find_available_port -from .constants import LOCAL_MODE -from .constants import OBLV_LOCALHOST_PORT -from .deployment_client import OblvMetadata -from .exceptions import OblvEnclaveError -from .exceptions import OblvProxyConnectPCRError -from .oblv_keys import OblvKeys -from .oblv_keys_stash import OblvKeysStash - -# caches the connection to Enclave using the deployment ID -OBLV_PROCESS_CACHE: dict[str, list] = {} - - -def connect_to_enclave( - oblv_keys_stash: OblvKeysStash, - verify_key: SyftVerifyKey, - oblv_client: OblvClient, - deployment_id: str, - connection_port: int, - oblv_key_name: str, -) -> subprocess.Popen | None: - global OBLV_PROCESS_CACHE - if deployment_id in OBLV_PROCESS_CACHE: - process = OBLV_PROCESS_CACHE[deployment_id][0] - if process.poll() is None: - return process - # If the process has been terminated create a new connection - del OBLV_PROCESS_CACHE[deployment_id] - - # Always create key file each time, which ensures consistency when there is key change in database - create_keys_from_db( - oblv_keys_stash=oblv_keys_stash, - verify_key=verify_key, - oblv_key_name=oblv_key_name, - ) - oblv_key_path = os.path.expanduser(os.getenv("OBLV_KEY_PATH", "~/.oblv")) - - public_file_name = oblv_key_path + "/" + oblv_key_name + "_public.der" - private_file_name = oblv_key_path + "/" + oblv_key_name + "_private.der" - - depl = oblv_client.deployment_info(deployment_id) - if depl.is_deleted: - raise OblvEnclaveError( - "User cannot connect to this deployment, as it is no longer available." - ) - if depl.is_dev_env: - process = subprocess.Popen( # nosec - [ - "oblv", - "connect", - "--private-key", - private_file_name, - "--public-key", - public_file_name, - "--url", - depl.instance.service_url, - "--pcr0", - depl.pcr_codes[0], - "--pcr1", - depl.pcr_codes[1], - "--pcr2", - depl.pcr_codes[2], - "--port", - "443", - "--lport", - str(connection_port), - "--disable-pcr-check", - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - else: - process = subprocess.Popen( # nosec - [ - "oblv", - "connect", - "--private-key", - private_file_name, - "--public-key", - public_file_name, - "--url", - depl.instance.service_url, - "--pcr0", - depl.pcr_codes[0], - "--pcr1", - depl.pcr_codes[1], - "--pcr2", - depl.pcr_codes[2], - "--port", - "443", - "--lport", - str(connection_port), - ], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - while process.poll() is None: - log_line = process.stderr.readline().decode() - if log_line.__contains__("Error: Invalid PCR Values"): - raise OblvProxyConnectPCRError() - elif log_line.lower().__contains__("error"): - raise OblvEnclaveError(message=log_line) - elif log_line.__contains__("listening on"): - break - - OBLV_PROCESS_CACHE[deployment_id] = [process, connection_port] - return None - - -def make_request_to_enclave( - oblv_keys_stash: OblvKeysStash, - verify_key: SyftVerifyKey, - deployment_id: str, - oblv_client: OblvClient, - request_method: Callable, - connection_string: str, - connection_port: int, - oblv_key_name: str, - params: dict | None = None, - files: dict | None = None, - data: dict | None = None, - json: dict | None = None, -) -> Any: - if not LOCAL_MODE: - _ = connect_to_enclave( - oblv_keys_stash=oblv_keys_stash, - verify_key=verify_key, - oblv_client=oblv_client, - deployment_id=deployment_id, - connection_port=connection_port, - oblv_key_name=oblv_key_name, - ) - req = request_method( - connection_string, - params=params, - files=files, - data=data, - json=json, - ) - - return req - else: - headers = {"x-oblv-user-name": "enclave-test", "x-oblv-user-role": "domain"} - - return request_method( - connection_string, - headers=headers, - params=params, - files=files, - data=data, - json=json, - ) - - -def create_keys_from_db( - oblv_keys_stash: OblvKeysStash, verify_key: SyftVerifyKey, oblv_key_name: str -) -> None: - oblv_key_path = os.path.expanduser(os.getenv("OBLV_KEY_PATH", "~/.oblv")) - - os.makedirs(oblv_key_path, exist_ok=True) - # Temporary new key name for the new service - - keys = oblv_keys_stash.get_all(verify_key) - if keys.is_ok(): - keys = keys.ok()[0] - else: - return keys.err() - - f_private = open(oblv_key_path + "/" + oblv_key_name + "_private.der", "w+b") - f_private.write(keys.private_key) - f_private.close() - f_public = open(oblv_key_path + "/" + oblv_key_name + "_public.der", "w+b") - f_public.write(keys.public_key) - f_public.close() - - -def generate_oblv_key(oblv_key_name: str) -> tuple[bytes, bytes]: - oblv_key_path = os.path.expanduser(os.getenv("OBLV_KEY_PATH", "~/.oblv")) - os.makedirs(oblv_key_path, exist_ok=True) - - result = subprocess.run( # nosec - [ - "oblv", - "keygen", - "--key-name", - oblv_key_name, - "--output", - oblv_key_path, - ], - capture_output=True, - ) - - if result.stderr: - raise Err( - subprocess.CalledProcessError( # nosec - returncode=result.returncode, cmd=result.args, stderr=result.stderr - ) - ) - f_private = open(oblv_key_path + "/" + oblv_key_name + "_private.der", "rb") - private_key = f_private.read() - f_private.close() - f_public = open(oblv_key_path + "/" + oblv_key_name + "_public.der", "rb") - public_key = f_public.read() - f_public.close() - - return (public_key, private_key) - - -@serializable() -class OblvService(AbstractService): - store: DocumentStore - oblv_keys_stash: OblvKeysStash - - def __init__(self, store: DocumentStore) -> None: - self.store = store - self.oblv_keys_stash = OblvKeysStash(store=store) - - @service_method(path="oblv.create_key", name="create_key", roles=GUEST_ROLE_LEVEL) - def create_key( - self, - context: AuthedServiceContext, - oblv_key_name: str, - override_existing_key: bool = False, - ) -> Result[Ok, Err]: - """Domain Public/Private Key pair creation""" - # TODO 🟣 Check for permission after it is fully integrated - public_key, private_key = generate_oblv_key(oblv_key_name) - - if override_existing_key: - self.oblv_keys_stash.clear() - oblv_keys = OblvKeys(public_key=public_key, private_key=private_key) - - res = self.oblv_keys_stash.set(context.credentials, oblv_keys) - - if res.is_ok(): - return Ok( - "Successfully created a new public/private RSA key-pair on the domain node" - ) - return res.err() - - @service_method( - path="oblv.get_public_key", name="get_public_key", roles=GUEST_ROLE_LEVEL - ) - def get_public_key( - self, - context: AuthedServiceContext, - ) -> Result[Ok, Err]: - "Retrieves the public key present on the Domain Node." - - if len(self.oblv_keys_stash): - # retrieve the public key from the stash using the node's verify key - # as the public should be accessible to all the users - oblv_keys = self.oblv_keys_stash.get_all(context.node.verify_key) - if oblv_keys.is_ok(): - oblv_keys = oblv_keys.ok()[0] - else: - return oblv_keys.err() - - public_key_str = ( - encodebytes(oblv_keys.public_key).decode("UTF-8").replace("\n", "") - ) - return Ok(public_key_str) - - return Err( - "Public Key not present for the domain node, Kindly request the admin to create a new one" - ) - - def get_api_for( - self, - enclave_metadata: OblvMetadata, - signing_key: SyftSigningKey, - worker_name: str, - ) -> SyftAPI: - deployment_id = enclave_metadata.deployment_id - oblv_client = enclave_metadata.oblv_client - if not LOCAL_MODE: - if ( - deployment_id in OBLV_PROCESS_CACHE - and OBLV_PROCESS_CACHE[deployment_id][0].poll() is None - ): - port = OBLV_PROCESS_CACHE[deployment_id][1] - else: - # randomized port staring point, to quickly find free port - port_start = 3000 + random.randint(1, 10_000) # nosec - port = find_available_port( - host="127.0.0.1", port=port_start, search=True - ) - connection_string = f"http://127.0.0.1:{port}" - else: - port = os.getenv("OBLV_LOCALHOST_PORT", OBLV_LOCALHOST_PORT) - connection_string = f"http://127.0.0.1:{port}" - - # To identify if we are in docker container - if "CONTAINER_HOST" in os.environ: - connection_string = connection_string.replace( - "127.0.0.1", "host.docker.internal" - ) - - params = {"verify_key": str(signing_key.verify_key)} - req = make_request_to_enclave( - connection_string=connection_string + Routes.ROUTE_API.value, - deployment_id=deployment_id, - oblv_client=oblv_client, - oblv_keys_stash=self.oblv_keys_stash, - verify_key=signing_key.verify_key, - request_method=requests.get, - connection_port=port, - oblv_key_name=worker_name, - params=params, - ) - - obj = deserialize(req.content, from_bytes=True) - # TODO 🟣 Retrieve of signing key of user after permission is fully integrated - obj.signing_key = signing_key - obj.connection = HTTPConnection(url=connection_string) - return cast(SyftAPI, obj) - - @service_method( - path="oblv.send_user_code_inputs_to_enclave", - name="send_user_code_inputs_to_enclave", - roles=GUEST_ROLE_LEVEL, - ) - def send_user_code_inputs_to_enclave( - self, - context: AuthedServiceContext, - user_code_id: UID, - inputs: dict, - node_name: str, - ) -> Result[Ok, Err]: - if not context.node or not context.node.signing_key: - return Err(f"{type(context)} has no node") - - user_code_service = context.node.get_service("usercodeservice") - action_service = context.node.get_service("actionservice") - user_code = user_code_service.stash.get_by_uid( - context.node.signing_key.verify_key, uid=user_code_id - ) - if user_code.is_err(): - return SyftError( - message=f"Unable to find {user_code_id} in {type(user_code_service)}" - ) - user_code = user_code.ok() - reason: str = context.extra_kwargs.get("reason", "") - res = user_code.status.mutate( - value=(UserCodeStatus.APPROVED, reason), - node_name=node_name, - verify_key=context.credentials, - ) - if res.is_err(): - return res - user_code.status = res.ok() - user_code_service.update_code_state(context=context, code_item=user_code) - - root_context = context.as_root_context() - - if not action_service.exists(context=context, obj_id=user_code_id): - dict_object = ActionObject.from_obj({}) - dict_object.id = user_code_id - dict_object[str(context.credentials)] = inputs - root_context.extra_kwargs = {"has_result_read_permission": True} - action_service.set(root_context, dict_object) - - else: - res = action_service.get(uid=user_code_id, context=context) - if res.is_ok(): - dict_object = res.ok() - dict_object[str(context.credentials)] = inputs - action_service.set(root_context, dict_object) - else: - return res - - return Ok(Ok(True)) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index 2dfc70c163f..79f803757a6 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -34,7 +34,6 @@ from ..client.api import debox_signed_syftapicall_response from ..client.client import SyftClient from ..exceptions.exception import PySyftException -from ..external import OblvServiceProvider from ..protocol.data_protocol import PROTOCOL_TYPE from ..protocol.data_protocol import get_data_protocol from ..service.action.action_object import Action @@ -875,7 +874,6 @@ def _construct_services(self) -> None: {"svc": OutputService}, {"svc": UserCodeStatusService}, {"svc": VeilidServiceProvider}, # this is lazy - {"svc": OblvServiceProvider}, # this is lazy ] for svc_kwargs in default_services: @@ -1501,33 +1499,6 @@ def create_admin_new( return None -# def create_oblv_key_pair( -# worker: Node, -# ) -> str | None: -# try: -# # relative -# from ..external.oblv.oblv_keys_stash import OblvKeys -# from ..external.oblv.oblv_keys_stash import OblvKeysStash -# from ..external.oblv.oblv_service import generate_oblv_key - -# oblv_keys_stash = OblvKeysStash(store=worker.document_store) - -# if not len(oblv_keys_stash) and worker.signing_key: -# public_key, private_key = generate_oblv_key(oblv_key_name=worker.name) -# oblv_keys = OblvKeys(public_key=public_key, private_key=private_key) -# res = oblv_keys_stash.set(worker.signing_key.verify_key, oblv_keys) -# if res.is_ok(): -# print("Successfully generated Oblv Key pair at startup") -# return res.err() -# else: -# print(f"Using Existing Public/Private Key pair: {len(oblv_keys_stash)}") -# except Exception as e: -# print("Unable to create Oblv Keys.", e) -# return None - -# return None - - class NodeRegistry: __node_registry__: dict[UID, Node] = {} diff --git a/packages/syft/src/syft/service/enclave/enclave_service.py b/packages/syft/src/syft/service/enclave/enclave_service.py index 052b81efa32..61621ba33c5 100644 --- a/packages/syft/src/syft/service/enclave/enclave_service.py +++ b/packages/syft/src/syft/service/enclave/enclave_service.py @@ -94,44 +94,10 @@ def send_user_code_inputs_to_enclave( return SyftSuccess(message="Enclave Code Status Updated Successfully") -def get_oblv_service() -> type[AbstractService] | SyftError: - # relative - from ...external import OBLV_ENABLED - - if OBLV_ENABLED: - # relative - from ...external.oblv.oblv_service import OblvService - - return OblvService - else: - return SyftError( - message="Oblivious is not enabled." - "To enable oblivious package, set sy.enable_external_lib('oblv') " - "on the client side" - "Or add --oblv when launching by hagrid" - ) - - -# Checks if the given user code would propogate value to enclave on acceptance +# Checks if the given user code would propogate value to enclave on acceptance def propagate_inputs_to_enclave( user_code: UserCode, context: ChangeContext ) -> SyftSuccess | SyftError: - # Temporarily disable Oblivious Enclave - # from ...external.oblv.deployment_client import OblvMetadata - - # if isinstance(user_code.enclave_metadata, OblvMetadata): - # # relative - # oblv_service_class = get_oblv_service() - # if isinstance(oblv_service_class, SyftError): - # return oblv_service_class - # method = context.node.get_service_method(oblv_service_class.get_api_for) - - # api = method( - # user_code.enclave_metadata, - # context.node.signing_key, - # worker_name=context.node.name, - # ) - # send_method = api.services.oblv.send_user_code_inputs_to_enclave if context.node is None: return SyftError(message=f"context {context}'s node is None") diff --git a/tests/integration/external/oblv/manual_code_submission_test.py b/tests/integration/external/oblv/manual_code_submission_test.py deleted file mode 100644 index fc1827df9cf..00000000000 --- a/tests/integration/external/oblv/manual_code_submission_test.py +++ /dev/null @@ -1,113 +0,0 @@ -# stdlib -import os -from textwrap import dedent - -# third party -import numpy as np - -# syft absolute -import syft as sy -from syft.service.action.numpy import NumpyArrayObject -from syft.service.code.user_code import SubmitUserCode - -LOCAL_ENCLAVE_PORT = os.environ.get("LOCAL_ENCLAVE_PORT", 8010) -# TODO: Should move to Docker Container tests - - -def load_dataset(domain_client) -> None: - dataset_name = f"{domain_client.name}'s... Private Data" - asset_name = "Secret data" - dataset = sy.Dataset(name=dataset_name) - asset = sy.Asset(name=asset_name) - - # Real Data - x = np.array([1, 2, 3]) - asset.set_obj(x) - - # Mock Data - y = np.array([1, 1, 1]) - asset.set_mock(y, mock_is_real=False) - - dataset.add_asset(asset) - - domain_client.upload_dataset(dataset) - - datasets = domain_client.datasets.get_all() - - assert len(datasets) == 1 - domain_dataset = datasets[0] - assert domain_dataset.name == dataset_name - assert len(domain_dataset.assets) == 1 - assert domain_dataset.assets[0].name == asset_name - - -def test_manual_code_submission_enclave() -> None: - # Step1: Login Phase - canada_root = sy.Worker.named(name="canada", local_db=True, reset=True).root_client - - italy_root = sy.Worker.named(name="italy", local_db=True, reset=True).root_client - - # Step 2: Uploading to Domain Nodes - load_dataset(canada_root) - load_dataset(italy_root) - - assert sy.enable_external_lib("oblv") - - # Step 3: Connection to Enclave - # TODO 🟣 Modify to use Data scientist account credentials - # after Permission are integrated - depl = sy.external.oblv.deployment_client.DeploymentClient( - deployment_id="d-2dfedbb1-7904-493b-8793-1a9554badae7", - oblv_client=None, - domain_clients=[canada_root, italy_root], - key_name="first", - ) # connection_port key can be added to set the port on which oblv_proxy will run - - depl.initiate_connection(LOCAL_ENCLAVE_PORT) - - depl.register( - name="Jane Doe", - email="jane@caltech.edu", - password="abc123", - institution="Caltech", - website="https://www.caltech.edu/", - ) - depl.login(email="jane@caltech.edu", password="abc123") - - # Step 4: Manual code preparation Phase - canada_data = canada_root.datasets[-1] - italy_data = italy_root.datasets[-1] - - @sy.syft_function( - input_policy=sy.ExactMatch( - canada_data=canada_data.assets[0], italy_data=italy_data.assets[0] - ), - output_policy=sy.SingleExecutionExactOutput(), - ) - def simple_function(canada_data, italy_data): - return canada_data + italy_data - - simple_function.code = dedent(simple_function.code) - assert isinstance(simple_function, SubmitUserCode) - - # Step 5 :Code Submission Phase - print(depl.request_code_execution(code=simple_function)) - - # Step 6: Code review phase - canada_requests = canada_root.api.services.request.get_all() - assert len(canada_requests) == 1 - assert canada_requests[0].approve() - - italy_requests = italy_root.api.services.request.get_all() - assert len(italy_requests) == 1 - assert italy_requests[0].approve() - - assert hasattr(depl.api.services.code, "simple_function") - res = depl.api.services.code.simple_function( - canada_data=canada_data.assets[0], italy_data=italy_data.assets[0] - ) - print(res, type(res)) - assert isinstance(res, NumpyArrayObject) - - canada_root.cleanup() - italy_root.cleanup() diff --git a/tox.ini b/tox.ini index 0e76c35476c..485a374a302 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,6 @@ envlist = syft.test.unit syft.test.notebook stack.test.notebook - stack.test.integration.enclave.oblv stack.test.integration.k8s stack.test.vm stack.test.podman @@ -412,36 +411,6 @@ commands = bash -c 'ulimit -n 4096 || true' pytest -n auto --dist loadgroup --durations=20 --disable-warnings -[testenv:stack.test.integration.enclave.oblv] -description = Integration Tests for Oblv Enclave -changedir = {toxinidir} -deps = - {[testenv:syft]deps} - {[testenv:hagrid]deps} - oblv-ctl==0.3.1 -allowlist_externals = - grep - bash -passenv=HOME, USER -setenv = - LOCAL_ENCLAVE_PORT=8010 - OBLV_ENABLED=true - OBLV_LOCALHOST_PORT=8010 - ENABLE_SIGNUP=True -commands = - # run at start to kill any process started beforehand - bash -c 'chmod +x scripts/kill_process_in_port.sh && ./scripts/kill_process_in_port.sh $LOCAL_ENCLAVE_PORT' - - bash -c 'rm -rf ~/.syft/syft-enclave' - bash -c 'git clone https://github.com/OpenMined/syft-enclave.git ~/.syft/syft-enclave || true' - bash -c 'cd ~/.syft/syft-enclave && git fetch && git checkout dev && git pull && uv pip install -r requirements_test.txt || true' - - # Starting FastAPI server locally - bash -c 'cd ~/.syft/syft-enclave/src && uvicorn app:app --host 0.0.0.0 --port $LOCAL_ENCLAVE_PORT > /dev/null 2>&1 &' - - bash -c 'cd tests/integration/external/oblv && pytest -p no:randomly -vvvv' - bash -c 'chmod +x scripts/kill_process_in_port.sh && ./scripts/kill_process_in_port.sh $LOCAL_ENCLAVE_PORT' - [testenv:syft.test.notebook] description = Syft Notebook Tests deps =