From bba87b16cd31e751ae722e5353bd8ef02f8bf33d Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 6 Feb 2024 15:58:26 +0800 Subject: [PATCH 01/59] Update to pydantic v2 first pass with bump-pydantic --- packages/syft/setup.cfg | 2 +- packages/syft/src/syft/client/client.py | 13 +++-- .../syft/src/syft/custom_worker/config.py | 8 +-- .../syft/external/oblv/deployment_client.py | 8 +-- packages/syft/src/syft/node/credentials.py | 7 +-- .../src/syft/service/action/action_graph.py | 20 +++----- .../src/syft/service/action/action_object.py | 50 +++++++------------ .../syft/src/syft/service/dataset/dataset.py | 47 ++++++++--------- .../syft/src/syft/service/job/job_stash.py | 33 +++++------- .../syft/service/metadata/node_metadata.py | 5 +- .../syft/src/syft/service/project/project.py | 29 +++++------ .../syft/src/syft/service/queue/zmq_queue.py | 9 ++-- packages/syft/src/syft/service/user/user.py | 15 ++---- .../src/syft/service/worker/image_registry.py | 9 +++- .../src/syft/store/sqlite_document_store.py | 5 +- packages/syft/src/syft/types/base.py | 4 +- packages/syft/src/syft/types/syft_object.py | 17 +++---- packages/syft/src/syft/types/twin_object.py | 34 +++++++------ 18 files changed, 151 insertions(+), 164 deletions(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index e88ce585f21..73b19e3096b 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -35,7 +35,7 @@ syft = packaging>=23.0 pyarrow==14.0.1 pycapnp==1.3.0 - pydantic[email]==1.10.13 + pydantic[email]==2.6.0 pymongo==4.6.1 pynacl==1.5.0 pyzmq>=23.2.1,<=25.1.1 diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index 11b4b2ab9a0..f9091e4d9d6 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -20,7 +20,7 @@ # third party from argon2 import PasswordHasher -import pydantic +from pydantic import field_validator import requests from requests import Response from requests import Session @@ -141,9 +141,14 @@ class HTTPConnection(NodeConnection): routes: Type[Routes] = Routes session_cache: Optional[Session] - @pydantic.validator("url", pre=True, always=True) - def make_url(cls, v: Union[GridURL, str]) -> GridURL: - return GridURL.from_url(v).as_container_host() + @field_validator("url", mode="before") + @classmethod + def make_url(cls, v: Any) -> Any: + return ( + GridURL.from_url(v).as_container_host() + if isinstance(v, (str, GridURL)) + else v + ) def with_proxy(self, proxy_target_uid: UID) -> Self: return HTTPConnection(url=self.url, proxy_target_uid=proxy_target_uid) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index c54d4f77c40..f60c19514d4 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -12,7 +12,7 @@ # third party import docker from packaging import version -from pydantic import validator +from pydantic import field_validator from typing_extensions import Self import yaml @@ -54,7 +54,8 @@ class CustomBuildConfig(SyftBaseModel): # f"Python version must be between {PYTHON_MIN_VER} and {PYTHON_MAX_VER}" # ) - @validator("python_packages") + @field_validator("python_packages") + @classmethod def validate_python_packages(cls, pkgs: List[str]) -> List[str]: for pkg in pkgs: ver_parts = () @@ -132,7 +133,8 @@ class DockerWorkerConfig(WorkerConfig): file_name: Optional[str] description: Optional[str] - @validator("dockerfile") + @field_validator("dockerfile") + @classmethod def validate_dockerfile(cls, dockerfile: str) -> str: if not dockerfile: raise ValueError("Dockerfile cannot be empty") diff --git a/packages/syft/src/syft/external/oblv/deployment_client.py b/packages/syft/src/syft/external/oblv/deployment_client.py index 6b4c1f6d304..172baee1cff 100644 --- a/packages/syft/src/syft/external/oblv/deployment_client.py +++ b/packages/syft/src/syft/external/oblv/deployment_client.py @@ -17,7 +17,7 @@ # third party from oblv_ctl import OblvClient -from pydantic import validator +from pydantic import field_validator import requests # relative @@ -46,7 +46,8 @@ class OblvMetadata(EnclaveMetadata): deployment_id: Optional[str] oblv_client: Optional[OblvClient] - @validator("deployment_id") + @field_validator("deployment_id") + @classmethod def check_valid_deployment_id(cls, deployment_id: str) -> str: if not deployment_id and not LOCAL_MODE: raise ValueError( @@ -56,7 +57,8 @@ def check_valid_deployment_id(cls, deployment_id: str) -> str: ) return deployment_id - @validator("oblv_client") + @field_validator("oblv_client") + @classmethod def check_valid_oblv_client(cls, oblv_client: OblvClient) -> OblvClient: if not oblv_client and not LOCAL_MODE: raise ValueError( diff --git a/packages/syft/src/syft/node/credentials.py b/packages/syft/src/syft/node/credentials.py index c60c57e1db2..d774f0f4c91 100644 --- a/packages/syft/src/syft/node/credentials.py +++ b/packages/syft/src/syft/node/credentials.py @@ -9,7 +9,7 @@ from nacl.encoding import HexEncoder from nacl.signing import SigningKey from nacl.signing import VerifyKey -import pydantic +from pydantic import field_validator # relative from ..serde.serializable import serializable @@ -54,8 +54,9 @@ def __hash__(self) -> int: class SyftSigningKey(SyftBaseModel): signing_key: SigningKey - @pydantic.validator("signing_key", pre=True, always=True) - def make_signing_key(cls, v: Union[str, SigningKey]) -> SigningKey: + @field_validator("signing_key", mode="before") + @classmethod + def make_signing_key(cls, v: Any) -> Any: return SigningKey(bytes.fromhex(v)) if isinstance(v, str) else v @property diff --git a/packages/syft/src/syft/service/action/action_graph.py b/packages/syft/src/syft/service/action/action_graph.py index 8456f6852a1..a64220a1980 100644 --- a/packages/syft/src/syft/service/action/action_graph.py +++ b/packages/syft/src/syft/service/action/action_graph.py @@ -15,9 +15,8 @@ # third party import matplotlib.pyplot as plt import networkx as nx -import pydantic from pydantic import Field -from pydantic import validator +from pydantic import field_validator from result import Err from result import Ok from result import Result @@ -66,18 +65,14 @@ class NodeActionData(SyftObject): type: NodeType status: ExecutionStatus = ExecutionStatus.PROCESSING retry: int = 0 - created_at: Optional[DateTime] - updated_at: Optional[DateTime] + created_at: DateTime = Field(default_factory=DateTime.now) + updated_at: DateTime = Field(default_factory=DateTime.now) user_verify_key: SyftVerifyKey is_mutated: bool = False # denotes that this node has been mutated is_mutagen: bool = False # denotes that this node is causing a mutation next_mutagen_node: Optional[UID] # next neighboring mutagen node last_nm_mutagen_node: Optional[UID] # last non mutated mutagen node - @pydantic.validator("created_at", pre=True, always=True) - def make_created_at(cls, v: Optional[DateTime]) -> DateTime: - return DateTime.now() if v is None else v - @classmethod def from_action(cls, action: Action, credentials: SyftVerifyKey) -> Self: is_mutagen = action.remote_self is not None and ( @@ -124,17 +119,13 @@ class NodeActionDataUpdate(PartialSyftObject): status: ExecutionStatus retry: int created_at: DateTime - updated_at: Optional[DateTime] + updated_at: DateTime = Field(default_factory=DateTime.now) credentials: SyftVerifyKey is_mutated: bool is_mutagen: bool next_mutagen_node: UID # next neighboring mutagen node last_nm_mutagen_node: UID # last non mutated mutagen node - @pydantic.validator("updated_at", pre=True, always=True) - def set_updated_at(cls, v: Optional[DateTime]) -> DateTime: - return DateTime.now() if v is None else v - @serializable() class BaseGraphStore: @@ -197,7 +188,8 @@ class InMemoryStoreClientConfig(StoreClientConfig): # We need this in addition to Field(default_factory=...) # so users can still do InMemoryStoreClientConfig(path=None) - @validator("path", pre=True) + @field_validator("path", mode="before") + @classmethod def __default_path(cls, path: Optional[Union[str, Path]]) -> Union[str, Path]: if path is None: return tempfile.gettempdir() diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index bbff783a49a..1a8c1247ad3 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -20,7 +20,10 @@ from typing import Union # third party -import pydantic +from pydantic import ConfigDict +from pydantic import Field +from pydantic import field_validator +from pydantic import model_validator from result import Err from result import Ok from result import Result @@ -144,20 +147,20 @@ class Action(SyftObject): remote_self: Optional[LineageID] args: List[LineageID] kwargs: Dict[str, LineageID] - result_id: Optional[LineageID] + result_id: LineageID = Field(default_factory=lambda: LineageID(UID())) action_type: Optional[ActionType] create_object: Optional[SyftObject] = None user_code_id: Optional[UID] = None - @pydantic.validator("id", pre=True, always=True) - def make_id(cls, v: Optional[UID]) -> UID: - """Generate or reuse an UID""" - return v if isinstance(v, UID) else UID() - - @pydantic.validator("result_id", pre=True, always=True) - def make_result_id(cls, v: Optional[Union[UID, LineageID]]) -> UID: + @field_validator("result_id", mode="before") + @classmethod + def make_result_id(cls, v: Any) -> LineageID: """Generate or reuse a LineageID""" - return v if isinstance(v, LineageID) else LineageID(v) + if isinstance(v, LineageID): + return v + if not isinstance(v, UID): + raise ValueError("result_id type must be a subclass of UID") + return LineageID(v) @property def full_path(self) -> str: @@ -612,6 +615,8 @@ class ActionObject(SyftObject): __version__ = SYFT_OBJECT_VERSION_2 __attr_searchable__: List[str] = [] + + id: UID syft_action_data_cache: Optional[Any] = None syft_blob_storage_entry_id: Optional[UID] = None syft_pointer_type: ClassVar[Type[ActionObjectPointer]] @@ -760,15 +765,10 @@ def syft_lineage_id(self) -> LineageID: """Compute the LineageID of the ActionObject, using the `id` and the `syft_history_hash` memebers""" return LineageID(self.id, self.syft_history_hash) - @pydantic.validator("id", pre=True, always=True) - def make_id(cls, v: Optional[UID]) -> UID: - """Generate or reuse an UID""" - return Action.make_id(v) + model_config = ConfigDict(validate_assignment=True) - class Config: - validate_assignment = True - - @pydantic.root_validator() + @model_validator(mode="before") + @classmethod def __check_action_data(cls, values: dict) -> dict: v = values.get("syft_action_data_cache") if values.get("syft_action_data_type", None) is None: @@ -798,18 +798,6 @@ def is_real(self) -> bool: def is_twin(self) -> bool: return self.syft_twin_type != TwinMode.NONE - # @pydantic.validator("syft_action_data", pre=True, always=True) - # def check_action_data( - # cls, v: ActionObject.syft_pointer_type - # ) -> ActionObject.syft_pointer_type: - # if cls == AnyActionObject or isinstance( - # v, (cls.syft_internal_type, ActionDataEmpty) - # ): - # return v - # raise SyftException( - # f"Must init {cls} with {cls.syft_internal_type} not {type(v)}" - # ) - def syft_point_to(self, node_uid: UID) -> ActionObject: """Set the syft_node_uid, used in the post hooks""" self.syft_node_uid = node_uid @@ -941,7 +929,7 @@ def _syft_prepare_obj_uid(self, obj: Any) -> LineageID: return obj.syft_lineage_id # We got a raw object. We need to create the ActionObject from scratch and save it in the store. - obj_id = Action.make_id(None) + obj_id = UID() lin_obj_id = Action.make_result_id(obj_id) act_obj = ActionObject.from_obj( obj, diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index a624956172b..b5972fd8671 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -15,12 +15,14 @@ from IPython.display import display import itables import pandas as pd +from pydantic import ConfigDict from pydantic import ValidationError -from pydantic import root_validator -from pydantic import validator +from pydantic import field_validator +from pydantic import model_validator from result import Err from result import Ok from result import Result +from typing_extensions import Self # relative from ...serde.serializable import serializable @@ -334,30 +336,29 @@ class CreateAsset(SyftObject): uploader: Optional[Contributor] __repr_attrs__ = ["name"] - - class Config: - validate_assignment = True + model_config = ConfigDict(validate_assignment=True) def __init__(self, description: Optional[str] = "", **data: Any) -> None: super().__init__(**data, description=MarkdownDescription(text=str(description))) - @root_validator() - def __empty_mock_cannot_be_real(cls, values: dict[str, Any]) -> Dict: + @model_validator(mode="after") + def __empty_mock_cannot_be_real(self) -> Self: """set mock_is_real to False whenever mock is None or empty""" - - if (mock := values.get("mock")) is None or _is_action_data_empty(mock): - values["mock_is_real"] = False - - return values - - @validator("mock_is_real") - def __mock_is_real_for_empty_mock_must_be_false( - cls, v: bool, values: dict[str, Any], **kwargs: Any - ) -> bool: - if v and ((mock := values.get("mock")) is None or _is_action_data_empty(mock)): + if self.mock is None or _is_action_data_empty(self.mock): + self.mock_is_real = False + + return self + + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. + @model_validator(mode="after") + def __mock_is_real_for_empty_mock_must_be_false(self) -> Self: + if self.mock_is_real and ( + self.mock is None or _is_action_data_empty(self.mock) + ): raise ValueError("mock_is_real must be False if mock is not provided") - return v + return self def add_data_subject(self, data_subject: DataSubject) -> None: self.data_subjects.append(data_subject) @@ -638,15 +639,15 @@ class CreateDataset(Dataset): id: Optional[UID] = None created_at: Optional[DateTime] - uploader: Optional[Contributor] # type: ignore[assignment] + uploader: Optional[Contributor] # type: ignore[assignment] - class Config: - validate_assignment = True + model_config = ConfigDict(validate_assignment=True) def _check_asset_must_contain_mock(self) -> None: _check_asset_must_contain_mock(self.asset_list) - @validator("asset_list") + @field_validator("asset_list") + @classmethod def __assets_must_contain_mock( cls, asset_list: List[CreateAsset] ) -> List[CreateAsset]: diff --git a/packages/syft/src/syft/service/job/job_stash.py b/packages/syft/src/syft/service/job/job_stash.py index 7fa85144ec7..9eb5c6561bb 100644 --- a/packages/syft/src/syft/service/job/job_stash.py +++ b/packages/syft/src/syft/service/job/job_stash.py @@ -10,7 +10,8 @@ from typing import Union # third party -import pydantic +from pydantic import Field +from pydantic import model_validator from result import Err from result import Ok from result import Result @@ -113,7 +114,7 @@ class Job(SyftObject): parent_job_id: Optional[UID] n_iters: Optional[int] = 0 current_iter: Optional[int] = None - creation_time: Optional[str] = None + creation_time: str = Field(default_factory=lambda: str(datetime.now())) action: Optional[Action] = None job_pid: Optional[int] = None job_worker_id: Optional[UID] = None @@ -123,26 +124,18 @@ class Job(SyftObject): __attr_searchable__ = ["parent_job_id", "job_worker_id", "status", "user_code_id"] __repr_attrs__ = ["id", "result", "resolved", "progress", "creation_time"] - @pydantic.root_validator() - def check_time(cls, values: dict) -> dict: - if values.get("creation_time", None) is None: - values["creation_time"] = str(datetime.now()) - return values - - @pydantic.root_validator() - def check_user_code_id(cls, values: dict) -> dict: - action = values.get("action") - user_code_id = values.get("user_code_id") - - if action is not None: - if user_code_id is None: - values["user_code_id"] = action.user_code_id - elif action.user_code_id != user_code_id: - raise pydantic.ValidationError( - "user_code_id does not match the action's user_code_id", cls + @model_validator(mode="after") + def check_user_code_id(self) -> Self: + if self.action is not None: + if self.user_code_id is None: + self.user_code_id = self.action.user_code_id + elif self.action.user_code_id != self.user_code_id: + raise ValueError( + "user_code_id does not match the action's user_code_id", + self.__class__, ) - return values + return self @property def action_display_name(self) -> str: diff --git a/packages/syft/src/syft/service/metadata/node_metadata.py b/packages/syft/src/syft/service/metadata/node_metadata.py index 13eb097b0ac..e9bb844ad41 100644 --- a/packages/syft/src/syft/service/metadata/node_metadata.py +++ b/packages/syft/src/syft/service/metadata/node_metadata.py @@ -9,7 +9,7 @@ # third party from packaging import version from pydantic import BaseModel -from pydantic import root_validator +from pydantic import model_validator # relative from ...abstract_node import NodeType @@ -164,7 +164,8 @@ class NodeMetadataJSON(BaseModel, StorableObjectType): show_warnings: bool supported_protocols: List = [] - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def add_protocol_versions(cls, values: dict) -> dict: if "supported_protocols" not in values: data_protocol = get_data_protocol() diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 4f55bc444f5..feaecf53a40 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -18,8 +18,8 @@ from typing import Union # third party -import pydantic -from pydantic import validator +from pydantic import Field +from pydantic import field_validator from rich.progress import Progress from typing_extensions import Self @@ -78,7 +78,7 @@ class ProjectEvent(SyftObject): # 1. Creation attrs id: UID - timestamp: DateTime + timestamp: DateTime = Field(default_factory=DateTime.now) allowed_sub_types: Optional[List] = [] # 2. Rebase attrs project_id: Optional[UID] @@ -96,12 +96,6 @@ def __repr_syft_nested__(self) -> tuple[str, str]: f"{str(self.id)[:4]}...{str(self.id)[-3:]}", ) - @pydantic.root_validator(pre=True) - def make_timestamp(cls, values: Dict[str, Any]) -> Dict[str, Any]: - if "timestamp" not in values or values["timestamp"] is None: - values["timestamp"] = DateTime.now() - return values - def _pre_add_update(self, project: Project) -> None: pass @@ -277,8 +271,9 @@ class ProjectRequest(ProjectEventAddObject): linked_request: LinkedObject allowed_sub_types: List[Type] = [ProjectRequestResponse] - @validator("linked_request", pre=True) - def _validate_linked_request(cls, v: Any) -> Union[Request, LinkedObject]: + @field_validator("linked_request", mode="before") + @classmethod + def _validate_linked_request(cls, v: Any) -> LinkedObject: if isinstance(v, Request): linked_request = LinkedObject.from_obj(v, node_uid=v.node_uid) return linked_request @@ -557,8 +552,9 @@ class ProjectMultipleChoicePoll(ProjectEventAddObject): choices: List[str] allowed_sub_types: List[Type] = [AnswerProjectPoll] - @validator("choices") - def choices_min_length(cls, v: str) -> str: + @field_validator("choices") + @classmethod + def choices_min_length(cls, v: list[str]) -> list[str]: if len(v) < 1: raise ValueError("choices must have at least one item") return v @@ -1228,10 +1224,9 @@ def _repr_html_(self) -> Any: + "" ) - @validator("members", pre=True) - def verify_members( - cls, val: Union[List[SyftClient], List[NodeIdentity]] - ) -> Union[List[SyftClient], List[NodeIdentity]]: + @field_validator("members", mode="before") + @classmethod + def verify_members(cls, val: Union[List[SyftClient], List[NodeIdentity]]) -> Union[List[SyftClient], List[NodeIdentity]]: # SyftClients must be logged in by the same emails clients = cls.get_syft_clients(val) if len(clients) > 0: diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 215dcdfe5c2..1136f34bdc0 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -17,7 +17,7 @@ # third party from loguru import logger -from pydantic import validator +from pydantic import field_validator from zmq import Frame from zmq import LINGER from zmq.error import ContextTerminated @@ -120,8 +120,11 @@ class Worker(SyftBaseModel): syft_worker_id: Optional[UID] = None expiry_t: Timeout = Timeout(WORKER_TIMEOUT_SEC) - @validator("syft_worker_id", pre=True, always=True) - def set_syft_worker_id(cls, v: Any, values: Any) -> Union[UID, Any]: + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. + @field_validator("syft_worker_id", mode="before") + @classmethod + def set_syft_worker_id(cls, v): if isinstance(v, str): return UID(v) return v diff --git a/packages/syft/src/syft/service/user/user.py b/packages/syft/src/syft/service/user/user.py index 71a07d2dfb0..5fae0d5d42d 100644 --- a/packages/syft/src/syft/service/user/user.py +++ b/packages/syft/src/syft/service/user/user.py @@ -12,9 +12,9 @@ from bcrypt import checkpw from bcrypt import gensalt from bcrypt import hashpw -import pydantic +from pydantic import EmailStr from pydantic import ValidationError -from pydantic.networks import EmailStr +from pydantic import field_validator # relative from ...client.api import APIRegistry @@ -64,10 +64,6 @@ class User(SyftObject): id: Optional[UID] - @pydantic.validator("email", pre=True, always=True) - def make_email(cls, v: EmailStr) -> EmailStr: - return EmailStr(v) - # fields email: Optional[EmailStr] name: Optional[str] @@ -145,11 +141,8 @@ class UserUpdate(PartialSyftObject): __canonical_name__ = "UserUpdate" __version__ = SYFT_OBJECT_VERSION_2 - @pydantic.validator("email", pre=True) - def make_email(cls, v: Any) -> Any: - return EmailStr(v) if isinstance(v, str) and not isinstance(v, EmailStr) else v - - @pydantic.validator("role", pre=True) + @field_validator(mode="before") + @classmethod def str_to_role(cls, v: Any) -> Any: if isinstance(v, str) and hasattr(ServiceRole, v.upper()): return getattr(ServiceRole, v.upper()) diff --git a/packages/syft/src/syft/service/worker/image_registry.py b/packages/syft/src/syft/service/worker/image_registry.py index 7292273c605..0b218ac8139 100644 --- a/packages/syft/src/syft/service/worker/image_registry.py +++ b/packages/syft/src/syft/service/worker/image_registry.py @@ -3,8 +3,12 @@ from urllib.parse import urlparse # third party +<<<<<<< HEAD from pydantic import validator from typing_extensions import Self +======= +from pydantic import field_validator +>>>>>>> d9b153bbdf (Update to pydantic v2 first pass with bump-pydantic) # relative from ...serde.serializable import serializable @@ -28,8 +32,9 @@ class SyftImageRegistry(SyftObject): id: UID url: str - @validator("url") - def validate_url(cls, val: str) -> str: + @field_validator("url") + @classmethod + def validate_url(cls, val: str): if not val: raise ValueError("Invalid Registry URL. Must not be empty") diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index fe1201ef92b..29fb9719b6e 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -16,7 +16,7 @@ # third party from pydantic import Field -from pydantic import validator +from pydantic import field_validator from result import Err from result import Ok from result import Result @@ -434,7 +434,8 @@ class SQLiteStoreClientConfig(StoreClientConfig): # We need this in addition to Field(default_factory=...) # so users can still do SQLiteStoreClientConfig(path=None) - @validator("path", pre=True) + @field_validator("path", mode="before") + @classmethod def __default_path(cls, path: Optional[Union[str, Path]]) -> Union[str, Path]: if path is None: return tempfile.gettempdir() diff --git a/packages/syft/src/syft/types/base.py b/packages/syft/src/syft/types/base.py index bb5160aebb0..764be4ae07a 100644 --- a/packages/syft/src/syft/types/base.py +++ b/packages/syft/src/syft/types/base.py @@ -2,8 +2,8 @@ # third party from pydantic import BaseModel +from pydantic import ConfigDict class SyftBaseModel(BaseModel): - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 9d8ed78a5d7..6c897cd0791 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -26,7 +26,9 @@ # third party import pandas as pd import pydantic +from pydantic import ConfigDict from pydantic import EmailStr +from pydantic import model_validator from pydantic.fields import Undefined from result import OkErr from typeguard import check_type @@ -91,8 +93,7 @@ def hash(self) -> str: class SyftBaseObject(pydantic.BaseModel, SyftHashableObject): - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) # the name which doesn't change even when there are multiple classes __canonical_name__: str @@ -342,19 +343,17 @@ def get_migration_for_version( class SyftObject(SyftBaseObject, SyftObjectRegistry, SyftMigrationRegistry): __canonical_name__ = "SyftObject" __version__ = SYFT_OBJECT_VERSION_1 - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) # all objects have a UID id: UID # # move this to transforms - @pydantic.root_validator(pre=True) + @model_validator(mode="before") def make_id(cls, values: Dict[str, Any]) -> Dict[str, Any]: - id_field = cls.__fields__["id"] - if "id" not in values and id_field.required: - values["id"] = id_field.type_() + id_field = cls.model_fields["id"] + if "id" not in values and id_field.is_required(): + values["id"] = id_field.annotation() return values __attr_searchable__: ClassVar[ diff --git a/packages/syft/src/syft/types/twin_object.py b/packages/syft/src/syft/types/twin_object.py index 5529daaff53..22b8b28c76d 100644 --- a/packages/syft/src/syft/types/twin_object.py +++ b/packages/syft/src/syft/types/twin_object.py @@ -3,11 +3,11 @@ # stdlib from typing import Any -from typing import Dict -from typing import Optional # third party -import pydantic +from pydantic import field_validator +from pydantic import model_validator +from typing_extensions import Self # relative from ..serde.serializable import serializable @@ -24,7 +24,7 @@ def to_action_object(obj: Any) -> ActionObject: if type(obj) in action_types: return action_types[type(obj)](syft_action_data_cache=obj) - raise Exception(f"{type(obj)} not in action_types") + raise ValueError(f"{type(obj)} not in action_types") @serializable() @@ -40,21 +40,27 @@ class TwinObject(SyftObject): mock_obj: ActionObject mock_obj_id: UID = None # type: ignore - @pydantic.validator("private_obj", pre=True, always=True) - def make_private_obj(cls, v: ActionObject) -> ActionObject: + @field_validator(mode="before") + @classmethod + def make_private_obj(cls, v: Any) -> ActionObject: return to_action_object(v) - @pydantic.validator("private_obj_id", pre=True, always=True) - def make_private_obj_id(cls, v: Optional[UID], values: Dict) -> UID: - return values["private_obj"].id if v is None else v + @model_validator(mode="after") + def make_private_obj_id(self) -> Self: + if self.private_obj_id is None: + self.private_obj_id = self.private_obj.id + return self - @pydantic.validator("mock_obj", pre=True, always=True) - def make_mock_obj(cls, v: ActionObject): + @field_validator(mode="before") + @classmethod + def make_mock_obj(cls, v: Any) -> ActionObject: return to_action_object(v) - @pydantic.validator("mock_obj_id", pre=True, always=True) - def make_mock_obj_id(cls, v: Optional[UID], values: Dict) -> UID: - return values["mock_obj"].id if v is None else v + @model_validator(mode="after") + def make_mock_obj_id(self) -> Self: + if self.mock_obj_id is None: + self.mock_obj_id = self.mock_obj.id + return self @property def private(self) -> ActionObject: From 1c58624ae7a3d325ba9a124aabd96852b618d7e0 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 6 Feb 2024 16:02:32 +0800 Subject: [PATCH 02/59] Set default values of Any/Optional fields to None --- packages/syft/src/syft/client/api.py | 16 ++-- .../syft/src/syft/client/gateway_client.py | 2 +- packages/syft/src/syft/custom_worker/k8s.py | 6 +- .../syft/src/syft/node/worker_settings.py | 6 +- .../src/syft/service/action/action_graph.py | 6 +- .../src/syft/service/action/action_object.py | 74 +++++++++---------- .../syft/src/syft/service/code/user_code.py | 24 +++--- packages/syft/src/syft/service/context.py | 8 +- .../syft/service/data_subject/data_subject.py | 4 +- .../syft/src/syft/service/dataset/dataset.py | 34 ++++----- .../syft/src/syft/service/job/job_stash.py | 18 ++--- .../syft/service/metadata/node_metadata.py | 24 +++--- .../src/syft/service/network/node_peer.py | 2 +- .../service/notification/notifications.py | 6 +- .../syft/src/syft/service/policy/policy.py | 2 +- .../syft/src/syft/service/project/project.py | 32 ++++---- .../src/syft/service/queue/queue_stash.py | 14 ++-- .../syft/src/syft/service/queue/zmq_queue.py | 8 +- .../syft/src/syft/service/request/request.py | 12 +-- packages/syft/src/syft/service/service.py | 6 +- packages/syft/src/syft/service/user/user.py | 42 +++++------ packages/syft/src/syft/service/warnings.py | 4 +- .../syft/service/worker/image_identifier.py | 2 +- .../src/syft/service/worker/worker_pool.py | 14 ++-- .../src/syft/store/blob_storage/__init__.py | 6 +- .../syft/src/syft/store/document_store.py | 4 +- packages/syft/src/syft/types/blob_storage.py | 14 ++-- packages/syft/src/syft/types/datetime.py | 2 +- packages/syft/src/syft/types/syft_object.py | 4 +- packages/syft/src/syft/types/transforms.py | 8 +- 30 files changed, 202 insertions(+), 202 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index cb4e2d8aa2f..1631356ca31 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -115,11 +115,11 @@ class APIEndpoint(SyftObject): module_path: str name: str description: str - doc_string: Optional[str] + doc_string: Optional[str] = None signature: Signature has_self: bool = False - pre_kwargs: Optional[Dict[str, Any]] - warning: Optional[APIEndpointWarning] + pre_kwargs: Optional[Dict[str, Any]] = None + warning: Optional[APIEndpointWarning] = None @serializable() @@ -132,10 +132,10 @@ class LibEndpoint(SyftBaseObject): module_path: str name: str description: str - doc_string: Optional[str] + doc_string: Optional[str] = None signature: Signature has_self: bool = False - pre_kwargs: Optional[Dict[str, Any]] + pre_kwargs: Optional[Dict[str, Any]] = None @serializable(attrs=["signature", "credentials", "serialized_message"]) @@ -205,7 +205,7 @@ class SyftAPIData(SyftBaseObject): __version__ = SYFT_OBJECT_VERSION_1 # fields - data: Any + data: Any = None def sign(self, credentials: SyftSigningKey) -> SignedSyftAPICall: signed_message = credentials.signing_key.sign(_serialize(self, to_bytes=True)) @@ -231,9 +231,9 @@ class RemoteFunction(SyftObject): signature: Signature path: str make_call: Callable - pre_kwargs: Optional[Dict[str, Any]] + pre_kwargs: Optional[Dict[str, Any]] = None communication_protocol: PROTOCOL_TYPE - warning: Optional[APIEndpointWarning] + warning: Optional[APIEndpointWarning] = None @property def __ipython_inspector_signature_override__(self) -> Optional[Signature]: diff --git a/packages/syft/src/syft/client/gateway_client.py b/packages/syft/src/syft/client/gateway_client.py index 2a569cabd22..abb42b69c59 100644 --- a/packages/syft/src/syft/client/gateway_client.py +++ b/packages/syft/src/syft/client/gateway_client.py @@ -157,7 +157,7 @@ class ProxyClient(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 routing_client: GatewayClient - node_type: Optional[NodeType] + node_type: Optional[NodeType] = None def retrieve_nodes(self) -> List[NodePeer]: if self.node_type in [NodeType.DOMAIN, NodeType.ENCLAVE]: diff --git a/packages/syft/src/syft/custom_worker/k8s.py b/packages/syft/src/syft/custom_worker/k8s.py index fb777f6ec6c..845ae95bc6c 100644 --- a/packages/syft/src/syft/custom_worker/k8s.py +++ b/packages/syft/src/syft/custom_worker/k8s.py @@ -62,9 +62,9 @@ class ContainerStatus(BaseModel): ready: bool running: bool waiting: bool - reason: Optional[str] # when waiting=True - message: Optional[str] # when waiting=True - startedAt: Optional[str] # when running=True + reason: Optional[str] = None # when waiting=True + message: Optional[str] = None # when waiting=True + startedAt: Optional[str] = None # when running=True @classmethod def from_status(cls, cstatus: dict): diff --git a/packages/syft/src/syft/node/worker_settings.py b/packages/syft/src/syft/node/worker_settings.py index 106e2e94821..7fd2299b177 100644 --- a/packages/syft/src/syft/node/worker_settings.py +++ b/packages/syft/src/syft/node/worker_settings.py @@ -37,7 +37,7 @@ class WorkerSettingsV1(SyftObject): signing_key: SyftSigningKey document_store_config: StoreConfig action_store_config: StoreConfig - blob_store_config: Optional[BlobStorageConfig] + blob_store_config: Optional[BlobStorageConfig] = None @serializable() @@ -52,8 +52,8 @@ class WorkerSettings(SyftObject): signing_key: SyftSigningKey document_store_config: StoreConfig action_store_config: StoreConfig - blob_store_config: Optional[BlobStorageConfig] - queue_config: Optional[QueueConfig] + blob_store_config: Optional[BlobStorageConfig] = None + queue_config: Optional[QueueConfig] = None @staticmethod def from_node(node: AbstractNode) -> Self: diff --git a/packages/syft/src/syft/service/action/action_graph.py b/packages/syft/src/syft/service/action/action_graph.py index a64220a1980..9fadcba3b90 100644 --- a/packages/syft/src/syft/service/action/action_graph.py +++ b/packages/syft/src/syft/service/action/action_graph.py @@ -61,7 +61,7 @@ class NodeActionData(SyftObject): __canonical_name__ = "NodeActionData" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] + id: Optional[UID] = None type: NodeType status: ExecutionStatus = ExecutionStatus.PROCESSING retry: int = 0 @@ -70,8 +70,8 @@ class NodeActionData(SyftObject): user_verify_key: SyftVerifyKey is_mutated: bool = False # denotes that this node has been mutated is_mutagen: bool = False # denotes that this node is causing a mutation - next_mutagen_node: Optional[UID] # next neighboring mutagen node - last_nm_mutagen_node: Optional[UID] # last non mutated mutagen node + next_mutagen_node: Optional[UID] = None # next neighboring mutagen node + last_nm_mutagen_node: Optional[UID] = None # last non mutated mutagen node @classmethod def from_action(cls, action: Action, credentials: SyftVerifyKey) -> Self: diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 1a8c1247ad3..ff066a01da5 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -110,11 +110,11 @@ class ActionV1(SyftObject): path: str op: str - remote_self: Optional[LineageID] + remote_self: Optional[LineageID] = None args: List[LineageID] kwargs: Dict[str, LineageID] - result_id: Optional[LineageID] - action_type: Optional[ActionType] + result_id: Optional[LineageID] = None + action_type: Optional[ActionType] = None create_object: Optional[SyftObject] = None @@ -142,13 +142,13 @@ class Action(SyftObject): __attr_searchable__: List[str] = [] - path: Optional[str] - op: Optional[str] - remote_self: Optional[LineageID] + path: Optional[str] = None + op: Optional[str] = None + remote_self: Optional[LineageID] = None args: List[LineageID] kwargs: Dict[str, LineageID] result_id: LineageID = Field(default_factory=lambda: LineageID(UID())) - action_type: Optional[ActionType] + action_type: Optional[ActionType] = None create_object: Optional[SyftObject] = None user_code_id: Optional[UID] = None @@ -360,13 +360,13 @@ class PreHookContext(SyftBaseObject): The action generated by the current hook """ - obj: Any + obj: Any = None op_name: str - node_uid: Optional[UID] - result_id: Optional[Union[UID, LineageID]] - result_twin_type: Optional[TwinMode] - action: Optional[Action] - action_type: Optional[ActionType] + node_uid: Optional[UID] = None + result_id: Optional[Union[UID, LineageID]] = None + result_twin_type: Optional[TwinMode] = None + action: Optional[Action] = None + action_type: Optional[ActionType] = None def make_action_side_effect( @@ -588,23 +588,23 @@ class ActionObjectV1(SyftObject): syft_pointer_type: ClassVar[Type[ActionObjectPointer]] # Help with calculating history hash for code verification - syft_parent_hashes: Optional[Union[int, List[int]]] - syft_parent_op: Optional[str] - syft_parent_args: Optional[Any] - syft_parent_kwargs: Optional[Any] - syft_history_hash: Optional[int] + syft_parent_hashes: Optional[Union[int, List[int]]] = None + syft_parent_op: Optional[str] = None + syft_parent_args: Optional[Any] = None + syft_parent_kwargs: Optional[Any] = None + syft_history_hash: Optional[int] = None syft_internal_type: ClassVar[Type[Any]] - syft_node_uid: Optional[UID] + syft_node_uid: Optional[UID] = None _syft_pre_hooks__: Dict[str, List] = {} _syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_action_data_type: Optional[Type] - syft_action_data_repr_: Optional[str] - syft_action_data_str_: Optional[str] - syft_has_bool_attr: Optional[bool] - syft_resolve_data: Optional[bool] - syft_created_at: Optional[DateTime] + syft_action_data_type: Optional[Type] = None + syft_action_data_repr_: Optional[str] = None + syft_action_data_str_: Optional[str] = None + syft_has_bool_attr: Optional[bool] = None + syft_resolve_data: Optional[bool] = None + syft_created_at: Optional[DateTime] = None @serializable() @@ -622,23 +622,23 @@ class ActionObject(SyftObject): syft_pointer_type: ClassVar[Type[ActionObjectPointer]] # Help with calculating history hash for code verification - syft_parent_hashes: Optional[Union[int, List[int]]] - syft_parent_op: Optional[str] - syft_parent_args: Optional[Any] - syft_parent_kwargs: Optional[Any] - syft_history_hash: Optional[int] + syft_parent_hashes: Optional[Union[int, List[int]]] = None + syft_parent_op: Optional[str] = None + syft_parent_args: Optional[Any] = None + syft_parent_kwargs: Optional[Any] = None + syft_history_hash: Optional[int] = None syft_internal_type: ClassVar[Type[Any]] - syft_node_uid: Optional[UID] + syft_node_uid: Optional[UID] = None _syft_pre_hooks__: Dict[str, List] = {} _syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_action_data_type: Optional[Type] - syft_action_data_repr_: Optional[str] - syft_action_data_str_: Optional[str] - syft_has_bool_attr: Optional[bool] - syft_resolve_data: Optional[bool] - syft_created_at: Optional[DateTime] + syft_action_data_type: Optional[Type] = None + syft_action_data_repr_: Optional[str] = None + syft_action_data_str_: Optional[str] = None + syft_has_bool_attr: Optional[bool] = None + syft_resolve_data: Optional[bool] = None + syft_created_at: Optional[DateTime] = None syft_resolved: bool = True # syft_dont_wrap_attrs = ["shape"] diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 6954e2c0dec..ffe8412cf45 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -260,7 +260,7 @@ class UserCodeV1(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 id: UID - node_uid: Optional[UID] + node_uid: Optional[UID] = None user_verify_key: SyftVerifyKey raw_code: str input_policy_type: Union[Type[InputPolicy], UserPolicy] @@ -278,7 +278,7 @@ class UserCodeV1(SyftObject): status: UserCodeStatusCollection input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None - submit_time: Optional[DateTime] + submit_time: Optional[DateTime] = None __attr_searchable__ = [ "user_verify_key", @@ -295,7 +295,7 @@ class UserCodeV2(SyftObject): __version__ = SYFT_OBJECT_VERSION_2 id: UID - node_uid: Optional[UID] + node_uid: Optional[UID] = None user_verify_key: SyftVerifyKey raw_code: str input_policy_type: Union[Type[InputPolicy], UserPolicy] @@ -313,7 +313,7 @@ class UserCodeV2(SyftObject): status: UserCodeStatusCollection input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None - submit_time: Optional[DateTime] + submit_time: Optional[DateTime] = None uses_domain = False # tracks if the code calls domain.something, variable is set during parsing nested_requests: Dict[str, str] = {} nested_codes: Optional[Dict[str, Tuple[LinkedObject, Dict]]] = {} @@ -326,7 +326,7 @@ class UserCodeV3(SyftObject): __version__ = SYFT_OBJECT_VERSION_3 id: UID - node_uid: Optional[UID] + node_uid: Optional[UID] = None user_verify_key: SyftVerifyKey raw_code: str input_policy_type: Union[Type[InputPolicy], UserPolicy] @@ -344,11 +344,11 @@ class UserCodeV3(SyftObject): status: UserCodeStatusCollection input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None - submit_time: Optional[DateTime] + submit_time: Optional[DateTime] = None uses_domain = False # tracks if the code calls domain.something, variable is set during parsing nested_requests: Dict[str, str] = {} nested_codes: Optional[Dict[str, Tuple[LinkedObject, Dict]]] = {} - worker_pool_name: Optional[str] + worker_pool_name: Optional[str] = None @serializable() @@ -747,7 +747,7 @@ class SubmitUserCodeV2(SyftObject): __canonical_name__ = "SubmitUserCode" __version__ = SYFT_OBJECT_VERSION_2 - id: Optional[UID] + id: Optional[UID] = None code: str func_name: str signature: inspect.Signature @@ -755,7 +755,7 @@ class SubmitUserCodeV2(SyftObject): input_policy_init_kwargs: Optional[Dict[Any, Any]] = {} output_policy_type: Union[SubmitUserPolicy, UID, Type[OutputPolicy]] output_policy_init_kwargs: Optional[Dict[Any, Any]] = {} - local_function: Optional[Callable] + local_function: Optional[Callable] = None input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None @@ -766,7 +766,7 @@ class SubmitUserCode(SyftObject): __canonical_name__ = "SubmitUserCode" __version__ = SYFT_OBJECT_VERSION_3 - id: Optional[UID] + id: Optional[UID] = None code: str func_name: str signature: inspect.Signature @@ -774,7 +774,7 @@ class SubmitUserCode(SyftObject): input_policy_init_kwargs: Optional[Dict[Any, Any]] = {} output_policy_type: Union[SubmitUserPolicy, UID, Type[OutputPolicy]] output_policy_init_kwargs: Optional[Dict[Any, Any]] = {} - local_function: Optional[Callable] + local_function: Optional[Callable] = None input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None worker_pool_name: Optional[str] = None @@ -1257,7 +1257,7 @@ class UserCodeExecutionResult(SyftObject): user_code_id: UID stdout: str stderr: str - result: Any + result: Any = None class SecureContext: diff --git a/packages/syft/src/syft/service/context.py b/packages/syft/src/syft/service/context.py index 5ac07649e37..2ca5d2a337e 100644 --- a/packages/syft/src/syft/service/context.py +++ b/packages/syft/src/syft/service/context.py @@ -24,8 +24,8 @@ class NodeServiceContext(Context, SyftObject): __canonical_name__ = "NodeServiceContext" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] - node: Optional[AbstractNode] + id: Optional[UID] = None + node: Optional[AbstractNode] = None class AuthedServiceContext(NodeServiceContext): @@ -77,8 +77,8 @@ class ChangeContext(SyftBaseObject): __version__ = SYFT_OBJECT_VERSION_1 node: Optional[AbstractNode] = None - approving_user_credentials: Optional[SyftVerifyKey] - requesting_user_credentials: Optional[SyftVerifyKey] + approving_user_credentials: Optional[SyftVerifyKey] = None + requesting_user_credentials: Optional[SyftVerifyKey] = None extra_kwargs: Dict = {} @classmethod diff --git a/packages/syft/src/syft/service/data_subject/data_subject.py b/packages/syft/src/syft/service/data_subject/data_subject.py index 301d24881f3..b06a2294f99 100644 --- a/packages/syft/src/syft/service/data_subject/data_subject.py +++ b/packages/syft/src/syft/service/data_subject/data_subject.py @@ -34,7 +34,7 @@ class DataSubject(SyftObject): node_uid: UID name: str - description: Optional[str] + description: Optional[str] = None aliases: List[str] = [] @property @@ -80,7 +80,7 @@ class DataSubjectCreate(SyftObject): id: Optional[UID] = None name: str - description: Optional[str] + description: Optional[str] = None aliases: Optional[List[str]] = [] members: Dict[str, "DataSubjectCreate"] = {} diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index b5972fd8671..12f058cffb3 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -64,10 +64,10 @@ class Contributor(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 name: str - role: Optional[str] + role: Optional[str] = None email: str - phone: Optional[str] - note: Optional[str] + phone: Optional[str] = None + note: Optional[str] = None __repr_attrs__ = ["name", "role", "email"] @@ -133,9 +133,9 @@ class Asset(SyftObject): contributors: Set[Contributor] = set() data_subjects: List[DataSubject] = [] mock_is_real: bool = False - shape: Optional[Tuple] + shape: Optional[Tuple] = None created_at: DateTime = DateTime.now() - uploader: Optional[Contributor] + uploader: Optional[Contributor] = None __repr_attrs__ = ["name", "shape"] @@ -326,14 +326,14 @@ class CreateAsset(SyftObject): description: Optional[MarkdownDescription] = None contributors: Set[Contributor] = set() data_subjects: List[DataSubjectCreate] = [] - node_uid: Optional[UID] - action_id: Optional[UID] - data: Optional[Any] - mock: Optional[Any] - shape: Optional[Tuple] + node_uid: Optional[UID] = None + action_id: Optional[UID] = None + data: Optional[Any] = None + mock: Optional[Any] = None + shape: Optional[Tuple] = None mock_is_real: bool = False - created_at: Optional[DateTime] - uploader: Optional[Contributor] + created_at: Optional[DateTime] = None + uploader: Optional[Contributor] = None __repr_attrs__ = ["name"] model_config = ConfigDict(validate_assignment=True) @@ -463,15 +463,15 @@ class Dataset(SyftObject): id: UID name: str - node_uid: Optional[UID] + node_uid: Optional[UID] = None asset_list: List[Asset] = [] contributors: Set[Contributor] = set() - citation: Optional[str] - url: Optional[str] + citation: Optional[str] = None + url: Optional[str] = None description: Optional[MarkdownDescription] = None - updated_at: Optional[str] + updated_at: Optional[str] = None requests: Optional[int] = 0 - mb_size: Optional[int] + mb_size: Optional[int] = None created_at: DateTime = DateTime.now() uploader: Contributor diff --git a/packages/syft/src/syft/service/job/job_stash.py b/packages/syft/src/syft/service/job/job_stash.py index 9eb5c6561bb..1269e037ced 100644 --- a/packages/syft/src/syft/service/job/job_stash.py +++ b/packages/syft/src/syft/service/job/job_stash.py @@ -70,11 +70,11 @@ class JobV1(SyftObject): id: UID node_uid: UID - result: Optional[Any] + result: Optional[Any] = None resolved: bool = False status: JobStatus = JobStatus.CREATED - log_id: Optional[UID] - parent_job_id: Optional[UID] + log_id: Optional[UID] = None + parent_job_id: Optional[UID] = None n_iters: Optional[int] = 0 current_iter: Optional[int] = None creation_time: Optional[str] = None @@ -88,11 +88,11 @@ class JobV2(SyftObject): id: UID node_uid: UID - result: Optional[Any] + result: Optional[Any] = None resolved: bool = False status: JobStatus = JobStatus.CREATED - log_id: Optional[UID] - parent_job_id: Optional[UID] + log_id: Optional[UID] = None + parent_job_id: Optional[UID] = None n_iters: Optional[int] = 0 current_iter: Optional[int] = None creation_time: Optional[str] = None @@ -107,11 +107,11 @@ class Job(SyftObject): id: UID node_uid: UID - result: Optional[Any] + result: Optional[Any] = None resolved: bool = False status: JobStatus = JobStatus.CREATED - log_id: Optional[UID] - parent_job_id: Optional[UID] + log_id: Optional[UID] = None + parent_job_id: Optional[UID] = None n_iters: Optional[int] = 0 current_iter: Optional[int] = None creation_time: str = Field(default_factory=lambda: str(datetime.now())) diff --git a/packages/syft/src/syft/service/metadata/node_metadata.py b/packages/syft/src/syft/service/metadata/node_metadata.py index e9bb844ad41..906096c0c4f 100644 --- a/packages/syft/src/syft/service/metadata/node_metadata.py +++ b/packages/syft/src/syft/service/metadata/node_metadata.py @@ -51,16 +51,16 @@ class NodeMetadataUpdate(SyftObject): __canonical_name__ = "NodeMetadataUpdate" __version__ = SYFT_OBJECT_VERSION_1 - name: Optional[str] - organization: Optional[str] - description: Optional[str] - on_board: Optional[bool] - id: Optional[UID] - verify_key: Optional[SyftVerifyKey] - highest_object_version: Optional[int] - lowest_object_version: Optional[int] - syft_version: Optional[str] - admin_email: Optional[str] + name: Optional[str] = None + organization: Optional[str] = None + description: Optional[str] = None + on_board: Optional[bool] = None + id: Optional[UID] = None + verify_key: Optional[SyftVerifyKey] = None + highest_object_version: Optional[int] = None + lowest_object_version: Optional[int] = None + syft_version: Optional[str] = None + admin_email: Optional[str] = None @serializable() @@ -152,8 +152,8 @@ class NodeMetadataJSON(BaseModel, StorableObjectType): name: str id: str verify_key: str - highest_object_version: Optional[int] - lowest_object_version: Optional[int] + highest_object_version: Optional[int] = None + lowest_object_version: Optional[int] = None syft_version: str node_type: str = NodeType.DOMAIN.value organization: str = "OpenMined" diff --git a/packages/syft/src/syft/service/network/node_peer.py b/packages/syft/src/syft/service/network/node_peer.py index cc9764947e5..52236824344 100644 --- a/packages/syft/src/syft/service/network/node_peer.py +++ b/packages/syft/src/syft/service/network/node_peer.py @@ -35,7 +35,7 @@ class NodePeer(SyftObject): __attr_unique__ = ["verify_key"] __repr_attrs__ = ["name", "node_type", "admin_email"] - id: Optional[UID] + id: Optional[UID] = None name: str verify_key: SyftVerifyKey node_routes: List[NodeRouteType] = [] diff --git a/packages/syft/src/syft/service/notification/notifications.py b/packages/syft/src/syft/service/notification/notifications.py index fd8be6675d1..f47f420484a 100644 --- a/packages/syft/src/syft/service/notification/notifications.py +++ b/packages/syft/src/syft/service/notification/notifications.py @@ -45,8 +45,8 @@ class ReplyNotification(SyftObject): text: str target_msg: UID - id: Optional[UID] - from_user_verify_key: Optional[SyftVerifyKey] + id: Optional[UID] = None + from_user_verify_key: Optional[SyftVerifyKey] = None @serializable() @@ -60,7 +60,7 @@ class Notification(SyftObject): to_user_verify_key: SyftVerifyKey created_at: DateTime status: NotificationStatus = NotificationStatus.UNREAD - linked_obj: Optional[LinkedObject] + linked_obj: Optional[LinkedObject] = None replies: Optional[List[ReplyNotification]] = [] __attr_searchable__ = [ diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index e0bcc0d0356..3f16c7d1ed5 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -324,7 +324,7 @@ class OutputHistory(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 output_time: DateTime - outputs: Optional[Union[List[UID], Dict[str, UID]]] + outputs: Optional[Union[List[UID], Dict[str, UID]]] = None executing_user_verify_key: SyftVerifyKey diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index feaecf53a40..2b4b489b745 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -81,14 +81,14 @@ class ProjectEvent(SyftObject): timestamp: DateTime = Field(default_factory=DateTime.now) allowed_sub_types: Optional[List] = [] # 2. Rebase attrs - project_id: Optional[UID] - seq_no: Optional[int] - prev_event_uid: Optional[UID] - prev_event_hash: Optional[str] - event_hash: Optional[str] + project_id: Optional[UID] = None + seq_no: Optional[int] = None + prev_event_uid: Optional[UID] = None + prev_event_hash: Optional[str] = None + event_hash: Optional[str] = None # 3. Signature attrs - creator_verify_key: Optional[SyftVerifyKey] - signature: Optional[bytes] # dont use in signing + creator_verify_key: Optional[SyftVerifyKey] = None + signature: Optional[bytes] = None # dont use in signing def __repr_syft_nested__(self) -> tuple[str, str]: return ( @@ -677,14 +677,14 @@ class Project(SyftObject): "event_id_hashmap", ] - id: Optional[UID] + id: Optional[UID] = None name: str - description: Optional[str] + description: Optional[str] = None members: List[NodeIdentity] users: List[UserIdentity] = [] - username: Optional[str] + username: Optional[str] = None created_by: str - start_hash: Optional[str] + start_hash: Optional[str] = None # WARNING: Do not add it to hash keys or print directly user_signing_key: Optional[SyftSigningKey] = None @@ -694,7 +694,7 @@ class Project(SyftObject): # Project sync state_sync_leader: NodeIdentity - leader_node_peer: Optional[NodePeer] + leader_node_peer: Optional[NodePeer] = None # Unused consensus_model: ConsensusModel @@ -1168,19 +1168,19 @@ class ProjectSubmit(SyftObject): # Init args name: str - description: Optional[str] + description: Optional[str] = None members: Union[List[SyftClient], List[NodeIdentity]] # These will be automatically populated users: List[UserIdentity] = [] created_by: Optional[str] = None - username: Optional[str] + username: Optional[str] = None clients: List[SyftClient] = [] # List of member clients start_hash: str = "" # Project sync args - leader_node_route: Optional[NodeRoute] - state_sync_leader: Optional[NodeIdentity] + leader_node_route: Optional[NodeRoute] = None + state_sync_leader: Optional[NodeIdentity] = None bootstrap_events: Optional[List[ProjectEvent]] = [] # Unused at the moment diff --git a/packages/syft/src/syft/service/queue/queue_stash.py b/packages/syft/src/syft/service/queue/queue_stash.py index e98641dc83a..31447da02c3 100644 --- a/packages/syft/src/syft/service/queue/queue_stash.py +++ b/packages/syft/src/syft/service/queue/queue_stash.py @@ -55,7 +55,7 @@ class QueueItemV1(SyftObject): id: UID node_uid: UID - result: Optional[Any] + result: Optional[Any] = None resolved: bool = False status: Status = Status.CREATED @@ -67,7 +67,7 @@ class QueueItemV2(SyftObject): id: UID node_uid: UID - result: Optional[Any] + result: Optional[Any] = None resolved: bool = False status: Status = Status.CREATED @@ -75,8 +75,8 @@ class QueueItemV2(SyftObject): service: str args: List kwargs: Dict[str, Any] - job_id: Optional[UID] - worker_settings: Optional[WorkerSettings] + job_id: Optional[UID] = None + worker_settings: Optional[WorkerSettings] = None has_execute_permissions: bool = False @@ -89,7 +89,7 @@ class QueueItem(SyftObject): id: UID node_uid: UID - result: Optional[Any] + result: Optional[Any] = None resolved: bool = False status: Status = Status.CREATED @@ -97,8 +97,8 @@ class QueueItem(SyftObject): service: str args: List kwargs: Dict[str, Any] - job_id: Optional[UID] - worker_settings: Optional[WorkerSettings] + job_id: Optional[UID] = None + worker_settings: Optional[WorkerSettings] = None has_execute_permissions: bool = False worker_pool: LinkedObject diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 1136f34bdc0..314522d36c3 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -796,7 +796,7 @@ class ZMQClientConfigV1(SyftObject, QueueClientConfig): __canonical_name__ = "ZMQClientConfig" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] + id: Optional[UID] = None hostname: str = "127.0.0.1" @@ -804,7 +804,7 @@ class ZMQClientConfigV2(SyftObject, QueueClientConfig): __canonical_name__ = "ZMQClientConfig" __version__ = SYFT_OBJECT_VERSION_2 - id: Optional[UID] + id: Optional[UID] = None hostname: str = "127.0.0.1" queue_port: Optional[int] = None # TODO: setting this to false until we can fix the ZMQ @@ -818,14 +818,14 @@ class ZMQClientConfig(SyftObject, QueueClientConfig): __canonical_name__ = "ZMQClientConfig" __version__ = SYFT_OBJECT_VERSION_3 - id: Optional[UID] + id: Optional[UID] = None hostname: str = "127.0.0.1" queue_port: Optional[int] = None # TODO: setting this to false until we can fix the ZMQ # port issue causing tests to randomly fail create_producer: bool = False n_consumers: int = 0 - consumer_service: Optional[str] + consumer_service: Optional[str] = None @migrate(ZMQClientConfig, ZMQClientConfigV1) diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 878f451e8e3..3d5f2f4947f 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -77,7 +77,7 @@ class Change(SyftObject): __canonical_name__ = "Change" __version__ = SYFT_OBJECT_VERSION_1 - linked_obj: Optional[LinkedObject] + linked_obj: Optional[LinkedObject] = None def is_type(self, type_: type) -> bool: return (self.linked_obj is not None) and (type_ == self.linked_obj.object_type) @@ -88,7 +88,7 @@ class ChangeStatus(SyftObject): __canonical_name__ = "ChangeStatus" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] + id: Optional[UID] = None change_id: UID applied: bool = False @@ -348,9 +348,9 @@ class Request(SyftObject): requesting_user_name: str = "" requesting_user_email: Optional[str] = "" requesting_user_institution: Optional[str] = "" - approving_user_verify_key: Optional[SyftVerifyKey] + approving_user_verify_key: Optional[SyftVerifyKey] = None request_time: DateTime - updated_at: Optional[DateTime] + updated_at: Optional[DateTime] = None node_uid: UID request_hash: str changes: List[Change] @@ -796,7 +796,7 @@ class RequestInfoFilter(SyftObject): __canonical_name__ = "RequestInfoFilter" __version__ = SYFT_OBJECT_VERSION_1 - name: Optional[str] + name: Optional[str] = None @serializable() @@ -805,7 +805,7 @@ class SubmitRequest(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 changes: List[Change] - requesting_user_verify_key: Optional[SyftVerifyKey] + requesting_user_verify_key: Optional[SyftVerifyKey] = None def hash_changes(context: TransformContext) -> TransformContext: diff --git a/packages/syft/src/syft/service/service.py b/packages/syft/src/syft/service/service.py index fa9a233e9fe..e377d5cdb93 100644 --- a/packages/syft/src/syft/service/service.py +++ b/packages/syft/src/syft/service/service.py @@ -87,10 +87,10 @@ class BaseConfig(SyftBaseObject): private_path: str public_name: str method_name: str - doc_string: Optional[str] - signature: Optional[Signature] + doc_string: Optional[str] = None + signature: Optional[Signature] = None is_from_lib: bool = False - warning: Optional[APIEndpointWarning] + warning: Optional[APIEndpointWarning] = None @serializable() diff --git a/packages/syft/src/syft/service/user/user.py b/packages/syft/src/syft/service/user/user.py index 5fae0d5d42d..4648bfb147b 100644 --- a/packages/syft/src/syft/service/user/user.py +++ b/packages/syft/src/syft/service/user/user.py @@ -44,14 +44,14 @@ class UserV1(SyftObject): __canonical_name__ = "User" __version__ = SYFT_OBJECT_VERSION_1 - email: Optional[EmailStr] - name: Optional[str] - hashed_password: Optional[str] - salt: Optional[str] - signing_key: Optional[SyftSigningKey] - verify_key: Optional[SyftVerifyKey] - role: Optional[ServiceRole] - institution: Optional[str] + email: Optional[EmailStr] = None + name: Optional[str] = None + hashed_password: Optional[str] = None + salt: Optional[str] = None + signing_key: Optional[SyftSigningKey] = None + verify_key: Optional[SyftVerifyKey] = None + role: Optional[ServiceRole] = None + institution: Optional[str] = None website: Optional[str] = None created_at: Optional[str] = None @@ -62,17 +62,17 @@ class User(SyftObject): __canonical_name__ = "User" __version__ = SYFT_OBJECT_VERSION_2 - id: Optional[UID] + id: Optional[UID] = None # fields - email: Optional[EmailStr] - name: Optional[str] - hashed_password: Optional[str] - salt: Optional[str] - signing_key: Optional[SyftSigningKey] - verify_key: Optional[SyftVerifyKey] - role: Optional[ServiceRole] - institution: Optional[str] + email: Optional[EmailStr] = None + name: Optional[str] = None + hashed_password: Optional[str] = None + salt: Optional[str] = None + signing_key: Optional[SyftSigningKey] = None + verify_key: Optional[SyftVerifyKey] = None + role: Optional[ServiceRole] = None + institution: Optional[str] = None website: Optional[str] = None created_at: Optional[str] = None # TODO where do we put this flag? @@ -211,8 +211,8 @@ class UserViewV1(SyftObject): email: EmailStr name: str role: ServiceRole # make sure role cant be set without uid - institution: Optional[str] - website: Optional[str] + institution: Optional[str] = None + website: Optional[str] = None @serializable() @@ -223,8 +223,8 @@ class UserView(SyftObject): email: EmailStr name: str role: ServiceRole # make sure role cant be set without uid - institution: Optional[str] - website: Optional[str] + institution: Optional[str] = None + website: Optional[str] = None mock_execution_permission: bool __repr_attrs__ = ["name", "email", "institution", "website", "role"] diff --git a/packages/syft/src/syft/service/warnings.py b/packages/syft/src/syft/service/warnings.py index 553531046a8..b0f753cfcbd 100644 --- a/packages/syft/src/syft/service/warnings.py +++ b/packages/syft/src/syft/service/warnings.py @@ -20,8 +20,8 @@ class WarningContext( Context, ): - node: Optional[AbstractNode] - credentials: Optional[SyftCredentials] + node: Optional[AbstractNode] = None + credentials: Optional[SyftCredentials] = None role: ServiceRole diff --git a/packages/syft/src/syft/service/worker/image_identifier.py b/packages/syft/src/syft/service/worker/image_identifier.py index 4651c3f4f2e..ac29f9ed3c9 100644 --- a/packages/syft/src/syft/service/worker/image_identifier.py +++ b/packages/syft/src/syft/service/worker/image_identifier.py @@ -29,7 +29,7 @@ class SyftWorkerImageIdentifier(SyftBaseModel): https://docs.docker.com/engine/reference/commandline/tag/#tag-an-image-referenced-by-name-and-tag """ - registry: Optional[Union[SyftImageRegistry, str]] + registry: Optional[Union[SyftImageRegistry, str]] = None repo: str tag: str diff --git a/packages/syft/src/syft/service/worker/worker_pool.py b/packages/syft/src/syft/service/worker/worker_pool.py index 13c1cf1c158..f8edf5c541a 100644 --- a/packages/syft/src/syft/service/worker/worker_pool.py +++ b/packages/syft/src/syft/service/worker/worker_pool.py @@ -69,14 +69,14 @@ class SyftWorker(SyftObject): id: UID name: str - container_id: Optional[str] + container_id: Optional[str] = None created_at: DateTime = DateTime.now() - healthcheck: Optional[WorkerHealth] + healthcheck: Optional[WorkerHealth] = None status: WorkerStatus - image: Optional[SyftWorkerImage] + image: Optional[SyftWorkerImage] = None worker_pool_name: str consumer_state: ConsumerState = ConsumerState.DETACHED - job_id: Optional[UID] + job_id: Optional[UID] = None @property def logs(self) -> Union[str, SyftError]: @@ -166,7 +166,7 @@ class WorkerPool(SyftObject): ] name: str - image_id: Optional[UID] + image_id: Optional[UID] = None max_count: int worker_list: List[LinkedObject] created_at: DateTime = DateTime.now() @@ -274,8 +274,8 @@ class ContainerSpawnStatus(SyftBaseModel): __repr_attrs__ = ["worker_name", "worker", "error"] worker_name: str - worker: Optional[SyftWorker] - error: Optional[str] + worker: Optional[SyftWorker] = None + error: Optional[str] = None def _get_worker_container( diff --git a/packages/syft/src/syft/store/blob_storage/__init__.py b/packages/syft/src/syft/store/blob_storage/__init__.py index bb86b6ef25a..3fef0371381 100644 --- a/packages/syft/src/syft/store/blob_storage/__init__.py +++ b/packages/syft/src/syft/store/blob_storage/__init__.py @@ -85,7 +85,7 @@ class BlobRetrievalV1(SyftObject): __canonical_name__ = "BlobRetrieval" __version__ = SYFT_OBJECT_VERSION_1 - type_: Optional[Type] + type_: Optional[Type] = None file_name: str @@ -94,10 +94,10 @@ class BlobRetrieval(SyftObject): __canonical_name__ = "BlobRetrieval" __version__ = SYFT_OBJECT_VERSION_2 - type_: Optional[Type] + type_: Optional[Type] = None file_name: str syft_blob_storage_entry_id: Optional[UID] = None - file_size: Optional[int] + file_size: Optional[int] = None @migrate(BlobRetrieval, BlobRetrievalV1) diff --git a/packages/syft/src/syft/store/document_store.py b/packages/syft/src/syft/store/document_store.py index 5151f43294c..c8cd5f23d20 100644 --- a/packages/syft/src/syft/store/document_store.py +++ b/packages/syft/src/syft/store/document_store.py @@ -140,7 +140,7 @@ def from_dict(cks_dict: Dict[str, type]) -> PartitionKeys: @serializable() class QueryKey(PartitionKey): - value: Any + value: Any = None def __eq__(self, other: Any) -> bool: return ( @@ -764,5 +764,5 @@ class StoreConfig(SyftBaseObject): __version__ = SYFT_OBJECT_VERSION_1 store_type: Type[DocumentStore] - client_config: Optional[StoreClientConfig] + client_config: Optional[StoreClientConfig] = None locking_config: LockingConfig = NoLockingConfig() diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index e36dc56df4e..a37e37229a7 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -76,7 +76,7 @@ class BlobFile(SyftObject): file_name: str syft_blob_storage_entry_id: Optional[UID] = None file_size: Optional[int] = None - path: Optional[Path] + path: Optional[Path] = None uploaded = False __repr_attrs__ = ["id", "file_name"] @@ -312,7 +312,7 @@ class BlobStorageEntryV1(SyftObject): id: UID location: Union[SecureFilePathLocation, SeaweedSecureFilePathLocation] - type_: Optional[Type] + type_: Optional[Type] = None mimetype: str = "bytes" file_size: int uploaded_by: SyftVerifyKey @@ -328,13 +328,13 @@ class BlobStorageEntry(SyftObject): id: UID location: Union[SecureFilePathLocation, SeaweedSecureFilePathLocation] - type_: Optional[Type] + type_: Optional[Type] = None mimetype: str = "bytes" file_size: int no_lines: Optional[int] = 0 uploaded_by: SyftVerifyKey created_at: DateTime = DateTime.now() - bucket_name: Optional[str] + bucket_name: Optional[str] = None __attr_searchable__ = ["bucket_name"] @@ -356,7 +356,7 @@ class BlobStorageMetadataV1(SyftObject): __canonical_name__ = "BlobStorageMetadata" __version__ = SYFT_OBJECT_VERSION_1 - type_: Optional[Type[SyftObject]] + type_: Optional[Type[SyftObject]] = None mimetype: str = "bytes" file_size: int @@ -366,7 +366,7 @@ class BlobStorageMetadata(SyftObject): __canonical_name__ = "BlobStorageMetadata" __version__ = SYFT_OBJECT_VERSION_2 - type_: Optional[Type[SyftObject]] + type_: Optional[Type[SyftObject]] = None mimetype: str = "bytes" file_size: int no_lines: Optional[int] = 0 @@ -390,7 +390,7 @@ class CreateBlobStorageEntry(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 id: UID - type_: Optional[Type] + type_: Optional[Type] = None mimetype: str = "bytes" file_size: int extensions: List[str] = [] diff --git a/packages/syft/src/syft/types/datetime.py b/packages/syft/src/syft/types/datetime.py index c03e1433fd0..67576afae7c 100644 --- a/packages/syft/src/syft/types/datetime.py +++ b/packages/syft/src/syft/types/datetime.py @@ -19,7 +19,7 @@ class DateTime(SyftObject): __canonical_name__ = "DateTime" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] + id: Optional[UID] = None utc_timestamp: float @classmethod diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 6c897cd0791..7eae41e43cb 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -99,8 +99,8 @@ class SyftBaseObject(pydantic.BaseModel, SyftHashableObject): __canonical_name__: str __version__: int # data is always versioned - syft_node_location: Optional[UID] - syft_client_verify_key: Optional[SyftVerifyKey] + syft_node_location: Optional[UID] = None + syft_client_verify_key: Optional[SyftVerifyKey] = None def _set_obj_location_(self, node_uid: UID, credentials: SyftVerifyKey): self.syft_node_location = node_uid diff --git a/packages/syft/src/syft/types/transforms.py b/packages/syft/src/syft/types/transforms.py index d4d7a0c38ca..3fb13b99d1b 100644 --- a/packages/syft/src/syft/types/transforms.py +++ b/packages/syft/src/syft/types/transforms.py @@ -28,10 +28,10 @@ class NotNone: class TransformContext(Context): - output: Optional[Dict[str, Any]] - node: Optional[AbstractNode] - credentials: Optional[SyftVerifyKey] - obj: Optional[Any] + output: Optional[Dict[str, Any]] = None + node: Optional[AbstractNode] = None + credentials: Optional[SyftVerifyKey] = None + obj: Optional[Any] = None @staticmethod def from_context(obj: Any, context: Optional[Context] = None) -> Self: From a286c3993f0393bcc1e0b3993f7ce0f88a61b45a Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 27 Feb 2024 16:25:28 +0800 Subject: [PATCH 03/59] Migrate PartialSyftObject to pydantic v2 --- packages/syft/src/syft/serde/third_party.py | 11 +- .../src/syft/service/worker/image_registry.py | 8 +- .../syft/src/syft/types/syft_metaclass.py | 124 ++++-------------- packages/syft/src/syft/types/syft_object.py | 89 ++++++------- 4 files changed, 83 insertions(+), 149 deletions(-) diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py index 2d70250cbe6..1080927c922 100644 --- a/packages/syft/src/syft/serde/third_party.py +++ b/packages/syft/src/syft/serde/third_party.py @@ -20,6 +20,7 @@ import pyarrow as pa import pyarrow.parquet as pq import pydantic +from pydantic._internal._model_construction import ModelMetaclass from pymongo.collection import Collection from result import Err from result import Ok @@ -29,6 +30,7 @@ # relative from ..types.dicttuple import DictTuple from ..types.dicttuple import _Meta as _DictTupleMetaClass +from ..types.syft_metaclass import _EmptyMetaclass from .deserialize import _deserialize as deserialize from .recursive_primitives import _serialize_kv_pairs from .recursive_primitives import deserialize_kv @@ -55,7 +57,7 @@ recursive_serde_register(Ok, serialize_attrs=["_value"]) recursive_serde_register(Err, serialize_attrs=["_value"]) -recursive_serde_register_type(pydantic.main.ModelMetaclass) +recursive_serde_register_type(ModelMetaclass) recursive_serde_register(Result) # exceptions @@ -148,6 +150,13 @@ def _serialize_dicttuple(x: DictTuple) -> bytes: ) +recursive_serde_register( + _EmptyMetaclass, + serialize=serialize_type, + deserialize=deserialize_type, +) + + def serialize_bytes_io(io: BytesIO) -> bytes: io.seek(0) return serialize(io.read(), to_bytes=True) diff --git a/packages/syft/src/syft/service/worker/image_registry.py b/packages/syft/src/syft/service/worker/image_registry.py index 0b218ac8139..bac6b8274a4 100644 --- a/packages/syft/src/syft/service/worker/image_registry.py +++ b/packages/syft/src/syft/service/worker/image_registry.py @@ -3,12 +3,8 @@ from urllib.parse import urlparse # third party -<<<<<<< HEAD -from pydantic import validator -from typing_extensions import Self -======= from pydantic import field_validator ->>>>>>> d9b153bbdf (Update to pydantic v2 first pass with bump-pydantic) +from typing_extensions import Self # relative from ...serde.serializable import serializable @@ -34,7 +30,7 @@ class SyftImageRegistry(SyftObject): @field_validator("url") @classmethod - def validate_url(cls, val: str): + def validate_url(cls, val: str) -> str: if not val: raise ValueError("Invalid Registry URL. Must not be empty") diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index 762213b7ba4..d6bfcc7b544 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -1,113 +1,41 @@ -# Reference: https://github.com/pydantic/pydantic/issues/1223#issuecomment-998160737 - - # stdlib -import inspect -import threading -from typing import Any -from typing import Dict -from typing import Generator -from typing import Tuple -from typing import Type +from typing import TypeVar +from typing import Union +from typing import final # third party -from pydantic.fields import UndefinedType -from pydantic.main import BaseModel -from pydantic.main import ModelField -from pydantic.main import ModelMetaclass +from pydantic import BaseModel +from pydantic._internal._model_construction import ModelMetaclass +from pydantic.fields import Field +from typing_extensions import dataclass_transform # relative -from ..serde.recursive_primitives import recursive_serde_register_type from ..serde.serializable import serializable -TupleGenerator = Generator[Tuple[str, Any], None, None] +T = TypeVar("T", bound=BaseModel) -@serializable() -class Empty: - pass - - -class PartialModelMetaclass(ModelMetaclass): - def __new__( - meta: Type["PartialModelMetaclass"], *args: Any, **kwargs: Any - ) -> "PartialModelMetaclass": - cls = super().__new__(meta, *args, *kwargs) - cls_init = cls.__init__ - # Because the class will be modified temporarily, need to lock __init__ - init_lock = threading.Lock() - # To preserve identical hashes of temporary nested partial models, - # only one instance of each temporary partial class can exist - - def __init__(self: BaseModel, *args: Any, **kwargs: Any) -> None: - with init_lock: - fields = self.__class__.__fields__ - fields_map: Dict[ModelField, Tuple[Any, bool]] = {} +class _EmptyMetaclass(type): + def __repr__(self) -> str: + return self.__name__ - def optionalize( - fields: Dict[str, ModelField], *, restore: bool = False - ) -> None: - for _, field in fields.items(): - if not restore: - if isinstance(field.required, UndefinedType): - raise Exception(f"{field.name} is a required field.") - fields_map[field] = (field.type_, field.required) - # If field has None allowed as a value - # then it becomes a required field. - if field.allow_none and field.name in kwargs: - field.required = True - else: - field.required = False - if inspect.isclass(field.type_) and issubclass( - field.type_, BaseModel - ): - field.populate_validators() - if field.sub_fields is not None: - for sub_field in field.sub_fields: - sub_field.type_ = field.type_ - sub_field.populate_validators() - optionalize(field.type_.__fields__) - else: - # No need to recursively de-optionalize once original types - # are restored - field.type_, field.required = fields_map[field] - if field.sub_fields is not None: - for sub_field in field.sub_fields: - sub_field.type_ = field.type_ - # Make fields and fields of nested model types optional - optionalize(fields) - - # Transform kwargs that are PartialModels to their dict() forms. This - # will exclude `None` (see below) from the dictionary used to construct - # the temporarily-partial model field, avoiding ValidationErrors of - # type type_error.none.not_allowed. - for kwarg, value in kwargs.items(): - if value.__class__.__class__ is PartialModelMetaclass: - kwargs[kwarg] = value.dict() - elif isinstance(value, (tuple, list)): - kwargs[kwarg] = value.__class__( - v.dict() - if v.__class__.__class__ is PartialModelMetaclass - else v - for v in value - ) - - # Validation is performed in __init__, for which all fields are now optional - cls_init(self, *args, **kwargs) - # Restore requiredness - optionalize(fields, restore=True) - - cls.__init__ = __init__ - - def iter_exclude_empty(self) -> TupleGenerator: - for key, value in self.__dict__.items(): - if value is not Empty: - yield key, value +@serializable +@final +class Empty(metaclass=_EmptyMetaclass): + pass - cls.__iter__ = iter_exclude_empty - return cls +@dataclass_transform(kw_only_default=True, field_specifiers=(Field,)) +class PartialModelMetaclass(ModelMetaclass): + def __call__(cls: type[T], *args, **kwargs) -> T: + for field, field_info in cls.model_fields.items(): + if field_info.annotation is not None and not field_info.is_required(): + cls.model_fields[field].annotation = Union[ + field_info.annotation, type[Empty] + ] + cls.model_fields[field].default = Empty + cls.model_rebuild(force=True) -recursive_serde_register_type(PartialModelMetaclass) + return super().__call__(*args, **kwargs) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 7eae41e43cb..8ec7cb4f131 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -29,7 +29,7 @@ from pydantic import ConfigDict from pydantic import EmailStr from pydantic import model_validator -from pydantic.fields import Undefined +from pydantic.fields import PydanticUndefined from result import OkErr from typeguard import check_type @@ -343,6 +343,7 @@ def get_migration_for_version( class SyftObject(SyftBaseObject, SyftObjectRegistry, SyftMigrationRegistry): __canonical_name__ = "SyftObject" __version__ = SYFT_OBJECT_VERSION_1 + model_config = ConfigDict(arbitrary_types_allowed=True) # all objects have a UID @@ -557,7 +558,7 @@ def _syft_set_validate_private_attrs_(self, **kwargs): for attr, decl in self.__private_attributes__.items(): value = kwargs.get(attr, decl.get_default()) var_annotation = self.__annotations__.get(attr) - if value is not Undefined: + if value is not PydanticUndefined: if decl.default_factory: # If the value is defined via PrivateAttr with default factory value = decl.default_factory(value) @@ -586,8 +587,8 @@ def __hash__(self) -> int: def _syft_keys_types_dict(cls, attr_name: str) -> Dict[str, type]: kt_dict = {} for key in getattr(cls, attr_name, []): - if key in cls.__fields__: - type_ = cls.__fields__[key].type_ + if key in cls.model_fields: + type_ = cls.model_fields[key].annotation else: try: method = getattr(cls, key) @@ -601,7 +602,7 @@ def _syft_keys_types_dict(cls, attr_name: str) -> Dict[str, type]: # EmailStr seems to be lost every time the value is set even with a validator # this means the incoming type is str so our validators fail - if type(type_) is type and issubclass(type_, EmailStr): + if isinstance(type_, type) and issubclass(type_, EmailStr): type_ = str kt_dict[key] = type_ return kt_dict @@ -811,45 +812,45 @@ class PartialSyftObject(SyftObject, metaclass=PartialModelMetaclass): __canonical_name__ = "PartialSyftObject" __version__ = SYFT_OBJECT_VERSION_1 - def __init__(self, *args, **kwargs) -> None: - # Filter out Empty values from args and kwargs - args_, kwargs_ = (), {} - for arg in args: - if arg is not Empty: - args_.append(arg) - - for key, val in kwargs.items(): - if val is not Empty: - kwargs_[key] = val - - super().__init__(*args_, **kwargs_) - - fields_with_default = set() - for _field_name, _field in self.__fields__.items(): - if _field.default is not None or _field.allow_none: - fields_with_default.add(_field_name) - - # Fields whose values are set via a validator hook - fields_set_via_validator = [] - - for _field_name in self.__validators__.keys(): - _field = self.__fields__[_field_name] - if self.__dict__[_field_name] is None: - # Since all fields are None, only allow None - # where either none is allowed or default is None - if _field.allow_none or _field.default is None: - fields_set_via_validator.append(_field) - - # Exclude unset fields - unset_fields = ( - set(self.__fields__) - - set(self.__fields_set__) - - set(fields_set_via_validator) - ) - - empty_fields = unset_fields - fields_with_default - for field_name in empty_fields: - self.__dict__[field_name] = Empty + # def __init__(self, *args, **kwargs) -> None: + # # Filter out Empty values from args and kwargs + # args_, kwargs_ = (), {} + # for arg in args: + # if arg is not Empty: + # args_.append(arg) + + # for key, val in kwargs.items(): + # if val is not Empty: + # kwargs_[key] = val + + # super().__init__(*args_, **kwargs_) + + # fields_with_default = set() + # for _field_name, _field in self.model_fields.items(): + # if _field.default is not None or _field.allow_none: + # fields_with_default.add(_field_name) + + # # Fields whose values are set via a validator hook + # fields_set_via_validator = [] + + # for _field_name in self.__validators__.keys(): + # _field = self.model_fields[_field_name] + # if self.__dict__[_field_name] is None: + # # Since all fields are None, only allow None + # # where either none is allowed or default is None + # if _field.allow_none or _field.default is None: + # fields_set_via_validator.append(_field) + + # # Exclude unset fields + # unset_fields = ( + # set(self.model_fields) + # - set(self.model_fields_set) + # - set(fields_set_via_validator) + # ) + + # empty_fields = unset_fields - fields_with_default + # for field_name in empty_fields: + # self.__dict__[field_name] = Empty recursive_serde_register_type(PartialSyftObject) From 9658f6890ba55b1605ec4b00cc619233421031bf Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 14:09:52 +0800 Subject: [PATCH 04/59] Exclude Empty fields from PartialSyftObject.__iter__ --- packages/syft/src/syft/types/syft_metaclass.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index d6bfcc7b544..b3f38e89326 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -1,4 +1,5 @@ # stdlib +import typing from typing import TypeVar from typing import Union from typing import final @@ -12,6 +13,7 @@ # relative from ..serde.serializable import serializable +TupleGenerator = typing.Generator[typing.Tuple[str, typing.Any], None, None] T = TypeVar("T", bound=BaseModel) @@ -38,4 +40,15 @@ def __call__(cls: type[T], *args, **kwargs) -> T: cls.model_rebuild(force=True) + def __iter__(self: T) -> TupleGenerator: + empty_fields = { + field + for field, field_info in self.model_fields.items() + if field_info.default is Empty + } + + yield from ((k, v) for k, v in super.__iter__() if k not in empty_fields) + + cls.__iter__ = __iter__ + return super().__call__(*args, **kwargs) From 91441f646bc4e201a39d524ea3314e5f923c070f Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 14:14:37 +0800 Subject: [PATCH 05/59] Fix pydantic.EmailStr import --- packages/syft/src/syft/serde/third_party.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py index 1080927c922..527de28cad8 100644 --- a/packages/syft/src/syft/serde/third_party.py +++ b/packages/syft/src/syft/serde/third_party.py @@ -189,9 +189,9 @@ def serialize_bytes_io(io: BytesIO) -> bytes: recursive_serde_register(np.core._ufunc_config._unspecified()) recursive_serde_register( - pydantic.networks.EmailStr, + pydantic.EmailStr, serialize=lambda x: x.encode(), - deserialize=lambda x: pydantic.networks.EmailStr(x.decode()), + deserialize=lambda x: pydantic.EmailStr(x.decode()), ) recursive_serde_register( From f869be9d78bda75308155ba9f42d9a89012781c1 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 14:29:34 +0800 Subject: [PATCH 06/59] Fix pydantic v2 recursive serde --- packages/syft/src/syft/serde/recursive.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index eeb1236b749..4dbcef91e5c 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -111,9 +111,14 @@ def recursive_serde_register( # if pydantic object and attrs are provided, the get attrs from __fields__ # cls.__fields__ auto inherits attrs pydantic_fields = [ - f.name - for f in cls.__fields__.values() - if f.outer_type_ not in (Callable, types.FunctionType, types.LambdaType) + field + for field, field_info in cls.model_fields.items() + if ( + field_info.annotation is not None + and hasattr(field_info.annotation, "__origin__") + and field_info.annotation.__origin__ + not in (Callable, types.FunctionType, types.LambdaType) + ) ] attribute_list.update(pydantic_fields) From 012d31c573ad8a8ce618582a7dd7d44a632b9a63 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 14:37:58 +0800 Subject: [PATCH 07/59] Fix MongoStoreConfig --- packages/syft/src/syft/store/mongo_document_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/store/mongo_document_store.py b/packages/syft/src/syft/store/mongo_document_store.py index efdd6496154..c6d148e88f3 100644 --- a/packages/syft/src/syft/store/mongo_document_store.py +++ b/packages/syft/src/syft/store/mongo_document_store.py @@ -859,6 +859,6 @@ class MongoStoreConfig(StoreConfig): client_config: MongoStoreClientConfig store_type: Type[DocumentStore] = MongoDocumentStore db_name: str = "app" - backing_store = MongoBackingStore + backing_store: Type[KeyValueBackingStore] = MongoBackingStore # TODO: should use a distributed lock, with RedisLockingConfig locking_config: LockingConfig = NoLockingConfig() From b59b6170c25b4a8d151b8a39c212429507f8e288 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 14:58:33 +0800 Subject: [PATCH 08/59] Fix pydantic v2 issues --- .../syft/service/action/action_data_empty.py | 5 ++- .../src/syft/service/action/action_object.py | 6 +-- .../syft/src/syft/service/action/numpy.py | 37 ++++++++++--------- .../syft/src/syft/service/action/pandas.py | 13 ++++--- packages/syft/src/syft/service/action/plan.py | 8 +++- .../syft/src/syft/service/code/user_code.py | 6 +-- .../syft/src/syft/service/dataset/dataset.py | 2 +- .../syft/src/syft/service/project/project.py | 4 +- .../syft/src/syft/service/queue/zmq_queue.py | 2 +- packages/syft/src/syft/service/user/user.py | 2 +- packages/syft/src/syft/types/blob_storage.py | 7 ++-- packages/syft/src/syft/types/twin_object.py | 4 +- 12 files changed, 54 insertions(+), 42 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_data_empty.py b/packages/syft/src/syft/service/action/action_data_empty.py index e32f4e339bb..7a1aee8cba5 100644 --- a/packages/syft/src/syft/service/action/action_data_empty.py +++ b/packages/syft/src/syft/service/action/action_data_empty.py @@ -2,7 +2,8 @@ from __future__ import annotations # stdlib -from typing import Optional +from typing import Any +from typing import ClassVar from typing import Type # relative @@ -19,7 +20,7 @@ class ActionDataEmpty(SyftObject): __canonical_name__ = "ActionDataEmpty" __version__ = SYFT_OBJECT_VERSION_1 - syft_internal_type: Optional[Type] = NoneType # type: ignore + syft_internal_type: ClassVar[Type[Any]] = NoneType # type: ignore def __repr__(self) -> str: return f"{type(self).__name__} <{self.syft_internal_type}>" diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index ff066a01da5..5be823f007b 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -598,7 +598,7 @@ class ActionObjectV1(SyftObject): _syft_pre_hooks__: Dict[str, List] = {} _syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS syft_action_data_type: Optional[Type] = None syft_action_data_repr_: Optional[str] = None syft_action_data_str_: Optional[str] = None @@ -632,7 +632,7 @@ class ActionObject(SyftObject): _syft_pre_hooks__: Dict[str, List] = {} _syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS syft_action_data_type: Optional[Type] = None syft_action_data_repr_: Optional[str] = None syft_action_data_str_: Optional[str] = None @@ -1926,7 +1926,7 @@ class AnyActionObject(ActionObject): syft_internal_type: ClassVar[Type[Any]] = NoneType # type: ignore # syft_passthrough_attrs: List[str] = [] syft_dont_wrap_attrs: List[str] = ["__str__", "__repr__", "syft_action_data_str_"] - syft_action_data_str_ = "" + syft_action_data_str_: str = "" def __float__(self) -> float: return float(self.syft_action_data) diff --git a/packages/syft/src/syft/service/action/numpy.py b/packages/syft/src/syft/service/action/numpy.py index dfd43907b92..eefdc9fa1fe 100644 --- a/packages/syft/src/syft/service/action/numpy.py +++ b/packages/syft/src/syft/service/action/numpy.py @@ -2,6 +2,7 @@ from typing import Any from typing import Callable from typing import ClassVar +from typing import List from typing import Type from typing import Union @@ -52,9 +53,9 @@ class NumpyArrayObjectV1(ActionObjectV1, np.lib.mixins.NDArrayOperatorsMixin): __version__ = SYFT_OBJECT_VERSION_1 syft_internal_type: ClassVar[Type[Any]] = np.ndarray - syft_pointer_type = NumpyArrayObjectPointer - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_pointer_type: ClassVar[Type[ActionObjectPointer]] = NumpyArrayObjectPointer + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] # 🔵 TODO 7: Map TPActionObjects and their 3rd Party types like numpy type to these @@ -65,9 +66,9 @@ class NumpyArrayObject(ActionObject, np.lib.mixins.NDArrayOperatorsMixin): __version__ = SYFT_OBJECT_VERSION_2 syft_internal_type: ClassVar[Type[Any]] = np.ndarray - syft_pointer_type = NumpyArrayObjectPointer - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_pointer_type: ClassVar[Type[ActionObjectPointer]] = NumpyArrayObjectPointer + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] # def __eq__(self, other: Any) -> bool: # # 🟡 TODO 8: move __eq__ to a Data / Serdeable type interface on ActionObject @@ -119,9 +120,9 @@ class NumpyScalarObjectV1(ActionObjectV1, np.lib.mixins.NDArrayOperatorsMixin): __canonical_name__ = "NumpyScalarObject" __version__ = SYFT_OBJECT_VERSION_1 - syft_internal_type = np.number - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_internal_type: ClassVar[Type] = np.number + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] @serializable() @@ -129,9 +130,9 @@ class NumpyScalarObject(ActionObject, np.lib.mixins.NDArrayOperatorsMixin): __canonical_name__ = "NumpyScalarObject" __version__ = SYFT_OBJECT_VERSION_2 - syft_internal_type = np.number - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_internal_type: ClassVar[Type] = np.number + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] def __float__(self) -> float: return float(self.syft_action_data) @@ -156,9 +157,9 @@ class NumpyBoolObjectV1(ActionObjectV1, np.lib.mixins.NDArrayOperatorsMixin): __canonical_name__ = "NumpyBoolObject" __version__ = SYFT_OBJECT_VERSION_1 - syft_internal_type = np.bool_ - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_internal_type: ClassVar[Type] = np.bool_ + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] @serializable() @@ -166,9 +167,9 @@ class NumpyBoolObject(ActionObject, np.lib.mixins.NDArrayOperatorsMixin): __canonical_name__ = "NumpyBoolObject" __version__ = SYFT_OBJECT_VERSION_2 - syft_internal_type = np.bool_ - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_internal_type: ClassVar[Type] = np.bool_ + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] @migrate(NumpyBoolObject, NumpyBoolObjectV1) diff --git a/packages/syft/src/syft/service/action/pandas.py b/packages/syft/src/syft/service/action/pandas.py index 2dac63f3b46..171f15ab27e 100644 --- a/packages/syft/src/syft/service/action/pandas.py +++ b/packages/syft/src/syft/service/action/pandas.py @@ -2,6 +2,7 @@ from typing import Any from typing import Callable from typing import ClassVar +from typing import List from typing import Type # third party @@ -27,7 +28,7 @@ class PandasDataFrameObjectV1(ActionObjectV1): __version__ = SYFT_OBJECT_VERSION_1 syft_internal_type: ClassVar[Type[Any]] = DataFrame - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS @serializable() @@ -36,7 +37,7 @@ class PandasDataFrameObject(ActionObject): __version__ = SYFT_OBJECT_VERSION_2 syft_internal_type: ClassVar[Type[Any]] = DataFrame - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS # this is added for instance checks for dataframes # syft_dont_wrap_attrs = ["shape"] @@ -75,8 +76,8 @@ class PandasSeriesObjectV1(ActionObjectV1): __canonical_name__ = "PandasSeriesObject" __version__ = SYFT_OBJECT_VERSION_1 - syft_internal_type = Series - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_internal_type: ClassVar[Type] = Series + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS @serializable() @@ -84,8 +85,8 @@ class PandasSeriesObject(ActionObject): __canonical_name__ = "PandasSeriesObject" __version__ = SYFT_OBJECT_VERSION_2 - syft_internal_type = Series - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_internal_type: ClassVar[Type] = Series + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS # name: Optional[str] = None # syft_dont_wrap_attrs = ["shape"] diff --git a/packages/syft/src/syft/service/action/plan.py b/packages/syft/src/syft/service/action/plan.py index a2a81b6f473..e2e920598cc 100644 --- a/packages/syft/src/syft/service/action/plan.py +++ b/packages/syft/src/syft/service/action/plan.py @@ -21,7 +21,13 @@ class Plan(SyftObject): __canonical_name__ = "Plan" __version__ = SYFT_OBJECT_VERSION_1 - syft_passthrough_attrs = ["inputs", "outputs", "code", "actions", "client"] + syft_passthrough_attrs: List[str] = [ + "inputs", + "outputs", + "code", + "actions", + "client", + ] inputs: Dict[str, ActionObject] outputs: List[ActionObject] diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index ffe8412cf45..61c7a1bb842 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -314,7 +314,7 @@ class UserCodeV2(SyftObject): input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None submit_time: Optional[DateTime] = None - uses_domain = False # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False # tracks if the code calls domain.something, variable is set during parsing nested_requests: Dict[str, str] = {} nested_codes: Optional[Dict[str, Tuple[LinkedObject, Dict]]] = {} @@ -345,7 +345,7 @@ class UserCodeV3(SyftObject): input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None submit_time: Optional[DateTime] = None - uses_domain = False # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False # tracks if the code calls domain.something, variable is set during parsing nested_requests: Dict[str, str] = {} nested_codes: Optional[Dict[str, Tuple[LinkedObject, Dict]]] = {} worker_pool_name: Optional[str] = None @@ -377,7 +377,7 @@ class UserCode(SyftObject): input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None submit_time: Optional[DateTime] - uses_domain = False # tracks if the code calls domain.something, variable is set during parsing + uses_domain: bool = False # tracks if the code calls domain.something, variable is set during parsing nested_codes: Optional[Dict[str, Tuple[LinkedObject, Dict]]] = {} worker_pool_name: Optional[str] diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 12f058cffb3..1306d1a1969 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -639,7 +639,7 @@ class CreateDataset(Dataset): id: Optional[UID] = None created_at: Optional[DateTime] - uploader: Optional[Contributor] # type: ignore[assignment] + uploader: Optional[Contributor] # type: ignore[assignment] model_config = ConfigDict(validate_assignment=True) diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index 2b4b489b745..f0798ffe4ba 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -1226,7 +1226,9 @@ def _repr_html_(self) -> Any: @field_validator("members", mode="before") @classmethod - def verify_members(cls, val: Union[List[SyftClient], List[NodeIdentity]]) -> Union[List[SyftClient], List[NodeIdentity]]: + def verify_members( + cls, val: Union[List[SyftClient], List[NodeIdentity]] + ) -> Union[List[SyftClient], List[NodeIdentity]]: # SyftClients must be logged in by the same emails clients = cls.get_syft_clients(val) if len(clients) > 0: diff --git a/packages/syft/src/syft/service/queue/zmq_queue.py b/packages/syft/src/syft/service/queue/zmq_queue.py index 314522d36c3..de78a9f87d0 100644 --- a/packages/syft/src/syft/service/queue/zmq_queue.py +++ b/packages/syft/src/syft/service/queue/zmq_queue.py @@ -124,7 +124,7 @@ class Worker(SyftBaseModel): # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @field_validator("syft_worker_id", mode="before") @classmethod - def set_syft_worker_id(cls, v): + def set_syft_worker_id(cls, v: Any) -> Any: if isinstance(v, str): return UID(v) return v diff --git a/packages/syft/src/syft/service/user/user.py b/packages/syft/src/syft/service/user/user.py index 4648bfb147b..490408cad6a 100644 --- a/packages/syft/src/syft/service/user/user.py +++ b/packages/syft/src/syft/service/user/user.py @@ -141,7 +141,7 @@ class UserUpdate(PartialSyftObject): __canonical_name__ = "UserUpdate" __version__ = SYFT_OBJECT_VERSION_2 - @field_validator(mode="before") + @field_validator("role", mode="before") @classmethod def str_to_role(cls, v: Any) -> Any: if isinstance(v, str) and hasattr(ServiceRole, v.upper()): diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index a37e37229a7..d68f9214d3c 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -25,6 +25,7 @@ from ..serde import serialize from ..serde.serializable import serializable from ..service.action.action_object import ActionObject +from ..service.action.action_object import ActionObjectPointer from ..service.action.action_object import BASE_PASSTHROUGH_ATTRS from ..service.action.action_types import action_types from ..service.response import SyftError @@ -77,7 +78,7 @@ class BlobFile(SyftObject): syft_blob_storage_entry_id: Optional[UID] = None file_size: Optional[int] = None path: Optional[Path] = None - uploaded = False + uploaded: bool = False __repr_attrs__ = ["id", "file_name"] @@ -210,8 +211,8 @@ class BlobFileObject(ActionObject): __version__ = SYFT_OBJECT_VERSION_1 syft_internal_type: ClassVar[Type[Any]] = BlobFile - syft_pointer_type = BlobFileObjectPointer - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_pointer_type: ClassVar[Type[ActionObjectPointer]] = BlobFileObjectPointer + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS @serializable() diff --git a/packages/syft/src/syft/types/twin_object.py b/packages/syft/src/syft/types/twin_object.py index 22b8b28c76d..43f1918d7cd 100644 --- a/packages/syft/src/syft/types/twin_object.py +++ b/packages/syft/src/syft/types/twin_object.py @@ -40,7 +40,7 @@ class TwinObject(SyftObject): mock_obj: ActionObject mock_obj_id: UID = None # type: ignore - @field_validator(mode="before") + @field_validator("private_obj", mode="before") @classmethod def make_private_obj(cls, v: Any) -> ActionObject: return to_action_object(v) @@ -51,7 +51,7 @@ def make_private_obj_id(self) -> Self: self.private_obj_id = self.private_obj.id return self - @field_validator(mode="before") + @field_validator("mock_obj", mode="before") @classmethod def make_mock_obj(cls, v: Any) -> ActionObject: return to_action_object(v) From 8b01c0f7dc408dfe8843bf236e12a6361fa8ccc6 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 15:01:29 +0800 Subject: [PATCH 09/59] Lint --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b9d577b39e7..6f3176dae92 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ certifi>=2023.7.22 # not directly required, pinned by Snyk to avoid a vulnerability ipython==8.10.0 +jinja2>=3.1.3 # not directly required, pinned by Snyk to avoid a vulnerability markupsafe==2.0.1 pydata-sphinx-theme==0.7.2 pygments>=2.15.0 # not directly required, pinned by Snyk to avoid a vulnerability @@ -10,4 +11,3 @@ sphinx-autoapi==1.8.4 sphinx-code-include==1.1.1 sphinx-copybutton==0.4.0 sphinx-panels==0.6.0 -jinja2>=3.1.3 # not directly required, pinned by Snyk to avoid a vulnerability From 1fdbeb99c2c430d545f36d47cbefe7e9e5bfe09e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 15:26:59 +0800 Subject: [PATCH 10/59] Fix pydantic v2 issues --- packages/grid/backend/grid/logger/config.py | 2 +- packages/syft/src/syft/client/client.py | 6 +- .../syft/src/syft/custom_worker/config.py | 6 +- .../syft/external/oblv/deployment_client.py | 4 +- .../syft/src/syft/service/code/user_code.py | 6 +- packages/syft/src/syft/service/context.py | 4 +- .../syft/src/syft/service/policy/policy.py | 4 +- .../syft/src/syft/service/request/request.py | 14 +-- .../syft/src/syft/types/syft_metaclass.py | 4 +- packages/syft/src/syft/types/syft_object.py | 8 +- packages/syft/src/syft/types/transforms.py | 8 +- packages/syft/src/syft/util/telemetry.py | 102 +++++++++--------- 12 files changed, 85 insertions(+), 83 deletions(-) diff --git a/packages/grid/backend/grid/logger/config.py b/packages/grid/backend/grid/logger/config.py index 7c5ea9ddb41..2a087216683 100644 --- a/packages/grid/backend/grid/logger/config.py +++ b/packages/grid/backend/grid/logger/config.py @@ -40,7 +40,7 @@ class LogConfig(BaseSettings): LOGURU_LEVEL: str = LogLevel.INFO.value LOGURU_SINK: Optional[str] = "/var/log/pygrid/grid.log" - LOGURU_COMPRESSION: Optional[str] + LOGURU_COMPRESSION: Optional[str] = None LOGURU_ROTATION: Union[ Optional[str], Optional[int], diff --git a/packages/syft/src/syft/client/client.py b/packages/syft/src/syft/client/client.py index f9091e4d9d6..a0f59c040b5 100644 --- a/packages/syft/src/syft/client/client.py +++ b/packages/syft/src/syft/client/client.py @@ -136,10 +136,10 @@ class HTTPConnection(NodeConnection): __canonical_name__ = "HTTPConnection" __version__ = SYFT_OBJECT_VERSION_1 - proxy_target_uid: Optional[UID] url: GridURL + proxy_target_uid: Optional[UID] = None routes: Type[Routes] = Routes - session_cache: Optional[Session] + session_cache: Optional[Session] = None @field_validator("url", mode="before") @classmethod @@ -335,7 +335,7 @@ class PythonConnection(NodeConnection): __version__ = SYFT_OBJECT_VERSION_1 node: AbstractNode - proxy_target_uid: Optional[UID] + proxy_target_uid: Optional[UID] = None def with_proxy(self, proxy_target_uid: UID) -> Self: return PythonConnection(node=self.node, proxy_target_uid=proxy_target_uid) diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index f60c19514d4..d43f135e2cc 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -115,7 +115,7 @@ def get_signature(self) -> str: class PrebuiltWorkerConfig(WorkerConfig): # tag that is already built and pushed in some registry tag: str - description: Optional[str] + description: Optional[str] = None def __str__(self) -> str: if self.description: @@ -130,8 +130,8 @@ def set_description(self, description_text: str) -> None: @serializable() class DockerWorkerConfig(WorkerConfig): dockerfile: str - file_name: Optional[str] - description: Optional[str] + file_name: Optional[str] = None + description: Optional[str] = None @field_validator("dockerfile") @classmethod diff --git a/packages/syft/src/syft/external/oblv/deployment_client.py b/packages/syft/src/syft/external/oblv/deployment_client.py index 172baee1cff..0d588211b41 100644 --- a/packages/syft/src/syft/external/oblv/deployment_client.py +++ b/packages/syft/src/syft/external/oblv/deployment_client.py @@ -43,8 +43,8 @@ class OblvMetadata(EnclaveMetadata): """Contains Metadata to connect to Oblivious Enclave""" - deployment_id: Optional[str] - oblv_client: Optional[OblvClient] + deployment_id: Optional[str] = None + oblv_client: Optional[OblvClient] = None @field_validator("deployment_id") @classmethod diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 61c7a1bb842..6d4e2b823e9 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -358,7 +358,7 @@ class UserCode(SyftObject): __version__ = SYFT_OBJECT_VERSION_4 id: UID - node_uid: Optional[UID] + node_uid: Optional[UID] = None user_verify_key: SyftVerifyKey raw_code: str input_policy_type: Union[Type[InputPolicy], UserPolicy] @@ -376,10 +376,10 @@ class UserCode(SyftObject): status: UserCodeStatusCollection input_kwargs: List[str] enclave_metadata: Optional[EnclaveMetadata] = None - submit_time: Optional[DateTime] + submit_time: Optional[DateTime] = None uses_domain: bool = False # tracks if the code calls domain.something, variable is set during parsing nested_codes: Optional[Dict[str, Tuple[LinkedObject, Dict]]] = {} - worker_pool_name: Optional[str] + worker_pool_name: Optional[str] = None __attr_searchable__ = [ "user_verify_key", diff --git a/packages/syft/src/syft/service/context.py b/packages/syft/src/syft/service/context.py index 2ca5d2a337e..6a33426eea1 100644 --- a/packages/syft/src/syft/service/context.py +++ b/packages/syft/src/syft/service/context.py @@ -34,7 +34,7 @@ class AuthedServiceContext(NodeServiceContext): credentials: SyftVerifyKey role: ServiceRole = ServiceRole.NONE - job_id: Optional[UID] + job_id: Optional[UID] = None extra_kwargs: Dict = {} has_execute_permissions: bool = False @@ -68,7 +68,7 @@ class UnauthedServiceContext(NodeServiceContext): __version__ = SYFT_OBJECT_VERSION_1 login_credentials: UserLoginCredentials - node: Optional[AbstractNode] + node: Optional[AbstractNode] = None role: ServiceRole = ServiceRole.NONE diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 3f16c7d1ed5..55970fe1bd7 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -335,7 +335,7 @@ class OutputPolicy(Policy): output_history: List[OutputHistory] = [] output_kwargs: List[str] = [] - node_uid: Optional[UID] + node_uid: Optional[UID] = None output_readers: List[SyftVerifyKey] = [] def apply_output( @@ -455,7 +455,7 @@ class UserPolicy(Policy): __version__ = SYFT_OBJECT_VERSION_1 id: UID - node_uid: Optional[UID] + node_uid: Optional[UID] = None user_verify_key: SyftVerifyKey raw_code: str parsed_code: str diff --git a/packages/syft/src/syft/service/request/request.py b/packages/syft/src/syft/service/request/request.py index 3d5f2f4947f..71d816f7829 100644 --- a/packages/syft/src/syft/service/request/request.py +++ b/packages/syft/src/syft/service/request/request.py @@ -199,7 +199,7 @@ class CreateCustomImageChange(Change): config: WorkerConfig tag: str - registry_uid: Optional[UID] + registry_uid: Optional[UID] = None __repr_attrs__ = ["config", "tag"] @@ -278,8 +278,8 @@ class CreateCustomWorkerPoolChange(Change): pool_name: str num_workers: int - image_uid: Optional[UID] - config: Optional[WorkerConfig] + image_uid: Optional[UID] = None + config: Optional[WorkerConfig] = None __repr_attrs__ = ["pool_name", "num_workers", "image_uid"] @@ -873,11 +873,11 @@ class ObjectMutation(Change): __canonical_name__ = "ObjectMutation" __version__ = SYFT_OBJECT_VERSION_1 - linked_obj: Optional[LinkedObject] + linked_obj: Optional[LinkedObject] = None attr_name: str - value: Optional[Any] + value: Optional[Any] = None match_type: bool - previous_value: Optional[Any] + previous_value: Optional[Any] = None __repr_attrs__ = ["linked_obj", "attr_name"] @@ -945,7 +945,7 @@ class EnumMutation(ObjectMutation): __version__ = SYFT_OBJECT_VERSION_1 enum_type: Type[Enum] - value: Optional[Enum] + value: Optional[Enum] = None match_type: bool = True __repr_attrs__ = ["linked_obj", "attr_name", "value"] diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index b3f38e89326..d2570572d08 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -22,7 +22,7 @@ def __repr__(self) -> str: return self.__name__ -@serializable +@serializable() @final class Empty(metaclass=_EmptyMetaclass): pass @@ -32,7 +32,7 @@ class Empty(metaclass=_EmptyMetaclass): class PartialModelMetaclass(ModelMetaclass): def __call__(cls: type[T], *args, **kwargs) -> T: for field, field_info in cls.model_fields.items(): - if field_info.annotation is not None and not field_info.is_required(): + if field_info.annotation is not None and field_info.is_required(): cls.model_fields[field].annotation = Union[ field_info.annotation, type[Empty] ] diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 8ec7cb4f131..439b950e07c 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -21,6 +21,7 @@ from typing import Tuple from typing import Type from typing import Union +from typing import get_args import warnings # third party @@ -529,7 +530,6 @@ def dict( include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, by_alias: bool = False, - skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, @@ -543,7 +543,6 @@ def dict( include=include, exclude=exclude, by_alias=by_alias, - skip_defaults=skip_defaults, exclude_unset=exclude_unset, exclude_defaults=exclude_defaults, exclude_none=exclude_none, @@ -602,8 +601,11 @@ def _syft_keys_types_dict(cls, attr_name: str) -> Dict[str, type]: # EmailStr seems to be lost every time the value is set even with a validator # this means the incoming type is str so our validators fail - if isinstance(type_, type) and issubclass(type_, EmailStr): + if (isinstance(type_, type) and type_ is EmailStr) or EmailStr in get_args( + type_ + ): type_ = str + kt_dict[key] = type_ return kt_dict diff --git a/packages/syft/src/syft/types/transforms.py b/packages/syft/src/syft/types/transforms.py index 3fb13b99d1b..1cfc48dde5d 100644 --- a/packages/syft/src/syft/types/transforms.py +++ b/packages/syft/src/syft/types/transforms.py @@ -8,7 +8,6 @@ from typing import Union # third party -from pydantic import EmailStr from typing_extensions import Self # relative @@ -38,9 +37,9 @@ def from_context(obj: Any, context: Optional[Context] = None) -> Self: t_context = TransformContext() t_context.obj = obj try: - t_context.output = dict(obj) + t_context.output = obj.to_dict(exclude_empty=True) except Exception: - t_context.output = obj.to_dict() + t_context.output = dict(obj) if hasattr(context, "credentials"): t_context.credentials = context.credentials if hasattr(context, "node"): @@ -140,8 +139,7 @@ def validate_url(context: TransformContext) -> TransformContext: def validate_email(context: TransformContext) -> TransformContext: if context.output["email"] is not None: - context.output["email"] = EmailStr(context.output["email"]) - EmailStr.validate(context.output["email"]) + context.output["email"] = context.output["email"] return context diff --git a/packages/syft/src/syft/util/telemetry.py b/packages/syft/src/syft/util/telemetry.py index c31c5c968d4..3e62409d165 100644 --- a/packages/syft/src/syft/util/telemetry.py +++ b/packages/syft/src/syft/util/telemetry.py @@ -27,53 +27,55 @@ def noop(__func_or_class: T, /, *args: Any, **kwargs: Any) -> T: if not TRACE_MODE: instrument = noop - -try: - print("OpenTelemetry Tracing enabled") - service_name = os.environ.get("SERVICE_NAME", "client") - jaeger_host = os.environ.get("JAEGER_HOST", "localhost") - jaeger_port = int(os.environ.get("JAEGER_PORT", "14268")) - - # third party - from opentelemetry import trace - from opentelemetry.exporter.jaeger.thrift import JaegerExporter - from opentelemetry.sdk.resources import Resource - from opentelemetry.sdk.resources import SERVICE_NAME - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - - trace.set_tracer_provider( - TracerProvider(resource=Resource.create({SERVICE_NAME: service_name})) - ) - jaeger_exporter = JaegerExporter( - # agent_host_name=jaeger_host, - # agent_port=jaeger_port, - collector_endpoint=f"http://{jaeger_host}:{jaeger_port}/api/traces?format=jaeger.thrift", - # udp_split_oversized_batches=True, - ) - - trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(jaeger_exporter)) - - # from opentelemetry.sdk.trace.export import ConsoleSpanExporter - # console_exporter = ConsoleSpanExporter() - # span_processor = BatchSpanProcessor(console_exporter) - # trace.get_tracer_provider().add_span_processor(span_processor) - - # third party - import opentelemetry.instrumentation.requests - - opentelemetry.instrumentation.requests.RequestsInstrumentor().instrument() - - # relative - # from opentelemetry.instrumentation.digma.trace_decorator import ( - # instrument as _instrument, - # ) - # - # until this is merged: - # https://github.com/digma-ai/opentelemetry-instrumentation-digma/pull/41 - from .trace_decorator import instrument as _instrument - - instrument = _instrument -except Exception: # nosec - print("Failed to import opentelemetry") - instrument = noop +else: + try: + print("OpenTelemetry Tracing enabled") + service_name = os.environ.get("SERVICE_NAME", "client") + jaeger_host = os.environ.get("JAEGER_HOST", "localhost") + jaeger_port = int(os.environ.get("JAEGER_PORT", "14268")) + + # third party + from opentelemetry import trace + from opentelemetry.exporter.jaeger.thrift import JaegerExporter + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.resources import SERVICE_NAME + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + trace.set_tracer_provider( + TracerProvider(resource=Resource.create({SERVICE_NAME: service_name})) + ) + jaeger_exporter = JaegerExporter( + # agent_host_name=jaeger_host, + # agent_port=jaeger_port, + collector_endpoint=f"http://{jaeger_host}:{jaeger_port}/api/traces?format=jaeger.thrift", + # udp_split_oversized_batches=True, + ) + + trace.get_tracer_provider().add_span_processor( + BatchSpanProcessor(jaeger_exporter) + ) + + # from opentelemetry.sdk.trace.export import ConsoleSpanExporter + # console_exporter = ConsoleSpanExporter() + # span_processor = BatchSpanProcessor(console_exporter) + # trace.get_tracer_provider().add_span_processor(span_processor) + + # third party + import opentelemetry.instrumentation.requests + + opentelemetry.instrumentation.requests.RequestsInstrumentor().instrument() + + # relative + # from opentelemetry.instrumentation.digma.trace_decorator import ( + # instrument as _instrument, + # ) + # + # until this is merged: + # https://github.com/digma-ai/opentelemetry-instrumentation-digma/pull/41 + from .trace_decorator import instrument as _instrument + + instrument = _instrument + except Exception: # nosec + print("Failed to import opentelemetry") + instrument = noop From 3febf9f7ee98e082ddf80d36ca8e127caef401c2 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 18:58:12 +0800 Subject: [PATCH 11/59] More conservative EmailStr to str --- packages/syft/src/syft/types/syft_object.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 439b950e07c..4d842c9a4cf 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -21,7 +21,6 @@ from typing import Tuple from typing import Type from typing import Union -from typing import get_args import warnings # third party @@ -601,9 +600,7 @@ def _syft_keys_types_dict(cls, attr_name: str) -> Dict[str, type]: # EmailStr seems to be lost every time the value is set even with a validator # this means the incoming type is str so our validators fail - if (isinstance(type_, type) and type_ is EmailStr) or EmailStr in get_args( - type_ - ): + if type_ is EmailStr or type_ == Union[EmailStr, None]: type_ = str kt_dict[key] = type_ From 50d125173773baecbd57bacb5083fd02d7027987 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 19:01:39 +0800 Subject: [PATCH 12/59] Rename metaclass for Empty --- packages/syft/src/syft/serde/third_party.py | 4 ++-- packages/syft/src/syft/types/syft_metaclass.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py index 527de28cad8..deffa73ec80 100644 --- a/packages/syft/src/syft/serde/third_party.py +++ b/packages/syft/src/syft/serde/third_party.py @@ -30,7 +30,7 @@ # relative from ..types.dicttuple import DictTuple from ..types.dicttuple import _Meta as _DictTupleMetaClass -from ..types.syft_metaclass import _EmptyMetaclass +from ..types.syft_metaclass import EmptyType from .deserialize import _deserialize as deserialize from .recursive_primitives import _serialize_kv_pairs from .recursive_primitives import deserialize_kv @@ -151,7 +151,7 @@ def _serialize_dicttuple(x: DictTuple) -> bytes: recursive_serde_register( - _EmptyMetaclass, + EmptyType, serialize=serialize_type, deserialize=deserialize_type, ) diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index d2570572d08..88e791d54f2 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -17,14 +17,14 @@ T = TypeVar("T", bound=BaseModel) -class _EmptyMetaclass(type): +class EmptyType(type): def __repr__(self) -> str: return self.__name__ @serializable() @final -class Empty(metaclass=_EmptyMetaclass): +class Empty(metaclass=EmptyType): pass @@ -34,7 +34,7 @@ def __call__(cls: type[T], *args, **kwargs) -> T: for field, field_info in cls.model_fields.items(): if field_info.annotation is not None and field_info.is_required(): cls.model_fields[field].annotation = Union[ - field_info.annotation, type[Empty] + field_info.annotation, EmptyType ] cls.model_fields[field].default = Empty From 6dda29add35513ad7c5d547dfd1d9fb6040451f0 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 19:02:38 +0800 Subject: [PATCH 13/59] Remove unused code --- packages/syft/src/syft/types/syft_object.py | 40 --------------------- 1 file changed, 40 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 4d842c9a4cf..f75cb1a8972 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -811,46 +811,6 @@ class PartialSyftObject(SyftObject, metaclass=PartialModelMetaclass): __canonical_name__ = "PartialSyftObject" __version__ = SYFT_OBJECT_VERSION_1 - # def __init__(self, *args, **kwargs) -> None: - # # Filter out Empty values from args and kwargs - # args_, kwargs_ = (), {} - # for arg in args: - # if arg is not Empty: - # args_.append(arg) - - # for key, val in kwargs.items(): - # if val is not Empty: - # kwargs_[key] = val - - # super().__init__(*args_, **kwargs_) - - # fields_with_default = set() - # for _field_name, _field in self.model_fields.items(): - # if _field.default is not None or _field.allow_none: - # fields_with_default.add(_field_name) - - # # Fields whose values are set via a validator hook - # fields_set_via_validator = [] - - # for _field_name in self.__validators__.keys(): - # _field = self.model_fields[_field_name] - # if self.__dict__[_field_name] is None: - # # Since all fields are None, only allow None - # # where either none is allowed or default is None - # if _field.allow_none or _field.default is None: - # fields_set_via_validator.append(_field) - - # # Exclude unset fields - # unset_fields = ( - # set(self.model_fields) - # - set(self.model_fields_set) - # - set(fields_set_via_validator) - # ) - - # empty_fields = unset_fields - fields_with_default - # for field_name in empty_fields: - # self.__dict__[field_name] = Empty - recursive_serde_register_type(PartialSyftObject) From e5e6566b6756c97cfc2529535040f32e41062469 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 19:14:08 +0800 Subject: [PATCH 14/59] Fix PartialSyftObject.__iter__ --- .../syft/src/syft/types/syft_metaclass.py | 21 +++---------------- packages/syft/src/syft/types/syft_object.py | 7 +++++++ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index 88e791d54f2..2a3de3518e6 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -1,5 +1,4 @@ # stdlib -import typing from typing import TypeVar from typing import Union from typing import final @@ -13,7 +12,6 @@ # relative from ..serde.serializable import serializable -TupleGenerator = typing.Generator[typing.Tuple[str, typing.Any], None, None] T = TypeVar("T", bound=BaseModel) @@ -31,24 +29,11 @@ class Empty(metaclass=EmptyType): @dataclass_transform(kw_only_default=True, field_specifiers=(Field,)) class PartialModelMetaclass(ModelMetaclass): def __call__(cls: type[T], *args, **kwargs) -> T: - for field, field_info in cls.model_fields.items(): + for field_info in cls.model_fields.values(): if field_info.annotation is not None and field_info.is_required(): - cls.model_fields[field].annotation = Union[ - field_info.annotation, EmptyType - ] - cls.model_fields[field].default = Empty + field_info.annotation = Union[field_info.annotation, EmptyType] + field_info.default = Empty cls.model_rebuild(force=True) - def __iter__(self: T) -> TupleGenerator: - empty_fields = { - field - for field, field_info in self.model_fields.items() - if field_info.default is Empty - } - - yield from ((k, v) for k, v in super.__iter__() if k not in empty_fields) - - cls.__iter__ = __iter__ - return super().__call__(*args, **kwargs) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index f75cb1a8972..220c0294221 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -14,6 +14,7 @@ from typing import Callable from typing import ClassVar from typing import Dict +from typing import Generator from typing import KeysView from typing import List from typing import Optional @@ -805,12 +806,18 @@ def to(self, projection: type, context: Optional[Context] = None) -> Any: return transform(self, context) +TupleGenerator = Generator[Tuple[str, Any], None, None] + + class PartialSyftObject(SyftObject, metaclass=PartialModelMetaclass): """Syft Object to which partial arguments can be provided.""" __canonical_name__ = "PartialSyftObject" __version__ = SYFT_OBJECT_VERSION_1 + def __iter__(self) -> TupleGenerator: + yield from ((k, v) for k, v in super().__iter__() if v is not Empty) + recursive_serde_register_type(PartialSyftObject) From 9b9cac9061700f0b8316278013f35ace7351351e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 19:16:49 +0800 Subject: [PATCH 15/59] Unexpose typevar --- packages/syft/src/syft/types/syft_metaclass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index 2a3de3518e6..74f59fa9336 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -12,7 +12,7 @@ # relative from ..serde.serializable import serializable -T = TypeVar("T", bound=BaseModel) +_T = TypeVar("_T", bound=BaseModel) class EmptyType(type): @@ -28,7 +28,7 @@ class Empty(metaclass=EmptyType): @dataclass_transform(kw_only_default=True, field_specifiers=(Field,)) class PartialModelMetaclass(ModelMetaclass): - def __call__(cls: type[T], *args, **kwargs) -> T: + def __call__(cls: type[_T], *args, **kwargs) -> _T: for field_info in cls.model_fields.values(): if field_info.annotation is not None and field_info.is_required(): field_info.annotation = Union[field_info.annotation, EmptyType] From 4bd0a0f28a5163c916b97578b167777ba282236b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 20:00:54 +0800 Subject: [PATCH 16/59] Make Empty falsy --- packages/syft/src/syft/types/syft_metaclass.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index 74f59fa9336..520c9842246 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -19,6 +19,9 @@ class EmptyType(type): def __repr__(self) -> str: return self.__name__ + def __bool__(self) -> bool: + return False + @serializable() @final From 0ac919547602f9f426bc4d77a8675bab96f53518 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Thu, 29 Feb 2024 21:50:05 +0800 Subject: [PATCH 17/59] Temporarily reset protocol --- .../src/syft/protocol/protocol_version.json | 9 - .../syft/src/syft/protocol/releases/.gitkeep | 0 .../src/syft/protocol/releases/0.8.2.json | 763 ------------------ .../src/syft/protocol/releases/0.8.3.json | 194 ----- .../src/syft/protocol/releases/0.8.4.json | 242 ------ 5 files changed, 1208 deletions(-) create mode 100644 packages/syft/src/syft/protocol/releases/.gitkeep delete mode 100644 packages/syft/src/syft/protocol/releases/0.8.2.json delete mode 100644 packages/syft/src/syft/protocol/releases/0.8.3.json delete mode 100644 packages/syft/src/syft/protocol/releases/0.8.4.json diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 1baa64af0a8..d339efb4005 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -1,13 +1,4 @@ { - "1": { - "release_name": "0.8.2.json" - }, - "2": { - "release_name": "0.8.3.json" - }, - "3": { - "release_name": "0.8.4.json" - }, "dev": { "object_versions": { "UserCode": { diff --git a/packages/syft/src/syft/protocol/releases/.gitkeep b/packages/syft/src/syft/protocol/releases/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/syft/src/syft/protocol/releases/0.8.2.json b/packages/syft/src/syft/protocol/releases/0.8.2.json deleted file mode 100644 index 0ea2060243e..00000000000 --- a/packages/syft/src/syft/protocol/releases/0.8.2.json +++ /dev/null @@ -1,763 +0,0 @@ -{ - "1": { - "object_versions": { - "PartialSyftObject": { - "1": { - "version": 1, - "hash": "008917584d8e1c09015cdbef02f59c0622f48e0618877c1b44425c8846befc13", - "action": "add" - } - }, - "NodeMetadataUpdate": { - "1": { - "version": 1, - "hash": "569d124c23590360bda240c19b53314ccc6204c5d1ab0d2898976a028e002191", - "action": "add" - } - }, - "NodeMetadata": { - "1": { - "version": 1, - "hash": "6bee018894dfdf697ea624740d0bf051750e0b0d8470ced59646f6d8812068ac", - "action": "add" - }, - "2": { - "version": 2, - "hash": "f856169fea72486cd436875ce4411ef935da11eb7c5af48121adfa00d4c0cdb6", - "action": "add" - }, - "3": { - "version": 3, - "hash": "3cc67abf394a805066a88aef0bea15bde609b9ecbe7ec15172eac5e7a0b7ef7c", - "action": "add" - } - }, - "StoreConfig": { - "1": { - "version": 1, - "hash": "17de8875cf590311ddb042140347ffc79d4a85028e504dad178ca4e1237ec861", - "action": "add" - } - }, - "MongoDict": { - "1": { - "version": 1, - "hash": "640734396edae801e1601fe7777710e67685e552acb0244ad8b4f689599baca9", - "action": "add" - } - }, - "MongoStoreConfig": { - "1": { - "version": 1, - "hash": "e52aa382e300b0b69aaa2d80aadb4e3a9a3c02b3c741b71d56f959c4d3891ce5", - "action": "add" - } - }, - "LinkedObject": { - "1": { - "version": 1, - "hash": "824567c6933c095d0e2f6995c8de3581c0fbd2e9e4ead35c8159f7964709c28e", - "action": "add" - } - }, - "BaseConfig": { - "1": { - "version": 1, - "hash": "4e5257080ce615aa4122b02bad8487e4c7d6d0f171ff77abbc9e8cd3e33df89a", - "action": "add" - } - }, - "ServiceConfig": { - "1": { - "version": 1, - "hash": "ca91f59bf045d949d82860f7d52655bfbede4cf6bdc5bae8f847f08a16f05d74", - "action": "add" - } - }, - "LibConfig": { - "1": { - "version": 1, - "hash": "c6ff229aea16874c5d9ae4d1f9e500d13f5cf984bbcee7abd16c5841707a2f78", - "action": "add" - } - }, - "APIEndpoint": { - "1": { - "version": 1, - "hash": "c0e83867b107113e6fed06364ba364c24b2f4af35b15a3869b176318d3be7989", - "action": "add" - } - }, - "LibEndpoint": { - "1": { - "version": 1, - "hash": "153eac6d8990774eebfffaa75a9895e7c4e1a0e09465d5da0baf4c3a3b03369d", - "action": "add" - } - }, - "SignedSyftAPICall": { - "1": { - "version": 1, - "hash": "e66a116de2fa44ebdd0d4c2d7d5a047dedb555fd201a0f431cd8017d9d33a61d", - "action": "add" - } - }, - "SyftAPICall": { - "1": { - "version": 1, - "hash": "014bd1d0933f6070888a313edba239170759de24eae49bf2374c1be4dbe2b4d7", - "action": "add" - } - }, - "SyftAPIData": { - "1": { - "version": 1, - "hash": "db101a75227e34750d7056785a1e87bb2e8ad6604f19c372d0cb6aa437243bf5", - "action": "add" - } - }, - "SyftAPI": { - "1": { - "version": 1, - "hash": "2bba1d9fcf677a58e35bf903de3da22ee4913af138aa3012af9c46b3609579cd", - "action": "add" - } - }, - "User": { - "1": { - "version": 1, - "hash": "078636e64f737e60245b39cf348d30fb006531e80c12b70aa7cf98254e1bb37a", - "action": "add" - } - }, - "UserUpdate": { - "1": { - "version": 1, - "hash": "839dd90aeb611e1dc471c8fd6daf230e913465c0625c6a297079cb7f0a271195", - "action": "add" - } - }, - "UserCreate": { - "1": { - "version": 1, - "hash": "dab78b63544ae91c09f9843c323cb237c0a6fcfeb71c1acf5f738e2fcf5c277f", - "action": "add" - } - }, - "UserSearch": { - "1": { - "version": 1, - "hash": "69d1e10b81c8a4143cf70e4f911d8562732af2458ebbc455ca64542f11373dd1", - "action": "add" - } - }, - "UserView": { - "1": { - "version": 1, - "hash": "63289383fe7e7584652f242a4362ce6e2f0ade52f6416ab6149b326a506b0675", - "action": "add" - } - }, - "UserViewPage": { - "1": { - "version": 1, - "hash": "16dac6209b19a934d286ef1efa874379e0040c324e71023c57d1bc6d2d367171", - "action": "add" - } - }, - "UserPrivateKey": { - "1": { - "version": 1, - "hash": "7cb196587887f0f3bffb298dd9f3b88509e9b2748792bf8dc03bdd0d6b98714a", - "action": "add" - } - }, - "NodeSettingsUpdate": { - "1": { - "version": 1, - "hash": "b6ddc66ff270a3c2c4760e31e1a55d72ed04ccae2d0115ebe2fba6f2bf9bd119", - "action": "add" - } - }, - "NodeSettings": { - "1": { - "version": 1, - "hash": "b662047bb278f4f5db77c102f94b733c3a929839271b3d6b82ea174a60e2aaf0", - "action": "add" - }, - "2": { - "version": 2, - "hash": "29a82afcb006a044b6ae04c6ea8a067d145d28b4210bb038ea9fa86ebde108c8", - "action": "add" - } - }, - "HTTPConnection": { - "1": { - "version": 1, - "hash": "5ee19eaf55ecbe7945ea45924c036ec0f500114a2f64176620961a8c2ec94cdb", - "action": "add" - } - }, - "PythonConnection": { - "1": { - "version": 1, - "hash": "011946fc9af0a6987f5c7bc9b0208b2fae9d65217531430bced7ba542788da1a", - "action": "add" - } - }, - "DateTime": { - "1": { - "version": 1, - "hash": "7e9d89309a10d2110a7ae4f97d8f25a7914853269e8fa0c531630790c1253f17", - "action": "add" - } - }, - "BlobFile": { - "1": { - "version": 1, - "hash": "47ed55183d619c6c624e35412360a41de42833e2c24223c1de1ad12a84fdafc2", - "action": "add" - } - }, - "SecureFilePathLocation": { - "1": { - "version": 1, - "hash": "7febc066e2ee5a3a4a891720afede3f5c155cacc0557662ac4d04bf67b964c6d", - "action": "add" - } - }, - "SeaweedSecureFilePathLocation": { - "1": { - "version": 1, - "hash": "5724a38b1a92b8a55da3d9cc34a720365a6d0c32683acda630fc44067173e201", - "action": "add" - } - }, - "BlobStorageEntry": { - "1": { - "version": 1, - "hash": "9f1b027cce390ee6f71c7a81e7420bb71a477b29c6c62ba74e781a97bc5434e6", - "action": "add" - } - }, - "BlobStorageMetadata": { - "1": { - "version": 1, - "hash": "6888943be3f97186190dd26d7eefbdf29b15c6f2fa459e13608065ebcdb799e2", - "action": "add" - } - }, - "CreateBlobStorageEntry": { - "1": { - "version": 1, - "hash": "61a373336e83645f1b6d78a320323d9ea4ee91b3d87b730cb0608fbfa0072262", - "action": "add" - } - }, - "BlobRetrieval": { - "1": { - "version": 1, - "hash": "a8d7e1d6483e7a9b5a130e837fa398862aa6cbb316cc5f4470450d835755fdd9", - "action": "add" - } - }, - "SyftObjectRetrieval": { - "1": { - "version": 1, - "hash": "7ccc62d5b434d2d438b3df661b4d753b0c7c8d593d451d8b86d364da83998c89", - "action": "add" - } - }, - "BlobRetrievalByURL": { - "1": { - "version": 1, - "hash": "18fd860cb9de296532fc9ff075932e6a4377cc8f043dd88ed4f620517321077d", - "action": "add" - } - }, - "BlobDeposit": { - "1": { - "version": 1, - "hash": "c98e6da658a3be01ead4ea6ee6a4c10046879f0ce0f5fc5f946346671579b229", - "action": "add" - } - }, - "WorkerSettings": { - "1": { - "version": 1, - "hash": "0dcd95422ec8a7c74e45ee68a125084c08f898dc94a13d25fe5a5fd0e4fc5027", - "action": "add" - } - }, - "HTTPNodeRoute": { - "1": { - "version": 1, - "hash": "1901b9f53f9970ce2bd8307ba9f7cafc0e7eba1d2ec82e4014c6120e605e3741", - "action": "add" - } - }, - "PythonNodeRoute": { - "1": { - "version": 1, - "hash": "15711e6e7a1ef726c8e8b5c35a6cb2d30b56ba5213cba489524bf63489e136cf", - "action": "add" - } - }, - "EnclaveMetadata": { - "1": { - "version": 1, - "hash": "39f85e475015e6f860ddcc5fea819423eba2db8f4b7d8e004c05a44d6f8444c6", - "action": "add" - } - }, - "DataSubject": { - "1": { - "version": 1, - "hash": "0b8b049d4627727b444c419f5d6a97b7cb97a433088ebf744c854b6a470dadf1", - "action": "add" - } - }, - "DataSubjectCreate": { - "1": { - "version": 1, - "hash": "5a94f9fcba75c50d78d71222f0235c5fd4d8003ae0db4d74bdbc4d56a99de3aa", - "action": "add" - } - }, - "DataSubjectMemberRelationship": { - "1": { - "version": 1, - "hash": "0a820edc9f1a87387acc3c611fe852752fcb3dab7608058f2bc48211be7bfbd2", - "action": "add" - } - }, - "Contributor": { - "1": { - "version": 1, - "hash": "d1d4f25bb87e59c0414501d3335097de66815c164c9ed5a7850ff8bec69fbcdc", - "action": "add" - } - }, - "MarkdownDescription": { - "1": { - "version": 1, - "hash": "519328a3952049f57004013e4fb00840695b24b8575cad983056412c9c9d9ba6", - "action": "add" - } - }, - "Asset": { - "1": { - "version": 1, - "hash": "24350b8d9597df49999918ad42e0eece1328ea30389311f1e0a420be8f39b8a1", - "action": "add" - } - }, - "CreateAsset": { - "1": { - "version": 1, - "hash": "1b4c71569b8da64258672483bd36dc4aa99a32d4cb519659241d15bc898041a6", - "action": "add" - } - }, - "Dataset": { - "1": { - "version": 1, - "hash": "99ca2fa3e46fd9810222d269fac6accb546f632e94d5d57529016ba5e55af5a8", - "action": "add" - } - }, - "DatasetPageView": { - "1": { - "version": 1, - "hash": "b1de14bb9b6a259648dfc59b6a48fa526116afe50a689c24b8bb36fd0e6a97f8", - "action": "add" - } - }, - "CreateDataset": { - "1": { - "version": 1, - "hash": "3b020d9b8928cbd7e91f41c749ab4c932e19520696a183f2c7cd1312ebb640d1", - "action": "add" - } - }, - "ActionDataEmpty": { - "1": { - "version": 1, - "hash": "89b5912fe5416f922051b8068be6071a03c87a4ab264959de524f1b86e95f028", - "action": "add" - } - }, - "ActionFileData": { - "1": { - "version": 1, - "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", - "action": "add" - } - }, - "Action": { - "1": { - "version": 1, - "hash": "5cf71ee35097f17fbb1dd05096f875211d71cf07161205d7f6a9c11fd49d5272", - "action": "add" - } - }, - "ActionObject": { - "1": { - "version": 1, - "hash": "632446f1415102490c93fafb56dd9eb29d79623bcc5e9f2e6e37c4f63c2c51c3", - "action": "add" - } - }, - "AnyActionObject": { - "1": { - "version": 1, - "hash": "bcb31f847907edc9c95d2d120dc5427854604f40940e3f41cd0474a1820ac65e", - "action": "add" - } - }, - "TwinObject": { - "1": { - "version": 1, - "hash": "c42455586b43724a7421becd99122b787a129798daf6081e96954ecaea228099", - "action": "add" - } - }, - "ExactMatch": { - "1": { - "version": 1, - "hash": "e497e2e2380db72766c5e219e8afd13136d8953933d6f1eaf83b14001e887cde", - "action": "add" - } - }, - "OutputHistory": { - "1": { - "version": 1, - "hash": "4ec6e6efd86a972b474251885151bdfe4ef262562174605e8ab6a8abba1aa867", - "action": "add" - } - }, - "OutputPolicyExecuteCount": { - "1": { - "version": 1, - "hash": "6bb24b3b35e19564c43b838ca3f46ccdeadb6596511917f2d220681a378e439d", - "action": "add" - } - }, - "OutputPolicyExecuteOnce": { - "1": { - "version": 1, - "hash": "32a40fc9966b277528eebc61c01041f3a5447417731954abdaffbb14dabc76bb", - "action": "add" - } - }, - "UserPolicy": { - "1": { - "version": 1, - "hash": "c69b17b1d96cace8b45da6d9639165f2da4aa7ff156b6fd922ac217bf7856d8a", - "action": "add" - } - }, - "SubmitUserPolicy": { - "1": { - "version": 1, - "hash": "96f7f39279fadc70c569b8d48ed4d6420a8132db51e37466d272fda19953554b", - "action": "add" - } - }, - "UserCode": { - "1": { - "version": 1, - "hash": "e14c22686cdc7d1fb2b0d01c0aebdea37e62a61b051677c1d30234214f05cd42", - "action": "add" - } - }, - "SubmitUserCode": { - "1": { - "version": 1, - "hash": "f572d32350d09e25b29572c591029d37a216818618c383094404f84bc9c15dd6", - "action": "add" - } - }, - "UserCodeExecutionResult": { - "1": { - "version": 1, - "hash": "49c32e85e78b7b189a7f13b7e26115ef94fcb0b60b578adcbe2b95e289f63a6e", - "action": "add" - } - }, - "CodeHistory": { - "1": { - "version": 1, - "hash": "a7baae93862ae0aa67675f1617574e31aafb15a9ebff633eb817278a3a867161", - "action": "add" - } - }, - "CodeHistoryView": { - "1": { - "version": 1, - "hash": "0ed1a2a04a962ecbcfa38b0b8a03c1e51e8946a4b80f6bf2557148ce658671ce", - "action": "add" - } - }, - "CodeHistoriesDict": { - "1": { - "version": 1, - "hash": "95288411cd5843834f3273a2fd66a7df2e603e980f4ab1d329f9ab17d5d2f643", - "action": "add" - } - }, - "UsersCodeHistoriesDict": { - "1": { - "version": 1, - "hash": "5e1f389c4565ee8558386dd5c934d81e0c68ab1434f86bb9065976b587ef44d1", - "action": "add" - } - }, - "NodePeer": { - "1": { - "version": 1, - "hash": "7b88de7e38490e2d69f31295137673e7ddabc16ab0e2272ff491f6cea1835d63", - "action": "add" - } - }, - "OnDiskBlobDeposit": { - "1": { - "version": 1, - "hash": "5efc230c1ee65c4626d334aa69ed458c796c45265e546a333844c6c2bcd0e6b0", - "action": "add" - } - }, - "SeaweedFSBlobDeposit": { - "1": { - "version": 1, - "hash": "382a9ac178deed2a9591e1ebbb39f265cbe67027fb93a420d473a4c26b7fda11", - "action": "add" - } - }, - "DictStoreConfig": { - "1": { - "version": 1, - "hash": "256e9c623ce0becd555ddd2a55a0c15514e162786b1549388cef98a92a9b18c9", - "action": "add" - } - }, - "NumpyArrayObject": { - "1": { - "version": 1, - "hash": "dcc7b44fa5ad22ae0bc576948f856c172dac1e9de2bc8e2a302e428f3309a278", - "action": "add" - } - }, - "NumpyScalarObject": { - "1": { - "version": 1, - "hash": "5c1b6b6e8ba88bc79e76646d621489b889fe8f9b9fd59f117d594be18a409633", - "action": "add" - } - }, - "NumpyBoolObject": { - "1": { - "version": 1, - "hash": "a5c822a6a3ca9eefd6a2b68f7fd0bc614fba7995f6bcc30bdc9dc882296b9b16", - "action": "add" - } - }, - "PandasDataframeObject": { - "1": { - "version": 1, - "hash": "35058924b3de2e0a604a92f91f4dd2e3cc0dac80c219d34f360e7cedd52f5f4c", - "action": "add" - } - }, - "PandasSeriesObject": { - "1": { - "version": 1, - "hash": "2a0d8a55f1c27bd8fccd276cbe01bf272c40cab10417d7027273983fed423caa", - "action": "add" - } - }, - "ReplyNotification": { - "1": { - "version": 1, - "hash": "34b2ad522f7406c2486573467d9c7acef5c1063a0d9f2177c3bda2d8c4f87572", - "action": "add" - } - }, - "Notification": { - "1": { - "version": 1, - "hash": "d13981f721fe2b3e2717640ee07dc716c596e4ecd442461665c3fdab0b85bf0e", - "action": "add" - } - }, - "CreateNotification": { - "1": { - "version": 1, - "hash": "b1f459de374fe674f873a4a5f3fb8a8aabe0d83faad84a933f0a77dd1141159a", - "action": "add" - } - }, - "Change": { - "1": { - "version": 1, - "hash": "aefebd1601cf5bfd4817b0db75300a78299cc4949ead735a90873cbd22c8d4bc", - "action": "add" - } - }, - "ChangeStatus": { - "1": { - "version": 1, - "hash": "627f6f8e42cc285336aa6fd4916285d796140f4ff901487b7cb3907ef0f116a6", - "action": "add" - } - }, - "ActionStoreChange": { - "1": { - "version": 1, - "hash": "17b865e75eb3fb2693924fb00ba87a25260be45d55a4eb2184c4ead22d787cbe", - "action": "add" - } - }, - "Request": { - "1": { - "version": 1, - "hash": "e054307eeb7f13683cde9ce7613d5ca2925a13fff7c345b1c9f729a12c955f90", - "action": "add" - } - }, - "RequestInfo": { - "1": { - "version": 1, - "hash": "b76075c138afc0563ce9ac7f6b1131f048951f7486cd516c02736dc1a2a23639", - "action": "add" - } - }, - "RequestInfoFilter": { - "1": { - "version": 1, - "hash": "7103abdc464ae71bb746410f5730f55dd8ed82268aa32bbb0a69e0070488a669", - "action": "add" - } - }, - "SubmitRequest": { - "1": { - "version": 1, - "hash": "96b4ec12beafd9d8a7c97399cb8a23dade4db16d8f521be3fe7b8fec99db5161", - "action": "add" - } - }, - "ObjectMutation": { - "1": { - "version": 1, - "hash": "0ee3dd38d6df0fe9a19d848e8f3aaaf13a6ba86afe3406c239caed6da185651a", - "action": "add" - } - }, - "EnumMutation": { - "1": { - "version": 1, - "hash": "4c02f956ec9b973064972cc57fc8dd9c525e683f93f804642b4e1bfee1b62e57", - "action": "add" - } - }, - "UserCodeStatusChange": { - "1": { - "version": 1, - "hash": "4f5b405cc2b3976ed8f7018df82e873435d9187dff15fa5a23bc85a738969f3f", - "action": "add" - } - }, - "SyftObjectMigrationState": { - "1": { - "version": 1, - "hash": "d3c8126bc15dae4dd243bb035530e3f56cd9e433d403dd6b5f3b45face6d281f", - "action": "add" - } - }, - "ProjectThreadMessage": { - "1": { - "version": 1, - "hash": "1118e935792e8e54103dbf91fa33edbf192a7767d2b1d4526dfa7d4a643cde2e", - "action": "add" - } - }, - "ProjectMessage": { - "1": { - "version": 1, - "hash": "55a3a5171b6949372b4125cc461bf39bc998565e07703804fca6c7ef99695ae4", - "action": "add" - } - }, - "ProjectRequestResponse": { - "1": { - "version": 1, - "hash": "d4c360e845697a0b24695143d0781626cd344cfde43162c90ae90fe67e00ae21", - "action": "add" - } - }, - "ProjectRequest": { - "1": { - "version": 1, - "hash": "514d189df335c68869eea36befcdcafec74bdc682eaf18871fe879e26da4dbb6", - "action": "add" - } - }, - "AnswerProjectPoll": { - "1": { - "version": 1, - "hash": "ff2e1ac7bb764c99d646b96eb3ebfbf9311599b7e3be07aa4a4eb4810bb6dd12", - "action": "add" - } - }, - "ProjectPoll": { - "1": { - "version": 1, - "hash": "b0ac8f1d9c06997374ddbc33fdf1d0af0da15fdb6899f52d91a8574106558964", - "action": "add" - } - }, - "Project": { - "1": { - "version": 1, - "hash": "ec5b7ac1c92808e266f06b175c6ebcd50be81777ad120c02ce8c6074d0004788", - "action": "add" - } - }, - "ProjectSubmit": { - "1": { - "version": 1, - "hash": "0374b37779497d7e0b2ffeabc38d35bfbae2ee762a7674a5a8af75e7c5545e61", - "action": "add" - } - }, - "QueueItem": { - "1": { - "version": 1, - "hash": "5aa94681d9d0715d5b605f9625a54e114927271378cf2ea7245f85c488035e0b", - "action": "add" - } - }, - "ZMQClientConfig": { - "1": { - "version": 1, - "hash": "e6054969b495791569caaf33239039beae3d116e1fe74e9575467c48b9007c45", - "action": "add" - } - }, - "SQLiteStoreConfig": { - "1": { - "version": 1, - "hash": "b656b26c14cf4e97aba702dd62a0927aec7f860c12eed512c2c688e1b7109aa5", - "action": "add" - } - }, - "Plan": { - "1": { - "version": 1, - "hash": "a0bba2b7792c9e08c453e9e256f0ac6e6185610726566bcd50b057ae83b42d9a", - "action": "add" - } - } - } - } -} diff --git a/packages/syft/src/syft/protocol/releases/0.8.3.json b/packages/syft/src/syft/protocol/releases/0.8.3.json deleted file mode 100644 index 0c74b349c3f..00000000000 --- a/packages/syft/src/syft/protocol/releases/0.8.3.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "2": { - "object_versions": { - "Action": { - "2": { - "version": 2, - "hash": "a13b50c4d23bd6deb7896e394f2a20e6cef4c33c5e6f4ee30f19eaffab708f21", - "action": "add" - } - }, - "ActionObject": { - "2": { - "version": 2, - "hash": "577aa1f010b90194958a18ec38ee21db3718bd96d9e036501c6ddeefabedf432", - "action": "add" - } - }, - "AnyActionObject": { - "2": { - "version": 2, - "hash": "002d8be821140befebbc0503e6bc1ef8779094e24e46305e5da5af6eecb56b13", - "action": "add" - } - }, - "BlobFile": { - "2": { - "version": 2, - "hash": "f2b29d28fe81a04bf5e946c819010283a9f98a97d50519358bead773865a2e09", - "action": "add" - } - }, - "BlobFileOBject": { - "1": { - "version": 1, - "hash": "8da2c80ced4f0414c671313c4b63d05846df1e397c763d99d803be86c29755bb", - "action": "add" - } - }, - "BlobStorageEntry": { - "2": { - "version": 2, - "hash": "5472bdd5bdce6d0b561543a6bac70d47bf0c05c141a21450751460cc538d6b55", - "action": "add" - } - }, - "BlobStorageMetadata": { - "2": { - "version": 2, - "hash": "674f4c52a8444289d5ef389b919008860e2b0e7acbaafa774d58e492d5b6741a", - "action": "add" - } - }, - "BlobRetrieval": { - "2": { - "version": 2, - "hash": "4c4fbdb6df5bb9fcbe914a9890bd1c1b6a1b3f382a04cbc8752a5a1b03130111", - "action": "add" - } - }, - "SyftObjectRetrieval": { - "2": { - "version": 2, - "hash": "d9d7a7e1b8843145c9687fd013c9223700285886073547734267e91ac53e0996", - "action": "add" - } - }, - "BlobRetrievalByURL": { - "1": { - "version": 1, - "hash": "18fd860cb9de296532fc9ff075932e6a4377cc8f043dd88ed4f620517321077d", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "8059ee03016c4d74e408dad9529e877f91829672e0cc42d8cfff9c8e14058adc", - "action": "add" - } - }, - "WorkerSettings": { - "2": { - "version": 2, - "hash": "d623a8a0d6c83b26ba49686bd8be10eccb126f54626fef334a85396c3b8a8ed6", - "action": "add" - } - }, - "QueueItem": { - "2": { - "version": 2, - "hash": "9503b878de4b5b7a1793580301353523b7d6219ebd27d38abe598061979b7570", - "action": "add" - } - }, - "ActionQueueItem": { - "1": { - "version": 1, - "hash": "11a43caf9164eb2a5a21f4bcb0ca361d0a5d134bf3c60173f2c502d0d80219de", - "action": "add" - } - }, - "ZMQClientConfig": { - "2": { - "version": 2, - "hash": "0f9bc88d56cd6eed6fc75459d1f914aed840c66e1195b9e41cc501b488fef2ed", - "action": "add" - } - }, - "JobItem": { - "1": { - "version": 1, - "hash": "7b8723861837b0b7e948b2cf9244159d232185f3407dd6bef108346f941ddf6e", - "action": "add" - }, - "2": { - "version": 2, - "hash": "e99cf5a78c6dd3a0adc37af3472c7c21570a9e747985dff540a2b06d24de6446", - "action": "add" - } - }, - "UserCode": { - "2": { - "version": 2, - "hash": "660e1abc15034f525e91ffdd820c2a2179bfddf83b7b9e3ce7823b2efc515c69", - "action": "add" - } - }, - "SubmitUserCode": { - "1": { - "version": 1, - "hash": "f572d32350d09e25b29572c591029d37a216818618c383094404f84bc9c15dd6", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "9b29e060973a3de8d3564a2b7d2bb5c53745aa445bf257576994b613505d7194", - "action": "add" - } - }, - "NumpyArrayObject": { - "2": { - "version": 2, - "hash": "2c631121d9211006edab5620b214dea83e2398bee92244d822227ee316647e22", - "action": "add" - } - }, - "NumpyScalarObject": { - "2": { - "version": 2, - "hash": "0d5d81b9d45c140f6e07b43ed68d31e0ef060d6b4d0431c9b4795997bb35c69d", - "action": "add" - } - }, - "NumpyBoolObject": { - "2": { - "version": 2, - "hash": "24839ba1c88ed833a134124750d5f299abcdf318670315028ed87b254f4578b3", - "action": "add" - } - }, - "PandasDataframeObject": { - "2": { - "version": 2, - "hash": "66729d4ba7a92210d45c5a5c24fbdb4c8e58138a515a7bdb71ac8f6e8b868544", - "action": "add" - } - }, - "PandasSeriesObject": { - "2": { - "version": 2, - "hash": "cb05a714f75b1140a943f56a3622fcc0477b3a1f504cd545a98510959ffe1528", - "action": "add" - } - }, - "UserCodeStatusChange": { - "2": { - "version": 2, - "hash": "d83e0905ae882c824ba8fbbf455cd3881906bf8b2ebbfff07bcf471ef869cedc", - "action": "add" - } - }, - "SyftLog": { - "1": { - "version": 1, - "hash": "bd3f62b8fe4b2718a6380c8f05a93c5c40169fc4ab174db291929298e588429e", - "action": "add" - }, - "2": { - "version": 2, - "hash": "d3ce45794da2e6c4b0cef63b98a553525af50c5d9db42d3d64caef3e7d22b4a9", - "action": "add" - } - } - } - } -} diff --git a/packages/syft/src/syft/protocol/releases/0.8.4.json b/packages/syft/src/syft/protocol/releases/0.8.4.json deleted file mode 100644 index b1581fef20b..00000000000 --- a/packages/syft/src/syft/protocol/releases/0.8.4.json +++ /dev/null @@ -1,242 +0,0 @@ -{ - "3": { - "object_versions": { - "SyftWorkerImage": { - "1": { - "version": 1, - "hash": "2a9585b6a286e24f1a9f3f943d0128730cf853edc549184dc1809d19e1eec54b", - "action": "add" - } - }, - "ActionDataLink": { - "1": { - "version": 1, - "hash": "10bf94e99637695f1ba283f0b10e70743a4ebcb9ee75aefb1a05e6d6e1d21a71", - "action": "add" - } - }, - "ObjectNotReady": { - "1": { - "version": 1, - "hash": "88207988639b11eaca686b6e079616d9caecc3dbc2a8112258e0f39ee5c3e113", - "action": "add" - } - }, - "JobItem": { - "3": { - "version": 3, - "hash": "5b93a59e28574691339d22826d5650969336a2e930b93d6b3fe6d5409ca0cfc4", - "action": "add" - } - }, - "SeaweedSecureFilePathLocation": { - "2": { - "version": 2, - "hash": "5fd63fed2a4efba8c2b6c7a7b5e9b5939181781c331230896aa130b6fd558739", - "action": "add" - } - }, - "AzureSecureFilePathLocation": { - "1": { - "version": 1, - "hash": "1bb15f3f9d7082779f1c9f58de94011487924cb8a8c9c2ec18fd7c161c27fd0e", - "action": "add" - } - }, - "RemoteConfig": { - "1": { - "version": 1, - "hash": "ad7bc4780a8ad52e14ce68601852c93d2fe07bda489809cad7cae786d2461754", - "action": "add" - } - }, - "AzureRemoteConfig": { - "1": { - "version": 1, - "hash": "c05c6caa27db4e385c642536d4b0ecabc0c71e91220d2e6ce21a2761ca68a673", - "action": "add" - } - }, - "BlobRetrievalByURL": { - "2": { - "version": 2, - "hash": "8059ee03016c4d74e408dad9529e877f91829672e0cc42d8cfff9c8e14058adc", - "action": "remove" - }, - "3": { - "version": 3, - "hash": "0b664100ea08413ca4ef04665ca910c2cf9535539617ea4ba33687d05cdfe747", - "action": "add" - } - }, - "QueueItem": { - "3": { - "version": 3, - "hash": "3495f406d2c97050ce86be80c230f49b6b846c63b9a9230cbd6631952f2bad0f", - "action": "add" - } - }, - "ActionQueueItem": { - "2": { - "version": 2, - "hash": "6413ed01e949cac169299a43ce40651f9bf8053e408b6942853f8afa8a693b3d", - "action": "add" - } - }, - "ZMQClientConfig": { - "2": { - "version": 2, - "hash": "0f9bc88d56cd6eed6fc75459d1f914aed840c66e1195b9e41cc501b488fef2ed", - "action": "remove" - }, - "3": { - "version": 3, - "hash": "91ce5953cced58e12c576aa5174d5ca0c91981b01cf42edd5283d347baa3390b", - "action": "add" - } - }, - "SyftWorker": { - "1": { - "version": 1, - "hash": "0d5b367162f3ce55ab090cc1b49bd30e50d4eb144e8431eadc679bd0e743aa70", - "action": "add" - } - }, - "WorkerPool": { - "1": { - "version": 1, - "hash": "250699eb4c452fc427995353d5c5ad6245fb3e9fdac8814f8348784816a0733b", - "action": "add" - } - }, - "SyftImageRegistry": { - "1": { - "version": 1, - "hash": "dc83910c91947e3d9eaa3e6f8592237448f0408668c7cca80450b5fcd54722e1", - "action": "add" - } - }, - "UserCode": { - "3": { - "version": 3, - "hash": "90fcae0f556f375ba1e91d2e345f57241660695c6e2b84c8e311df89d09e6c66", - "action": "add" - } - }, - "SubmitUserCode": { - "3": { - "version": 3, - "hash": "a29160c16d2e2620800d42cdcd9f3637d063a570c477a5d05217a2e64b4bb396", - "action": "add" - } - }, - "CreateCustomImageChange": { - "1": { - "version": 1, - "hash": "bc09dca7995938f3b3a2bd9c8b3c2feffc8484df466144a425cb69cadb2ab635", - "action": "add" - } - }, - "CreateCustomWorkerPoolChange": { - "1": { - "version": 1, - "hash": "86894f8ccc037de61f44f9698fd113ba02c3cf3870a3048c00a46e15dcd1941c", - "action": "add" - } - }, - "JobInfo": { - "1": { - "version": 1, - "hash": "cf26eeac3d9254dfa439917493b816341f8a379a77d182bbecba3b7ed2c1d00a", - "action": "add" - } - }, - "User": { - "1": { - "version": 1, - "hash": "078636e64f737e60245b39cf348d30fb006531e80c12b70aa7cf98254e1bb37a", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "ded970c92f202716ed33a2117cf541789f35fad66bd4b1db39da5026b1d7d0e7", - "action": "add" - } - }, - "UserUpdate": { - "1": { - "version": 1, - "hash": "839dd90aeb611e1dc471c8fd6daf230e913465c0625c6a297079cb7f0a271195", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "32cba8fbd786c575f92e26c31384d282e68e3ebfe5c4b0a0e793820b1228d246", - "action": "add" - } - }, - "UserCreate": { - "1": { - "version": 1, - "hash": "dab78b63544ae91c09f9843c323cb237c0a6fcfeb71c1acf5f738e2fcf5c277f", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "2540188c5aaea866914dccff459df6e0f4727108a503414bb1567ff6297d4646", - "action": "add" - } - }, - "UserView": { - "1": { - "version": 1, - "hash": "63289383fe7e7584652f242a4362ce6e2f0ade52f6416ab6149b326a506b0675", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "e410de583bb15bc5af57acef7be55ea5fc56b5b0fc169daa3869f4203c4d7473", - "action": "add" - } - }, - "BlobFile": { - "2": { - "version": 2, - "hash": "f2b29d28fe81a04bf5e946c819010283a9f98a97d50519358bead773865a2e09", - "action": "remove" - }, - "3": { - "version": 3, - "hash": "8f1710c754bb3b39f546b97fd69c4826291398b247976bbc41fa873af431bca9", - "action": "add" - } - }, - "SyftObjectRetrieval": { - "1": { - "version": 1, - "hash": "7ccc62d5b434d2d438b3df661b4d753b0c7c8d593d451d8b86d364da83998c89", - "action": "remove" - }, - "3": { - "version": 3, - "hash": "952958e9afae007bef3cb89aa15be95dddc4c310e3a8ce4191576f90ac6fcbc8", - "action": "add" - } - }, - "ActionFileData": { - "1": { - "version": 1, - "hash": "1f32d94b75b0a6b4e86cec93d94aa905738219e3e7e75f51dd335ee832a6ed3e", - "action": "remove" - } - }, - "SeaweedFSBlobDeposit": { - "2": { - "version": 2, - "hash": "07d84a95324d95d9c868cd7d1c33c908f77aa468671d76c144586aab672bcbb5", - "action": "add" - } - } - } - } -} From 99820e1d93bb8465692c1b41d0440ecfd303ea56 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 1 Mar 2024 12:09:02 +0800 Subject: [PATCH 18/59] Fix pydantic v2 serde --- packages/syft/src/syft/serde/recursive.py | 4 ++-- packages/syft/src/syft/serde/third_party.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index 4dbcef91e5c..dbc563d68d0 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -113,11 +113,11 @@ def recursive_serde_register( pydantic_fields = [ field for field, field_info in cls.model_fields.items() - if ( + if not ( field_info.annotation is not None and hasattr(field_info.annotation, "__origin__") and field_info.annotation.__origin__ - not in (Callable, types.FunctionType, types.LambdaType) + in (Callable, types.FunctionType, types.LambdaType) ) ] attribute_list.update(pydantic_fields) diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py index deffa73ec80..13adc94c383 100644 --- a/packages/syft/src/syft/serde/third_party.py +++ b/packages/syft/src/syft/serde/third_party.py @@ -31,6 +31,7 @@ from ..types.dicttuple import DictTuple from ..types.dicttuple import _Meta as _DictTupleMetaClass from ..types.syft_metaclass import EmptyType +from ..types.syft_metaclass import PartialModelMetaclass from .deserialize import _deserialize as deserialize from .recursive_primitives import _serialize_kv_pairs from .recursive_primitives import deserialize_kv @@ -56,8 +57,6 @@ # result Ok and Err recursive_serde_register(Ok, serialize_attrs=["_value"]) recursive_serde_register(Err, serialize_attrs=["_value"]) - -recursive_serde_register_type(ModelMetaclass) recursive_serde_register(Result) # exceptions @@ -157,6 +156,10 @@ def _serialize_dicttuple(x: DictTuple) -> bytes: ) +recursive_serde_register_type(ModelMetaclass) +recursive_serde_register(PartialModelMetaclass) + + def serialize_bytes_io(io: BytesIO) -> bytes: io.seek(0) return serialize(io.read(), to_bytes=True) From 4c3427945d07d6ee0c1af8d6cdc4453f60b07b69 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 1 Mar 2024 12:10:18 +0800 Subject: [PATCH 19/59] Fix error message --- packages/syft/src/syft/node/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/node/node.py b/packages/syft/src/syft/node/node.py index fe5872f1aba..a5c89fe0a09 100644 --- a/packages/syft/src/syft/node/node.py +++ b/packages/syft/src/syft/node/node.py @@ -1188,8 +1188,8 @@ def handle_api_call_with_unsigned_result( if api_call.path not in user_config_registry: if ServiceConfigRegistry.path_exists(api_call.path): return SyftError( - message=f"As a `{role}`," - f"you have has no access to: {api_call.path}" + message=f"As a `{role}`, " + f"you have no access to: {api_call.path}" ) else: return SyftError( From d22ef15cf6ef3f81be65cdf5c7e046da8c3bd7a9 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 1 Mar 2024 13:49:00 +0800 Subject: [PATCH 20/59] Fix SyftObject keys for pydantic v2 --- packages/syft/src/syft/types/syft_object.py | 27 +++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 220c0294221..51a298939dd 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -8,8 +8,10 @@ import inspect from inspect import Signature import re +import sys import traceback import types +from types import NoneType from typing import Any from typing import Callable from typing import ClassVar @@ -22,6 +24,8 @@ from typing import Tuple from typing import Type from typing import Union +from typing import get_args +from typing import get_origin import warnings # third party @@ -49,6 +53,12 @@ from .syft_metaclass import PartialModelMetaclass from .uid import UID +if sys.version_info >= (3, 10): + # stdlib + from types import UnionType +else: + UnionType = Union + IntStr = Union[int, str] AbstractSetIntStr = Set[IntStr] MappingIntStrAny = Mapping[IntStr, Any] @@ -78,6 +88,19 @@ ] +def _get_optional_inner_type(x: Any) -> Any: + if get_origin(x) not in (Optional, UnionType, Union): + return x + + args = get_args(x) + + if not any(arg is NoneType for arg in args): + return x + + non_none = [arg for arg in args if arg is not NoneType] + return non_none[0] if len(non_none) == 1 else x + + class SyftHashableObject: __hash_exclude_attrs__ = [] @@ -587,7 +610,7 @@ def _syft_keys_types_dict(cls, attr_name: str) -> Dict[str, type]: kt_dict = {} for key in getattr(cls, attr_name, []): if key in cls.model_fields: - type_ = cls.model_fields[key].annotation + type_ = _get_optional_inner_type(cls.model_fields[key].annotation) else: try: method = getattr(cls, key) @@ -601,7 +624,7 @@ def _syft_keys_types_dict(cls, attr_name: str) -> Dict[str, type]: # EmailStr seems to be lost every time the value is set even with a validator # this means the incoming type is str so our validators fail - if type_ is EmailStr or type_ == Union[EmailStr, None]: + if type_ is EmailStr: type_ = str kt_dict[key] = type_ From 4ed8b374ca08df7bf19aede0912bf004bf6e9a62 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 1 Mar 2024 13:58:34 +0800 Subject: [PATCH 21/59] Fix UnionType and NoneType for python < 3.10 --- packages/syft/src/syft/types/syft_object.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 51a298939dd..dd787e1eaa6 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -11,7 +11,6 @@ import sys import traceback import types -from types import NoneType from typing import Any from typing import Callable from typing import ClassVar @@ -55,9 +54,11 @@ if sys.version_info >= (3, 10): # stdlib + from types import NoneType from types import UnionType else: UnionType = Union + NoneType = type(None) IntStr = Union[int, str] AbstractSetIntStr = Set[IntStr] From bf5f52f7fdc0b19dd183888d3018e041c6f32acc Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 1 Mar 2024 15:44:48 +0800 Subject: [PATCH 22/59] Fix pydantic v2 migration issues (cont.) Co-authored-by: Shubham Gupta --- packages/syft/src/syft/serde/recursive.py | 4 +- .../src/syft/service/action/action_object.py | 101 +++++++++++------- .../syft/src/syft/service/dataset/dataset.py | 4 +- packages/syft/src/syft/types/syft_object.py | 14 ++- packages/syft/src/syft/types/transforms.py | 3 +- .../syft/service/action/action_object_test.py | 10 +- .../syft/transforms/transform_methods_test.py | 6 +- packages/syft/tests/syft/worker_test.py | 8 +- 8 files changed, 89 insertions(+), 61 deletions(-) diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index dbc563d68d0..9efd64e02c0 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -130,7 +130,9 @@ def recursive_serde_register( attribute_list.update(["value"]) exclude_attrs = [] if exclude_attrs is None else exclude_attrs - attribute_list = attribute_list - set(exclude_attrs) + attribute_list = ( + attribute_list - set(exclude_attrs) - {"syft_pre_hooks__", "syft_post_hooks__"} + ) if inheritable_attrs and attribute_list and not is_pydantic: # only set __syft_serializable__ for non-pydantic classes because diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 5be823f007b..96871258160 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -310,6 +310,23 @@ class ActionObjectPointer: "__include_fields__", # pydantic "_calculate_keys", # pydantic "_get_value", # pydantic + "__pydantic_validator__", # pydantic + "__class_vars__", # pydantic + "__private_attributes__", # pydantic + "__signature__", # pydantic + "__pydantic_complete__", # pydantic + "__pydantic_core_schema__", # pydantic + "__pydantic_custom_init__", # pydantic + "__pydantic_decorators__", # pydantic + "__pydantic_generic_metadata__", # pydantic + "__pydantic_parent_namespace__", # pydantic + "__pydantic_post_init__", # pydantic + "__pydantic_root_model__", # pydantic + "__pydantic_serializer__", # pydantic + "__pydantic_validator__", # pydantic + "__pydantic_extra__", # pydantic + "__pydantic_fields_set__", # pydantic + "__pydantic_private__", # pydantic ] dont_wrap_output_attrs = [ "__repr__", @@ -595,8 +612,8 @@ class ActionObjectV1(SyftObject): syft_history_hash: Optional[int] = None syft_internal_type: ClassVar[Type[Any]] syft_node_uid: Optional[UID] = None - _syft_pre_hooks__: Dict[str, List] = {} - _syft_post_hooks__: Dict[str, List] = {} + syft_pre_hooks__: Dict[str, List] = {} + syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS syft_action_data_type: Optional[Type] = None @@ -607,7 +624,7 @@ class ActionObjectV1(SyftObject): syft_created_at: Optional[DateTime] = None -@serializable() +@serializable(without=["syft_pre_hooks__", "syft_post_hooks__"]) class ActionObject(SyftObject): """Action object for remote execution.""" @@ -629,8 +646,8 @@ class ActionObject(SyftObject): syft_history_hash: Optional[int] = None syft_internal_type: ClassVar[Type[Any]] syft_node_uid: Optional[UID] = None - _syft_pre_hooks__: Dict[str, List] = {} - _syft_post_hooks__: Dict[str, List] = {} + syft_pre_hooks__: Dict[str, List] = {} + syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS syft_action_data_type: Optional[Type] = None @@ -1200,13 +1217,13 @@ def from_obj( @classmethod def add_trace_hook(cls) -> bool: return True - # if trace_action_side_effect not in self._syft_pre_hooks__[HOOK_ALWAYS]: - # self._syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) + # if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: + # self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) @classmethod def remove_trace_hook(cls) -> bool: return True - # self._syft_pre_hooks__[HOOK_ALWAYS].pop(trace_action_side_effct, None) + # self.syft_pre_hooks__[HOOK_ALWAYS].pop(trace_action_side_effct, None) def as_empty_data(self) -> ActionDataEmpty: return ActionDataEmpty(syft_internal_type=self.syft_internal_type) @@ -1253,14 +1270,15 @@ def obj_not_ready( ) return res - @staticmethod + @classmethod def empty( # TODO: fix the mypy issue + cls, syft_internal_type: Optional[Type[Any]] = None, id: Optional[UID] = None, syft_lineage_id: Optional[LineageID] = None, syft_resolved: Optional[bool] = True, - ) -> ActionObject: + ) -> Self: """Create an ActionObject from a type, using a ActionDataEmpty object Parameters: @@ -1275,8 +1293,8 @@ def empty( syft_internal_type = ( type(None) if syft_internal_type is None else syft_internal_type ) - empty = ActionDataEmpty(syft_internal_type=syft_internal_type) - res = ActionObject.from_obj( + empty = cls(syft_internal_type=syft_internal_type) + res = cls.from_obj( id=id, syft_lineage_id=syft_lineage_id, syft_action_data=empty, @@ -1287,33 +1305,33 @@ def empty( def __post_init__(self) -> None: """Add pre/post hooks.""" - if HOOK_ALWAYS not in self._syft_pre_hooks__: - self._syft_pre_hooks__[HOOK_ALWAYS] = [] + if HOOK_ALWAYS not in self.syft_pre_hooks__: + self.syft_pre_hooks__[HOOK_ALWAYS] = [] - if HOOK_ON_POINTERS not in self._syft_post_hooks__: - self._syft_pre_hooks__[HOOK_ON_POINTERS] = [] + if HOOK_ON_POINTERS not in self.syft_post_hooks__: + self.syft_pre_hooks__[HOOK_ON_POINTERS] = [] # this should be a list as orders matters for side_effect in [make_action_side_effect]: - if side_effect not in self._syft_pre_hooks__[HOOK_ALWAYS]: - self._syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) + if side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: + self.syft_pre_hooks__[HOOK_ALWAYS].append(side_effect) for side_effect in [send_action_side_effect]: - if side_effect not in self._syft_pre_hooks__[HOOK_ON_POINTERS]: - self._syft_pre_hooks__[HOOK_ON_POINTERS].append(side_effect) + if side_effect not in self.syft_pre_hooks__[HOOK_ON_POINTERS]: + self.syft_pre_hooks__[HOOK_ON_POINTERS].append(side_effect) - if trace_action_side_effect not in self._syft_pre_hooks__[HOOK_ALWAYS]: - self._syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) + if trace_action_side_effect not in self.syft_pre_hooks__[HOOK_ALWAYS]: + self.syft_pre_hooks__[HOOK_ALWAYS].append(trace_action_side_effect) - if HOOK_ALWAYS not in self._syft_post_hooks__: - self._syft_post_hooks__[HOOK_ALWAYS] = [] + if HOOK_ALWAYS not in self.syft_post_hooks__: + self.syft_post_hooks__[HOOK_ALWAYS] = [] - if HOOK_ON_POINTERS not in self._syft_post_hooks__: - self._syft_post_hooks__[HOOK_ON_POINTERS] = [] + if HOOK_ON_POINTERS not in self.syft_post_hooks__: + self.syft_post_hooks__[HOOK_ON_POINTERS] = [] for side_effect in [propagate_node_uid]: - if side_effect not in self._syft_post_hooks__[HOOK_ALWAYS]: - self._syft_post_hooks__[HOOK_ALWAYS].append(side_effect) + if side_effect not in self.syft_post_hooks__[HOOK_ALWAYS]: + self.syft_post_hooks__[HOOK_ALWAYS].append(side_effect) if isinstance(self.syft_action_data_type, ActionObject): raise Exception("Nested ActionObjects", self.syft_action_data_repr_) @@ -1325,16 +1343,16 @@ def _syft_run_pre_hooks__( ) -> Tuple[PreHookContext, Tuple[Any, ...], Dict[str, Any]]: """Hooks executed before the actual call""" result_args, result_kwargs = args, kwargs - if name in self._syft_pre_hooks__: - for hook in self._syft_pre_hooks__[name]: + if name in self.syft_pre_hooks__: + for hook in self.syft_pre_hooks__[name]: result = hook(context, *result_args, **result_kwargs) if result.is_ok(): context, result_args, result_kwargs = result.ok() else: debug(f"Pre-hook failed with {result.err()}") if name not in self._syft_dont_wrap_attrs(): - if HOOK_ALWAYS in self._syft_pre_hooks__: - for hook in self._syft_pre_hooks__[HOOK_ALWAYS]: + if HOOK_ALWAYS in self.syft_pre_hooks__: + for hook in self.syft_pre_hooks__[HOOK_ALWAYS]: result = hook(context, *result_args, **result_kwargs) if result.is_ok(): context, result_args, result_kwargs = result.ok() @@ -1344,8 +1362,8 @@ def _syft_run_pre_hooks__( if self.is_pointer: if name not in self._syft_dont_wrap_attrs(): - if HOOK_ALWAYS in self._syft_pre_hooks__: - for hook in self._syft_pre_hooks__[HOOK_ON_POINTERS]: + if HOOK_ALWAYS in self.syft_pre_hooks__: + for hook in self.syft_pre_hooks__[HOOK_ON_POINTERS]: result = hook(context, *result_args, **result_kwargs) if result.is_ok(): context, result_args, result_kwargs = result.ok() @@ -1360,8 +1378,8 @@ def _syft_run_post_hooks__( ) -> Any: """Hooks executed after the actual call""" new_result = result - if name in self._syft_post_hooks__: - for hook in self._syft_post_hooks__[name]: + if name in self.syft_post_hooks__: + for hook in self.syft_post_hooks__[name]: result = hook(context, name, new_result) if result.is_ok(): new_result = result.ok() @@ -1369,8 +1387,8 @@ def _syft_run_post_hooks__( debug(f"Post hook failed with {result.err()}") if name not in self._syft_dont_wrap_attrs(): - if HOOK_ALWAYS in self._syft_post_hooks__: - for hook in self._syft_post_hooks__[HOOK_ALWAYS]: + if HOOK_ALWAYS in self.syft_post_hooks__: + for hook in self.syft_post_hooks__[HOOK_ALWAYS]: result = hook(context, name, new_result) if result.is_ok(): new_result = result.ok() @@ -1379,8 +1397,8 @@ def _syft_run_post_hooks__( if self.is_pointer: if name not in self._syft_dont_wrap_attrs(): - if HOOK_ALWAYS in self._syft_post_hooks__: - for hook in self._syft_post_hooks__[HOOK_ON_POINTERS]: + if HOOK_ALWAYS in self.syft_post_hooks__: + for hook in self.syft_post_hooks__[HOOK_ON_POINTERS]: result = hook(context, name, new_result) if result.is_ok(): new_result = result.ok() @@ -1654,6 +1672,9 @@ def __getattribute__(self, name: str) -> Any: if name.startswith("_syft") or name.startswith("syft"): return object.__getattribute__(self, name) + if name in passthrough_attrs: + return object.__getattribute__(self, name) + # third party if name in self._syft_passthrough_attrs(): return object.__getattribute__(self, name) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 1306d1a1969..b3a973bfc23 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -638,8 +638,8 @@ class CreateDataset(Dataset): __repr_attrs__ = ["name", "url"] id: Optional[UID] = None - created_at: Optional[DateTime] - uploader: Optional[Contributor] # type: ignore[assignment] + created_at: Optional[DateTime] = None + uploader: Optional[Contributor] = None # type: ignore[assignment] model_config = ConfigDict(validate_assignment=True) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index dd787e1eaa6..a85800a4f8f 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -376,10 +376,12 @@ class SyftObject(SyftBaseObject, SyftObjectRegistry, SyftMigrationRegistry): # # move this to transforms @model_validator(mode="before") - def make_id(cls, values: Dict[str, Any]) -> Dict[str, Any]: - id_field = cls.model_fields["id"] - if "id" not in values and id_field.is_required(): - values["id"] = id_field.annotation() + @classmethod + def make_id(cls, values: Any) -> Any: + if isinstance(values, dict): + id_field = cls.model_fields["id"] + if "id" not in values and id_field.is_required(): + values["id"] = id_field.annotation() return values __attr_searchable__: ClassVar[ @@ -584,7 +586,9 @@ def _syft_set_validate_private_attrs_(self, **kwargs): if value is not PydanticUndefined: if decl.default_factory: # If the value is defined via PrivateAttr with default factory - value = decl.default_factory(value) + value = decl.default_factory() + elif decl.default: + value = decl.default elif var_annotation is not None: # Otherwise validate value against the variable annotation check_type(attr, value, var_annotation) diff --git a/packages/syft/src/syft/types/transforms.py b/packages/syft/src/syft/types/transforms.py index 1cfc48dde5d..7e69a894c6f 100644 --- a/packages/syft/src/syft/types/transforms.py +++ b/packages/syft/src/syft/types/transforms.py @@ -8,6 +8,7 @@ from typing import Union # third party +from pydantic import EmailStr from typing_extensions import Self # relative @@ -139,7 +140,7 @@ def validate_url(context: TransformContext) -> TransformContext: def validate_email(context: TransformContext) -> TransformContext: if context.output["email"] is not None: - context.output["email"] = context.output["email"] + EmailStr._validate(context.output["email"]) return context diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index 131b93d70da..5cd0d2f3880 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -163,12 +163,12 @@ def test_actionobject_make_empty_sanity(dtype: Type): def test_actionobject_hooks_init(orig_obj: Any): obj = ActionObject.from_obj(orig_obj) - assert HOOK_ALWAYS in obj._syft_pre_hooks__ - assert HOOK_ALWAYS in obj._syft_post_hooks__ + assert HOOK_ALWAYS in obj.syft_pre_hooks__ + assert HOOK_ALWAYS in obj.syft_post_hooks__ - assert make_action_side_effect in obj._syft_pre_hooks__[HOOK_ALWAYS] - assert send_action_side_effect in obj._syft_pre_hooks__[HOOK_ON_POINTERS] - assert propagate_node_uid in obj._syft_post_hooks__[HOOK_ALWAYS] + assert make_action_side_effect in obj.syft_pre_hooks__[HOOK_ALWAYS] + assert send_action_side_effect in obj.syft_pre_hooks__[HOOK_ON_POINTERS] + assert propagate_node_uid in obj.syft_post_hooks__[HOOK_ALWAYS] @pytest.mark.parametrize( diff --git a/packages/syft/tests/syft/transforms/transform_methods_test.py b/packages/syft/tests/syft/transforms/transform_methods_test.py index 4010e454ce3..40669b0db5d 100644 --- a/packages/syft/tests/syft/transforms/transform_methods_test.py +++ b/packages/syft/tests/syft/transforms/transform_methods_test.py @@ -5,8 +5,8 @@ from typing import Optional # third party -from pydantic import EmailError from pydantic import EmailStr +from pydantic_core import PydanticCustomError import pytest # syft absolute @@ -421,7 +421,7 @@ def __iter__(self): ) result = validate_email(transform_context) assert isinstance(result, TransformContext) - assert isinstance(result.output["email"], EmailStr) + assert EmailStr._validate(result.output["email"]) assert result.output["email"] == mock_obj.email mock_obj = MockObject(email=faker.name()) @@ -429,5 +429,5 @@ def __iter__(self): obj=mock_obj, context=node_context ) - with pytest.raises(EmailError): + with pytest.raises(PydanticCustomError): validate_email(transform_context) diff --git a/packages/syft/tests/syft/worker_test.py b/packages/syft/tests/syft/worker_test.py index 03772951d71..268e03c10c5 100644 --- a/packages/syft/tests/syft/worker_test.py +++ b/packages/syft/tests/syft/worker_test.py @@ -210,8 +210,8 @@ def post_add(context: Any, name: str, new_result: Any) -> Any: # change return type to sum return Ok(sum(new_result)) - action_object._syft_pre_hooks__["__add__"] = [pre_add] - action_object._syft_post_hooks__["__add__"] = [post_add] + action_object.syft_pre_hooks__["__add__"] = [pre_add] + action_object.syft_post_hooks__["__add__"] = [post_add] result = action_object + action_object x = result.syft_action_data @@ -219,8 +219,8 @@ def post_add(context: Any, name: str, new_result: Any) -> Any: assert y == 18 assert x == y - action_object._syft_pre_hooks__["__add__"] = [] - action_object._syft_post_hooks__["__add__"] = [] + action_object.syft_pre_hooks__["__add__"] = [] + action_object.syft_post_hooks__["__add__"] = [] def test_worker_serde() -> None: From 59e38fdaf10796325d3d2b3f600073fc0a075ab7 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Fri, 1 Mar 2024 16:26:32 +0800 Subject: [PATCH 23/59] Fix ActionObject Co-authored-by: Shubham Gupta --- .../src/syft/service/action/action_object.py | 11 +------ .../syft/service/action/action_object_test.py | 33 +++++++++---------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 96871258160..58fa66dd8e0 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -155,12 +155,7 @@ class Action(SyftObject): @field_validator("result_id", mode="before") @classmethod def make_result_id(cls, v: Any) -> LineageID: - """Generate or reuse a LineageID""" - if isinstance(v, LineageID): - return v - if not isinstance(v, UID): - raise ValueError("result_id type must be a subclass of UID") - return LineageID(v) + return v if isinstance(v, LineageID) else LineageID(v) @property def full_path(self) -> str: @@ -946,12 +941,8 @@ def _syft_prepare_obj_uid(self, obj: Any) -> LineageID: return obj.syft_lineage_id # We got a raw object. We need to create the ActionObject from scratch and save it in the store. - obj_id = UID() - lin_obj_id = Action.make_result_id(obj_id) act_obj = ActionObject.from_obj( obj, - id=obj_id, - syft_lineage_id=lin_obj_id, syft_client_verify_key=self.syft_client_verify_key, syft_node_location=self.syft_node_location, ) diff --git a/packages/syft/tests/syft/service/action/action_object_test.py b/packages/syft/tests/syft/service/action/action_object_test.py index 5cd0d2f3880..d5eefcd7f77 100644 --- a/packages/syft/tests/syft/service/action/action_object_test.py +++ b/packages/syft/tests/syft/service/action/action_object_test.py @@ -25,13 +25,12 @@ from syft.service.action.action_object import propagate_node_uid from syft.service.action.action_object import send_action_side_effect from syft.service.action.action_types import action_type_for_type +from syft.types.uid import LineageID +from syft.types.uid import UID def helper_make_action_obj(orig_obj: Any): - obj_id = Action.make_id(None) - lin_obj_id = Action.make_result_id(obj_id) - - return ActionObject.from_obj(orig_obj, id=obj_id, syft_lineage_id=lin_obj_id) + return ActionObject.from_obj(orig_obj) def helper_make_action_pointers(worker, obj, *args, **kwargs): @@ -62,15 +61,13 @@ def helper_make_action_pointers(worker, obj, *args, **kwargs): def test_action_sanity(path_op: Tuple[str, str]): path, op = path_op - remote_self = Action.make_result_id(None) - result_id = Action.make_result_id(None) + remote_self = LineageID() new_action = Action( path=path, op=op, remote_self=remote_self, args=[], kwargs={}, - result_id=result_id, ) assert new_action is not None assert new_action.full_path == f"{path}.{op}" @@ -99,22 +96,22 @@ def test_actionobject_from_obj_sanity(orig_obj: Any): assert obj.syft_history_hash is not None # with id - obj_id = Action.make_id(None) + obj_id = UID() obj = ActionObject.from_obj(orig_obj, id=obj_id) assert obj.id == obj_id assert obj.syft_history_hash == hash(obj_id) # with id and lineage id - obj_id = Action.make_id(None) - lin_obj_id = Action.make_result_id(obj_id) + obj_id = UID() + lin_obj_id = LineageID(obj_id) obj = ActionObject.from_obj(orig_obj, id=obj_id, syft_lineage_id=lin_obj_id) assert obj.id == obj_id assert obj.syft_history_hash == lin_obj_id.syft_history_hash def test_actionobject_from_obj_fail_id_mismatch(): - obj_id = Action.make_id(None) - lineage_id = Action.make_result_id(None) + obj_id = UID() + lineage_id = LineageID() with pytest.raises(ValueError): ActionObject.from_obj("abc", id=obj_id, syft_lineage_id=lineage_id) @@ -131,14 +128,14 @@ def test_actionobject_make_empty_sanity(dtype: Type): assert obj.syft_history_hash is not None # with id - obj_id = Action.make_id(None) + obj_id = UID() obj = ActionObject.empty(syft_internal_type=syft_type, id=obj_id) assert obj.id == obj_id assert obj.syft_history_hash == hash(obj_id) # with id and lineage id - obj_id = Action.make_id(None) - lin_obj_id = Action.make_result_id(obj_id) + obj_id = UID() + lin_obj_id = LineageID(obj_id) obj = ActionObject.empty( syft_internal_type=syft_type, id=obj_id, syft_lineage_id=lin_obj_id ) @@ -302,7 +299,7 @@ def test_actionobject_hooks_propagate_node_uid_ok(): orig_obj = "abc" op = "capitalize" - obj_id = Action.make_id(None) + obj_id = UID() obj = ActionObject.from_obj(orig_obj) obj.syft_point_to(obj_id) @@ -315,7 +312,7 @@ def test_actionobject_hooks_propagate_node_uid_ok(): def test_actionobject_syft_point_to(): orig_obj = "abc" - obj_id = Action.make_id(None) + obj_id = UID() obj = ActionObject.from_obj(orig_obj) obj.syft_point_to(obj_id) @@ -587,7 +584,7 @@ def test_actionobject_syft_execute_hooks(worker, testcase): ) assert context.result_id is not None - context.obj.syft_node_uid = Action.make_id(None) + context.obj.syft_node_uid = UID() result = obj_pointer._syft_run_post_hooks__(context, name=op, result=obj_pointer) assert result.syft_node_uid == context.obj.syft_node_uid From 0d58c766e12bde02e6eda709d8b4092470598987 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 15:51:02 +0800 Subject: [PATCH 24/59] Fix SyftObject private attr --- packages/syft/src/syft/types/syft_object.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index a85800a4f8f..4565e79a040 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -89,6 +89,12 @@ ] +def _is_optional(x: Any) -> bool: + return get_origin(x) in (Optional, UnionType, Union) and any( + arg is NoneType for arg in get_args(x) + ) + + def _get_optional_inner_type(x: Any) -> Any: if get_origin(x) not in (Optional, UnionType, Union): return x @@ -584,19 +590,12 @@ def _syft_set_validate_private_attrs_(self, **kwargs): value = kwargs.get(attr, decl.get_default()) var_annotation = self.__annotations__.get(attr) if value is not PydanticUndefined: - if decl.default_factory: - # If the value is defined via PrivateAttr with default factory - value = decl.default_factory() - elif decl.default: - value = decl.default - elif var_annotation is not None: + if var_annotation is not None: # Otherwise validate value against the variable annotation check_type(attr, value, var_annotation) setattr(self, attr, value) else: - # check if the private is optional - is_optional_attr = type(None) in getattr(var_annotation, "__args__", []) - if not is_optional_attr: + if not _is_optional(var_annotation): raise ValueError( f"{attr}\n field required (type=value_error.missing)" ) From 122b9d5701eada78b23742bf675a082c4a755fb8 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 16:12:23 +0800 Subject: [PATCH 25/59] Correct PartialModelMetaclass serde --- packages/syft/src/syft/serde/third_party.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/serde/third_party.py b/packages/syft/src/syft/serde/third_party.py index 13adc94c383..fddbb5ae755 100644 --- a/packages/syft/src/syft/serde/third_party.py +++ b/packages/syft/src/syft/serde/third_party.py @@ -157,7 +157,7 @@ def _serialize_dicttuple(x: DictTuple) -> bytes: recursive_serde_register_type(ModelMetaclass) -recursive_serde_register(PartialModelMetaclass) +recursive_serde_register_type(PartialModelMetaclass) def serialize_bytes_io(io: BytesIO) -> bytes: From b4143eedf73824184270c1d75a35ff6a37ce59b8 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 16:38:52 +0800 Subject: [PATCH 26/59] Fix infinite loop cause by model_validator after and validate_assignment=True --- packages/syft/src/syft/service/dataset/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index b3a973bfc23..ef657109c28 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -345,7 +345,7 @@ def __init__(self, description: Optional[str] = "", **data: Any) -> None: def __empty_mock_cannot_be_real(self) -> Self: """set mock_is_real to False whenever mock is None or empty""" if self.mock is None or _is_action_data_empty(self.mock): - self.mock_is_real = False + self.__dict__["mock_is_real"] = False return self From 023e2a69cd84c5c7ebe619c1dfea21cb55e2252b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 17:23:07 +0800 Subject: [PATCH 27/59] Migrate CreateNotification tp pydantic v2 --- .../syft/src/syft/service/notification/notifications.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/notification/notifications.py b/packages/syft/src/syft/service/notification/notifications.py index f47f420484a..81b81fe1ee6 100644 --- a/packages/syft/src/syft/service/notification/notifications.py +++ b/packages/syft/src/syft/service/notification/notifications.py @@ -132,10 +132,10 @@ class CreateNotification(Notification): __canonical_name__ = "CreateNotification" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] - node_uid: Optional[UID] - from_user_verify_key: Optional[SyftVerifyKey] - created_at: Optional[DateTime] + id: Optional[UID] = None + node_uid: Optional[UID] = None + from_user_verify_key: Optional[SyftVerifyKey] = None + created_at: Optional[DateTime] = None def add_msg_creation_time(context: TransformContext) -> TransformContext: From fdd2b21f752011dce6f4097c847e3f0a9e089c81 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 17:24:14 +0800 Subject: [PATCH 28/59] Use correct pydantic v2 attribute --- packages/syft/src/syft/types/syft_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index ae1f010d57c..18db6dfb788 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -437,7 +437,7 @@ def __str__(self) -> str: def _repr_debug_(self) -> str: class_name = get_qualname_for(type(self)) _repr_str = f"class {class_name}:\n" - fields = getattr(self, "__fields__", {}) + fields = getattr(self, "model_fields", {}) for attr in fields.keys(): if attr in DYNAMIC_SYFT_ATTRIBUTES: continue From 3473e29297718e864c1143501d204c3818188f76 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 17:48:48 +0800 Subject: [PATCH 29/59] Revert Job.creation_time validator change --- packages/syft/src/syft/service/job/job_stash.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/service/job/job_stash.py b/packages/syft/src/syft/service/job/job_stash.py index 1269e037ced..332c995e7b6 100644 --- a/packages/syft/src/syft/service/job/job_stash.py +++ b/packages/syft/src/syft/service/job/job_stash.py @@ -10,7 +10,7 @@ from typing import Union # third party -from pydantic import Field +from pydantic import field_validator from pydantic import model_validator from result import Err from result import Ok @@ -114,7 +114,7 @@ class Job(SyftObject): parent_job_id: Optional[UID] = None n_iters: Optional[int] = 0 current_iter: Optional[int] = None - creation_time: str = Field(default_factory=lambda: str(datetime.now())) + creation_time: Optional[str] = None action: Optional[Action] = None job_pid: Optional[int] = None job_worker_id: Optional[UID] = None @@ -124,6 +124,11 @@ class Job(SyftObject): __attr_searchable__ = ["parent_job_id", "job_worker_id", "status", "user_code_id"] __repr_attrs__ = ["id", "result", "resolved", "progress", "creation_time"] + @field_validator("creation_time") + @classmethod + def check_time(cls, time: Any) -> Any: + return str(datetime.now()) if time is None else time + @model_validator(mode="after") def check_user_code_id(self) -> Self: if self.action is not None: From 4954ed43bbc1cef84612edc149be417f608a1dc1 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 18:22:00 +0800 Subject: [PATCH 30/59] Passthrough pydantic.BaseModel attributes in ActionObject --- .../src/syft/service/action/action_object.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 58fa66dd8e0..a969ebe7a64 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -322,6 +322,22 @@ class ActionObjectPointer: "__pydantic_extra__", # pydantic "__pydantic_fields_set__", # pydantic "__pydantic_private__", # pydantic + "model_config", # pydantic + "model_computed_fields", # pydantic + "model_extra", # pydantic + "model_fields", # pydantic + "model_fields_set", # pydantic + "model_construct", # pydantic + "model_copy", # pydantic + "model_dump", # pydantic + "model_dump_json", # pydantic + "model_json_schema", # pydantic + "model_parametrized_name", # pydantic + "model_post_init", # pydantic + "model_rebuild", # pydantic + "model_validate", # pydantic + "model_validate_json", # pydantic + "copy", # pydantic ] dont_wrap_output_attrs = [ "__repr__", From d8e281cc98b75571fe215ab5566b1876df986299 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 18:22:21 +0800 Subject: [PATCH 31/59] mb_size should be float --- packages/syft/src/syft/service/dataset/dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index ef657109c28..4abf448b644 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -471,7 +471,7 @@ class Dataset(SyftObject): description: Optional[MarkdownDescription] = None updated_at: Optional[str] = None requests: Optional[int] = 0 - mb_size: Optional[int] = None + mb_size: Optional[float] = None created_at: DateTime = DateTime.now() uploader: Contributor From 8f6790bb27494d61bba817c59771dfeb0403af83 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 18:57:22 +0800 Subject: [PATCH 32/59] Make ActionDataEmpty.syft_internal_type instance var --- .../src/syft/service/action/action_data_empty.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_data_empty.py b/packages/syft/src/syft/service/action/action_data_empty.py index 7a1aee8cba5..c8f0e143e3d 100644 --- a/packages/syft/src/syft/service/action/action_data_empty.py +++ b/packages/syft/src/syft/service/action/action_data_empty.py @@ -2,8 +2,8 @@ from __future__ import annotations # stdlib -from typing import Any -from typing import ClassVar +import sys +from typing import Optional from typing import Type # relative @@ -12,7 +12,11 @@ from ...types.syft_object import SyftObject from ...types.uid import UID -NoneType = type(None) +if sys.version_info >= (3, 10): + # stdlib + from types import NoneType +else: + NoneType = type(None) @serializable() @@ -20,7 +24,7 @@ class ActionDataEmpty(SyftObject): __canonical_name__ = "ActionDataEmpty" __version__ = SYFT_OBJECT_VERSION_1 - syft_internal_type: ClassVar[Type[Any]] = NoneType # type: ignore + syft_internal_type: Optional[Type] = NoneType # type: ignore def __repr__(self) -> str: return f"{type(self).__name__} <{self.syft_internal_type}>" From 2a0113ef7cd3ae5cf3c6ba19bd079cbcfc337e5b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 19:40:19 +0800 Subject: [PATCH 33/59] Fix ActionObject.empty --- packages/syft/src/syft/service/action/action_object.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index a969ebe7a64..d17d988cb45 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -1300,7 +1300,7 @@ def empty( syft_internal_type = ( type(None) if syft_internal_type is None else syft_internal_type ) - empty = cls(syft_internal_type=syft_internal_type) + empty = ActionDataEmpty(syft_internal_type=syft_internal_type) res = cls.from_obj( id=id, syft_lineage_id=syft_lineage_id, From 4eb7e4eb7319b5e561e481f8b0526d0e934520ee Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 21:45:17 +0800 Subject: [PATCH 34/59] Fix Dataset tests --- .../syft/src/syft/service/dataset/dataset.py | 24 ++++--------------- .../service/dataset/dataset_service_test.py | 16 +++++++------ 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/packages/syft/src/syft/service/dataset/dataset.py b/packages/syft/src/syft/service/dataset/dataset.py index 4abf448b644..7e2a478fb50 100644 --- a/packages/syft/src/syft/service/dataset/dataset.py +++ b/packages/syft/src/syft/service/dataset/dataset.py @@ -16,7 +16,6 @@ import itables import pandas as pd from pydantic import ConfigDict -from pydantic import ValidationError from pydantic import field_validator from pydantic import model_validator from result import Err @@ -341,22 +340,12 @@ class CreateAsset(SyftObject): def __init__(self, description: Optional[str] = "", **data: Any) -> None: super().__init__(**data, description=MarkdownDescription(text=str(description))) - @model_validator(mode="after") - def __empty_mock_cannot_be_real(self) -> Self: - """set mock_is_real to False whenever mock is None or empty""" - if self.mock is None or _is_action_data_empty(self.mock): - self.__dict__["mock_is_real"] = False - - return self - - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @model_validator(mode="after") def __mock_is_real_for_empty_mock_must_be_false(self) -> Self: if self.mock_is_real and ( self.mock is None or _is_action_data_empty(self.mock) ): - raise ValueError("mock_is_real must be False if mock is not provided") + self.__dict__["mock_is_real"] = False return self @@ -400,14 +389,11 @@ def set_mock(self, mock_data: Any, mock_is_real: bool) -> None: if isinstance(mock_data, SyftError): raise SyftException(mock_data) - current_mock = self.mock - self.mock = mock_data + if mock_is_real and (mock_data is None or _is_action_data_empty(mock_data)): + raise SyftException("`mock_is_real` must be False if mock is empty") - try: - self.mock_is_real = mock_is_real - except ValidationError as e: - self.mock = current_mock - raise e + self.mock = mock_data + self.mock_is_real = mock_is_real def no_mock(self) -> None: # relative diff --git a/packages/syft/tests/syft/service/dataset/dataset_service_test.py b/packages/syft/tests/syft/service/dataset/dataset_service_test.py index 8b4f1380961..a60bc653c13 100644 --- a/packages/syft/tests/syft/service/dataset/dataset_service_test.py +++ b/packages/syft/tests/syft/service/dataset/dataset_service_test.py @@ -16,6 +16,7 @@ from syft.service.dataset.dataset import CreateDataset as Dataset from syft.service.dataset.dataset import _ASSET_WITH_NONE_MOCK_ERROR_MESSAGE from syft.service.response import SyftError +from syft.service.response import SyftException from syft.service.response import SyftSuccess from syft.types.twin_object import TwinMode @@ -62,8 +63,9 @@ def make_asset_with_empty_mock() -> dict[str, Any]: def test_asset_without_mock_mock_is_real_must_be_false( asset_without_mock: dict[str, Any], ): - with pytest.raises(ValidationError): - Asset(**asset_without_mock, mock_is_real=True) + asset = Asset(**asset_without_mock, mock_is_real=True) + asset.mock_is_real = True + assert not asset.mock_is_real def test_mock_always_not_real_after_calling_no_mock( @@ -85,8 +87,8 @@ def test_mock_always_not_real_after_set_mock_to_empty( asset.no_mock() assert not asset.mock_is_real - with pytest.raises(ValidationError): - asset.mock_is_real = True + asset.mock_is_real = True + assert not asset.mock_is_real asset.mock = mock() asset.mock_is_real = True @@ -102,8 +104,8 @@ def test_mock_always_not_real_after_set_to_empty( asset.mock = ActionObject.empty() assert not asset.mock_is_real - with pytest.raises(ValidationError): - asset.mock_is_real = True + asset.mock_is_real = True + assert not asset.mock_is_real asset.mock = mock() asset.mock_is_real = True @@ -123,7 +125,7 @@ def test_cannot_set_empty_mock_with_true_mock_is_real( asset = Asset(**asset_with_mock, mock_is_real=True) assert asset.mock_is_real - with pytest.raises(ValidationError): + with pytest.raises(SyftException): asset.set_mock(empty_mock, mock_is_real=True) assert asset.mock is asset_with_mock["mock"] From bb2b675149c462a489e9e40d5aa5424f4e7c1f6a Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sat, 2 Mar 2024 22:43:39 +0800 Subject: [PATCH 35/59] Add None as default value for Optional[UID] --- packages/syft/src/syft/service/policy/policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index 55970fe1bd7..f770235ac9c 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -525,7 +525,7 @@ class SubmitUserPolicy(Policy): __canonical_name__ = "SubmitUserPolicy" __version__ = SYFT_OBJECT_VERSION_1 - id: Optional[UID] + id: Optional[UID] = None code: str class_name: str input_kwargs: List[str] From b04f1abb0188ee8d57e075d7d2ee8dc26ee002e6 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 11:35:22 +0800 Subject: [PATCH 36/59] Debug container node launch --- packages/hagrid/hagrid/orchestra.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py index 8a0e74c06b6..8ee771c0036 100644 --- a/packages/hagrid/hagrid/orchestra.py +++ b/packages/hagrid/hagrid/orchestra.py @@ -9,6 +9,7 @@ import inspect import os import subprocess # nosec +import sys from threading import Thread from typing import Any from typing import Callable @@ -598,7 +599,10 @@ def shutdown( elif "No resource found to remove for project" in land_output: print(f" ✅ {snake_name} Container does not exist") else: - print(f"❌ Unable to remove container: {snake_name} :{land_output}") + print( + f"❌ Unable to remove container: {snake_name} :{land_output}", + file=sys.stderr, + ) @staticmethod def reset(name: str, deployment_type_enum: DeploymentType) -> None: From 26580ce3440d9f683b5fc6554cfd6652ce6caeae Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 12:18:49 +0800 Subject: [PATCH 37/59] Upgrade grid to pydantic v2 --- packages/grid/backend/grid/core/config.py | 43 +++++++++++---------- packages/grid/backend/grid/logger/config.py | 6 +-- packages/syft/setup.cfg | 1 + 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 9f13b7b54fd..9d41011c344 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -2,17 +2,19 @@ import os import secrets from typing import Any -from typing import Dict from typing import List from typing import Optional from typing import Union # third party from pydantic import AnyHttpUrl -from pydantic import BaseSettings from pydantic import EmailStr from pydantic import HttpUrl -from pydantic import validator +from pydantic import field_validator +from pydantic import model_validator +from pydantic_settings import BaseSettings +from pydantic_settings import SettingsConfigDict +from typing_extensions import Self _truthy = {"yes", "y", "true", "t", "on", "1"} _falsy = {"no", "n", "false", "f", "off", "0"} @@ -50,7 +52,8 @@ class Settings(BaseSettings): # "http://localhost:8080", "http://local.dockertoolbox.tiangolo.com"]' BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = [] - @validator("BACKEND_CORS_ORIGINS", pre=True) + @field_validator("BACKEND_CORS_ORIGINS", mode="before") + @classmethod def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str]: if isinstance(v, str) and not v.startswith("["): return [i.strip() for i in v.split(",")] @@ -62,7 +65,8 @@ def assemble_cors_origins(cls, v: Union[str, List[str]]) -> Union[List[str], str SENTRY_DSN: Optional[HttpUrl] = None - @validator("SENTRY_DSN", pre=True) + @field_validator("SENTRY_DSN", mode="before") + @classmethod def sentry_dsn_can_be_blank(cls, v: str) -> Optional[str]: if v is None or len(v) == 0: return None @@ -76,11 +80,12 @@ def sentry_dsn_can_be_blank(cls, v: str) -> Optional[str]: EMAILS_FROM_EMAIL: Optional[EmailStr] = None EMAILS_FROM_NAME: Optional[str] = None - @validator("EMAILS_FROM_NAME") - def get_project_name(cls, v: Optional[str], values: Dict[str, Any]) -> str: - if not v: - return values["PROJECT_NAME"] - return v + @model_validator(mode="after") + def get_project_name(self) -> Self: + if not self.EMAILS_FROM_NAME: + self.EMAILS_FROM_NAME = self.PROJECT_NAME + + return self EMAIL_RESET_TOKEN_EXPIRE_HOURS: int = 48 EMAIL_TEMPLATES_DIR: str = os.path.expandvars( @@ -88,15 +93,15 @@ def get_project_name(cls, v: Optional[str], values: Dict[str, Any]) -> str: ) EMAILS_ENABLED: bool = False - @validator("EMAILS_ENABLED", pre=True) - def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: - return bool( - values.get("SMTP_HOST") - and values.get("SMTP_PORT") - and values.get("EMAILS_FROM_EMAIL") + @model_validator(mode="after") + def get_emails_enabled(self) -> Self: + self.EMAILS_ENABLED = bool( + self.SMTP_HOST and self.SMTP_PORT and self.EMAILS_FROM_EMAIL ) - DEFAULT_ROOT_EMAIL: EmailStr = EmailStr("info@openmined.org") + return self + + DEFAULT_ROOT_EMAIL: EmailStr = "info@openmined.org" DEFAULT_ROOT_PASSWORD: str = "changethis" USERS_OPEN_REGISTRATION: bool = False @@ -149,9 +154,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: True if os.getenv("TEST_MODE", "false").lower() == "true" else False ) ASSOCIATION_TIMEOUT: int = 10 - - class Config: - case_sensitive = True + model_config = SettingsConfigDict(case_sensitive=True) settings = Settings() diff --git a/packages/grid/backend/grid/logger/config.py b/packages/grid/backend/grid/logger/config.py index 2a087216683..9845536bbdd 100644 --- a/packages/grid/backend/grid/logger/config.py +++ b/packages/grid/backend/grid/logger/config.py @@ -11,7 +11,7 @@ from typing import Union # third party -from pydantic import BaseSettings +from pydantic_settings import BaseSettings # LOGURU_LEVEL type for version>3.8 @@ -46,8 +46,8 @@ class LogConfig(BaseSettings): Optional[int], Optional[time], Optional[timedelta], - ] - LOGURU_RETENTION: Union[Optional[str], Optional[int], Optional[timedelta]] + ] = None + LOGURU_RETENTION: Union[Optional[str], Optional[int], Optional[timedelta]] = None LOGURU_COLORIZE: Optional[bool] = True LOGURU_SERIALIZE: Optional[bool] = False LOGURU_BACKTRACE: Optional[bool] = True diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 6a68ddc3b08..fd6cb7bcd80 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -37,6 +37,7 @@ syft = # pycapnp is beta version, update to stable version when available pycapnp==2.0.0b2 pydantic[email]==2.6.0 + pydantic-settings==2.2.1 pymongo==4.6.1 pynacl==1.5.0 pyzmq>=23.2.1,<=25.1.1 From 31e579771f3b4b85cf0fe46e98d3f0b9105a277d Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 13:13:11 +0800 Subject: [PATCH 38/59] Simplify type --- packages/grid/backend/grid/logger/config.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/grid/backend/grid/logger/config.py b/packages/grid/backend/grid/logger/config.py index 9845536bbdd..5f2376a9615 100644 --- a/packages/grid/backend/grid/logger/config.py +++ b/packages/grid/backend/grid/logger/config.py @@ -41,13 +41,8 @@ class LogConfig(BaseSettings): LOGURU_LEVEL: str = LogLevel.INFO.value LOGURU_SINK: Optional[str] = "/var/log/pygrid/grid.log" LOGURU_COMPRESSION: Optional[str] = None - LOGURU_ROTATION: Union[ - Optional[str], - Optional[int], - Optional[time], - Optional[timedelta], - ] = None - LOGURU_RETENTION: Union[Optional[str], Optional[int], Optional[timedelta]] = None + LOGURU_ROTATION: Union[str, int, time, timedelta, None] = None + LOGURU_RETENTION: Union[str, int, timedelta, None] = None LOGURU_COLORIZE: Optional[bool] = True LOGURU_SERIALIZE: Optional[bool] = False LOGURU_BACKTRACE: Optional[bool] = True From 4834147e4b77954f4d5fe50d2a3aeea05629cf0b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 16:29:29 +0800 Subject: [PATCH 39/59] Exclude DYNAMIC_SYFT_ATTRIBUTES from pydantic model_dump() --- packages/syft/src/syft/types/syft_object.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 18db6dfb788..210042ca876 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -32,6 +32,7 @@ import pydantic from pydantic import ConfigDict from pydantic import EmailStr +from pydantic import Field from pydantic import model_validator from pydantic.fields import PydanticUndefined from result import OkErr @@ -130,8 +131,8 @@ class SyftBaseObject(pydantic.BaseModel, SyftHashableObject): __canonical_name__: str __version__: int # data is always versioned - syft_node_location: Optional[UID] = None - syft_client_verify_key: Optional[SyftVerifyKey] = None + syft_node_location: Optional[UID] = Field(default=None, exclude=True) + syft_client_verify_key: Optional[SyftVerifyKey] = Field(default=None, exclude=True) def _set_obj_location_(self, node_uid: UID, credentials: SyftVerifyKey): self.syft_node_location = node_uid From 02a3f7e74dbeb8d2c758e3785dfa1e92b358e65b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 16:44:57 +0800 Subject: [PATCH 40/59] Add JSON encoder for UID to SyftObject so that pydantic model_dump(mode="json") works --- packages/syft/src/syft/types/syft_object.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 210042ca876..04018683377 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -376,7 +376,10 @@ class SyftObject(SyftBaseObject, SyftObjectRegistry, SyftMigrationRegistry): __canonical_name__ = "SyftObject" __version__ = SYFT_OBJECT_VERSION_1 - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict( + arbitrary_types_allowed=True, + json_encoders={UID: str}, + ) # all objects have a UID id: UID From c749e46c60734158338dbd16dee7055386dfd69c Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 16:46:06 +0800 Subject: [PATCH 41/59] Fix settings service test --- .../syft/tests/syft/settings/settings_service_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/syft/tests/syft/settings/settings_service_test.py b/packages/syft/tests/syft/settings/settings_service_test.py index d0a6def902d..d359eb2848f 100644 --- a/packages/syft/tests/syft/settings/settings_service_test.py +++ b/packages/syft/tests/syft/settings/settings_service_test.py @@ -158,8 +158,12 @@ def mock_stash_get_all(root_verify_key) -> Ok: assert response.is_ok() is True assert len(response.ok()) == len(mock_stash_get_all_output) - assert updated_settings == new_settings # the first settings is updated - assert not_updated_settings == settings # the second settings is not updated + assert ( + updated_settings.model_dump() == new_settings.model_dump() + ) # the first settings is updated + assert ( + not_updated_settings.model_dump() == settings.model_dump() + ) # the second settings is not updated def test_settingsservice_update_stash_get_all_fail( From 450e342fba0362852ed3c687e2a2752ad55e38bf Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 20:53:42 +0800 Subject: [PATCH 42/59] Adapt user tests to changes in pydantic v2 comparison --- .../syft/tests/syft/users/user_code_test.py | 4 ++- .../tests/syft/users/user_service_test.py | 33 +++++++++++++++---- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/syft/tests/syft/users/user_code_test.py b/packages/syft/tests/syft/users/user_code_test.py index b4d05f66498..cc601226466 100644 --- a/packages/syft/tests/syft/users/user_code_test.py +++ b/packages/syft/tests/syft/users/user_code_test.py @@ -124,7 +124,9 @@ def func(asset): c for c in request.changes if (isinstance(c, UserCodeStatusChange)) ) - assert status_change.linked_obj.resolve.assets[0] == asset_input + assert status_change.linked_obj.resolve.assets[0].model_dump( + mode="json" + ) == asset_input.model_dump(mode="json") @sy.syft_function() diff --git a/packages/syft/tests/syft/users/user_service_test.py b/packages/syft/tests/syft/users/user_service_test.py index 94e3d7a5deb..b372fa5d690 100644 --- a/packages/syft/tests/syft/users/user_service_test.py +++ b/packages/syft/tests/syft/users/user_service_test.py @@ -172,7 +172,7 @@ def mock_get_by_uid(credentials: SyftVerifyKey, uid: UID) -> Ok: monkeypatch.setattr(user_service.stash, "get_by_uid", mock_get_by_uid) response = user_service.view(authed_context, uid_to_view) assert isinstance(response, UserView) - assert response == expected_output + assert response.model_dump() == expected_output.model_dump() def test_userservice_get_all_success( @@ -192,7 +192,10 @@ def mock_get_all(credentials: SyftVerifyKey) -> Ok: response = user_service.get_all(authed_context) assert isinstance(response, List) assert len(response) == len(expected_output) - assert response == expected_output + assert all( + r.model_dump() == expected.model_dump() + for r, expected in zip(response, expected_output) + ) def test_userservice_get_all_error( @@ -230,17 +233,27 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Union[Ok, Err]: # Search via id response = user_service.search(authed_context, id=guest_user.id) assert isinstance(response, List) - assert response == expected_output + assert all( + r.model_dump() == expected.model_dump() + for r, expected in zip(response, expected_output) + ) + # assert response.model_dump() == expected_output.model_dump() # Search via email response = user_service.search(authed_context, email=guest_user.email) assert isinstance(response, List) - assert response == expected_output + assert all( + r.model_dump() == expected.model_dump() + for r, expected in zip(response, expected_output) + ) # Search via name response = user_service.search(authed_context, name=guest_user.name) assert isinstance(response, List) - assert response == expected_output + assert all( + r.model_dump() == expected.model_dump() + for r, expected in zip(response, expected_output) + ) # Search via verify_key response = user_service.search( @@ -248,14 +261,20 @@ def mock_find_all(credentials: SyftVerifyKey, **kwargs) -> Union[Ok, Err]: verify_key=guest_user.verify_key, ) assert isinstance(response, List) - assert response == expected_output + assert all( + r.model_dump() == expected.model_dump() + for r, expected in zip(response, expected_output) + ) # Search via multiple kwargs response = user_service.search( authed_context, name=guest_user.name, email=guest_user.email ) assert isinstance(response, List) - assert response == expected_output + assert all( + r.model_dump() == expected.model_dump() + for r, expected in zip(response, expected_output) + ) def test_userservice_search_with_invalid_kwargs( From 5d1d00dbd0fea67a19b13bbc0c0f8d5235792633 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 21:07:46 +0800 Subject: [PATCH 43/59] Add missing defaults to UserCreate optional attributes --- packages/syft/src/syft/service/user/user.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/service/user/user.py b/packages/syft/src/syft/service/user/user.py index 490408cad6a..7f7f10327ca 100644 --- a/packages/syft/src/syft/service/user/user.py +++ b/packages/syft/src/syft/service/user/user.py @@ -168,10 +168,10 @@ class UserCreateV1(UserUpdateV1): role: Optional[ServiceRole] = None # type: ignore[assignment] password: str password_verify: Optional[str] = None # type: ignore[assignment] - verify_key: Optional[SyftVerifyKey] - institution: Optional[str] # type: ignore[assignment] - website: Optional[str] # type: ignore[assignment] - created_by: Optional[SyftSigningKey] + verify_key: Optional[SyftVerifyKey] = None + institution: Optional[str] = None # type: ignore[assignment] + website: Optional[str] = None # type: ignore[assignment] + created_by: Optional[SyftSigningKey] = None @serializable() @@ -184,10 +184,10 @@ class UserCreate(UserUpdate): role: Optional[ServiceRole] = None # type: ignore[assignment] password: str password_verify: Optional[str] = None # type: ignore[assignment] - verify_key: Optional[SyftVerifyKey] - institution: Optional[str] # type: ignore[assignment] - website: Optional[str] # type: ignore[assignment] - created_by: Optional[SyftSigningKey] + verify_key: Optional[SyftVerifyKey] = None + institution: Optional[str] = None # type: ignore[assignment] + website: Optional[str] = None # type: ignore[assignment] + created_by: Optional[SyftSigningKey] = None mock_execution_permission: bool = False __repr_attrs__ = ["name", "email"] From 4f5a4abb2555b18bd77f5e09f7939207edef0e1b Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Sun, 3 Mar 2024 21:53:40 +0800 Subject: [PATCH 44/59] Remove unused and replaced Policy.byte_code attribute which was replaced by the byte_code property which was causing a serialization issue. --- packages/syft/src/syft/service/policy/policy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/service/policy/policy.py b/packages/syft/src/syft/service/policy/policy.py index f770235ac9c..89790c2e9ee 100644 --- a/packages/syft/src/syft/service/policy/policy.py +++ b/packages/syft/src/syft/service/policy/policy.py @@ -463,7 +463,6 @@ class UserPolicy(Policy): class_name: str unique_name: str code_hash: str - byte_code: PyCodeObject status: UserPolicyStatus = UserPolicyStatus.SUBMITTED # TODO: fix the mypy issue From c0ef9bfc43af8ae208cd5ae0d609a080278f1ebd Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 4 Mar 2024 00:26:26 +0800 Subject: [PATCH 45/59] Reorder integration tests Move network tests before container workload --- tox.ini | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tox.ini b/tox.ini index 6577929ee25..7439cb216e7 100644 --- a/tox.ini +++ b/tox.ini @@ -328,16 +328,6 @@ commands = exit $return; \ fi' - ; container workload - bash -c 'if [[ "$PYTEST_MODULES" == *"container_workload"* ]]; then \ - echo "Starting Container Workload test"; date; \ - pytest tests/integration -m container_workload -p no:randomly --co; \ - pytest tests/integration -m container_workload -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \ - return=$?; \ - echo "Finished container workload"; date; \ - exit $return; \ - fi' - ; network bash -c 'if [[ "$PYTEST_MODULES" == *"network"* ]]; then \ echo "Starting network"; date; \ @@ -348,6 +338,16 @@ commands = exit $return; \ fi' + ; container workload + bash -c 'if [[ "$PYTEST_MODULES" == *"container_workload"* ]]; then \ + echo "Starting Container Workload test"; date; \ + pytest tests/integration -m container_workload -p no:randomly --co; \ + pytest tests/integration -m container_workload -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \ + return=$?; \ + echo "Finished container workload"; date; \ + exit $return; \ + fi' + ; shutdown bash -c "echo Killing Nodes; date" bash -c 'HAGRID_ART=false hagrid land all --force' @@ -1153,4 +1153,4 @@ commands = fi" - pytest notebooks/api/0.8 --nbmake -p no:randomly -vvvv --nbmake-timeout=1000 -k '{env:EXCLUDE_NOTEBOOKS:}' \ No newline at end of file + pytest notebooks/api/0.8 --nbmake -p no:randomly -vvvv --nbmake-timeout=1000 -k '{env:EXCLUDE_NOTEBOOKS:}' From 927bf3e0108feaf5e9eb0781278f20b4f8cd7bc6 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 4 Mar 2024 02:21:14 +0800 Subject: [PATCH 46/59] Use model_rebuild instead of the deprecated update_forward_ref --- packages/syft/src/syft/client/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 8503ebb9be1..8a4a5638bcf 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -1063,5 +1063,5 @@ def validate_callable_args_and_kwargs( return _valid_args, _valid_kwargs -RemoteFunction.update_forward_refs() -RemoteUserCodeFunction.update_forward_refs() +RemoteFunction.model_rebuild(force=True) +RemoteUserCodeFunction.model_rebuild(force=True) From 9c599d7bed74033f1f7bc5c4876870cc23f0965a Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 4 Mar 2024 14:26:07 +0800 Subject: [PATCH 47/59] Fix action graph in-memory config test Co-authored-by: khoaguin --- packages/syft/tests/syft/action_graph/action_graph_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/syft/tests/syft/action_graph/action_graph_test.py b/packages/syft/tests/syft/action_graph/action_graph_test.py index d01466afa75..8e7e235105a 100644 --- a/packages/syft/tests/syft/action_graph/action_graph_test.py +++ b/packages/syft/tests/syft/action_graph/action_graph_test.py @@ -30,7 +30,7 @@ from syft.service.action.action_object import Action from syft.service.action.action_object import ActionObject from syft.store.document_store import QueryKeys -from syft.store.locks import NoLockingConfig +from syft.store.locks import ThreadingLockingConfig from syft.types.datetime import DateTime from syft.types.syft_metaclass import Empty from syft.types.uid import UID @@ -164,7 +164,7 @@ def test_in_memory_store_client_config() -> None: def test_in_memory_graph_config() -> None: store_config = InMemoryGraphConfig() default_client_conf = InMemoryStoreClientConfig() - locking_config = NoLockingConfig() + locking_config = ThreadingLockingConfig() assert store_config.client_config == default_client_conf assert store_config.store_type == NetworkXBackingStore From 734be88d1f20fd7717051fa02884908b425436a1 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 4 Mar 2024 14:32:45 +0800 Subject: [PATCH 48/59] Fix action graph service tests Co-authored-by: khoaguin --- .../syft/tests/syft/action_graph/action_graph_service_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/syft/tests/syft/action_graph/action_graph_service_test.py b/packages/syft/tests/syft/action_graph/action_graph_service_test.py index 22199150ad3..3cac37f975e 100644 --- a/packages/syft/tests/syft/action_graph/action_graph_service_test.py +++ b/packages/syft/tests/syft/action_graph/action_graph_service_test.py @@ -105,7 +105,6 @@ def test_action_graph_service_add_action_no_mutagen( assert action_node.status == ExecutionStatus.PROCESSING assert action_node.retry == 0 assert isinstance(action_node.created_at, DateTime) - assert action_node.updated_at is None assert action_node.user_verify_key == authed_context.credentials assert action_node.is_mutated is False assert action_node.is_mutagen is False @@ -117,7 +116,6 @@ def test_action_graph_service_add_action_no_mutagen( assert result_node.status == ExecutionStatus.PROCESSING assert result_node.retry == 0 assert isinstance(result_node.created_at, DateTime) - assert result_node.updated_at is None assert result_node.user_verify_key == authed_context.credentials assert result_node.is_mutated is False assert result_node.is_mutagen is False @@ -168,7 +166,6 @@ def test_action_graph_service_add_action_mutagen( assert result_node.id == action.result_id.id assert action_node.type == NodeType.ACTION assert result_node.type == NodeType.ACTION_OBJECT - assert result_node.updated_at is None assert result_node.is_mutated is False assert result_node.is_mutagen is False assert result_node.next_mutagen_node is None From 2ffe0a8beb9b6621416a31a5644660fde6e4fe54 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 4 Mar 2024 16:44:05 +0800 Subject: [PATCH 49/59] Fix ActionObject getattr Co-authored-by: Shubham Gupta --- packages/syft/src/syft/types/syft_object.py | 48 +++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 04018683377..606bb6c9406 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -11,6 +11,7 @@ import sys import traceback import types +import typing from typing import Any from typing import Callable from typing import ClassVar @@ -659,6 +660,53 @@ def migrate_to(self, version: int, context: Optional[Context] = None) -> Any: ) return self + ## OVERRIDING pydantic.BaseModel.__getattr__ + ## return super().__getattribute__(item) -> return self.__getattribute__(item) + ## so that ActionObject.__getattribute__ works properly, + ## raising AttributeError when underlying object does not have the attribute + if not typing.TYPE_CHECKING: + # We put `__getattr__` in a non-TYPE_CHECKING block because otherwise, mypy allows arbitrary attribute access + + def __getattr__(self, item: str) -> Any: + private_attributes = object.__getattribute__(self, "__private_attributes__") + if item in private_attributes: + attribute = private_attributes[item] + if hasattr(attribute, "__get__"): + return attribute.__get__(self, type(self)) # type: ignore + + try: + # Note: self.__pydantic_private__ cannot be None if self.__private_attributes__ has items + return self.__pydantic_private__[item] # type: ignore + except KeyError as exc: + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {item!r}" + ) from exc + else: + # `__pydantic_extra__` can fail to be set if the model is not yet fully initialized. + # See `BaseModel.__repr_args__` for more details + try: + pydantic_extra = object.__getattribute__(self, "__pydantic_extra__") + except AttributeError: + pydantic_extra = None + + if pydantic_extra is not None: + try: + return pydantic_extra[item] + except KeyError as exc: + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {item!r}" + ) from exc + else: + if hasattr(self.__class__, item): + return self.__getattribute__( + item + ) # Raises AttributeError if appropriate + else: + # this is the current error + raise AttributeError( + f"{type(self).__name__!r} object has no attribute {item!r}" + ) + def short_qual_name(name: str) -> str: # If the name is a qualname of formax a.b.c.d we will only get d From 496af8a14558e463877790db03d0b9b66058632e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Mon, 4 Mar 2024 19:32:16 +0800 Subject: [PATCH 50/59] Remove the now redundant SyftObject.dict --- packages/syft/src/syft/types/syft_object.py | 24 --------------------- 1 file changed, 24 deletions(-) diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 606bb6c9406..dfa3014e038 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -563,30 +563,6 @@ def to_dict( new_dict[k] = v return new_dict - def dict( - self, - *, - include: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - exclude: Optional[Union["AbstractSetIntStr", "MappingIntStrAny"]] = None, - by_alias: bool = False, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ): - if exclude is None: - exclude = set() - - for attr in DYNAMIC_SYFT_ATTRIBUTES: - exclude.add(attr) - return super().dict( - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - def __post_init__(self) -> None: pass From aac9fcccb17449048185e43be162e7ff46de4025 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 5 Mar 2024 12:26:59 +0800 Subject: [PATCH 51/59] Upgrade Node Syncing to pydantic v2 --- .../src/syft/service/action/action_object.py | 4 ++-- packages/syft/src/syft/service/action/numpy.py | 14 +++++++------- .../syft/src/syft/service/action/pandas.py | 2 +- .../syft/src/syft/service/code/user_code.py | 13 +++++++------ .../src/syft/service/output/output_service.py | 7 ++++--- .../syft/src/syft/service/sync/diff_state.py | 18 +++++++++--------- .../syft/src/syft/service/sync/sync_state.py | 2 +- packages/syft/src/syft/types/blob_storage.py | 4 ++-- 8 files changed, 33 insertions(+), 31 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 4dd41e6961f..b218ab50e62 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -716,7 +716,7 @@ class ActionObject(SyftObject): syft_pre_hooks__: Dict[str, List] = {} syft_post_hooks__: Dict[str, List] = {} syft_twin_type: TwinMode = TwinMode.NONE - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS syft_action_data_type: Optional[Type] = None syft_action_data_repr_: Optional[str] = None syft_action_data_str_: Optional[str] = None @@ -2074,7 +2074,7 @@ class AnyActionObject(ActionObject): syft_internal_type: ClassVar[Type[Any]] = NoneType # type: ignore # syft_passthrough_attrs: List[str] = [] syft_dont_wrap_attrs: List[str] = ["__str__", "__repr__", "syft_action_data_str_"] - syft_action_data_str_ = "" + syft_action_data_str_: str = "" def __float__(self) -> float: return float(self.syft_action_data) diff --git a/packages/syft/src/syft/service/action/numpy.py b/packages/syft/src/syft/service/action/numpy.py index dc26eaebb53..b8dd97aa766 100644 --- a/packages/syft/src/syft/service/action/numpy.py +++ b/packages/syft/src/syft/service/action/numpy.py @@ -67,8 +67,8 @@ class NumpyArrayObjectV2(ActionObjectV2, np.lib.mixins.NDArrayOperatorsMixin): syft_internal_type: ClassVar[Type[Any]] = np.ndarray syft_pointer_type = NumpyArrayObjectPointer - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] # 🔵 TODO 7: Map TPActionObjects and their 3rd Party types like numpy type to these @@ -153,9 +153,9 @@ class NumpyScalarObject(ActionObject, np.lib.mixins.NDArrayOperatorsMixin): __canonical_name__ = "NumpyScalarObject" __version__ = SYFT_OBJECT_VERSION_3 - syft_internal_type = np.number - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_internal_type: ClassVar[Type] = np.number + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] def __float__(self) -> float: return float(self.syft_action_data) @@ -201,8 +201,8 @@ class NumpyBoolObject(ActionObject, np.lib.mixins.NDArrayOperatorsMixin): __version__ = SYFT_OBJECT_VERSION_3 syft_internal_type = np.bool_ - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS - syft_dont_wrap_attrs = ["dtype", "shape"] + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS + syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] @migrate(NumpyBoolObject, NumpyBoolObjectV1) diff --git a/packages/syft/src/syft/service/action/pandas.py b/packages/syft/src/syft/service/action/pandas.py index 5fedbbaef52..9da7f351ecc 100644 --- a/packages/syft/src/syft/service/action/pandas.py +++ b/packages/syft/src/syft/service/action/pandas.py @@ -106,7 +106,7 @@ class PandasSeriesObject(ActionObject): __version__ = SYFT_OBJECT_VERSION_3 syft_internal_type = Series - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS # name: Optional[str] = None # syft_dont_wrap_attrs = ["shape"] diff --git a/packages/syft/src/syft/service/code/user_code.py b/packages/syft/src/syft/service/code/user_code.py index 5795b1983e5..f3b74b0e95f 100644 --- a/packages/syft/src/syft/service/code/user_code.py +++ b/packages/syft/src/syft/service/code/user_code.py @@ -30,7 +30,7 @@ # third party from IPython.display import display -import pydantic +from pydantic import field_validator from result import Err from typing_extensions import Self @@ -879,10 +879,11 @@ class SubmitUserCode(SyftObject): __repr_attrs__ = ["func_name", "code"] - @pydantic.root_validator(pre=True) - def add_output_policy_ids(cls, values: dict) -> dict: - if "id" not in values["output_policy_init_kwargs"]: - values["output_policy_init_kwargs"]["id"] = UID() + @field_validator("output_policy_init_kwargs", mode="after") + @classmethod + def add_output_policy_ids(cls, values: Any) -> Any: + if isinstance(values, dict) and "id" not in values: + values["id"] = UID() return values @property @@ -1395,7 +1396,7 @@ class UserCodeExecutionOutput(SyftObject): user_code_id: UID stdout: str stderr: str - result: Any + result: Any = None class SecureContext: diff --git a/packages/syft/src/syft/service/output/output_service.py b/packages/syft/src/syft/service/output/output_service.py index 15253fe8f1e..fe86430d67f 100644 --- a/packages/syft/src/syft/service/output/output_service.py +++ b/packages/syft/src/syft/service/output/output_service.py @@ -7,7 +7,7 @@ from typing import Union # third party -import pydantic +from pydantic import model_validator from result import Result # relative @@ -48,7 +48,7 @@ class ExecutionOutput(SyftObject): job_link: Optional[LinkedObject] = None created_at: DateTime = DateTime.now() - # Required for __attr_searchable__, set by root_validator + # Required for __attr_searchable__, set by model_validator user_code_id: UID # Output policy is not a linked object because its saved on the usercode @@ -57,7 +57,8 @@ class ExecutionOutput(SyftObject): __attr_searchable__: List[str] = ["user_code_id", "created_at", "output_policy_id"] __repr_attrs__: List[str] = ["created_at", "user_code_id", "job_id", "output_ids"] - @pydantic.root_validator(pre=True) + @model_validator(mode="before") + @classmethod def add_user_code_id(cls, values: dict) -> dict: if "user_code_link" in values: values["user_code_id"] = values["user_code_link"].object_uid diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 54beb360d41..457aa3da4a0 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -14,13 +14,14 @@ from typing import Dict from typing import List from typing import Optional +from typing import Self from typing import Set from typing import Tuple from typing import Type from typing import Union # third party -import pydantic +from pydantic import model_validator from rich import box from rich.console import Console from rich.console import Group @@ -56,8 +57,8 @@ class AttrDiff(SyftObject): __canonical_name__ = "AttrDiff" __version__ = SYFT_OBJECT_VERSION_1 attr_name: str - low_attr: Any - high_attr: Any + low_attr: Any = None + high_attr: Any = None def _repr_html_(self) -> str: return f"""{self.attr_name}: @@ -436,15 +437,14 @@ def visual_hierarchy(self) -> Tuple[Type, dict]: } raise ValueError(f"Unknown root type: {self.root.obj_type}") - @pydantic.root_validator - def make_dependents(cls, values: dict) -> dict: - dependencies = values.get("dependencies", {}) + @model_validator(mode="after") + def make_dependents(self) -> Self: dependents: Dict = {} - for parent, children in dependencies.items(): + for parent, children in self.dependencies.items(): for child in children: dependents[child] = dependents.get(child, []) + [parent] - values["dependents"] = dependents - return values + self.dependents = dependents + return self @property def root(self) -> ObjectDiff: diff --git a/packages/syft/src/syft/service/sync/sync_state.py b/packages/syft/src/syft/service/sync/sync_state.py index 84968f98910..0e6ecb28074 100644 --- a/packages/syft/src/syft/service/sync/sync_state.py +++ b/packages/syft/src/syft/service/sync/sync_state.py @@ -37,7 +37,7 @@ class SyncStateRow(SyftObject): __version__ = SYFT_OBJECT_VERSION_1 object: SyftObject - previous_object: Optional[SyftObject] + previous_object: Optional[SyftObject] = None current_state: str previous_state: str level: int = 0 diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index bc0350d1ec5..6690ab4fd37 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -222,8 +222,8 @@ class BlobFileObject(ActionObject): __version__ = SYFT_OBJECT_VERSION_2 syft_internal_type: ClassVar[Type[Any]] = BlobFile - syft_pointer_type = BlobFileObjectPointer - syft_passthrough_attrs = BASE_PASSTHROUGH_ATTRS + syft_pointer_type: ClassVar[Type[ActionObjectPointer]] = BlobFileObjectPointer + syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS @serializable() From 0b749fb953f357ac070c96f718219297b652d8f0 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 5 Mar 2024 13:11:44 +0800 Subject: [PATCH 52/59] Fix import Self for python<3.10 --- packages/syft/src/syft/service/sync/diff_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 457aa3da4a0..741e231f241 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -14,7 +14,6 @@ from typing import Dict from typing import List from typing import Optional -from typing import Self from typing import Set from typing import Tuple from typing import Type @@ -28,6 +27,7 @@ from rich.markdown import Markdown from rich.padding import Padding from rich.panel import Panel +from typing_extensions import Self # relative from ...types.syft_object import SYFT_OBJECT_VERSION_1 From 5dd433a2eec4f23298b3ddf3a733cfcd95caa117 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 5 Mar 2024 13:37:22 +0800 Subject: [PATCH 53/59] Fix pydantic.ValidationError deprecation warning --- packages/syft/src/syft/service/action/action_graph_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/action_graph_service.py b/packages/syft/src/syft/service/action/action_graph_service.py index 6e06a9c84a0..6527c27b0d8 100644 --- a/packages/syft/src/syft/service/action/action_graph_service.py +++ b/packages/syft/src/syft/service/action/action_graph_service.py @@ -4,7 +4,7 @@ from typing import Union # third party -from pydantic.error_wrappers import ValidationError +from pydantic import ValidationError # relative from ...node.credentials import SyftVerifyKey From d0c4a87f843280339b1748a64aa953c770e77b3e Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 5 Mar 2024 13:37:50 +0800 Subject: [PATCH 54/59] Add type annotation --- packages/syft/src/syft/service/action/numpy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/action/numpy.py b/packages/syft/src/syft/service/action/numpy.py index b8dd97aa766..cd75f103c82 100644 --- a/packages/syft/src/syft/service/action/numpy.py +++ b/packages/syft/src/syft/service/action/numpy.py @@ -200,7 +200,7 @@ class NumpyBoolObject(ActionObject, np.lib.mixins.NDArrayOperatorsMixin): __canonical_name__ = "NumpyBoolObject" __version__ = SYFT_OBJECT_VERSION_3 - syft_internal_type = np.bool_ + syft_internal_type: ClassVar[Type] = np.bool_ syft_passthrough_attrs: List[str] = BASE_PASSTHROUGH_ATTRS syft_dont_wrap_attrs: List[str] = ["dtype", "shape"] From dd19367caed60c491d3d9053838c87b1d2a3ab56 Mon Sep 17 00:00:00 2001 From: Shubham Gupta Date: Tue, 5 Mar 2024 12:23:08 +0530 Subject: [PATCH 55/59] fix __hash__ method not working with ActionObject - add retry to test_sync_flow and skip in case of windows --- packages/syft/src/syft/service/action/action_object.py | 8 ++++++-- packages/syft/src/syft/service/action/numpy.py | 8 +++++--- packages/syft/tests/syft/service/sync/sync_flow_test.py | 4 ++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index b218ab50e62..89d79020b71 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -346,8 +346,8 @@ class ActionObjectPointer: "model_validate", # pydantic "model_validate_json", # pydantic "copy", # pydantic - "__sha256__", - "__hash_exclude_attrs__", + "__sha256__", # syft + "__hash_exclude_attrs__", # syft ] dont_wrap_output_attrs = [ "__repr__", @@ -623,6 +623,7 @@ def debox_args_and_kwargs(args: Any, kwargs: Any) -> Tuple[Any, Any]: "node_uid", "__sha256__", "__hash_exclude_attrs__", + "__hash__", ] @@ -1887,6 +1888,9 @@ def __str__(self) -> str: def __len__(self) -> int: return self.__len__() + def __hash__(self, *args: Any, **kwargs: Any) -> int: + return super().__hash__(*args, **kwargs) + def __getitem__(self, key: Any) -> Any: return self._syft_output_action_object(self.__getitem__(key)) diff --git a/packages/syft/src/syft/service/action/numpy.py b/packages/syft/src/syft/service/action/numpy.py index cd75f103c82..afbf7c866b5 100644 --- a/packages/syft/src/syft/service/action/numpy.py +++ b/packages/syft/src/syft/service/action/numpy.py @@ -96,9 +96,11 @@ def __array_ufunc__( self, ufunc: Any, method: str, *inputs: Any, **kwargs: Any ) -> Union[Self, tuple[Self, ...]]: inputs = tuple( - np.array(x.syft_action_data, dtype=x.dtype) - if isinstance(x, NumpyArrayObject) - else x + ( + np.array(x.syft_action_data, dtype=x.dtype) + if isinstance(x, NumpyArrayObject) + else x + ) for x in inputs ) diff --git a/packages/syft/tests/syft/service/sync/sync_flow_test.py b/packages/syft/tests/syft/service/sync/sync_flow_test.py index 3882a0905f5..e007baf6a03 100644 --- a/packages/syft/tests/syft/service/sync/sync_flow_test.py +++ b/packages/syft/tests/syft/service/sync/sync_flow_test.py @@ -1,8 +1,10 @@ # stdlib +import sys from textwrap import dedent # third party import numpy as np +import pytest # syft absolute import syft as sy @@ -12,6 +14,8 @@ from syft.service.action.action_object import ActionObject +@pytest.mark.flaky(reruns=5, reruns_delay=1) +@pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") def test_sync_flow(): low_worker = sy.Worker( name="low-test", From b741d4243f459a16680383f14dd6605fe7d835dd Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 5 Mar 2024 21:28:21 +0800 Subject: [PATCH 56/59] Fix typing --- .../syft/src/syft/service/action/action_graph_service.py | 5 +---- packages/syft/src/syft/types/syft_metaclass.py | 5 +++-- packages/syft/src/syft/types/twin_object.py | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_graph_service.py b/packages/syft/src/syft/service/action/action_graph_service.py index 6527c27b0d8..886669f7deb 100644 --- a/packages/syft/src/syft/service/action/action_graph_service.py +++ b/packages/syft/src/syft/service/action/action_graph_service.py @@ -118,10 +118,7 @@ def _extract_input_and_output_from_action( for _, kwarg in action.kwargs.items(): input_uids.add(kwarg.id) - if action.result_id is not None: - output_uid = action.result_id.id - else: - output_uid = None + output_uid = action.result_id.id if action.result_id is not None else None return input_uids, output_uid diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index 520c9842246..0d3058dbeca 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -1,4 +1,5 @@ # stdlib +from typing import Any from typing import TypeVar from typing import Union from typing import final @@ -31,7 +32,7 @@ class Empty(metaclass=EmptyType): @dataclass_transform(kw_only_default=True, field_specifiers=(Field,)) class PartialModelMetaclass(ModelMetaclass): - def __call__(cls: type[_T], *args, **kwargs) -> _T: + def __call__(cls: type[_T], *args: Any, **kwargs: Any) -> _T: for field_info in cls.model_fields.values(): if field_info.annotation is not None and field_info.is_required(): field_info.annotation = Union[field_info.annotation, EmptyType] @@ -39,4 +40,4 @@ def __call__(cls: type[_T], *args, **kwargs) -> _T: cls.model_rebuild(force=True) - return super().__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) # type: ignore[misc] diff --git a/packages/syft/src/syft/types/twin_object.py b/packages/syft/src/syft/types/twin_object.py index c95ed59aaee..d06d97d8b77 100644 --- a/packages/syft/src/syft/types/twin_object.py +++ b/packages/syft/src/syft/types/twin_object.py @@ -51,7 +51,7 @@ def make_private_obj(cls, v: Any) -> ActionObject: @model_validator(mode="after") def make_private_obj_id(self) -> Self: if self.private_obj_id is None: - self.private_obj_id = self.private_obj.id + self.private_obj_id = self.private_obj.id # type: ignore[unreachable] return self @field_validator("mock_obj", mode="before") @@ -62,7 +62,7 @@ def make_mock_obj(cls, v: Any) -> ActionObject: @model_validator(mode="after") def make_mock_obj_id(self) -> Self: if self.mock_obj_id is None: - self.mock_obj_id = self.mock_obj.id + self.mock_obj_id = self.mock_obj.id # type: ignore[unreachable] return self @property From 202bee21b8135eae04ce1238dcd537b307ca1bdf Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Tue, 5 Mar 2024 21:56:43 +0800 Subject: [PATCH 57/59] Fix PartialSyftObject.__init__ typing --- packages/syft/src/syft/types/syft_metaclass.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/syft/src/syft/types/syft_metaclass.py b/packages/syft/src/syft/types/syft_metaclass.py index 0d3058dbeca..08ac3ce32de 100644 --- a/packages/syft/src/syft/types/syft_metaclass.py +++ b/packages/syft/src/syft/types/syft_metaclass.py @@ -7,8 +7,6 @@ # third party from pydantic import BaseModel from pydantic._internal._model_construction import ModelMetaclass -from pydantic.fields import Field -from typing_extensions import dataclass_transform # relative from ..serde.serializable import serializable @@ -30,7 +28,6 @@ class Empty(metaclass=EmptyType): pass -@dataclass_transform(kw_only_default=True, field_specifiers=(Field,)) class PartialModelMetaclass(ModelMetaclass): def __call__(cls: type[_T], *args: Any, **kwargs: Any) -> _T: for field_info in cls.model_fields.values(): From 2a9d4256feb90be305ea86584ee72650404724a5 Mon Sep 17 00:00:00 2001 From: Kien Dang Date: Wed, 6 Mar 2024 11:22:09 +0800 Subject: [PATCH 58/59] Temporarily reset protocol_version --- .../src/syft/protocol/protocol_version.json | 1016 ++++++++++++++++- 1 file changed, 969 insertions(+), 47 deletions(-) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 34b62666e2e..e5b262643d3 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -1,101 +1,796 @@ { - "1": { - "release_name": "0.8.2.json" - }, - "2": { - "release_name": "0.8.3.json" - }, - "3": { - "release_name": "0.8.4.json" - }, "dev": { "object_versions": { + "PartialSyftObject": { + "1": { + "version": 1, + "hash": "008917584d8e1c09015cdbef02f59c0622f48e0618877c1b44425c8846befc13", + "action": "add" + } + }, + "NodeMetadataUpdate": { + "1": { + "version": 1, + "hash": "569d124c23590360bda240c19b53314ccc6204c5d1ab0d2898976a028e002191", + "action": "add" + } + }, + "NodeMetadata": { + "1": { + "version": 1, + "hash": "6bee018894dfdf697ea624740d0bf051750e0b0d8470ced59646f6d8812068ac", + "action": "add" + }, + "2": { + "version": 2, + "hash": "f856169fea72486cd436875ce4411ef935da11eb7c5af48121adfa00d4c0cdb6", + "action": "add" + }, + "3": { + "version": 3, + "hash": "3cc67abf394a805066a88aef0bea15bde609b9ecbe7ec15172eac5e7a0b7ef7c", + "action": "add" + } + }, + "StoreConfig": { + "1": { + "version": 1, + "hash": "17de8875cf590311ddb042140347ffc79d4a85028e504dad178ca4e1237ec861", + "action": "add" + } + }, + "MongoDict": { + "1": { + "version": 1, + "hash": "640734396edae801e1601fe7777710e67685e552acb0244ad8b4f689599baca9", + "action": "add" + } + }, + "MongoStoreConfig": { + "1": { + "version": 1, + "hash": "e52aa382e300b0b69aaa2d80aadb4e3a9a3c02b3c741b71d56f959c4d3891ce5", + "action": "add" + } + }, + "LinkedObject": { + "1": { + "version": 1, + "hash": "824567c6933c095d0e2f6995c8de3581c0fbd2e9e4ead35c8159f7964709c28e", + "action": "add" + } + }, + "BaseConfig": { + "1": { + "version": 1, + "hash": "4e5257080ce615aa4122b02bad8487e4c7d6d0f171ff77abbc9e8cd3e33df89a", + "action": "add" + } + }, + "ServiceConfig": { + "1": { + "version": 1, + "hash": "ca91f59bf045d949d82860f7d52655bfbede4cf6bdc5bae8f847f08a16f05d74", + "action": "add" + } + }, + "LibConfig": { + "1": { + "version": 1, + "hash": "c6ff229aea16874c5d9ae4d1f9e500d13f5cf984bbcee7abd16c5841707a2f78", + "action": "add" + } + }, + "APIEndpoint": { + "1": { + "version": 1, + "hash": "c0e83867b107113e6fed06364ba364c24b2f4af35b15a3869b176318d3be7989", + "action": "add" + } + }, + "LibEndpoint": { + "1": { + "version": 1, + "hash": "153eac6d8990774eebfffaa75a9895e7c4e1a0e09465d5da0baf4c3a3b03369d", + "action": "add" + } + }, + "SignedSyftAPICall": { + "1": { + "version": 1, + "hash": "e66a116de2fa44ebdd0d4c2d7d5a047dedb555fd201a0f431cd8017d9d33a61d", + "action": "add" + } + }, + "SyftAPICall": { + "1": { + "version": 1, + "hash": "014bd1d0933f6070888a313edba239170759de24eae49bf2374c1be4dbe2b4d7", + "action": "add" + } + }, + "SyftAPIData": { + "1": { + "version": 1, + "hash": "db101a75227e34750d7056785a1e87bb2e8ad6604f19c372d0cb6aa437243bf5", + "action": "add" + } + }, + "SyftAPI": { + "1": { + "version": 1, + "hash": "2bba1d9fcf677a58e35bf903de3da22ee4913af138aa3012af9c46b3609579cd", + "action": "add" + } + }, + "User": { + "2": { + "version": 2, + "hash": "ded970c92f202716ed33a2117cf541789f35fad66bd4b1db39da5026b1d7d0e7", + "action": "add" + } + }, + "UserUpdate": { + "2": { + "version": 2, + "hash": "32cba8fbd786c575f92e26c31384d282e68e3ebfe5c4b0a0e793820b1228d246", + "action": "add" + } + }, + "UserCreate": { + "2": { + "version": 2, + "hash": "2540188c5aaea866914dccff459df6e0f4727108a503414bb1567ff6297d4646", + "action": "add" + } + }, + "UserSearch": { + "1": { + "version": 1, + "hash": "69d1e10b81c8a4143cf70e4f911d8562732af2458ebbc455ca64542f11373dd1", + "action": "add" + } + }, + "UserView": { + "2": { + "version": 2, + "hash": "e410de583bb15bc5af57acef7be55ea5fc56b5b0fc169daa3869f4203c4d7473", + "action": "add" + } + }, + "UserViewPage": { + "1": { + "version": 1, + "hash": "16dac6209b19a934d286ef1efa874379e0040c324e71023c57d1bc6d2d367171", + "action": "add" + } + }, + "UserPrivateKey": { + "1": { + "version": 1, + "hash": "7cb196587887f0f3bffb298dd9f3b88509e9b2748792bf8dc03bdd0d6b98714a", + "action": "add" + } + }, + "NodeSettingsUpdate": { + "1": { + "version": 1, + "hash": "b6ddc66ff270a3c2c4760e31e1a55d72ed04ccae2d0115ebe2fba6f2bf9bd119", + "action": "add" + } + }, + "NodeSettings": { + "1": { + "version": 1, + "hash": "b662047bb278f4f5db77c102f94b733c3a929839271b3d6b82ea174a60e2aaf0", + "action": "add" + }, + "2": { + "version": 2, + "hash": "29a82afcb006a044b6ae04c6ea8a067d145d28b4210bb038ea9fa86ebde108c8", + "action": "add" + } + }, + "HTTPConnection": { + "1": { + "version": 1, + "hash": "5ee19eaf55ecbe7945ea45924c036ec0f500114a2f64176620961a8c2ec94cdb", + "action": "add" + } + }, + "PythonConnection": { + "1": { + "version": 1, + "hash": "011946fc9af0a6987f5c7bc9b0208b2fae9d65217531430bced7ba542788da1a", + "action": "add" + } + }, + "DateTime": { + "1": { + "version": 1, + "hash": "7e9d89309a10d2110a7ae4f97d8f25a7914853269e8fa0c531630790c1253f17", + "action": "add" + } + }, + "ActionDataEmpty": { + "1": { + "version": 1, + "hash": "89b5912fe5416f922051b8068be6071a03c87a4ab264959de524f1b86e95f028", + "action": "add" + } + }, + "ObjectNotReady": { + "1": { + "version": 1, + "hash": "88207988639b11eaca686b6e079616d9caecc3dbc2a8112258e0f39ee5c3e113", + "action": "add" + } + }, + "ActionDataLink": { + "1": { + "version": 1, + "hash": "10bf94e99637695f1ba283f0b10e70743a4ebcb9ee75aefb1a05e6d6e1d21a71", + "action": "add" + } + }, + "Action": { + "1": { + "version": 1, + "hash": "5cf71ee35097f17fbb1dd05096f875211d71cf07161205d7f6a9c11fd49d5272", + "action": "add" + }, + "2": { + "version": 2, + "hash": "a13b50c4d23bd6deb7896e394f2a20e6cef4c33c5e6f4ee30f19eaffab708f21", + "action": "add" + } + }, "ActionObject": { + "1": { + "version": 1, + "hash": "632446f1415102490c93fafb56dd9eb29d79623bcc5e9f2e6e37c4f63c2c51c3", + "action": "add" + }, + "2": { + "version": 2, + "hash": "577aa1f010b90194958a18ec38ee21db3718bd96d9e036501c6ddeefabedf432", + "action": "add" + }, "3": { "version": 3, "hash": "9cc29f4b637e7a9407ff5f56e282b5411c2709e8195214c452e44b65f9051d56", "action": "add" } }, - "AnyActionObject": { - "3": { - "version": 3, - "hash": "ad6b897da9bdf2fcacc853fad73b7d7b98c8d0ecb58c35110e236edefd99f561", + "AnyActionObject": { + "1": { + "version": 1, + "hash": "bcb31f847907edc9c95d2d120dc5427854604f40940e3f41cd0474a1820ac65e", + "action": "add" + }, + "2": { + "version": 2, + "hash": "002d8be821140befebbc0503e6bc1ef8779094e24e46305e5da5af6eecb56b13", + "action": "add" + }, + "3": { + "version": 3, + "hash": "ad6b897da9bdf2fcacc853fad73b7d7b98c8d0ecb58c35110e236edefd99f561", + "action": "add" + } + }, + "SyftImageRegistry": { + "1": { + "version": 1, + "hash": "dc83910c91947e3d9eaa3e6f8592237448f0408668c7cca80450b5fcd54722e1", + "action": "add" + } + }, + "SyftWorkerImage": { + "1": { + "version": 1, + "hash": "2a9585b6a286e24f1a9f3f943d0128730cf853edc549184dc1809d19e1eec54b", + "action": "add" + } + }, + "SyftWorker": { + "1": { + "version": 1, + "hash": "0d5b367162f3ce55ab090cc1b49bd30e50d4eb144e8431eadc679bd0e743aa70", + "action": "add" + } + }, + "WorkerPool": { + "1": { + "version": 1, + "hash": "250699eb4c452fc427995353d5c5ad6245fb3e9fdac8814f8348784816a0733b", + "action": "add" + } + }, + "BlobFile": { + "1": { + "version": 1, + "hash": "47ed55183d619c6c624e35412360a41de42833e2c24223c1de1ad12a84fdafc2", + "action": "add" + }, + "3": { + "version": 3, + "hash": "8f1710c754bb3b39f546b97fd69c4826291398b247976bbc41fa873af431bca9", + "action": "add" + } + }, + "BlobFileOBject": { + "1": { + "version": 1, + "hash": "8da2c80ced4f0414c671313c4b63d05846df1e397c763d99d803be86c29755bb", + "action": "add" + }, + "2": { + "version": 2, + "hash": "eb895881610723d674755033b0f1798219725d5ad70791d25a44f473ddff318b", + "action": "add" + } + }, + "SecureFilePathLocation": { + "1": { + "version": 1, + "hash": "7febc066e2ee5a3a4a891720afede3f5c155cacc0557662ac4d04bf67b964c6d", + "action": "add" + } + }, + "SeaweedSecureFilePathLocation": { + "1": { + "version": 1, + "hash": "5724a38b1a92b8a55da3d9cc34a720365a6d0c32683acda630fc44067173e201", + "action": "add" + }, + "2": { + "version": 2, + "hash": "5fd63fed2a4efba8c2b6c7a7b5e9b5939181781c331230896aa130b6fd558739", + "action": "add" + } + }, + "AzureSecureFilePathLocation": { + "1": { + "version": 1, + "hash": "1bb15f3f9d7082779f1c9f58de94011487924cb8a8c9c2ec18fd7c161c27fd0e", + "action": "add" + } + }, + "BlobStorageEntry": { + "1": { + "version": 1, + "hash": "9f1b027cce390ee6f71c7a81e7420bb71a477b29c6c62ba74e781a97bc5434e6", + "action": "add" + }, + "2": { + "version": 2, + "hash": "5472bdd5bdce6d0b561543a6bac70d47bf0c05c141a21450751460cc538d6b55", + "action": "add" + } + }, + "BlobStorageMetadata": { + "1": { + "version": 1, + "hash": "6888943be3f97186190dd26d7eefbdf29b15c6f2fa459e13608065ebcdb799e2", + "action": "add" + }, + "2": { + "version": 2, + "hash": "674f4c52a8444289d5ef389b919008860e2b0e7acbaafa774d58e492d5b6741a", + "action": "add" + } + }, + "CreateBlobStorageEntry": { + "1": { + "version": 1, + "hash": "61a373336e83645f1b6d78a320323d9ea4ee91b3d87b730cb0608fbfa0072262", + "action": "add" + } + }, + "BlobRetrieval": { + "1": { + "version": 1, + "hash": "a8d7e1d6483e7a9b5a130e837fa398862aa6cbb316cc5f4470450d835755fdd9", + "action": "add" + }, + "2": { + "version": 2, + "hash": "4c4fbdb6df5bb9fcbe914a9890bd1c1b6a1b3f382a04cbc8752a5a1b03130111", + "action": "add" + } + }, + "SyftObjectRetrieval": { + "2": { + "version": 2, + "hash": "d9d7a7e1b8843145c9687fd013c9223700285886073547734267e91ac53e0996", + "action": "add" + }, + "3": { + "version": 3, + "hash": "952958e9afae007bef3cb89aa15be95dddc4c310e3a8ce4191576f90ac6fcbc8", + "action": "add" + } + }, + "BlobRetrievalByURL": { + "3": { + "version": 3, + "hash": "0b664100ea08413ca4ef04665ca910c2cf9535539617ea4ba33687d05cdfe747", + "action": "add" + } + }, + "BlobDeposit": { + "1": { + "version": 1, + "hash": "c98e6da658a3be01ead4ea6ee6a4c10046879f0ce0f5fc5f946346671579b229", + "action": "add" + } + }, + "WorkerSettings": { + "1": { + "version": 1, + "hash": "0dcd95422ec8a7c74e45ee68a125084c08f898dc94a13d25fe5a5fd0e4fc5027", + "action": "add" + }, + "2": { + "version": 2, + "hash": "d623a8a0d6c83b26ba49686bd8be10eccb126f54626fef334a85396c3b8a8ed6", + "action": "add" + } + }, + "QueueItem": { + "1": { + "version": 1, + "hash": "5aa94681d9d0715d5b605f9625a54e114927271378cf2ea7245f85c488035e0b", + "action": "add" + }, + "2": { + "version": 2, + "hash": "9503b878de4b5b7a1793580301353523b7d6219ebd27d38abe598061979b7570", + "action": "add" + }, + "3": { + "version": 3, + "hash": "3495f406d2c97050ce86be80c230f49b6b846c63b9a9230cbd6631952f2bad0f", + "action": "add" + } + }, + "ActionQueueItem": { + "1": { + "version": 1, + "hash": "11a43caf9164eb2a5a21f4bcb0ca361d0a5d134bf3c60173f2c502d0d80219de", + "action": "add" + }, + "2": { + "version": 2, + "hash": "6413ed01e949cac169299a43ce40651f9bf8053e408b6942853f8afa8a693b3d", + "action": "add" + } + }, + "ZMQClientConfig": { + "1": { + "version": 1, + "hash": "e6054969b495791569caaf33239039beae3d116e1fe74e9575467c48b9007c45", + "action": "add" + }, + "3": { + "version": 3, + "hash": "91ce5953cced58e12c576aa5174d5ca0c91981b01cf42edd5283d347baa3390b", + "action": "add" + } + }, + "HTTPNodeRoute": { + "1": { + "version": 1, + "hash": "1901b9f53f9970ce2bd8307ba9f7cafc0e7eba1d2ec82e4014c6120e605e3741", + "action": "add" + } + }, + "PythonNodeRoute": { + "1": { + "version": 1, + "hash": "15711e6e7a1ef726c8e8b5c35a6cb2d30b56ba5213cba489524bf63489e136cf", + "action": "add" + } + }, + "EnclaveMetadata": { + "1": { + "version": 1, + "hash": "39f85e475015e6f860ddcc5fea819423eba2db8f4b7d8e004c05a44d6f8444c6", + "action": "add" + } + }, + "DataSubject": { + "1": { + "version": 1, + "hash": "0b8b049d4627727b444c419f5d6a97b7cb97a433088ebf744c854b6a470dadf1", + "action": "add" + } + }, + "DataSubjectCreate": { + "1": { + "version": 1, + "hash": "5a94f9fcba75c50d78d71222f0235c5fd4d8003ae0db4d74bdbc4d56a99de3aa", + "action": "add" + } + }, + "DataSubjectMemberRelationship": { + "1": { + "version": 1, + "hash": "0a820edc9f1a87387acc3c611fe852752fcb3dab7608058f2bc48211be7bfbd2", + "action": "add" + } + }, + "Contributor": { + "1": { + "version": 1, + "hash": "d1d4f25bb87e59c0414501d3335097de66815c164c9ed5a7850ff8bec69fbcdc", + "action": "add" + } + }, + "MarkdownDescription": { + "1": { + "version": 1, + "hash": "519328a3952049f57004013e4fb00840695b24b8575cad983056412c9c9d9ba6", + "action": "add" + } + }, + "Asset": { + "1": { + "version": 1, + "hash": "24350b8d9597df49999918ad42e0eece1328ea30389311f1e0a420be8f39b8a1", + "action": "add" + } + }, + "CreateAsset": { + "1": { + "version": 1, + "hash": "1b4c71569b8da64258672483bd36dc4aa99a32d4cb519659241d15bc898041a6", + "action": "add" + } + }, + "Dataset": { + "1": { + "version": 1, + "hash": "99ca2fa3e46fd9810222d269fac6accb546f632e94d5d57529016ba5e55af5a8", + "action": "add" + } + }, + "DatasetPageView": { + "1": { + "version": 1, + "hash": "b1de14bb9b6a259648dfc59b6a48fa526116afe50a689c24b8bb36fd0e6a97f8", + "action": "add" + } + }, + "CreateDataset": { + "1": { + "version": 1, + "hash": "3b020d9b8928cbd7e91f41c749ab4c932e19520696a183f2c7cd1312ebb640d1", + "action": "add" + } + }, + "JobItem": { + "1": { + "version": 1, + "hash": "7b8723861837b0b7e948b2cf9244159d232185f3407dd6bef108346f941ddf6e", + "action": "add" + }, + "2": { + "version": 2, + "hash": "e99cf5a78c6dd3a0adc37af3472c7c21570a9e747985dff540a2b06d24de6446", + "action": "add" + }, + "3": { + "version": 3, + "hash": "5b93a59e28574691339d22826d5650969336a2e930b93d6b3fe6d5409ca0cfc4", + "action": "add" + } + }, + "JobInfo": { + "1": { + "version": 1, + "hash": "b2b88e49c6f6e5d4587d52099b521b258b62c9d685f769207c0b05cd4079507b", + "action": "add" + } + }, + "ExecutionOutput": { + "1": { + "version": 1, + "hash": "833addc66807a638939aac00a4be306c93bd8d80a8f4ce6fcdb16d98e87ceb8b", + "action": "add" + } + }, + "TwinObject": { + "1": { + "version": 1, + "hash": "c42455586b43724a7421becd99122b787a129798daf6081e96954ecaea228099", + "action": "add" + } + }, + "ExactMatch": { + "1": { + "version": 1, + "hash": "e497e2e2380db72766c5e219e8afd13136d8953933d6f1eaf83b14001e887cde", + "action": "add" + } + }, + "OutputHistory": { + "1": { + "version": 1, + "hash": "4ec6e6efd86a972b474251885151bdfe4ef262562174605e8ab6a8abba1aa867", + "action": "add" + } + }, + "OutputPolicyExecuteCount": { + "1": { + "version": 1, + "hash": "4fd95093a1fa04c3a6582df60db27931b6d38dd4f59b7e4151847dfcce8b57de", + "action": "add" + } + }, + "OutputPolicyExecuteOnce": { + "1": { + "version": 1, + "hash": "89884332bb98f64e9c992e69dd260c9c2083cb59f9863b001b670da03b26d673", + "action": "add" + } + }, + "UserPolicy": { + "1": { + "version": 1, + "hash": "c69b17b1d96cace8b45da6d9639165f2da4aa7ff156b6fd922ac217bf7856d8a", + "action": "add" + } + }, + "SubmitUserPolicy": { + "1": { + "version": 1, + "hash": "96f7f39279fadc70c569b8d48ed4d6420a8132db51e37466d272fda19953554b", + "action": "add" + } + }, + "UserCodeStatusCollection": { + "1": { + "version": 1, + "hash": "4afcdcebd4b0ba95a8ac65eda9fcaa88129b7c520e8e6b093c6ab5208641a617", + "action": "add" + } + }, + "UserCode": { + "1": { + "version": 1, + "hash": "e14c22686cdc7d1fb2b0d01c0aebdea37e62a61b051677c1d30234214f05cd42", "action": "add" - } - }, - "BlobFileOBject": { + }, "2": { "version": 2, - "hash": "eb895881610723d674755033b0f1798219725d5ad70791d25a44f473ddff318b", + "hash": "660e1abc15034f525e91ffdd820c2a2179bfddf83b7b9e3ce7823b2efc515c69", + "action": "add" + }, + "4": { + "version": 4, + "hash": "4acb1fa6856da943966b6a93eb7874000f785b29f12ecbed9025606f8fe51aa4", "action": "add" } }, - "JobInfo": { + "SubmitUserCode": { "2": { "version": 2, - "hash": "5c1f7d5e6a991123a1907c1823be14a75458ba06af1fe5a1b77aaac7fa546c78", + "hash": "9b29e060973a3de8d3564a2b7d2bb5c53745aa445bf257576994b613505d7194", + "action": "add" + }, + "3": { + "version": 3, + "hash": "a29160c16d2e2620800d42cdcd9f3637d063a570c477a5d05217a2e64b4bb396", "action": "add" } }, - "ExecutionOutput": { + "UserCodeExecutionResult": { "1": { "version": 1, - "hash": "833addc66807a638939aac00a4be306c93bd8d80a8f4ce6fcdb16d98e87ceb8b", + "hash": "49c32e85e78b7b189a7f13b7e26115ef94fcb0b60b578adcbe2b95e289f63a6e", "action": "add" } }, - "OutputPolicyExecuteCount": { + "UserCodeExecutionOutput": { "1": { "version": 1, - "hash": "6bb24b3b35e19564c43b838ca3f46ccdeadb6596511917f2d220681a378e439d", - "action": "remove" - }, - "2": { - "version": 2, - "hash": "ca0ba249f4f32379f5b83279a27df4a21eb23c531a86538c821a10ddf2c799ff", + "hash": "94c18d2dec05b39993c1a7a70bca2c991c95bd168005a93e578a810e57ef3164", "action": "add" } }, - "OutputPolicyExecuteOnce": { + "CodeHistory": { "1": { "version": 1, - "hash": "32a40fc9966b277528eebc61c01041f3a5447417731954abdaffbb14dabc76bb", - "action": "remove" + "hash": "a7baae93862ae0aa67675f1617574e31aafb15a9ebff633eb817278a3a867161", + "action": "add" + } + }, + "CodeHistoryView": { + "1": { + "version": 1, + "hash": "0ed1a2a04a962ecbcfa38b0b8a03c1e51e8946a4b80f6bf2557148ce658671ce", + "action": "add" + } + }, + "CodeHistoriesDict": { + "1": { + "version": 1, + "hash": "95288411cd5843834f3273a2fd66a7df2e603e980f4ab1d329f9ab17d5d2f643", + "action": "add" + } + }, + "UsersCodeHistoriesDict": { + "1": { + "version": 1, + "hash": "5e1f389c4565ee8558386dd5c934d81e0c68ab1434f86bb9065976b587ef44d1", + "action": "add" + } + }, + "SyftLog": { + "1": { + "version": 1, + "hash": "bd3f62b8fe4b2718a6380c8f05a93c5c40169fc4ab174db291929298e588429e", + "action": "add" }, "2": { "version": 2, - "hash": "e6b0f23047037734c1cc448771bc2770f5bf6c8b8f80cf46939eb7ba66dd377e", + "hash": "d3ce45794da2e6c4b0cef63b98a553525af50c5d9db42d3d64caef3e7d22b4a9", "action": "add" } }, - "UserCodeStatusCollection": { + "OnDiskBlobDeposit": { "1": { "version": 1, - "hash": "4afcdcebd4b0ba95a8ac65eda9fcaa88129b7c520e8e6b093c6ab5208641a617", + "hash": "5efc230c1ee65c4626d334aa69ed458c796c45265e546a333844c6c2bcd0e6b0", "action": "add" } }, - "UserCode": { - "3": { - "version": 3, - "hash": "90fcae0f556f375ba1e91d2e345f57241660695c6e2b84c8e311df89d09e6c66", - "action": "remove" + "RemoteConfig": { + "1": { + "version": 1, + "hash": "ad7bc4780a8ad52e14ce68601852c93d2fe07bda489809cad7cae786d2461754", + "action": "add" + } + }, + "AzureRemoteConfig": { + "1": { + "version": 1, + "hash": "c05c6caa27db4e385c642536d4b0ecabc0c71e91220d2e6ce21a2761ca68a673", + "action": "add" + } + }, + "SeaweedFSBlobDeposit": { + "1": { + "version": 1, + "hash": "382a9ac178deed2a9591e1ebbb39f265cbe67027fb93a420d473a4c26b7fda11", + "action": "add" }, - "4": { - "version": 4, - "hash": "4acb1fa6856da943966b6a93eb7874000f785b29f12ecbed9025606f8fe51aa4", + "2": { + "version": 2, + "hash": "07d84a95324d95d9c868cd7d1c33c908f77aa468671d76c144586aab672bcbb5", "action": "add" } }, - "UserCodeExecutionOutput": { + "DictStoreConfig": { "1": { "version": 1, - "hash": "94c18d2dec05b39993c1a7a70bca2c991c95bd168005a93e578a810e57ef3164", + "hash": "256e9c623ce0becd555ddd2a55a0c15514e162786b1549388cef98a92a9b18c9", "action": "add" } }, "NumpyArrayObject": { + "1": { + "version": 1, + "hash": "dcc7b44fa5ad22ae0bc576948f856c172dac1e9de2bc8e2a302e428f3309a278", + "action": "add" + }, + "2": { + "version": 2, + "hash": "2c631121d9211006edab5620b214dea83e2398bee92244d822227ee316647e22", + "action": "add" + }, "3": { "version": 3, "hash": "8572cdfb6a9f8b20c276aeebaf12cd26e424703ea624cf52efdbe33a8ab3d4ef", @@ -103,6 +798,16 @@ } }, "NumpyScalarObject": { + "1": { + "version": 1, + "hash": "5c1b6b6e8ba88bc79e76646d621489b889fe8f9b9fd59f117d594be18a409633", + "action": "add" + }, + "2": { + "version": 2, + "hash": "0d5d81b9d45c140f6e07b43ed68d31e0ef060d6b4d0431c9b4795997bb35c69d", + "action": "add" + }, "3": { "version": 3, "hash": "d3107530ef42a6f3b234423a3c5efcb13f4b43a5959642a0b1e3c093992d59d1", @@ -110,6 +815,16 @@ } }, "NumpyBoolObject": { + "1": { + "version": 1, + "hash": "a5c822a6a3ca9eefd6a2b68f7fd0bc614fba7995f6bcc30bdc9dc882296b9b16", + "action": "add" + }, + "2": { + "version": 2, + "hash": "24839ba1c88ed833a134124750d5f299abcdf318670315028ed87b254f4578b3", + "action": "add" + }, "3": { "version": 3, "hash": "b9cd5c27630b0c67465d21e1d67a0b2591cd59aa6ac7d60ab1f5f64a746114df", @@ -117,6 +832,16 @@ } }, "PandasDataframeObject": { + "1": { + "version": 1, + "hash": "35058924b3de2e0a604a92f91f4dd2e3cc0dac80c219d34f360e7cedd52f5f4c", + "action": "add" + }, + "2": { + "version": 2, + "hash": "66729d4ba7a92210d45c5a5c24fbdb4c8e58138a515a7bdb71ac8f6e8b868544", + "action": "add" + }, "3": { "version": 3, "hash": "f1ec7d6debb6d2d81e752418dd84544995b16d0386da791c4770914a227700d9", @@ -124,16 +849,129 @@ } }, "PandasSeriesObject": { + "1": { + "version": 1, + "hash": "2a0d8a55f1c27bd8fccd276cbe01bf272c40cab10417d7027273983fed423caa", + "action": "add" + }, + "2": { + "version": 2, + "hash": "cb05a714f75b1140a943f56a3622fcc0477b3a1f504cd545a98510959ffe1528", + "action": "add" + }, "3": { "version": 3, "hash": "96f75160ec2a966fc3866cf2e86b151914012514e2b89f618933cdfbb639e528", "action": "add" } }, + "ReplyNotification": { + "1": { + "version": 1, + "hash": "34b2ad522f7406c2486573467d9c7acef5c1063a0d9f2177c3bda2d8c4f87572", + "action": "add" + } + }, + "Notification": { + "1": { + "version": 1, + "hash": "d13981f721fe2b3e2717640ee07dc716c596e4ecd442461665c3fdab0b85bf0e", + "action": "add" + } + }, + "CreateNotification": { + "1": { + "version": 1, + "hash": "b1f459de374fe674f873a4a5f3fb8a8aabe0d83faad84a933f0a77dd1141159a", + "action": "add" + } + }, + "Change": { + "1": { + "version": 1, + "hash": "aefebd1601cf5bfd4817b0db75300a78299cc4949ead735a90873cbd22c8d4bc", + "action": "add" + } + }, + "ChangeStatus": { + "1": { + "version": 1, + "hash": "627f6f8e42cc285336aa6fd4916285d796140f4ff901487b7cb3907ef0f116a6", + "action": "add" + } + }, + "ActionStoreChange": { + "1": { + "version": 1, + "hash": "17b865e75eb3fb2693924fb00ba87a25260be45d55a4eb2184c4ead22d787cbe", + "action": "add" + } + }, + "CreateCustomImageChange": { + "1": { + "version": 1, + "hash": "bc09dca7995938f3b3a2bd9c8b3c2feffc8484df466144a425cb69cadb2ab635", + "action": "add" + } + }, + "CreateCustomWorkerPoolChange": { + "1": { + "version": 1, + "hash": "86894f8ccc037de61f44f9698fd113ba02c3cf3870a3048c00a46e15dcd1941c", + "action": "add" + } + }, + "Request": { + "1": { + "version": 1, + "hash": "e054307eeb7f13683cde9ce7613d5ca2925a13fff7c345b1c9f729a12c955f90", + "action": "add" + } + }, + "RequestInfo": { + "1": { + "version": 1, + "hash": "b76075c138afc0563ce9ac7f6b1131f048951f7486cd516c02736dc1a2a23639", + "action": "add" + } + }, + "RequestInfoFilter": { + "1": { + "version": 1, + "hash": "7103abdc464ae71bb746410f5730f55dd8ed82268aa32bbb0a69e0070488a669", + "action": "add" + } + }, + "SubmitRequest": { + "1": { + "version": 1, + "hash": "96b4ec12beafd9d8a7c97399cb8a23dade4db16d8f521be3fe7b8fec99db5161", + "action": "add" + } + }, + "ObjectMutation": { + "1": { + "version": 1, + "hash": "0ee3dd38d6df0fe9a19d848e8f3aaaf13a6ba86afe3406c239caed6da185651a", + "action": "add" + } + }, + "EnumMutation": { + "1": { + "version": 1, + "hash": "4c02f956ec9b973064972cc57fc8dd9c525e683f93f804642b4e1bfee1b62e57", + "action": "add" + } + }, "UserCodeStatusChange": { - "3": { - "version": 3, - "hash": "999ab977d4fe5a7b74ee2d90370599ce9caa1b38fd6e6c29bd543d379c4dae31", + "1": { + "version": 1, + "hash": "4f5b405cc2b3976ed8f7018df82e873435d9187dff15fa5a23bc85a738969f3f", + "action": "add" + }, + "2": { + "version": 2, + "hash": "4193847a32c3aa358ff67934ee661a60055688c1340051fbd46d2a5e3aac7898", "action": "add" } }, @@ -150,6 +988,90 @@ "hash": "6da39adb0ecffb4ca7873c0d95ed31c8bf037610cde144662285b921de5d8f04", "action": "add" } + }, + "NodePeer": { + "1": { + "version": 1, + "hash": "7b88de7e38490e2d69f31295137673e7ddabc16ab0e2272ff491f6cea1835d63", + "action": "add" + } + }, + "SyftObjectMigrationState": { + "1": { + "version": 1, + "hash": "d3c8126bc15dae4dd243bb035530e3f56cd9e433d403dd6b5f3b45face6d281f", + "action": "add" + } + }, + "ProjectThreadMessage": { + "1": { + "version": 1, + "hash": "1118e935792e8e54103dbf91fa33edbf192a7767d2b1d4526dfa7d4a643cde2e", + "action": "add" + } + }, + "ProjectMessage": { + "1": { + "version": 1, + "hash": "55a3a5171b6949372b4125cc461bf39bc998565e07703804fca6c7ef99695ae4", + "action": "add" + } + }, + "ProjectRequestResponse": { + "1": { + "version": 1, + "hash": "d4c360e845697a0b24695143d0781626cd344cfde43162c90ae90fe67e00ae21", + "action": "add" + } + }, + "ProjectRequest": { + "1": { + "version": 1, + "hash": "514d189df335c68869eea36befcdcafec74bdc682eaf18871fe879e26da4dbb6", + "action": "add" + } + }, + "AnswerProjectPoll": { + "1": { + "version": 1, + "hash": "ff2e1ac7bb764c99d646b96eb3ebfbf9311599b7e3be07aa4a4eb4810bb6dd12", + "action": "add" + } + }, + "ProjectPoll": { + "1": { + "version": 1, + "hash": "b0ac8f1d9c06997374ddbc33fdf1d0af0da15fdb6899f52d91a8574106558964", + "action": "add" + } + }, + "Project": { + "1": { + "version": 1, + "hash": "ec5b7ac1c92808e266f06b175c6ebcd50be81777ad120c02ce8c6074d0004788", + "action": "add" + } + }, + "ProjectSubmit": { + "1": { + "version": 1, + "hash": "0374b37779497d7e0b2ffeabc38d35bfbae2ee762a7674a5a8af75e7c5545e61", + "action": "add" + } + }, + "SQLiteStoreConfig": { + "1": { + "version": 1, + "hash": "b656b26c14cf4e97aba702dd62a0927aec7f860c12eed512c2c688e1b7109aa5", + "action": "add" + } + }, + "Plan": { + "1": { + "version": 1, + "hash": "a0bba2b7792c9e08c453e9e256f0ac6e6185610726566bcd50b057ae83b42d9a", + "action": "add" + } } } } From 06226e1170db0cb593b352385034ef257d7cfbf0 Mon Sep 17 00:00:00 2001 From: alfred-openmined-bot <145415986+alfred-openmined-bot@users.noreply.github.com> Date: Wed, 6 Mar 2024 06:19:11 +0000 Subject: [PATCH 59/59] bump protocol and remove notebooks --- .../src/syft/protocol/protocol_version.json | 84 +++++++++++-------- 1 file changed, 47 insertions(+), 37 deletions(-) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index e5b262643d3..a24d3320068 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -49,7 +49,7 @@ "MongoStoreConfig": { "1": { "version": 1, - "hash": "e52aa382e300b0b69aaa2d80aadb4e3a9a3c02b3c741b71d56f959c4d3891ce5", + "hash": "a6d48634dbfce836754b60ca1c35db30c86df795ffdbe87bb7758ba4f809f6c1", "action": "add" } }, @@ -241,41 +241,41 @@ }, "2": { "version": 2, - "hash": "a13b50c4d23bd6deb7896e394f2a20e6cef4c33c5e6f4ee30f19eaffab708f21", + "hash": "125da21171e5bdac0b1547d804df98e1de6ed2942ea181c7f7883372fdf23620", "action": "add" } }, "ActionObject": { "1": { "version": 1, - "hash": "632446f1415102490c93fafb56dd9eb29d79623bcc5e9f2e6e37c4f63c2c51c3", + "hash": "49079c4481865c54f198fbd3ef41f1babdf71d5878f14ec8556cdace6203b30f", "action": "add" }, "2": { "version": 2, - "hash": "577aa1f010b90194958a18ec38ee21db3718bd96d9e036501c6ddeefabedf432", + "hash": "7876ae48df101386df1e3edb075bae5f0b348f5bce516e1467ea6a3cddf9d4ec", "action": "add" }, "3": { "version": 3, - "hash": "9cc29f4b637e7a9407ff5f56e282b5411c2709e8195214c452e44b65f9051d56", + "hash": "0fe8c63c7ebf317c9b3791563eede28ce301dc0a2a1a98b13e657f34ed1e9edb", "action": "add" } }, "AnyActionObject": { "1": { "version": 1, - "hash": "bcb31f847907edc9c95d2d120dc5427854604f40940e3f41cd0474a1820ac65e", + "hash": "0a181a8950a9e91e06217a2b9bece1bfbb06ba732da4a267ce05b20dd2945b0e", "action": "add" }, "2": { "version": 2, - "hash": "002d8be821140befebbc0503e6bc1ef8779094e24e46305e5da5af6eecb56b13", + "hash": "60e1e4be3fc1486cddd73ae3594af46d40943c48bbe6f81f42322258712120bd", "action": "add" }, "3": { "version": 3, - "hash": "ad6b897da9bdf2fcacc853fad73b7d7b98c8d0ecb58c35110e236edefd99f561", + "hash": "0ac9122d40743966890247c7444c1033ba52bdbb0d2396daf8767adbe42faaad", "action": "add" } }, @@ -322,12 +322,12 @@ "BlobFileOBject": { "1": { "version": 1, - "hash": "8da2c80ced4f0414c671313c4b63d05846df1e397c763d99d803be86c29755bb", + "hash": "0fbf142f99e18fecc72e145f1d86c592dea1289c60eeef82fcf69c5c74b261fb", "action": "add" }, "2": { "version": 2, - "hash": "eb895881610723d674755033b0f1798219725d5ad70791d25a44f473ddff318b", + "hash": "cf3789022517ea88c968672566e7e3ae1dbf35c9f8ac5f09fd1ff7ca79534444", "action": "add" } }, @@ -510,7 +510,7 @@ "DataSubjectCreate": { "1": { "version": 1, - "hash": "5a94f9fcba75c50d78d71222f0235c5fd4d8003ae0db4d74bdbc4d56a99de3aa", + "hash": "f8f7f130fc2fcc54c5f984828cd3e2c7004c7d66e6ba7218adfa5b19d8d4b4a6", "action": "add" } }, @@ -552,7 +552,7 @@ "Dataset": { "1": { "version": 1, - "hash": "99ca2fa3e46fd9810222d269fac6accb546f632e94d5d57529016ba5e55af5a8", + "hash": "d16d64aa2b3b179f4bcdf88b837d4bda26d7a66d9505f473b56982634deb3808", "action": "add" } }, @@ -566,7 +566,7 @@ "CreateDataset": { "1": { "version": 1, - "hash": "3b020d9b8928cbd7e91f41c749ab4c932e19520696a183f2c7cd1312ebb640d1", + "hash": "1a53db706b173758f47097d9fd47b435228cf73a50315c54741015652805be52", "action": "add" } }, @@ -590,7 +590,12 @@ "JobInfo": { "1": { "version": 1, - "hash": "b2b88e49c6f6e5d4587d52099b521b258b62c9d685f769207c0b05cd4079507b", + "hash": "cf26eeac3d9254dfa439917493b816341f8a379a77d182bbecba3b7ed2c1d00a", + "action": "add" + }, + "2": { + "version": 2, + "hash": "5c1f7d5e6a991123a1907c1823be14a75458ba06af1fe5a1b77aaac7fa546c78", "action": "add" } }, @@ -623,16 +628,16 @@ } }, "OutputPolicyExecuteCount": { - "1": { - "version": 1, - "hash": "4fd95093a1fa04c3a6582df60db27931b6d38dd4f59b7e4151847dfcce8b57de", + "2": { + "version": 2, + "hash": "ca0ba249f4f32379f5b83279a27df4a21eb23c531a86538c821a10ddf2c799ff", "action": "add" } }, "OutputPolicyExecuteOnce": { - "1": { - "version": 1, - "hash": "89884332bb98f64e9c992e69dd260c9c2083cb59f9863b001b670da03b26d673", + "2": { + "version": 2, + "hash": "e6b0f23047037734c1cc448771bc2770f5bf6c8b8f80cf46939eb7ba66dd377e", "action": "add" } }, @@ -783,85 +788,85 @@ "NumpyArrayObject": { "1": { "version": 1, - "hash": "dcc7b44fa5ad22ae0bc576948f856c172dac1e9de2bc8e2a302e428f3309a278", + "hash": "6154a7b13c177ae6584965648bf1bf32e187e8a97f297bce1cdfca63f83a86a5", "action": "add" }, "2": { "version": 2, - "hash": "2c631121d9211006edab5620b214dea83e2398bee92244d822227ee316647e22", + "hash": "4139e24efefd95a64d853c9701d3cc742704217a36e25313ab7f9d3b54027cd4", "action": "add" }, "3": { "version": 3, - "hash": "8572cdfb6a9f8b20c276aeebaf12cd26e424703ea624cf52efdbe33a8ab3d4ef", + "hash": "b6c27c63285f55425942296a91bb16010fd359909fb82fcd52efa9e744e5f2a4", "action": "add" } }, "NumpyScalarObject": { "1": { "version": 1, - "hash": "5c1b6b6e8ba88bc79e76646d621489b889fe8f9b9fd59f117d594be18a409633", + "hash": "713fc4e6e4429ed34e49a0cf156a300ff73b53673af4cba7443bd83887c4ac19", "action": "add" }, "2": { "version": 2, - "hash": "0d5d81b9d45c140f6e07b43ed68d31e0ef060d6b4d0431c9b4795997bb35c69d", + "hash": "8cbd24a20bb40b58c68bea330306a51cf6376f077b298f16d185c3e9470150f1", "action": "add" }, "3": { "version": 3, - "hash": "d3107530ef42a6f3b234423a3c5efcb13f4b43a5959642a0b1e3c093992d59d1", + "hash": "028e645eea21425a049a56393218c2e89343edf09e9ff70d7fed6561c6508a43", "action": "add" } }, "NumpyBoolObject": { "1": { "version": 1, - "hash": "a5c822a6a3ca9eefd6a2b68f7fd0bc614fba7995f6bcc30bdc9dc882296b9b16", + "hash": "4c4562a531fe6c067b5319c2c9735e4108474eae599ca3f75622df6550f322f1", "action": "add" }, "2": { "version": 2, - "hash": "24839ba1c88ed833a134124750d5f299abcdf318670315028ed87b254f4578b3", + "hash": "1c1c6991d0ed36be43185bd991be33f54c7b5d938b1400a2b59b85e37c00a2b0", "action": "add" }, "3": { "version": 3, - "hash": "b9cd5c27630b0c67465d21e1d67a0b2591cd59aa6ac7d60ab1f5f64a746114df", + "hash": "e36b44d1829aff0e127bb1ba7b8e8f6853d6cf94cc86ef11c521019f1eec7e96", "action": "add" } }, "PandasDataframeObject": { "1": { "version": 1, - "hash": "35058924b3de2e0a604a92f91f4dd2e3cc0dac80c219d34f360e7cedd52f5f4c", + "hash": "0c677a65a31ec6f8b74c8d575edc4013950380cde7bb78e0fe206d16205658ec", "action": "add" }, "2": { "version": 2, - "hash": "66729d4ba7a92210d45c5a5c24fbdb4c8e58138a515a7bdb71ac8f6e8b868544", + "hash": "fe98ec3d8e49b9c1302f29fede6c98b01753ecee0e035996eccb158e16f5b3d3", "action": "add" }, "3": { "version": 3, - "hash": "f1ec7d6debb6d2d81e752418dd84544995b16d0386da791c4770914a227700d9", + "hash": "90fb7e7e5c7b03f37573012029c6979ccaaa44e720a48a7f829d83c6a41393e5", "action": "add" } }, "PandasSeriesObject": { "1": { "version": 1, - "hash": "2a0d8a55f1c27bd8fccd276cbe01bf272c40cab10417d7027273983fed423caa", + "hash": "4b9a9a980fe48bb7692db4d268421894e9e65606fd3633e9a8c752152d8a0aed", "action": "add" }, "2": { "version": 2, - "hash": "cb05a714f75b1140a943f56a3622fcc0477b3a1f504cd545a98510959ffe1528", + "hash": "58c33bfb4963a371afc8c18a1368753339b6c2b5731801ab9ef47d6211924f53", "action": "add" }, "3": { "version": 3, - "hash": "96f75160ec2a966fc3866cf2e86b151914012514e2b89f618933cdfbb639e528", + "hash": "50d5d68c0b4d57f8ecf594ee9761a6b4a9cd726354a4c8e3ff28e4e0a2fe58a4", "action": "add" } }, @@ -971,7 +976,12 @@ }, "2": { "version": 2, - "hash": "4193847a32c3aa358ff67934ee661a60055688c1340051fbd46d2a5e3aac7898", + "hash": "d83e0905ae882c824ba8fbbf455cd3881906bf8b2ebbfff07bcf471ef869cedc", + "action": "add" + }, + "3": { + "version": 3, + "hash": "999ab977d4fe5a7b74ee2d90370599ce9caa1b38fd6e6c29bd543d379c4dae31", "action": "add" } }, @@ -1069,7 +1079,7 @@ "Plan": { "1": { "version": 1, - "hash": "a0bba2b7792c9e08c453e9e256f0ac6e6185610726566bcd50b057ae83b42d9a", + "hash": "1aecbc00bbbb99745a1919f98a48bd27ba020d4968dd7c215e4f422dbfed0d92", "action": "add" } }