From 095fb4e72d59787d704af5330dcae5d2e8fc1148 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Tue, 16 Jul 2024 23:03:03 +0200 Subject: [PATCH 01/11] make serialization of type vars less dynamic --- packages/syft/src/syft/client/api.py | 1 + .../syft/src/syft/custom_worker/config.py | 1 + .../src/syft/protocol/protocol_version.json | 14 +++++ .../src/syft/serde/recursive_primitives.py | 53 ++++++++++++++----- .../src/syft/service/action/action_object.py | 1 + .../syft/src/syft/service/network/routes.py | 1 + .../src/syft/service/notifier/notifier.py | 1 + .../syft/src/syft/service/project/project.py | 1 + packages/syft/src/syft/types/syft_object.py | 1 + .../src/syft/types/syft_object_registry.py | 13 +++-- 10 files changed, 72 insertions(+), 15 deletions(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 6f7253076c4..55c38a3d207 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -1401,6 +1401,7 @@ def validate_callable_args_and_kwargs( pass else: _type_str = getattr(t, "__name__", str(t)) + raise msg = f"Arg is `{arg}`. \nIt must be of type `{_type_str}`, not `{type(arg).__name__}`" if msg: diff --git a/packages/syft/src/syft/custom_worker/config.py b/packages/syft/src/syft/custom_worker/config.py index 254d64b702e..f01266221bd 100644 --- a/packages/syft/src/syft/custom_worker/config.py +++ b/packages/syft/src/syft/custom_worker/config.py @@ -79,6 +79,7 @@ def merged_custom_cmds(self, sep: str = ";") -> str: return sep.join(self.custom_cmds) +@serializable(canonical_name="WorkerConfig", version=1) class WorkerConfig(SyftBaseModel): pass diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index 6602155d200..c500d823f3a 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -1190,6 +1190,20 @@ "hash": "ae07a6345762b8ebe9d2a100776e2405fd17516c9d224913a3358c96480ba889", "action": "add" } + }, + "SyftObject": { + "2": { + "version": 2, + "hash": "26efd140219eb651d18467b0430b9dbde17c57670f050e300823075f6e44d3e5", + "action": "add" + } + }, + "ProjectEvent": { + "2": { + "version": 2, + "hash": "cbe61714b816b769a856fe7b47c3001e6da716883b2845e03485e3e7b0a02b91", + "action": "add" + } } } } diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index 9385219b8e0..8808979c45c 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -8,6 +8,7 @@ from enum import Enum from enum import EnumMeta import functools +import inspect import pathlib from pathlib import PurePath import sys @@ -24,8 +25,12 @@ from typing import _SpecialGenericAlias from typing import _UnionGenericAlias from typing import cast +import typing import weakref +from result import Result +from syft.types.syft_object_registry import SyftObjectRegistry + # relative from .capnp import get_capnp_schema from .recursive import chunk_bytes @@ -169,22 +174,29 @@ def deserialize_enum(enum_type: type, enum_buf: bytes) -> Enum: return enum_type(enum_value) -def serialize_type(serialized_type: type) -> bytes: +def serialize_type(_type_to_serialize: type) -> bytes: # relative - from ..util.util import full_name_with_qualname + type_to_serialize = typing.get_origin(_type_to_serialize) or _type_to_serialize + canonical_name, version = SyftObjectRegistry.get_identifier_for_type(type_to_serialize) + return f"{canonical_name}:{version}".encode() - fqn = full_name_with_qualname(klass=serialized_type) - module_parts = fqn.split(".") - return ".".join(module_parts).encode() + # from ..util.util import full_name_with_qualname + + # fqn = full_name_with_qualname(klass=serialized_type) + # module_parts = fqn.split(".") + # return ".".join(module_parts).encode() def deserialize_type(type_blob: bytes) -> type: deserialized_type = type_blob.decode() - module_parts = deserialized_type.split(".") - klass = module_parts.pop() - klass = "None" if klass == "NoneType" else klass - exception_type = getattr(sys.modules[".".join(module_parts)], klass) - return exception_type + canonical_name, version = deserialized_type.split(":", 1) + return SyftObjectRegistry.get_serde_class(canonical_name, int(version)) + + # module_parts = deserialized_type.split(".") + # klass = module_parts.pop() + # klass = "None" if klass == "NoneType" else klass + # exception_type = getattr(sys.modules[".".join(module_parts)], klass) + # return exception_type TPath = TypeVar("TPath", bound=PurePath) @@ -470,6 +482,19 @@ def deserialize_union_type(type_blob: bytes) -> type: args = _deserialize(type_blob, from_bytes=True) return functools.reduce(lambda x, y: x | y, args) +def serialize_union(serialized_type: UnionType) -> bytes: + return b'' + +def deserialize_union(type_blob: bytes) -> type: + return Union + +def serialize_typevar(serialized_type: TypeVar) -> bytes: + return f'{serialized_type.__name__}'.encode() + +def deserialize_typevar(type_blob: bytes) -> type: + name = type_blob.decode() + return TypeVar(name=name) # type: ignore + recursive_serde_register( UnionType, @@ -481,8 +506,8 @@ def deserialize_union_type(type_blob: bytes) -> type: recursive_serde_register_type(_SpecialForm, canonical_name="_SpecialForm", version=1) recursive_serde_register_type(_GenericAlias, canonical_name="_GenericAlias", version=1) -recursive_serde_register_type(Union, canonical_name="Union", version=1) -recursive_serde_register_type(TypeVar, canonical_name="TypeVar", version=1) +recursive_serde_register(Union, canonical_name="Union", serialize=serialize_union, deserialize=deserialize_union, version=1) +recursive_serde_register(TypeVar, canonical_name="TypeVar", serialize=serialize_typevar, deserialize=deserialize_typevar, version=1) recursive_serde_register_type( _UnionGenericAlias, @@ -507,3 +532,7 @@ def deserialize_union_type(type_blob: bytes) -> type: recursive_serde_register_type(EnumMeta, canonical_name="EnumMeta", version=1) recursive_serde_register_type(ABCMeta, canonical_name="ABCMeta", version=1) + +recursive_serde_register_type(inspect._empty, canonical_name="inspect_empty", version=1) + + diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index ca277149620..3ef0c9fc414 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -235,6 +235,7 @@ def repr_uid(_id: LineageID) -> str: ) +@serializable(canonical_name="ActionObjectPointer", version=1) class ActionObjectPointer: pass diff --git a/packages/syft/src/syft/service/network/routes.py b/packages/syft/src/syft/service/network/routes.py index e64c63ac77a..372ceaa915c 100644 --- a/packages/syft/src/syft/service/network/routes.py +++ b/packages/syft/src/syft/service/network/routes.py @@ -31,6 +31,7 @@ from .server_peer import ServerPeer +@serializable(canonical_name="ServerRoute", version=1) class ServerRoute: def client_with_context( self, context: ServerServiceContext diff --git a/packages/syft/src/syft/service/notifier/notifier.py b/packages/syft/src/syft/service/notifier/notifier.py index 01a601d0da9..26dafe34e44 100644 --- a/packages/syft/src/syft/service/notifier/notifier.py +++ b/packages/syft/src/syft/service/notifier/notifier.py @@ -31,6 +31,7 @@ def send( TBaseNotifier = TypeVar("TBaseNotifier", bound=BaseNotifier) +@serializable(canonical_name="EmailNotifier", version=1) class EmailNotifier(BaseNotifier): smtp_client: SMTPClient sender = "" diff --git a/packages/syft/src/syft/service/project/project.py b/packages/syft/src/syft/service/project/project.py index e3a3d292951..d81f9e8ea07 100644 --- a/packages/syft/src/syft/service/project/project.py +++ b/packages/syft/src/syft/service/project/project.py @@ -65,6 +65,7 @@ def metadata_to_server_identity() -> list[Callable]: return [rename("id", "server_id"), rename("name", "server_name")] +@serializable() class ProjectEvent(SyftObject): __canonical_name__ = "ProjectEvent" __version__ = SYFT_OBJECT_VERSION_2 diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 4e4218cb07b..64625b45bbb 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -305,6 +305,7 @@ def get_migration_for_version( ] +@serializable() class SyftObject(SyftBaseObject, SyftMigrationRegistry): __canonical_name__ = "SyftObject" __version__ = SYFT_OBJECT_VERSION_2 diff --git a/packages/syft/src/syft/types/syft_object_registry.py b/packages/syft/src/syft/types/syft_object_registry.py index 7226e6a246f..d3dc9219a3b 100644 --- a/packages/syft/src/syft/types/syft_object_registry.py +++ b/packages/syft/src/syft/types/syft_object_registry.py @@ -38,6 +38,14 @@ def get_versions(cls, canonical_name: str) -> list[int]: ) return list(available_versions.keys()) + + @classmethod + def get_identifier_for_type(cls, obj: Any) -> tuple[str, int]: + """ + This is to create the string in nonrecursiveBlob + """ + return cls.__type_to_canonical_name__[obj] + @classmethod def get_canonical_name_version(cls, obj: Any) -> tuple[str, int]: """ @@ -53,7 +61,7 @@ def get_canonical_name_version(cls, obj: Any) -> tuple[str, int]: get_canonical_name_version([1,2,3]) -> "list" get_canonical_name_version(list) -> "type" get_canonical_name_version(MyEnum.A) -> "MyEnum" - get_canonical_name_version(MyEnum) -> "EnumMeta" + get_canonical_name_version(MyEnum) -> "type" Args: obj: The object or type for which to get the canonical name. @@ -62,8 +70,7 @@ def get_canonical_name_version(cls, obj: Any) -> tuple[str, int]: The canonical name and version of the object or type. """ - # NOTE the metaclass of the object is not needed during serde - # so we can safely ignore it + # for types we return "type" if isinstance(obj, type): return cls.__type_to_canonical_name__[type] From 911b4eab57e2ce476bc58bf2afc2b57a4e9bb567 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 17 Jul 2024 10:58:30 +0200 Subject: [PATCH 02/11] add serialization to a few classes --- .../src/syft/serde/recursive_primitives.py | 44 ++++++++++++------- .../service/notification/email_templates.py | 4 ++ packages/syft/src/syft/types/blob_storage.py | 2 + 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index 8808979c45c..e1036eb4f94 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -15,6 +15,7 @@ import tempfile from types import MappingProxyType from types import UnionType +import typing from typing import Any from typing import GenericAlias from typing import Optional @@ -25,13 +26,10 @@ from typing import _SpecialGenericAlias from typing import _UnionGenericAlias from typing import cast -import typing import weakref -from result import Result -from syft.types.syft_object_registry import SyftObjectRegistry - # relative +from ..types.syft_object_registry import SyftObjectRegistry from .capnp import get_capnp_schema from .recursive import chunk_bytes from .recursive import combine_bytes @@ -176,8 +174,10 @@ def deserialize_enum(enum_type: type, enum_buf: bytes) -> Enum: def serialize_type(_type_to_serialize: type) -> bytes: # relative - type_to_serialize = typing.get_origin(_type_to_serialize) or _type_to_serialize - canonical_name, version = SyftObjectRegistry.get_identifier_for_type(type_to_serialize) + type_to_serialize = typing.get_origin(_type_to_serialize) or _type_to_serialize + canonical_name, version = SyftObjectRegistry.get_identifier_for_type( + type_to_serialize + ) return f"{canonical_name}:{version}".encode() # from ..util.util import full_name_with_qualname @@ -482,18 +482,22 @@ def deserialize_union_type(type_blob: bytes) -> type: args = _deserialize(type_blob, from_bytes=True) return functools.reduce(lambda x, y: x | y, args) + def serialize_union(serialized_type: UnionType) -> bytes: - return b'' + return b"" + + +def deserialize_union(type_blob: bytes) -> type: # type: ignore + return Union # type: ignore -def deserialize_union(type_blob: bytes) -> type: - return Union def serialize_typevar(serialized_type: TypeVar) -> bytes: - return f'{serialized_type.__name__}'.encode() + return f"{serialized_type.__name__}".encode() + def deserialize_typevar(type_blob: bytes) -> type: name = type_blob.decode() - return TypeVar(name=name) # type: ignore + return TypeVar(name=name) # type: ignore recursive_serde_register( @@ -506,8 +510,20 @@ def deserialize_typevar(type_blob: bytes) -> type: recursive_serde_register_type(_SpecialForm, canonical_name="_SpecialForm", version=1) recursive_serde_register_type(_GenericAlias, canonical_name="_GenericAlias", version=1) -recursive_serde_register(Union, canonical_name="Union", serialize=serialize_union, deserialize=deserialize_union, version=1) -recursive_serde_register(TypeVar, canonical_name="TypeVar", serialize=serialize_typevar, deserialize=deserialize_typevar, version=1) +recursive_serde_register( + Union, + canonical_name="Union", + serialize=serialize_union, + deserialize=deserialize_union, + version=1, +) +recursive_serde_register( + TypeVar, + canonical_name="TypeVar", + serialize=serialize_typevar, + deserialize=deserialize_typevar, + version=1, +) recursive_serde_register_type( _UnionGenericAlias, @@ -534,5 +550,3 @@ def deserialize_typevar(type_blob: bytes) -> type: recursive_serde_register_type(ABCMeta, canonical_name="ABCMeta", version=1) recursive_serde_register_type(inspect._empty, canonical_name="inspect_empty", version=1) - - diff --git a/packages/syft/src/syft/service/notification/email_templates.py b/packages/syft/src/syft/service/notification/email_templates.py index 1a6965365dc..f8baceee38a 100644 --- a/packages/syft/src/syft/service/notification/email_templates.py +++ b/packages/syft/src/syft/service/notification/email_templates.py @@ -3,6 +3,7 @@ from typing import cast # relative +from ...serde.serializable import serializable from ...store.linked_obj import LinkedObject from ..context import AuthedServiceContext @@ -21,6 +22,7 @@ def email_body(notification: "Notification", context: AuthedServiceContext) -> s return "" +@serializable(canonical_name="OnboardEmailTemplate", version=1) class OnBoardEmailTemplate(EmailTemplate): @staticmethod def email_title(notification: "Notification", context: AuthedServiceContext) -> str: @@ -107,6 +109,7 @@ def email_body(notification: "Notification", context: AuthedServiceContext) -> s return f"""{head} {body}""" +@serializable(canonical_name="RequestEmailTemplate", version=1) class RequestEmailTemplate(EmailTemplate): @staticmethod def email_title(notification: "Notification", context: AuthedServiceContext) -> str: @@ -254,6 +257,7 @@ def email_body(notification: "Notification", context: AuthedServiceContext) -> s return f"""{head} {body}""" +@serializable(canonical_name="RequestUpdateEmailTemplate", version=1) class RequestUpdateEmailTemplate(EmailTemplate): @staticmethod def email_title(notification: "Notification", context: AuthedServiceContext) -> str: diff --git a/packages/syft/src/syft/types/blob_storage.py b/packages/syft/src/syft/types/blob_storage.py index 364a3428b0b..247cf94b365 100644 --- a/packages/syft/src/syft/types/blob_storage.py +++ b/packages/syft/src/syft/types/blob_storage.py @@ -187,10 +187,12 @@ def _coll_repr_(self) -> dict[str, str]: return {"file_name": self.file_name} +@serializable(canonical_name="BlobFileType", version=1) class BlobFileType(type): pass +@serializable(canonical_name="BlobFileObjectPointer", version=1) class BlobFileObjectPointer(ActionObjectPointer): pass From 9cedc5122346761fe557c6a61286d7541a498b9b Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 17 Jul 2024 11:05:08 +0200 Subject: [PATCH 03/11] lint lint lint --- packages/syft/src/syft/client/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index 55c38a3d207..6f7253076c4 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -1401,7 +1401,6 @@ def validate_callable_args_and_kwargs( pass else: _type_str = getattr(t, "__name__", str(t)) - raise msg = f"Arg is `{arg}`. \nIt must be of type `{_type_str}`, not `{type(arg).__name__}`" if msg: From 29f5a17661aa1729680a93a3ba21800c867340b1 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 17 Jul 2024 11:10:08 +0200 Subject: [PATCH 04/11] lint lint lint --- packages/syft/src/syft/types/syft_object_registry.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/src/syft/types/syft_object_registry.py b/packages/syft/src/syft/types/syft_object_registry.py index d3dc9219a3b..68952b5874b 100644 --- a/packages/syft/src/syft/types/syft_object_registry.py +++ b/packages/syft/src/syft/types/syft_object_registry.py @@ -38,7 +38,6 @@ def get_versions(cls, canonical_name: str) -> list[int]: ) return list(available_versions.keys()) - @classmethod def get_identifier_for_type(cls, obj: Any) -> tuple[str, int]: """ From 5ea3cbd6230573e21ef35ca14ff76f41334de206 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Wed, 17 Jul 2024 11:36:34 +0200 Subject: [PATCH 05/11] fix numpy test --- packages/syft/src/syft/serde/array.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/syft/src/syft/serde/array.py b/packages/syft/src/syft/serde/array.py index fa1ed27e74b..3f19e575b97 100644 --- a/packages/syft/src/syft/serde/array.py +++ b/packages/syft/src/syft/serde/array.py @@ -162,6 +162,14 @@ version=SYFT_OBJECT_VERSION_1, ) +recursive_serde_register( + np.number, + serialize=lambda x: x.tobytes(), + deserialize=lambda buffer: frombuffer(buffer, dtype=np.number)[0], + canonical_name="numpy_number", + version=SYFT_OBJECT_VERSION_1, +) + # TODO: There is an incorrect mapping in looping,which makes it not work. # numpy_scalar_types = [ # np.bool_, From 01f51e62e0c5c7aa7924b3b3ec4656329decea1f Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Thu, 18 Jul 2024 14:26:49 +0200 Subject: [PATCH 06/11] fix py310 any serialization --- notebooks/api/0.8/00-load-data.ipynb | 19 +++++++++++-------- .../src/syft/serde/recursive_primitives.py | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/notebooks/api/0.8/00-load-data.ipynb b/notebooks/api/0.8/00-load-data.ipynb index 5370bd2e595..0d7d6b21a57 100644 --- a/notebooks/api/0.8/00-load-data.ipynb +++ b/notebooks/api/0.8/00-load-data.ipynb @@ -609,11 +609,7 @@ }, { "cell_type": "markdown", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "source": [ "### Reading the Syft Dataset from Datasite Server\n", "\n", @@ -708,6 +704,13 @@ " server.land()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -718,9 +721,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python [conda env:syft310] *", "language": "python", - "name": "python3" + "name": "conda-env-syft310-py" }, "language_info": { "codemirror_mode": { @@ -732,7 +735,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.2" + "version": "3.10.14" }, "toc": { "base_numbering": 1, diff --git a/packages/syft/src/syft/serde/recursive_primitives.py b/packages/syft/src/syft/serde/recursive_primitives.py index e1036eb4f94..38d8281434d 100644 --- a/packages/syft/src/syft/serde/recursive_primitives.py +++ b/packages/syft/src/syft/serde/recursive_primitives.py @@ -446,6 +446,8 @@ def recursive_serde_register_type( canonical_name: str | None = None, version: int | None = None, ) -> None: + # former case is for instance for _GerericAlias itself or UnionGenericAlias + # Latter case is true for for instance List[str], which is currently not used if (isinstance(t, type) and issubclass(t, _GenericAlias)) or issubclass( type(t), _GenericAlias ): @@ -500,6 +502,14 @@ def deserialize_typevar(type_blob: bytes) -> type: return TypeVar(name=name) # type: ignore +def serialize_any(serialized_type: TypeVar) -> bytes: + return b"" + + +def deserialize_any(type_blob: bytes) -> type: # type: ignore + return Any # type: ignore + + recursive_serde_register( UnionType, serialize=serialize_union_type, @@ -524,6 +534,13 @@ def deserialize_typevar(type_blob: bytes) -> type: deserialize=deserialize_typevar, version=1, ) +recursive_serde_register( + Any, + canonical_name="Any", + serialize=serialize_any, + deserialize=deserialize_any, + version=1, +) recursive_serde_register_type( _UnionGenericAlias, @@ -544,7 +561,7 @@ def deserialize_typevar(type_blob: bytes) -> type: ) recursive_serde_register_type(GenericAlias, canonical_name="GenericAlias", version=1) -recursive_serde_register_type(Any, canonical_name="Any", version=1) +# recursive_serde_register_type(Any, canonical_name="Any", version=1) recursive_serde_register_type(EnumMeta, canonical_name="EnumMeta", version=1) recursive_serde_register_type(ABCMeta, canonical_name="ABCMeta", version=1) From 3cf2ddf2528e380635302ac5f9a17e6e860c9a77 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Thu, 18 Jul 2024 14:51:16 +0200 Subject: [PATCH 07/11] merge undo nb changes --- notebooks/api/0.8/00-load-data.ipynb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/notebooks/api/0.8/00-load-data.ipynb b/notebooks/api/0.8/00-load-data.ipynb index 0d7d6b21a57..5370bd2e595 100644 --- a/notebooks/api/0.8/00-load-data.ipynb +++ b/notebooks/api/0.8/00-load-data.ipynb @@ -609,7 +609,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "jupyter": { + "source_hidden": true + } + }, "source": [ "### Reading the Syft Dataset from Datasite Server\n", "\n", @@ -704,13 +708,6 @@ " server.land()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, { "cell_type": "code", "execution_count": null, @@ -721,9 +718,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:syft310] *", + "display_name": "Python 3 (ipykernel)", "language": "python", - "name": "conda-env-syft310-py" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -735,7 +732,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.12.2" }, "toc": { "base_numbering": 1, From 6d290905c989aebeaef546d4c67a3d5aec2fe120 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Thu, 18 Jul 2024 15:21:48 +0200 Subject: [PATCH 08/11] fix syftobject --- .../src/syft/protocol/protocol_version.json | 19 +++++++++++++------ packages/syft/src/syft/types/syft_object.py | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index a1098e600ee..b4793310320 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -1052,16 +1052,23 @@ } }, "SyftObject": { - "2": { - "version": 2, - "hash": "26efd140219eb651d18467b0430b9dbde17c57670f050e300823075f6e44d3e5", + "1": { + "version": 1, + "hash": "bb70d874355988908d3a92a3941d6613a6995a4850be3b6a0147f4d387724406", "action": "add" } }, "ProjectEvent": { - "2": { - "version": 2, - "hash": "cbe61714b816b769a856fe7b47c3001e6da716883b2845e03485e3e7b0a02b91", + "1": { + "version": 1, + "hash": "dc0486c52daebd5e98c2b3b03ffd9a9a14bc3d86d8dc0c23e41ebf6c31fe2ffb", + "action": "add" + } + }, + "SyftObjectVersioned": { + "1": { + "version": 1, + "hash": "7c842dcdbb57e2528ffa690ea18c19fff3c8a591811d40cad2b19be3100e2ff4", "action": "add" } } diff --git a/packages/syft/src/syft/types/syft_object.py b/packages/syft/src/syft/types/syft_object.py index 3a84ef5bc3e..f987479cafc 100644 --- a/packages/syft/src/syft/types/syft_object.py +++ b/packages/syft/src/syft/types/syft_object.py @@ -346,6 +346,7 @@ def __lt__(self, other: Self) -> bool: return self.utc_timestamp < other.utc_timestamp +@serializable() class SyftObject(SyftObjectVersioned): __canonical_name__ = "SyftObject" __version__ = SYFT_OBJECT_VERSION_1 From 3fbbdf301747696b1480f8a3b1aa77b8dc3078c1 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Thu, 18 Jul 2024 18:17:20 +0200 Subject: [PATCH 09/11] fix any --- packages/syft/src/syft/types/syft_object_registry.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/types/syft_object_registry.py b/packages/syft/src/syft/types/syft_object_registry.py index 68952b5874b..a86bed7ddee 100644 --- a/packages/syft/src/syft/types/syft_object_registry.py +++ b/packages/syft/src/syft/types/syft_object_registry.py @@ -80,7 +80,15 @@ def get_canonical_name_version(cls, obj: Any) -> tuple[str, int]: @classmethod def get_serde_properties(cls, canonical_name: str, version: int) -> tuple: - return cls.__object_serialization_registry__[canonical_name][version] + try: + return cls.__object_serialization_registry__[canonical_name][version] + except Exception: + # This is a hack for python 3.10 in which Any is not a type + # if the server uses py>3.10 and the client 3.10 this goes wrong + if canonical_name == "Any_typing._SpecialForm": + return cls.__object_serialization_registry__["Any"][version] + else: + raise @classmethod def get_serde_class(cls, canonical_name: str, version: int) -> type["SyftObject"]: From b1910663dd27c1100b1eb89e816fa02f5805d052 Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Fri, 19 Jul 2024 11:33:36 +0200 Subject: [PATCH 10/11] remove fullyQualifiedName --- .../syft/src/syft/capnp/recursive_serde.capnp | 7 +- .../src/syft/protocol/protocol_version.json | 42 +++++------ packages/syft/src/syft/serde/recursive.py | 55 ++++---------- .../src/syft/types/syft_object_registry.py | 72 ++----------------- 4 files changed, 43 insertions(+), 133 deletions(-) diff --git a/packages/syft/src/syft/capnp/recursive_serde.capnp b/packages/syft/src/syft/capnp/recursive_serde.capnp index c29ba57aae6..5b6fadb5c65 100644 --- a/packages/syft/src/syft/capnp/recursive_serde.capnp +++ b/packages/syft/src/syft/capnp/recursive_serde.capnp @@ -3,8 +3,7 @@ struct RecursiveSerde { fieldsName @0 :List(Text); fieldsData @1 :List(List(Data)); - fullyQualifiedName @2 :Text; - nonrecursiveBlob @3 :List(Data); - canonicalName @4 :Text; - version @5 :Int32; + nonrecursiveBlob @2 :List(Data); + canonicalName @3 :Text; + version @4 :Int32; } diff --git a/packages/syft/src/syft/protocol/protocol_version.json b/packages/syft/src/syft/protocol/protocol_version.json index b4793310320..fa34ec9c538 100644 --- a/packages/syft/src/syft/protocol/protocol_version.json +++ b/packages/syft/src/syft/protocol/protocol_version.json @@ -1,6 +1,13 @@ { "dev": { "object_versions": { + "SyftObjectVersioned": { + "1": { + "version": 1, + "hash": "7c842dcdbb57e2528ffa690ea18c19fff3c8a591811d40cad2b19be3100e2ff4", + "action": "add" + } + }, "BaseDateTime": { "1": { "version": 1, @@ -8,6 +15,13 @@ "action": "add" } }, + "SyftObject": { + "1": { + "version": 1, + "hash": "bb70d874355988908d3a92a3941d6613a6995a4850be3b6a0147f4d387724406", + "action": "add" + } + }, "PartialSyftObject": { "1": { "version": 1, @@ -988,6 +1002,13 @@ "action": "add" } }, + "ProjectEvent": { + "1": { + "version": 1, + "hash": "dc0486c52daebd5e98c2b3b03ffd9a9a14bc3d86d8dc0c23e41ebf6c31fe2ffb", + "action": "add" + } + }, "ProjectThreadMessage": { "1": { "version": 1, @@ -1050,27 +1071,6 @@ "hash": "ed05cb87aec832098fc464ac36cd6bceaab705463d0d2fa1b2d8e1ccc510018c", "action": "add" } - }, - "SyftObject": { - "1": { - "version": 1, - "hash": "bb70d874355988908d3a92a3941d6613a6995a4850be3b6a0147f4d387724406", - "action": "add" - } - }, - "ProjectEvent": { - "1": { - "version": 1, - "hash": "dc0486c52daebd5e98c2b3b03ffd9a9a14bc3d86d8dc0c23e41ebf6c31fe2ffb", - "action": "add" - } - }, - "SyftObjectVersioned": { - "1": { - "version": 1, - "hash": "7c842dcdbb57e2528ffa690ea18c19fff3c8a591811d40cad2b19be3100e2ff4", - "action": "add" - } } } } diff --git a/packages/syft/src/syft/serde/recursive.py b/packages/syft/src/syft/serde/recursive.py index 19ede3d0040..4c438975245 100644 --- a/packages/syft/src/syft/serde/recursive.py +++ b/packages/syft/src/syft/serde/recursive.py @@ -3,7 +3,6 @@ from enum import Enum from enum import EnumMeta import os -import sys import tempfile import types from typing import Any @@ -17,7 +16,6 @@ # relative from ..types.syft_object_registry import SyftObjectRegistry -from ..util.util import index_syft_by_module_name from .capnp import get_capnp_schema from .util import compatible_with_large_file_writes_capnp @@ -285,7 +283,7 @@ def rs_object2proto(self: Any, for_hashing: bool = False) -> _DynamicStructBuild # todo: rewrite and make sure every object has a canonical name and version canonical_name, version = SyftObjectRegistry.get_canonical_name_version(self) - if not SyftObjectRegistry.has_serde_class("", canonical_name, version): + if not SyftObjectRegistry.has_serde_class(canonical_name, version): # third party raise Exception( f"obj2proto: {canonical_name} version {version} not in SyftObjectRegistry" @@ -382,34 +380,12 @@ def rs_proto2object(proto: _DynamicStructBuilder) -> Any: # relative from .deserialize import _deserialize - # clean this mess, Tudor - module_parts = proto.fullyQualifiedName.split(".") - klass = module_parts.pop() class_type: type | Any = type(None) - if klass != "NoneType": - try: - class_type = index_syft_by_module_name(proto.fullyQualifiedName) # type: ignore[assignment,unused-ignore] - except Exception: # nosec - try: - class_type = getattr(sys.modules[".".join(module_parts)], klass) - except Exception: # nosec - if "syft.user" in proto.fullyQualifiedName: - # relative - from ..server.server import CODE_RELOADER - - for load_user_code in CODE_RELOADER.values(): - load_user_code() - try: - class_type = getattr(sys.modules[".".join(module_parts)], klass) - except Exception: # nosec - pass - canonical_name = proto.canonicalName version = getattr(proto, "version", -1) - fqn = getattr(proto, "fullyQualifiedName", "") - fqn = map_fqns_for_backward_compatibility(fqn) - if not SyftObjectRegistry.has_serde_class(fqn, canonical_name, version): + + if not SyftObjectRegistry.has_serde_class(canonical_name, version): # third party raise Exception( f"proto2obj: {canonical_name} version {version} not in SyftObjectRegistry" @@ -431,13 +407,9 @@ def rs_proto2object(proto: _DynamicStructBuilder) -> Any: cls, _, version, - ) = SyftObjectRegistry.get_serde_properties_bw_compatible( - fqn, canonical_name, version - ) + ) = SyftObjectRegistry.get_serde_properties(canonical_name, version) - if class_type == type(None) or fqn != "": - # yes this looks stupid but it works and the opposite breaks - class_type = cls + class_type = cls if nonrecursive: if deserialize is None: @@ -468,14 +440,15 @@ def rs_proto2object(proto: _DynamicStructBuilder) -> Any: # if we skip the __new__ flow of BaseModel we get the error # AttributeError: object has no attribute '__fields_set__' - if "syft.user" in proto.fullyQualifiedName: - # weird issues with pydantic and ForwardRef on user classes being inited - # with custom state args / kwargs - obj = class_type() - for attr_name, attr_value in kwargs.items(): - setattr(obj, attr_name, attr_value) - else: - obj = class_type(**kwargs) + # if "syft.user" in proto.fullyQualifiedName: + # # weird issues with pydantic and ForwardRef on user classes being inited + # # with custom state args / kwargs + # obj = class_type() + # for attr_name, attr_value in kwargs.items(): + # setattr(obj, attr_name, attr_value) + # else: + # obj = class_type(**kwargs) + obj = class_type(**kwargs) else: obj = class_type.__new__(class_type) # type: ignore diff --git a/packages/syft/src/syft/types/syft_object_registry.py b/packages/syft/src/syft/types/syft_object_registry.py index a86bed7ddee..d5cc342635e 100644 --- a/packages/syft/src/syft/types/syft_object_registry.py +++ b/packages/syft/src/syft/types/syft_object_registry.py @@ -96,74 +96,12 @@ def get_serde_class(cls, canonical_name: str, version: int) -> type["SyftObject" return serde_properties[7] @classmethod - def get_serde_properties_bw_compatible( - cls, fqn: str, canonical_name: str, version: int - ) -> tuple: + def has_serde_class(cls, canonical_name: str | None, version: int) -> bool: # relative - from ..serde.recursive import TYPE_BANK - - if canonical_name != "" and canonical_name is not None: - return cls.get_serde_properties(canonical_name, version) - else: - # this is for backward compatibility with 0.8.6 - try: - # relative - from ..protocol.data_protocol import get_data_protocol - - serde_props = TYPE_BANK[fqn] - klass = serde_props[7] - is_syftobject = hasattr(klass, "__canonical_name__") - if is_syftobject: - canonical_name = klass.__canonical_name__ - dp = get_data_protocol() - try: - version_mutations = dp.protocol_history[ - SYFT_086_PROTOCOL_VERSION - ]["object_versions"][canonical_name] - except Exception: - print(f"could not find {canonical_name} in protocol history") - raise - - version_086 = max( - [ - int(k) - for k, v in version_mutations.items() - if v["action"] == "add" - ] - ) - try: - res = cls.get_serde_properties(canonical_name, version_086) - - except Exception: - print( - f"could not find {canonical_name} {version_086} in ObjectRegistry" - ) - raise - return res - else: - # TODO, add refactoring for non syftobject versions - canonical_name = fqn - version = 1 - return cls.get_serde_properties(canonical_name, version) - except Exception as e: - print(e) - raise - - @classmethod - def has_serde_class( - cls, fqn: str, canonical_name: str | None, version: int - ) -> bool: - # relative - from ..serde.recursive import TYPE_BANK - - if canonical_name != "" and canonical_name is not None: - return ( - canonical_name in cls.__object_serialization_registry__ - and version in cls.__object_serialization_registry__[canonical_name] - ) - else: - # this is for backward compatibility with 0.8.6 - return fqn in TYPE_BANK + return ( + canonical_name in cls.__object_serialization_registry__ + and version in cls.__object_serialization_registry__[canonical_name] + ) @classmethod def add_transform( From 330a7000736c925e73ba0c461a314428a6f3e50a Mon Sep 17 00:00:00 2001 From: Koen van der Veen Date: Fri, 19 Jul 2024 11:51:33 +0200 Subject: [PATCH 11/11] lint --- packages/syft/src/syft/client/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index d60416fdf3b..0ff9299bdfe 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -32,7 +32,6 @@ from ..protocol.data_protocol import get_data_protocol from ..protocol.data_protocol import migrate_args_and_kwargs from ..serde.deserialize import _deserialize -from ..serde.recursive import index_syft_by_module_name from ..serde.serializable import serializable from ..serde.serialize import _serialize from ..serde.signature import Signature @@ -63,6 +62,7 @@ from ..util.markdown import as_markdown_python_code from ..util.notebook_ui.components.tabulator_template import build_tabulator_table from ..util.telemetry import instrument +from ..util.util import index_syft_by_module_name from ..util.util import prompt_warning_message from .connection import ServerConnection