From 6df91071d01be67cfb76b55731aff62f4fab8536 Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Sun, 31 Oct 2021 23:10:47 +0300 Subject: [PATCH 1/8] Add support for PEP 563 --- mashumaro/config.py | 1 + mashumaro/exceptions.py | 16 +++++++ mashumaro/meta/helpers.py | 20 ++++++++- mashumaro/meta/macros.py | 2 + mashumaro/serializer/base/dict.py | 23 ++++++---- mashumaro/serializer/base/metaprogramming.py | 9 +++- tests/test_pep_563.py | 46 ++++++++++++++++++++ 7 files changed, 106 insertions(+), 11 deletions(-) create mode 100644 tests/test_pep_563.py diff --git a/mashumaro/config.py b/mashumaro/config.py index cbe41b32..c6cc97a5 100644 --- a/mashumaro/config.py +++ b/mashumaro/config.py @@ -31,6 +31,7 @@ class BaseConfig: aliases: Dict[str, str] = {} serialize_by_alias: bool = False namedtuple_as_dict: bool = False + allow_postponed_evaluation: bool = True __all__ = [ diff --git a/mashumaro/exceptions.py b/mashumaro/exceptions.py index 24d68716..4807a066 100644 --- a/mashumaro/exceptions.py +++ b/mashumaro/exceptions.py @@ -121,3 +121,19 @@ def __str__(self): f"in {self.holder_class_name}" ) return s + + +class UnresolvedTypeReferenceError(NameError): + def __init__(self, holder_class, unresolved_type_name): + self.holder_class = holder_class + self.unresolved_type_name = unresolved_type_name + + @property + def holder_class_name(self): + return type_name(self.holder_class, short=True) + + def __str__(self): + return ( + f"Class {self.holder_class_name} has unresolved type reference " + f"{self.unresolved_type_name} in some of its fields" + ) diff --git a/mashumaro/meta/helpers.py b/mashumaro/meta/helpers.py index a2891766..bfdaef2c 100644 --- a/mashumaro/meta/helpers.py +++ b/mashumaro/meta/helpers.py @@ -1,5 +1,6 @@ import dataclasses import inspect +import re import types import typing from contextlib import suppress @@ -9,7 +10,15 @@ import typing_extensions -from .macros import PY_36, PY_37, PY_37_MIN, PY_38, PY_38_MIN, PY_39_MIN +from .macros import ( + PY_36, + PY_37, + PY_37_MIN, + PY_38, + PY_38_MIN, + PY_39_MIN, + PY_310_MIN, +) DataClassDictMixinPath = "mashumaro.serializer.base.dict.DataClassDictMixin" NoneType = type(None) @@ -290,8 +299,16 @@ def resolve_type_vars(cls, arg_types=(), is_cls_created=False): return result +def get_name_error_name(e: NameError) -> str: + if PY_310_MIN: + return e.name + else: + return re.search("'(.*)'", e.args[0]).group(1) + + __all__ = [ "get_type_origin", + "get_args", "type_name", "is_special_typing_primitive", "is_generic", @@ -309,4 +326,5 @@ def resolve_type_vars(cls, arg_types=(), is_cls_created=False): "is_dataclass_dict_mixin_subclass", "resolve_type_vars", "get_generic_name", + "get_name_error_name", ] diff --git a/mashumaro/meta/macros.py b/mashumaro/meta/macros.py index ccf10596..54fbf9e8 100644 --- a/mashumaro/meta/macros.py +++ b/mashumaro/meta/macros.py @@ -9,6 +9,7 @@ PY_37_MIN = PY_37 or PY_38 or PY_39 or PY_310 PY_38_MIN = PY_38 or PY_39 or PY_310 PY_39_MIN = PY_39 or PY_310 +PY_310_MIN = PY_310 PEP_585_COMPATIBLE = PY_39_MIN # Type Hinting Generics In Standard Collections PEP_586_COMPATIBLE = PY_38_MIN # Literal Types @@ -23,6 +24,7 @@ "PY_37_MIN", "PY_38_MIN", "PY_39_MIN", + "PY_310_MIN", "PEP_585_COMPATIBLE", "PEP_586_COMPATIBLE", ] diff --git a/mashumaro/serializer/base/dict.py b/mashumaro/serializer/base/dict.py index 64961072..2bf25aca 100644 --- a/mashumaro/serializer/base/dict.py +++ b/mashumaro/serializer/base/dict.py @@ -1,5 +1,6 @@ from typing import Any, Dict, Mapping, Type, TypeVar +from mashumaro.exceptions import UnresolvedTypeReferenceError from mashumaro.serializer.base.metaprogramming import CodeBuilder T = TypeVar("T", bound="DataClassDictMixin") @@ -10,17 +11,17 @@ class DataClassDictMixin: def __init_subclass__(cls: Type[T], **kwargs): builder = CodeBuilder(cls) - exc = None + config = builder.get_config() try: builder.add_from_dict() - except Exception as e: - exc = e + except UnresolvedTypeReferenceError: + if not config.allow_postponed_evaluation: + raise try: builder.add_to_dict() - except Exception as e: - exc = e - if exc: - raise exc + except UnresolvedTypeReferenceError: + if not config.allow_postponed_evaluation: + raise def to_dict( self: T, @@ -33,7 +34,9 @@ def to_dict( # by_alias: bool = False **kwargs, ) -> dict: - ... + builder = CodeBuilder(self.__class__) + builder.add_to_dict() + return self.to_dict(use_bytes, use_enum, use_datetime, **kwargs) @classmethod def from_dict( @@ -43,7 +46,9 @@ def from_dict( use_enum: bool = False, use_datetime: bool = False, ) -> T: - ... + builder = CodeBuilder(cls) + builder.add_from_dict() + return cls.from_dict(d, use_bytes, use_enum, use_datetime) @classmethod def __pre_deserialize__(cls: Type[T], d: Dict[Any, Any]) -> Dict[Any, Any]: diff --git a/mashumaro/serializer/base/metaprogramming.py b/mashumaro/serializer/base/metaprogramming.py index ca0073a0..76e546f0 100644 --- a/mashumaro/serializer/base/metaprogramming.py +++ b/mashumaro/serializer/base/metaprogramming.py @@ -31,6 +31,7 @@ InvalidFieldValue, MissingField, ThirdPartyModuleNotFoundError, + UnresolvedTypeReferenceError, UnserializableDataError, UnserializableField, UnsupportedDeserializationEngine, @@ -40,6 +41,7 @@ get_args, get_class_that_defines_field, get_class_that_defines_method, + get_name_error_name, get_type_origin, is_class_var, is_dataclass_dict_mixin, @@ -143,7 +145,12 @@ def __get_field_types( fields = {} globalns = sys.modules[self.cls.__module__].__dict__.copy() globalns[self.cls.__name__] = self.cls - for fname, ftype in typing.get_type_hints(self.cls, globalns).items(): + try: + field_type_hints = typing.get_type_hints(self.cls, globalns) + except NameError as e: + name = get_name_error_name(e) + raise UnresolvedTypeReferenceError(self.cls, name) from None + for fname, ftype in field_type_hints.items(): if is_class_var(ftype) or is_init_var(ftype): continue if recursive or fname in self.annotations: diff --git a/tests/test_pep_563.py b/tests/test_pep_563.py new file mode 100644 index 00000000..04e15ae5 --- /dev/null +++ b/tests/test_pep_563.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import pytest +from dataclasses import dataclass + +from mashumaro import DataClassDictMixin +from mashumaro.exceptions import UnresolvedTypeReferenceError +from mashumaro.config import BaseConfig + + +@dataclass +class A(DataClassDictMixin): + x: B + + +@dataclass +class B(DataClassDictMixin): + x: int + + +def test_postponed_annotation_evaluation(): + obj = A(x=B(x=1)) + assert obj.to_dict() == {"x": {"x": 1}} + assert A.from_dict({"x": {"x": 1}}) == obj + + +def test_unresolved_type_with_allowed_postponed_annotation_evaluation(): + @dataclass + class DataClass(DataClassDictMixin): + x: X + + with pytest.raises(UnresolvedTypeReferenceError): + DataClass.from_dict({}) + + with pytest.raises(UnresolvedTypeReferenceError): + DataClass(x=1).to_dict() + + +def test_unresolved_type_with_disallowed_postponed_annotation_evaluation(): + with pytest.raises(UnresolvedTypeReferenceError): + @dataclass + class DataClass(DataClassDictMixin): + x: X + + class Config(BaseConfig): + allow_postponed_evaluation = False From 0e014489556b952e7df8432fad2a3c52a4897096 Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Sun, 31 Oct 2021 23:14:51 +0300 Subject: [PATCH 2/8] Fix mypy --- mashumaro/meta/helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mashumaro/meta/helpers.py b/mashumaro/meta/helpers.py index bfdaef2c..14d8660c 100644 --- a/mashumaro/meta/helpers.py +++ b/mashumaro/meta/helpers.py @@ -301,9 +301,10 @@ def resolve_type_vars(cls, arg_types=(), is_cls_created=False): def get_name_error_name(e: NameError) -> str: if PY_310_MIN: - return e.name + return e.name # type: ignore else: - return re.search("'(.*)'", e.args[0]).group(1) + match = re.search("'(.*)'", e.args[0]) + return match.group(1) if match else '' __all__ = [ From 80d7ddb9d2e8d983ba64de815f4d4c038816c71d Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Sun, 31 Oct 2021 23:17:50 +0300 Subject: [PATCH 3/8] Fix black --- mashumaro/meta/helpers.py | 2 +- tests/test_pep_563.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mashumaro/meta/helpers.py b/mashumaro/meta/helpers.py index 14d8660c..6d9bba90 100644 --- a/mashumaro/meta/helpers.py +++ b/mashumaro/meta/helpers.py @@ -304,7 +304,7 @@ def get_name_error_name(e: NameError) -> str: return e.name # type: ignore else: match = re.search("'(.*)'", e.args[0]) - return match.group(1) if match else '' + return match.group(1) if match else "" __all__ = [ diff --git a/tests/test_pep_563.py b/tests/test_pep_563.py index 04e15ae5..1c3e3c37 100644 --- a/tests/test_pep_563.py +++ b/tests/test_pep_563.py @@ -38,6 +38,7 @@ class DataClass(DataClassDictMixin): def test_unresolved_type_with_disallowed_postponed_annotation_evaluation(): with pytest.raises(UnresolvedTypeReferenceError): + @dataclass class DataClass(DataClassDictMixin): x: X From 8219e3001a03f6b7f192030f12b31bba8ce31c97 Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Sun, 31 Oct 2021 23:43:38 +0300 Subject: [PATCH 4/8] Ignore PEP 563 tests on Python 3.6 --- tests/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..48a73104 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +from mashumaro.meta.macros import PY_37_MIN + + +if not PY_37_MIN: + collect_ignore = ["test_pep_563.py"] From 55f18e88437fc7d196c838d771e61d6171501e00 Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Mon, 1 Nov 2021 21:21:34 +0300 Subject: [PATCH 5/8] Fix test_no_code_builder --- tests/test_meta.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_meta.py b/tests/test_meta.py index 00f37a1f..d228c314 100644 --- a/tests/test_meta.py +++ b/tests/test_meta.py @@ -72,8 +72,6 @@ def test_no_code_builder(): class DataClass(DataClassDictMixin): pass - assert DataClass.from_dict({}) is None - assert DataClass().to_dict() is None assert DataClass.__pre_deserialize__({}) is None assert DataClass.__post_deserialize__(DataClass()) is None assert DataClass().__pre_serialize__() is None From 15e701ab2fc7b17bd659e20eaf3941b4ee1ee2ee Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Mon, 1 Nov 2021 22:21:18 +0300 Subject: [PATCH 6/8] Update README --- README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/README.md b/README.md index d430bb3d..8413f864 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Table of contents * [`aliases` config option](#aliases-config-option) * [`serialize_by_alias` config option](#serialize_by_alias-config-option) * [`namedtuple_as_dict` config option](#namedtuple_as_dict-config-option) + * [`allow_postponed_evaluation` config option](#allow_postponed_evaluation-config-option) * [Code generation options](#code-generation-options) * [Add `omit_none` keyword argument](#add-omit_none-keyword-argument) * [Add `by_alias` keyword argument](#add-by_alias-keyword-argument) @@ -847,6 +848,56 @@ If you want to serialize only certain named tuple fields as dictionaries, you can use the corresponding [serialization](#serialize-option) and [deserialization](#deserialize-option) engines. +#### `allow_postponed_evaluation` config option + +[PEP 563](https://www.python.org/dev/peps/pep-0563/) solved the problem of forward references by postponing the evaluation +of annotations, so you can write the following code: + +```python +from __future__ import annotations +from dataclasses import dataclass +from mashumaro import DataClassDictMixin + +@dataclass +class A(DataClassDictMixin): + x: B + +@dataclass +class B(DataClassDictMixin): + y: int + +obj = A.from_dict({'x': {'y': 1}}) +``` + +You don't need to write anything special here, forward references work out of +the box. If a field of a dataclass has a forward reference in the type +annotations, building of `from_dict` and `to_dict` methods of this dataclass +will be postponed until they are called once. However, if for some reason you +don't want the evaluation to be possibly postponed, you can disable it using +`allow_postponed_evaluation` option: + +```python +from __future__ import annotations +from dataclasses import dataclass +from mashumaro import DataClassDictMixin + +@dataclass +class A(DataClassDictMixin): + x: B + + class Config: + allow_postponed_evaluation = False + +# UnresolvedTypeReferenceError: Class A has unresolved type reference B +# in some of its fields + +@dataclass +class B(DataClassDictMixin): + y: int +``` + +In this case you will get `UnresolvedTypeReferenceError` regardless of whether class B is declared below or not. + ### Code generation options #### Add `omit_none` keyword argument From b7dd17aa06c00b575e24daeb00de5b964bf97d5b Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Mon, 1 Nov 2021 22:32:05 +0300 Subject: [PATCH 7/8] Increase coverage --- mashumaro/exceptions.py | 4 ++-- mashumaro/serializer/base/dict.py | 2 +- tests/conftest.py | 1 - tests/test_exceptions.py | 11 +++++++++++ tests/test_pep_563.py | 5 +++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/mashumaro/exceptions.py b/mashumaro/exceptions.py index 4807a066..062f9dfc 100644 --- a/mashumaro/exceptions.py +++ b/mashumaro/exceptions.py @@ -126,7 +126,7 @@ def __str__(self): class UnresolvedTypeReferenceError(NameError): def __init__(self, holder_class, unresolved_type_name): self.holder_class = holder_class - self.unresolved_type_name = unresolved_type_name + self.name = unresolved_type_name @property def holder_class_name(self): @@ -135,5 +135,5 @@ def holder_class_name(self): def __str__(self): return ( f"Class {self.holder_class_name} has unresolved type reference " - f"{self.unresolved_type_name} in some of its fields" + f"{self.name} in some of its fields" ) diff --git a/mashumaro/serializer/base/dict.py b/mashumaro/serializer/base/dict.py index 2bf25aca..4386418d 100644 --- a/mashumaro/serializer/base/dict.py +++ b/mashumaro/serializer/base/dict.py @@ -21,7 +21,7 @@ def __init_subclass__(cls: Type[T], **kwargs): builder.add_to_dict() except UnresolvedTypeReferenceError: if not config.allow_postponed_evaluation: - raise + raise # pragma no cover def to_dict( self: T, diff --git a/tests/conftest.py b/tests/conftest.py index 48a73104..db9e777b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ from mashumaro.meta.macros import PY_37_MIN - if not PY_37_MIN: collect_ignore = ["test_pep_563.py"] diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 61abd442..c6efae27 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -4,6 +4,7 @@ InvalidFieldValue, MissingField, ThirdPartyModuleNotFoundError, + UnresolvedTypeReferenceError, UnserializableField, UnsupportedDeserializationEngine, UnsupportedSerializationEngine, @@ -127,3 +128,13 @@ def test_unsupported_serialization_engine(): str(exc) == 'Field "x" of type int in object is not serializable: ' 'Unsupported serialization engine "engine_name"' ) + + +def test_unresolved_type_reference_error(): + exc = UnresolvedTypeReferenceError(object, "x") + assert exc.holder_class_name == "object" + assert exc.name == "x" + assert ( + str(exc) == "Class object has unresolved type reference " + "x in some of its fields" + ) diff --git a/tests/test_pep_563.py b/tests/test_pep_563.py index 1c3e3c37..e70b914d 100644 --- a/tests/test_pep_563.py +++ b/tests/test_pep_563.py @@ -1,11 +1,12 @@ from __future__ import annotations -import pytest from dataclasses import dataclass +import pytest + from mashumaro import DataClassDictMixin -from mashumaro.exceptions import UnresolvedTypeReferenceError from mashumaro.config import BaseConfig +from mashumaro.exceptions import UnresolvedTypeReferenceError @dataclass From d453afc6db583b1d3680b57c3d63a1d463d2269d Mon Sep 17 00:00:00 2001 From: Alexander Tikhonov Date: Mon, 1 Nov 2021 23:35:14 +0300 Subject: [PATCH 8/8] Increase coverage --- mashumaro/serializer/base/dict.py | 2 +- tests/conftest.py | 8 ++++ tests/test_data_types.py | 67 +++++++++++++++++++++++++++++++ tests/test_pep_563.py | 12 ++++++ 4 files changed, 88 insertions(+), 1 deletion(-) diff --git a/mashumaro/serializer/base/dict.py b/mashumaro/serializer/base/dict.py index 4386418d..2bf25aca 100644 --- a/mashumaro/serializer/base/dict.py +++ b/mashumaro/serializer/base/dict.py @@ -21,7 +21,7 @@ def __init_subclass__(cls: Type[T], **kwargs): builder.add_to_dict() except UnresolvedTypeReferenceError: if not config.allow_postponed_evaluation: - raise # pragma no cover + raise def to_dict( self: T, diff --git a/tests/conftest.py b/tests/conftest.py index db9e777b..2c0a2d8a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,12 @@ +from unittest.mock import patch + from mashumaro.meta.macros import PY_37_MIN if not PY_37_MIN: collect_ignore = ["test_pep_563.py"] + + +fake_add_from_dict = patch( + "mashumaro.serializer.base.metaprogramming." "CodeBuilder.add_from_dict", + lambda *args, **kwargs: ..., +) diff --git a/tests/test_data_types.py b/tests/test_data_types.py index a829e89f..d5e2803c 100644 --- a/tests/test_data_types.py +++ b/tests/test_data_types.py @@ -37,6 +37,8 @@ Tuple, ) +from .conftest import fake_add_from_dict + try: from typing import OrderedDict # New in version 3.7.2 except ImportError: @@ -729,6 +731,14 @@ class _(DataClassDictMixin): # noinspection PyTypeChecker x: generic_type[x_type] + with fake_add_from_dict: + with pytest.raises(UnserializableField): + + @dataclass + class _(DataClassDictMixin): + # noinspection PyTypeChecker + x: generic_type[x_type] + @pytest.mark.parametrize("x_type", unsupported_typing_primitives) @pytest.mark.parametrize("generic_type", generic_sequence_types) @@ -740,6 +750,14 @@ class _(DataClassDictMixin): # noinspection PyTypeChecker x: generic_type[x_type] + with fake_add_from_dict: + with pytest.raises(UnserializableDataError): + + @dataclass + class _(DataClassDictMixin): + # noinspection PyTypeChecker + x: generic_type[x_type] + @pytest.mark.parametrize("x_type", unsupported_field_types) def test_unsupported_field_types(x_type): @@ -749,6 +767,13 @@ def test_unsupported_field_types(x_type): class _(DataClassDictMixin): x: x_type + with fake_add_from_dict: + with pytest.raises(UnserializableField): + + @dataclass + class _(DataClassDictMixin): + x: x_type + @pytest.mark.parametrize("x_type", unsupported_typing_primitives) def test_unsupported_typing_primitives(x_type): @@ -758,6 +783,13 @@ def test_unsupported_typing_primitives(x_type): class _(DataClassDictMixin): x: x_type + with fake_add_from_dict: + with pytest.raises(UnserializableDataError): + + @dataclass + class _(DataClassDictMixin): + x: x_type + @pytest.mark.parametrize("generic_type", generic_mapping_types) def test_data_class_as_mapping_key(generic_type): @@ -771,6 +803,13 @@ class Key(DataClassDictMixin): class _(DataClassDictMixin): x: generic_type[Key, int] + with fake_add_from_dict: + with pytest.raises(UnserializableDataError): + + @dataclass + class _(DataClassDictMixin): + x: generic_type[Key, int] + def test_data_class_as_mapping_key_for_counter(): @dataclass @@ -783,6 +822,13 @@ class Key(DataClassDictMixin): class _(DataClassDictMixin): x: Counter[Key] + with fake_add_from_dict: + with pytest.raises(UnserializableDataError): + + @dataclass + class _(DataClassDictMixin): + x: Counter[Key] + def test_data_class_as_chain_map_key(): @dataclass @@ -795,6 +841,13 @@ class Key(DataClassDictMixin): class _(DataClassDictMixin): x: ChainMap[Key, int] + with fake_add_from_dict: + with pytest.raises(UnserializableDataError): + + @dataclass + class _(DataClassDictMixin): + x: ChainMap[Key, int] + @pytest.mark.parametrize("use_datetime", [True, False]) @pytest.mark.parametrize("use_enum", [True, False]) @@ -902,6 +955,13 @@ def test_weird_field_type(): class _(DataClassDictMixin): x: 123 + with fake_add_from_dict: + with pytest.raises(UnserializableDataError): + + @dataclass + class _(DataClassDictMixin): + x: 123 + @pytest.mark.parametrize( "rounding", [None, decimal.ROUND_UP, decimal.ROUND_DOWN] @@ -1186,6 +1246,13 @@ def test_dataclass_field_without_mixin(): class _(DataClassDictMixin): p: DataClassWithoutMixin + with fake_add_from_dict: + with pytest.raises(UnserializableField): + + @dataclass + class _(DataClassDictMixin): + p: DataClassWithoutMixin + def test_serializable_type_dataclass(): @dataclass diff --git a/tests/test_pep_563.py b/tests/test_pep_563.py index e70b914d..55624ec5 100644 --- a/tests/test_pep_563.py +++ b/tests/test_pep_563.py @@ -8,6 +8,8 @@ from mashumaro.config import BaseConfig from mashumaro.exceptions import UnresolvedTypeReferenceError +from .conftest import fake_add_from_dict + @dataclass class A(DataClassDictMixin): @@ -46,3 +48,13 @@ class DataClass(DataClassDictMixin): class Config(BaseConfig): allow_postponed_evaluation = False + + with fake_add_from_dict: + with pytest.raises(UnresolvedTypeReferenceError): + + @dataclass + class DataClass(DataClassDictMixin): + x: X + + class Config(BaseConfig): + allow_postponed_evaluation = False