fix(python): run tests in python 3.8 and fix mypy errors (#4670)

.github/workflows/python-generator.yml

@@ -48,16 +48,16 @@ jobs:
 
           docker build -f ./pydantic/Dockerfile -t ferntest/fern-pydantic-model:0.0.0 .
           cd tests/utils/example_models
-          fern generate --group dummy_types --local
+          fern generate --group dummy_types --local || true
 
           cd ../typeddict_models
-          fern generate --group dummy_td_types --local
+          fern generate --group dummy_td_types --local || true
 
           cd ../union_utils
-          fern generate --group dummy_union_types --local
+          fern generate --group dummy_union_types --local || true
 
           cd ../unaliased_models
-          fern generate --group dummy_unaliased_types --local
+          fern generate --group dummy_unaliased_types --local || true
 
       - name: Install Dependencies - Pydantic V1
         working-directory: ./generators/python

docker/seed/Dockerfile.python

@@ -1,4 +1,4 @@
-FROM mwalbeck/python-poetry:1.7-3.11
+FROM mwalbeck/python-poetry:1.7-3.8
 
 RUN apt update && apt upgrade -y
 RUN apt install nodejs npm -y

generators/python/core_utilities/shared/pydantic_utilities.py

@@ -95,15 +95,15 @@ class Config:
 
     @classmethod
     def model_construct(
-        cls: type[Model], _fields_set: typing.Optional[typing.Set[str]] = None, **values: typing.Any
-    ) -> Model:
+        cls: typing.Type["Model"], _fields_set: typing.Optional[typing.Set[str]] = None, **values: typing.Any
+    ) -> "Model":
         dealiased_object = convert_and_respect_annotation_metadata(object_=values, annotation=cls, direction="read")
         return cls.construct(_fields_set, **dealiased_object)
 
     @classmethod
     def construct(
-        cls: type[Model], _fields_set: typing.Optional[typing.Set[str]] = None, **values: typing.Any
-    ) -> Model:
+        cls: typing.Type["Model"], _fields_set: typing.Optional[typing.Set[str]] = None, **values: typing.Any
+    ) -> "Model":
         dealiased_object = convert_and_respect_annotation_metadata(object_=values, annotation=cls, direction="read")
         if IS_PYDANTIC_V2:
             return super().model_construct(_fields_set, **dealiased_object)  # type: ignore # Pydantic v2
@@ -238,9 +238,7 @@ def decorator(func: AnyCallable) -> AnyCallable:
 def universal_field_validator(field_name: str, pre: bool = False) -> typing.Callable[[AnyCallable], AnyCallable]:
     def decorator(func: AnyCallable) -> AnyCallable:
         if IS_PYDANTIC_V2:
-            return pydantic.field_validator(field_name, mode="before" if pre else "after")(
-                func
-            )  # type: ignore # Pydantic v2
+            return pydantic.field_validator(field_name, mode="before" if pre else "after")(func)  # type: ignore # Pydantic v2
         else:
             return pydantic.validator(field_name, pre=pre)(func)  # type: ignore # Pydantic v1
 

generators/python/sdk/Dockerfile

@@ -1,10 +1,11 @@
 FROM python:3.9.14
 
 # Install node and npm.
-ENV NODE_VERSION=18.20.0
+ENV NODE_VERSION=18.20.1
 RUN apt install -y curl
 RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash
 ENV NVM_DIR=/root/.nvm
+RUN . "$NVM_DIR/nvm.sh" && nvm ls-remote
 RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
 RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
 RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}

generators/python/sdk/versions.yml

@@ -1,4 +1,12 @@
 # For unreleased changes, use unreleased.yml
+- version: 4.2.4
+  irVersion: 53
+  changelogEntry:
+    - type: fix
+      summary: |
+        Datetime examples are generated correctly once again.
+        The `pydantic_utilites` file is python 3.8 compatible.
+
 - version: 4.2.3
   irVersion: 53
   changelogEntry:

generators/python/src/fern_python/snippet/snippet_writer.py

@@ -170,7 +170,7 @@ def _get_snippet_for_primitive(
                             "fromisoformat",
                         ),
                     ),
-                    args=[AST.Expression(f'"{str(datetime)}"')],
+                    args=[AST.Expression(f'"{str(datetime.datetime)}"')],
                 ),
             ),
             date=lambda date: AST.Expression(

generators/python/tests/ir/fixtures/fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "examples",
-  "version": "0.37.4"
+  "version": "0.41.16"
 }

generators/python/tests/utils/example_models/fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "fern",
-  "version": "0.40.4"
+  "version": "0.41.16"
 }

generators/python/tests/utils/example_models/types/core/pydantic_utilities.py

@@ -81,15 +81,52 @@ def to_jsonable_with_fallback(
 class UniversalBaseModel(pydantic.BaseModel):
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(
+            # Allow fields begining with `model_` to be used in the model
             protected_namespaces=(),
-            json_encoders={dt.datetime: serialize_datetime},
         )  # type: ignore # Pydantic v2
+
+        @pydantic.model_serializer(mode="wrap", when_used="json")  # type: ignore # Pydantic v2
+        def serialize_model(
+            self, handler: pydantic.SerializerFunctionWrapHandler
+        ) -> typing.Any:  # type: ignore # Pydantic v2
+            serialized = handler(self)
+            data = {
+                k: serialize_datetime(v) if isinstance(v, dt.datetime) else v
+                for k, v in serialized.items()
+            }
+            return data
+
     else:
 
         class Config:
             smart_union = True
             json_encoders = {dt.datetime: serialize_datetime}
 
+    @classmethod
+    def model_construct(
+        cls: typing.Type["Model"],
+        _fields_set: typing.Optional[typing.Set[str]] = None,
+        **values: typing.Any,
+    ) -> "Model":
+        dealiased_object = convert_and_respect_annotation_metadata(
+            object_=values, annotation=cls, direction="read"
+        )
+        return cls.construct(_fields_set, **dealiased_object)
+
+    @classmethod
+    def construct(
+        cls: typing.Type["Model"],
+        _fields_set: typing.Optional[typing.Set[str]] = None,
+        **values: typing.Any,
+    ) -> "Model":
+        dealiased_object = convert_and_respect_annotation_metadata(
+            object_=values, annotation=cls, direction="read"
+        )
+        if IS_PYDANTIC_V2:
+            return super().model_construct(_fields_set, **dealiased_object)  # type: ignore # Pydantic v2
+        else:
+            return super().construct(_fields_set, **dealiased_object)
+
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {
             "by_alias": True,
@@ -113,19 +150,22 @@ def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
         # that we have less control over, and this is less intrusive than custom serializers for now.
         if IS_PYDANTIC_V2:
             kwargs_with_defaults_exclude_unset: typing.Any = {
+                **kwargs,
                 "by_alias": True,
                 "exclude_unset": True,
-                **kwargs,
+                "exclude_none": False,
             }
             kwargs_with_defaults_exclude_none: typing.Any = {
+                **kwargs,
                 "by_alias": True,
                 "exclude_none": True,
-                **kwargs,
+                "exclude_unset": False,
             }
             dict_dump = deep_union_pydantic_dicts(
                 super().model_dump(**kwargs_with_defaults_exclude_unset),  # type: ignore # Pydantic v2
                 super().model_dump(**kwargs_with_defaults_exclude_none),  # type: ignore # Pydantic v2
             )
+
         else:
             _fields_set = self.__fields_set__
 
@@ -193,11 +233,11 @@ def encode_by_type(o: typing.Any) -> typing.Any:
             return encoder(o)
 
 
-def update_forward_refs(model: typing.Type["Model"]) -> None:
+def update_forward_refs(model: typing.Type["Model"], **localns: typing.Any) -> None:
     if IS_PYDANTIC_V2:
         model.model_rebuild(raise_errors=False)  # type: ignore # Pydantic v2
     else:
-        model.update_forward_refs()
+        model.update_forward_refs(**localns)
 
 
 # Mirrors Pydantic's internal typing

generators/python/tests/utils/example_models/types/resources/types/object_with_optional_field.py

@@ -2,45 +2,33 @@
 
 from ...core.unchecked_base_model import UncheckedBaseModel
 import typing
-import typing_extensions
-from ...core.serialization import FieldMetadata
+import pydantic
 import datetime as dt
 import uuid
 from .color import Color
 from .shape import Shape
 from .undiscriminated_shape import UndiscriminatedShape
 from ...core.pydantic_utilities import IS_PYDANTIC_V2
-import pydantic
 
 
 class ObjectWithOptionalField(UncheckedBaseModel):
     literal: typing.Literal["lit_one"] = "lit_one"
     string: typing.Optional[str] = None
     integer: typing.Optional[int] = None
-    long_: typing_extensions.Annotated[
-        typing.Optional[int], FieldMetadata(alias="long")
-    ] = None
+    long_: typing.Optional[int] = pydantic.Field(alias="long", default=None)
     double: typing.Optional[float] = None
-    bool_: typing_extensions.Annotated[
-        typing.Optional[bool], FieldMetadata(alias="bool")
-    ] = None
+    bool_: typing.Optional[bool] = pydantic.Field(alias="bool", default=None)
     datetime: typing.Optional[dt.datetime] = None
     date: typing.Optional[dt.date] = None
-    uuid_: typing_extensions.Annotated[
-        typing.Optional[uuid.UUID], FieldMetadata(alias="uuid")
-    ] = None
-    base_64: typing_extensions.Annotated[
-        typing.Optional[str], FieldMetadata(alias="base64")
-    ] = None
-    list_: typing_extensions.Annotated[
-        typing.Optional[typing.List[str]], FieldMetadata(alias="list")
-    ] = None
-    set_: typing_extensions.Annotated[
-        typing.Optional[typing.Set[str]], FieldMetadata(alias="set")
-    ] = None
-    map_: typing_extensions.Annotated[
-        typing.Optional[typing.Dict[int, str]], FieldMetadata(alias="map")
-    ] = None
+    uuid_: typing.Optional[uuid.UUID] = pydantic.Field(alias="uuid", default=None)
+    base_64: typing.Optional[str] = pydantic.Field(alias="base64", default=None)
+    list_: typing.Optional[typing.List[str]] = pydantic.Field(
+        alias="list", default=None
+    )
+    set_: typing.Optional[typing.Set[str]] = pydantic.Field(alias="set", default=None)
+    map_: typing.Optional[typing.Dict[int, str]] = pydantic.Field(
+        alias="map", default=None
+    )
     enum: typing.Optional[Color] = None
     union: typing.Optional[Shape] = None
     second_union: typing.Optional[Shape] = None

generators/python/tests/utils/typeddict_models/fern/fern.config.json

@@ -1,4 +1,4 @@
 {
   "organization": "fern",
-  "version": "0.40.4"
+  "version": "0.41.16"
 }

generators/python/tests/utils/typeddict_models/types/core/pydantic_utilities.py

@@ -81,15 +81,52 @@ def to_jsonable_with_fallback(
 class UniversalBaseModel(pydantic.BaseModel):
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(
+            # Allow fields begining with `model_` to be used in the model
             protected_namespaces=(),
-            json_encoders={dt.datetime: serialize_datetime},
         )  # type: ignore # Pydantic v2
+
+        @pydantic.model_serializer(mode="wrap", when_used="json")  # type: ignore # Pydantic v2
+        def serialize_model(
+            self, handler: pydantic.SerializerFunctionWrapHandler
+        ) -> typing.Any:  # type: ignore # Pydantic v2
+            serialized = handler(self)
+            data = {
+                k: serialize_datetime(v) if isinstance(v, dt.datetime) else v
+                for k, v in serialized.items()
+            }
+            return data
+
     else:
 
         class Config:
             smart_union = True
             json_encoders = {dt.datetime: serialize_datetime}
 
+    @classmethod
+    def model_construct(
+        cls: typing.Type["Model"],
+        _fields_set: typing.Optional[typing.Set[str]] = None,
+        **values: typing.Any,
+    ) -> "Model":
+        dealiased_object = convert_and_respect_annotation_metadata(
+            object_=values, annotation=cls, direction="read"
+        )
+        return cls.construct(_fields_set, **dealiased_object)
+
+    @classmethod
+    def construct(
+        cls: typing.Type["Model"],
+        _fields_set: typing.Optional[typing.Set[str]] = None,
+        **values: typing.Any,
+    ) -> "Model":
+        dealiased_object = convert_and_respect_annotation_metadata(
+            object_=values, annotation=cls, direction="read"
+        )
+        if IS_PYDANTIC_V2:
+            return super().model_construct(_fields_set, **dealiased_object)  # type: ignore # Pydantic v2
+        else:
+            return super().construct(_fields_set, **dealiased_object)
+
     def json(self, **kwargs: typing.Any) -> str:
         kwargs_with_defaults: typing.Any = {
             "by_alias": True,
@@ -113,19 +150,22 @@ def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]:
         # that we have less control over, and this is less intrusive than custom serializers for now.
         if IS_PYDANTIC_V2:
             kwargs_with_defaults_exclude_unset: typing.Any = {
+                **kwargs,
                 "by_alias": True,
                 "exclude_unset": True,
-                **kwargs,
+                "exclude_none": False,
             }
             kwargs_with_defaults_exclude_none: typing.Any = {
+                **kwargs,
                 "by_alias": True,
                 "exclude_none": True,
-                **kwargs,
+                "exclude_unset": False,
             }
             dict_dump = deep_union_pydantic_dicts(
                 super().model_dump(**kwargs_with_defaults_exclude_unset),  # type: ignore # Pydantic v2
                 super().model_dump(**kwargs_with_defaults_exclude_none),  # type: ignore # Pydantic v2
             )
+
         else:
             _fields_set = self.__fields_set__
 
@@ -193,11 +233,11 @@ def encode_by_type(o: typing.Any) -> typing.Any:
             return encoder(o)
 
 
-def update_forward_refs(model: typing.Type["Model"]) -> None:
+def update_forward_refs(model: typing.Type["Model"], **localns: typing.Any) -> None:
     if IS_PYDANTIC_V2:
         model.model_rebuild(raise_errors=False)  # type: ignore # Pydantic v2
     else:
-        model.update_forward_refs()
+        model.update_forward_refs(**localns)
 
 
 # Mirrors Pydantic's internal typing

generators/python/tests/utils/typeddict_models/types/resources/types/circle.py

@@ -1,17 +1,13 @@
 # This file was auto-generated by Fern from our API Definition.
 
 from ...core.unchecked_base_model import UncheckedBaseModel
-import typing_extensions
-from ...core.serialization import FieldMetadata
+import pydantic
 from ...core.pydantic_utilities import IS_PYDANTIC_V2
 import typing
-import pydantic
 
 
 class Circle(UncheckedBaseModel):
-    radius_measurement: typing_extensions.Annotated[
-        float, FieldMetadata(alias="radiusMeasurement")
-    ]
+    radius_measurement: float = pydantic.Field(alias="radiusMeasurement")
 
     if IS_PYDANTIC_V2:
         model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(