fix: replace marshmallow_jsonapi which is now unmaintained

src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py

@@ -1,4 +1,5 @@
 import asyncio
+import traceback
 from typing import Any, Awaitable, Dict, List, Literal, Optional, Tuple, Union, cast
 from uuid import UUID
 
@@ -27,6 +28,12 @@
 from forestadmin.agent_toolkit.services.permissions.permission_service import PermissionService
 from forestadmin.agent_toolkit.services.serializers import add_search_metadata
 from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException, JsonApiSerializer
+from forestadmin.agent_toolkit.services.serializers.json_api_deserializer import (
+    JsonApiDeserializer as JsonApiDeserializerHomeMade,
+)
+from forestadmin.agent_toolkit.services.serializers.json_api_serializer import (
+    JsonApiSerializer as JsonApiSerializerHomeMade,
+)
 from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response, User
 from forestadmin.agent_toolkit.utils.csv import Csv, CsvException
 from forestadmin.agent_toolkit.utils.id import unpack_id
@@ -124,7 +131,7 @@ async def get(self, request: RequestCollection) -> Response:
             return HttpResponseBuilder.build_unknown_response()
 
         try:
-            dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships(
+            dumped: Dict[str, Any] = self._serialize_records_with_relationships(
                 records, request.collection, projections, many=False
             )
         except JsonApiException as e:
@@ -141,6 +148,17 @@ async def add(self, request: RequestCollection) -> Response:
         schema = JsonApiSerializer.get(collection)
         try:
             data: RecordsDataAlias = schema().load(request.body)  # type: ignore
+            try:
+                new_data = JsonApiDeserializerHomeMade(self.datasource).deserialize(request.body, collection)
+                from dictdiffer import diff as differ
+
+                diff = list(differ(data, new_data))
+                ForestLogger.log("info", f"creating new_ret({collection.name}) ... diff({len(diff)})")
+                data = new_data
+            except Exception as exc:
+                traceback.print_exc()
+                pass
+                # raise
         except JsonApiException as e:
             ForestLogger.log("exception", e)
             return HttpResponseBuilder.build_client_error_response([e])
@@ -166,7 +184,7 @@ async def add(self, request: RequestCollection) -> Response:
             return HttpResponseBuilder.build_client_error_response([e])
 
         return HttpResponseBuilder.build_success_response(
-            CrudResource._serialize_records_with_relationships(
+            self._serialize_records_with_relationships(
                 records, request.collection, Projection(*list(records[0].keys())), many=False
             )
         )
@@ -193,7 +211,7 @@ async def list(self, request: RequestCollection) -> Response:
         records = await request.collection.list(request.user, paginated_filter, projections)
 
         try:
-            dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships(
+            dumped: Dict[str, Any] = self._serialize_records_with_relationships(
                 records, request.collection, projections, many=True
             )
         except JsonApiException as e:
@@ -275,6 +293,18 @@ async def update(self, request: RequestCollection) -> Response:
             # if the id change it will be in 'data.attributes', otherwise, we get the id by from the request url.
             request.body["data"].pop("id", None)  # type: ignore
             data: RecordsDataAlias = schema().load(request.body)  # type: ignore
+            try:
+                new_data = JsonApiDeserializerHomeMade(self.datasource).deserialize(request.body, collection)
+                from dictdiffer import diff as differ
+
+                diff = list(differ(data, new_data))
+                ForestLogger.log("info", f"creating new_ret({collection.name}) ... diff({len(diff)})")
+                data = new_data
+            except Exception as exc:
+                traceback.print_exc()
+                pass
+                # raise
+
         except JsonApiException as e:
             ForestLogger.log("exception", e)
             return HttpResponseBuilder.build_client_error_response([e])
@@ -290,7 +320,7 @@ async def update(self, request: RequestCollection) -> Response:
         records = await collection.list(request.user, PaginatedFilter.from_base_filter(filter), projection)
 
         try:
-            dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships(
+            dumped: Dict[str, Any] = self._serialize_records_with_relationships(
                 records, request.collection, projection, many=False
             )
         except JsonApiException as e:
@@ -428,8 +458,8 @@ async def extract_data(
 
         return record, one_to_one_relations
 
-    @staticmethod
     def _serialize_records_with_relationships(
+        self,
         records: List[RecordsDataAlias],
         collection: Union[Collection, CollectionCustomizer],
         projection: Projection,
@@ -450,7 +480,23 @@ def _serialize_records_with_relationships(
                 record[name] = None
 
         schema = JsonApiSerializer.get(collection)
-        return schema(projections=projection).dump(records if many is True else records[0], many=many)
+        ret = schema(projections=projection).dump(records if many is True else records[0], many=many)
+
+        try:
+            new_ret = JsonApiSerializerHomeMade(self.datasource, projection).serialize(
+                records if many is True else records[0], collection
+            )
+            from dictdiffer import diff as differ
+
+            diff = list(differ(ret, new_ret))
+            ForestLogger.log("info", f"returning new_ret({collection.name}) ... diff({len(diff)})")
+            return cast(Dict[str, Any], new_ret)
+        except Exception as exc:
+            traceback.print_exc()
+            pass
+            # raise
+
+        return ret
 
     async def _handle_live_query_segment(
         self, request: RequestCollection, condition_tree: Optional[ConditionTree]

src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py

@@ -1,4 +1,5 @@
 import sys
+import traceback
 from typing import Any, Dict, List, Literal, Union, cast
 
 if sys.version_info >= (3, 9):
@@ -28,6 +29,9 @@
 )
 from forestadmin.agent_toolkit.services.serializers import DumpedResult, add_search_metadata
 from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException, JsonApiSerializer
+from forestadmin.agent_toolkit.services.serializers.json_api_serializer import (
+    JsonApiSerializer as JsonApiSerializerHomeMade,
+)
 from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response
 from forestadmin.agent_toolkit.utils.csv import Csv, CsvException
 from forestadmin.agent_toolkit.utils.id import unpack_id
@@ -113,6 +117,24 @@ async def list(self, request: RequestRelationCollection) -> Response:
         if paginated_filter.search:
             dumped = add_search_metadata(dumped, paginated_filter.search)
 
+        try:
+            new_ret = JsonApiSerializerHomeMade(self.datasource, projection).serialize(
+                records, request.foreign_collection
+            )
+            if paginated_filter.search:
+                dumped = add_search_metadata(dumped, paginated_filter.search)
+            from dictdiffer import diff as differ
+
+            diff = list(differ(dumped, new_ret))
+            ForestLogger.log("info", f"returning new_ret({request.foreign_collection.name}) ... diff({len(diff)})")
+            if len(diff) > 0:
+                pass
+            return HttpResponseBuilder.build_success_response(cast(Dict[str, Any], dumped))
+        except Exception as exc:
+            traceback.print_exc()
+            pass
+            # raise
+
         return HttpResponseBuilder.build_success_response(cast(Dict[str, Any], dumped))
 
     @authenticate

src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py

@@ -1,18 +1,28 @@
-from typing import Any, Dict, List, Optional, TypedDict
+from typing import Any, Dict, List, Union
+
+from typing_extensions import NotRequired, TypedDict
 
 
 class Data(TypedDict):
     type: str
-    relationships: Dict[str, Any]
+    id: int
     attributes: Dict[str, Any]
+    relationships: Dict[str, Any]
+    links: Dict[str, Any]
+
+
+class IncludedData(TypedDict):
+    type: str
     id: int
     links: Dict[str, Any]
+    attributes: NotRequired[Dict[str, Any]]
+    relationships: NotRequired[Dict[str, Any]]
 
 
 class DumpedResult(TypedDict):
-    data: List[Data]
-    included: Dict[str, Any]
-    meta: Optional[Dict[str, Any]]
+    data: Union[List[Data], Data]
+    included: NotRequired[List[IncludedData]]
+    meta: NotRequired[Dict[str, Any]]
 
 
 def add_search_metadata(dumped: DumpedResult, search_value: str):

src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py

@@ -0,0 +1,10 @@
+class JsonApiException(Exception):
+    pass
+
+
+class JsonApiSerializerException(JsonApiException):
+    pass
+
+
+class JsonApiDeserializerException(JsonApiException):
+    pass

src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py

@@ -0,0 +1,73 @@
+from datetime import date, datetime, time
+from typing import Any, Callable, Dict, Union, cast
+from uuid import UUID
+
+from forestadmin.agent_toolkit.exceptions import AgentToolkitException
+from forestadmin.agent_toolkit.forest_logger import ForestLogger
+from forestadmin.agent_toolkit.services.serializers import Data, DumpedResult
+from forestadmin.datasource_toolkit.collections import Collection
+from forestadmin.datasource_toolkit.datasources import Datasource
+from forestadmin.datasource_toolkit.interfaces.fields import Column, PrimitiveType, is_one_to_one
+from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias
+from forestadmin.datasource_toolkit.utils.schema import SchemaUtils
+
+
+class JsonApiDeserializer:
+    def __init__(self, datasource: Datasource) -> None:
+        self.datasource = datasource
+
+    def deserialize(self, data: DumpedResult, collection: Collection) -> RecordsDataAlias:
+        ret = {}
+        data["data"] = cast(Data, data["data"])
+
+        for key, value in data["data"]["attributes"].items():
+            ret[key] = self._deserialize_value(value, cast(Column, collection.schema["fields"][key]))
+
+        # TODO: relationships
+        for key, value in data["data"].get("relationships", {}).items():
+            try:
+                ret[key] = int(value["data"]["id"])
+            except ValueError:
+                ret[key] = value["data"]["id"]
+
+        return ret
+
+    def _deserialize_value(self, value: Union[str, int, float, bool, None], schema: Column) -> Any:
+        if value is None:
+            return None
+
+        def number_parser(val):
+            try:
+                return int(value)
+            except ValueError:
+                return float(value)
+
+        parser_map: Dict[PrimitiveType, Callable] = {
+            PrimitiveType.STRING: str,
+            PrimitiveType.ENUM: str,
+            PrimitiveType.BOOLEAN: bool,
+            PrimitiveType.NUMBER: number_parser,
+            PrimitiveType.UUID: UUID,
+            PrimitiveType.DATE_ONLY: date.fromisoformat,
+            PrimitiveType.TIME_ONLY: time.fromisoformat,
+            PrimitiveType.DATE: datetime.fromisoformat,
+            PrimitiveType.POINT: lambda v: (int(v) for v in cast(str, value).split(",")),
+            PrimitiveType.BINARY: lambda v: v,  # should not be called
+            PrimitiveType.JSON: lambda v: v,
+        }
+
+        if schema["column_type"] in parser_map.keys():
+            return parser_map[cast(PrimitiveType, schema["column_type"])](value)
+        elif isinstance(schema["column_type"], dict) or isinstance(schema["column_type"], list):
+            return value
+        else:
+            ForestLogger.log("error", f"Unknown column type {schema['column_type']}")
+            raise AgentToolkitException(f"Unknown column type {schema['column_type']}")
+
+    def _deserialize_relationships(self, name: str, value, collection: Collection):
+        ret = {}
+        if is_one_to_one(collection.schema):
+            return value["data"].get(pk)
+        for pk in SchemaUtils.get_primary_keys(collection.schema):
+            ret[pk] = value["data"].get(pk)
+        return ret