diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py index 8a3a09e7d..537122abb 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py @@ -1,4 +1,5 @@ import asyncio +import traceback from typing import Any, Awaitable, Dict, List, Literal, Optional, Tuple, Union, cast from uuid import UUID @@ -27,6 +28,12 @@ from forestadmin.agent_toolkit.services.permissions.permission_service import PermissionService from forestadmin.agent_toolkit.services.serializers import add_search_metadata from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException, JsonApiSerializer +from forestadmin.agent_toolkit.services.serializers.json_api_deserializer import ( + JsonApiDeserializer as JsonApiDeserializerHomeMade, +) +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import ( + JsonApiSerializer as JsonApiSerializerHomeMade, +) from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response, User from forestadmin.agent_toolkit.utils.csv import Csv, CsvException from forestadmin.agent_toolkit.utils.id import unpack_id @@ -124,7 +131,7 @@ async def get(self, request: RequestCollection) -> Response: return HttpResponseBuilder.build_unknown_response() try: - dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships( + dumped: Dict[str, Any] = self._serialize_records_with_relationships( records, request.collection, projections, many=False ) except JsonApiException as e: @@ -141,6 +148,17 @@ async def add(self, request: RequestCollection) -> Response: schema = JsonApiSerializer.get(collection) try: data: RecordsDataAlias = schema().load(request.body) # type: ignore + try: + new_data = JsonApiDeserializerHomeMade(self.datasource).deserialize(request.body, collection) + from dictdiffer import diff as differ + + diff = list(differ(data, new_data)) + ForestLogger.log("info", f"creating new_ret({collection.name}) ... diff({len(diff)})") + data = new_data + except Exception as exc: + traceback.print_exc() + pass + # raise except JsonApiException as e: ForestLogger.log("exception", e) return HttpResponseBuilder.build_client_error_response([e]) @@ -166,7 +184,7 @@ async def add(self, request: RequestCollection) -> Response: return HttpResponseBuilder.build_client_error_response([e]) return HttpResponseBuilder.build_success_response( - CrudResource._serialize_records_with_relationships( + self._serialize_records_with_relationships( records, request.collection, Projection(*list(records[0].keys())), many=False ) ) @@ -193,7 +211,7 @@ async def list(self, request: RequestCollection) -> Response: records = await request.collection.list(request.user, paginated_filter, projections) try: - dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships( + dumped: Dict[str, Any] = self._serialize_records_with_relationships( records, request.collection, projections, many=True ) except JsonApiException as e: @@ -275,6 +293,18 @@ async def update(self, request: RequestCollection) -> Response: # if the id change it will be in 'data.attributes', otherwise, we get the id by from the request url. request.body["data"].pop("id", None) # type: ignore data: RecordsDataAlias = schema().load(request.body) # type: ignore + try: + new_data = JsonApiDeserializerHomeMade(self.datasource).deserialize(request.body, collection) + from dictdiffer import diff as differ + + diff = list(differ(data, new_data)) + ForestLogger.log("info", f"creating new_ret({collection.name}) ... diff({len(diff)})") + data = new_data + except Exception as exc: + traceback.print_exc() + pass + # raise + except JsonApiException as e: ForestLogger.log("exception", e) return HttpResponseBuilder.build_client_error_response([e]) @@ -290,7 +320,7 @@ async def update(self, request: RequestCollection) -> Response: records = await collection.list(request.user, PaginatedFilter.from_base_filter(filter), projection) try: - dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships( + dumped: Dict[str, Any] = self._serialize_records_with_relationships( records, request.collection, projection, many=False ) except JsonApiException as e: @@ -428,8 +458,8 @@ async def extract_data( return record, one_to_one_relations - @staticmethod def _serialize_records_with_relationships( + self, records: List[RecordsDataAlias], collection: Union[Collection, CollectionCustomizer], projection: Projection, @@ -450,7 +480,23 @@ def _serialize_records_with_relationships( record[name] = None schema = JsonApiSerializer.get(collection) - return schema(projections=projection).dump(records if many is True else records[0], many=many) + ret = schema(projections=projection).dump(records if many is True else records[0], many=many) + + try: + new_ret = JsonApiSerializerHomeMade(self.datasource, projection).serialize( + records if many is True else records[0], collection + ) + from dictdiffer import diff as differ + + diff = list(differ(ret, new_ret)) + ForestLogger.log("info", f"returning new_ret({collection.name}) ... diff({len(diff)})") + return cast(Dict[str, Any], new_ret) + except Exception as exc: + traceback.print_exc() + pass + # raise + + return ret async def _handle_live_query_segment( self, request: RequestCollection, condition_tree: Optional[ConditionTree] diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py index 29ee9e181..f53b7bc6f 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py @@ -1,4 +1,5 @@ import sys +import traceback from typing import Any, Dict, List, Literal, Union, cast if sys.version_info >= (3, 9): @@ -28,6 +29,9 @@ ) from forestadmin.agent_toolkit.services.serializers import DumpedResult, add_search_metadata from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException, JsonApiSerializer +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import ( + JsonApiSerializer as JsonApiSerializerHomeMade, +) from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response from forestadmin.agent_toolkit.utils.csv import Csv, CsvException from forestadmin.agent_toolkit.utils.id import unpack_id @@ -113,6 +117,24 @@ async def list(self, request: RequestRelationCollection) -> Response: if paginated_filter.search: dumped = add_search_metadata(dumped, paginated_filter.search) + try: + new_ret = JsonApiSerializerHomeMade(self.datasource, projection).serialize( + records, request.foreign_collection + ) + if paginated_filter.search: + dumped = add_search_metadata(dumped, paginated_filter.search) + from dictdiffer import diff as differ + + diff = list(differ(dumped, new_ret)) + ForestLogger.log("info", f"returning new_ret({request.foreign_collection.name}) ... diff({len(diff)})") + if len(diff) > 0: + pass + return HttpResponseBuilder.build_success_response(cast(Dict[str, Any], dumped)) + except Exception as exc: + traceback.print_exc() + pass + # raise + return HttpResponseBuilder.build_success_response(cast(Dict[str, Any], dumped)) @authenticate diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py index 9ad33ccfa..140a6114f 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py @@ -1,18 +1,28 @@ -from typing import Any, Dict, List, Optional, TypedDict +from typing import Any, Dict, List, Union + +from typing_extensions import NotRequired, TypedDict class Data(TypedDict): type: str - relationships: Dict[str, Any] + id: int attributes: Dict[str, Any] + relationships: Dict[str, Any] + links: Dict[str, Any] + + +class IncludedData(TypedDict): + type: str id: int links: Dict[str, Any] + attributes: NotRequired[Dict[str, Any]] + relationships: NotRequired[Dict[str, Any]] class DumpedResult(TypedDict): - data: List[Data] - included: Dict[str, Any] - meta: Optional[Dict[str, Any]] + data: Union[List[Data], Data] + included: NotRequired[List[IncludedData]] + meta: NotRequired[Dict[str, Any]] def add_search_metadata(dumped: DumpedResult, search_value: str): diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py new file mode 100644 index 000000000..3d6fd22c1 --- /dev/null +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py @@ -0,0 +1,10 @@ +class JsonApiException(Exception): + pass + + +class JsonApiSerializerException(JsonApiException): + pass + + +class JsonApiDeserializerException(JsonApiException): + pass diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py new file mode 100644 index 000000000..871201988 --- /dev/null +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py @@ -0,0 +1,73 @@ +from datetime import date, datetime, time +from typing import Any, Callable, Dict, Union, cast +from uuid import UUID + +from forestadmin.agent_toolkit.exceptions import AgentToolkitException +from forestadmin.agent_toolkit.forest_logger import ForestLogger +from forestadmin.agent_toolkit.services.serializers import Data, DumpedResult +from forestadmin.datasource_toolkit.collections import Collection +from forestadmin.datasource_toolkit.datasources import Datasource +from forestadmin.datasource_toolkit.interfaces.fields import Column, PrimitiveType, is_one_to_one +from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias +from forestadmin.datasource_toolkit.utils.schema import SchemaUtils + + +class JsonApiDeserializer: + def __init__(self, datasource: Datasource) -> None: + self.datasource = datasource + + def deserialize(self, data: DumpedResult, collection: Collection) -> RecordsDataAlias: + ret = {} + data["data"] = cast(Data, data["data"]) + + for key, value in data["data"]["attributes"].items(): + ret[key] = self._deserialize_value(value, cast(Column, collection.schema["fields"][key])) + + # TODO: relationships + for key, value in data["data"].get("relationships", {}).items(): + try: + ret[key] = int(value["data"]["id"]) + except ValueError: + ret[key] = value["data"]["id"] + + return ret + + def _deserialize_value(self, value: Union[str, int, float, bool, None], schema: Column) -> Any: + if value is None: + return None + + def number_parser(val): + try: + return int(value) + except ValueError: + return float(value) + + parser_map: Dict[PrimitiveType, Callable] = { + PrimitiveType.STRING: str, + PrimitiveType.ENUM: str, + PrimitiveType.BOOLEAN: bool, + PrimitiveType.NUMBER: number_parser, + PrimitiveType.UUID: UUID, + PrimitiveType.DATE_ONLY: date.fromisoformat, + PrimitiveType.TIME_ONLY: time.fromisoformat, + PrimitiveType.DATE: datetime.fromisoformat, + PrimitiveType.POINT: lambda v: (int(v) for v in cast(str, value).split(",")), + PrimitiveType.BINARY: lambda v: v, # should not be called + PrimitiveType.JSON: lambda v: v, + } + + if schema["column_type"] in parser_map.keys(): + return parser_map[cast(PrimitiveType, schema["column_type"])](value) + elif isinstance(schema["column_type"], dict) or isinstance(schema["column_type"], list): + return value + else: + ForestLogger.log("error", f"Unknown column type {schema['column_type']}") + raise AgentToolkitException(f"Unknown column type {schema['column_type']}") + + def _deserialize_relationships(self, name: str, value, collection: Collection): + ret = {} + if is_one_to_one(collection.schema): + return value["data"].get(pk) + for pk in SchemaUtils.get_primary_keys(collection.schema): + ret[pk] = value["data"].get(pk) + return ret diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_serializer.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_serializer.py new file mode 100644 index 000000000..82dd04553 --- /dev/null +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_serializer.py @@ -0,0 +1,265 @@ +from datetime import date, datetime, time +from typing import Any, Dict, List, Optional, Tuple, Union, cast +from uuid import uuid4 + +from forestadmin.agent_toolkit.exceptions import AgentToolkitException +from forestadmin.agent_toolkit.forest_logger import ForestLogger +from forestadmin.agent_toolkit.services.serializers import Data, DumpedResult, IncludedData +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiSerializerException +from forestadmin.agent_toolkit.utils.id import pack_id +from forestadmin.datasource_toolkit.collections import Collection +from forestadmin.datasource_toolkit.datasources import Datasource +from forestadmin.datasource_toolkit.interfaces.chart import Chart +from forestadmin.datasource_toolkit.interfaces.fields import ( + Column, + ManyToOne, + OneToOne, + PolymorphicManyToOne, + PolymorphicOneToOne, + PrimitiveType, + RelationAlias, + is_column, + is_many_to_many, + is_many_to_one, + is_one_to_many, + is_one_to_one, + is_polymorphic_many_to_one, + is_polymorphic_one_to_one, +) +from forestadmin.datasource_toolkit.interfaces.query.projections import Projection +from forestadmin.datasource_toolkit.interfaces.query.projections.factory import ProjectionFactory +from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias +from forestadmin.datasource_toolkit.utils.schema import SchemaUtils + + +def render_chart(chart: Chart): + return {"id": str(uuid4()), "type": "stats", "attributes": {"value": chart}} + + +class JsonApiSerializer: + def __init__(self, datasource: Datasource, projection: Projection) -> None: + self.datasource = datasource + self.projection = projection + + def serialize(self, data, collection: Collection) -> DumpedResult: + if isinstance(data, list): + ret = self._serialize_many(data, collection) + else: + ret = self._serialize_one(data, collection) + + if ret.get("included") == []: + del ret["included"] + return cast(DumpedResult, ret) + + @classmethod + def _get_id(cls, collection: Collection, data: RecordsDataAlias) -> Union[int, str]: + pk = pack_id(collection.schema, data) + try: + pk = int(pk) + except ValueError: + pass + return pk + + @classmethod + def _is_in_included(cls, included: List[Dict[str, Any]], item: IncludedData) -> bool: + if "id" not in item or "type" not in item: + raise JsonApiSerializerException("Included item must have an id and a type") + for included_item in included: + if included_item["id"] == item["id"] and included_item["type"] == item["type"]: + return True + return False + + def _serialize_many(self, data, collection: Collection) -> DumpedResult: + ret = {"data": [], "included": []} + for item in data: + serialized = self._serialize_one(item, collection) + ret["data"].append(serialized["data"]) + for included in serialized.get("included", []): + if not self._is_in_included(ret["included"], included): + ret["included"].append(included) + + return cast(DumpedResult, ret) + + def _serialize_one( + self, data: RecordsDataAlias, collection: Collection, projection: Optional[Projection] = None + ) -> DumpedResult: + projection = projection if projection is not None else self.projection + primary_keys = SchemaUtils.get_primary_keys(collection.schema) + if len(primary_keys) > 1: + primary_keys = [] + pk_value = self._get_id(collection, data) + ret = { + "data": { + "id": pk_value, + "attributes": {}, + "links": {"self": f"/forest/{collection.name}/{pk_value}"}, + "relationships": {}, + "type": collection.name, + }, + "included": [], + "links": {"self": f"/forest/{collection.name}/{pk_value}"}, + } + + first_level_projection = [*projection.relations.keys(), *projection.columns] + for key, value in data.items(): + if key in primary_keys or key not in collection.schema["fields"]: + continue + if is_column(collection.schema["fields"][key]) and key in first_level_projection: + ret["data"]["attributes"][key] = self._serialize_value( + value, cast(Column, collection.schema["fields"][key]) + ) + elif not is_column(collection.schema["fields"][key]): + relation, included = self._serialize_relation( + key, + data, + cast(RelationAlias, collection.schema["fields"][key]), + f"/forest/{collection.name}/{pk_value}", + ) + ret["data"]["relationships"][key] = relation + if included is not None and not self._is_in_included(ret["included"], included): + ret["included"].append(included) + + if ret["data"].get("attributes") == {}: + del ret["data"]["attributes"] + if ret["data"].get("relationships") == {}: + del ret["data"]["relationships"] + return cast(DumpedResult, ret) + + def _serialize_value(self, value: Any, schema: Column) -> Union[str, int, float, bool, None]: + if value is None: + return None + if schema["column_type"] in [ + PrimitiveType.STRING, + PrimitiveType.NUMBER, + PrimitiveType.BOOLEAN, + PrimitiveType.JSON, + PrimitiveType.BINARY, + PrimitiveType.POINT, + ]: + return value + elif schema["column_type"] in [PrimitiveType.UUID, PrimitiveType.ENUM, PrimitiveType.POINT]: + return str(value) + elif schema["column_type"] == PrimitiveType.DATE: + if isinstance(value, date): + return value.isoformat() + return str(value) + elif schema["column_type"] == PrimitiveType.DATE_ONLY: + if isinstance(value, datetime): + return value.date().isoformat() + return str(value) + elif schema["column_type"] == PrimitiveType.TIME_ONLY: + if isinstance(value, time) or isinstance(value, datetime): + return value.isoformat() + # return value.strftime("%H:%M:%S") # This format is in forest developer guide + return str(value) + elif isinstance(schema["column_type"], dict) or isinstance(schema["column_type"], list): + return value + else: + ForestLogger.log("error", f"Unknown column type {schema['column_type']}") + raise AgentToolkitException(f"Unknown column type {schema['column_type']}") + + def _serialize_relation( + self, name: str, data: Any, schema: RelationAlias, current_link: str + ) -> Tuple[Dict[str, Any], Optional[IncludedData]]: + relation, included = {}, None + sub_data = data[name] + if sub_data is None: + return { + "data": None if is_polymorphic_many_to_one(schema) or is_polymorphic_one_to_one(schema) else [], + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + }, included + + if is_polymorphic_many_to_one(schema): + relation, included = self._serialize_polymorphic_many_to_one_relationship(name, data, schema, current_link) + elif is_many_to_one(schema) or is_one_to_one(schema) or is_polymorphic_one_to_one(schema): + relation, included = self._serialize_to_one_relationships(name, sub_data, schema, current_link) + elif is_many_to_many(schema) or is_one_to_many(schema): + relation = { + "data": [], + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + } + + return relation, included + + def _serialize_to_one_relationships( + self, + name: str, + data: Any, + schema: Union[PolymorphicOneToOne, OneToOne, ManyToOne], + current_link: str, + ) -> Tuple[Dict[str, Any], IncludedData]: + """return (relationships, included)""" + foreign_collection = self.datasource.get_collection(schema["foreign_collection"]) + + relation = { + "data": { + "id": pack_id(foreign_collection.schema, data), + # "id": self._get_id(foreign_collection, data), + "type": schema["foreign_collection"], + }, + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + } + + sub_projection = self.projection.relations[name] + included_attributes = {} + for key, value in data.items(): + if key not in sub_projection or key in SchemaUtils.get_primary_keys(foreign_collection.schema): + continue + included_attributes[key] = self._serialize_value(value, foreign_collection.schema["fields"][key]) + + included = { + "id": self._get_id(foreign_collection, data), + "links": { + "self": f"/forest/{foreign_collection.name}/{self._get_id(foreign_collection, data)}", + }, + "type": foreign_collection.name, + } + if included_attributes != {}: + included["attributes"] = included_attributes + return relation, cast(IncludedData, included) + + def _serialize_polymorphic_many_to_one_relationship( + self, + name: str, + data: Any, + schema: PolymorphicManyToOne, + current_link: str, + ) -> Tuple[Dict[str, Any], IncludedData]: + """return (relationships, included)""" + sub_data = data[name] + foreign_collection = self.datasource.get_collection(data[schema["foreign_key_type_field"]]) + + relation = { + "data": { + "id": pack_id(foreign_collection.schema, sub_data), # TODO: validate + # "id": self._get_id(foreign_collection, sub_data), + "type": foreign_collection.name, + }, + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + } + included = self._serialize_one( + sub_data, foreign_collection, ProjectionFactory.all(foreign_collection, allow_nested=True) + ) + included["data"] = cast(Data, included["data"]) + included = { + "type": included["data"]["type"], + "id": included["data"]["id"], + "attributes": included["data"]["attributes"], + # **included.get("data", {}), # type: ignore for serialize_one it's a dict + "links": included.get("links", {}), + "relationships": {}, + } + + # add relationships key in included + for foreign_relation_name, foreign_relation_schema in foreign_collection.schema["fields"].items(): + if not is_column(foreign_relation_schema): + included["relationships"][foreign_relation_name] = { + "links": { + "related": { + "href": f"/forest/{foreign_collection.name}/{self._get_id(foreign_collection, sub_data)}" + f"/relationships/{foreign_relation_name}" + } + } + } + + return relation, cast(IncludedData, included)