From 62365dcec4c9d457b245041111c9d33d1dbeedbd Mon Sep 17 00:00:00 2001 From: Mateusz Kulas Date: Tue, 19 Sep 2023 16:11:52 +0200 Subject: [PATCH 1/3] Change the way JSONs are parsed --- odd_collector_sdk/utils/metadata.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/odd_collector_sdk/utils/metadata.py b/odd_collector_sdk/utils/metadata.py index ea712ed..a38dccb 100644 --- a/odd_collector_sdk/utils/metadata.py +++ b/odd_collector_sdk/utils/metadata.py @@ -23,12 +23,12 @@ class DefinitionType(Enum): def extract_metadata( - datasource: str, - entity: HasMetadata, - definition: DefinitionType, - flatten: Optional[bool] = False, - jsonify: Optional[bool] = False, - json_encoder: Optional[Type[json.JSONEncoder]] = None, + datasource: str, + entity: HasMetadata, + definition: DefinitionType, + flatten: Optional[bool] = False, + jsonify: Optional[bool] = False, + json_encoder: Optional[Type[json.JSONEncoder]] = None, ) -> MetadataExtension: """ :param datasource: name of datasource. @@ -48,13 +48,13 @@ def extract_metadata( if not isinstance(data, dict): raise TypeError(f"Metadata must be a dict, got {type(data)}") - if flatten: - data = flat_dict(data) - if jsonify: encode = partial(to_json, encoder=json_encoder) data = walk_values(encode, data) + if flatten: + data = flat_dict(data) + not_none = select_values(complement(is_none), data) return MetadataExtension(schema_url=schema_url, metadata=not_none) @@ -68,8 +68,10 @@ def is_none(value) -> bool: def to_json(value, encoder: Optional[Type[json.JSONEncoder]]) -> Optional[str]: + if isinstance(value, (str, int, float, bool, type(None))): + return value # Return primitives as-is try: - return json.dumps(value, cls=encoder or CustomJSONEncoder) + return json.loads(json.dumps(value, cls=encoder or CustomJSONEncoder)) # Deserialize back to Python objects except Exception as error: logger.error(f"Could not jsonfy {value=}. {error}.\n SKIP.") return None From cc9770441bbe41e4b6d435044969dd8f3a0c68ec Mon Sep 17 00:00:00 2001 From: Mateusz Kulas Date: Tue, 19 Sep 2023 16:17:42 +0200 Subject: [PATCH 2/3] Refactor --- odd_collector_sdk/utils/metadata.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/odd_collector_sdk/utils/metadata.py b/odd_collector_sdk/utils/metadata.py index a38dccb..8de1ffa 100644 --- a/odd_collector_sdk/utils/metadata.py +++ b/odd_collector_sdk/utils/metadata.py @@ -23,12 +23,12 @@ class DefinitionType(Enum): def extract_metadata( - datasource: str, - entity: HasMetadata, - definition: DefinitionType, - flatten: Optional[bool] = False, - jsonify: Optional[bool] = False, - json_encoder: Optional[Type[json.JSONEncoder]] = None, + datasource: str, + entity: HasMetadata, + definition: DefinitionType, + flatten: Optional[bool] = False, + jsonify: Optional[bool] = False, + json_encoder: Optional[Type[json.JSONEncoder]] = None, ) -> MetadataExtension: """ :param datasource: name of datasource. From 4bbf38f646fa3b751b40f936330cbaeb1f4e13b7 Mon Sep 17 00:00:00 2001 From: Mateusz Kulas Date: Wed, 20 Sep 2023 10:05:35 +0200 Subject: [PATCH 3/3] Update tests --- odd_collector_sdk/utils/metadata.py | 4 ++-- tests/utils/test_metadata.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/odd_collector_sdk/utils/metadata.py b/odd_collector_sdk/utils/metadata.py index 8de1ffa..975d9a6 100644 --- a/odd_collector_sdk/utils/metadata.py +++ b/odd_collector_sdk/utils/metadata.py @@ -49,7 +49,7 @@ def extract_metadata( raise TypeError(f"Metadata must be a dict, got {type(data)}") if jsonify: - encode = partial(to_json, encoder=json_encoder) + encode = partial(parse_complex_types, encoder=json_encoder) data = walk_values(encode, data) if flatten: @@ -67,7 +67,7 @@ def is_none(value) -> bool: return value is None -def to_json(value, encoder: Optional[Type[json.JSONEncoder]]) -> Optional[str]: +def parse_complex_types(value, encoder: Optional[Type[json.JSONEncoder]]) -> Optional[str]: if isinstance(value, (str, int, float, bool, type(None))): return value # Return primitives as-is try: diff --git a/tests/utils/test_metadata.py b/tests/utils/test_metadata.py index 2d5934b..9179100 100644 --- a/tests/utils/test_metadata.py +++ b/tests/utils/test_metadata.py @@ -24,7 +24,7 @@ def partial_extract_metadata(): "nested_field_one": 1, "nested_field_two": "string", "nested_field_three": [{"foo": "bat"}, {"foo": {}}], - "nested_field_four": {"more_nested_field": 1, "more_nested_field_two": {}}, + "nested_field_four": {"more_nested_field": 1, "more_nested_field_two": {}, "more_nested_field_three": "string"}, "nested_field_five": {}, }, } @@ -36,11 +36,12 @@ def partial_extract_metadata(): "nested_meta.nested_field_three": [{"foo": "bat"}, {"foo": {}}], "nested_meta.nested_field_four.more_nested_field": 1, "nested_meta.nested_field_four.more_nested_field_two": {}, + "nested_meta.nested_field_four.more_nested_field_three": "string", "nested_meta.nested_field_five": {}, } -ENCODED_METADATA = walk_values(json.dumps, COMPLEX_METADATA) -FLATTEN_ENCODED_METADATA = walk_values(json.dumps, FLATTEN) +ENCODED_METADATA = walk_values(lambda x: json.loads(json.dumps(x)), COMPLEX_METADATA) +FLATTEN_ENCODED_METADATA = walk_values(lambda x: json.loads(json.dumps(x)), FLATTEN) class Entity: