fix: json parsing in extract_metadata function (#38)

* Change the way JSONs are parsed * Refactor * Update tests --------- Co-authored-by: Mateusz Kulas <[email protected]>
opendatadiscovery · Sep 20, 2023 · 6693dcf · 6693dcf
1 parent 86dedaf
commit 6693dcf
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 9 deletions.
diff --git a/odd_collector_sdk/utils/metadata.py b/odd_collector_sdk/utils/metadata.py
@@ -48,13 +48,13 @@ def extract_metadata(
         if not isinstance(data, dict):
             raise TypeError(f"Metadata must be a dict, got {type(data)}")
 
-        if flatten:
-            data = flat_dict(data)
-
         if jsonify:
-            encode = partial(to_json, encoder=json_encoder)
+            encode = partial(parse_complex_types, encoder=json_encoder)
             data = walk_values(encode, data)
 
+        if flatten:
+            data = flat_dict(data)
+
         not_none = select_values(complement(is_none), data)
 
         return MetadataExtension(schema_url=schema_url, metadata=not_none)
@@ -67,9 +67,11 @@ def is_none(value) -> bool:
     return value is None
 
 
-def to_json(value, encoder: Optional[Type[json.JSONEncoder]]) -> Optional[str]:
+def parse_complex_types(value, encoder: Optional[Type[json.JSONEncoder]]) -> Optional[str]:
+    if isinstance(value, (str, int, float, bool, type(None))):
+        return value  # Return primitives as-is
     try:
-        return json.dumps(value, cls=encoder or CustomJSONEncoder)
+        return json.loads(json.dumps(value, cls=encoder or CustomJSONEncoder))  # Deserialize back to Python objects
     except Exception as error:
         logger.error(f"Could not jsonfy {value=}. {error}.\n SKIP.")
         return None

diff --git a/tests/utils/test_metadata.py b/tests/utils/test_metadata.py
@@ -24,7 +24,7 @@ def partial_extract_metadata():
         "nested_field_one": 1,
         "nested_field_two": "string",
         "nested_field_three": [{"foo": "bat"}, {"foo": {}}],
-        "nested_field_four": {"more_nested_field": 1, "more_nested_field_two": {}},
+        "nested_field_four": {"more_nested_field": 1, "more_nested_field_two": {}, "more_nested_field_three": "string"},
         "nested_field_five": {},
     },
 }
@@ -36,11 +36,12 @@ def partial_extract_metadata():
     "nested_meta.nested_field_three": [{"foo": "bat"}, {"foo": {}}],
     "nested_meta.nested_field_four.more_nested_field": 1,
     "nested_meta.nested_field_four.more_nested_field_two": {},
+    "nested_meta.nested_field_four.more_nested_field_three": "string",
     "nested_meta.nested_field_five": {},
 }
 
-ENCODED_METADATA = walk_values(json.dumps, COMPLEX_METADATA)
-FLATTEN_ENCODED_METADATA = walk_values(json.dumps, FLATTEN)
+ENCODED_METADATA = walk_values(lambda x: json.loads(json.dumps(x)), COMPLEX_METADATA)
+FLATTEN_ENCODED_METADATA = walk_values(lambda x: json.loads(json.dumps(x)), FLATTEN)
 
 
 class Entity: