Deduplicate keys before populating PyDict

Author: friendlymatthew

Cargo.lock

@@ -45,6 +45,7 @@ num-bigint = "0.4.6"
 num-traits = "0.2.19"
 uuid = "1.18.1"
 jiter = { version = "0.11.0", features = ["python"] }
+indexmap = "2.7.1"
 hex = "0.4.3"
 percent-encoding = "2.3.1"
 

Cargo.toml

@@ -1,5 +1,7 @@
 use std::borrow::Cow;
 
+use indexmap::IndexMap;
+
 use jiter::{JsonArray, JsonObject, JsonValue};
 use num_traits::cast::ToPrimitive;
 use pyo3::prelude::*;
@@ -59,18 +61,25 @@ impl<'py, 'data> Input<'py> for JsonValue<'data> {
     }
 
     fn as_kwargs(&self, py: Python<'py>) -> Option<Bound<'py, PyDict>> {
-        match self {
-            JsonValue::Object(object) => {
-                let dict = PyDict::new(py);
-                for (k, v) in object.as_slice() {
-                    // TODO: jiter doesn't deduplicate keys, so we should probably do that here to
-                    // avoid potential wasted work creating Python objects.
-                    dict.set_item(k, v).unwrap();
-                }
-                Some(dict)
-            }
-            _ => None,
+        let JsonValue::Object(object) = self else {
+            return None;
+        };
+
+        // deduplicate keys before creating objects to avoid wasted work
+        // jiter doesn't deduplicate keys, so duplicate keys in JSON will appear multiple times
+        // in the slice. We use an IndexMap to keep only the last value for each key while preserving order
+        let mut unique_object = IndexMap::with_capacity(object.len());
+
+        for (k, v) in object.as_slice() {
+            unique_object.insert(k, v);
         }
+
+        let dict = PyDict::new(py);
+        for (k, v) in unique_object {
+            dict.set_item(k, v).unwrap();
+        }
+
+        Some(dict)
     }
 
     type Arguments<'a>

src/input/input_json.rs

@@ -1845,3 +1845,46 @@ class MyDataclass:
         assert dataclasses.asdict(
             s.validate_python({'my_field': 1}, by_alias=runtime_by_alias, by_name=runtime_by_name)
         ) == {'my_field': 1}
+
+
+def test_dataclass_json_duplicate_keys():
+    """Test that duplicate keys in JSON are handled correctly (last value wins).
+
+    We want to ensure that:
+    1. The last value for a duplicate key is used (standard JSON behavior)
+    2. We don't waste work creating Python objects for values that get overwritten
+    """
+
+    @dataclasses.dataclass
+    class MyDataclass:
+        name: str
+        age: int
+
+    schema = core_schema.dataclass_schema(
+        MyDataclass,
+        core_schema.dataclass_args_schema(
+            'MyDataclass',
+            [
+                core_schema.dataclass_field(name='name', schema=core_schema.str_schema()),
+                core_schema.dataclass_field(name='age', schema=core_schema.int_schema()),
+            ],
+        ),
+        ['name', 'age'],
+    )
+    v = SchemaValidator(schema)
+
+    # json with duplicate keys - the last value should win
+    json_with_duplicates = '{"name": "Alice", "age": 30, "name": "Bob", "age": 25}'
+    result = v.validate_json(json_with_duplicates)
+
+    assert result.name == 'Bob', "Last value for 'name' should win"
+    assert result.age == 25, "Last value for 'age' should win"
+    assert dataclasses.asdict(result) == {'name': 'Bob', 'age': 25}
+
+    # test with multiple duplicates of the same key
+    json_multiple_duplicates = '{"name": "First", "age": 1, "name": "Second", "name": "Third", "age": 3}'
+    result2 = v.validate_json(json_multiple_duplicates)
+
+    assert result2.name == 'Third', 'Last value among multiple duplicates should win'
+    assert result2.age == 3
+    assert dataclasses.asdict(result2) == {'name': 'Third', 'age': 3}