Merge pull request #7 from tzoiker/feature/additional-properties

Support additionalProperties

CHANGES.md

@@ -1 +1,6 @@
 # Changelog
+
+## 0.1.1
+
+### Features
+- Support `additionalProperties` (`dict[str, Model]`)

README.md

@@ -59,9 +59,9 @@ Supports
 * `python>=3.8,<3.12`
 * `simdjson>=2,<6` (with caveats)
 
-Does not support complex schemas (yet), e.g.
+Does not support complex schemas (it may be not very reasonable from the
+practical standpoint anyway), e.g.,
 * `anyOf` (`Union[Model1, Model2]`)
-* `additionalProperties` (`dict[str, Model]`)
 * ...
 
 In such cases it will fully (not lazily) load the underlying objects.
@@ -125,6 +125,38 @@ assert parsed == [
 ]
 ```
 
+Example with `additionalProperties`:
+
+<!--  name: test_basic -->
+```python
+schema = {
+  "type": "object",
+  "additionalProperties": {
+    "$ref": "#/definitions/Model",
+  },
+  "definitions": {
+    "Model": {
+      "type": "object",
+      "properties": {
+        "key": {"type": "integer"},
+      }
+    }
+  }
+}
+
+data = json.dumps({
+    "some": {"key": 0, "other": 1},
+    "other": {"missing": 2},
+})
+
+parsed = loads(data, schema=schema)
+
+assert parsed == {
+    "some": {"key": 0},
+    "other": {},
+}
+```
+
 ### <a name="usage_reusing_parser"/> Reusing parser
 
 With re-used simdjson parser **(recommended when used in a single thread,
@@ -137,10 +169,10 @@ from simdjson import Parser
 parser = Parser()
 parsed = loads(data, schema=schema, parser=parser)
 
-assert parsed == [
-    {"key": 0},
-    {},
-]
+assert parsed == {
+    "some": {"key": 0},
+    "other": {},
+}
 ```
 
 ### <a name="usage_pydantic_v1"/> Pydantic v1

simdjson_schemaful/parser.py

@@ -10,22 +10,12 @@
 _FuncSet = Callable[..., None]
 
 
-_CACHE: Dict[str, Schema] = {}
-
-# TODO: handle additionalProperties, for example Dict[str, Model]
-# https://json-schema.org/understanding-json-schema/reference/object
-
-# TODO: handle anyOf
+# TODO: handle anyOf?
 
 
 def _get_definition(definitions: Dict[str, Schema], schema: Schema) -> Schema:
-    ref = schema.get("$ref")
-    if ref:
-        definition = _CACHE.get(ref)
-        if not definition:
-            definition = definitions[ref.split("/")[-1]]
-            _CACHE[ref] = definition
-        return definition
+    if ref := schema.get("$ref"):
+        return definitions[ref.split("/")[-1]]
     return schema
 
 
@@ -65,7 +55,9 @@ def _process_prop(
         return
 
     if (not type_ and not prop_data.get("$ref")) or (
-        type_ == "object" and not prop_data.get("properties")
+        type_ == "object"
+        and not prop_data.get("properties")
+        and not prop_data.get("additionalProperties")
     ):
         if not isinstance(value, simdjson.Object):
             raise ValueError(
@@ -127,26 +119,45 @@ def _loads(  # noqa: C901
                 )
 
             properties = schema.get("properties", {})
-            if not properties:
-                target.update(source.as_dict())  # type: ignore
+
+            if properties:
+                for prop_name, prop_data in properties.items():
+                    value = None
+                    try:
+                        value = source[prop_name]
+                    except KeyError:
+                        continue
+
+                    _process_prop(
+                        prop_data=prop_data,
+                        prop=prop_name,
+                        value=value,
+                        target=target,
+                        func_set=_set_dict,
+                        definitions=definitions,
+                        queue=queue,
+                    )
                 continue
 
-            for prop_name, prop_data in properties.items():
-                value = None
-                try:
-                    value = source[prop_name]
-                except KeyError:
-                    continue
+            additional_properties = _get_definition(
+                definitions=definitions,
+                schema=schema.get("additionalProperties", {}),
+            )
+            if additional_properties:
+                for prop_name in source.keys():
+                    value = source.get(prop_name)
+                    _process_prop(
+                        prop_data=additional_properties,
+                        prop=prop_name,
+                        value=value,
+                        target=target,
+                        func_set=_set_dict,
+                        definitions=definitions,
+                        queue=queue,
+                    )
+                continue
 
-                _process_prop(
-                    prop_data=prop_data,
-                    prop=prop_name,
-                    value=value,
-                    target=target,
-                    func_set=_set_dict,
-                    definitions=definitions,
-                    queue=queue,
-                )
+            target.update(source.as_dict())  # type: ignore
 
         elif type_ == "array":
             if not isinstance(source, simdjson.Array):

tests/pydantic/v1/conftest.py

@@ -1,5 +1,5 @@
 from importlib.util import find_spec
-from typing import Optional, Sequence
+from typing import Dict, Optional, Sequence
 
 from pydantic import Extra
 
@@ -22,6 +22,7 @@ class Model2(BaseModel):
                 f: float
 
             l2: Model2
+            l2_model_values: Optional[Dict[str, Model2]]
 
         l1_list: Sequence[Model1]
         l1_dict: Optional[Model1]

tests/pydantic/v1/test_model.py

@@ -41,7 +41,13 @@ def test_nested_not_an_array():
 def test_ok():
     data = {
         "l1_list": [
-            {"l2": {"s": "0", "i": 0, "f": 0.0, "other": "value"}},
+            {
+                "l2": {"s": "0", "i": 0, "f": 0.0, "other": "value"},
+                "l2_model_values": {
+                    "some": {"s": "0", "i": 0, "f": 0.0, "other": "value"},
+                    "other": {"s": "1", "i": 1, "f": 1.0, "other": "value"},
+                },
+            },
             {"l2": {"s": "1", "i": 1, "f": 1.0, "another": "value"}},
         ],
         "l1_dict": {
@@ -56,15 +62,21 @@ def test_ok():
     }
     expected = {
         "l1_list": [
-            {"l2": {"s": "0", "i": 0, "f": 0.0}},
+            {
+                "l2": {"s": "0", "i": 0, "f": 0.0},
+                "l2_model_values": {
+                    "some": {"s": "0", "i": 0, "f": 0.0},
+                    "other": {"s": "1", "i": 1, "f": 1.0},
+                },
+            },
             {"l2": {"s": "1", "i": 1, "f": 1.0}},
         ],
         "l1_dict": {
             "l2": {"s": "2", "i": 2, "f": 2.0},
         },
     }
     parsed = ModelNested.parse_raw_simdjson(dumps(data))
-    assert parsed.dict() == expected
+    assert parsed.dict(exclude_none=True) == expected
 
 
 def test_missing_required():

tests/pydantic/v1/test_parse.py

@@ -23,32 +23,16 @@ def test_union_fail():
         parse_raw_simdjson_as(model, data)
 
 
-def test_dict_model_value_fail():
+def test_dict_model_value_ok():
     model = Dict[str, Model]
     data = dumps({"key": {"some": 0, "value": 1}})
-    with pytest.raises(
-        ValidationError,
-        match=re.escape(
-            "1 validation error for ParsingModel[Dict[str, tests.pydantic.v1.conftest."
-            "Model]]\n__root__ -> key -> some\n  extra fields not permitted (type=va"
-            "lue_error.extra)"
-        ),
-    ):
-        parse_raw_simdjson_as(model, data)
+    assert parse_raw_simdjson_as(model, data)
 
 
-def test_nested_dict_model_value_fail():
+def test_nested_dict_model_value_ok():
     model = List[Dict[str, Model]]
     data = dumps([{"key": {"some": 0, "value": 1}}])
-    with pytest.raises(
-        ValidationError,
-        match=re.escape(
-            "1 validation error for ParsingModel[List[Dict[str, tests.pydantic.v1.conft"
-            "est.Model]]]\n__root__ -> 0 -> key -> some\n  extra fields not permitt"
-            "ed (type=value_error.extra)"
-        ),
-    ):
-        parse_raw_simdjson_as(model, data)
+    assert parse_raw_simdjson_as(model, data)
 
 
 def test_not_an_object():
@@ -97,22 +81,33 @@ def test_ok():
     data = [
         {
             "l1_list": [
-                {"l2": {"s": "0", "i": 0, "f": 0.0, "other": "value"}},
+                {
+                    "l2": {"s": "0", "i": 0, "f": 0.0, "other": "value"},
+                    "l2_model_values": {
+                        "some": {"s": "0", "i": 0, "f": 0.0, "other": "value"},
+                        "other": {"s": "1", "i": 1, "f": 1.0, "other": "value"},
+                    },
+                },
                 {"l2": {"s": "1", "i": 1, "f": 1.0, "another": "value"}},
             ],
         }
     ]
     expected = [
         {
             "l1_list": [
-                {"l2": {"s": "0", "i": 0, "f": 0.0}},
+                {
+                    "l2": {"s": "0", "i": 0, "f": 0.0},
+                    "l2_model_values": {
+                        "some": {"s": "0", "i": 0, "f": 0.0},
+                        "other": {"s": "1", "i": 1, "f": 1.0},
+                    },
+                },
                 {"l2": {"s": "1", "i": 1, "f": 1.0}},
             ],
-            "l1_dict": None,
         }
     ]
     (parsed,) = parse_raw_simdjson_as(List[ModelNested], dumps(data))
-    assert [parsed.dict()] == expected
+    assert [parsed.dict(exclude_none=True)] == expected
 
 
 def test_raw_missing_required():

tests/pydantic/v2/conftest.py

@@ -1,5 +1,5 @@
 from importlib.util import find_spec
-from typing import Optional, Sequence
+from typing import Dict, Optional, Sequence
 
 from pydantic import ConfigDict
 
@@ -21,6 +21,7 @@ class Model2(BaseModel):
                 f: float
 
             l2: Model2
+            l2_model_values: Optional[Dict[str, Model2]] = None
 
         l1_list: Sequence[Model1]
         l1_dict: Optional[Model1] = None

tests/pydantic/v2/test_model.py

@@ -46,7 +46,13 @@ def test_nested_not_an_array():
 def test_ok():
     data = {
         "l1_list": [
-            {"l2": {"s": "0", "i": 0, "f": 0.0, "other": "value"}},
+            {
+                "l2": {"s": "0", "i": 0, "f": 0.0, "other": "value"},
+                "l2_model_values": {
+                    "some": {"s": "0", "i": 0, "f": 0.0, "other": "value"},
+                    "other": {"s": "1", "i": 1, "f": 1.0, "other": "value"},
+                },
+            },
             {"l2": {"s": "1", "i": 1, "f": 1.0, "another": "value"}},
         ],
         "l1_dict": {
@@ -61,15 +67,21 @@ def test_ok():
     }
     expected = {
         "l1_list": [
-            {"l2": {"s": "0", "i": 0, "f": 0.0}},
+            {
+                "l2": {"s": "0", "i": 0, "f": 0.0},
+                "l2_model_values": {
+                    "some": {"s": "0", "i": 0, "f": 0.0},
+                    "other": {"s": "1", "i": 1, "f": 1.0},
+                },
+            },
             {"l2": {"s": "1", "i": 1, "f": 1.0}},
         ],
         "l1_dict": {
             "l2": {"s": "2", "i": 2, "f": 2.0},
         },
     }
     parsed = ModelNested.model_validate_simdjson(dumps(data))
-    assert parsed.model_dump() == expected
+    assert parsed.model_dump(exclude_none=True) == expected
 
 
 def test_missing_required():