diff --git a/lib/dl_model_tools/dl_model_tools/schema/oneofschema.py b/lib/dl_model_tools/dl_model_tools/schema/oneofschema.py index 6fb142a0d..97c2263fd 100644 --- a/lib/dl_model_tools/dl_model_tools/schema/oneofschema.py +++ b/lib/dl_model_tools/dl_model_tools/schema/oneofschema.py @@ -23,7 +23,7 @@ from marshmallow_oneofschema import OneOfSchema -class OneOfSchemaWithDumpLoadHooks(OneOfSchema): # TODO: Move to bi_model_tools +class OneOfSchemaWithDumpLoadHooks(OneOfSchema): """ OneOfSchema has disabled all pre/post_dump/load hooks by default so we add the following copy-paste from marshmallow to be able to use those hooks diff --git a/lib/dl_model_tools/dl_model_tools/serialization.py b/lib/dl_model_tools/dl_model_tools/serialization.py index 7694731ef..d8536b016 100644 --- a/lib/dl_model_tools/dl_model_tools/serialization.py +++ b/lib/dl_model_tools/dl_model_tools/serialization.py @@ -14,8 +14,10 @@ import json from typing import ( Any, + Callable, ClassVar, Generic, + Optional, Type, TypeVar, Union, @@ -27,6 +29,8 @@ TJSONExt, TJSONLike, ) +from dl_type_transformer.native_type import GenericNativeType +from dl_type_transformer.native_type_schema import OneOfNativeTypeSchema _TS_TV = TypeVar("_TS_TV") @@ -245,6 +249,21 @@ def from_jsonable(value: TJSONLike) -> ipaddress.IPv6Interface: return ipaddress.IPv6Interface(value) +class NativeTypeSerializer(TypeSerializer[GenericNativeType]): + typename = "dl_native_type" + + schema = OneOfNativeTypeSchema() + + @staticmethod + def to_jsonable(value: GenericNativeType) -> TJSONLike: + return NativeTypeSerializer.schema.dump(value) + + @staticmethod + def from_jsonable(value: TJSONLike) -> GenericNativeType: + assert isinstance(value, dict) + return NativeTypeSerializer.schema.load(value) + + COMMON_SERIALIZERS: list[Type[TypeSerializer]] = [ DateSerializer, DatetimeSerializer, @@ -259,6 +278,7 @@ def from_jsonable(value: TJSONLike) -> ipaddress.IPv6Interface: IPv6NetworkSerializer, IPv4InterfaceSerializer, IPv6InterfaceSerializer, + NativeTypeSerializer, ] assert len(set(cls.typename for cls in COMMON_SERIALIZERS)) == len(COMMON_SERIALIZERS), "uniqueness check" @@ -268,7 +288,11 @@ class RedisDatalensDataJSONEncoder(json.JSONEncoder): def default(self, obj: Any) -> Any: typeobj = type(obj) - preprocessor = self.JSONABLERS_MAP.get(typeobj) + preprocessor: Optional[Type[TypeSerializer]] + if issubclass(typeobj, GenericNativeType): + preprocessor = NativeTypeSerializer + else: + preprocessor = self.JSONABLERS_MAP.get(typeobj) if preprocessor is not None: return dict(__dl_type__=preprocessor.typename, value=preprocessor.to_jsonable(obj)) @@ -278,8 +302,25 @@ def default(self, obj: Any) -> Any: class RedisDatalensDataJSONDecoder(json.JSONDecoder): DEJSONABLERS_MAP = {cls.typename: cls for cls in COMMON_SERIALIZERS} - def __init__(self, *args, **kwargs) -> None: # type: ignore # TODO: fix - super().__init__(*args, object_hook=self.object_hook, **kwargs) + def __init__( + self, + *, + object_hook: Optional[Callable[[dict[str, Any]], Any]] = None, + parse_float: Optional[Callable[[str], Any]] = None, + parse_int: Optional[Callable[[str], Any]] = None, + parse_constant: Optional[Callable[[str], Any]] = None, + strict: bool = True, + object_pairs_hook: Optional[Callable[[list[tuple[str, Any]]], Any]] = None, + ) -> None: + assert object_hook is None + super().__init__( + object_hook=self.object_hook, + parse_float=parse_float, + parse_int=parse_int, + parse_constant=parse_constant, + strict=strict, + object_pairs_hook=object_pairs_hook, + ) def object_hook(self, obj: dict[str, Any]) -> Any: # WARNING: this might collide with some unexpected objects that have a `__dl_type__` key. diff --git a/lib/dl_core/dl_core_tests/unit/test_json_serializer.py b/lib/dl_model_tools/dl_model_tools_tests/unit/test_json_serializer.py similarity index 51% rename from lib/dl_core/dl_core_tests/unit/test_json_serializer.py rename to lib/dl_model_tools/dl_model_tools_tests/unit/test_json_serializer.py index 2e44580d2..b4ef9aaa9 100644 --- a/lib/dl_core/dl_core_tests/unit/test_json_serializer.py +++ b/lib/dl_model_tools/dl_model_tools_tests/unit/test_json_serializer.py @@ -10,6 +10,15 @@ RedisDatalensDataJSONDecoder, RedisDatalensDataJSONEncoder, ) +from dl_type_transformer.native_type import ( + ClickHouseDateTime64NativeType, + ClickHouseDateTime64WithTZNativeType, + ClickHouseDateTimeWithTZNativeType, + ClickHouseNativeType, + CommonNativeType, + GenericNativeType, + LengthedNativeType, +) TZINFO = datetime.timezone(datetime.timedelta(seconds=-1320)) @@ -34,6 +43,31 @@ some_ipv6_network=ipaddress.IPv6Network("2001:db8::/64"), some_ipv4_interface=ipaddress.IPv4Interface("192.0.2.5/24"), some_ipv6_interface=ipaddress.IPv6Interface("2001:db8::1000/24"), + some_native_types=[ + GenericNativeType(name="tinyblob"), + CommonNativeType(name="double_precision", nullable=True), + LengthedNativeType(name="nvarchar2", nullable=False, length=121), + ClickHouseNativeType(name="uint64", nullable=True, lowcardinality=True), + ClickHouseDateTimeWithTZNativeType( + name="datetimewithtz", + nullable=False, + lowcardinality=True, + timezone_name="Europe/Moscow", + ), + ClickHouseDateTime64NativeType( + name="datetime64", + nullable=False, + lowcardinality=True, + precision=3, + ), + ClickHouseDateTime64WithTZNativeType( + name="datetime64withtz", + nullable=False, + lowcardinality=True, + precision=3, + timezone_name="Europe/Moscow", + ), + ], ) @@ -61,6 +95,67 @@ some_ipv6_network={"__dl_type__": "ipv6_network", "value": "2001:db8::/64"}, some_ipv4_interface={"__dl_type__": "ipv4_interface", "value": "192.0.2.5/24"}, some_ipv6_interface={"__dl_type__": "ipv6_interface", "value": "2001:db8::1000/24"}, + some_native_types=[ + { + "__dl_type__": "dl_native_type", + "value": {"name": "tinyblob", "native_type_class_name": "generic_native_type"}, + }, + { + "__dl_type__": "dl_native_type", + "value": {"name": "double_precision", "native_type_class_name": "common_native_type", "nullable": True}, + }, + { + "__dl_type__": "dl_native_type", + "value": { + "length": 121, + "name": "nvarchar2", + "native_type_class_name": "lengthed_native_type", + "nullable": False, + }, + }, + { + "__dl_type__": "dl_native_type", + "value": { + "lowcardinality": True, + "name": "uint64", + "native_type_class_name": "clickhouse_native_type", + "nullable": True, + }, + }, + { + "__dl_type__": "dl_native_type", + "value": { + "explicit_timezone": True, + "lowcardinality": True, + "name": "datetimewithtz", + "native_type_class_name": "clickhouse_datetimewithtz_native_type", + "nullable": False, + "timezone_name": "Europe/Moscow", + }, + }, + { + "__dl_type__": "dl_native_type", + "value": { + "lowcardinality": True, + "name": "datetime64", + "native_type_class_name": "clickhouse_datetime64_native_type", + "nullable": False, + "precision": 3, + }, + }, + { + "__dl_type__": "dl_native_type", + "value": { + "explicit_timezone": True, + "lowcardinality": True, + "name": "datetime64withtz", + "native_type_class_name": "clickhouse_datetime64withtz_native_type", + "nullable": False, + "precision": 3, + "timezone_name": "Europe/Moscow", + }, + }, + ], ) diff --git a/lib/dl_model_tools/pyproject.toml b/lib/dl_model_tools/pyproject.toml index 2af9d96a0..8cbeeff17 100644 --- a/lib/dl_model_tools/pyproject.toml +++ b/lib/dl_model_tools/pyproject.toml @@ -12,6 +12,7 @@ readme = "README.md" [tool.poetry.dependencies] attrs = ">=22.2.0" dl-constants = {path = "../dl_constants"} +dl-type-transformer = {path = "../dl_type_transformer"} dynamic-enum = {path = "../dynamic_enum"} marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index 6c7ae00c0..99c5da905 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -2282,6 +2282,7 @@ develop = false [package.dependencies] attrs = ">=22.2.0" dl-constants = {path = "../dl_constants"} +dl-type-transformer = {path = "../dl_type_transformer"} dynamic-enum = {path = "../dynamic_enum"} marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" @@ -5540,6 +5541,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},