refactor(connectors): BI-5673 move the type transformer definition to…

… a separate package (#571)

lib/dl_api_connector/dl_api_connector/api_schema/source_base.py

@@ -27,7 +27,6 @@
     IndexInfo,
     SchemaColumn,
 )
-from dl_core.db.native_type_schema import OneOfNativeTypeSchema
 from dl_core.marshmallow import FrozenSetField
 from dl_core.multisource import (
     BinaryCondition,
@@ -37,6 +36,7 @@
 )
 from dl_model_tools.schema.base import BaseSchema
 from dl_model_tools.schema.dynamic_enum_field import DynamicEnumField
+from dl_type_transformer.native_type_schema import OneOfNativeTypeSchema
 
 
 if TYPE_CHECKING:

lib/dl_api_connector/pyproject.toml

@@ -23,6 +23,7 @@ dl-configs = {path = "../dl_configs"}
 dl-core = {path = "../dl_core"}
 dl-query-processing = {path = "../dl_query_processing"}
 dl-model-tools = {path = "../dl_model_tools"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 
 [tool.poetry.group.tests.dependencies]
 pytest = ">=7.2.2"

lib/dl_api_lib/dl_api_lib/error_handling.py

@@ -26,7 +26,8 @@
 from dl_dashsql import exc as dashsql_exc
 from dl_formula.core import exc as formula_exc
 import dl_query_processing.exc
-from dl_rls import exc as rls_exc
+import dl_rls.exc
+import dl_type_transformer.exc
 
 
 LOGGER = logging.getLogger(__name__)
@@ -57,7 +58,7 @@
     common_exc.InvalidFieldError: status.BAD_REQUEST,
     common_exc.FieldNotFound: status.BAD_REQUEST,
     common_exc.USPermissionRequired: status.FORBIDDEN,
-    rls_exc.RLSError: status.BAD_REQUEST,
+    dl_rls.exc.RLSError: status.BAD_REQUEST,
     exc.FeatureNotAvailable: status.BAD_REQUEST,
     dl_query_processing.exc.FilterError: status.BAD_REQUEST,
     exc.UnsupportedForEntityType: status.BAD_REQUEST,
@@ -75,7 +76,7 @@
     exc.DatasetActionNotAllowedError: status.BAD_REQUEST,
     dl_query_processing.exc.LegendError: status.BAD_REQUEST,
     dl_query_processing.exc.PivotError: status.BAD_REQUEST,
-    common_exc.TypeCastFailed: status.BAD_REQUEST,
+    dl_type_transformer.exc.TypeCastFailed: status.BAD_REQUEST,
     dl_query_processing.exc.BlockSpecError: status.BAD_REQUEST,
     dl_query_processing.exc.TreeError: status.BAD_REQUEST,
     dl_query_processing.exc.ParameterError: status.BAD_REQUEST,

lib/dl_api_lib/pyproject.toml

@@ -39,6 +39,7 @@ dl-cache-engine = {path = "../dl_cache_engine"}
 dynamic-enum = {path = "../dynamic_enum"}
 typing-extensions = ">=4.9.0"
 dl-rls = {path = "../dl_rls"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 
 [tool.poetry.group.tests.dependencies]
 pytest = ">=7.2.2"

lib/dl_compeng_pg/dl_compeng_pg/compeng_pg_base/exec_adapter_base.py

@@ -19,7 +19,7 @@
 from dl_constants.enums import UserDataType
 from dl_core.connectors.base.query_compiler import QueryCompiler
 from dl_core.data_processing.processing.db_base.exec_adapter_base import ProcessorDbExecAdapterBase
-from dl_core.db.sa_types import make_sa_type
+from dl_type_transformer.sa_types import make_sa_type
 from dl_utils.streaming import AsyncChunkedBase
 
 from dl_connector_postgresql.core.postgresql.constants import BACKEND_TYPE_POSTGRES

lib/dl_compeng_pg/pyproject.toml

@@ -19,6 +19,7 @@ dl-constants = {path = "../dl_constants"}
 dl-connector-postgresql = {path = "../dl_connector_postgresql"}
 dl-core = {path = "../dl_core"}
 dl-sqlalchemy-postgres = {path = "../dl_sqlalchemy_postgres"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 dl-utils = {path = "../dl_utils"}
 python = ">=3.10, <3.13"
 sqlalchemy = ">=1.4.46, <2.0"

lib/dl_connector_bigquery/dl_connector_bigquery/core/sa_types.py

@@ -1,9 +1,7 @@
 import sqlalchemy_bigquery._types as bq_types
 
-from dl_core.db.sa_types_base import (
-    make_native_type,
-    simple_instantiator,
-)
+from dl_type_transformer.sa_types_base import simple_instantiator
+from dl_type_transformer.type_transformer import make_native_type
 
 from dl_connector_bigquery.core.constants import BACKEND_TYPE_BIGQUERY
 

lib/dl_connector_bigquery/dl_connector_bigquery/core/type_transformer.py

@@ -1,7 +1,7 @@
 import sqlalchemy_bigquery._types as bq_types
 
 from dl_constants.enums import UserDataType
-from dl_core.db.conversion_base import (
+from dl_type_transformer.type_transformer import (
     TypeTransformer,
     make_native_type,
 )

lib/dl_connector_bigquery/pyproject.toml

@@ -32,6 +32,7 @@ dl-api-connector = {path = "../dl_api_connector"}
 dl-core = {path = "../dl_core"}
 dl-model-tools = {path = "../dl_model_tools"}
 dl-db-testing = {path = "../dl_db_testing"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 
 [tool.deptry.per_rule_ignores]
 DEP001 = ["google"]

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/adapter.py

@@ -39,8 +39,8 @@
     TableIdent,
 )
 from dl_core.connectors.base.error_handling import ETBasedExceptionMaker
-from dl_core.db.native_type import CommonNativeType
 from dl_core.exc import DatabaseQueryError
+from dl_type_transformer.native_type import CommonNativeType
 
 from dl_connector_bitrix_gds.core.caches import (
     bitrix_cache_deserializer,

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/type_transformer.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dl_constants.enums import UserDataType
-from dl_core.db.conversion_base import (
+from dl_type_transformer.type_transformer import (
     TypeTransformer,
     make_native_type,
 )

lib/dl_connector_bitrix_gds/pyproject.toml

@@ -27,6 +27,7 @@ dl-sqlalchemy-bitrix = {path = "../dl_sqlalchemy_bitrix"}
 dl-cache-engine = {path = "../dl_cache_engine"}
 dl-formula = {path = "../dl_formula"}
 dl-model-tools = {path = "../dl_model_tools"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 
 [tool.poetry.plugins]
 [tool.poetry.plugins."dl_api_lib.connectors"]

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/type_transformer.py

@@ -10,7 +10,7 @@
 from clickhouse_sqlalchemy import types as ch_types
 
 from dl_constants.enums import UserDataType
-from dl_core.db.conversion_base import (
+from dl_type_transformer.type_transformer import (
     BooleanTypeCaster,
     DateTypeCaster,
     TypeCaster,

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py

@@ -20,10 +20,7 @@
 )
 from dl_configs.settings_submodels import S3Settings
 from dl_constants.enums import DataSourceType
-from dl_core.db import (
-    SchemaColumn,
-    get_type_transformer,
-)
+from dl_core.db import SchemaColumn
 from dl_core.services_registry import ServicesRegistry
 from dl_core_testing.configuration import RedisSettingMaker
 from dl_core_testing.database import DbTable
@@ -39,6 +36,7 @@
     create_s3_bucket,
     create_s3_client,
 )
+from dl_type_transformer.type_transformer import get_type_transformer
 
 from dl_connector_bundle_chs3.chs3_base.core.testing.utils import create_s3_native_from_ch_table
 from dl_connector_bundle_chs3.chs3_base.core.us_connection import BaseFileS3Connection

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/data_source.py

@@ -5,9 +5,9 @@
 
 from dl_constants.enums import UserDataType
 from dl_core.db import SchemaColumn
-from dl_core.db.native_type import GenericNativeType
 from dl_core_testing.fixtures.sample_tables import TABLE_SPEC_SAMPLE_SUPERSTORE
 from dl_core_testing.testcases.data_source import DefaultDataSourceTestClass
+from dl_type_transformer.native_type import GenericNativeType
 
 from dl_connector_bundle_chs3.chs3_base.core.data_source import BaseFileS3DataSource
 from dl_connector_bundle_chs3.chs3_base.core.data_source_spec import BaseFileS3DataSourceSpec

lib/dl_connector_bundle_chs3/pyproject.toml

@@ -29,6 +29,7 @@ dl-core = {path = "../dl_core"}
 dl-model-tools = {path = "../dl_model_tools"}
 dl-core-testing = {path = "../dl_core_testing"}
 dl-task-processor = {path = "../dl_task_processor"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 sqlalchemy = ">=1.4.46, <2.0"
 
 [tool.poetry.plugins]

lib/dl_connector_chyt/dl_connector_chyt/core/data_source.py

@@ -33,12 +33,12 @@
     SchemaColumn,
     SchemaInfo,
 )
-from dl_core.db.native_type import ClickHouseNativeType
 from dl_sqlalchemy_chyt import (
     CHYTTablesConcat,
     CHYTTablesRange,
     CHYTTableSubselect,
 )
+from dl_type_transformer.native_type import ClickHouseNativeType
 
 from dl_connector_chyt.core.constants import (
     CONNECTION_TYPE_CHYT,

lib/dl_connector_chyt/dl_connector_chyt/core/type_transformer.py

@@ -1,13 +1,13 @@
 from typing import ClassVar
 
 from dl_constants.enums import UserDataType
-from dl_core.db.conversion_base import (
+from dl_sqlalchemy_chyt import types as chyt_types
+from dl_type_transformer.native_type import GenericNativeType
+from dl_type_transformer.type_transformer import (
     TypeCaster,
     YTBooleanTypeCaster,
     make_native_type,
 )
-from dl_core.db.native_type import GenericNativeType
-from dl_sqlalchemy_chyt import types as chyt_types
 
 from dl_connector_clickhouse.core.clickhouse_base.type_transformer import ClickHouseTypeTransformer
 

lib/dl_connector_chyt/pyproject.toml

@@ -25,6 +25,7 @@ dl-configs = {path = "../dl_configs"}
 dl-api-connector = {path = "../dl_api_connector"}
 dl-core = {path = "../dl_core"}
 dl-db-testing = {path = "../dl_db_testing"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 aiohttp = ">=3.9.1"
 
 [tool.poetry.plugins]

lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py

@@ -60,17 +60,17 @@
 )
 from dl_core.connectors.base.error_transformer import DBExcKWArgs
 from dl_core.connectors.ssl_common.adapter import BaseSSLCertAdapter
-from dl_core.db.conversion_base import get_type_transformer
-from dl_core.db.native_type import (
+from dl_dashsql.formatting.placeholder_dbapi import DBAPIQueryFormatterFactory
+from dl_dashsql.registry import get_dash_sql_param_literalizer
+from dl_type_transformer.native_type import (
     ClickHouseDateTime64NativeType,
     ClickHouseDateTime64WithTZNativeType,
     ClickHouseDateTimeWithTZNativeType,
     ClickHouseNativeType,
     GenericNativeType,
     norm_native_type,
 )
-from dl_dashsql.formatting.placeholder_dbapi import DBAPIQueryFormatterFactory
-from dl_dashsql.registry import get_dash_sql_param_literalizer
+from dl_type_transformer.type_transformer import get_type_transformer
 from dl_utils.utils import make_url
 
 from dl_connector_clickhouse.core.clickhouse_base.ch_commons import (

lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/ch_commons.py

@@ -18,10 +18,8 @@
 from dl_core import exc
 from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI
 from dl_core.connectors.base.error_transformer import DBExcKWArgs
-from dl_core.db import (
-    SchemaColumn,
-    make_sa_type,
-)
+from dl_core.db import SchemaColumn
+from dl_type_transformer.sa_types import make_sa_type
 
 from dl_connector_clickhouse.core.clickhouse_base.constants import BACKEND_TYPE_CLICKHOUSE
 from dl_connector_clickhouse.core.clickhouse_base.exc import (
@@ -36,7 +34,7 @@
 
 
 if TYPE_CHECKING:
-    from dl_core.db.conversion_base import TypeTransformer
+    from dl_type_transformer.type_transformer import TypeTransformer
 
 LOGGER = logging.getLogger(__name__)
 

lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/data_source.py

@@ -2,10 +2,7 @@
 
 import abc
 import logging
-from typing import (
-    TYPE_CHECKING,
-    ClassVar,
-)
+from typing import ClassVar
 
 from dl_constants.enums import JoinType
 from dl_core.data_source.sql import (
@@ -17,14 +14,6 @@
 from dl_connector_clickhouse.core.clickhouse_base.constants import CONNECTION_TYPE_CLICKHOUSE
 
 
-if TYPE_CHECKING:
-    from dl_core import us_connection  # noqa
-    from dl_core.db import (  # noqa
-        SchemaColumn,
-        SchemaInfo,
-    )
-
-
 LOGGER = logging.getLogger(__name__)
 
 

lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/sa_types.py

@@ -12,15 +12,15 @@
 from sqlalchemy.types import TypeEngine
 
 from dl_constants.enums import SourceBackendType
-from dl_core.db.native_type import (
+from dl_type_transformer.native_type import (
     ClickHouseDateTime64NativeType,
     ClickHouseDateTime64WithTZNativeType,
     ClickHouseDateTimeWithTZNativeType,
     ClickHouseNativeType,
     CommonNativeType,
     GenericNativeType,
 )
-from dl_core.db.sa_types_base import make_native_type
+from dl_type_transformer.type_transformer import make_native_type
 
 from dl_connector_clickhouse.core.clickhouse_base.constants import BACKEND_TYPE_CLICKHOUSE
 from dl_connector_clickhouse.core.clickhouse_base.type_transformer import (

lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/type_transformer.py

@@ -4,7 +4,7 @@
 import sqlalchemy as sa
 
 from dl_constants.enums import UserDataType
-from dl_core.db.conversion_base import (
+from dl_type_transformer.type_transformer import (
     TypeTransformer,
     make_native_type,
 )

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_connection_executor.py

@@ -21,19 +21,19 @@
     SyncConnExecutorBase,
 )
 from dl_core.connection_models.common_models import DBIdent
-from dl_core.db.native_type import (
-    ClickHouseDateTime64WithTZNativeType,
-    ClickHouseDateTimeWithTZNativeType,
-    ClickHouseNativeType,
-    GenericNativeType,
-    norm_native_type,
-)
 from dl_core_testing.testcases.connection_executor import (
     DefaultAsyncConnectionExecutorTestSuite,
     DefaultSyncAsyncConnectionExecutorCheckBase,
     DefaultSyncConnectionExecutorTestSuite,
 )
 from dl_testing.regulated_test import RegulatedTestParams
+from dl_type_transformer.native_type import (
+    ClickHouseDateTime64WithTZNativeType,
+    ClickHouseDateTimeWithTZNativeType,
+    ClickHouseNativeType,
+    GenericNativeType,
+    norm_native_type,
+)
 
 from dl_connector_clickhouse.core.clickhouse.us_connection import ConnectionClickhouse
 from dl_connector_clickhouse_tests.db.config import CoreConnectionSettings

lib/dl_connector_clickhouse/pyproject.toml

@@ -34,6 +34,7 @@ dl-dashsql = {path = "../dl_dashsql"}
 dl-formula-testing = {path = "../dl_formula_testing"}
 dl-query-processing = {path = "../dl_query_processing"}
 dl-testing = {path = "../dl_testing"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 dl-utils = {path = "../dl_utils"}
 pytest = ">=7.2.2"
 pytz = ">=2022.7.1"

lib/dl_connector_greenplum/pyproject.toml

@@ -19,6 +19,7 @@ dl-connector-postgresql = {path = "../dl_connector_postgresql"}
 dl-configs = {path = "../dl_configs"}
 dl-api-connector = {path = "../dl_api_connector"}
 dl-core = {path = "../dl_core"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 asyncpg = ">=0.29.0"
 sqlalchemy = ">=1.4.46, <2.0"
 

lib/dl_connector_metrica/dl_connector_metrica/core/type_transformer.py

@@ -1,6 +1,8 @@
 from dl_constants.enums import UserDataType
-from dl_core.db import TypeTransformer
-from dl_core.db.conversion_base import make_native_type
+from dl_type_transformer.type_transformer import (
+    TypeTransformer,
+    make_native_type,
+)
 
 
 class MetrikaApiTypeTransformer(TypeTransformer):

lib/dl_connector_metrica/dl_connector_metrica/core/us_connection.py

@@ -17,10 +17,7 @@
     UserDataType,
 )
 from dl_core import exc
-from dl_core.db import (
-    SchemaColumn,
-    get_type_transformer,
-)
+from dl_core.db import SchemaColumn
 from dl_core.us_connection_base import (
     ConnectionBase,
     DataSourceTemplate,
@@ -30,6 +27,7 @@
 from dl_sqlalchemy_metrica_api import api_client as metrika_api_client
 from dl_sqlalchemy_metrica_api.api_info.appmetrica import AppMetricaFieldsNamespaces
 from dl_sqlalchemy_metrica_api.api_info.metrika import MetrikaApiCounterSource
+from dl_type_transformer.type_transformer import get_type_transformer
 from dl_utils.utils import DataKey
 
 from dl_connector_metrica.core.constants import (

lib/dl_connector_metrica/pyproject.toml

@@ -26,6 +26,7 @@ dl-api-lib = {path = "../dl_api_lib"}
 dl-model-tools = {path = "../dl_model_tools"}
 dl-query-processing = {path = "../dl_query_processing"}
 dl-app-tools = {path = "../dl_app_tools"}
+dl-type-transformer = {path = "../dl_type_transformer"}
 
 [tool.poetry.plugins]
 [tool.poetry.plugins."dl_api_lib.connectors"]