Moved backend_type-related properties from connectors to separate obj…

…ects

lib/dl_api_connector/dl_api_connector/connector.py

@@ -20,6 +20,7 @@
 from dl_api_lib.query.registry import MQMFactorySettingItem
 from dl_constants.enums import QueryProcessingMode
 from dl_core.connectors.base.connector import (
+    CoreBackendDefinition,
     CoreConnectionDefinition,
     CoreConnector,
     CoreSourceDefinition,
@@ -53,8 +54,9 @@ class ApiConnectionDefinition(abc.ABC):
     form_factory_cls: ClassVar[Optional[Type[ConnectionFormFactory]]] = None
 
 
-class ApiConnector(abc.ABC):
-    # backend_type-bound properties - TODO: move to a separate entity
+class ApiBackendDefinition(abc.ABC):
+    core_backend_definition: Type[CoreBackendDefinition]
+
     formula_dialect_name: ClassVar[DialectName] = DialectName.DUMMY
     multi_query_mutation_factories: tuple[MQMFactorySettingItem, ...] = (
         MQMFactorySettingItem(
@@ -70,8 +72,10 @@ class ApiConnector(abc.ABC):
     is_compeng_executable: ClassVar[bool] = False
     filter_formula_compiler_cls: ClassVar[Type[FilterFormulaCompiler]] = MainFilterFormulaCompiler
     dashsql_literalizer_cls: ClassVar[Type[DashSQLParamLiteralizer]] = DefaultDashSQLParamLiteralizer
-    # others
-    core_connector_cls: ClassVar[Type[CoreConnector]]
+
+
+class ApiConnector(abc.ABC):
+    backend_definition: Type[ApiBackendDefinition]
     connection_definitions: ClassVar[Tuple[Type[ApiConnectionDefinition], ...]] = ()
     source_definitions: ClassVar[Tuple[Type[ApiSourceDefinition], ...]] = ()
     translation_configs: ClassVar[frozenset[TranslationConfig]] = frozenset()

lib/dl_api_lib/dl_api_lib/connector_registrator.py

@@ -5,6 +5,7 @@
     register_source_template_api_schema,
 )
 from dl_api_connector.connector import (
+    ApiBackendDefinition,
     ApiConnectionDefinition,
     ApiConnector,
     ApiSourceDefinition,
@@ -47,29 +48,33 @@ def register_connection_definition(cls, conn_def: Type[ApiConnectionDefinition])
             register_connection_form_factory_cls(conn_type=conn_type, factory_cls=conn_def.form_factory_cls)
 
     @classmethod
-    def register_connector(cls, connector: Type[ApiConnector]) -> None:
-        # backend_type-related stuff - TODO: Move to a separate entity
-        backend_type = connector.core_connector_cls.backend_type
-        register_dialect_name(backend_type=backend_type, dialect_name=connector.formula_dialect_name)
-        for mqm_setting_item in connector.multi_query_mutation_factories:
+    def register_backend_definition(cls, backend_def: Type[ApiBackendDefinition]) -> None:
+        backend_type = backend_def.core_backend_definition.backend_type
+        register_dialect_name(backend_type=backend_type, dialect_name=backend_def.formula_dialect_name)
+        for mqm_setting_item in backend_def.multi_query_mutation_factories:
             register_multi_query_mutator_factory_cls(
                 query_proc_mode=mqm_setting_item.query_proc_mode,
                 backend_type=backend_type,
                 dialects=mqm_setting_item.dialects,
                 factory_cls=mqm_setting_item.factory_cls,
             )
-        register_is_forkable_source(backend_type=backend_type, is_forkable=connector.is_forkable)
-        register_is_compeng_executable(backend_type=backend_type, is_compeng_executable=connector.is_compeng_executable)
+        register_is_forkable_source(backend_type=backend_type, is_forkable=backend_def.is_forkable)
+        register_is_compeng_executable(
+            backend_type=backend_type,
+            is_compeng_executable=backend_def.is_compeng_executable,
+        )
         register_filter_formula_compiler_cls(
             backend_type=backend_type,
-            filter_compiler_cls=connector.filter_formula_compiler_cls,
+            filter_compiler_cls=backend_def.filter_formula_compiler_cls,
         )
         register_dash_sql_param_literalizer_cls(
             backend_type=backend_type,
-            literalizer_cls=connector.dashsql_literalizer_cls,
+            literalizer_cls=backend_def.dashsql_literalizer_cls,
         )
 
-        # everything else
+    @classmethod
+    def register_connector(cls, connector: Type[ApiConnector]) -> None:
+        cls.register_backend_definition(backend_def=connector.backend_definition)
         for source_def in connector.source_definitions:
             cls.register_source_definition(source_def=source_def)
         for conn_def in connector.connection_definitions:

lib/dl_connector_bigquery/dl_connector_bigquery/api/connector.py

@@ -3,6 +3,7 @@
     SubselectDataSourceTemplateSchema,
 )
 from dl_api_connector.connector import (
+    ApiBackendDefinition,
     ApiConnectionDefinition,
     ApiConnector,
     ApiSourceDefinition,
@@ -17,6 +18,7 @@
 from dl_connector_bigquery.api.connection_info import BigQueryConnectionInfoProvider
 from dl_connector_bigquery.api.i18n.localizer import CONFIGS
 from dl_connector_bigquery.core.connector import (
+    BigQueryCoreBackendDefinition,
     BigQueryCoreConnectionDefinition,
     BigQueryCoreConnector,
     BigQueryCoreSubselectSourceDefinition,
@@ -44,12 +46,16 @@ class BigQueryApiConnectionDefinition(ApiConnectionDefinition):
     form_factory_cls = BigQueryConnectionFormFactory
 
 
+class BigQueryApiBackendDefinition(ApiBackendDefinition):
+    core_backend_definition = BigQueryCoreBackendDefinition
+    formula_dialect_name = DIALECT_NAME_BIGQUERY
+
+
 class BigQueryApiConnector(ApiConnector):
-    core_connector_cls = BigQueryCoreConnector
+    backend_definition = BigQueryApiBackendDefinition
     source_definitions = (
         BigQueryApiTableSourceDefinition,
         BigQueryApiSubselectSourceDefinition,
     )
     connection_definitions = (BigQueryApiConnectionDefinition,)
-    formula_dialect_name = DIALECT_NAME_BIGQUERY
     translation_configs = frozenset(CONFIGS)

lib/dl_connector_bigquery/dl_connector_bigquery/core/connector.py

@@ -1,4 +1,5 @@
 from dl_core.connectors.base.connector import (
+    CoreBackendDefinition,
     CoreConnectionDefinition,
     CoreConnector,
     CoreSourceDefinition,
@@ -55,9 +56,13 @@ class BigQueryCoreSubselectSourceDefinition(CoreSourceDefinition):
     us_storage_schema_cls = BigQuerySubselectDataSourceSpecStorageSchema
 
 
-class BigQueryCoreConnector(CoreConnector):
+class BigQueryCoreBackendDefinition(CoreBackendDefinition):
     backend_type = BACKEND_TYPE_BIGQUERY
     compiler_cls = BigQueryQueryCompiler
+
+
+class BigQueryCoreConnector(CoreConnector):
+    backend_definition = BigQueryCoreBackendDefinition
     connection_definitions = (BigQueryCoreConnectionDefinition,)
     source_definitions = (
         BigQueryCoreTableSourceDefinition,

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py

@@ -5,6 +5,7 @@
     SQLDataSourceTemplateSchema,
 )
 from dl_api_connector.connector import (
+    ApiBackendDefinition,
     ApiConnectionDefinition,
     ApiConnector,
     ApiSourceDefinition,
@@ -20,6 +21,7 @@
 from dl_connector_bitrix_gds.api.i18n.localizer import CONFIGS
 from dl_connector_bitrix_gds.api.multi_query import BitrixGDSMultiQueryMutatorFactory
 from dl_connector_bitrix_gds.core.connector import (
+    BitrixGDSCoreBackendDefinition,
     BitrixGDSCoreConnectionDefinition,
     BitrixGDSCoreConnector,
     BitrixGDSCoreSourceDefinition,
@@ -40,8 +42,8 @@ class BitrixGDSApiConnectionDefinition(ApiConnectionDefinition):
     info_provider_cls = BitrixGDSConnectionInfoProvider
 
 
-class BitrixGDSApiConnector(ApiConnector):
-    core_connector_cls = BitrixGDSCoreConnector
+class BitrixGDSApiBackendDefinition(ApiBackendDefinition):
+    core_backend_definition = BitrixGDSCoreBackendDefinition
     formula_dialect_name = DIALECT_NAME_BITRIX
     multi_query_mutation_factories = (
         MQMFactorySettingItem(
@@ -53,9 +55,13 @@ class BitrixGDSApiConnector(ApiConnector):
             factory_cls=NoCompengMultiQueryMutatorFactory,
         ),
     )
-    connection_definitions = (BitrixGDSApiConnectionDefinition,)
-    source_definitions = (BitrixGDSApiSourceDefinition,)
     is_forkable = False
     is_compeng_executable = True
     filter_formula_compiler_cls = BitrixGDSFilterFormulaCompiler
+
+
+class BitrixGDSApiConnector(ApiConnector):
+    backend_definition = BitrixGDSApiBackendDefinition
+    connection_definitions = (BitrixGDSApiConnectionDefinition,)
+    source_definitions = (BitrixGDSApiSourceDefinition,)
     translation_configs = frozenset(CONFIGS)

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/connector.py

@@ -1,4 +1,5 @@
 from dl_core.connectors.base.connector import (
+    CoreBackendDefinition,
     CoreConnectionDefinition,
     CoreConnector,
     CoreSourceDefinition,
@@ -36,8 +37,12 @@ class BitrixGDSCoreSourceDefinition(CoreSourceDefinition):
     us_storage_schema_cls = SQLDataSourceSpecStorageSchema
 
 
-class BitrixGDSCoreConnector(CoreConnector):
+class BitrixGDSCoreBackendDefinition(CoreBackendDefinition):
     backend_type = BACKEND_TYPE_BITRIX_GDS
+
+
+class BitrixGDSCoreConnector(CoreConnector):
+    backend_definition = BitrixGDSCoreBackendDefinition
     connection_definitions = (BitrixGDSCoreConnectionDefinition,)
     source_definitions = (BitrixGDSCoreSourceDefinition,)
     rqe_adapter_classes = frozenset({BitrixGDSDefaultAdapter})

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/api/connector.py

@@ -17,7 +17,7 @@
     BaseFileS3CoreConnector,
     BaseFileS3TableCoreSourceDefinition,
 )
-from dl_connector_clickhouse.formula.constants import DIALECT_NAME_CLICKHOUSE
+from dl_connector_clickhouse.api.connector import ClickHouseApiBackendDefinition
 
 
 class BaseFileS3TableApiSourceDefinition(ApiSourceDefinition):
@@ -32,8 +32,7 @@ class BaseFileS3ApiConnectionDefinition(ApiConnectionDefinition):
 
 
 class BaseFileS3ApiConnector(ApiConnector):
-    core_connector_cls = BaseFileS3CoreConnector
-    formula_dialect_name = DIALECT_NAME_CLICKHOUSE
+    backend_definition = ClickHouseApiBackendDefinition
     connection_definitions = (BaseFileS3ApiConnectionDefinition,)
     source_definitions = (BaseFileS3TableApiSourceDefinition,)
     translation_configs = frozenset(CONFIGS)

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/connector.py

@@ -5,6 +5,7 @@
     NonUserInputConnectionSafetyChecker,
 )
 from dl_core.connectors.base.connector import (
+    CoreBackendDefinition,
     CoreConnectionDefinition,
     CoreConnector,
     CoreSourceDefinition,
@@ -25,10 +26,14 @@ class BaseFileS3TableCoreSourceDefinition(CoreSourceDefinition):
     pass
 
 
-class BaseFileS3CoreConnector(CoreConnector):
+class CHS3CoreBackendDefinition(CoreBackendDefinition):  # TODO: Why not just use CH??
     backend_type = BACKEND_TYPE_CHS3
+    query_cls = CHQuery
+    compiler_cls = ClickHouseQueryCompiler
+
+
+class BaseFileS3CoreConnector(CoreConnector):
+    backend_definition = CHS3CoreBackendDefinition
     conn_security = frozenset(
         {ConnSecuritySettings(NonUserInputConnectionSafetyChecker, frozenset({BaseFileS3ConnDTO}))}
     )
-    query_cls = CHQuery
-    compiler_cls = ClickHouseQueryCompiler

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_gsheets/api/connector.py

@@ -25,6 +25,5 @@ class GSheetsFileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition):
 
 
 class GSheetsFileS3ApiConnector(BaseFileS3ApiConnector):
-    core_connector_cls = GSheetsFileS3CoreConnector
     connection_definitions = (GSheetsFileS3ApiConnectionDefinition,)
     source_definitions = (GSheetsFileS3TableApiSourceDefinition,)

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/connector.py

@@ -25,6 +25,5 @@ class YaDocsFileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition):
 
 
 class YaDocsFileS3ApiConnector(BaseFileS3ApiConnector):
-    core_connector_cls = YaDocsFileS3CoreConnector
     connection_definitions = (YaDocsFileS3ApiConnectionDefinition,)
     source_definitions = (YaDocsFileS3TableApiSourceDefinition,)

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/file/api/connector.py

@@ -25,6 +25,5 @@ class FileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition):
 
 
 class FileS3ApiConnector(BaseFileS3ApiConnector):
-    core_connector_cls = FileS3CoreConnector
     connection_definitions = (FileS3ApiConnectionDefinition,)
     source_definitions = (FileS3TableApiSourceDefinition,)

lib/dl_connector_chyt/dl_connector_chyt/api/connector.py

@@ -23,14 +23,15 @@
 from dl_connector_chyt.api.connection_info import CHYTConnectionInfoProvider
 from dl_connector_chyt.api.i18n.localizer import CONFIGS
 from dl_connector_chyt.core.connector import (
+    CHYTCoreBackendDefinition,
     CHYTCoreConnectionDefinition,
     CHYTCoreConnector,
     CHYTTableCoreSourceDefinition,
     CHYTTableListCoreSourceDefinition,
     CHYTTableRangeCoreSourceDefinition,
     CHYTTableSubselectCoreSourceDefinition,
 )
-from dl_connector_clickhouse.formula.constants import DIALECT_NAME_CLICKHOUSE
+from dl_connector_clickhouse.api.connector import ClickHouseApiBackendDefinition
 
 
 class CHYTApiConnectionDefinition(ApiConnectionDefinition):
@@ -64,9 +65,12 @@ class CHYTSubselectApiSourceDefinition(ApiSourceDefinition):
     template_api_schema_cls = SubselectDataSourceTemplateSchema
 
 
+class CHYTApiBackendDefinition(ClickHouseApiBackendDefinition):
+    core_backend_definition = CHYTCoreBackendDefinition
+
+
 class CHYTApiConnector(ApiConnector):
-    core_connector_cls = CHYTCoreConnector
-    formula_dialect_name = DIALECT_NAME_CLICKHOUSE
+    backend_definition = CHYTApiBackendDefinition
     connection_definitions = (CHYTApiConnectionDefinition,)
     source_definitions = (
         CHYTTableApiSourceDefinition,

lib/dl_connector_chyt/dl_connector_chyt/core/connector.py

@@ -1,6 +1,7 @@
 from clickhouse_sqlalchemy.orm.query import Query as CHQuery
 
 from dl_core.connectors.base.connector import (
+    CoreBackendDefinition,
     CoreConnectionDefinition,
     CoreConnector,
     CoreSourceDefinition,
@@ -86,8 +87,18 @@ class CHYTTableSubselectCoreSourceDefinition(CoreSourceDefinition):
     us_storage_schema_cls = CHYTSubselectDataSourceSpecStorageSchema
 
 
-class CHYTCoreConnector(CoreConnector):
+class CHYTTableSubselectCoreSourceDefinition(CoreSourceDefinition):
+    source_type = SOURCE_TYPE_CHYT_YTSAURUS_SUBSELECT
+
+
+class CHYTCoreBackendDefinition(CoreBackendDefinition):
     backend_type = BACKEND_TYPE_CHYT
+    query_cls = CHQuery
+    compiler_cls = ClickHouseQueryCompiler
+
+
+class CHYTCoreConnector(CoreConnector):
+    backend_definition = CHYTCoreBackendDefinition
     connection_definitions = (CHYTCoreConnectionDefinition,)
     source_definitions = (
         CHYTTableCoreSourceDefinition,
@@ -101,5 +112,3 @@ class CHYTCoreConnector(CoreConnector):
             AsyncCHYTAdapter,
         }
     )
-    query_cls = CHQuery
-    compiler_cls = ClickHouseQueryCompiler

lib/dl_connector_clickhouse/dl_connector_clickhouse/api/connector.py

@@ -5,6 +5,7 @@
     SubselectDataSourceTemplateSchema,
 )
 from dl_api_connector.connector import (
+    ApiBackendDefinition,
     ApiConnectionDefinition,
     ApiConnector,
     ApiSourceDefinition,
@@ -23,6 +24,7 @@
     ClickHouseSubselectCoreSourceDefinition,
     ClickHouseTableCoreSourceDefinition,
 )
+from dl_connector_clickhouse.core.clickhouse_base.connector import ClickHouseCoreBackendDefinition
 from dl_connector_clickhouse.formula.constants import (
     DIALECT_NAME_CLICKHOUSE,
     ClickHouseDialect,
@@ -48,19 +50,23 @@ class ClickHouseApiConnectionDefinition(ApiConnectionDefinition):
     form_factory_cls = ClickHouseConnectionFormFactory
 
 
-class ClickHouseApiConnector(ApiConnector):
-    core_connector_cls = ClickHouseCoreConnector
-    connection_definitions = (ClickHouseApiConnectionDefinition,)
-    source_definitions = (
-        ClickHouseApiTableSourceDefinition,
-        ClickHouseApiSubselectSourceDefinition,
-    )
+class ClickHouseApiBackendDefinition(ApiBackendDefinition):
+    core_backend_definition = ClickHouseCoreBackendDefinition
     formula_dialect_name = DIALECT_NAME_CLICKHOUSE
-    translation_configs = frozenset(CONFIGS)
-    multi_query_mutation_factories = ApiConnector.multi_query_mutation_factories + (
+    multi_query_mutation_factories = ApiBackendDefinition.multi_query_mutation_factories + (
         MQMFactorySettingItem(
             query_proc_mode=QueryProcessingMode.native_wf,
             dialects=ClickHouseDialect.and_above(ClickHouseDialect.CLICKHOUSE_22_10).to_list(),
             factory_cls=NoCompengMultiQueryMutatorFactory,
         ),
     )
+
+
+class ClickHouseApiConnector(ApiConnector):
+    backend_definition = ClickHouseApiBackendDefinition
+    connection_definitions = (ClickHouseApiConnectionDefinition,)
+    source_definitions = (
+        ClickHouseApiTableSourceDefinition,
+        ClickHouseApiSubselectSourceDefinition,
+    )
+    translation_configs = frozenset(CONFIGS)