diff --git a/lib/dl_api_connector/dl_api_connector/connector.py b/lib/dl_api_connector/dl_api_connector/connector.py index 9a996641d..2312f013a 100644 --- a/lib/dl_api_connector/dl_api_connector/connector.py +++ b/lib/dl_api_connector/dl_api_connector/connector.py @@ -20,6 +20,7 @@ from dl_api_lib.query.registry import MQMFactorySettingItem from dl_constants.enums import QueryProcessingMode from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -53,8 +54,9 @@ class ApiConnectionDefinition(abc.ABC): form_factory_cls: ClassVar[Optional[Type[ConnectionFormFactory]]] = None -class ApiConnector(abc.ABC): - # backend_type-bound properties - TODO: move to a separate entity +class ApiBackendDefinition(abc.ABC): + core_backend_definition: Type[CoreBackendDefinition] + formula_dialect_name: ClassVar[DialectName] = DialectName.DUMMY multi_query_mutation_factories: tuple[MQMFactorySettingItem, ...] = ( MQMFactorySettingItem( @@ -70,8 +72,10 @@ class ApiConnector(abc.ABC): is_compeng_executable: ClassVar[bool] = False filter_formula_compiler_cls: ClassVar[Type[FilterFormulaCompiler]] = MainFilterFormulaCompiler dashsql_literalizer_cls: ClassVar[Type[DashSQLParamLiteralizer]] = DefaultDashSQLParamLiteralizer - # others - core_connector_cls: ClassVar[Type[CoreConnector]] + + +class ApiConnector(abc.ABC): + backend_definition: Type[ApiBackendDefinition] connection_definitions: ClassVar[Tuple[Type[ApiConnectionDefinition], ...]] = () source_definitions: ClassVar[Tuple[Type[ApiSourceDefinition], ...]] = () translation_configs: ClassVar[frozenset[TranslationConfig]] = frozenset() diff --git a/lib/dl_api_lib/dl_api_lib/connector_registrator.py b/lib/dl_api_lib/dl_api_lib/connector_registrator.py index d416b9ecd..25d386f6a 100644 --- a/lib/dl_api_lib/dl_api_lib/connector_registrator.py +++ b/lib/dl_api_lib/dl_api_lib/connector_registrator.py @@ -5,6 +5,7 @@ register_source_template_api_schema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -47,29 +48,33 @@ def register_connection_definition(cls, conn_def: Type[ApiConnectionDefinition]) register_connection_form_factory_cls(conn_type=conn_type, factory_cls=conn_def.form_factory_cls) @classmethod - def register_connector(cls, connector: Type[ApiConnector]) -> None: - # backend_type-related stuff - TODO: Move to a separate entity - backend_type = connector.core_connector_cls.backend_type - register_dialect_name(backend_type=backend_type, dialect_name=connector.formula_dialect_name) - for mqm_setting_item in connector.multi_query_mutation_factories: + def register_backend_definition(cls, backend_def: Type[ApiBackendDefinition]) -> None: + backend_type = backend_def.core_backend_definition.backend_type + register_dialect_name(backend_type=backend_type, dialect_name=backend_def.formula_dialect_name) + for mqm_setting_item in backend_def.multi_query_mutation_factories: register_multi_query_mutator_factory_cls( query_proc_mode=mqm_setting_item.query_proc_mode, backend_type=backend_type, dialects=mqm_setting_item.dialects, factory_cls=mqm_setting_item.factory_cls, ) - register_is_forkable_source(backend_type=backend_type, is_forkable=connector.is_forkable) - register_is_compeng_executable(backend_type=backend_type, is_compeng_executable=connector.is_compeng_executable) + register_is_forkable_source(backend_type=backend_type, is_forkable=backend_def.is_forkable) + register_is_compeng_executable( + backend_type=backend_type, + is_compeng_executable=backend_def.is_compeng_executable, + ) register_filter_formula_compiler_cls( backend_type=backend_type, - filter_compiler_cls=connector.filter_formula_compiler_cls, + filter_compiler_cls=backend_def.filter_formula_compiler_cls, ) register_dash_sql_param_literalizer_cls( backend_type=backend_type, - literalizer_cls=connector.dashsql_literalizer_cls, + literalizer_cls=backend_def.dashsql_literalizer_cls, ) - # everything else + @classmethod + def register_connector(cls, connector: Type[ApiConnector]) -> None: + cls.register_backend_definition(backend_def=connector.backend_definition) for source_def in connector.source_definitions: cls.register_source_definition(source_def=source_def) for conn_def in connector.connection_definitions: diff --git a/lib/dl_connector_bigquery/dl_connector_bigquery/api/connector.py b/lib/dl_connector_bigquery/dl_connector_bigquery/api/connector.py index 9c362a1d4..16a43eeb3 100644 --- a/lib/dl_connector_bigquery/dl_connector_bigquery/api/connector.py +++ b/lib/dl_connector_bigquery/dl_connector_bigquery/api/connector.py @@ -3,6 +3,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -17,6 +18,7 @@ from dl_connector_bigquery.api.connection_info import BigQueryConnectionInfoProvider from dl_connector_bigquery.api.i18n.localizer import CONFIGS from dl_connector_bigquery.core.connector import ( + BigQueryCoreBackendDefinition, BigQueryCoreConnectionDefinition, BigQueryCoreConnector, BigQueryCoreSubselectSourceDefinition, @@ -44,12 +46,16 @@ class BigQueryApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = BigQueryConnectionFormFactory +class BigQueryApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = BigQueryCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_BIGQUERY + + class BigQueryApiConnector(ApiConnector): - core_connector_cls = BigQueryCoreConnector + backend_definition = BigQueryApiBackendDefinition source_definitions = ( BigQueryApiTableSourceDefinition, BigQueryApiSubselectSourceDefinition, ) connection_definitions = (BigQueryApiConnectionDefinition,) - formula_dialect_name = DIALECT_NAME_BIGQUERY translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_bigquery/dl_connector_bigquery/core/connector.py b/lib/dl_connector_bigquery/dl_connector_bigquery/core/connector.py index c4d557c27..fde19ae4b 100644 --- a/lib/dl_connector_bigquery/dl_connector_bigquery/core/connector.py +++ b/lib/dl_connector_bigquery/dl_connector_bigquery/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -55,9 +56,13 @@ class BigQueryCoreSubselectSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = BigQuerySubselectDataSourceSpecStorageSchema -class BigQueryCoreConnector(CoreConnector): +class BigQueryCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_BIGQUERY compiler_cls = BigQueryQueryCompiler + + +class BigQueryCoreConnector(CoreConnector): + backend_definition = BigQueryCoreBackendDefinition connection_definitions = (BigQueryCoreConnectionDefinition,) source_definitions = ( BigQueryCoreTableSourceDefinition, diff --git a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py index 67277a734..d4573f966 100644 --- a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py +++ b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py @@ -5,6 +5,7 @@ SQLDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -20,6 +21,7 @@ from dl_connector_bitrix_gds.api.i18n.localizer import CONFIGS from dl_connector_bitrix_gds.api.multi_query import BitrixGDSMultiQueryMutatorFactory from dl_connector_bitrix_gds.core.connector import ( + BitrixGDSCoreBackendDefinition, BitrixGDSCoreConnectionDefinition, BitrixGDSCoreConnector, BitrixGDSCoreSourceDefinition, @@ -40,8 +42,8 @@ class BitrixGDSApiConnectionDefinition(ApiConnectionDefinition): info_provider_cls = BitrixGDSConnectionInfoProvider -class BitrixGDSApiConnector(ApiConnector): - core_connector_cls = BitrixGDSCoreConnector +class BitrixGDSApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = BitrixGDSCoreBackendDefinition formula_dialect_name = DIALECT_NAME_BITRIX multi_query_mutation_factories = ( MQMFactorySettingItem( @@ -53,9 +55,13 @@ class BitrixGDSApiConnector(ApiConnector): factory_cls=NoCompengMultiQueryMutatorFactory, ), ) - connection_definitions = (BitrixGDSApiConnectionDefinition,) - source_definitions = (BitrixGDSApiSourceDefinition,) is_forkable = False is_compeng_executable = True filter_formula_compiler_cls = BitrixGDSFilterFormulaCompiler + + +class BitrixGDSApiConnector(ApiConnector): + backend_definition = BitrixGDSApiBackendDefinition + connection_definitions = (BitrixGDSApiConnectionDefinition,) + source_definitions = (BitrixGDSApiSourceDefinition,) translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/connector.py b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/connector.py index 34730b74c..f4a21aba5 100644 --- a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/connector.py +++ b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -36,8 +37,12 @@ class BitrixGDSCoreSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = SQLDataSourceSpecStorageSchema -class BitrixGDSCoreConnector(CoreConnector): +class BitrixGDSCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_BITRIX_GDS + + +class BitrixGDSCoreConnector(CoreConnector): + backend_definition = BitrixGDSCoreBackendDefinition connection_definitions = (BitrixGDSCoreConnectionDefinition,) source_definitions = (BitrixGDSCoreSourceDefinition,) rqe_adapter_classes = frozenset({BitrixGDSDefaultAdapter}) diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/api/connector.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/api/connector.py index 1ee0fe1cc..780d57117 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/api/connector.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/api/connector.py @@ -17,7 +17,7 @@ BaseFileS3CoreConnector, BaseFileS3TableCoreSourceDefinition, ) -from dl_connector_clickhouse.formula.constants import DIALECT_NAME_CLICKHOUSE +from dl_connector_clickhouse.api.connector import ClickHouseApiBackendDefinition class BaseFileS3TableApiSourceDefinition(ApiSourceDefinition): @@ -32,8 +32,7 @@ class BaseFileS3ApiConnectionDefinition(ApiConnectionDefinition): class BaseFileS3ApiConnector(ApiConnector): - core_connector_cls = BaseFileS3CoreConnector - formula_dialect_name = DIALECT_NAME_CLICKHOUSE + backend_definition = ClickHouseApiBackendDefinition connection_definitions = (BaseFileS3ApiConnectionDefinition,) source_definitions = (BaseFileS3TableApiSourceDefinition,) translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/connector.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/connector.py index a8b38d9bf..63885e32c 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/connector.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/connector.py @@ -5,6 +5,7 @@ NonUserInputConnectionSafetyChecker, ) from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -25,10 +26,14 @@ class BaseFileS3TableCoreSourceDefinition(CoreSourceDefinition): pass -class BaseFileS3CoreConnector(CoreConnector): +class CHS3CoreBackendDefinition(CoreBackendDefinition): # TODO: Why not just use CH?? backend_type = BACKEND_TYPE_CHS3 + query_cls = CHQuery + compiler_cls = ClickHouseQueryCompiler + + +class BaseFileS3CoreConnector(CoreConnector): + backend_definition = CHS3CoreBackendDefinition conn_security = frozenset( {ConnSecuritySettings(NonUserInputConnectionSafetyChecker, frozenset({BaseFileS3ConnDTO}))} ) - query_cls = CHQuery - compiler_cls = ClickHouseQueryCompiler diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_gsheets/api/connector.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_gsheets/api/connector.py index 0a9e1badc..a6350ccd7 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_gsheets/api/connector.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_gsheets/api/connector.py @@ -25,6 +25,5 @@ class GSheetsFileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition): class GSheetsFileS3ApiConnector(BaseFileS3ApiConnector): - core_connector_cls = GSheetsFileS3CoreConnector connection_definitions = (GSheetsFileS3ApiConnectionDefinition,) source_definitions = (GSheetsFileS3TableApiSourceDefinition,) diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/connector.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/connector.py index 8a9e583e7..fcf721ba7 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/connector.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_yadocs/api/connector.py @@ -25,6 +25,5 @@ class YaDocsFileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition): class YaDocsFileS3ApiConnector(BaseFileS3ApiConnector): - core_connector_cls = YaDocsFileS3CoreConnector connection_definitions = (YaDocsFileS3ApiConnectionDefinition,) source_definitions = (YaDocsFileS3TableApiSourceDefinition,) diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/file/api/connector.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/file/api/connector.py index 081f1e165..9e20ac077 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/file/api/connector.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/file/api/connector.py @@ -25,6 +25,5 @@ class FileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition): class FileS3ApiConnector(BaseFileS3ApiConnector): - core_connector_cls = FileS3CoreConnector connection_definitions = (FileS3ApiConnectionDefinition,) source_definitions = (FileS3TableApiSourceDefinition,) diff --git a/lib/dl_connector_chyt/dl_connector_chyt/api/connector.py b/lib/dl_connector_chyt/dl_connector_chyt/api/connector.py index e6542dbb0..a226fc712 100644 --- a/lib/dl_connector_chyt/dl_connector_chyt/api/connector.py +++ b/lib/dl_connector_chyt/dl_connector_chyt/api/connector.py @@ -23,6 +23,7 @@ from dl_connector_chyt.api.connection_info import CHYTConnectionInfoProvider from dl_connector_chyt.api.i18n.localizer import CONFIGS from dl_connector_chyt.core.connector import ( + CHYTCoreBackendDefinition, CHYTCoreConnectionDefinition, CHYTCoreConnector, CHYTTableCoreSourceDefinition, @@ -30,7 +31,7 @@ CHYTTableRangeCoreSourceDefinition, CHYTTableSubselectCoreSourceDefinition, ) -from dl_connector_clickhouse.formula.constants import DIALECT_NAME_CLICKHOUSE +from dl_connector_clickhouse.api.connector import ClickHouseApiBackendDefinition class CHYTApiConnectionDefinition(ApiConnectionDefinition): @@ -64,9 +65,12 @@ class CHYTSubselectApiSourceDefinition(ApiSourceDefinition): template_api_schema_cls = SubselectDataSourceTemplateSchema +class CHYTApiBackendDefinition(ClickHouseApiBackendDefinition): + core_backend_definition = CHYTCoreBackendDefinition + + class CHYTApiConnector(ApiConnector): - core_connector_cls = CHYTCoreConnector - formula_dialect_name = DIALECT_NAME_CLICKHOUSE + backend_definition = CHYTApiBackendDefinition connection_definitions = (CHYTApiConnectionDefinition,) source_definitions = ( CHYTTableApiSourceDefinition, diff --git a/lib/dl_connector_chyt/dl_connector_chyt/core/connector.py b/lib/dl_connector_chyt/dl_connector_chyt/core/connector.py index 19547c66c..2f65a5ee7 100644 --- a/lib/dl_connector_chyt/dl_connector_chyt/core/connector.py +++ b/lib/dl_connector_chyt/dl_connector_chyt/core/connector.py @@ -1,6 +1,7 @@ from clickhouse_sqlalchemy.orm.query import Query as CHQuery from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -86,8 +87,18 @@ class CHYTTableSubselectCoreSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = CHYTSubselectDataSourceSpecStorageSchema -class CHYTCoreConnector(CoreConnector): +class CHYTTableSubselectCoreSourceDefinition(CoreSourceDefinition): + source_type = SOURCE_TYPE_CHYT_YTSAURUS_SUBSELECT + + +class CHYTCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_CHYT + query_cls = CHQuery + compiler_cls = ClickHouseQueryCompiler + + +class CHYTCoreConnector(CoreConnector): + backend_definition = CHYTCoreBackendDefinition connection_definitions = (CHYTCoreConnectionDefinition,) source_definitions = ( CHYTTableCoreSourceDefinition, @@ -101,5 +112,3 @@ class CHYTCoreConnector(CoreConnector): AsyncCHYTAdapter, } ) - query_cls = CHQuery - compiler_cls = ClickHouseQueryCompiler diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse/api/connector.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse/api/connector.py index 552e7702d..d78edc5f2 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse/api/connector.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse/api/connector.py @@ -5,6 +5,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -23,6 +24,7 @@ ClickHouseSubselectCoreSourceDefinition, ClickHouseTableCoreSourceDefinition, ) +from dl_connector_clickhouse.core.clickhouse_base.connector import ClickHouseCoreBackendDefinition from dl_connector_clickhouse.formula.constants import ( DIALECT_NAME_CLICKHOUSE, ClickHouseDialect, @@ -48,15 +50,9 @@ class ClickHouseApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = ClickHouseConnectionFormFactory -class ClickHouseApiConnector(ApiConnector): - core_connector_cls = ClickHouseCoreConnector - connection_definitions = (ClickHouseApiConnectionDefinition,) - source_definitions = ( - ClickHouseApiTableSourceDefinition, - ClickHouseApiSubselectSourceDefinition, - ) +class ClickHouseApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = ClickHouseCoreBackendDefinition formula_dialect_name = DIALECT_NAME_CLICKHOUSE - translation_configs = frozenset(CONFIGS) multi_query_mutation_factories = ApiConnector.multi_query_mutation_factories + ( MQMFactorySettingItem( query_proc_mode=QueryProcessingMode.native_wf, @@ -64,3 +60,13 @@ class ClickHouseApiConnector(ApiConnector): factory_cls=NoCompengMultiQueryMutatorFactory, ), ) + + +class ClickHouseApiConnector(ApiConnector): + backend_definition = ClickHouseApiBackendDefinition + connection_definitions = (ClickHouseApiConnectionDefinition,) + source_definitions = ( + ClickHouseApiTableSourceDefinition, + ClickHouseApiSubselectSourceDefinition, + ) + translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/connector.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/connector.py index 499c93853..530870996 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/connector.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/connector.py @@ -1,6 +1,9 @@ from clickhouse_sqlalchemy.orm.query import Query as CHQuery -from dl_core.connectors.base.connector import CoreConnector +from dl_core.connectors.base.connector import ( + CoreBackendDefinition, + CoreConnector, +) from dl_connector_clickhouse.core.clickhouse_base.adapters import ( AsyncClickHouseAdapter, @@ -11,9 +14,13 @@ from dl_connector_clickhouse.core.clickhouse_base.sa_types import SQLALCHEMY_CLICKHOUSE_TYPES -class ClickHouseCoreConnectorBase(CoreConnector): +class ClickHouseCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_CLICKHOUSE compiler_cls = ClickHouseQueryCompiler query_cls = CHQuery + + +class ClickHouseCoreConnectorBase(CoreConnector): + backend_definition = ClickHouseCoreBackendDefinition rqe_adapter_classes = frozenset({ClickHouseAdapter, AsyncClickHouseAdapter}) sa_types = SQLALCHEMY_CLICKHOUSE_TYPES diff --git a/lib/dl_connector_greenplum/dl_connector_greenplum/api/connector.py b/lib/dl_connector_greenplum/dl_connector_greenplum/api/connector.py index 5fbded38e..f820839ac 100644 --- a/lib/dl_connector_greenplum/dl_connector_greenplum/api/connector.py +++ b/lib/dl_connector_greenplum/dl_connector_greenplum/api/connector.py @@ -15,12 +15,13 @@ from dl_connector_greenplum.api.connection_info import GreenplumConnectionInfoProvider from dl_connector_greenplum.api.i18n.localizer import CONFIGS from dl_connector_greenplum.core.connector import ( + GreenplumCoreBackendDefinition, GreenplumCoreConnectionDefinition, GreenplumCoreConnector, GreenplumSubselectCoreSourceDefinition, GreenplumTableCoreSourceDefinition, ) -from dl_connector_postgresql.formula.constants import DIALECT_NAME_POSTGRESQL +from dl_connector_postgresql.api.connector import PostgreSQLApiBackendDefinition class GreenplumApiTableSourceDefinition(ApiSourceDefinition): @@ -42,12 +43,15 @@ class GreenplumApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = GreenplumConnectionFormFactory +class GreenplumApiBackendDefinition(PostgreSQLApiBackendDefinition): + core_backend_definition = GreenplumCoreBackendDefinition + + class GreenplumApiConnector(ApiConnector): - core_connector_cls = GreenplumCoreConnector + backend_definition = GreenplumApiBackendDefinition connection_definitions = (GreenplumApiConnectionDefinition,) source_definitions = ( GreenplumApiTableSourceDefinition, GreenplumApiSubselectSourceDefinition, ) - formula_dialect_name = DIALECT_NAME_POSTGRESQL translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_greenplum/dl_connector_greenplum/core/connector.py b/lib/dl_connector_greenplum/dl_connector_greenplum/core/connector.py index 29d467cdf..7c6335fdf 100644 --- a/lib/dl_connector_greenplum/dl_connector_greenplum/core/connector.py +++ b/lib/dl_connector_greenplum/dl_connector_greenplum/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, ) @@ -52,8 +53,13 @@ class GreenplumSubselectCoreSourceDefinition(SQLSubselectCoreSourceDefinitionBas source_cls = GreenplumSubselectDataSource -class GreenplumCoreConnector(CoreConnector): +class GreenplumCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_GREENPLUM + compiler_cls = PostgreSQLQueryCompiler + + +class GreenplumCoreConnector(CoreConnector): + backend_definition = GreenplumCoreBackendDefinition connection_definitions = (GreenplumCoreConnectionDefinition,) source_definitions = ( GreenplumTableCoreSourceDefinition, @@ -61,4 +67,3 @@ class GreenplumCoreConnector(CoreConnector): ) rqe_adapter_classes = frozenset({PostgresAdapter, AsyncPostgresAdapter}) sa_types = SQLALCHEMY_POSTGRES_TYPES - compiler_cls = PostgreSQLQueryCompiler diff --git a/lib/dl_connector_metrica/dl_connector_metrica/api/connector.py b/lib/dl_connector_metrica/dl_connector_metrica/api/connector.py index db86331c4..bb857732a 100644 --- a/lib/dl_connector_metrica/dl_connector_metrica/api/connector.py +++ b/lib/dl_connector_metrica/dl_connector_metrica/api/connector.py @@ -3,6 +3,7 @@ SQLDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -23,9 +24,11 @@ from dl_connector_metrica.api.filter_compiler import MetricaApiFilterFormulaCompiler from dl_connector_metrica.api.i18n.localizer import CONFIGS from dl_connector_metrica.core.connector import ( + AppMetricaApiCoreBackendDefinition, AppMetricaApiCoreConnectionDefinition, AppMetricaApiCoreConnector, AppMetricaApiCoreSourceDefinition, + MetricaApiCoreBackendDefinition, MetricaApiCoreConnectionDefinition, MetricaApiCoreConnector, MetricaApiCoreSourceDefinition, @@ -47,12 +50,16 @@ class MetricaApiApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = MetricaAPIConnectionFormFactory +class MetricaApiApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = MetricaApiCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_METRICAAPI + filter_formula_compiler_cls = MetricaApiFilterFormulaCompiler + + class MetricaApiApiConnector(ApiConnector): - core_connector_cls = MetricaApiCoreConnector + backend_definition = MetricaApiApiBackendDefinition connection_definitions = (MetricaApiApiConnectionDefinition,) source_definitions = (MetricaApiFilteredApiTableSourceDefinition,) - filter_formula_compiler_cls = MetricaApiFilterFormulaCompiler - formula_dialect_name = DIALECT_NAME_METRICAAPI translation_configs = frozenset(CONFIGS) @@ -70,10 +77,14 @@ class AppMetricaApiApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = AppMetricaAPIConnectionFormFactory +class AppMetricaApiApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = AppMetricaApiCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_METRICAAPI + filter_formula_compiler_cls = MetricaApiFilterFormulaCompiler + + class AppMetricaApiApiConnector(ApiConnector): - core_connector_cls = AppMetricaApiCoreConnector + backend_definition = AppMetricaApiApiBackendDefinition connection_definitions = (AppMetricaApiApiConnectionDefinition,) source_definitions = (AppMetricaApiFilteredApiTableSourceDefinition,) - filter_formula_compiler_cls = MetricaApiFilterFormulaCompiler - formula_dialect_name = DIALECT_NAME_METRICAAPI translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_metrica/dl_connector_metrica/core/connector.py b/lib/dl_connector_metrica/dl_connector_metrica/core/connector.py index bf6c86233..b3090b5ce 100644 --- a/lib/dl_connector_metrica/dl_connector_metrica/core/connector.py +++ b/lib/dl_connector_metrica/dl_connector_metrica/core/connector.py @@ -5,6 +5,7 @@ NonUserInputConnectionSafetyChecker, ) from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -76,8 +77,12 @@ class MetricaApiCoreSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = SQLDataSourceSpecStorageSchema -class MetricaApiCoreConnector(CoreConnector): +class MetricaApiCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_METRICA_API + + +class MetricaApiCoreConnector(CoreConnector): + backend_definition = MetricaApiCoreBackendDefinition connection_definitions = (MetricaApiCoreConnectionDefinition,) source_definitions = (MetricaApiCoreSourceDefinition,) rqe_adapter_classes = frozenset({MetricaAPIDefaultAdapter}) @@ -108,8 +113,12 @@ class AppMetricaApiCoreSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = SQLDataSourceSpecStorageSchema -class AppMetricaApiCoreConnector(CoreConnector): +class AppMetricaApiCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_APPMETRICA_API + + +class AppMetricaApiCoreConnector(CoreConnector): + backend_definition = AppMetricaApiCoreBackendDefinition connection_definitions = (AppMetricaApiCoreConnectionDefinition,) source_definitions = (AppMetricaApiCoreSourceDefinition,) rqe_adapter_classes = frozenset({AppMetricaAPIDefaultAdapter}) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py b/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py index 88d5e28e3..8de338933 100644 --- a/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py +++ b/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py @@ -5,6 +5,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -15,6 +16,7 @@ from dl_connector_mssql.api.connection_info import MSSQLConnectionInfoProvider from dl_connector_mssql.api.i18n.localizer import CONFIGS from dl_connector_mssql.core.connector import ( + MSSQLCoreBackendDefinition, MSSQLCoreConnectionDefinition, MSSQLCoreConnector, MSSQLSubselectCoreSourceDefinition, @@ -42,12 +44,16 @@ class MSSQLApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = MSSQLConnectionFormFactory +class MSSQLApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = MSSQLCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_MSSQLSRV + + class MSSQLApiConnector(ApiConnector): - core_connector_cls = MSSQLCoreConnector + backend_definition = MSSQLApiBackendDefinition connection_definitions = (MSSQLApiConnectionDefinition,) source_definitions = ( MSSQLApiTableSourceDefinition, MSSQLApiSubselectSourceDefinition, ) - formula_dialect_name = DIALECT_NAME_MSSQLSRV translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py b/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py index f03e746ce..1e1f1d65e 100644 --- a/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py @@ -1,6 +1,7 @@ import pyodbc from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, ) @@ -50,8 +51,13 @@ class MSSQLSubselectCoreSourceDefinition(SQLSubselectCoreSourceDefinitionBase): source_cls = MSSQLSubselectDataSource -class MSSQLCoreConnector(CoreConnector): +class MSSQLCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_MSSQL + compiler_cls = MSSQLQueryCompiler + + +class MSSQLCoreConnector(CoreConnector): + backend_definition = MSSQLCoreBackendDefinition connection_definitions = (MSSQLCoreConnectionDefinition,) source_definitions = ( MSSQLTableCoreSourceDefinition, @@ -60,4 +66,3 @@ class MSSQLCoreConnector(CoreConnector): rqe_adapter_classes = frozenset({MSSQLDefaultAdapter}) sa_types = SQLALCHEMY_MSSQL_TYPES query_fail_exceptions = frozenset({pyodbc.Error}) - compiler_cls = MSSQLQueryCompiler diff --git a/lib/dl_connector_mysql/dl_connector_mysql/api/connector.py b/lib/dl_connector_mysql/dl_connector_mysql/api/connector.py index 42ebf2507..d56321baf 100644 --- a/lib/dl_connector_mysql/dl_connector_mysql/api/connector.py +++ b/lib/dl_connector_mysql/dl_connector_mysql/api/connector.py @@ -5,6 +5,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -18,6 +19,7 @@ from dl_connector_mysql.api.connection_info import MySQLConnectionInfoProvider from dl_connector_mysql.api.i18n.localizer import CONFIGS from dl_connector_mysql.core.connector import ( + MySQLCoreBackendDefinition, MySQLCoreConnectionDefinition, MySQLCoreConnector, MySQLSubselectCoreSourceDefinition, @@ -48,15 +50,9 @@ class MySQLApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = MySQLConnectionFormFactory -class MySQLApiConnector(ApiConnector): - core_connector_cls = MySQLCoreConnector - connection_definitions = (MySQLApiConnectionDefinition,) - source_definitions = ( - MySQLApiTableSourceDefinition, - MySQLApiSubselectSourceDefinition, - ) +class MySQLApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = MySQLCoreBackendDefinition formula_dialect_name = DIALECT_NAME_MYSQL - translation_configs = frozenset(CONFIGS) multi_query_mutation_factories = ApiConnector.multi_query_mutation_factories + ( MQMFactorySettingItem( query_proc_mode=QueryProcessingMode.native_wf, @@ -64,3 +60,13 @@ class MySQLApiConnector(ApiConnector): factory_cls=NoCompengMultiQueryMutatorFactory, ), ) + + +class MySQLApiConnector(ApiConnector): + backend_definition = MySQLApiBackendDefinition + connection_definitions = (MySQLApiConnectionDefinition,) + source_definitions = ( + MySQLApiTableSourceDefinition, + MySQLApiSubselectSourceDefinition, + ) + translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_mysql/dl_connector_mysql/core/connector.py b/lib/dl_connector_mysql/dl_connector_mysql/core/connector.py index 9c4a55848..a68718865 100644 --- a/lib/dl_connector_mysql/dl_connector_mysql/core/connector.py +++ b/lib/dl_connector_mysql/dl_connector_mysql/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, ) @@ -52,8 +53,13 @@ class MySQLSubselectCoreSourceDefinition(SQLSubselectCoreSourceDefinitionBase): source_cls = MySQLSubselectDataSource -class MySQLCoreConnector(CoreConnector): +class MySQLCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_MYSQL + compiler_cls = MySQLQueryCompiler + + +class MySQLCoreConnector(CoreConnector): + backend_definition = MySQLCoreBackendDefinition connection_definitions = (MySQLCoreConnectionDefinition,) source_definitions = ( MySQLTableCoreSourceDefinition, @@ -61,4 +67,3 @@ class MySQLCoreConnector(CoreConnector): ) rqe_adapter_classes = frozenset({MySQLAdapter, AsyncMySQLAdapter}) sa_types = SQLALCHEMY_MYSQL_TYPES - compiler_cls = MySQLQueryCompiler diff --git a/lib/dl_connector_oracle/dl_connector_oracle/api/connector.py b/lib/dl_connector_oracle/dl_connector_oracle/api/connector.py index 4934afc81..effe0f21c 100644 --- a/lib/dl_connector_oracle/dl_connector_oracle/api/connector.py +++ b/lib/dl_connector_oracle/dl_connector_oracle/api/connector.py @@ -5,6 +5,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -16,6 +17,7 @@ from dl_connector_oracle.api.dashsql import OracleDashSQLParamLiteralizer from dl_connector_oracle.api.i18n.localizer import CONFIGS from dl_connector_oracle.core.connector import ( + OracleCoreBackendDefinition, OracleCoreConnectionDefinition, OracleCoreConnector, OracleSubselectCoreSourceDefinition, @@ -43,13 +45,16 @@ class OracleApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = OracleConnectionFormFactory +class OracleApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = OracleCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_ORACLE + + class OracleApiConnector(ApiConnector): - core_connector_cls = OracleCoreConnector connection_definitions = (OracleApiConnectionDefinition,) source_definitions = ( OracleApiTableSourceDefinition, OracleApiSubselectSourceDefinition, ) - formula_dialect_name = DIALECT_NAME_ORACLE translation_configs = frozenset(CONFIGS) dashsql_literalizer_cls = OracleDashSQLParamLiteralizer diff --git a/lib/dl_connector_oracle/dl_connector_oracle/core/connector.py b/lib/dl_connector_oracle/dl_connector_oracle/core/connector.py index 3d55bad06..f5e995bd4 100644 --- a/lib/dl_connector_oracle/dl_connector_oracle/core/connector.py +++ b/lib/dl_connector_oracle/dl_connector_oracle/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, ) @@ -48,8 +49,13 @@ class OracleSubselectCoreSourceDefinition(SQLSubselectCoreSourceDefinitionBase): source_cls = OracleSubselectDataSource -class OracleCoreConnector(CoreConnector): +class OracleCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_ORACLE + compiler_cls = OracleQueryCompiler + + +class OracleCoreConnector(CoreConnector): + backend_definition = OracleCoreBackendDefinition connection_definitions = (OracleCoreConnectionDefinition,) source_definitions = ( OracleTableCoreSourceDefinition, @@ -57,4 +63,3 @@ class OracleCoreConnector(CoreConnector): ) rqe_adapter_classes = frozenset({OracleDefaultAdapter}) sa_types = SQLALCHEMY_ORACLE_TYPES - compiler_cls = OracleQueryCompiler diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql/api/connector.py b/lib/dl_connector_postgresql/dl_connector_postgresql/api/connector.py index 5c8439f9c..6dd472842 100644 --- a/lib/dl_connector_postgresql/dl_connector_postgresql/api/connector.py +++ b/lib/dl_connector_postgresql/dl_connector_postgresql/api/connector.py @@ -5,6 +5,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -18,6 +19,7 @@ from dl_connector_postgresql.api.connection_info import PostgreSQLConnectionInfoProvider from dl_connector_postgresql.api.i18n.localizer import CONFIGS from dl_connector_postgresql.core.postgresql.connector import ( + PostgreSQLCoreBackendDefinition, PostgreSQLCoreConnectionDefinition, PostgreSQLCoreConnector, PostgreSQLSubselectCoreSourceDefinition, @@ -48,16 +50,9 @@ class PostgreSQLApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = PostgreSQLConnectionFormFactory -class PostgreSQLApiConnector(ApiConnector): - core_connector_cls = PostgreSQLCoreConnector - connection_definitions = (PostgreSQLApiConnectionDefinition,) - source_definitions = ( - PostgreSQLApiTableSourceDefinition, - PostgreSQLApiSubselectSourceDefinition, - ) +class PostgreSQLApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = PostgreSQLCoreBackendDefinition formula_dialect_name = DIALECT_NAME_POSTGRESQL - translation_configs = frozenset(CONFIGS) - compeng_dialect = PostgreSQLDialect.COMPENG multi_query_mutation_factories = ApiConnector.multi_query_mutation_factories + ( MQMFactorySettingItem( query_proc_mode=QueryProcessingMode.native_wf, @@ -65,3 +60,14 @@ class PostgreSQLApiConnector(ApiConnector): factory_cls=NoCompengMultiQueryMutatorFactory, ), ) + + +class PostgreSQLApiConnector(ApiConnector): + backend_definition = PostgreSQLApiBackendDefinition + connection_definitions = (PostgreSQLApiConnectionDefinition,) + source_definitions = ( + PostgreSQLApiTableSourceDefinition, + PostgreSQLApiSubselectSourceDefinition, + ) + translation_configs = frozenset(CONFIGS) + compeng_dialect = PostgreSQLDialect.COMPENG diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql/connector.py b/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql/connector.py index fac3022c6..1ec0d71ad 100644 --- a/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql/connector.py +++ b/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, ) @@ -52,8 +53,13 @@ class PostgreSQLSubselectCoreSourceDefinition(SQLSubselectCoreSourceDefinitionBa source_cls = PostgreSQLSubselectDataSource -class PostgreSQLCoreConnector(CoreConnector): +class PostgreSQLCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_POSTGRES + compiler_cls = PostgreSQLQueryCompiler + + +class PostgreSQLCoreConnector(CoreConnector): + backend_definition = PostgreSQLCoreBackendDefinition connection_definitions = (PostgreSQLCoreConnectionDefinition,) source_definitions = ( PostgreSQLTableCoreSourceDefinition, @@ -61,4 +67,3 @@ class PostgreSQLCoreConnector(CoreConnector): ) rqe_adapter_classes = frozenset({PostgresAdapter, AsyncPostgresAdapter}) sa_types = SQLALCHEMY_POSTGRES_TYPES - compiler_cls = PostgreSQLQueryCompiler diff --git a/lib/dl_connector_promql/dl_connector_promql/api/connector.py b/lib/dl_connector_promql/dl_connector_promql/api/connector.py index 5436d5772..39b8b7e43 100644 --- a/lib/dl_connector_promql/dl_connector_promql/api/connector.py +++ b/lib/dl_connector_promql/dl_connector_promql/api/connector.py @@ -5,6 +5,7 @@ SimpleDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -15,6 +16,7 @@ from dl_connector_promql.api.connection_info import PromQLConnectionInfoProvider from dl_connector_promql.api.i18n.localizer import CONFIGS from dl_connector_promql.core.connector import ( + PromQLCoreBackendDefinition, PromQLCoreConnectionDefinition, PromQLCoreConnector, PromQLCoreSourceDefinition, @@ -34,8 +36,12 @@ class PromQLApiSourceDefinition(ApiSourceDefinition): template_api_schema_cls = SimpleDataSourceTemplateSchema +class PromQLApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = PromQLCoreBackendDefinition + + class PromQLApiConnector(ApiConnector): - core_connector_cls = PromQLCoreConnector + backend_definition = PromQLApiBackendDefinition connection_definitions = (PromQLApiConnectionDefinition,) source_definitions = (PromQLApiSourceDefinition,) translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_promql/dl_connector_promql/core/connector.py b/lib/dl_connector_promql/dl_connector_promql/core/connector.py index 30369090d..41cb23db9 100644 --- a/lib/dl_connector_promql/dl_connector_promql/core/connector.py +++ b/lib/dl_connector_promql/dl_connector_promql/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -39,8 +40,12 @@ class PromQLCoreSourceDefinition(CoreSourceDefinition): source_cls = PromQLDataSource -class PromQLCoreConnector(CoreConnector): +class PromQLCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_PROMQL + + +class PromQLCoreConnector(CoreConnector): + backend_definition = PromQLCoreBackendDefinition connection_definitions = (PromQLCoreConnectionDefinition,) source_definitions = (PromQLCoreSourceDefinition,) rqe_adapter_classes = frozenset({AsyncPromQLAdapter, PromQLAdapter}) diff --git a/lib/dl_connector_snowflake/dl_connector_snowflake/api/connector.py b/lib/dl_connector_snowflake/dl_connector_snowflake/api/connector.py index 6494b62df..bdd33a5f0 100644 --- a/lib/dl_connector_snowflake/dl_connector_snowflake/api/connector.py +++ b/lib/dl_connector_snowflake/dl_connector_snowflake/api/connector.py @@ -1,4 +1,5 @@ from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -13,6 +14,7 @@ from dl_connector_snowflake.api.connection_info import SnowflakeConnectionInfoProvider from dl_connector_snowflake.api.i18n.localizer import CONFIGS from dl_connector_snowflake.core.connector import ( + SnowflakeCoreBackendDefinition, SnowFlakeCoreConnectionDefinition, SnowFlakeCoreConnector, SnowFlakeCoreTableSourceDefinition, @@ -33,9 +35,13 @@ class SnowFlakeApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = SnowFlakeConnectionFormFactory +class SnowFlakeApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = SnowflakeCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_SNOWFLAKE + + class SnowFlakeApiConnector(ApiConnector): - core_connector_cls = SnowFlakeCoreConnector + backend_definition = SnowFlakeApiBackendDefinition connection_definitions = (SnowFlakeApiConnectionDefinition,) source_definitions = (SnowFlakeApiTableSourceDefinition,) - formula_dialect_name = DIALECT_NAME_SNOWFLAKE translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_snowflake/dl_connector_snowflake/core/connector.py b/lib/dl_connector_snowflake/dl_connector_snowflake/core/connector.py index 305373991..66d9f79a5 100644 --- a/lib/dl_connector_snowflake/dl_connector_snowflake/core/connector.py +++ b/lib/dl_connector_snowflake/dl_connector_snowflake/core/connector.py @@ -1,4 +1,5 @@ from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -55,8 +56,12 @@ class SnowFlakeCoreSubselectSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = SnowFlakeSubselectDataSourceSpecStorageSchema -class SnowFlakeCoreConnector(CoreConnector): +class SnowflakeCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_SNOWFLAKE + + +class SnowFlakeCoreConnector(CoreConnector): + backend_definition = SnowflakeCoreBackendDefinition connection_definitions = (SnowFlakeCoreConnectionDefinition,) source_definitions = ( SnowFlakeCoreTableSourceDefinition, diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py index dd945884e..ab6e4b148 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py +++ b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py @@ -5,6 +5,7 @@ SubselectDataSourceTemplateSchema, ) from dl_api_connector.connector import ( + ApiBackendDefinition, ApiConnectionDefinition, ApiConnector, ApiSourceDefinition, @@ -15,6 +16,7 @@ from dl_connector_ydb.api.ydb.connection_info import YDBConnectionInfoProvider from dl_connector_ydb.api.ydb.i18n.localizer import CONFIGS from dl_connector_ydb.core.ydb.connector import ( + YDBCoreBackendDefinition, YDBCoreConnectionDefinition, YDBCoreConnector, YDBCoreSourceDefinition, @@ -42,12 +44,16 @@ class YDBApiConnectionDefinition(ApiConnectionDefinition): form_factory_cls = YDBConnectionFormFactory +class YDBApiBackendDefinition(ApiBackendDefinition): + core_backend_definition = YDBCoreBackendDefinition + formula_dialect_name = DIALECT_NAME_YDB + + class YDBApiConnector(ApiConnector): - core_connector_cls = YDBCoreConnector + backend_definition = YDBApiBackendDefinition connection_definitions = (YDBApiConnectionDefinition,) source_definitions = ( YDBApiTableSourceDefinition, YDBApiSubselectSourceDefinition, ) - formula_dialect_name = DIALECT_NAME_YDB translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py index 12f1c080d..1aa347cd0 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py @@ -1,6 +1,7 @@ from ydb.sqlalchemy import register_dialect as yql_register_dialect from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -58,15 +59,19 @@ class YDBCoreSubselectSourceDefinition(CoreSourceDefinition): us_storage_schema_cls = SubselectDataSourceSpecStorageSchema -class YDBCoreConnector(CoreConnector): +class YDBCoreBackendDefinition(CoreBackendDefinition): backend_type = BACKEND_TYPE_YDB + compiler_cls = YQLQueryCompiler + + +class YDBCoreConnector(CoreConnector): + backend_definition = YDBCoreBackendDefinition connection_definitions = (YDBCoreConnectionDefinition,) source_definitions = ( YDBCoreSourceDefinition, YDBCoreSubselectSourceDefinition, ) rqe_adapter_classes = frozenset({YDBAdapter}) - compiler_cls = YQLQueryCompiler @classmethod def registration_hook(cls) -> None: diff --git a/lib/dl_core/dl_core/connectors/base/connector.py b/lib/dl_core/dl_core/connectors/base/connector.py index 4a3adc7b6..3feb18e27 100644 --- a/lib/dl_core/dl_core/connectors/base/connector.py +++ b/lib/dl_core/dl_core/connectors/base/connector.py @@ -67,12 +67,15 @@ class CoreConnectionDefinition(abc.ABC): custom_dashsql_key_names: frozenset[str] = frozenset() -class CoreConnector(abc.ABC): - # backend_type-bound properties - TODO: move to a separate entity +class CoreBackendDefinition(abc.ABC): backend_type: ClassVar[SourceBackendType] = SourceBackendType.NONE compiler_cls: ClassVar[Type[QueryCompiler]] = QueryCompiler query_cls: ClassVar[Type[Query]] = Query + + +class CoreConnector(abc.ABC): # others + backend_definition: Type[CoreBackendDefinition] connection_definitions: ClassVar[tuple[Type[CoreConnectionDefinition], ...]] = () source_definitions: ClassVar[tuple[Type[CoreSourceDefinition], ...]] = () sa_types: ClassVar[Optional[dict[GenericNativeType, Callable[[GenericNativeType], TypeEngine]]]] = None diff --git a/lib/dl_core/dl_core/connectors/base/registrator.py b/lib/dl_core/dl_core/connectors/base/registrator.py index df1d0db4c..12376a52d 100644 --- a/lib/dl_core/dl_core/connectors/base/registrator.py +++ b/lib/dl_core/dl_core/connectors/base/registrator.py @@ -3,10 +3,12 @@ Type, ) +from dl_constants.enums import SourceBackendType from dl_core.backend_types import register_connection_backend_type from dl_core.connection_executors.adapters.common_base import register_dialect_string from dl_core.connection_executors.remote_query_executor.commons import register_adapter_class from dl_core.connectors.base.connector import ( + CoreBackendDefinition, CoreConnectionDefinition, CoreConnector, CoreSourceDefinition, @@ -57,7 +59,7 @@ def register_source_definition(cls, source_def: Type[CoreSourceDefinition]) -> N def register_connection_definition( cls, conn_def: Type[CoreConnectionDefinition], - connector: Type[CoreConnector], + backend_type: SourceBackendType, ) -> None: register_connection_class( new_conn_cls=conn_def.connection_cls, @@ -65,7 +67,7 @@ def register_connection_definition( conn_type=conn_def.conn_type, lifecycle_manager_cls=conn_def.lifecycle_manager_cls, ) - register_connection_backend_type(conn_type=conn_def.conn_type, backend_type=connector.backend_type) + register_connection_backend_type(conn_type=conn_def.conn_type, backend_type=backend_type) register_connection_schema(conn_cls=conn_def.connection_cls, schema_cls=conn_def.us_storage_schema_cls) # type: ignore register_type_transformer_class(conn_type=conn_def.conn_type, tt_cls=conn_def.type_transformer_cls) # type: ignore if conn_def.sync_conn_executor_cls is not None: @@ -86,12 +88,23 @@ def register_connection_definition( register_data_source_migrator(conn_type=conn_def.conn_type, migrator_cls=conn_def.data_source_migrator_cls) register_custom_dash_sql_key_names(conn_type=conn_def.conn_type, key_names=conn_def.custom_dashsql_key_names) + @classmethod + def register_backend_definition(cls, backend_def: Type[CoreBackendDefinition]) -> None: + register_sa_query_cls(backend_type=backend_def.backend_type, query_cls=backend_def.query_cls) + register_sa_query_compiler_cls( + backend_type=backend_def.backend_type, + sa_query_compiler_cls=backend_def.compiler_cls, + ) + @classmethod def register_connector(cls, connector: Type[CoreConnector]) -> None: + backend_def = connector.backend_definition + cls.register_backend_definition(backend_def=backend_def) for source_def in connector.source_definitions: cls.register_source_definition(source_def=source_def) for conn_def in connector.connection_definitions: - cls.register_connection_definition(conn_def=conn_def, connector=connector) + cls.register_connection_definition(conn_def=conn_def, backend_type=backend_def.backend_type) + register_sa_types(connector.sa_types or {}) for conn_sec_settings in connector.conn_security: # type: ConnSecuritySettings dto_types = conn_sec_settings.dtos @@ -102,11 +115,6 @@ def register_connector(cls, connector: Type[CoreConnector]) -> None: for notification_cls in connector.notification_classes: register_notification()(notification_cls) # it is a parameterized decorator - # backend_type-dependent properties - backend_type = connector.backend_type - register_sa_query_cls(backend_type=backend_type, query_cls=connector.query_cls) - register_sa_query_compiler_cls(backend_type=backend_type, sa_query_compiler_cls=connector.compiler_cls) - connector.registration_hook() # for custom actions