Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Moved backend_type-related properties from connectors to separate objects #154

Merged
merged 1 commit into from
Dec 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions lib/dl_api_connector/dl_api_connector/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from dl_api_lib.query.registry import MQMFactorySettingItem
from dl_constants.enums import QueryProcessingMode
from dl_core.connectors.base.connector import (
CoreBackendDefinition,
CoreConnectionDefinition,
CoreConnector,
CoreSourceDefinition,
Expand Down Expand Up @@ -53,8 +54,9 @@ class ApiConnectionDefinition(abc.ABC):
form_factory_cls: ClassVar[Optional[Type[ConnectionFormFactory]]] = None


class ApiConnector(abc.ABC):
# backend_type-bound properties - TODO: move to a separate entity
class ApiBackendDefinition(abc.ABC):
core_backend_definition: Type[CoreBackendDefinition]

formula_dialect_name: ClassVar[DialectName] = DialectName.DUMMY
multi_query_mutation_factories: tuple[MQMFactorySettingItem, ...] = (
MQMFactorySettingItem(
Expand All @@ -70,8 +72,10 @@ class ApiConnector(abc.ABC):
is_compeng_executable: ClassVar[bool] = False
filter_formula_compiler_cls: ClassVar[Type[FilterFormulaCompiler]] = MainFilterFormulaCompiler
dashsql_literalizer_cls: ClassVar[Type[DashSQLParamLiteralizer]] = DefaultDashSQLParamLiteralizer
# others
core_connector_cls: ClassVar[Type[CoreConnector]]


class ApiConnector(abc.ABC):
backend_definition: Type[ApiBackendDefinition]
connection_definitions: ClassVar[Tuple[Type[ApiConnectionDefinition], ...]] = ()
source_definitions: ClassVar[Tuple[Type[ApiSourceDefinition], ...]] = ()
translation_configs: ClassVar[frozenset[TranslationConfig]] = frozenset()
Expand Down
28 changes: 18 additions & 10 deletions lib/dl_api_lib/dl_api_lib/connector_registrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
register_source_template_api_schema,
)
from dl_api_connector.connector import (
ApiBackendDefinition,
ApiConnectionDefinition,
ApiConnector,
ApiSourceDefinition,
Expand Down Expand Up @@ -47,29 +48,36 @@ def register_connection_definition(cls, conn_def: Type[ApiConnectionDefinition])
register_connection_form_factory_cls(conn_type=conn_type, factory_cls=conn_def.form_factory_cls)

@classmethod
def register_connector(cls, connector: Type[ApiConnector]) -> None:
# backend_type-related stuff - TODO: Move to a separate entity
backend_type = connector.core_connector_cls.backend_type
register_dialect_name(backend_type=backend_type, dialect_name=connector.formula_dialect_name)
for mqm_setting_item in connector.multi_query_mutation_factories:
def register_backend_definition(cls, backend_def: Type[ApiBackendDefinition]) -> None:
backend_type = backend_def.core_backend_definition.backend_type
register_dialect_name(backend_type=backend_type, dialect_name=backend_def.formula_dialect_name)
for mqm_setting_item in backend_def.multi_query_mutation_factories:
register_multi_query_mutator_factory_cls(
query_proc_mode=mqm_setting_item.query_proc_mode,
backend_type=backend_type,
dialects=mqm_setting_item.dialects,
factory_cls=mqm_setting_item.factory_cls,
)
register_forkable_dialect_name(dialect_name=connector.formula_dialect_name, is_forkable=connector.is_forkable)
register_is_compeng_executable(backend_type=backend_type, is_compeng_executable=connector.is_compeng_executable)
register_forkable_dialect_name(
dialect_name=backend_def.formula_dialect_name,
is_forkable=backend_def.is_forkable,
)
register_is_compeng_executable(
backend_type=backend_type,
is_compeng_executable=backend_def.is_compeng_executable,
)
register_filter_formula_compiler_cls(
backend_type=backend_type,
filter_compiler_cls=connector.filter_formula_compiler_cls,
filter_compiler_cls=backend_def.filter_formula_compiler_cls,
)
register_dash_sql_param_literalizer_cls(
backend_type=backend_type,
literalizer_cls=connector.dashsql_literalizer_cls,
literalizer_cls=backend_def.dashsql_literalizer_cls,
)

# everything else
@classmethod
def register_connector(cls, connector: Type[ApiConnector]) -> None:
cls.register_backend_definition(backend_def=connector.backend_definition)
for source_def in connector.source_definitions:
cls.register_source_definition(source_def=source_def)
for conn_def in connector.connection_definitions:
Expand Down
10 changes: 8 additions & 2 deletions lib/dl_connector_bigquery/dl_connector_bigquery/api/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
SubselectDataSourceTemplateSchema,
)
from dl_api_connector.connector import (
ApiBackendDefinition,
ApiConnectionDefinition,
ApiConnector,
ApiSourceDefinition,
Expand All @@ -17,6 +18,7 @@
from dl_connector_bigquery.api.connection_info import BigQueryConnectionInfoProvider
from dl_connector_bigquery.api.i18n.localizer import CONFIGS
from dl_connector_bigquery.core.connector import (
BigQueryCoreBackendDefinition,
BigQueryCoreConnectionDefinition,
BigQueryCoreConnector,
BigQueryCoreSubselectSourceDefinition,
Expand Down Expand Up @@ -44,12 +46,16 @@ class BigQueryApiConnectionDefinition(ApiConnectionDefinition):
form_factory_cls = BigQueryConnectionFormFactory


class BigQueryApiBackendDefinition(ApiBackendDefinition):
core_backend_definition = BigQueryCoreBackendDefinition
formula_dialect_name = DIALECT_NAME_BIGQUERY


class BigQueryApiConnector(ApiConnector):
core_connector_cls = BigQueryCoreConnector
backend_definition = BigQueryApiBackendDefinition
source_definitions = (
BigQueryApiTableSourceDefinition,
BigQueryApiSubselectSourceDefinition,
)
connection_definitions = (BigQueryApiConnectionDefinition,)
formula_dialect_name = DIALECT_NAME_BIGQUERY
translation_configs = frozenset(CONFIGS)
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from dl_core.connectors.base.connector import (
CoreBackendDefinition,
CoreConnectionDefinition,
CoreConnector,
CoreSourceDefinition,
Expand Down Expand Up @@ -55,9 +56,13 @@ class BigQueryCoreSubselectSourceDefinition(CoreSourceDefinition):
us_storage_schema_cls = BigQuerySubselectDataSourceSpecStorageSchema


class BigQueryCoreConnector(CoreConnector):
class BigQueryCoreBackendDefinition(CoreBackendDefinition):
backend_type = BACKEND_TYPE_BIGQUERY
compiler_cls = BigQueryQueryCompiler


class BigQueryCoreConnector(CoreConnector):
backend_definition = BigQueryCoreBackendDefinition
connection_definitions = (BigQueryCoreConnectionDefinition,)
source_definitions = (
BigQueryCoreTableSourceDefinition,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
SQLDataSourceTemplateSchema,
)
from dl_api_connector.connector import (
ApiBackendDefinition,
ApiConnectionDefinition,
ApiConnector,
ApiSourceDefinition,
Expand All @@ -20,6 +21,7 @@
from dl_connector_bitrix_gds.api.i18n.localizer import CONFIGS
from dl_connector_bitrix_gds.api.multi_query import BitrixGDSMultiQueryMutatorFactory
from dl_connector_bitrix_gds.core.connector import (
BitrixGDSCoreBackendDefinition,
BitrixGDSCoreConnectionDefinition,
BitrixGDSCoreConnector,
BitrixGDSCoreSourceDefinition,
Expand All @@ -40,8 +42,8 @@ class BitrixGDSApiConnectionDefinition(ApiConnectionDefinition):
info_provider_cls = BitrixGDSConnectionInfoProvider


class BitrixGDSApiConnector(ApiConnector):
core_connector_cls = BitrixGDSCoreConnector
class BitrixGDSApiBackendDefinition(ApiBackendDefinition):
core_backend_definition = BitrixGDSCoreBackendDefinition
formula_dialect_name = DIALECT_NAME_BITRIX
multi_query_mutation_factories = (
MQMFactorySettingItem(
Expand All @@ -53,9 +55,13 @@ class BitrixGDSApiConnector(ApiConnector):
factory_cls=NoCompengMultiQueryMutatorFactory,
),
)
connection_definitions = (BitrixGDSApiConnectionDefinition,)
source_definitions = (BitrixGDSApiSourceDefinition,)
is_forkable = False
is_compeng_executable = True
filter_formula_compiler_cls = BitrixGDSFilterFormulaCompiler


class BitrixGDSApiConnector(ApiConnector):
backend_definition = BitrixGDSApiBackendDefinition
connection_definitions = (BitrixGDSApiConnectionDefinition,)
source_definitions = (BitrixGDSApiSourceDefinition,)
translation_configs = frozenset(CONFIGS)
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from dl_core.connectors.base.connector import (
CoreBackendDefinition,
CoreConnectionDefinition,
CoreConnector,
CoreSourceDefinition,
Expand Down Expand Up @@ -36,8 +37,12 @@ class BitrixGDSCoreSourceDefinition(CoreSourceDefinition):
us_storage_schema_cls = SQLDataSourceSpecStorageSchema


class BitrixGDSCoreConnector(CoreConnector):
class BitrixGDSCoreBackendDefinition(CoreBackendDefinition):
backend_type = BACKEND_TYPE_BITRIX_GDS


class BitrixGDSCoreConnector(CoreConnector):
backend_definition = BitrixGDSCoreBackendDefinition
connection_definitions = (BitrixGDSCoreConnectionDefinition,)
source_definitions = (BitrixGDSCoreSourceDefinition,)
rqe_adapter_classes = frozenset({BitrixGDSDefaultAdapter})
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
BaseFileS3CoreConnector,
BaseFileS3TableCoreSourceDefinition,
)
from dl_connector_clickhouse.formula.constants import DIALECT_NAME_CLICKHOUSE
from dl_connector_clickhouse.api.connector import ClickHouseApiBackendDefinition


class BaseFileS3TableApiSourceDefinition(ApiSourceDefinition):
Expand All @@ -32,8 +32,7 @@ class BaseFileS3ApiConnectionDefinition(ApiConnectionDefinition):


class BaseFileS3ApiConnector(ApiConnector):
core_connector_cls = BaseFileS3CoreConnector
formula_dialect_name = DIALECT_NAME_CLICKHOUSE
backend_definition = ClickHouseApiBackendDefinition
connection_definitions = (BaseFileS3ApiConnectionDefinition,)
source_definitions = (BaseFileS3TableApiSourceDefinition,)
translation_configs = frozenset(CONFIGS)
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
NonUserInputConnectionSafetyChecker,
)
from dl_core.connectors.base.connector import (
CoreBackendDefinition,
CoreConnectionDefinition,
CoreConnector,
CoreSourceDefinition,
Expand All @@ -25,10 +26,14 @@ class BaseFileS3TableCoreSourceDefinition(CoreSourceDefinition):
pass


class BaseFileS3CoreConnector(CoreConnector):
class CHS3CoreBackendDefinition(CoreBackendDefinition): # TODO: Why not just use CH??
backend_type = BACKEND_TYPE_CHS3
query_cls = CHQuery
compiler_cls = ClickHouseQueryCompiler


class BaseFileS3CoreConnector(CoreConnector):
backend_definition = CHS3CoreBackendDefinition
conn_security = frozenset(
{ConnSecuritySettings(NonUserInputConnectionSafetyChecker, frozenset({BaseFileS3ConnDTO}))}
)
query_cls = CHQuery
compiler_cls = ClickHouseQueryCompiler
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,5 @@ class GSheetsFileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition):


class GSheetsFileS3ApiConnector(BaseFileS3ApiConnector):
core_connector_cls = GSheetsFileS3CoreConnector
connection_definitions = (GSheetsFileS3ApiConnectionDefinition,)
source_definitions = (GSheetsFileS3TableApiSourceDefinition,)
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,5 @@ class YaDocsFileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition):


class YaDocsFileS3ApiConnector(BaseFileS3ApiConnector):
core_connector_cls = YaDocsFileS3CoreConnector
connection_definitions = (YaDocsFileS3ApiConnectionDefinition,)
source_definitions = (YaDocsFileS3TableApiSourceDefinition,)
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,5 @@ class FileS3ApiConnectionDefinition(BaseFileS3ApiConnectionDefinition):


class FileS3ApiConnector(BaseFileS3ApiConnector):
core_connector_cls = FileS3CoreConnector
connection_definitions = (FileS3ApiConnectionDefinition,)
source_definitions = (FileS3TableApiSourceDefinition,)
10 changes: 7 additions & 3 deletions lib/dl_connector_chyt/dl_connector_chyt/api/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@
from dl_connector_chyt.api.connection_info import CHYTConnectionInfoProvider
from dl_connector_chyt.api.i18n.localizer import CONFIGS
from dl_connector_chyt.core.connector import (
CHYTCoreBackendDefinition,
CHYTCoreConnectionDefinition,
CHYTCoreConnector,
CHYTTableCoreSourceDefinition,
CHYTTableListCoreSourceDefinition,
CHYTTableRangeCoreSourceDefinition,
CHYTTableSubselectCoreSourceDefinition,
)
from dl_connector_clickhouse.formula.constants import DIALECT_NAME_CLICKHOUSE
from dl_connector_clickhouse.api.connector import ClickHouseApiBackendDefinition


class CHYTApiConnectionDefinition(ApiConnectionDefinition):
Expand Down Expand Up @@ -64,9 +65,12 @@ class CHYTSubselectApiSourceDefinition(ApiSourceDefinition):
template_api_schema_cls = SubselectDataSourceTemplateSchema


class CHYTApiBackendDefinition(ClickHouseApiBackendDefinition):
core_backend_definition = CHYTCoreBackendDefinition


class CHYTApiConnector(ApiConnector):
core_connector_cls = CHYTCoreConnector
formula_dialect_name = DIALECT_NAME_CLICKHOUSE
backend_definition = CHYTApiBackendDefinition
connection_definitions = (CHYTApiConnectionDefinition,)
source_definitions = (
CHYTTableApiSourceDefinition,
Expand Down
15 changes: 12 additions & 3 deletions lib/dl_connector_chyt/dl_connector_chyt/core/connector.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from clickhouse_sqlalchemy.orm.query import Query as CHQuery

from dl_core.connectors.base.connector import (
CoreBackendDefinition,
CoreConnectionDefinition,
CoreConnector,
CoreSourceDefinition,
Expand Down Expand Up @@ -86,8 +87,18 @@ class CHYTTableSubselectCoreSourceDefinition(CoreSourceDefinition):
us_storage_schema_cls = CHYTSubselectDataSourceSpecStorageSchema


class CHYTCoreConnector(CoreConnector):
class CHYTTableSubselectCoreSourceDefinition(CoreSourceDefinition):
source_type = SOURCE_TYPE_CHYT_YTSAURUS_SUBSELECT


class CHYTCoreBackendDefinition(CoreBackendDefinition):
backend_type = BACKEND_TYPE_CHYT
query_cls = CHQuery
compiler_cls = ClickHouseQueryCompiler


class CHYTCoreConnector(CoreConnector):
backend_definition = CHYTCoreBackendDefinition
connection_definitions = (CHYTCoreConnectionDefinition,)
source_definitions = (
CHYTTableCoreSourceDefinition,
Expand All @@ -101,5 +112,3 @@ class CHYTCoreConnector(CoreConnector):
AsyncCHYTAdapter,
}
)
query_cls = CHQuery
compiler_cls = ClickHouseQueryCompiler
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
SubselectDataSourceTemplateSchema,
)
from dl_api_connector.connector import (
ApiBackendDefinition,
ApiConnectionDefinition,
ApiConnector,
ApiSourceDefinition,
Expand All @@ -23,6 +24,7 @@
ClickHouseSubselectCoreSourceDefinition,
ClickHouseTableCoreSourceDefinition,
)
from dl_connector_clickhouse.core.clickhouse_base.connector import ClickHouseCoreBackendDefinition
from dl_connector_clickhouse.formula.constants import (
DIALECT_NAME_CLICKHOUSE,
ClickHouseDialect,
Expand All @@ -48,19 +50,23 @@ class ClickHouseApiConnectionDefinition(ApiConnectionDefinition):
form_factory_cls = ClickHouseConnectionFormFactory


class ClickHouseApiConnector(ApiConnector):
core_connector_cls = ClickHouseCoreConnector
connection_definitions = (ClickHouseApiConnectionDefinition,)
source_definitions = (
ClickHouseApiTableSourceDefinition,
ClickHouseApiSubselectSourceDefinition,
)
class ClickHouseApiBackendDefinition(ApiBackendDefinition):
core_backend_definition = ClickHouseCoreBackendDefinition
formula_dialect_name = DIALECT_NAME_CLICKHOUSE
translation_configs = frozenset(CONFIGS)
multi_query_mutation_factories = ApiConnector.multi_query_mutation_factories + (
multi_query_mutation_factories = ApiBackendDefinition.multi_query_mutation_factories + (
MQMFactorySettingItem(
query_proc_mode=QueryProcessingMode.native_wf,
dialects=ClickHouseDialect.and_above(ClickHouseDialect.CLICKHOUSE_22_10).to_list(),
factory_cls=NoCompengMultiQueryMutatorFactory,
),
)


class ClickHouseApiConnector(ApiConnector):
backend_definition = ClickHouseApiBackendDefinition
connection_definitions = (ClickHouseApiConnectionDefinition,)
source_definitions = (
ClickHouseApiTableSourceDefinition,
ClickHouseApiSubselectSourceDefinition,
)
translation_configs = frozenset(CONFIGS)
Loading
Loading