Filter connectors whitelist before load call to avoid unnecessary connectors imports

lib/dl_api_lib/dl_api_lib/app/data_api/resources/ping.py

@@ -52,7 +52,8 @@ class PingReadyView(BaseView):
     async def is_pg_ready(self) -> bool:
         compeng: CompEngPgService = CompEngPgService.get_app_instance(self.request.app)
         processor = compeng.get_data_processor(
-            service_registry=self.dl_request.services_registry, reporting_enabled=False,
+            service_registry=self.dl_request.services_registry,
+            reporting_enabled=False,
         )
 
         try:

lib/dl_api_lib/dl_api_lib/app_connectors.py

@@ -25,8 +25,6 @@ class ApiConnectorEntrypointManager(EntrypointClassManager[ApiConnector]):
 
 
 def register_all_connectors(connector_ep_names: Optional[Collection[str]] = None) -> None:
-    connectors = ApiConnectorEntrypointManager().get_all_ep_classes()
+    connectors = ApiConnectorEntrypointManager().get_all_ep_classes(connector_ep_names)
     for ep_name, connector_cls in sorted(connectors.items()):
-        if connector_ep_names is not None and ep_name not in connector_ep_names:
-            continue
         CONN_REG_BI_API.register_connector(connector_cls)

lib/dl_api_lib_testing/dl_api_lib_testing/connector/complex_queries.py

@@ -30,12 +30,14 @@
 from dl_core_testing.testcases.service_base import DbServiceFixtureTextClass
 from dl_testing.regulated_test import (
     Feature,
-    for_features,
     RegulatedTestCase,
+    for_features,
 )
 
 
-class DefaultBasicExtAggregationTestSuite(RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass):
+class DefaultBasicExtAggregationTestSuite(
+    RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass
+):
     def test_lod_fixed_single_dim_in_two_dim_query(self, control_api, data_api, saved_dataset):
         ds = add_formulas_to_dataset(
             api_v1=control_api,
@@ -264,7 +266,9 @@ def get_data(order_by: list[ResultField]) -> list:
         assert ordered_data_rows == data_rows
 
 
-class DefaultBasicLookupFunctionTestSuite(RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass):
+class DefaultBasicLookupFunctionTestSuite(
+    RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass
+):
     def test_ago_any_db(self, request, saved_connection_id, control_api, data_api, db):
         db_table = make_table(db=db)
         request.addfinalizer(functools.partial(db.drop_table, db_table.table))
@@ -460,7 +464,9 @@ def test_month_ago_for_shorter_month(self, request, db, saved_connection_id, con
         assert len(data_rows) == 1
 
 
-class DefaultBasicWindowFunctionTestSuite(RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass):
+class DefaultBasicWindowFunctionTestSuite(
+    RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass
+):
     feature_window_functions = Feature("window_functions")
 
     @for_features(feature_window_functions)

lib/dl_core/dl_core/connectors/registry.py

@@ -1,4 +1,8 @@
-from typing import Type
+from typing import (
+    Collection,
+    Optional,
+    Type,
+)
 
 import attr
 
@@ -15,9 +19,9 @@ class CoreConnectorEntrypointManager(EntrypointClassManager[CoreConnector]):
     entrypoint_group_name = attr.ib(init=False, default=_CONNECTOR_EP_GROUP)
 
 
-def get_all_connectors() -> dict[str, Type[CoreConnector]]:
+def get_all_connectors(ep_filter: Optional[Collection[str]] = None) -> dict[str, Type[CoreConnector]]:
     ep_mgr = CoreConnectorEntrypointManager()
-    return ep_mgr.get_all_ep_classes()
+    return ep_mgr.get_all_ep_classes(ep_filter)
 
 
 def get_connector_cls(name: str) -> Type[CoreConnector]:

lib/dl_core/dl_core/core_connectors.py

@@ -18,7 +18,5 @@ def load_all_connectors() -> None:
 
 
 def register_all_connectors(connector_ep_names: Optional[Collection[str]] = None) -> None:
-    for ep_name, connector_cls in sorted(get_all_connectors().items()):
-        if connector_ep_names is not None and ep_name not in connector_ep_names:
-            continue
+    for ep_name, connector_cls in sorted(get_all_connectors(connector_ep_names).items()):
         _register_connector(connector_cls)

lib/dl_formula/dl_formula/formula_connectors.py

@@ -5,26 +5,31 @@
     Type,
 )
 
+import attr
+
 import dl_formula as package
 from dl_formula.connectors.base.connector import FormulaConnector
 from dl_formula.connectors.registration import CONN_REG_FORMULA
+from dl_utils.entrypoints import EntrypointClassManager
 
 
 _CONNECTOR_EP_GROUP = f"{package.__name__}.connectors"
 
 
-def _get_all_ep_connectors() -> dict[str, Type[FormulaConnector]]:
-    entrypoints = list(metadata.entry_points().select(group=_CONNECTOR_EP_GROUP))  # type: ignore
-    return {ep.name: ep.load() for ep in entrypoints}
+@attr.s
+class FormulaConnectorEntrypointManager(EntrypointClassManager[FormulaConnector]):
+    entrypoint_group_name = attr.ib(init=False, default=_CONNECTOR_EP_GROUP)
+
+
+def _get_all_ep_connectors(ep_filter: Optional[Collection[str]] = None) -> dict[str, Type[FormulaConnector]]:
+    ep_mgr = FormulaConnectorEntrypointManager()
+    return ep_mgr.get_all_ep_classes(ep_filter)
 
 
 def load_all_connectors() -> None:
     _get_all_ep_connectors()
 
 
 def register_all_connectors(connector_ep_names: Optional[Collection[str]] = None) -> None:
-    connectors = _get_all_ep_connectors()
-    for ep_name, connector_cls in sorted(connectors.items()):
-        if connector_ep_names is not None and ep_name not in connector_ep_names:
-            continue
+    for ep_name, connector_cls in sorted(_get_all_ep_connectors(connector_ep_names).items()):
         CONN_REG_FORMULA.register_connector(connector_cls)

lib/dl_formula/pyproject.toml

@@ -18,6 +18,7 @@ jinja2 = ">=3.1.2"
 python = ">=3.10, <3.12"
 pytz = ">=2022.7.1"
 sqlalchemy = ">=1.4.46, <2.0"
+datalens-utils = {path = "../dl_utils"}
 dynamic-enum = {path = "../dynamic_enum"}
 
 [tool.poetry.group]

lib/dl_utils/dl_utils/entrypoints.py

@@ -1,7 +1,9 @@
 import abc
 from importlib import metadata
 from typing import (
+    Collection,
     Generic,
+    Optional,
     Type,
     TypeVar,
 )
@@ -20,8 +22,10 @@ class EntrypointClassManager(abc.ABC, Generic[_EP_CLS_TV]):
 
     entrypoint_group_name: str = attr.ib(kw_only=True)
 
-    def get_all_ep_classes(self) -> dict[str, Type[_EP_CLS_TV]]:
+    def get_all_ep_classes(self, ep_filter: Optional[Collection[str]] = None) -> dict[str, Type[_EP_CLS_TV]]:
         entrypoints = list(metadata.entry_points().select(group=self.entrypoint_group_name))  # type: ignore
+        if ep_filter is not None:
+            entrypoints = [ep for ep in entrypoints if ep.name in ep_filter]
         return {ep.name: ep.load() for ep in entrypoints}
 
     def get_ep_class(self, name: str) -> Type[_EP_CLS_TV]:

lib/dl_utils/dl_utils/utils.py

@@ -14,7 +14,6 @@
     Generator,
     Iterable,
     List,
-    NoReturn,
     Optional,
     Tuple,
     Type,

terrarium/bi_ci/bi_ci/execute_mypy_multi.py

@@ -3,9 +3,7 @@
 from pathlib import Path
 import subprocess
 import sys
-from typing import (
-    Iterable,
-)
+from typing import Iterable
 
 import clize
 import tomlkit