From 287595afddc69db60685e7c0ed6cbdac81b9f4d7 Mon Sep 17 00:00:00 2001 From: Andrey Snytin Date: Wed, 8 Nov 2023 01:45:49 +0100 Subject: [PATCH 1/3] Avoid unnecessary connectors loading --- lib/dl_api_lib/dl_api_lib/app_connectors.py | 4 +--- lib/dl_core/dl_core/connectors/registry.py | 10 +++++++--- lib/dl_core/dl_core/core_connectors.py | 4 +--- lib/dl_utils/dl_utils/entrypoints.py | 6 +++++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/dl_api_lib/dl_api_lib/app_connectors.py b/lib/dl_api_lib/dl_api_lib/app_connectors.py index 70a5badc3..7089a28df 100644 --- a/lib/dl_api_lib/dl_api_lib/app_connectors.py +++ b/lib/dl_api_lib/dl_api_lib/app_connectors.py @@ -25,8 +25,6 @@ class ApiConnectorEntrypointManager(EntrypointClassManager[ApiConnector]): def register_all_connectors(connector_ep_names: Optional[Collection[str]] = None) -> None: - connectors = ApiConnectorEntrypointManager().get_all_ep_classes() + connectors = ApiConnectorEntrypointManager().get_all_ep_classes(connector_ep_names) for ep_name, connector_cls in sorted(connectors.items()): - if connector_ep_names is not None and ep_name not in connector_ep_names: - continue CONN_REG_BI_API.register_connector(connector_cls) diff --git a/lib/dl_core/dl_core/connectors/registry.py b/lib/dl_core/dl_core/connectors/registry.py index 4be619f71..17a88d208 100644 --- a/lib/dl_core/dl_core/connectors/registry.py +++ b/lib/dl_core/dl_core/connectors/registry.py @@ -1,4 +1,8 @@ -from typing import Type +from typing import ( + Collection, + Optional, + Type, +) import attr @@ -15,9 +19,9 @@ class CoreConnectorEntrypointManager(EntrypointClassManager[CoreConnector]): entrypoint_group_name = attr.ib(init=False, default=_CONNECTOR_EP_GROUP) -def get_all_connectors() -> dict[str, Type[CoreConnector]]: +def get_all_connectors(ep_filter: Optional[Collection[str]] = None) -> dict[str, Type[CoreConnector]]: ep_mgr = CoreConnectorEntrypointManager() - return ep_mgr.get_all_ep_classes() + return ep_mgr.get_all_ep_classes(ep_filter) def get_connector_cls(name: str) -> Type[CoreConnector]: diff --git a/lib/dl_core/dl_core/core_connectors.py b/lib/dl_core/dl_core/core_connectors.py index 7e61bfe0a..cfea100ea 100644 --- a/lib/dl_core/dl_core/core_connectors.py +++ b/lib/dl_core/dl_core/core_connectors.py @@ -18,7 +18,5 @@ def load_all_connectors() -> None: def register_all_connectors(connector_ep_names: Optional[Collection[str]] = None) -> None: - for ep_name, connector_cls in sorted(get_all_connectors().items()): - if connector_ep_names is not None and ep_name not in connector_ep_names: - continue + for ep_name, connector_cls in sorted(get_all_connectors(connector_ep_names).items()): _register_connector(connector_cls) diff --git a/lib/dl_utils/dl_utils/entrypoints.py b/lib/dl_utils/dl_utils/entrypoints.py index ba3d436dc..36a50d70c 100644 --- a/lib/dl_utils/dl_utils/entrypoints.py +++ b/lib/dl_utils/dl_utils/entrypoints.py @@ -1,7 +1,9 @@ import abc from importlib import metadata from typing import ( + Collection, Generic, + Optional, Type, TypeVar, ) @@ -20,8 +22,10 @@ class EntrypointClassManager(abc.ABC, Generic[_EP_CLS_TV]): entrypoint_group_name: str = attr.ib(kw_only=True) - def get_all_ep_classes(self) -> dict[str, Type[_EP_CLS_TV]]: + def get_all_ep_classes(self, ep_filter: Optional[Collection[str]] = None) -> dict[str, Type[_EP_CLS_TV]]: entrypoints = list(metadata.entry_points().select(group=self.entrypoint_group_name)) # type: ignore + if ep_filter is not None: + entrypoints = [ep for ep in entrypoints if ep.name in ep_filter] return {ep.name: ep.load() for ep in entrypoints} def get_ep_class(self, name: str) -> Type[_EP_CLS_TV]: From 20820cb01078a04290babfb16d3444bb1231501a Mon Sep 17 00:00:00 2001 From: Andrey Snytin Date: Wed, 8 Nov 2023 01:54:14 +0100 Subject: [PATCH 2/3] cq --- .../dl_api_lib/app/data_api/resources/ping.py | 3 ++- .../connector/complex_queries.py | 14 ++++++++++---- terrarium/bi_ci/bi_ci/execute_mypy_multi.py | 4 +--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/lib/dl_api_lib/dl_api_lib/app/data_api/resources/ping.py b/lib/dl_api_lib/dl_api_lib/app/data_api/resources/ping.py index a5d1c85bb..d59d94eee 100644 --- a/lib/dl_api_lib/dl_api_lib/app/data_api/resources/ping.py +++ b/lib/dl_api_lib/dl_api_lib/app/data_api/resources/ping.py @@ -52,7 +52,8 @@ class PingReadyView(BaseView): async def is_pg_ready(self) -> bool: compeng: CompEngPgService = CompEngPgService.get_app_instance(self.request.app) processor = compeng.get_data_processor( - service_registry=self.dl_request.services_registry, reporting_enabled=False, + service_registry=self.dl_request.services_registry, + reporting_enabled=False, ) try: diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/connector/complex_queries.py b/lib/dl_api_lib_testing/dl_api_lib_testing/connector/complex_queries.py index c0eb886d9..2154367d0 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/connector/complex_queries.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/connector/complex_queries.py @@ -30,12 +30,14 @@ from dl_core_testing.testcases.service_base import DbServiceFixtureTextClass from dl_testing.regulated_test import ( Feature, - for_features, RegulatedTestCase, + for_features, ) -class DefaultBasicExtAggregationTestSuite(RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass): +class DefaultBasicExtAggregationTestSuite( + RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass +): def test_lod_fixed_single_dim_in_two_dim_query(self, control_api, data_api, saved_dataset): ds = add_formulas_to_dataset( api_v1=control_api, @@ -264,7 +266,9 @@ def get_data(order_by: list[ResultField]) -> list: assert ordered_data_rows == data_rows -class DefaultBasicLookupFunctionTestSuite(RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass): +class DefaultBasicLookupFunctionTestSuite( + RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass +): def test_ago_any_db(self, request, saved_connection_id, control_api, data_api, db): db_table = make_table(db=db) request.addfinalizer(functools.partial(db.drop_table, db_table.table)) @@ -460,7 +464,9 @@ def test_month_ago_for_shorter_month(self, request, db, saved_connection_id, con assert len(data_rows) == 1 -class DefaultBasicWindowFunctionTestSuite(RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass): +class DefaultBasicWindowFunctionTestSuite( + RegulatedTestCase, DataApiTestBase, DatasetTestBase, DbServiceFixtureTextClass +): feature_window_functions = Feature("window_functions") @for_features(feature_window_functions) diff --git a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py index 53db415c8..1284fd710 100644 --- a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py +++ b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py @@ -3,9 +3,7 @@ from pathlib import Path import subprocess import sys -from typing import ( - Iterable, -) +from typing import Iterable import clize import tomlkit From 4a3a2c9c3eb6ed6e3e73b648834b58b00538d2c8 Mon Sep 17 00:00:00 2001 From: Andrey Snytin Date: Wed, 8 Nov 2023 14:05:13 +0100 Subject: [PATCH 3/3] dl_formula --- .../dl_formula/connector_registration.py | 0 .../dl_formula/formula_connectors.py | 19 ++++++++++++------- lib/dl_formula/pyproject.toml | 1 + lib/dl_utils/dl_utils/utils.py | 1 - metapkg/poetry.lock | 11 +---------- 5 files changed, 14 insertions(+), 18 deletions(-) delete mode 100644 lib/dl_formula/dl_formula/connector_registration.py diff --git a/lib/dl_formula/dl_formula/connector_registration.py b/lib/dl_formula/dl_formula/connector_registration.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/dl_formula/dl_formula/formula_connectors.py b/lib/dl_formula/dl_formula/formula_connectors.py index 296fe0176..fb83986e0 100644 --- a/lib/dl_formula/dl_formula/formula_connectors.py +++ b/lib/dl_formula/dl_formula/formula_connectors.py @@ -5,17 +5,25 @@ Type, ) +import attr + import dl_formula as package from dl_formula.connectors.base.connector import FormulaConnector from dl_formula.connectors.registration import CONN_REG_FORMULA +from dl_utils.entrypoints import EntrypointClassManager _CONNECTOR_EP_GROUP = f"{package.__name__}.connectors" -def _get_all_ep_connectors() -> dict[str, Type[FormulaConnector]]: - entrypoints = list(metadata.entry_points().select(group=_CONNECTOR_EP_GROUP)) # type: ignore - return {ep.name: ep.load() for ep in entrypoints} +@attr.s +class FormulaConnectorEntrypointManager(EntrypointClassManager[FormulaConnector]): + entrypoint_group_name = attr.ib(init=False, default=_CONNECTOR_EP_GROUP) + + +def _get_all_ep_connectors(ep_filter: Optional[Collection[str]] = None) -> dict[str, Type[FormulaConnector]]: + ep_mgr = FormulaConnectorEntrypointManager() + return ep_mgr.get_all_ep_classes(ep_filter) def load_all_connectors() -> None: @@ -23,8 +31,5 @@ def load_all_connectors() -> None: def register_all_connectors(connector_ep_names: Optional[Collection[str]] = None) -> None: - connectors = _get_all_ep_connectors() - for ep_name, connector_cls in sorted(connectors.items()): - if connector_ep_names is not None and ep_name not in connector_ep_names: - continue + for ep_name, connector_cls in sorted(_get_all_ep_connectors(connector_ep_names).items()): CONN_REG_FORMULA.register_connector(connector_cls) diff --git a/lib/dl_formula/pyproject.toml b/lib/dl_formula/pyproject.toml index 89cd01254..bd473659d 100644 --- a/lib/dl_formula/pyproject.toml +++ b/lib/dl_formula/pyproject.toml @@ -18,6 +18,7 @@ jinja2 = ">=3.1.2" python = ">=3.10, <3.12" pytz = ">=2022.7.1" sqlalchemy = ">=1.4.46, <2.0" +datalens-utils = {path = "../dl_utils"} dynamic-enum = {path = "../dynamic_enum"} [tool.poetry.group] diff --git a/lib/dl_utils/dl_utils/utils.py b/lib/dl_utils/dl_utils/utils.py index 3a360387e..f54affdf3 100644 --- a/lib/dl_utils/dl_utils/utils.py +++ b/lib/dl_utils/dl_utils/utils.py @@ -14,7 +14,6 @@ Generator, Iterable, List, - NoReturn, Optional, Tuple, Type, diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index 20090b5d2..79b0c1f41 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -1910,6 +1910,7 @@ develop = false antlr4-python3-runtime = ">=4.11.1" attrs = ">=22.2.0" ciso8601 = ">=2.2.0" +datalens-utils = {path = "../dl_utils"} dynamic-enum = {path = "../dynamic_enum"} graphviz = ">=0.17" jinja2 = ">=3.1.2" @@ -4841,7 +4842,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4849,15 +4849,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4874,7 +4867,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4882,7 +4874,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},