diff --git a/lib/dl_api_lib/dl_api_lib/app/data_api/app.py b/lib/dl_api_lib/dl_api_lib/app/data_api/app.py index 31a18820c..0ec6178fa 100644 --- a/lib/dl_api_lib/dl_api_lib/app/data_api/app.py +++ b/lib/dl_api_lib/dl_api_lib/app/data_api/app.py @@ -71,7 +71,6 @@ RedisSentinelService, SingleHostSimpleRedisService, ) -from dl_core.utils import make_url LOGGER = logging.getLogger(__name__) @@ -248,12 +247,7 @@ def create_app( if self._settings.CACHES_REDIS.MODE == RedisMode.single_host: redis_server_single_host = SingleHostSimpleRedisService( instance_kind=RedisInstanceKind.caches, - url=make_url( - protocol="rediss" if self._settings.CACHES_REDIS.SSL else "redis", - host=self._settings.CACHES_REDIS.HOSTS[0], - port=self._settings.CACHES_REDIS.PORT, - path=str(self._settings.CACHES_REDIS.DB), - ), + url=self._settings.CACHES_REDIS.as_single_host_url(), password=self._settings.CACHES_REDIS.PASSWORD, ssl=self._settings.CACHES_REDIS.SSL, ) @@ -278,12 +272,7 @@ def create_app( if self._settings.MUTATIONS_REDIS.MODE == RedisMode.single_host: mutations_redis_server_single_host = SingleHostSimpleRedisService( instance_kind=RedisInstanceKind.mutations, - url=make_url( - protocol="rediss" if self._settings.MUTATIONS_REDIS.SSL else "redis", - host=self._settings.MUTATIONS_REDIS.HOSTS[0], - port=self._settings.MUTATIONS_REDIS.PORT, - path=str(self._settings.MUTATIONS_REDIS.DB), - ), + url=self._settings.MUTATIONS_REDIS.as_single_host_url(), password=self._settings.MUTATIONS_REDIS.PASSWORD, ssl=self._settings.MUTATIONS_REDIS.SSL, ) diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py index 69da45d86..307c5635a 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py @@ -98,33 +98,6 @@ def rqe_config_subprocess_cm(self) -> Generator[RQEConfig, None, None]: ) -@attr.s -class RedisSettingMaker: - bi_test_config: ApiTestEnvironmentConfiguration = attr.ib(kw_only=True) - - def get_redis_settings(self, db: int) -> RedisSettings: - return RedisSettings( # type: ignore # TODO: fix compatibility of models using `s_attrib` with mypy - MODE=RedisMode.single_host, - CLUSTER_NAME="", - HOSTS=(self.bi_test_config.redis_host,), - PORT=self.bi_test_config.redis_port, - DB=db, - PASSWORD=self.bi_test_config.redis_password, - ) - - def get_redis_settings_default(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_default) - - def get_redis_settings_cache(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_cache) - - def get_redis_settings_mutation(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_mutation) - - def get_redis_settings_arq(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_arq) - - class TestingSRFactoryBuilder(SRFactoryBuilder[AppSettings]): def _get_required_services(self, settings: AppSettings) -> set[RequiredService]: return {RequiredService.RQE_INT_SYNC, RequiredService.RQE_EXT_SYNC} diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/base.py index eb85f299a..068b2db29 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/base.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/base.py @@ -26,7 +26,6 @@ from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig from dl_api_lib.loader import preload_api_lib from dl_api_lib_testing.app import ( - RedisSettingMaker, RQEConfigurationMaker, TestingControlApiAppFactory, ) @@ -104,7 +103,7 @@ def create_control_api_settings( core_test_config = bi_test_config.core_test_config us_config = core_test_config.get_us_config() - redis_setting_maker = RedisSettingMaker(bi_test_config=bi_test_config) + redis_setting_maker = core_test_config.get_redis_setting_maker() settings = ControlApiAppSettings( CONNECTOR_AVAILABILITY=ConnectorAvailabilityConfig.from_settings( diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py index 48a9ed3ec..d61dee412 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py @@ -25,14 +25,6 @@ class ApiTestEnvironmentConfiguration: file_uploader_api_host: str = attr.ib(default="http://127.0.0.1") file_uploader_api_port: int = attr.ib(default=9999) - redis_host: str = attr.ib(default="") - redis_port: int = attr.ib(default=6379) - redis_password: str = attr.ib(default="") - redis_db_default: int = attr.ib(default=0) - redis_db_cache: int = attr.ib(default=1) - redis_db_mutation: int = attr.ib(default=2) - redis_db_arq: int = attr.ib(default=11) - connector_availability_settings: ConnectorAvailabilityConfigSettings = attr.ib( factory=ConnectorAvailabilityConfigSettings, ) diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py index 8c00f652e..b10309931 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py @@ -21,10 +21,7 @@ from dl_api_client.dsmaker.primitives import Dataset from dl_api_lib.app.data_api.app import DataApiAppFactory from dl_api_lib.app_settings import DataApiAppSettings -from dl_api_lib_testing.app import ( - RedisSettingMaker, - TestingDataApiAppFactory, -) +from dl_api_lib_testing.app import TestingDataApiAppFactory from dl_api_lib_testing.base import ApiTestBase from dl_api_lib_testing.client import ( TestClientConverterAiohttpToFlask, @@ -67,7 +64,7 @@ def create_data_api_settings( ) -> DataApiAppSettings: core_test_config = bi_test_config.core_test_config us_config = core_test_config.get_us_config() - redis_setting_maker = RedisSettingMaker(bi_test_config=bi_test_config) + redis_setting_maker = core_test_config.get_redis_setting_maker() return DataApiAppSettings( SENTRY_ENABLED=False, diff --git a/lib/dl_configs/dl_configs/settings_submodels.py b/lib/dl_configs/dl_configs/settings_submodels.py index 319371f07..21234c88d 100644 --- a/lib/dl_configs/dl_configs/settings_submodels.py +++ b/lib/dl_configs/dl_configs/settings_submodels.py @@ -6,6 +6,7 @@ from dl_configs.settings_loaders.meta_definition import s_attrib from dl_configs.settings_loaders.settings_obj_base import SettingsBase from dl_configs.utils import split_by_comma +from dl_utils.utils import make_url def redis_mode_env_var_converter(env_value: str) -> RedisMode: @@ -25,6 +26,14 @@ class RedisSettings(SettingsBase): PASSWORD: str = s_attrib("PASSWORD", sensitive=True, missing=None) SSL: Optional[bool] = s_attrib("SSL", missing=None) + def as_single_host_url(self) -> str: + return make_url( + protocol="rediss" if self.SSL else "redis", + host=self.HOSTS[0], + port=self.PORT, + path=str(self.DB), + ) + @attr.s(frozen=True) class CorsSettings(SettingsBase): diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py index db5506f3c..3a662632f 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py @@ -18,7 +18,6 @@ RequestContextInfo, TenantCommon, ) -from dl_api_lib_testing.app import RedisSettingMaker from dl_configs.settings_submodels import S3Settings from dl_constants.enums import DataSourceType from dl_core.db import ( @@ -26,6 +25,7 @@ get_type_transformer, ) from dl_core.services_registry import ServicesRegistry +from dl_core_testing.configuration import RedisSettingMaker from dl_core_testing.database import DbTable from dl_core_testing.fixtures.primitives import FixtureTableSpec from dl_core_testing.fixtures.sample_tables import TABLE_SPEC_SAMPLE_SUPERSTORE @@ -89,8 +89,8 @@ def conn_bi_context(self) -> RequestContextInfo: @pytest.fixture(scope="session") def redis_setting_maker(self) -> RedisSettingMaker: - bi_test_config = test_config.API_TEST_CONFIG - return RedisSettingMaker(bi_test_config=bi_test_config) + core_test_config = test_config.CORE_TEST_CONFIG + return core_test_config.get_redis_setting_maker() @pytest.fixture(scope="session") def s3_settings(self) -> S3Settings: diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py index 9a52eb713..df42bcf2d 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py @@ -12,6 +12,9 @@ port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=52610).port, us_master_token="AC1ofiek8coB", core_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"], + redis_host=get_test_container_hostport("redis", fallback_port=52604).host, + redis_port=get_test_container_hostport("redis", fallback_port=52604).port, + redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5", ) SR_CONNECTION_SETTINGS = FileS3ConnectorSettings( @@ -37,7 +40,4 @@ api_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"], core_test_config=CORE_TEST_CONFIG, ext_query_executer_secret_key="_some_test_secret_key_", - redis_host=get_test_container_hostport("redis", fallback_port=52604).host, - redis_port=get_test_container_hostport("redis", fallback_port=52604).port, - redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5", ) diff --git a/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py b/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py index 05a7bf6d5..0e7d84b76 100644 --- a/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py +++ b/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py @@ -16,11 +16,9 @@ from dl_core.connection_executors.models.db_adapter_data import RawIndexInfo from dl_core.connection_models import TableIdent from dl_core.connectors.base.error_transformer import DBExcKWArgs -from dl_core.utils import ( - get_current_w3c_tracing_headers, - make_url, -) +from dl_core.utils import get_current_w3c_tracing_headers from dl_utils.aio import await_sync +from dl_utils.utils import make_url from dl_connector_chyt.core.constants import CONNECTION_TYPE_CHYT from dl_connector_chyt.core.target_dto import ( diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py index a9f0a7a71..312400958 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py @@ -63,7 +63,7 @@ GenericNativeType, norm_native_type, ) -from dl_core.utils import make_url +from dl_utils.utils import make_url from dl_connector_clickhouse.core.clickhouse_base.ch_commons import ( ClickHouseBaseUtils, diff --git a/lib/dl_connector_promql/dl_connector_promql/core/adapter.py b/lib/dl_connector_promql/dl_connector_promql/core/adapter.py index 424deff73..e460979a3 100644 --- a/lib/dl_connector_promql/dl_connector_promql/core/adapter.py +++ b/lib/dl_connector_promql/dl_connector_promql/core/adapter.py @@ -29,7 +29,7 @@ from dl_core.connection_executors.adapters.async_adapters_base import AsyncRawExecutionResult from dl_core.db.native_type import GenericNativeType from dl_core.exc import DatabaseQueryError -from dl_core.utils import make_url +from dl_utils.utils import make_url from dl_connector_promql.core.constants import CONNECTION_TYPE_PROMQL diff --git a/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py b/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py index 756e6eb69..cdbdfeea9 100644 --- a/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py +++ b/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py @@ -35,7 +35,6 @@ generic_profiler_async, ) from dl_core import exc as common_exc -from dl_core import utils from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter from dl_core.connection_executors.adapters.async_adapters_base import ( AsyncDBAdapter, @@ -62,6 +61,7 @@ from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest from dl_core.connection_models.conn_options import ConnectOptions from dl_core.enums import RQEEventType +from dl_utils.utils import make_url if TYPE_CHECKING: @@ -147,14 +147,14 @@ async def _make_request( url: str if self._use_sync_rqe: - url = utils.make_url( + url = make_url( protocol=qe.sync_protocol, host=qe.sync_host, port=qe.sync_port, path=rel_path, ) else: - url = utils.make_url( + url = make_url( protocol=qe.async_protocol, host=qe.async_host, port=qe.async_port, diff --git a/lib/dl_core/dl_core/utils.py b/lib/dl_core/dl_core/utils.py index 5953d1b08..d874688d3 100644 --- a/lib/dl_core/dl_core/utils.py +++ b/lib/dl_core/dl_core/utils.py @@ -2,7 +2,6 @@ import ipaddress import logging -import os import re from typing import ( Any, @@ -42,25 +41,12 @@ stringify_dl_cookies, stringify_dl_headers, ) -from dl_configs.settings_loaders.env_remap import remap_env from dl_constants.api_constants import DLHeadersCommon LOGGER = logging.getLogger(__name__) -def make_url( - protocol: str, - host: str, - port: int, - path: Optional[str] = None, -) -> str: - # TODO FIX: Sanitize/use urllib - if path is None: - path = "" - return f"{protocol}://{host}:{port}/{path.lstrip('/')}" - - def get_requests_session() -> requests.Session: session = requests.Session() ua = "{}, Datalens".format(requests.utils.default_user_agent()) diff --git a/lib/dl_core_testing/dl_core_testing/configuration.py b/lib/dl_core_testing/dl_core_testing/configuration.py index e14085902..0851f1b7c 100644 --- a/lib/dl_core_testing/dl_core_testing/configuration.py +++ b/lib/dl_core_testing/dl_core_testing/configuration.py @@ -13,6 +13,8 @@ CryptoKeysConfig, get_single_key_crypto_keys_config, ) +from dl_configs.enums import RedisMode +from dl_configs.settings_submodels import RedisSettings from dl_core.loader import CoreLibraryConfig @@ -27,6 +29,39 @@ class UnitedStorageConfiguration: force: bool = attr.ib(kw_only=True, default=True) +@attr.s(frozen=True) +class RedisSettingMaker: + redis_host: str = attr.ib(default="") + redis_port: int = attr.ib(default=6379) + redis_password: str = attr.ib(default="") + redis_db_default: int = attr.ib(default=0) + redis_db_cache: int = attr.ib(default=1) + redis_db_mutation: int = attr.ib(default=2) + redis_db_arq: int = attr.ib(default=11) + + def get_redis_settings(self, db: int) -> RedisSettings: + return RedisSettings( # type: ignore # TODO: fix compatibility of models using `s_attrib` with mypy + MODE=RedisMode.single_host, + CLUSTER_NAME="", + HOSTS=(self.redis_host,), + PORT=self.redis_port, + DB=db, + PASSWORD=self.redis_password, + ) + + def get_redis_settings_default(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_default) + + def get_redis_settings_cache(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_cache) + + def get_redis_settings_mutation(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_mutation) + + def get_redis_settings_arq(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_arq) + + @attr.s(frozen=True) class CoreTestEnvironmentConfigurationBase(abc.ABC): @abc.abstractmethod @@ -41,6 +76,10 @@ def get_crypto_keys_config(self) -> CryptoKeysConfig: def get_core_library_config(self) -> CoreLibraryConfig: raise NotImplementedError + @abc.abstractmethod + def get_redis_setting_maker(self) -> RedisSettingMaker: + raise NotImplementedError + # These are used only for creation of local environments in tests, not actual external ones DEFAULT_FERNET_KEY = "h1ZpilcYLYRdWp7Nk8X1M1kBPiUi8rdjz9oBfHyUKIk=" @@ -54,8 +93,17 @@ class DefaultCoreTestConfiguration(CoreTestEnvironmentConfigurationBase): port_us_pg_5432: int = attr.ib(kw_only=True) us_master_token: str = attr.ib(kw_only=True) fernet_key: str = attr.ib(kw_only=True, default=DEFAULT_FERNET_KEY) + core_connector_ep_names: Optional[Collection[str]] = attr.ib(kw_only=True, default=None) + redis_host: str = attr.ib(default="") + redis_port: int = attr.ib(default=6379) + redis_password: str = attr.ib(default="") + redis_db_default: int = attr.ib(default=0) + redis_db_cache: int = attr.ib(default=1) + redis_db_mutation: int = attr.ib(default=2) + redis_db_arq: int = attr.ib(default=11) + def get_us_config(self) -> UnitedStorageConfiguration: return UnitedStorageConfiguration( us_master_token=self.us_master_token, @@ -66,6 +114,17 @@ def get_us_config(self) -> UnitedStorageConfiguration: def get_crypto_keys_config(self) -> CryptoKeysConfig: return get_single_key_crypto_keys_config(key_id="0", key_value=self.fernet_key) + def get_redis_setting_maker(self) -> RedisSettingMaker: + return RedisSettingMaker( + redis_host=self.redis_host, + redis_port=self.redis_port, + redis_password=self.redis_password, + redis_db_default=self.redis_db_default, + redis_db_cache=self.redis_db_cache, + redis_db_mutation=self.redis_db_mutation, + redis_db_arq=self.redis_db_arq, + ) + def get_core_library_config(self) -> CoreLibraryConfig: return CoreLibraryConfig( core_connector_ep_names=self.core_connector_ep_names, diff --git a/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py b/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py index 2e1f00abc..1fd0b0b6e 100644 --- a/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py +++ b/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py @@ -5,7 +5,6 @@ RedisSentinelService, SingleHostSimpleRedisService, ) -from dl_core.utils import make_url from dl_file_uploader_lib.settings import FileUploaderBaseSettings @@ -14,12 +13,7 @@ def init_redis_service(settings: FileUploaderBaseSettings) -> RedisBaseService: if settings.REDIS_APP.MODE == RedisMode.single_host: assert len(settings.REDIS_APP.HOSTS) == 1 redis_service = SingleHostSimpleRedisService( - url=make_url( - protocol="rediss" if settings.REDIS_APP.SSL else "redis", - host=settings.REDIS_APP.HOSTS[0], - port=settings.REDIS_APP.PORT, - path=str(settings.REDIS_APP.DB), - ), + url=settings.REDIS_APP.as_single_host_url(), password=settings.REDIS_APP.PASSWORD, instance_kind=RedisInstanceKind.persistent, ssl=settings.REDIS_APP.SSL, diff --git a/lib/dl_utils/dl_utils/utils.py b/lib/dl_utils/dl_utils/utils.py index f54affdf3..1b45e9ca3 100644 --- a/lib/dl_utils/dl_utils/utils.py +++ b/lib/dl_utils/dl_utils/utils.py @@ -221,3 +221,15 @@ def time_it_cm(label: str) -> Generator[None, None, None]: delta = time() - t0 if delta >= 0.01: print(f"Time elapsed for {label}: {delta}") + + +def make_url( + protocol: str, + host: str, + port: int, + path: Optional[str] = None, +) -> str: + # TODO FIX: Sanitize/use urllib + if path is None: + path = "" + return f"{protocol}://{host}:{port}/{path.lstrip('/')}"