Moved redis settings from api test config to core test config

lib/dl_api_lib/dl_api_lib/app/data_api/app.py

@@ -71,7 +71,6 @@
     RedisSentinelService,
     SingleHostSimpleRedisService,
 )
-from dl_core.utils import make_url
 
 
 LOGGER = logging.getLogger(__name__)
@@ -248,12 +247,7 @@ def create_app(
             if self._settings.CACHES_REDIS.MODE == RedisMode.single_host:
                 redis_server_single_host = SingleHostSimpleRedisService(
                     instance_kind=RedisInstanceKind.caches,
-                    url=make_url(
-                        protocol="rediss" if self._settings.CACHES_REDIS.SSL else "redis",
-                        host=self._settings.CACHES_REDIS.HOSTS[0],
-                        port=self._settings.CACHES_REDIS.PORT,
-                        path=str(self._settings.CACHES_REDIS.DB),
-                    ),
+                    url=self._settings.CACHES_REDIS.as_url(),
                     password=self._settings.CACHES_REDIS.PASSWORD,
                     ssl=self._settings.CACHES_REDIS.SSL,
                 )
@@ -278,12 +272,7 @@ def create_app(
             if self._settings.MUTATIONS_REDIS.MODE == RedisMode.single_host:
                 mutations_redis_server_single_host = SingleHostSimpleRedisService(
                     instance_kind=RedisInstanceKind.mutations,
-                    url=make_url(
-                        protocol="rediss" if self._settings.MUTATIONS_REDIS.SSL else "redis",
-                        host=self._settings.MUTATIONS_REDIS.HOSTS[0],
-                        port=self._settings.MUTATIONS_REDIS.PORT,
-                        path=str(self._settings.MUTATIONS_REDIS.DB),
-                    ),
+                    url=self._settings.MUTATIONS_REDIS.as_url(),
                     password=self._settings.MUTATIONS_REDIS.PASSWORD,
                     ssl=self._settings.MUTATIONS_REDIS.SSL,
                 )

lib/dl_api_lib_testing/dl_api_lib_testing/app.py

@@ -98,33 +98,6 @@ def rqe_config_subprocess_cm(self) -> Generator[RQEConfig, None, None]:
             )
 
 
-@attr.s
-class RedisSettingMaker:
-    bi_test_config: ApiTestEnvironmentConfiguration = attr.ib(kw_only=True)
-
-    def get_redis_settings(self, db: int) -> RedisSettings:
-        return RedisSettings(  # type: ignore  # TODO: fix compatibility of models using `s_attrib` with mypy
-            MODE=RedisMode.single_host,
-            CLUSTER_NAME="",
-            HOSTS=(self.bi_test_config.redis_host,),
-            PORT=self.bi_test_config.redis_port,
-            DB=db,
-            PASSWORD=self.bi_test_config.redis_password,
-        )
-
-    def get_redis_settings_default(self) -> RedisSettings:
-        return self.get_redis_settings(self.bi_test_config.redis_db_default)
-
-    def get_redis_settings_cache(self) -> RedisSettings:
-        return self.get_redis_settings(self.bi_test_config.redis_db_cache)
-
-    def get_redis_settings_mutation(self) -> RedisSettings:
-        return self.get_redis_settings(self.bi_test_config.redis_db_mutation)
-
-    def get_redis_settings_arq(self) -> RedisSettings:
-        return self.get_redis_settings(self.bi_test_config.redis_db_arq)
-
-
 class TestingSRFactoryBuilder(SRFactoryBuilder[AppSettings]):
     def _get_required_services(self, settings: AppSettings) -> set[RequiredService]:
         return {RequiredService.RQE_INT_SYNC, RequiredService.RQE_EXT_SYNC}

lib/dl_api_lib_testing/dl_api_lib_testing/base.py

@@ -26,7 +26,6 @@
 from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig
 from dl_api_lib.loader import preload_api_lib
 from dl_api_lib_testing.app import (
-    RedisSettingMaker,
     RQEConfigurationMaker,
     TestingControlApiAppFactory,
 )
@@ -104,7 +103,7 @@ def create_control_api_settings(
         core_test_config = bi_test_config.core_test_config
         us_config = core_test_config.get_us_config()
 
-        redis_setting_maker = RedisSettingMaker(bi_test_config=bi_test_config)
+        redis_setting_maker = core_test_config.get_redis_setting_maker()
 
         settings = ControlApiAppSettings(
             CONNECTOR_AVAILABILITY=ConnectorAvailabilityConfig.from_settings(

lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py

@@ -25,14 +25,6 @@ class ApiTestEnvironmentConfiguration:
     file_uploader_api_host: str = attr.ib(default="http://127.0.0.1")
     file_uploader_api_port: int = attr.ib(default=9999)
 
-    redis_host: str = attr.ib(default="")
-    redis_port: int = attr.ib(default=6379)
-    redis_password: str = attr.ib(default="")
-    redis_db_default: int = attr.ib(default=0)
-    redis_db_cache: int = attr.ib(default=1)
-    redis_db_mutation: int = attr.ib(default=2)
-    redis_db_arq: int = attr.ib(default=11)
-
     connector_availability_settings: ConnectorAvailabilityConfigSettings = attr.ib(
         factory=ConnectorAvailabilityConfigSettings,
     )

lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py

@@ -21,10 +21,7 @@
 from dl_api_client.dsmaker.primitives import Dataset
 from dl_api_lib.app.data_api.app import DataApiAppFactory
 from dl_api_lib.app_settings import DataApiAppSettings
-from dl_api_lib_testing.app import (
-    RedisSettingMaker,
-    TestingDataApiAppFactory,
-)
+from dl_api_lib_testing.app import TestingDataApiAppFactory
 from dl_api_lib_testing.base import ApiTestBase
 from dl_api_lib_testing.client import (
     TestClientConverterAiohttpToFlask,
@@ -67,7 +64,7 @@ def create_data_api_settings(
     ) -> DataApiAppSettings:
         core_test_config = bi_test_config.core_test_config
         us_config = core_test_config.get_us_config()
-        redis_setting_maker = RedisSettingMaker(bi_test_config=bi_test_config)
+        redis_setting_maker = core_test_config.get_redis_setting_maker()
 
         return DataApiAppSettings(
             SENTRY_ENABLED=False,

lib/dl_configs/dl_configs/settings_submodels.py

@@ -6,6 +6,7 @@
 from dl_configs.settings_loaders.meta_definition import s_attrib
 from dl_configs.settings_loaders.settings_obj_base import SettingsBase
 from dl_configs.utils import split_by_comma
+from dl_utils.utils import make_url
 
 
 def redis_mode_env_var_converter(env_value: str) -> RedisMode:
@@ -25,6 +26,14 @@ class RedisSettings(SettingsBase):
     PASSWORD: str = s_attrib("PASSWORD", sensitive=True, missing=None)
     SSL: Optional[bool] = s_attrib("SSL", missing=None)
 
+    def as_url(self) -> str:
+        return make_url(
+            protocol="rediss" if self.SSL else "redis",
+            host=self.HOSTS[0],
+            port=self.PORT,
+            path=str(self.DB),
+        )
+
 
 @attr.s(frozen=True)
 class CorsSettings(SettingsBase):

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py

@@ -18,14 +18,14 @@
     RequestContextInfo,
     TenantCommon,
 )
-from dl_api_lib_testing.app import RedisSettingMaker
 from dl_configs.settings_submodels import S3Settings
 from dl_constants.enums import DataSourceType
 from dl_core.db import (
     SchemaColumn,
     get_type_transformer,
 )
 from dl_core.services_registry import ServicesRegistry
+from dl_core_testing.configuration import RedisSettingMaker
 from dl_core_testing.database import DbTable
 from dl_core_testing.fixtures.primitives import FixtureTableSpec
 from dl_core_testing.fixtures.sample_tables import TABLE_SPEC_SAMPLE_SUPERSTORE
@@ -89,8 +89,8 @@ def conn_bi_context(self) -> RequestContextInfo:
 
     @pytest.fixture(scope="session")
     def redis_setting_maker(self) -> RedisSettingMaker:
-        bi_test_config = test_config.API_TEST_CONFIG
-        return RedisSettingMaker(bi_test_config=bi_test_config)
+        core_test_config = test_config.CORE_TEST_CONFIG
+        return core_test_config.get_redis_setting_maker()
 
     @pytest.fixture(scope="session")
     def s3_settings(self) -> S3Settings:

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py

@@ -12,6 +12,9 @@
     port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=52610).port,
     us_master_token="AC1ofiek8coB",
     core_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"],
+    redis_host=get_test_container_hostport("redis", fallback_port=52604).host,
+    redis_port=get_test_container_hostport("redis", fallback_port=52604).port,
+    redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5",
 )
 
 SR_CONNECTION_SETTINGS = FileS3ConnectorSettings(
@@ -37,7 +40,4 @@
     api_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"],
     core_test_config=CORE_TEST_CONFIG,
     ext_query_executer_secret_key="_some_test_secret_key_",
-    redis_host=get_test_container_hostport("redis", fallback_port=52604).host,
-    redis_port=get_test_container_hostport("redis", fallback_port=52604).port,
-    redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5",
 )

lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py

@@ -18,8 +18,8 @@
 from dl_core.connectors.base.error_transformer import DBExcKWArgs
 from dl_core.utils import (
     get_current_w3c_tracing_headers,
-    make_url,
 )
+from dl_utils.utils import make_url
 from dl_utils.aio import await_sync
 
 from dl_connector_chyt.core.constants import CONNECTION_TYPE_CHYT

lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py

@@ -63,7 +63,7 @@
     GenericNativeType,
     norm_native_type,
 )
-from dl_core.utils import make_url
+from dl_utils.utils import make_url
 
 from dl_connector_clickhouse.core.clickhouse_base.ch_commons import (
     ClickHouseBaseUtils,

lib/dl_connector_promql/dl_connector_promql/core/adapter.py

@@ -29,7 +29,7 @@
 from dl_core.connection_executors.adapters.async_adapters_base import AsyncRawExecutionResult
 from dl_core.db.native_type import GenericNativeType
 from dl_core.exc import DatabaseQueryError
-from dl_core.utils import make_url
+from dl_utils.utils import make_url
 
 from dl_connector_promql.core.constants import CONNECTION_TYPE_PROMQL
 

lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py

@@ -35,7 +35,6 @@
     generic_profiler_async,
 )
 from dl_core import exc as common_exc
-from dl_core import utils
 from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter
 from dl_core.connection_executors.adapters.async_adapters_base import (
     AsyncDBAdapter,
@@ -62,6 +61,7 @@
 from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest
 from dl_core.connection_models.conn_options import ConnectOptions
 from dl_core.enums import RQEEventType
+from dl_utils.utils import make_url
 
 
 if TYPE_CHECKING:
@@ -147,14 +147,14 @@ async def _make_request(
 
         url: str
         if self._use_sync_rqe:
-            url = utils.make_url(
+            url = make_url(
                 protocol=qe.sync_protocol,
                 host=qe.sync_host,
                 port=qe.sync_port,
                 path=rel_path,
             )
         else:
-            url = utils.make_url(
+            url = make_url(
                 protocol=qe.async_protocol,
                 host=qe.async_host,
                 port=qe.async_port,

lib/dl_core/dl_core/utils.py

@@ -2,7 +2,6 @@
 
 import ipaddress
 import logging
-import os
 import re
 from typing import (
     Any,
@@ -42,25 +41,12 @@
     stringify_dl_cookies,
     stringify_dl_headers,
 )
-from dl_configs.settings_loaders.env_remap import remap_env
 from dl_constants.api_constants import DLHeadersCommon
 
 
 LOGGER = logging.getLogger(__name__)
 
 
-def make_url(
-    protocol: str,
-    host: str,
-    port: int,
-    path: Optional[str] = None,
-) -> str:
-    # TODO FIX: Sanitize/use urllib
-    if path is None:
-        path = ""
-    return f"{protocol}://{host}:{port}/{path.lstrip('/')}"
-
-
 def get_requests_session() -> requests.Session:
     session = requests.Session()
     ua = "{}, Datalens".format(requests.utils.default_user_agent())

lib/dl_core_testing/dl_core_testing/configuration.py

@@ -13,6 +13,8 @@
     CryptoKeysConfig,
     get_single_key_crypto_keys_config,
 )
+from dl_configs.enums import RedisMode
+from dl_configs.settings_submodels import RedisSettings
 from dl_core.loader import CoreLibraryConfig
 
 
@@ -27,6 +29,39 @@ class UnitedStorageConfiguration:
     force: bool = attr.ib(kw_only=True, default=True)
 
 
+@attr.s(frozen=True)
+class RedisSettingMaker:
+    redis_host: str = attr.ib(default="")
+    redis_port: int = attr.ib(default=6379)
+    redis_password: str = attr.ib(default="")
+    redis_db_default: int = attr.ib(default=0)
+    redis_db_cache: int = attr.ib(default=1)
+    redis_db_mutation: int = attr.ib(default=2)
+    redis_db_arq: int = attr.ib(default=11)
+
+    def get_redis_settings(self, db: int) -> RedisSettings:
+        return RedisSettings(  # type: ignore  # TODO: fix compatibility of models using `s_attrib` with mypy
+            MODE=RedisMode.single_host,
+            CLUSTER_NAME="",
+            HOSTS=(self.redis_host,),
+            PORT=self.redis_port,
+            DB=db,
+            PASSWORD=self.redis_password,
+        )
+
+    def get_redis_settings_default(self) -> RedisSettings:
+        return self.get_redis_settings(self.redis_db_default)
+
+    def get_redis_settings_cache(self) -> RedisSettings:
+        return self.get_redis_settings(self.redis_db_cache)
+
+    def get_redis_settings_mutation(self) -> RedisSettings:
+        return self.get_redis_settings(self.redis_db_mutation)
+
+    def get_redis_settings_arq(self) -> RedisSettings:
+        return self.get_redis_settings(self.redis_db_arq)
+
+
 @attr.s(frozen=True)
 class CoreTestEnvironmentConfigurationBase(abc.ABC):
     @abc.abstractmethod
@@ -41,6 +76,10 @@ def get_crypto_keys_config(self) -> CryptoKeysConfig:
     def get_core_library_config(self) -> CoreLibraryConfig:
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def get_redis_setting_maker(self) -> RedisSettingMaker:
+        raise NotImplementedError
+
 
 # These are used only for creation of local environments in tests, not actual external ones
 DEFAULT_FERNET_KEY = "h1ZpilcYLYRdWp7Nk8X1M1kBPiUi8rdjz9oBfHyUKIk="
@@ -54,8 +93,17 @@ class DefaultCoreTestConfiguration(CoreTestEnvironmentConfigurationBase):
     port_us_pg_5432: int = attr.ib(kw_only=True)
     us_master_token: str = attr.ib(kw_only=True)
     fernet_key: str = attr.ib(kw_only=True, default=DEFAULT_FERNET_KEY)
+
     core_connector_ep_names: Optional[Collection[str]] = attr.ib(kw_only=True, default=None)
 
+    redis_host: str = attr.ib(default="")
+    redis_port: int = attr.ib(default=6379)
+    redis_password: str = attr.ib(default="")
+    redis_db_default: int = attr.ib(default=0)
+    redis_db_cache: int = attr.ib(default=1)
+    redis_db_mutation: int = attr.ib(default=2)
+    redis_db_arq: int = attr.ib(default=11)
+
     def get_us_config(self) -> UnitedStorageConfiguration:
         return UnitedStorageConfiguration(
             us_master_token=self.us_master_token,
@@ -66,6 +114,17 @@ def get_us_config(self) -> UnitedStorageConfiguration:
     def get_crypto_keys_config(self) -> CryptoKeysConfig:
         return get_single_key_crypto_keys_config(key_id="0", key_value=self.fernet_key)
 
+    def get_redis_setting_maker(self) -> RedisSettingMaker:
+        return RedisSettingMaker(
+            redis_host=self.redis_host,
+            redis_port=self.redis_port,
+            redis_password=self.redis_password,
+            redis_db_default=self.redis_db_default,
+            redis_db_cache=self.redis_db_cache,
+            redis_db_mutation=self.redis_db_mutation,
+            redis_db_arq=self.redis_db_arq,
+        )
+
     def get_core_library_config(self) -> CoreLibraryConfig:
         return CoreLibraryConfig(
             core_connector_ep_names=self.core_connector_ep_names,