Added compeng cache test to dl_core

datalens-tech · Dec 6, 2023 · 0cdc661 · 0cdc661
1 parent 5b83b71
commit 0cdc661
Show file tree

Hide file tree

Showing 24 changed files with 352 additions and 29 deletions.
diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/base.py b/lib/dl_api_lib/dl_api_lib_tests/db/base.py
@@ -24,7 +24,7 @@
 class DefaultApiTestBase(DataApiTestBase, DatasetTestBase, ConnectionTestBase):
     """The knowledge that this is a ClickHouse connector should not go beyond this class"""
 
-    bi_compeng_pg_on = True
+    compeng_enabled = True
     conn_type = CONNECTION_TYPE_CLICKHOUSE
 
     raw_sql_level: ClassVar[RawSQLLevel] = RawSQLLevel.off

diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/config.py b/lib/dl_api_lib/dl_api_lib_tests/db/config.py
@@ -14,6 +14,10 @@
     port_us_pg_5432=get_test_container_hostport("db-postgres", fallback_port=52509).port,
     us_master_token="AC1ofiek8coB",
     core_connector_ep_names=["clickhouse", "postgresql"],
+    compeng_url=(
+        f"postgresql://us:us@"
+        f'{get_test_container_hostport("db-postgres", fallback_port=52509).as_pair()}/us-db-ci_purgeable'
+    ),
 )
 
 
@@ -35,8 +39,4 @@ class CoreConnectionSettings:
     api_connector_ep_names=["clickhouse", "postgresql"],
     core_test_config=CORE_TEST_CONFIG,
     ext_query_executer_secret_key="_some_test_secret_key_",
-    bi_compeng_pg_url=(
-        f"postgresql://us:us@"
-        f'{get_test_container_hostport("db-postgres", fallback_port=52509).as_pair()}/us-db-ci_purgeable'
-    ),
 )
diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/base.py
@@ -52,7 +52,6 @@ class ApiTestBase(abc.ABC):
     Base class defining the basic fixtures of bi-api tests
     """
 
-    bi_compeng_pg_on: ClassVar[bool] = True
     query_processing_mode: ClassVar[QueryProcessingMode] = QueryProcessingMode.basic
 
     @pytest.fixture(scope="function", autouse=True)
@@ -115,8 +114,8 @@ def create_control_api_settings(
             BI_API_CONNECTOR_WHITELIST=bi_test_config.get_api_library_config().api_connector_ep_names,
             CORE_CONNECTOR_WHITELIST=core_test_config.get_core_library_config().core_connector_ep_names,
             RQE_CONFIG=rqe_config_subprocess,
-            BI_COMPENG_PG_ON=cls.bi_compeng_pg_on,
-            BI_COMPENG_PG_URL=bi_test_config.bi_compeng_pg_url,
+            BI_COMPENG_PG_ON=cls.compeng_enabled,
+            BI_COMPENG_PG_URL=core_test_config.get_compeng_url(),
             DO_DSRC_IDX_FETCH=True,
             FIELD_ID_GENERATOR_TYPE=FieldIdGeneratorType.suffix,
             REDIS_ARQ=redis_setting_maker.get_redis_settings_arq(),

diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py
@@ -20,8 +20,6 @@ class ApiTestEnvironmentConfiguration:
 
     mutation_cache_enabled: bool = attr.ib(default=True)
 
-    bi_compeng_pg_url: str = attr.ib(default="")
-
     file_uploader_api_host: str = attr.ib(default="http://127.0.0.1")
     file_uploader_api_port: int = attr.ib(default=9999)
 

diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py
@@ -79,8 +79,8 @@ def create_data_api_settings(
             CORE_CONNECTOR_WHITELIST=core_test_config.get_core_library_config().core_connector_ep_names,
             MUTATIONS_CACHES_ON=cls.mutation_caches_on,
             CACHES_REDIS=redis_setting_maker.get_redis_settings_cache(),
-            BI_COMPENG_PG_ON=cls.bi_compeng_pg_on,
-            BI_COMPENG_PG_URL=bi_test_config.bi_compeng_pg_url,
+            BI_COMPENG_PG_ON=cls.compeng_enabled,
+            BI_COMPENG_PG_URL=core_test_config.get_compeng_url(),
             FIELD_ID_GENERATOR_TYPE=FieldIdGeneratorType.suffix,
             FILE_UPLOADER_BASE_URL=f"{bi_test_config.file_uploader_api_host}:{bi_test_config.file_uploader_api_port}",
             FILE_UPLOADER_MASTER_TOKEN="qwerty",

diff --git a/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py b/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py
@@ -15,7 +15,7 @@
 
 class BigQueryConnectionTestBase(BaseBigQueryTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_BIGQUERY
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py
@@ -3,6 +3,8 @@
 from dl_testing.containers import get_test_container_hostport
 
 
+COMPENG_URL = f'postgresql://datalens:qwerty@{get_test_container_hostport("db-postgres-13", fallback_port=52301).as_pair()}/test_data'
+
 # Infra settings
 CORE_TEST_CONFIG = DefaultCoreTestConfiguration(
     host_us_http=get_test_container_hostport("us", fallback_port=51911).host,
@@ -11,14 +13,13 @@
     port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=51910).port,
     us_master_token="AC1ofiek8coB",
     core_connector_ep_names=["bitrix_gds", "postgresql"],
+    compeng_url=COMPENG_URL,
 )
 
-COMPENG_URL = f'postgresql://datalens:qwerty@{get_test_container_hostport("db-postgres-13", fallback_port=52301).as_pair()}/test_data'
 API_TEST_CONFIG = ApiTestEnvironmentConfiguration(
     api_connector_ep_names=["bitrix_gds", "postgresql"],
     core_test_config=CORE_TEST_CONFIG,
     ext_query_executer_secret_key="_some_test_secret_key_",
-    bi_compeng_pg_url=COMPENG_URL,
 )
 
 BITRIX_PORTALS = dict(

diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/base.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/base.py
@@ -66,7 +66,7 @@ async def get_internal_params(self, src: FileSourceDesc) -> SourceInternalParams
 
 
 class CHS3ConnectionApiTestBase(BaseCHS3TestClass[FILE_CONN_TV], ConnectionTestBase, metaclass=abc.ABCMeta):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py
@@ -17,7 +17,7 @@
 
 class ClickHouseConnectionTestBase(BaseClickHouseTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_CLICKHOUSE
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py b/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py
@@ -23,7 +23,7 @@
 class GreenplumConnectionTestBase(ConnectionTestBase, ServiceFixtureTextClass):
     conn_type = CONNECTION_TYPE_GREENPLUM
     core_test_config = CORE_TEST_CONFIG
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def db_url(self) -> str:

diff --git a/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py b/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py
@@ -29,7 +29,7 @@
 
 class MetricaConnectionTestBase(BaseMetricaTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_METRICA_API
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -71,7 +71,7 @@ def data_api_test_params(self) -> DataApiTestParams:
 
 class AppMetricaConnectionTestBase(BaseAppMetricaTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_APPMETRICA_API
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py
@@ -19,7 +19,7 @@
 
 class MSSQLConnectionTestBase(BaseMSSQLTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_MSSQL
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py
@@ -19,7 +19,7 @@
 
 class MySQLConnectionTestBase(BaseMySQLTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_MYSQL
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py
@@ -20,7 +20,7 @@
 
 class OracleConnectionTestBase(BaseOracleTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_ORACLE
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py
@@ -19,7 +19,7 @@
 
 class PostgreSQLConnectionTestBase(BasePostgreSQLTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_POSTGRES
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

diff --git a/lib/dl_connector_promql/dl_connector_promql_tests/db/api/base.py b/lib/dl_connector_promql/dl_connector_promql_tests/db/api/base.py
@@ -20,7 +20,7 @@
 
 
 class PromQLConnectionTestBase(ConnectionTestBase):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
     conn_type = CONNECTION_TYPE_PROMQL
 
     @pytest.fixture(scope="class")

diff --git a/lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/api/base.py b/lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/api/base.py
@@ -63,7 +63,7 @@ def dataset_params(self, sf_secrets) -> dict:
 
 
 class SnowFlakeDataApiTestBase(SnowFlakeDatasetTestBase, StandardizedDataApiTestBase):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:

diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py
@@ -30,7 +30,7 @@
 
 
 class YDBConnectionTestBase(ConnectionTestBase):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
     conn_type = CONNECTION_TYPE_YDB
 
     @pytest.fixture(scope="class")

diff --git a/lib/dl_core/dl_core_tests/db/compeng/__init__.py b/lib/dl_core/dl_core_tests/db/compeng/__init__.py