Added a compeng cache test to dl_core and an api cache test to dl_api_lib

lib/dl_api_lib/dl_api_lib_tests/db/base.py

@@ -24,7 +24,7 @@
 class DefaultApiTestBase(DataApiTestBase, DatasetTestBase, ConnectionTestBase):
     """The knowledge that this is a ClickHouse connector should not go beyond this class"""
 
-    bi_compeng_pg_on = True
+    compeng_enabled = True
     conn_type = CONNECTION_TYPE_CLICKHOUSE
 
     raw_sql_level: ClassVar[RawSQLLevel] = RawSQLLevel.off

lib/dl_api_lib/dl_api_lib_tests/db/config.py

@@ -14,6 +14,12 @@
     port_us_pg_5432=get_test_container_hostport("db-postgres", fallback_port=52509).port,
     us_master_token="AC1ofiek8coB",
     core_connector_ep_names=["clickhouse", "postgresql"],
+    compeng_url=(
+        f"postgresql://us:us@"
+        f'{get_test_container_hostport("db-postgres", fallback_port=52509).as_pair()}/us-db-ci_purgeable'
+    ),
+    redis_host=get_test_container_hostport("redis-caches").host,
+    redis_port=get_test_container_hostport("redis-caches", fallback_port=52505).port,
 )
 
 
@@ -35,8 +41,4 @@ class CoreConnectionSettings:
     api_connector_ep_names=["clickhouse", "postgresql"],
     core_test_config=CORE_TEST_CONFIG,
     ext_query_executer_secret_key="_some_test_secret_key_",
-    bi_compeng_pg_url=(
-        f"postgresql://us:us@"
-        f'{get_test_container_hostport("db-postgres", fallback_port=52509).as_pair()}/us-db-ci_purgeable'
-    ),
 )

lib/dl_api_lib/dl_api_lib_tests/db/data_api/caches/test_caches.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from dl_api_client.dsmaker.primitives import Dataset
+from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows
+from dl_api_lib_testing.helpers.data_source import data_source_settings_from_table
+from dl_api_lib_tests.db.base import DefaultApiTestBase
+from dl_core_testing.database import make_table
+
+
+class TestDataCaches(DefaultApiTestBase):
+    data_caches_enabled = True
+
+    def test_cache_by_deleting_table(self, db, control_api, data_api, saved_connection_id):
+        db_table = make_table(db)
+        ds = Dataset()
+        ds.sources["source_1"] = ds.source(
+            connection_id=saved_connection_id, **data_source_settings_from_table(db_table)
+        )
+        ds.source_avatars["avatar_1"] = ds.sources["source_1"].avatar()
+        ds.result_schema["measure"] = ds.field(formula="SUM([int_value])")
+        ds = control_api.apply_updates(dataset=ds).dataset
+        ds = control_api.save_dataset(ds).dataset
+
+        def get_data():
+            result_resp = data_api.get_result(
+                dataset=ds,
+                fields=[
+                    ds.find_field(title="int_value"),
+                    ds.find_field(title="measure"),
+                ],
+            )
+            assert result_resp.status_code == 200, result_resp.response_errors
+            data = get_data_rows(response=result_resp)
+            return data
+
+        data_rows = get_data()
+
+        # Now delete the table.
+        # This will make real DB queries impossible,
+        # however the cache should still return the same data
+        db_table.db.drop_table(db_table.table)
+
+        data_rows_after_deletion = get_data()
+        assert data_rows_after_deletion == data_rows

lib/dl_api_lib/docker-compose.yml

@@ -4,6 +4,14 @@ x-constants:
   US_MASTER_TOKEN: &c-us-master-token "AC1ofiek8coB"
 
 services:
+  redis-caches:
+    # image: "bitnami/redis:5.0.8"
+    image: "bitnami/redis:5.0.8@sha256:3127620da977815556439a9dc347fff89432a79b6bb6e93a16f20ac4a34ce337"
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+    ports:
+     - "52505:6379"
+
   db-clickhouse:
     ports:
      - "52510:8123"

lib/dl_api_lib_testing/dl_api_lib_testing/base.py

@@ -52,7 +52,7 @@ class ApiTestBase(abc.ABC):
     Base class defining the basic fixtures of bi-api tests
     """
 
-    bi_compeng_pg_on: ClassVar[bool] = True
+    compeng_enabled: ClassVar[bool] = True
     query_processing_mode: ClassVar[QueryProcessingMode] = QueryProcessingMode.basic
 
     @pytest.fixture(scope="function", autouse=True)
@@ -115,8 +115,8 @@ def create_control_api_settings(
             BI_API_CONNECTOR_WHITELIST=bi_test_config.get_api_library_config().api_connector_ep_names,
             CORE_CONNECTOR_WHITELIST=core_test_config.get_core_library_config().core_connector_ep_names,
             RQE_CONFIG=rqe_config_subprocess,
-            BI_COMPENG_PG_ON=cls.bi_compeng_pg_on,
-            BI_COMPENG_PG_URL=bi_test_config.bi_compeng_pg_url,
+            BI_COMPENG_PG_ON=cls.compeng_enabled,
+            BI_COMPENG_PG_URL=core_test_config.get_compeng_url(),
             DO_DSRC_IDX_FETCH=True,
             FIELD_ID_GENERATOR_TYPE=FieldIdGeneratorType.suffix,
             REDIS_ARQ=redis_setting_maker.get_redis_settings_arq(),

lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py

@@ -20,8 +20,6 @@ class ApiTestEnvironmentConfiguration:
 
     mutation_cache_enabled: bool = attr.ib(default=True)
 
-    bi_compeng_pg_url: str = attr.ib(default="")
-
     file_uploader_api_host: str = attr.ib(default="http://127.0.0.1")
     file_uploader_api_port: int = attr.ib(default=9999)
 

lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py

@@ -45,8 +45,8 @@ class DataApiTestParams(NamedTuple):
 
 
 class DataApiTestBase(ApiTestBase, metaclass=abc.ABCMeta):
-    mutation_caches_on: ClassVar[bool] = True
-    data_caches_on: ClassVar[bool] = True
+    mutation_caches_enabled: ClassVar[bool] = True
+    data_caches_enabled: ClassVar[bool] = True
 
     @pytest.fixture
     def loop(self, event_loop: asyncio.AbstractEventLoop) -> Generator[asyncio.AbstractEventLoop, None, None]:
@@ -72,15 +72,15 @@ def create_data_api_settings(
             US_MASTER_TOKEN=us_config.us_master_token,
             CRYPTO_KEYS_CONFIG=core_test_config.get_crypto_keys_config(),
             # TODO FIX: Configure caches
-            CACHES_ON=cls.data_caches_on,
+            CACHES_ON=cls.data_caches_enabled,
             SAMPLES_CH_HOSTS=(),
             RQE_CONFIG=rqe_config_subprocess,
             BI_API_CONNECTOR_WHITELIST=bi_test_config.get_api_library_config().api_connector_ep_names,
             CORE_CONNECTOR_WHITELIST=core_test_config.get_core_library_config().core_connector_ep_names,
-            MUTATIONS_CACHES_ON=cls.mutation_caches_on,
+            MUTATIONS_CACHES_ON=cls.mutation_caches_enabled,
             CACHES_REDIS=redis_setting_maker.get_redis_settings_cache(),
-            BI_COMPENG_PG_ON=cls.bi_compeng_pg_on,
-            BI_COMPENG_PG_URL=bi_test_config.bi_compeng_pg_url,
+            BI_COMPENG_PG_ON=cls.compeng_enabled,
+            BI_COMPENG_PG_URL=core_test_config.get_compeng_url(),
             FIELD_ID_GENERATOR_TYPE=FieldIdGeneratorType.suffix,
             FILE_UPLOADER_BASE_URL=f"{bi_test_config.file_uploader_api_host}:{bi_test_config.file_uploader_api_port}",
             FILE_UPLOADER_MASTER_TOKEN="qwerty",

lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py

@@ -15,7 +15,7 @@
 
 class BigQueryConnectionTestBase(BaseBigQueryTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_BIGQUERY
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -42,4 +42,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class BigQueryDataApiTestBase(BigQueryDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/api/base.py

@@ -95,7 +95,7 @@ def dataset_params(self) -> dict:
 
 
 class BitrixDataApiTestBase(BitrixDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:
@@ -109,4 +109,4 @@ def data_api_test_params(self) -> DataApiTestParams:
 
 
 class BitrixSmartTablesDataApiTestBase(BitrixSmartTablesDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py

@@ -3,6 +3,8 @@
 from dl_testing.containers import get_test_container_hostport
 
 
+COMPENG_URL = f'postgresql://datalens:qwerty@{get_test_container_hostport("db-postgres-13", fallback_port=52301).as_pair()}/test_data'
+
 # Infra settings
 CORE_TEST_CONFIG = DefaultCoreTestConfiguration(
     host_us_http=get_test_container_hostport("us", fallback_port=51911).host,
@@ -11,14 +13,13 @@
     port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=51910).port,
     us_master_token="AC1ofiek8coB",
     core_connector_ep_names=["bitrix_gds", "postgresql"],
+    compeng_url=COMPENG_URL,
 )
 
-COMPENG_URL = f'postgresql://datalens:qwerty@{get_test_container_hostport("db-postgres-13", fallback_port=52301).as_pair()}/test_data'
 API_TEST_CONFIG = ApiTestEnvironmentConfiguration(
     api_connector_ep_names=["bitrix_gds", "postgresql"],
     core_test_config=CORE_TEST_CONFIG,
     ext_query_executer_secret_key="_some_test_secret_key_",
-    bi_compeng_pg_url=COMPENG_URL,
 )
 
 BITRIX_PORTALS = dict(

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/base.py

@@ -66,7 +66,7 @@ async def get_internal_params(self, src: FileSourceDesc) -> SourceInternalParams
 
 
 class CHS3ConnectionApiTestBase(BaseCHS3TestClass[FILE_CONN_TV], ConnectionTestBase, metaclass=abc.ABCMeta):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/data.py

@@ -24,7 +24,7 @@
 
 
 class CHS3DataApiTestBase(CHS3DatasetTestBase[FILE_CONN_TV], StandardizedDataApiTestBase, metaclass=abc.ABCMeta):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
 
 class CHS3DataResultTestSuite(

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py

@@ -17,7 +17,7 @@
 
 class ClickHouseConnectionTestBase(BaseClickHouseTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_CLICKHOUSE
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -52,4 +52,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class ClickHouseDataApiTestBase(ClickHouseDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py

@@ -23,7 +23,7 @@
 class GreenplumConnectionTestBase(ConnectionTestBase, ServiceFixtureTextClass):
     conn_type = CONNECTION_TYPE_GREENPLUM
     core_test_config = CORE_TEST_CONFIG
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def db_url(self) -> str:
@@ -57,4 +57,4 @@ def dataset_params(self, sample_table: DbTable) -> dict:
 
 
 class GreenplumDataApiTestBase(GreenplumDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py

@@ -29,7 +29,7 @@
 
 class MetricaConnectionTestBase(BaseMetricaTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_METRICA_API
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -56,7 +56,7 @@ def dataset_params(self) -> dict:
 
 
 class MetricaDataApiTestBase(MetricaDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:
@@ -71,7 +71,7 @@ def data_api_test_params(self) -> DataApiTestParams:
 
 class AppMetricaConnectionTestBase(BaseAppMetricaTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_APPMETRICA_API
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:

lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py

@@ -19,7 +19,7 @@
 
 class MSSQLConnectionTestBase(BaseMSSQLTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_MSSQL
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -55,4 +55,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class MSSQLDataApiTestBase(MSSQLDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py

@@ -19,7 +19,7 @@
 
 class MySQLConnectionTestBase(BaseMySQLTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_MYSQL
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -54,4 +54,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class MySQLDataApiTestBase(MySQLDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py

@@ -20,7 +20,7 @@
 
 class OracleConnectionTestBase(BaseOracleTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_ORACLE
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -57,4 +57,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class OracleDataApiTestBase(OracleDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py

@@ -19,7 +19,7 @@
 
 class PostgreSQLConnectionTestBase(BasePostgreSQLTestClass, ConnectionTestBase):
     conn_type = CONNECTION_TYPE_POSTGRES
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
@@ -54,4 +54,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class PostgreSQLDataApiTestBase(PostgreSQLDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_promql/dl_connector_promql_tests/db/api/base.py

@@ -20,7 +20,7 @@
 
 
 class PromQLConnectionTestBase(ConnectionTestBase):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
     conn_type = CONNECTION_TYPE_PROMQL
 
     @pytest.fixture(scope="class")

lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/api/base.py

@@ -63,7 +63,7 @@ def dataset_params(self, sf_secrets) -> dict:
 
 
 class SnowFlakeDataApiTestBase(SnowFlakeDatasetTestBase, StandardizedDataApiTestBase):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:

lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py

@@ -30,7 +30,7 @@
 
 
 class YDBConnectionTestBase(ConnectionTestBase):
-    bi_compeng_pg_on = False
+    compeng_enabled = False
     conn_type = CONNECTION_TYPE_YDB
 
     @pytest.fixture(scope="class")
@@ -77,7 +77,7 @@ def dataset_params(self, sample_table: DbTable) -> dict:
 
 
 class YDBDataApiTestBase(YDBDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams: