Added a compeng cache test to dl_core and an api cache test to dl_api…

…_lib

lib/dl_api_lib/dl_api_lib_tests/db/config.py

@@ -18,6 +18,8 @@
         f"postgresql://us:us@"
         f'{get_test_container_hostport("db-postgres", fallback_port=52509).as_pair()}/us-db-ci_purgeable'
     ),
+    redis_host=get_test_container_hostport("redis-caches").host,
+    redis_port=get_test_container_hostport("redis-caches", fallback_port=52505).port,
 )
 
 

lib/dl_api_lib/dl_api_lib_tests/db/data_api/caches/test_caches.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from dl_api_client.dsmaker.shortcuts.dataset import add_formulas_to_dataset
+from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows
+from dl_api_lib_tests.db.base import DefaultApiTestBase
+
+
+class TestDataCaches(DefaultApiTestBase):
+    data_caches_enabled = True
+
+    def test_cache_by_deleting_table(self, saved_dataset, control_api, data_api, sample_table):
+        ds = saved_dataset
+        ds = add_formulas_to_dataset(dataset=ds, api_v1=control_api, formulas={"sales sum": "SUM([sales])"})
+
+        def get_data():
+            result_resp = data_api.get_result(
+                dataset=ds,
+                fields=[
+                    ds.find_field(title="city"),
+                    ds.find_field(title="sales sum"),
+                ],
+            )
+            assert result_resp.status_code == 200, result_resp.response_errors
+            data = get_data_rows(response=result_resp)
+            return data
+
+        data_rows = get_data()
+
+        # Now delete the table.
+        # This will make real DB queries impossible,
+        # however the cache should still return the same data
+        sample_table.db.drop_table(sample_table.table)
+
+        data_rows_after_deletion = get_data()
+        assert data_rows_after_deletion == data_rows

lib/dl_api_lib/docker-compose.yml

@@ -4,6 +4,14 @@ x-constants:
   US_MASTER_TOKEN: &c-us-master-token "AC1ofiek8coB"
 
 services:
+  redis-caches:
+    # image: "bitnami/redis:5.0.8"
+    image: "bitnami/redis:5.0.8@sha256:3127620da977815556439a9dc347fff89432a79b6bb6e93a16f20ac4a34ce337"
+    environment:
+      ALLOW_EMPTY_PASSWORD: "yes"
+    ports:
+     - "52505:6379"
+
   db-clickhouse:
     ports:
      - "52510:8123"

lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py

@@ -45,8 +45,8 @@ class DataApiTestParams(NamedTuple):
 
 
 class DataApiTestBase(ApiTestBase, metaclass=abc.ABCMeta):
-    mutation_caches_on: ClassVar[bool] = True
-    data_caches_on: ClassVar[bool] = True
+    mutation_caches_enabled: ClassVar[bool] = True
+    data_caches_enabled: ClassVar[bool] = True
 
     @pytest.fixture
     def loop(self, event_loop: asyncio.AbstractEventLoop) -> Generator[asyncio.AbstractEventLoop, None, None]:
@@ -72,12 +72,12 @@ def create_data_api_settings(
             US_MASTER_TOKEN=us_config.us_master_token,
             CRYPTO_KEYS_CONFIG=core_test_config.get_crypto_keys_config(),
             # TODO FIX: Configure caches
-            CACHES_ON=cls.data_caches_on,
+            CACHES_ON=cls.data_caches_enabled,
             SAMPLES_CH_HOSTS=(),
             RQE_CONFIG=rqe_config_subprocess,
             BI_API_CONNECTOR_WHITELIST=bi_test_config.get_api_library_config().api_connector_ep_names,
             CORE_CONNECTOR_WHITELIST=core_test_config.get_core_library_config().core_connector_ep_names,
-            MUTATIONS_CACHES_ON=cls.mutation_caches_on,
+            MUTATIONS_CACHES_ON=cls.mutation_caches_enabled,
             CACHES_REDIS=redis_setting_maker.get_redis_settings_cache(),
             BI_COMPENG_PG_ON=cls.compeng_enabled,
             BI_COMPENG_PG_URL=core_test_config.get_compeng_url(),

lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py

@@ -42,4 +42,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class BigQueryDataApiTestBase(BigQueryDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/api/base.py

@@ -95,7 +95,7 @@ def dataset_params(self) -> dict:
 
 
 class BitrixDataApiTestBase(BitrixDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:
@@ -109,4 +109,4 @@ def data_api_test_params(self) -> DataApiTestParams:
 
 
 class BitrixSmartTablesDataApiTestBase(BitrixSmartTablesDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/data.py

@@ -24,7 +24,7 @@
 
 
 class CHS3DataApiTestBase(CHS3DatasetTestBase[FILE_CONN_TV], StandardizedDataApiTestBase, metaclass=abc.ABCMeta):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
 
 class CHS3DataResultTestSuite(

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py

@@ -52,4 +52,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class ClickHouseDataApiTestBase(ClickHouseDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py

@@ -57,4 +57,4 @@ def dataset_params(self, sample_table: DbTable) -> dict:
 
 
 class GreenplumDataApiTestBase(GreenplumDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py

@@ -56,7 +56,7 @@ def dataset_params(self) -> dict:
 
 
 class MetricaDataApiTestBase(MetricaDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:

lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py

@@ -55,4 +55,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class MSSQLDataApiTestBase(MSSQLDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py

@@ -54,4 +54,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class MySQLDataApiTestBase(MySQLDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py

@@ -57,4 +57,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class OracleDataApiTestBase(OracleDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py

@@ -54,4 +54,4 @@ def dataset_params(self, sample_table) -> dict:
 
 
 class PostgreSQLDataApiTestBase(PostgreSQLDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False

lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py

@@ -77,7 +77,7 @@ def dataset_params(self, sample_table: DbTable) -> dict:
 
 
 class YDBDataApiTestBase(YDBDatasetTestBase, StandardizedDataApiTestBase):
-    mutation_caches_on = False
+    mutation_caches_enabled = False
 
     @pytest.fixture(scope="class")
     def data_api_test_params(self) -> DataApiTestParams:

lib/dl_core/dl_core_tests/db/compeng/test_compeng_cache.py

@@ -47,7 +47,7 @@
 
 
 class TestCompengCache(DefaultCoreTestClass):
-    caches_enabled = True
+    data_caches_enabled = True
     compeng_enabled = True
 
     @pytest.mark.asyncio

lib/dl_core_testing/dl_core_testing/testcases/service_base.py

@@ -61,7 +61,7 @@ class ServiceFixtureTextClass(metaclass=abc.ABCMeta):
     core_test_config: ClassVar[CoreTestEnvironmentConfigurationBase]
     connection_settings: ClassVar[Optional[ConnectorSettingsBase]] = None
     inst_specific_sr_factory: ClassVar[Optional[InstallationSpecificServiceRegistryFactory]] = None
-    caches_enabled: ClassVar[bool] = False
+    data_caches_enabled: ClassVar[bool] = False
     compeng_enabled: ClassVar[bool] = False
 
     @pytest.fixture(scope="session")
@@ -98,7 +98,7 @@ async def _make_redis(self, redis_settings: RedisSettings) -> AsyncGenerator[Red
 
     @pytest.fixture(scope="function")
     async def caches_redis_client_factory(self) -> Optional[Callable[[bool], Redis]]:
-        if not self.caches_enabled:
+        if not self.data_caches_enabled:
             yield None
 
         else: