diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/config.py b/lib/dl_api_lib/dl_api_lib_tests/db/config.py index 2dd5670e3..c8983927a 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/db/config.py +++ b/lib/dl_api_lib/dl_api_lib_tests/db/config.py @@ -18,6 +18,8 @@ f"postgresql://us:us@" f'{get_test_container_hostport("db-postgres", fallback_port=52509).as_pair()}/us-db-ci_purgeable' ), + redis_host=get_test_container_hostport("redis-caches").host, + redis_port=get_test_container_hostport("redis-caches", fallback_port=52505).port, ) diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/caches/test_caches.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/caches/test_caches.py new file mode 100644 index 000000000..936455a34 --- /dev/null +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/caches/test_caches.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from dl_api_client.dsmaker.shortcuts.dataset import add_formulas_to_dataset +from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows +from dl_api_lib_tests.db.base import DefaultApiTestBase + + +class TestDataCaches(DefaultApiTestBase): + data_caches_enabled = True + + def test_cache_by_deleting_table(self, saved_dataset, control_api, data_api, sample_table): + ds = saved_dataset + ds = add_formulas_to_dataset(dataset=ds, api_v1=control_api, formulas={"sales sum": "SUM([sales])"}) + + def get_data(): + result_resp = data_api.get_result( + dataset=ds, + fields=[ + ds.find_field(title="city"), + ds.find_field(title="sales sum"), + ], + ) + assert result_resp.status_code == 200, result_resp.response_errors + data = get_data_rows(response=result_resp) + return data + + data_rows = get_data() + + # Now delete the table. + # This will make real DB queries impossible, + # however the cache should still return the same data + sample_table.db.drop_table(sample_table.table) + + data_rows_after_deletion = get_data() + assert data_rows_after_deletion == data_rows diff --git a/lib/dl_api_lib/docker-compose.yml b/lib/dl_api_lib/docker-compose.yml index ef2d3427f..42119d82f 100644 --- a/lib/dl_api_lib/docker-compose.yml +++ b/lib/dl_api_lib/docker-compose.yml @@ -4,6 +4,14 @@ x-constants: US_MASTER_TOKEN: &c-us-master-token "AC1ofiek8coB" services: + redis-caches: + # image: "bitnami/redis:5.0.8" + image: "bitnami/redis:5.0.8@sha256:3127620da977815556439a9dc347fff89432a79b6bb6e93a16f20ac4a34ce337" + environment: + ALLOW_EMPTY_PASSWORD: "yes" + ports: + - "52505:6379" + db-clickhouse: ports: - "52510:8123" diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py index f104cb4fc..ca6128c40 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py @@ -45,8 +45,8 @@ class DataApiTestParams(NamedTuple): class DataApiTestBase(ApiTestBase, metaclass=abc.ABCMeta): - mutation_caches_on: ClassVar[bool] = True - data_caches_on: ClassVar[bool] = True + mutation_caches_enabled: ClassVar[bool] = True + data_caches_enabled: ClassVar[bool] = True @pytest.fixture def loop(self, event_loop: asyncio.AbstractEventLoop) -> Generator[asyncio.AbstractEventLoop, None, None]: @@ -72,12 +72,12 @@ def create_data_api_settings( US_MASTER_TOKEN=us_config.us_master_token, CRYPTO_KEYS_CONFIG=core_test_config.get_crypto_keys_config(), # TODO FIX: Configure caches - CACHES_ON=cls.data_caches_on, + CACHES_ON=cls.data_caches_enabled, SAMPLES_CH_HOSTS=(), RQE_CONFIG=rqe_config_subprocess, BI_API_CONNECTOR_WHITELIST=bi_test_config.get_api_library_config().api_connector_ep_names, CORE_CONNECTOR_WHITELIST=core_test_config.get_core_library_config().core_connector_ep_names, - MUTATIONS_CACHES_ON=cls.mutation_caches_on, + MUTATIONS_CACHES_ON=cls.mutation_caches_enabled, CACHES_REDIS=redis_setting_maker.get_redis_settings_cache(), BI_COMPENG_PG_ON=cls.compeng_enabled, BI_COMPENG_PG_URL=core_test_config.get_compeng_url(), diff --git a/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py b/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py index 838744ee4..7a25d8d1e 100644 --- a/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py +++ b/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/api/base.py @@ -42,4 +42,4 @@ def dataset_params(self, sample_table) -> dict: class BigQueryDataApiTestBase(BigQueryDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/api/base.py b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/api/base.py index 399f84924..0bc46df8e 100644 --- a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/api/base.py +++ b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/api/base.py @@ -95,7 +95,7 @@ def dataset_params(self) -> dict: class BitrixDataApiTestBase(BitrixDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False @pytest.fixture(scope="class") def data_api_test_params(self) -> DataApiTestParams: @@ -109,4 +109,4 @@ def data_api_test_params(self) -> DataApiTestParams: class BitrixSmartTablesDataApiTestBase(BitrixSmartTablesDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/data.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/data.py index a7fd3ac24..c26fc11c2 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/data.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/api/data.py @@ -24,7 +24,7 @@ class CHS3DataApiTestBase(CHS3DatasetTestBase[FILE_CONN_TV], StandardizedDataApiTestBase, metaclass=abc.ABCMeta): - mutation_caches_on = False + mutation_caches_enabled = False class CHS3DataResultTestSuite( diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py index 49325561c..f73ed12c9 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py @@ -52,4 +52,4 @@ def dataset_params(self, sample_table) -> dict: class ClickHouseDataApiTestBase(ClickHouseDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py b/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py index 838f8c590..782c00c9e 100644 --- a/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py +++ b/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/api/base.py @@ -57,4 +57,4 @@ def dataset_params(self, sample_table: DbTable) -> dict: class GreenplumDataApiTestBase(GreenplumDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py b/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py index 4baad5ce7..7eaa26cb3 100644 --- a/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py +++ b/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/api/base.py @@ -56,7 +56,7 @@ def dataset_params(self) -> dict: class MetricaDataApiTestBase(MetricaDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False @pytest.fixture(scope="class") def data_api_test_params(self) -> DataApiTestParams: diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py index 9c5cc8ff1..6dce3ff6b 100644 --- a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py @@ -55,4 +55,4 @@ def dataset_params(self, sample_table) -> dict: class MSSQLDataApiTestBase(MSSQLDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py index c41209aca..3ebd5ff53 100644 --- a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py +++ b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/api/base.py @@ -54,4 +54,4 @@ def dataset_params(self, sample_table) -> dict: class MySQLDataApiTestBase(MySQLDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py index f89d77fc4..7e3a180d4 100644 --- a/lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py +++ b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/api/base.py @@ -57,4 +57,4 @@ def dataset_params(self, sample_table) -> dict: class OracleDataApiTestBase(OracleDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py index dd78f253f..c6e33104d 100644 --- a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py +++ b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/api/base.py @@ -54,4 +54,4 @@ def dataset_params(self, sample_table) -> dict: class PostgreSQLDataApiTestBase(PostgreSQLDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py index ebf8bc25a..dbb51cc9a 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py @@ -77,7 +77,7 @@ def dataset_params(self, sample_table: DbTable) -> dict: class YDBDataApiTestBase(YDBDatasetTestBase, StandardizedDataApiTestBase): - mutation_caches_on = False + mutation_caches_enabled = False @pytest.fixture(scope="class") def data_api_test_params(self) -> DataApiTestParams: diff --git a/lib/dl_core/dl_core_tests/db/compeng/test_compeng_cache.py b/lib/dl_core/dl_core_tests/db/compeng/test_compeng_cache.py index 1be3bc2a7..feb5dd9ba 100644 --- a/lib/dl_core/dl_core_tests/db/compeng/test_compeng_cache.py +++ b/lib/dl_core/dl_core_tests/db/compeng/test_compeng_cache.py @@ -47,7 +47,7 @@ class TestCompengCache(DefaultCoreTestClass): - caches_enabled = True + data_caches_enabled = True compeng_enabled = True @pytest.mark.asyncio diff --git a/lib/dl_core_testing/dl_core_testing/testcases/service_base.py b/lib/dl_core_testing/dl_core_testing/testcases/service_base.py index 08ed378d1..12fd5ac6b 100644 --- a/lib/dl_core_testing/dl_core_testing/testcases/service_base.py +++ b/lib/dl_core_testing/dl_core_testing/testcases/service_base.py @@ -61,7 +61,7 @@ class ServiceFixtureTextClass(metaclass=abc.ABCMeta): core_test_config: ClassVar[CoreTestEnvironmentConfigurationBase] connection_settings: ClassVar[Optional[ConnectorSettingsBase]] = None inst_specific_sr_factory: ClassVar[Optional[InstallationSpecificServiceRegistryFactory]] = None - caches_enabled: ClassVar[bool] = False + data_caches_enabled: ClassVar[bool] = False compeng_enabled: ClassVar[bool] = False @pytest.fixture(scope="session") @@ -98,7 +98,7 @@ async def _make_redis(self, redis_settings: RedisSettings) -> AsyncGenerator[Red @pytest.fixture(scope="function") async def caches_redis_client_factory(self) -> Optional[Callable[[bool], Redis]]: - if not self.caches_enabled: + if not self.data_caches_enabled: yield None else: