diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/config.py b/lib/dl_api_lib/dl_api_lib_tests/db/config.py index d056b3a98..84c30bc6b 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/db/config.py +++ b/lib/dl_api_lib/dl_api_lib_tests/db/config.py @@ -7,13 +7,13 @@ SectionSettings, TranslatableSettings, ) -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_clickhouse.formula.constants import ClickHouseDialect as D -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=52500).host, port_us_http=get_test_container_hostport("us", fallback_port=52500).port, host_us_pg=get_test_container_hostport("db-postgres", fallback_port=52509).host, diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py index 7be36fc7a..eed4961c1 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py @@ -9,13 +9,13 @@ from dl_api_lib.loader import ApiLibraryConfig from dl_configs.connector_availability import ConnectorAvailabilityConfigSettings -from dl_core_testing.configuration import CoreTestEnvironmentConfigurationBase +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_formula_testing.configuration import FormulaTestEnvironmentConfiguration @attr.s(kw_only=True) class ApiTestEnvironmentConfiguration: - core_test_config: CoreTestEnvironmentConfigurationBase = attr.ib(factory=CoreTestEnvironmentConfigurationBase) + core_test_config: CoreTestEnvironmentConfiguration = attr.ib() formula_test_config: FormulaTestEnvironmentConfiguration = attr.ib(factory=FormulaTestEnvironmentConfiguration) ext_query_executer_secret_key: str = attr.ib() diff --git a/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/config.py b/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/config.py index e2b94ea78..42fd4386b 100644 --- a/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/config.py +++ b/lib/dl_connector_bigquery/dl_connector_bigquery_tests/ext/config.py @@ -1,10 +1,10 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51600).host, port_us_http=get_test_container_hostport("us", fallback_port=51600).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51609).host, diff --git a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py index b2f9f9f87..da406f830 100644 --- a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py +++ b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds_tests/ext/config.py @@ -1,12 +1,12 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport COMPENG_URL = f'postgresql://datalens:qwerty@{get_test_container_hostport("db-postgres-13", fallback_port=52301).as_pair()}/test_data' # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51911).host, port_us_http=get_test_container_hostport("us", fallback_port=51911).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51910).host, diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py index df42bcf2d..62483ab2a 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py @@ -1,11 +1,11 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_bundle_chs3.chs3_base.core.settings import FileS3ConnectorSettings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=52611).host, port_us_http=get_test_container_hostport("us", fallback_port=52611).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=52610).host, diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py index df1116e82..13a984480 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py @@ -2,7 +2,7 @@ from typing import ClassVar from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_clickhouse.formula.constants import ClickHouseDialect as D @@ -23,7 +23,7 @@ def get_clickhouse_ssl_ca() -> str: return f.read() -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=52211).host, port_us_http=get_test_container_hostport("us", fallback_port=52211).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=52210).host, diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_remote_query_executor.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_remote_query_executor.py new file mode 100644 index 000000000..3455c2f15 --- /dev/null +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_remote_query_executor.py @@ -0,0 +1,16 @@ +import pytest + +from dl_core_testing.testcases.remote_query_executor import BaseRemoteQueryExecutorTestClass + +from dl_connector_clickhouse.core.clickhouse_base.adapters import ClickHouseAdapter +from dl_connector_clickhouse_tests.db.config import DASHSQL_QUERY_FULL +from dl_connector_clickhouse_tests.db.core.base import BaseClickHouseTestClass + + +class TestClickHouseRemoteQueryExecutor(BaseClickHouseTestClass, BaseRemoteQueryExecutorTestClass): + ADAPTER_CLS = ClickHouseAdapter + + @pytest.mark.asyncio + async def test_qe_result(self, remote_adapter): + result = await self.execute_request(remote_adapter, query=DASHSQL_QUERY_FULL) + assert len(result) == 7 diff --git a/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/config.py b/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/config.py index e1c265760..090319760 100644 --- a/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/config.py +++ b/lib/dl_connector_greenplum/dl_connector_greenplum_tests/db/config.py @@ -1,10 +1,10 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51911).host, port_us_http=get_test_container_hostport("us", fallback_port=51911).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51910).host, diff --git a/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/config.py b/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/config.py index 23d3d29b1..3c202647d 100644 --- a/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/config.py +++ b/lib/dl_connector_metrica/dl_connector_metrica_tests/ext/config.py @@ -1,5 +1,5 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport @@ -7,7 +7,7 @@ APPMETRICA_SAMPLE_COUNTER_ID = "1111" # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51911).host, port_us_http=get_test_container_hostport("us", fallback_port=51911).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51910).host, diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py index a7b4fb259..e60fcf4cd 100644 --- a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py @@ -2,14 +2,14 @@ from typing import ClassVar from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_mssql.formula.constants import MssqlDialect as D # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=52111).host, port_us_http=get_test_container_hostport("us", fallback_port=52111).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=52110).host, diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_remote_query_executor.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_remote_query_executor.py new file mode 100644 index 000000000..92f4f4b44 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_remote_query_executor.py @@ -0,0 +1,16 @@ +import pytest + +from dl_core_testing.testcases.remote_query_executor import BaseRemoteQueryExecutorTestClass + +from dl_connector_mssql.core.adapters_mssql import MSSQLDefaultAdapter +from dl_connector_mssql_tests.db.config import SUBSELECT_QUERY_FULL +from dl_connector_mssql_tests.db.core.base import BaseMSSQLTestClass + + +class TestMSSQLRemoteQueryExecutor(BaseMSSQLTestClass, BaseRemoteQueryExecutorTestClass): + ADAPTER_CLS = MSSQLDefaultAdapter + + @pytest.mark.asyncio + async def test_qe_result(self, remote_adapter): + result = await self.execute_request(remote_adapter, query=SUBSELECT_QUERY_FULL) + assert len(result) == 3 diff --git a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/config.py b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/config.py index 2a6694069..18ab48971 100644 --- a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/config.py +++ b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/config.py @@ -1,14 +1,14 @@ from typing import ClassVar from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_mysql.formula.constants import MySQLDialect as D # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=52011).host, port_us_http=get_test_container_hostport("us", fallback_port=52011).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=52010).host, diff --git a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_remote_query_executor.py b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_remote_query_executor.py new file mode 100644 index 000000000..bcc5d6305 --- /dev/null +++ b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_remote_query_executor.py @@ -0,0 +1,16 @@ +import pytest + +from dl_core_testing.testcases.remote_query_executor import BaseRemoteQueryExecutorTestClass + +from dl_connector_mysql.core.adapters_mysql import MySQLAdapter +from dl_connector_mysql_tests.db.config import SUBSELECT_QUERY_FULL +from dl_connector_mysql_tests.db.core.base import BaseMySQLTestClass + + +class TestMySQLRemoteQueryExecutor(BaseMySQLTestClass, BaseRemoteQueryExecutorTestClass): + ADAPTER_CLS = MySQLAdapter + + @pytest.mark.asyncio + async def test_qe_result(self, remote_adapter): + result = await self.execute_request(remote_adapter, query=SUBSELECT_QUERY_FULL) + assert len(result) == 3 diff --git a/lib/dl_connector_oracle/dl_connector_oracle_tests/db/config.py b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/config.py index 1a1324e93..0f883289e 100644 --- a/lib/dl_connector_oracle/dl_connector_oracle_tests/db/config.py +++ b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/config.py @@ -2,14 +2,14 @@ from typing import ClassVar from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_oracle.formula.constants import OracleDialect as D # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51811).host, port_us_http=get_test_container_hostport("us", fallback_port=51811).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51810).host, diff --git a/lib/dl_connector_oracle/dl_connector_oracle_tests/db/core/test_remote_query_executor.py b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/core/test_remote_query_executor.py new file mode 100644 index 000000000..27100b266 --- /dev/null +++ b/lib/dl_connector_oracle/dl_connector_oracle_tests/db/core/test_remote_query_executor.py @@ -0,0 +1,16 @@ +import pytest + +from dl_core_testing.testcases.remote_query_executor import BaseRemoteQueryExecutorTestClass + +from dl_connector_oracle.core.adapters_oracle import OracleDefaultAdapter +from dl_connector_oracle_tests.db.config import SUBSELECT_QUERY_FULL +from dl_connector_oracle_tests.db.core.base import BaseOracleTestClass + + +class TestOracleRemoteQueryExecutor(BaseOracleTestClass, BaseRemoteQueryExecutorTestClass): + ADAPTER_CLS = OracleDefaultAdapter + + @pytest.mark.asyncio + async def test_qe_result(self, remote_adapter): + result = await self.execute_request(remote_adapter, query=SUBSELECT_QUERY_FULL) + assert len(result) == 3 diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/config.py b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/config.py index 15331f600..9cc1b7737 100644 --- a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/config.py +++ b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/config.py @@ -2,14 +2,14 @@ from typing import ClassVar from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_postgresql.formula.constants import PostgreSQLDialect as D # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=52311).host, port_us_http=get_test_container_hostport("us", fallback_port=52311).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=52310).host, diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_remote_query_executor.py b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_remote_query_executor.py new file mode 100644 index 000000000..95bf2292f --- /dev/null +++ b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_remote_query_executor.py @@ -0,0 +1,16 @@ +import pytest + +from dl_core_testing.testcases.remote_query_executor import BaseRemoteQueryExecutorTestClass + +from dl_connector_postgresql.core.postgresql_base.adapters_postgres import PostgresAdapter +from dl_connector_postgresql_tests.db.config import SUBSELECT_QUERY_FULL +from dl_connector_postgresql_tests.db.core.base import BasePostgreSQLTestClass + + +class TestPostgreSQLRemoteQueryExecutor(BasePostgreSQLTestClass, BaseRemoteQueryExecutorTestClass): + ADAPTER_CLS = PostgresAdapter + + @pytest.mark.asyncio + async def test_qe_result(self, remote_adapter): + result = await self.execute_request(remote_adapter, query=SUBSELECT_QUERY_FULL) + assert len(result) == 7 diff --git a/lib/dl_connector_promql/dl_connector_promql_tests/db/config.py b/lib/dl_connector_promql/dl_connector_promql_tests/db/config.py index 3d26bf6e3..521cd5daf 100644 --- a/lib/dl_connector_promql/dl_connector_promql_tests/db/config.py +++ b/lib/dl_connector_promql/dl_connector_promql_tests/db/config.py @@ -1,10 +1,10 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51911).host, port_us_http=get_test_container_hostport("us", fallback_port=51911).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51910).host, diff --git a/lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/config.py b/lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/config.py index f9c811e17..742d3b2f6 100644 --- a/lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/config.py +++ b/lib/dl_connector_snowflake/dl_connector_snowflake_tests/ext/config.py @@ -1,11 +1,11 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration from dl_constants.enums import UserDataType -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51700).host, port_us_http=get_test_container_hostport("us", fallback_port=51700).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51709).host, diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py index 92825577a..f4582da8a 100644 --- a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py @@ -2,14 +2,14 @@ from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration from dl_constants.enums import UserDataType -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport from dl_connector_ydb.formula.constants import YqlDialect as D # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=51911).host, port_us_http=get_test_container_hostport("us", fallback_port=51911).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=51910).host, diff --git a/lib/dl_core/dl_core_tests/db/components/test_rqe.py b/lib/dl_core/dl_core_tests/db/components/test_rqe.py index dc5e955ac..dd09554f4 100644 --- a/lib/dl_core/dl_core_tests/db/components/test_rqe.py +++ b/lib/dl_core/dl_core_tests/db/components/test_rqe.py @@ -2,75 +2,21 @@ import pytest from dl_app_tools.log.context import log_context -from dl_configs.rqe import RQEExecuteRequestMode -from dl_core.connection_executors.adapters.async_adapters_remote import RemoteAsyncAdapter -from dl_core.connection_executors.models.common import RemoteQueryExecutorData -from dl_core.connection_executors.models.db_adapter_data import DBAdapterQuery from dl_core.connection_executors.models.exc import QueryExecutorException -from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI -from dl_core.connection_executors.remote_query_executor.app_async import create_async_qe_app -from dl_core_testing.rqe import RQEConfigurationMaker +from dl_core_testing.testcases.remote_query_executor import BaseRemoteQueryExecutorTestClass from dl_core_tests.db.base import DefaultCoreTestClass from dl_connector_clickhouse.core.clickhouse_base.adapters import ClickHouseAdapter -EXT_QUERY_EXECUTER_SECRET_KEY = "very_secret_key" - - -class TestRQE(DefaultCoreTestClass): - @pytest.fixture(scope="function") - def query_executor_app(self, loop, aiohttp_client): - app = create_async_qe_app(hmac_key=EXT_QUERY_EXECUTER_SECRET_KEY.encode()) - return loop.run_until_complete(aiohttp_client(app)) - - @pytest.fixture(scope="function") - def sync_rqe_netloc_subprocess(self): - with RQEConfigurationMaker( - ext_query_executer_secret_key=EXT_QUERY_EXECUTER_SECRET_KEY, - core_connector_whitelist=self.core_test_config.core_connector_ep_names, - ).sync_rqe_netloc_subprocess_cm() as sync_rqe_config: - yield sync_rqe_config - - @pytest.fixture(scope="function", params=[RQEExecuteRequestMode.STREAM, RQEExecuteRequestMode.NON_STREAM]) - def query_executor_options(self, query_executor_app, sync_rqe_netloc_subprocess, request): - return RemoteQueryExecutorData( - hmac_key=EXT_QUERY_EXECUTER_SECRET_KEY.encode(), - # Async RQE - async_protocol="http", - async_host=query_executor_app.host, - async_port=query_executor_app.port, - # Sync RQE - sync_protocol="http", - sync_host=sync_rqe_netloc_subprocess.host, - sync_port=sync_rqe_netloc_subprocess.port, - execute_request_mode=request.param, - ) - - @pytest.fixture(scope="function") - async def conn_target_dto(self, async_conn_executor_factory): - conn_executor = async_conn_executor_factory() - target_conn_dto_pool = await conn_executor._make_target_conn_dto_pool() - yield next(iter(target_conn_dto_pool)) - await conn_executor.close() - - @pytest.fixture(scope="function", params=[True, False]) - def remote_adapter(self, conn_target_dto, query_executor_options, request): - return RemoteAsyncAdapter( - target_dto=conn_target_dto, - dba_cls=ClickHouseAdapter, - rqe_data=query_executor_options, - req_ctx_info=DBAdapterScopedRCI(), - force_async_rqe=request.param, - ) +class TestRQE(DefaultCoreTestClass, BaseRemoteQueryExecutorTestClass): + ADAPTER_CLS = ClickHouseAdapter @pytest.mark.asyncio async def test_body_signature_validation(self, remote_adapter, query_executor_options): # OK case - async with remote_adapter: - res = await remote_adapter.execute(DBAdapterQuery("select 1")) - async for row in res.raw_chunk_generator: - assert row + result = await self.execute_request(remote_adapter, query="select 1") + assert result == [(1,)] # Not OK case remote_adapter = attr.evolve( @@ -79,7 +25,7 @@ async def test_body_signature_validation(self, remote_adapter, query_executor_op ) async with remote_adapter: with pytest.raises(QueryExecutorException, match=r"Invalid signature"): - await remote_adapter.execute(DBAdapterQuery("select 1")) + await self.execute_request(remote_adapter, query="select 1") @staticmethod def _validate_logging_ctx(record, outer_logging_ctx): @@ -95,9 +41,8 @@ async def test_qe_logging_ctx_propagation(self, remote_adapter, caplog): outer_logging_ctx = dict(some_str_key="some_val", some_int_key=123) with log_context(**outer_logging_ctx): - res = await remote_adapter.execute(DBAdapterQuery("select 1")) - async for row in res.raw_chunk_generator: - assert row + result = await self.execute_request(remote_adapter, query="select 1") + assert result == [(1,)] qe_logs = list( filter(lambda r: r.name == "dl_core.connection_executors.remote_query_executor.app_async", caplog.records) diff --git a/lib/dl_core/dl_core_tests/db/config.py b/lib/dl_core/dl_core_tests/db/config.py index 5748b2607..47eed5ee7 100644 --- a/lib/dl_core/dl_core_tests/db/config.py +++ b/lib/dl_core/dl_core_tests/db/config.py @@ -1,11 +1,11 @@ from typing import ClassVar -from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_testing.containers import get_test_container_hostport # Infra settings -CORE_TEST_CONFIG = DefaultCoreTestConfiguration( +CORE_TEST_CONFIG = CoreTestEnvironmentConfiguration( host_us_http=get_test_container_hostport("us", fallback_port=50300).host, port_us_http=get_test_container_hostport("us", fallback_port=50300).port, host_us_pg=get_test_container_hostport("pg-us", fallback_port=50309).host, diff --git a/lib/dl_core_testing/dl_core_testing/configuration.py b/lib/dl_core_testing/dl_core_testing/configuration.py index a414d5792..d25d8941d 100644 --- a/lib/dl_core_testing/dl_core_testing/configuration.py +++ b/lib/dl_core_testing/dl_core_testing/configuration.py @@ -1,6 +1,5 @@ from __future__ import annotations -import abc import logging from typing import ( Collection, @@ -62,35 +61,12 @@ def get_redis_settings_arq(self) -> RedisSettings: return self.get_redis_settings(self.redis_db_arq) -@attr.s(frozen=True) -class CoreTestEnvironmentConfigurationBase(abc.ABC): - @abc.abstractmethod - def get_us_config(self) -> UnitedStorageConfiguration: - raise NotImplementedError - - @abc.abstractmethod - def get_crypto_keys_config(self) -> CryptoKeysConfig: - raise NotImplementedError - - @abc.abstractmethod - def get_core_library_config(self) -> CoreLibraryConfig: - raise NotImplementedError - - @abc.abstractmethod - def get_redis_setting_maker(self) -> RedisSettingMaker: - raise NotImplementedError - - @abc.abstractmethod - def get_compeng_url(self) -> str: - raise NotImplementedError - - # These are used only for creation of local environments in tests, not actual external ones DEFAULT_FERNET_KEY = "h1ZpilcYLYRdWp7Nk8X1M1kBPiUi8rdjz9oBfHyUKIk=" @attr.s(frozen=True) -class DefaultCoreTestConfiguration(CoreTestEnvironmentConfigurationBase): +class CoreTestEnvironmentConfiguration: host_us_http: str = attr.ib(kw_only=True) port_us_http: int = attr.ib(kw_only=True) host_us_pg: str = attr.ib(kw_only=True) diff --git a/lib/dl_core_testing/dl_core_testing/initialization.py b/lib/dl_core_testing/dl_core_testing/initialization.py index f4650350f..f19a4ed11 100644 --- a/lib/dl_core_testing/dl_core_testing/initialization.py +++ b/lib/dl_core_testing/dl_core_testing/initialization.py @@ -3,12 +3,12 @@ from dl_core.loader import load_core_lib from dl_core.logging_config import add_log_context_scoped -from dl_core_testing.configuration import CoreTestEnvironmentConfigurationBase +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_core_testing.environment import prepare_united_storage_from_config from dl_db_testing.loader import load_db_testing_lib -def initialize_core_test(pytest_config: Config, core_test_config: CoreTestEnvironmentConfigurationBase) -> None: +def initialize_core_test(pytest_config: Config, core_test_config: CoreTestEnvironmentConfiguration) -> None: # Configure pytest itself pytest_config.addinivalue_line("markers", "slow: ...") pytest_config.addinivalue_line("markers", "yt: ...") diff --git a/lib/dl_core_testing/dl_core_testing/testcases/remote_query_executor.py b/lib/dl_core_testing/dl_core_testing/testcases/remote_query_executor.py new file mode 100644 index 000000000..bc924f75e --- /dev/null +++ b/lib/dl_core_testing/dl_core_testing/testcases/remote_query_executor.py @@ -0,0 +1,105 @@ +import asyncio +from typing import ( + AsyncGenerator, + ClassVar, + Generator, + Generic, + Type, + TypeVar, +) + +from aiohttp.pytest_plugin import AiohttpClient +from aiohttp.test_utils import TestClient +import pytest + +from dl_configs.rqe import ( + RQEBaseURL, + RQEExecuteRequestMode, +) +from dl_constants.types import TBIDataRow +from dl_core.connection_executors import DefaultSqlAlchemyConnExecutor +from dl_core.connection_executors.adapters.async_adapters_remote import RemoteAsyncAdapter +from dl_core.connection_executors.adapters.common_base import CommonBaseDirectAdapter +from dl_core.connection_executors.async_base import AsyncConnExecutorBase +from dl_core.connection_executors.models.common import RemoteQueryExecutorData +from dl_core.connection_executors.models.connection_target_dto_base import ConnTargetDTO +from dl_core.connection_executors.models.db_adapter_data import DBAdapterQuery +from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI +from dl_core.connection_executors.remote_query_executor.app_async import create_async_qe_app +from dl_core.us_connection_base import ConnectionBase +from dl_core_testing.rqe import RQEConfigurationMaker +from dl_core_testing.testcases.connection_executor import BaseConnectionExecutorTestClass +from dl_utils.aio import alist + + +_CONN_TV = TypeVar("_CONN_TV", bound=ConnectionBase) + + +class BaseRemoteQueryExecutorTestClass(BaseConnectionExecutorTestClass[_CONN_TV], Generic[_CONN_TV]): + ADAPTER_CLS: ClassVar[Type[CommonBaseDirectAdapter]] + + EXT_QUERY_EXECUTER_SECRET_KEY: ClassVar[str] = "very_secret_key" + + @pytest.fixture(scope="function") + def query_executor_app(self, loop: asyncio.AbstractEventLoop, aiohttp_client: AiohttpClient) -> TestClient: + app = create_async_qe_app(hmac_key=self.EXT_QUERY_EXECUTER_SECRET_KEY.encode()) + return loop.run_until_complete(aiohttp_client(app)) + + @pytest.fixture(scope="function") + def sync_rqe_netloc_subprocess(self) -> Generator[RQEBaseURL, None, None]: + with RQEConfigurationMaker( + ext_query_executer_secret_key=self.EXT_QUERY_EXECUTER_SECRET_KEY, + core_connector_whitelist=self.core_test_config.core_connector_ep_names, + ).sync_rqe_netloc_subprocess_cm() as sync_rqe_config: + yield sync_rqe_config + + @pytest.fixture(scope="function", params=[RQEExecuteRequestMode.STREAM, RQEExecuteRequestMode.NON_STREAM]) + def query_executor_options( + self, + query_executor_app: TestClient, + sync_rqe_netloc_subprocess: RQEBaseURL, + request: pytest.FixtureRequest, + ) -> RemoteQueryExecutorData: + assert query_executor_app.port is not None + return RemoteQueryExecutorData( + hmac_key=self.EXT_QUERY_EXECUTER_SECRET_KEY.encode(), + # Async RQE + async_protocol="http", + async_host=query_executor_app.host, + async_port=query_executor_app.port, + # Sync RQE + sync_protocol="http", + sync_host=sync_rqe_netloc_subprocess.host, + sync_port=sync_rqe_netloc_subprocess.port, + execute_request_mode=request.param, + ) + + @pytest.fixture(scope="function") + async def conn_target_dto( + self, + async_connection_executor: AsyncConnExecutorBase, + ) -> AsyncGenerator[ConnTargetDTO, None]: + assert isinstance(async_connection_executor, DefaultSqlAlchemyConnExecutor) + target_conn_dto_pool = await async_connection_executor._make_target_conn_dto_pool() + yield next(iter(target_conn_dto_pool)) + + @pytest.fixture(scope="function", params=[True, False]) + def remote_adapter( + self, + conn_target_dto: ConnTargetDTO, + query_executor_options: RemoteQueryExecutorData, + request: pytest.FixtureRequest, + ) -> RemoteAsyncAdapter: + return RemoteAsyncAdapter( + target_dto=conn_target_dto, + dba_cls=self.ADAPTER_CLS, + rqe_data=query_executor_options, + req_ctx_info=DBAdapterScopedRCI(), + force_async_rqe=request.param, + ) + + async def execute_request(self, remote_adapter: RemoteAsyncAdapter, query: str) -> list[TBIDataRow]: + async with remote_adapter: + resp = await remote_adapter.execute(DBAdapterQuery(query)) + result = await alist(resp.get_all_rows()) + return result diff --git a/lib/dl_core_testing/dl_core_testing/testcases/service_base.py b/lib/dl_core_testing/dl_core_testing/testcases/service_base.py index afb4017a9..84884c97f 100644 --- a/lib/dl_core_testing/dl_core_testing/testcases/service_base.py +++ b/lib/dl_core_testing/dl_core_testing/testcases/service_base.py @@ -40,7 +40,7 @@ from dl_core.us_manager.us_manager_async import AsyncUSManager from dl_core.us_manager.us_manager_sync import SyncUSManager from dl_core.utils import FutureRef -from dl_core_testing.configuration import CoreTestEnvironmentConfigurationBase +from dl_core_testing.configuration import CoreTestEnvironmentConfiguration from dl_core_testing.database import ( CoreDbConfig, CoreReInitableDbDispenser, @@ -60,7 +60,7 @@ class USConfig(NamedTuple): class ServiceFixtureTextClass(metaclass=abc.ABCMeta): conn_type: ClassVar[ConnectionType] - core_test_config: ClassVar[CoreTestEnvironmentConfigurationBase] + core_test_config: ClassVar[CoreTestEnvironmentConfiguration] connection_settings: ClassVar[Optional[ConnectorSettingsBase]] = None inst_specific_sr_factory: ClassVar[Optional[InstallationSpecificServiceRegistryFactory]] = None data_caches_enabled: ClassVar[bool] = False