diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/file/core/test_adapter.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/file/core/test_adapter.py new file mode 100644 index 000000000..d8bdad074 --- /dev/null +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/file/core/test_adapter.py @@ -0,0 +1,12 @@ +from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass + +from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO +from dl_connector_bundle_chs3.file.core.adapter import AsyncFileS3Adapter +from dl_connector_bundle_chs3_tests.db.file.core.base import BaseFileS3TestClass + + +class TestAsyncFileS3Adapter( + BaseFileS3TestClass, + BaseAsyncAdapterTestClass[BaseFileS3ConnTargetDTO], +): + ASYNC_ADAPTER_CLS = AsyncFileS3Adapter diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/gsheets_v2/core/test_adapter.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/gsheets_v2/core/test_adapter.py new file mode 100644 index 000000000..c6b628339 --- /dev/null +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/gsheets_v2/core/test_adapter.py @@ -0,0 +1,12 @@ +from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass + +from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO +from dl_connector_bundle_chs3.chs3_gsheets.core.adapter import AsyncGSheetsFileS3Adapter +from dl_connector_bundle_chs3_tests.db.gsheets_v2.core.base import BaseGSheetsFileS3TestClass + + +class TestAsyncGSheetsFileS3Adapter( + BaseGSheetsFileS3TestClass, + BaseAsyncAdapterTestClass[BaseFileS3ConnTargetDTO], +): + ASYNC_ADAPTER_CLS = AsyncGSheetsFileS3Adapter diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_adapter.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_adapter.py new file mode 100644 index 000000000..273c2495a --- /dev/null +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_adapter.py @@ -0,0 +1,12 @@ +from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass + +from dl_connector_clickhouse.core.clickhouse_base.adapters import AsyncClickHouseAdapter +from dl_connector_clickhouse.core.clickhouse_base.target_dto import ClickHouseConnTargetDTO +from dl_connector_clickhouse_tests.db.core.base import BaseClickHouseTestClass + + +class TestAsyncClickHouseAdapter( + BaseClickHouseTestClass, + BaseAsyncAdapterTestClass[ClickHouseConnTargetDTO], +): + ASYNC_ADAPTER_CLS = AsyncClickHouseAdapter diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/async_adapters_postgres.py b/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/async_adapters_postgres.py index e71e6df05..b60dc7a62 100644 --- a/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/async_adapters_postgres.py +++ b/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/async_adapters_postgres.py @@ -274,7 +274,10 @@ def make_record(raw_rec: asyncpg.Record, query_attrs: Iterable[asyncpg.Attribute # and exclude STRINGS because of our BI ENUMS (it's strings) # in asyncpg we can skip type annotations for strings, it should work like in psycopg compiled_query, params = compile_pg_query(q, self._dialect, exclude_types={DBAPIMock.ENUM, DBAPIMock.STRING}) - debug_query = make_debug_query(compiled_query, params) + debug_query = None + if self._target_dto.pass_db_query_to_user: + debug_query = query.debug_compiled_query or make_debug_query(compiled_query, params) + with self.handle_execution_error(debug_query), self.execution_context(): async with self._get_connection(query.db_name) as conn: # prepare works only inside a transaction diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/error_transformer.py b/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/error_transformer.py index 7162523d9..ab3050f94 100644 --- a/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/error_transformer.py +++ b/lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/error_transformer.py @@ -51,6 +51,7 @@ def make_async_pg_error_transformer() -> DbErrorTransformer: ((asyncpg_exc.FDWTableNotFoundError, None), PostgresSourceDoesNotExistError), # todo: test for this error ((asyncpg_exc.SyntaxOrAccessError, None), exc.DatabaseOperationalError), ((TimeoutError, None), exc.SourceConnectError), + ((OSError, "Connect call failed"), exc.SourceConnectError), ((OSError, "Name or service not known"), exc.SourceHostNotKnownError), ) return ChainedDbErrorTransformer([make_rule_from_descr(d) for d in rule_descriptions]) diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_adapter.py b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_adapter.py new file mode 100644 index 000000000..e0502b61f --- /dev/null +++ b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_adapter.py @@ -0,0 +1,19 @@ +from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass +from dl_testing.regulated_test import RegulatedTestParams + +from dl_connector_postgresql.core.postgresql_base.async_adapters_postgres import AsyncPostgresAdapter +from dl_connector_postgresql.core.postgresql_base.target_dto import PostgresConnTargetDTO +from dl_connector_postgresql_tests.db.core.base import BasePostgreSQLTestClass + + +class TestAsyncPostgreSQLAdapter( + BasePostgreSQLTestClass, + BaseAsyncAdapterTestClass[PostgresConnTargetDTO], +): + test_params = RegulatedTestParams( + mark_tests_skipped={ + BaseAsyncAdapterTestClass.test_default_pass_db_query_to_user: "Not relevant", + }, + ) + + ASYNC_ADAPTER_CLS = AsyncPostgresAdapter diff --git a/lib/dl_core_testing/dl_core_testing/testcases/adapter.py b/lib/dl_core_testing/dl_core_testing/testcases/adapter.py new file mode 100644 index 000000000..fcc345aa3 --- /dev/null +++ b/lib/dl_core_testing/dl_core_testing/testcases/adapter.py @@ -0,0 +1,107 @@ +import abc +from typing import ( + ClassVar, + Generic, + Optional, + Type, + TypeVar, +) + +import pytest + +from dl_api_commons.base_models import RequestContextInfo +from dl_core import exc +from dl_core.connection_executors import AsyncConnExecutorBase +from dl_core.connection_executors.adapters.async_adapters_base import AsyncDirectDBAdapter +from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO +from dl_core.connection_executors.models.db_adapter_data import DBAdapterQuery +from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI +from dl_core_testing.testcases.connection_executor import BaseConnectionExecutorTestClass + + +_TARGET_DTO_TV = TypeVar("_TARGET_DTO_TV", bound=BaseSQLConnTargetDTO) + + +class BaseAsyncAdapterTestClass(BaseConnectionExecutorTestClass, Generic[_TARGET_DTO_TV], metaclass=abc.ABCMeta): + """ + Most of the testable adapter behaviour is covered by conn executor tests + Here are the tests, that are just easier to implement at the adapter level + """ + + ASYNC_ADAPTER_CLS: ClassVar[Type[AsyncDirectDBAdapter]] # TODO add tests for other adapters + + @pytest.fixture + async def target_conn_dto(self, async_connection_executor: AsyncConnExecutorBase) -> _TARGET_DTO_TV: + target_conn_dto_pool = await async_connection_executor._make_target_conn_dto_pool() # noqa + return next(iter(target_conn_dto_pool)) + + def _make_dba(self, target_dto: _TARGET_DTO_TV, rci: RequestContextInfo) -> AsyncDirectDBAdapter: + return self.ASYNC_ADAPTER_CLS.create( + req_ctx_info=DBAdapterScopedRCI.from_full_rci(rci), + target_dto=target_dto, + default_chunk_size=10, + ) + + async def _test_pass_db_query_to_user( + self, + pass_db_query_to_user: Optional[bool], + query_to_send: str, + expected_query: Optional[str], + conn_bi_context: RequestContextInfo, + target_conn_dto: _TARGET_DTO_TV, + ): + debug_query = expected_query + + target_conn_dto = target_conn_dto.clone(port="65535") # not the actual db port to raise connect error + if pass_db_query_to_user is not None: + target_conn_dto = target_conn_dto.clone(pass_db_query_to_user=pass_db_query_to_user) + + dba = self._make_dba(target_conn_dto, conn_bi_context) + + with pytest.raises(exc.SourceConnectError) as exception_info: + await dba.execute(DBAdapterQuery(query=query_to_send, debug_compiled_query=debug_query)) + + assert exception_info.value.query == expected_query, exception_info.value.query + + @pytest.mark.parametrize( + "pass_db_query_to_user, expected_query", + ( + (False, None), + (True, "select 1 from "), + ), + ) + async def test_pass_db_query_to_user( + self, + pass_db_query_to_user: bool, + expected_query: str, + conn_bi_context: RequestContextInfo, + target_conn_dto: _TARGET_DTO_TV, + ) -> None: + query_to_send = "select 1 from very_secret_table" + await self._test_pass_db_query_to_user( + pass_db_query_to_user, query_to_send, expected_query, conn_bi_context, target_conn_dto + ) + + async def test_default_pass_db_query_to_user( + self, + conn_bi_context: RequestContextInfo, + target_conn_dto: _TARGET_DTO_TV, + ) -> None: + await self._test_pass_db_query_to_user( + pass_db_query_to_user=None, + query_to_send="select 1 from very_secret_table", + expected_query=None, + conn_bi_context=conn_bi_context, + target_conn_dto=target_conn_dto, + ) + + async def test_timeout( + self, + conn_bi_context: RequestContextInfo, + target_conn_dto: _TARGET_DTO_TV, + ) -> None: + target_conn_dto = target_conn_dto.clone(port="65535") + dba = self._make_dba(target_conn_dto, conn_bi_context) + + with pytest.raises(exc.SourceConnectError): + await dba.execute(DBAdapterQuery(query="select 1"))