Add async adapter tests (#11)

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/file/core/test_adapter.py

@@ -0,0 +1,12 @@
+from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass
+
+from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO
+from dl_connector_bundle_chs3.file.core.adapter import AsyncFileS3Adapter
+from dl_connector_bundle_chs3_tests.db.file.core.base import BaseFileS3TestClass
+
+
+class TestAsyncFileS3Adapter(
+    BaseFileS3TestClass,
+    BaseAsyncAdapterTestClass[BaseFileS3ConnTargetDTO],
+):
+    ASYNC_ADAPTER_CLS = AsyncFileS3Adapter

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/gsheets_v2/core/test_adapter.py

@@ -0,0 +1,12 @@
+from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass
+
+from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO
+from dl_connector_bundle_chs3.chs3_gsheets.core.adapter import AsyncGSheetsFileS3Adapter
+from dl_connector_bundle_chs3_tests.db.gsheets_v2.core.base import BaseGSheetsFileS3TestClass
+
+
+class TestAsyncGSheetsFileS3Adapter(
+    BaseGSheetsFileS3TestClass,
+    BaseAsyncAdapterTestClass[BaseFileS3ConnTargetDTO],
+):
+    ASYNC_ADAPTER_CLS = AsyncGSheetsFileS3Adapter

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/core/test_adapter.py

@@ -0,0 +1,12 @@
+from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass
+
+from dl_connector_clickhouse.core.clickhouse_base.adapters import AsyncClickHouseAdapter
+from dl_connector_clickhouse.core.clickhouse_base.target_dto import ClickHouseConnTargetDTO
+from dl_connector_clickhouse_tests.db.core.base import BaseClickHouseTestClass
+
+
+class TestAsyncClickHouseAdapter(
+    BaseClickHouseTestClass,
+    BaseAsyncAdapterTestClass[ClickHouseConnTargetDTO],
+):
+    ASYNC_ADAPTER_CLS = AsyncClickHouseAdapter

lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/async_adapters_postgres.py

@@ -274,7 +274,10 @@ def make_record(raw_rec: asyncpg.Record, query_attrs: Iterable[asyncpg.Attribute
         # and exclude STRINGS because of our BI ENUMS (it's strings)
         # in asyncpg we can skip type annotations for strings, it should work like in psycopg
         compiled_query, params = compile_pg_query(q, self._dialect, exclude_types={DBAPIMock.ENUM, DBAPIMock.STRING})
-        debug_query = make_debug_query(compiled_query, params)
+        debug_query = None
+        if self._target_dto.pass_db_query_to_user:
+            debug_query = query.debug_compiled_query or make_debug_query(compiled_query, params)
+
         with self.handle_execution_error(debug_query), self.execution_context():
             async with self._get_connection(query.db_name) as conn:
                 # prepare works only inside a transaction

lib/dl_connector_postgresql/dl_connector_postgresql/core/postgresql_base/error_transformer.py

@@ -51,6 +51,7 @@ def make_async_pg_error_transformer() -> DbErrorTransformer:
         ((asyncpg_exc.FDWTableNotFoundError, None), PostgresSourceDoesNotExistError),  # todo: test for this error
         ((asyncpg_exc.SyntaxOrAccessError, None), exc.DatabaseOperationalError),
         ((TimeoutError, None), exc.SourceConnectError),
+        ((OSError, "Connect call failed"), exc.SourceConnectError),
         ((OSError, "Name or service not known"), exc.SourceHostNotKnownError),
     )
     return ChainedDbErrorTransformer([make_rule_from_descr(d) for d in rule_descriptions])

lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_adapter.py

@@ -0,0 +1,19 @@
+from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass
+from dl_testing.regulated_test import RegulatedTestParams
+
+from dl_connector_postgresql.core.postgresql_base.async_adapters_postgres import AsyncPostgresAdapter
+from dl_connector_postgresql.core.postgresql_base.target_dto import PostgresConnTargetDTO
+from dl_connector_postgresql_tests.db.core.base import BasePostgreSQLTestClass
+
+
+class TestAsyncPostgreSQLAdapter(
+    BasePostgreSQLTestClass,
+    BaseAsyncAdapterTestClass[PostgresConnTargetDTO],
+):
+    test_params = RegulatedTestParams(
+        mark_tests_skipped={
+            BaseAsyncAdapterTestClass.test_default_pass_db_query_to_user: "Not relevant",
+        },
+    )
+
+    ASYNC_ADAPTER_CLS = AsyncPostgresAdapter

lib/dl_core_testing/dl_core_testing/testcases/adapter.py

@@ -0,0 +1,107 @@
+import abc
+from typing import (
+    ClassVar,
+    Generic,
+    Optional,
+    Type,
+    TypeVar,
+)
+
+import pytest
+
+from dl_api_commons.base_models import RequestContextInfo
+from dl_core import exc
+from dl_core.connection_executors import AsyncConnExecutorBase
+from dl_core.connection_executors.adapters.async_adapters_base import AsyncDirectDBAdapter
+from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO
+from dl_core.connection_executors.models.db_adapter_data import DBAdapterQuery
+from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI
+from dl_core_testing.testcases.connection_executor import BaseConnectionExecutorTestClass
+
+
+_TARGET_DTO_TV = TypeVar("_TARGET_DTO_TV", bound=BaseSQLConnTargetDTO)
+
+
+class BaseAsyncAdapterTestClass(BaseConnectionExecutorTestClass, Generic[_TARGET_DTO_TV], metaclass=abc.ABCMeta):
+    """
+    Most of the testable adapter behaviour is covered by conn executor tests
+    Here are the tests, that are just easier to implement at the adapter level
+    """
+
+    ASYNC_ADAPTER_CLS: ClassVar[Type[AsyncDirectDBAdapter]]  # TODO add tests for other adapters
+
+    @pytest.fixture
+    async def target_conn_dto(self, async_connection_executor: AsyncConnExecutorBase) -> _TARGET_DTO_TV:
+        target_conn_dto_pool = await async_connection_executor._make_target_conn_dto_pool()  # noqa
+        return next(iter(target_conn_dto_pool))
+
+    def _make_dba(self, target_dto: _TARGET_DTO_TV, rci: RequestContextInfo) -> AsyncDirectDBAdapter:
+        return self.ASYNC_ADAPTER_CLS.create(
+            req_ctx_info=DBAdapterScopedRCI.from_full_rci(rci),
+            target_dto=target_dto,
+            default_chunk_size=10,
+        )
+
+    async def _test_pass_db_query_to_user(
+        self,
+        pass_db_query_to_user: Optional[bool],
+        query_to_send: str,
+        expected_query: Optional[str],
+        conn_bi_context: RequestContextInfo,
+        target_conn_dto: _TARGET_DTO_TV,
+    ):
+        debug_query = expected_query
+
+        target_conn_dto = target_conn_dto.clone(port="65535")  # not the actual db port to raise connect error
+        if pass_db_query_to_user is not None:
+            target_conn_dto = target_conn_dto.clone(pass_db_query_to_user=pass_db_query_to_user)
+
+        dba = self._make_dba(target_conn_dto, conn_bi_context)
+
+        with pytest.raises(exc.SourceConnectError) as exception_info:
+            await dba.execute(DBAdapterQuery(query=query_to_send, debug_compiled_query=debug_query))
+
+        assert exception_info.value.query == expected_query, exception_info.value.query
+
+    @pytest.mark.parametrize(
+        "pass_db_query_to_user, expected_query",
+        (
+            (False, None),
+            (True, "select 1 from <hidden>"),
+        ),
+    )
+    async def test_pass_db_query_to_user(
+        self,
+        pass_db_query_to_user: bool,
+        expected_query: str,
+        conn_bi_context: RequestContextInfo,
+        target_conn_dto: _TARGET_DTO_TV,
+    ) -> None:
+        query_to_send = "select 1 from very_secret_table"
+        await self._test_pass_db_query_to_user(
+            pass_db_query_to_user, query_to_send, expected_query, conn_bi_context, target_conn_dto
+        )
+
+    async def test_default_pass_db_query_to_user(
+        self,
+        conn_bi_context: RequestContextInfo,
+        target_conn_dto: _TARGET_DTO_TV,
+    ) -> None:
+        await self._test_pass_db_query_to_user(
+            pass_db_query_to_user=None,
+            query_to_send="select 1 from very_secret_table",
+            expected_query=None,
+            conn_bi_context=conn_bi_context,
+            target_conn_dto=target_conn_dto,
+        )
+
+    async def test_timeout(
+        self,
+        conn_bi_context: RequestContextInfo,
+        target_conn_dto: _TARGET_DTO_TV,
+    ) -> None:
+        target_conn_dto = target_conn_dto.clone(port="65535")
+        dba = self._make_dba(target_conn_dto, conn_bi_context)
+
+        with pytest.raises(exc.SourceConnectError):
+            await dba.execute(DBAdapterQuery(query="select 1"))