Skip to content

Commit

Permalink
Add async adapter tests
Browse files Browse the repository at this point in the history
  • Loading branch information
KonstantAnxiety committed Oct 11, 2023
1 parent 483d11c commit 94255d2
Show file tree
Hide file tree
Showing 7 changed files with 142 additions and 1 deletion.
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass

from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO
from dl_connector_bundle_chs3.file.core.adapter import AsyncFileS3Adapter
from dl_connector_bundle_chs3_tests.db.file.core.base import BaseFileS3TestClass


class TestAsyncFileS3Adapter(
BaseFileS3TestClass,
BaseAsyncAdapterTestClass[BaseFileS3ConnTargetDTO],
):
ASYNC_ADAPTER_CLS = AsyncFileS3Adapter
test_debug_query_with_default_value = True
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass

from dl_connector_bundle_chs3.chs3_base.core.target_dto import BaseFileS3ConnTargetDTO
from dl_connector_bundle_chs3.chs3_gsheets.core.adapter import AsyncGSheetsFileS3Adapter
from dl_connector_bundle_chs3_tests.db.gsheets_v2.core.base import BaseGSheetsFileS3TestClass


class TestAsyncGSheetsFileS3Adapter(
BaseGSheetsFileS3TestClass,
BaseAsyncAdapterTestClass[BaseFileS3ConnTargetDTO],
):
ASYNC_ADAPTER_CLS = AsyncGSheetsFileS3Adapter
test_debug_query_with_default_value = True
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass

from dl_connector_clickhouse.core.clickhouse_base.adapters import AsyncClickHouseAdapter
from dl_connector_clickhouse.core.clickhouse_base.target_dto import ClickHouseConnTargetDTO
from dl_connector_clickhouse_tests.db.core.base import BaseClickHouseTestClass


class TestAsyncClickHouseAdapter(
BaseClickHouseTestClass,
BaseAsyncAdapterTestClass[ClickHouseConnTargetDTO],
):
ASYNC_ADAPTER_CLS = AsyncClickHouseAdapter
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,10 @@ def make_record(raw_rec: asyncpg.Record, query_attrs: Iterable[asyncpg.Attribute
# and exclude STRINGS because of our BI ENUMS (it's strings)
# in asyncpg we can skip type annotations for strings, it should work like in psycopg
compiled_query, params = compile_pg_query(q, self._dialect, exclude_types={DBAPIMock.ENUM, DBAPIMock.STRING})
debug_query = make_debug_query(compiled_query, params)
debug_query = None
if self._target_dto.pass_db_query_to_user:
debug_query = query.debug_compiled_query or make_debug_query(compiled_query, params)

with self.handle_execution_error(debug_query), self.execution_context():
async with self._get_connection(query.db_name) as conn:
# prepare works only inside a transaction
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ def make_async_pg_error_transformer() -> DbErrorTransformer:
((asyncpg_exc.FDWTableNotFoundError, None), PostgresSourceDoesNotExistError), # todo: test for this error
((asyncpg_exc.SyntaxOrAccessError, None), exc.DatabaseOperationalError),
((TimeoutError, None), exc.SourceConnectError),
((OSError, "Connect call failed"), exc.SourceConnectError),
((OSError, "Name or service not known"), exc.SourceHostNotKnownError),
)
return ChainedDbErrorTransformer([make_rule_from_descr(d) for d in rule_descriptions])
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass

from dl_connector_postgresql.core.postgresql_base.async_adapters_postgres import AsyncPostgresAdapter
from dl_connector_postgresql.core.postgresql_base.target_dto import PostgresConnTargetDTO
from dl_connector_postgresql_tests.db.core.base import BasePostgreSQLTestClass


class TestAsyncPostgreSQLAdapter(
BasePostgreSQLTestClass,
BaseAsyncAdapterTestClass[PostgresConnTargetDTO],
):
ASYNC_ADAPTER_CLS = AsyncPostgresAdapter
87 changes: 87 additions & 0 deletions lib/dl_core_testing/dl_core_testing/testcases/adapter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import abc
from typing import (
ClassVar,
Generic,
Type,
TypeVar,
)

import pytest

from dl_api_commons.base_models import RequestContextInfo
from dl_core import exc
from dl_core.connection_executors import AsyncConnExecutorBase
from dl_core.connection_executors.adapters.async_adapters_base import AsyncDirectDBAdapter
from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO
from dl_core.connection_executors.models.db_adapter_data import DBAdapterQuery
from dl_core.connection_executors.models.scoped_rci import DBAdapterScopedRCI
from dl_core_testing.testcases.connection_executor import BaseConnectionExecutorTestClass


_TARGET_DTO_TV = TypeVar("_TARGET_DTO_TV", bound=BaseSQLConnTargetDTO)


class BaseAsyncAdapterTestClass(BaseConnectionExecutorTestClass, Generic[_TARGET_DTO_TV], metaclass=abc.ABCMeta):
"""
Most of the testable adapter behaviour is covered by conn executor tests
Here are the tests, that are just easier to implement at the adapter level
"""

ASYNC_ADAPTER_CLS: ClassVar[Type[AsyncDirectDBAdapter]] # TODO add tests for other adapters

@pytest.fixture
async def target_conn_dto(self, async_connection_executor: AsyncConnExecutorBase) -> _TARGET_DTO_TV:
return next(iter(await async_connection_executor._make_target_conn_dto_pool())) # noqa

def _make_dba(self, target_dto: _TARGET_DTO_TV, rci: RequestContextInfo) -> AsyncDirectDBAdapter:
return self.ASYNC_ADAPTER_CLS.create(
req_ctx_info=DBAdapterScopedRCI.from_full_rci(rci),
target_dto=target_dto,
default_chunk_size=10,
)

test_debug_query_with_default_value: ClassVar[bool] = False

@pytest.mark.parametrize(
"pass_db_query_to_user, expected_query",
(
(None, None),
(False, None),
(True, "select 1 from <hidden>"),
),
)
async def test_pass_db_query_to_user(
self,
pass_db_query_to_user: bool,
expected_query: str,
conn_bi_context: RequestContextInfo,
target_conn_dto: _TARGET_DTO_TV,
) -> None:
if pass_db_query_to_user is None and not self.test_debug_query_with_default_value:
pytest.skip(f"Skipping the test with no value override ({self.test_debug_query_with_default_value=})")

query_to_send = "select 1 from very_secret_table"
debug_query = "select 1 from <hidden>"

target_conn_dto = target_conn_dto.clone(
port="65535", # not the actual db port to raise connect error
pass_db_query_to_user=pass_db_query_to_user,
)

dba = self._make_dba(target_conn_dto, conn_bi_context)

with pytest.raises(exc.SourceConnectError) as exception_info:
await dba.execute(DBAdapterQuery(query=query_to_send, debug_compiled_query=debug_query))

assert exception_info.value.query == expected_query, exception_info.value.query

async def test_timeout(
self,
conn_bi_context: RequestContextInfo,
target_conn_dto: _TARGET_DTO_TV,
) -> None:
target_conn_dto = target_conn_dto.clone(port="65535")
dba = self._make_dba(target_conn_dto, conn_bi_context)

with pytest.raises(exc.SourceConnectError):
await dba.execute(DBAdapterQuery(query="select 1"))

0 comments on commit 94255d2

Please sign in to comment.