diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/base.py b/lib/dl_api_lib/dl_api_lib_tests/db/base.py index 5da2df725..1233b7e04 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/db/base.py +++ b/lib/dl_api_lib/dl_api_lib_tests/db/base.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import ClassVar + import pytest from dl_api_client.dsmaker.primitives import Dataset @@ -12,6 +14,7 @@ DB_CORE_URL, CoreConnectionSettings, ) +from dl_constants.enums import RawSQLLevel from dl_connector_clickhouse.core.clickhouse.constants import SOURCE_TYPE_CH_TABLE from dl_connector_clickhouse.core.clickhouse_base.constants import CONNECTION_TYPE_CLICKHOUSE @@ -24,6 +27,8 @@ class DefaultApiTestBase(DataApiTestBase, DatasetTestBase, ConnectionTestBase): bi_compeng_pg_on = True conn_type = CONNECTION_TYPE_CLICKHOUSE + raw_sql_level: ClassVar[RawSQLLevel] = RawSQLLevel.off + @pytest.fixture(scope="class") def bi_test_config(self) -> ApiTestEnvironmentConfiguration: return API_TEST_CONFIG @@ -48,6 +53,7 @@ def connection_params(self) -> dict: port=CoreConnectionSettings.PORT, username=CoreConnectionSettings.USERNAME, password=CoreConnectionSettings.PASSWORD, + raw_sql_level=self.raw_sql_level.value, ) @pytest.fixture(scope="class") diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_errors.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_errors.py index 373432bc0..495529723 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_errors.py +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_errors.py @@ -1,8 +1,11 @@ +import pytest + from dl_api_client.dsmaker.primitives import ResultField from dl_api_lib_tests.db.base import DefaultApiTestBase from dl_constants.enums import ( AggregationFunction, CalcMode, + RawSQLLevel, ) @@ -74,3 +77,58 @@ def test_invalid_group_by_configuration(self, saved_dataset, data_api): assert result_resp.status_code == 400 assert result_resp.bi_status_code == "ERR.DS_API.INVALID_GROUP_BY_CONFIGURATION" assert result_resp.json["message"] == "Invalid parameter disable_group_by for dataset with measure fields" + + @pytest.mark.asyncio + async def test_disallowed_dashsql(self, data_api_lowlevel_aiohttp_client, saved_connection_id): + client = data_api_lowlevel_aiohttp_client + conn_id = saved_connection_id + req_data = {"sql_query": "select 1, 2, 3"} + + resp = await client.post(f"/api/v1/connections/{conn_id}/dashsql", json=req_data) + resp_data = await resp.json() + assert resp.status == 400 + assert resp_data["code"] == "ERR.DS_API.CONNECTION_CONFIG.DASHSQL_NOT_ALLOWED" + + +class TestDashSQLErrors(DefaultApiTestBase): + raw_sql_level = RawSQLLevel.dashsql + + @pytest.mark.asyncio + async def test_invalid_param_value(self, data_api_lowlevel_aiohttp_client, saved_connection_id): + client = data_api_lowlevel_aiohttp_client + conn_id = saved_connection_id + req_data = { + "sql_query": r"SELECT {{date}}", + "params": { + "date": { + "type_name": "date", + "value": "Invalid date", + }, + }, + } + + resp = await client.post(f"/api/v1/connections/{conn_id}/dashsql", json=req_data) + resp_data = await resp.json() + assert resp.status == 400 + assert resp_data["code"] == "ERR.DS_API.DASHSQL" + assert resp_data["message"] == "Unsupported value for type 'date': 'Invalid date'" + + @pytest.mark.asyncio + async def test_invalid_param_format(self, data_api_lowlevel_aiohttp_client, saved_connection_id): + client = data_api_lowlevel_aiohttp_client + conn_id = saved_connection_id + req_data = { + "sql_query": r"SELECT 'some_{{value}}'", + "params": { + "value": { + "type_name": "string", + "value": "value", + }, + }, + } + + resp = await client.post(f"/api/v1/connections/{conn_id}/dashsql", json=req_data) + resp_data = await resp.json() + assert resp.status == 400 + assert resp_data["code"] == "ERR.DS_API.DB.WRONG_QUERY_PARAMETERIZATION" + assert resp_data["message"] == "Wrong query parameterization. Parameter was not found" diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py index 47b713f45..738487fbb 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py @@ -1,5 +1,7 @@ import pytest +from dl_constants.enums import RawSQLLevel + from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration from dl_api_lib_testing.connection_base import ConnectionTestBase from dl_api_lib_testing.data_api_base import StandardizedDataApiTestBase @@ -30,9 +32,14 @@ def connection_params(self) -> dict: port=CoreConnectionSettings.PORT, username=CoreConnectionSettings.USERNAME, password=CoreConnectionSettings.PASSWORD, + **(dict(raw_sql_level=self.raw_sql_level.value) if self.raw_sql_level is not None else {}), ) +class ClickHouseDashSQLConnectionTest(ClickHouseConnectionTestBase): + raw_sql_level = RawSQLLevel.dashsql + + class ClickHouseDatasetTestBase(ClickHouseConnectionTestBase, DatasetTestBase): @pytest.fixture(scope="class") def dataset_params(self, sample_table) -> dict: diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/test_dashsql.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/test_dashsql.py new file mode 100644 index 000000000..4ebade60a --- /dev/null +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/test_dashsql.py @@ -0,0 +1,62 @@ +from aiohttp.test_utils import TestClient +import pytest + +from dl_api_lib_testing.connector.dashsql_suite import DefaultDashSQLTestSuite + +from dl_connector_clickhouse_tests.db.api.base import ClickHouseDashSQLConnectionTest +from dl_connector_clickhouse_tests.db.config import ( + DASHSQL_QUERY, + DASHSQL_QUERY_FULL, +) + + +class TestClickHouseDashSQL(ClickHouseDashSQLConnectionTest, DefaultDashSQLTestSuite): + @pytest.mark.asyncio + async def test_result(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str): + resp = await self.get_dashsql_response( + data_api_aio=data_api_lowlevel_aiohttp_client, + conn_id=saved_connection_id, + query=DASHSQL_QUERY, + ) + + resp_data = await resp.json() + assert resp_data[0]["event"] == "metadata", resp_data + assert resp_data[0]["data"]["names"] == ["a", "b", "ts"] + assert resp_data[0]["data"]["driver_types"] == ["Nullable(UInt8)", "Array(UInt8)", "Nullable(DateTime('UTC'))"] + assert resp_data[0]["data"]["db_types"] == ["uint8", "array", "datetime"] + assert resp_data[0]["data"]["bi_types"] == ["integer", "unsupported", "genericdatetime"] + + assert resp_data[0]["data"]["clickhouse_headers"]["X-ClickHouse-Timezone"] == "UTC", resp_data + assert resp_data[1] == {"event": "row", "data": [11, [33, 44], "2020-01-02 03:04:16"]}, resp_data + assert resp_data[2] == {"event": "row", "data": [22, [33, 44], "2020-01-02 03:04:27"]}, resp_data + assert resp_data[-1]["event"] == "footer", resp_data + assert isinstance(resp_data[-1]["data"], dict) + + @pytest.mark.asyncio + async def test_result_extended(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str): + await self.get_dashsql_response( + data_api_aio=data_api_lowlevel_aiohttp_client, + conn_id=saved_connection_id, + query=DASHSQL_QUERY_FULL, + ) + + @pytest.mark.asyncio + async def test_invalid_alias(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str): + """ + Clickhouse doesn't support unicode aliases + https://clickhouse.com/docs/en/sql-reference/syntax#identifiers + """ + + resp = await self.get_dashsql_response( + data_api_aio=data_api_lowlevel_aiohttp_client, + conn_id=saved_connection_id, + query="SELECT 1 AS русский текст", + fail_ok=True, + ) + + assert resp.status == 400 + resp_data = await resp.json() + assert resp_data["code"] == "ERR.DS_API.DB.INVALID_QUERY" + assert resp_data["message"] == "Invalid SQL query to the database." + assert "Unrecognized token" in resp_data["details"]["db_message"] + assert "русский текст" in resp_data["details"]["db_message"] diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py index 42aa437e2..0c315c0f7 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py @@ -50,6 +50,47 @@ class CoreSslConnectionSettings: PASSWORD: ClassVar[str] = "qwerty" +DASHSQL_QUERY = r""" +select + arrayJoin([11, 22, NULL]) as a, + [33, 44] as b, + toDateTime('2020-01-02 03:04:05', 'UTC') + a as ts +""" +DASHSQL_QUERY_FULL = r""" +select + arrayJoin(range(7)) as number, + 'test' || toString(number) as str, + cast(number as Int64) as num_int64, + cast(number as Int32) as num_int32, + cast(number as Int16) as num_int16, + cast(number as Int8) as num_int8, + cast(number as UInt64) as num_uint64, + cast(number as UInt32) as num_uint32, + cast(number as UInt16) as num_uint16, + cast(number as Nullable(UInt8)) as num_uint8_n, + cast(number as Nullable(Date)) as num_date, + cast(number as Nullable(DateTime)) as num_datetime, + cast(number as Float64) as num_float64, + cast(number as Nullable(Float32)) as num_float32_n, + cast(number as Decimal(3, 3)) as num_decimal, + cast(number as String) as num_string, + cast('bcc3de04-d31a-4e17-8485-8ef423f646be' as UUID) as num_uuid, + cast(number as IPv4) as num_ipv4, + cast('20:43:ff::40:1bc' as IPv6) as num_ipv6, + cast(toString(number) as FixedString(10)) as num_fixedstring, + cast(number as Enum8('a'=0, 'b'=1, 'c'=2, 'd'=3, 'e'=4, 'f'=5, 'g'=6)) as num_enum8, + cast(number as Enum16('a'=0, 'b'=1, 'c'=2, 'd'=3, 'e'=4, 'f'=5, 'g'=6)) as num_enum16, + (number, 'x') as num_tuple, + [number, -2] as num_intarray, + [toString(number), '-2'] as num_strarray, + cast(toString(number) as LowCardinality(Nullable(String))) as num_lc, + cast(number as DateTime('Pacific/Chatham')) as num_dt_tz, + cast(number as DateTime64(6)) as num_dt64, + cast(number as DateTime64(6, 'America/New_York')) as num_dt64_tz +limit 10 +""" + + DB_URLS = { D.CLICKHOUSE_21_8: f"clickhouse://datalens:qwerty@" f'{get_test_container_hostport("db-clickhouse-21-8", fallback_port=52202).as_pair()}/test_data',