BI-4817: add more dashsql tests (#71)

lib/dl_api_lib/dl_api_lib_tests/db/base.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import ClassVar
+
 import pytest
 
 from dl_api_client.dsmaker.primitives import Dataset
@@ -12,6 +14,7 @@
     DB_CORE_URL,
     CoreConnectionSettings,
 )
+from dl_constants.enums import RawSQLLevel
 
 from dl_connector_clickhouse.core.clickhouse.constants import SOURCE_TYPE_CH_TABLE
 from dl_connector_clickhouse.core.clickhouse_base.constants import CONNECTION_TYPE_CLICKHOUSE
@@ -24,6 +27,8 @@ class DefaultApiTestBase(DataApiTestBase, DatasetTestBase, ConnectionTestBase):
     bi_compeng_pg_on = True
     conn_type = CONNECTION_TYPE_CLICKHOUSE
 
+    raw_sql_level: ClassVar[RawSQLLevel] = RawSQLLevel.off
+
     @pytest.fixture(scope="class")
     def bi_test_config(self) -> ApiTestEnvironmentConfiguration:
         return API_TEST_CONFIG
@@ -48,6 +53,7 @@ def connection_params(self) -> dict:
             port=CoreConnectionSettings.PORT,
             username=CoreConnectionSettings.USERNAME,
             password=CoreConnectionSettings.PASSWORD,
+            raw_sql_level=self.raw_sql_level.value,
         )
 
     @pytest.fixture(scope="class")

lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_errors.py

@@ -1,8 +1,11 @@
+import pytest
+
 from dl_api_client.dsmaker.primitives import ResultField
 from dl_api_lib_tests.db.base import DefaultApiTestBase
 from dl_constants.enums import (
     AggregationFunction,
     CalcMode,
+    RawSQLLevel,
 )
 
 
@@ -74,3 +77,58 @@ def test_invalid_group_by_configuration(self, saved_dataset, data_api):
         assert result_resp.status_code == 400
         assert result_resp.bi_status_code == "ERR.DS_API.INVALID_GROUP_BY_CONFIGURATION"
         assert result_resp.json["message"] == "Invalid parameter disable_group_by for dataset with measure fields"
+
+    @pytest.mark.asyncio
+    async def test_disallowed_dashsql(self, data_api_lowlevel_aiohttp_client, saved_connection_id):
+        client = data_api_lowlevel_aiohttp_client
+        conn_id = saved_connection_id
+        req_data = {"sql_query": "select 1, 2, 3"}
+
+        resp = await client.post(f"/api/v1/connections/{conn_id}/dashsql", json=req_data)
+        resp_data = await resp.json()
+        assert resp.status == 400
+        assert resp_data["code"] == "ERR.DS_API.CONNECTION_CONFIG.DASHSQL_NOT_ALLOWED"
+
+
+class TestDashSQLErrors(DefaultApiTestBase):
+    raw_sql_level = RawSQLLevel.dashsql
+
+    @pytest.mark.asyncio
+    async def test_invalid_param_value(self, data_api_lowlevel_aiohttp_client, saved_connection_id):
+        client = data_api_lowlevel_aiohttp_client
+        conn_id = saved_connection_id
+        req_data = {
+            "sql_query": r"SELECT {{date}}",
+            "params": {
+                "date": {
+                    "type_name": "date",
+                    "value": "Invalid date",
+                },
+            },
+        }
+
+        resp = await client.post(f"/api/v1/connections/{conn_id}/dashsql", json=req_data)
+        resp_data = await resp.json()
+        assert resp.status == 400
+        assert resp_data["code"] == "ERR.DS_API.DASHSQL"
+        assert resp_data["message"] == "Unsupported value for type 'date': 'Invalid date'"
+
+    @pytest.mark.asyncio
+    async def test_invalid_param_format(self, data_api_lowlevel_aiohttp_client, saved_connection_id):
+        client = data_api_lowlevel_aiohttp_client
+        conn_id = saved_connection_id
+        req_data = {
+            "sql_query": r"SELECT 'some_{{value}}'",
+            "params": {
+                "value": {
+                    "type_name": "string",
+                    "value": "value",
+                },
+            },
+        }
+
+        resp = await client.post(f"/api/v1/connections/{conn_id}/dashsql", json=req_data)
+        resp_data = await resp.json()
+        assert resp.status == 400
+        assert resp_data["code"] == "ERR.DS_API.DB.WRONG_QUERY_PARAMETERIZATION"
+        assert resp_data["message"] == "Wrong query parameterization. Parameter was not found"

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/base.py

@@ -1,5 +1,7 @@
 import pytest
 
+from dl_constants.enums import RawSQLLevel
+
 from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration
 from dl_api_lib_testing.connection_base import ConnectionTestBase
 from dl_api_lib_testing.data_api_base import StandardizedDataApiTestBase
@@ -30,9 +32,14 @@ def connection_params(self) -> dict:
             port=CoreConnectionSettings.PORT,
             username=CoreConnectionSettings.USERNAME,
             password=CoreConnectionSettings.PASSWORD,
+            **(dict(raw_sql_level=self.raw_sql_level.value) if self.raw_sql_level is not None else {}),
         )
 
 
+class ClickHouseDashSQLConnectionTest(ClickHouseConnectionTestBase):
+    raw_sql_level = RawSQLLevel.dashsql
+
+
 class ClickHouseDatasetTestBase(ClickHouseConnectionTestBase, DatasetTestBase):
     @pytest.fixture(scope="class")
     def dataset_params(self, sample_table) -> dict:

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/api/test_dashsql.py

@@ -0,0 +1,62 @@
+from aiohttp.test_utils import TestClient
+import pytest
+
+from dl_api_lib_testing.connector.dashsql_suite import DefaultDashSQLTestSuite
+
+from dl_connector_clickhouse_tests.db.api.base import ClickHouseDashSQLConnectionTest
+from dl_connector_clickhouse_tests.db.config import (
+    DASHSQL_QUERY,
+    DASHSQL_QUERY_FULL,
+)
+
+
+class TestClickHouseDashSQL(ClickHouseDashSQLConnectionTest, DefaultDashSQLTestSuite):
+    @pytest.mark.asyncio
+    async def test_result(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str):
+        resp = await self.get_dashsql_response(
+            data_api_aio=data_api_lowlevel_aiohttp_client,
+            conn_id=saved_connection_id,
+            query=DASHSQL_QUERY,
+        )
+
+        resp_data = await resp.json()
+        assert resp_data[0]["event"] == "metadata", resp_data
+        assert resp_data[0]["data"]["names"] == ["a", "b", "ts"]
+        assert resp_data[0]["data"]["driver_types"] == ["Nullable(UInt8)", "Array(UInt8)", "Nullable(DateTime('UTC'))"]
+        assert resp_data[0]["data"]["db_types"] == ["uint8", "array", "datetime"]
+        assert resp_data[0]["data"]["bi_types"] == ["integer", "unsupported", "genericdatetime"]
+
+        assert resp_data[0]["data"]["clickhouse_headers"]["X-ClickHouse-Timezone"] == "UTC", resp_data
+        assert resp_data[1] == {"event": "row", "data": [11, [33, 44], "2020-01-02 03:04:16"]}, resp_data
+        assert resp_data[2] == {"event": "row", "data": [22, [33, 44], "2020-01-02 03:04:27"]}, resp_data
+        assert resp_data[-1]["event"] == "footer", resp_data
+        assert isinstance(resp_data[-1]["data"], dict)
+
+    @pytest.mark.asyncio
+    async def test_result_extended(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str):
+        await self.get_dashsql_response(
+            data_api_aio=data_api_lowlevel_aiohttp_client,
+            conn_id=saved_connection_id,
+            query=DASHSQL_QUERY_FULL,
+        )
+
+    @pytest.mark.asyncio
+    async def test_invalid_alias(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str):
+        """
+        Clickhouse doesn't support unicode aliases
+        https://clickhouse.com/docs/en/sql-reference/syntax#identifiers
+        """
+
+        resp = await self.get_dashsql_response(
+            data_api_aio=data_api_lowlevel_aiohttp_client,
+            conn_id=saved_connection_id,
+            query="SELECT 1 AS русский текст",
+            fail_ok=True,
+        )
+
+        assert resp.status == 400
+        resp_data = await resp.json()
+        assert resp_data["code"] == "ERR.DS_API.DB.INVALID_QUERY"
+        assert resp_data["message"] == "Invalid SQL query to the database."
+        assert "Unrecognized token" in resp_data["details"]["db_message"]
+        assert "русский текст" in resp_data["details"]["db_message"]

lib/dl_connector_clickhouse/dl_connector_clickhouse_tests/db/config.py

@@ -50,6 +50,47 @@ class CoreSslConnectionSettings:
     PASSWORD: ClassVar[str] = "qwerty"
 
 
+DASHSQL_QUERY = r"""
+select
+    arrayJoin([11, 22, NULL]) as a,
+    [33, 44] as b,
+    toDateTime('2020-01-02 03:04:05', 'UTC') + a as ts
+"""
+DASHSQL_QUERY_FULL = r"""
+select
+    arrayJoin(range(7)) as number,
+    'test' || toString(number) as str,
+    cast(number as Int64) as num_int64,
+    cast(number as Int32) as num_int32,
+    cast(number as Int16) as num_int16,
+    cast(number as Int8) as num_int8,
+    cast(number as UInt64) as num_uint64,
+    cast(number as UInt32) as num_uint32,
+    cast(number as UInt16) as num_uint16,
+    cast(number as Nullable(UInt8)) as num_uint8_n,
+    cast(number as Nullable(Date)) as num_date,
+    cast(number as Nullable(DateTime)) as num_datetime,
+    cast(number as Float64) as num_float64,
+    cast(number as Nullable(Float32)) as num_float32_n,
+    cast(number as Decimal(3, 3)) as num_decimal,
+    cast(number as String) as num_string,
+    cast('bcc3de04-d31a-4e17-8485-8ef423f646be' as UUID) as num_uuid,
+    cast(number as IPv4) as num_ipv4,
+    cast('20:43:ff::40:1bc' as IPv6) as num_ipv6,
+    cast(toString(number) as FixedString(10)) as num_fixedstring,
+    cast(number as Enum8('a'=0, 'b'=1, 'c'=2, 'd'=3, 'e'=4, 'f'=5, 'g'=6)) as num_enum8,
+    cast(number as Enum16('a'=0, 'b'=1, 'c'=2, 'd'=3, 'e'=4, 'f'=5, 'g'=6)) as num_enum16,
+    (number, 'x') as num_tuple,
+    [number, -2] as num_intarray,
+    [toString(number), '-2'] as num_strarray,
+    cast(toString(number) as LowCardinality(Nullable(String))) as num_lc,
+    cast(number as DateTime('Pacific/Chatham')) as num_dt_tz,
+    cast(number as DateTime64(6)) as num_dt64,
+    cast(number as DateTime64(6, 'America/New_York')) as num_dt64_tz
+limit 10
+"""
+
+
 DB_URLS = {
     D.CLICKHOUSE_21_8: f"clickhouse://datalens:qwerty@"
     f'{get_test_container_hostport("db-clickhouse-21-8", fallback_port=52202).as_pair()}/test_data',