Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix tests #357

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

Fix tests #357

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b791e35
fix test enum classes collection by pytest
Dec 22, 2024
a7405bc
allow_experimental_json
Dec 22, 2024
4732612
skip test if a JSON type is unknown
Dec 22, 2024
84831a9
eliminate pytest-asyncio loop scope warning
Dec 22, 2024
9eb1f45
fix test skipping only when code 50 is raised
Dec 22, 2024
1914607
refactor DatabaseException class
Dec 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions tests/types/test_enum16.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@


class TestEnum(enum.IntEnum):
__test__ = False

First = 1
Second = 2

Expand Down
2 changes: 2 additions & 0 deletions tests/types/test_enum8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@


class TestEnum(enum.IntEnum):
__test__ = False
stankudrow marked this conversation as resolved.
Show resolved Hide resolved

First = 1
Second = 2

Expand Down
3 changes: 3 additions & 0 deletions tests/types/test_json.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ def test_select_insert(self):
self.session.execute(
text('SET allow_experimental_object_type = 1;')
)
self.session.execute(
text('SET allow_experimental_json_type = 1;')
)
Comment on lines +51 to +53
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without:

_____________________ JSONTestCaseNative.test_select_insert ______________________

self = <clickhouse_sqlalchemy.drivers.native.connector.Cursor object at 0x7a79ffb51dc0>
operation = 'CREATE TABLE test (x JSON) ENGINE = Memory', parameters = {}
context = <clickhouse_sqlalchemy.drivers.native.base.ClickHouseExecutionContext object at 0x7a79ffb51280>

    def execute(self, operation, parameters=None, context=None):
        self._reset_state()
        self._begin_query()
    
        try:
            execute, execute_kwargs = self._prepare(context)
>           response = execute(
                operation, params=parameters, with_column_types=True,
                **execute_kwargs
            )

clickhouse_sqlalchemy/drivers/native/connector.py:153: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:382: in execute
    rv = self.process_ordinary_query(
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:580: in process_ordinary_query
    return self.receive_result(with_column_types=with_column_types,
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:212: in receive_result
    return result.get_result()
.venv/lib/python3.9/site-packages/clickhouse_driver/result.py:50: in get_result
    for packet in self.packet_generator:
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:228: in packet_generator
    packet = self.receive_packet()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <clickhouse_driver.client.Client object at 0x7a79ffd19ee0>

    def receive_packet(self):
        packet = self.connection.receive_packet()
    
        if packet.type == ServerPacketTypes.EXCEPTION:
>           raise packet.exception
E           clickhouse_driver.errors.ServerException: Code: 44.
E           DB::Exception: Cannot create column with type 'JSON' because experimental JSON type is not allowed. Set setting allow_experimental_json_type = 1 in order to allow it. Stack trace:
E           
E           0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000cf7c73b
E           1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007ea788c
E           2. DB::Exception::Exception<String>(int, FormatStringHelperImpl<std::type_identity<String>::type>, String&&) @ 0x0000000007ea73cb
E           3. DB::validateDataType(std::shared_ptr<DB::IDataType const> const&, DB::DataTypeValidationSettings const&)::$_0::operator()(DB::IDataType const&) const (.llvm.2336510096372259121) @ 0x00000000117810cd
E           4. DB::InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(DB::ASTCreateQuery&, DB::LoadingStrictnessLevel) const @ 0x000000001114e26f
E           5. DB::InterpreterCreateQuery::createTable(DB::ASTCreateQuery&) @ 0x00000000111550bf
E           6. DB::InterpreterCreateQuery::execute() @ 0x0000000011167bb0
E           7. DB::executeQueryImpl(char const*, char const*, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum, DB::ReadBuffer*) @ 0x00000000117351c3
E           8. DB::executeQuery(String const&, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum) @ 0x0000000011730ffa
E           9. DB::TCPHandler::runImpl() @ 0x0000000012922614
E           10. DB::TCPHandler::run() @ 0x000000001293da78
E           11. Poco::Net::TCPServerConnection::start() @ 0x000000001580b827
E           12. Poco::Net::TCPServerDispatcher::run() @ 0x000000001580bcb9
E           13. Poco::PooledThread::run() @ 0x00000000157d8821
E           14. Poco::ThreadImpl::runnableEntry(void*) @ 0x00000000157d6ddd
E           15. ? @ 0x00007c17125aa609
E           16. ? @ 0x00007c17124cf353

.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:245: ServerException

During handling of the above exception, another exception occurred:

self = <tests.types.test_json.JSONTestCaseNative testMethod=test_select_insert>

    def test_select_insert(self):
        data = {'k1': 1, 'k2': '2', 'k3': True}
    
        self.table.drop(bind=self.session.bind, if_exists=True)
        try:
            # http session is unsupport
            self.session.execute(
                text('SET allow_experimental_object_type = 1;')
            )
            # self.session.execute(
            #     text('SET allow_experimental_json_type = 1;')
            # )
>           self.session.execute(text(self.compile(CreateTable(self.table))))

tests/types/test_json.py:51: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.venv/lib/python3.9/site-packages/sqlalchemy/orm/session.py:2229: in execute
    return self._execute_internal(
.venv/lib/python3.9/site-packages/sqlalchemy/orm/session.py:2133: in _execute_internal
    result = conn.execute(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1414: in execute
    return meth(
.venv/lib/python3.9/site-packages/sqlalchemy/sql/elements.py:489: in _execute_on_connection
    return connection._execute_clauseelement(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1638: in _execute_clauseelement
    ret = self._execute_context(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1842: in _execute_context
    return self._exec_single_context(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1983: in _exec_single_context
    self._handle_dbapi_exception(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:2328: in _handle_dbapi_exception
    raise exc_info[1].with_traceback(exc_info[2])
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1964: in _exec_single_context
    self.dialect.do_execute(
clickhouse_sqlalchemy/drivers/base.py:499: in do_execute
    cursor.execute(statement, parameters, context=context)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <clickhouse_sqlalchemy.drivers.native.connector.Cursor object at 0x7a79ffb51dc0>
operation = 'CREATE TABLE test (x JSON) ENGINE = Memory', parameters = {}
context = <clickhouse_sqlalchemy.drivers.native.base.ClickHouseExecutionContext object at 0x7a79ffb51280>

    def execute(self, operation, parameters=None, context=None):
        self._reset_state()
        self._begin_query()
    
        try:
            execute, execute_kwargs = self._prepare(context)
            response = execute(
                operation, params=parameters, with_column_types=True,
                **execute_kwargs
            )
    
        except DriverError as orig:
>           raise DatabaseException(orig)
E           clickhouse_sqlalchemy.exceptions.DatabaseException: Orig exception: Code: 44.
E           DB::Exception: Cannot create column with type 'JSON' because experimental JSON type is not allowed. Set setting allow_experimental_json_type = 1 in order to allow it. Stack trace:
E           
E           0. DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000cf7c73b
E           1. DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000007ea788c
E           2. DB::Exception::Exception<String>(int, FormatStringHelperImpl<std::type_identity<String>::type>, String&&) @ 0x0000000007ea73cb
E           3. DB::validateDataType(std::shared_ptr<DB::IDataType const> const&, DB::DataTypeValidationSettings const&)::$_0::operator()(DB::IDataType const&) const (.llvm.2336510096372259121) @ 0x00000000117810cd
E           4. DB::InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(DB::ASTCreateQuery&, DB::LoadingStrictnessLevel) const @ 0x000000001114e26f
E           5. DB::InterpreterCreateQuery::createTable(DB::ASTCreateQuery&) @ 0x00000000111550bf
E           6. DB::InterpreterCreateQuery::execute() @ 0x0000000011167bb0
E           7. DB::executeQueryImpl(char const*, char const*, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum, DB::ReadBuffer*) @ 0x00000000117351c3
E           8. DB::executeQuery(String const&, std::shared_ptr<DB::Context>, DB::QueryFlags, DB::QueryProcessingStage::Enum) @ 0x0000000011730ffa
E           9. DB::TCPHandler::runImpl() @ 0x0000000012922614
E           10. DB::TCPHandler::run() @ 0x000000001293da78
E           11. Poco::Net::TCPServerConnection::start() @ 0x000000001580b827
E           12. Poco::Net::TCPServerDispatcher::run() @ 0x000000001580bcb9
E           13. Poco::PooledThread::run() @ 0x00000000157d8821
E           14. Poco::ThreadImpl::runnableEntry(void*) @ 0x00000000157d6ddd
E           15. ? @ 0x00007c17125aa609
E           16. ? @ 0x00007c17124cf353

clickhouse_sqlalchemy/drivers/native/connector.py:159: DatabaseException

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But with the experimental JSON support enabled, another exceptions occurs:

_____________________ JSONTestCaseNative.test_select_insert ______________________

spec = 'JSON'
column_options = {'context': <Context(server_info=<ServerInfo(name=ClickHouse, version=24.10.2, revision=54471, used revision=54468, ti...null_as_default': False, 'namedtuple_as_json': True, 'server_side_params': False}, settings={})>, 'types_check': False}
use_numpy = False

    def get_column_by_spec(spec, column_options, use_numpy=None):
        context = column_options['context']
    
        if use_numpy is None:
            use_numpy = context.client_settings['use_numpy'] if context else False
    
        if use_numpy:
            from .numpy.service import get_numpy_column_by_spec
    
            try:
                return get_numpy_column_by_spec(spec, column_options)
            except errors.UnknownTypeError:
                use_numpy = False
                logger.warning('NumPy support is not implemented for %s. '
                               'Using generic column', spec)
    
        def create_column_with_options(x):
            return get_column_by_spec(x, column_options, use_numpy=use_numpy)
    
        if spec == 'String' or spec.startswith('FixedString'):
            return create_string_column(spec, column_options)
    
        elif spec.startswith('Enum'):
            return create_enum_column(spec, column_options)
    
        elif spec.startswith('DateTime'):
            return create_datetime_column(spec, column_options)
    
        elif spec.startswith('Decimal'):
            return create_decimal_column(spec, column_options)
    
        elif spec.startswith('Array'):
            return create_array_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('Tuple'):
            return create_tuple_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('Nested'):
            return create_nested_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('Nullable'):
            return create_nullable_column(spec, create_column_with_options)
    
        elif spec.startswith('LowCardinality'):
            return create_low_cardinality_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('SimpleAggregateFunction'):
            return create_simple_aggregate_function_column(
                spec, create_column_with_options
            )
    
        elif spec.startswith('Map'):
            return create_map_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith("Object('json')"):
            return create_json_column(
                spec, create_column_with_options, column_options
            )
    
        else:
            for alias, primitive in aliases:
                if spec.startswith(alias):
                    return create_column_with_options(
                        primitive + spec[len(alias):]
                    )
    
            try:
>               cls = column_by_type[spec]
E               KeyError: 'JSON'

.venv/lib/python3.9/site-packages/clickhouse_driver/columns/service.py:139: KeyError

During handling of the above exception, another exception occurred:

self = <clickhouse_sqlalchemy.drivers.native.connector.Cursor object at 0x7b7886ba0760>
operation = 'INSERT INTO test (x) VALUES'
seq_of_parameters = [{'x': '{"k1": 1, "k2": "2", "k3": true}'}]
context = <clickhouse_sqlalchemy.drivers.native.base.ClickHouseExecutionContext object at 0x7b7886ba08e0>

    def executemany(self, operation, seq_of_parameters, context=None):
        self._reset_state()
        self._begin_query()
    
        try:
            execute, execute_kwargs = self._prepare(context)
    
>           response = execute(
                operation, params=seq_of_parameters, **execute_kwargs
            )

clickhouse_sqlalchemy/drivers/native/connector.py:171: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:376: in execute
    rv = self.process_insert_query(
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:607: in process_insert_query
    rv = self.send_data(sample_block, data,
.venv/lib/python3.9/site-packages/clickhouse_driver/client.py:660: in send_data
    self.connection.send_data(block)
.venv/lib/python3.9/site-packages/clickhouse_driver/connection.py:688: in send_data
    self.block_out.write(block)
.venv/lib/python3.9/site-packages/clickhouse_driver/streams/native.py:48: in write
    write_column(self.context, col_name, col_type, items,
.venv/lib/python3.9/site-packages/clickhouse_driver/columns/service.py:163: in write_column
    column = get_column_by_spec(column_spec, column_options)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

spec = 'JSON'
column_options = {'context': <Context(server_info=<ServerInfo(name=ClickHouse, version=24.10.2, revision=54471, used revision=54468, ti...null_as_default': False, 'namedtuple_as_json': True, 'server_side_params': False}, settings={})>, 'types_check': False}
use_numpy = False

    def get_column_by_spec(spec, column_options, use_numpy=None):
        context = column_options['context']
    
        if use_numpy is None:
            use_numpy = context.client_settings['use_numpy'] if context else False
    
        if use_numpy:
            from .numpy.service import get_numpy_column_by_spec
    
            try:
                return get_numpy_column_by_spec(spec, column_options)
            except errors.UnknownTypeError:
                use_numpy = False
                logger.warning('NumPy support is not implemented for %s. '
                               'Using generic column', spec)
    
        def create_column_with_options(x):
            return get_column_by_spec(x, column_options, use_numpy=use_numpy)
    
        if spec == 'String' or spec.startswith('FixedString'):
            return create_string_column(spec, column_options)
    
        elif spec.startswith('Enum'):
            return create_enum_column(spec, column_options)
    
        elif spec.startswith('DateTime'):
            return create_datetime_column(spec, column_options)
    
        elif spec.startswith('Decimal'):
            return create_decimal_column(spec, column_options)
    
        elif spec.startswith('Array'):
            return create_array_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('Tuple'):
            return create_tuple_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('Nested'):
            return create_nested_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('Nullable'):
            return create_nullable_column(spec, create_column_with_options)
    
        elif spec.startswith('LowCardinality'):
            return create_low_cardinality_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith('SimpleAggregateFunction'):
            return create_simple_aggregate_function_column(
                spec, create_column_with_options
            )
    
        elif spec.startswith('Map'):
            return create_map_column(
                spec, create_column_with_options, column_options
            )
    
        elif spec.startswith("Object('json')"):
            return create_json_column(
                spec, create_column_with_options, column_options
            )
    
        else:
            for alias, primitive in aliases:
                if spec.startswith(alias):
                    return create_column_with_options(
                        primitive + spec[len(alias):]
                    )
    
            try:
                cls = column_by_type[spec]
                return cls(**column_options)
    
            except KeyError:
>               raise errors.UnknownTypeError('Unknown type {}'.format(spec))
E               clickhouse_driver.errors.UnknownTypeError: Code: 50. Unknown type JSON

.venv/lib/python3.9/site-packages/clickhouse_driver/columns/service.py:143: UnknownTypeError

During handling of the above exception, another exception occurred:

self = <tests.types.test_json.JSONTestCaseNative testMethod=test_select_insert>

    def test_select_insert(self):
        data = {'k1': 1, 'k2': '2', 'k3': True}
    
        self.table.drop(bind=self.session.bind, if_exists=True)
        try:
            # http session is unsupport
            self.session.execute(
                text('SET allow_experimental_object_type = 1;')
            )
            self.session.execute(
                text('SET allow_experimental_json_type = 1;')
            )
            self.session.execute(text(self.compile(CreateTable(self.table))))
>           self.session.execute(self.table.insert(), [{'x': data}])

tests/types/test_json.py:52: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.venv/lib/python3.9/site-packages/sqlalchemy/orm/session.py:2229: in execute
    return self._execute_internal(
.venv/lib/python3.9/site-packages/sqlalchemy/orm/session.py:2133: in _execute_internal
    result = conn.execute(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1414: in execute
    return meth(
.venv/lib/python3.9/site-packages/sqlalchemy/sql/elements.py:489: in _execute_on_connection
    return connection._execute_clauseelement(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1638: in _execute_clauseelement
    ret = self._execute_context(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1842: in _execute_context
    return self._exec_single_context(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1983: in _exec_single_context
    self._handle_dbapi_exception(
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:2328: in _handle_dbapi_exception
    raise exc_info[1].with_traceback(exc_info[2])
.venv/lib/python3.9/site-packages/sqlalchemy/engine/base.py:1933: in _exec_single_context
    self.dialect.do_executemany(
clickhouse_sqlalchemy/drivers/base.py:496: in do_executemany
    cursor.executemany(statement, parameters, context=context)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <clickhouse_sqlalchemy.drivers.native.connector.Cursor object at 0x7b7886ba0760>
operation = 'INSERT INTO test (x) VALUES'
seq_of_parameters = [{'x': '{"k1": 1, "k2": "2", "k3": true}'}]
context = <clickhouse_sqlalchemy.drivers.native.base.ClickHouseExecutionContext object at 0x7b7886ba08e0>

    def executemany(self, operation, seq_of_parameters, context=None):
        self._reset_state()
        self._begin_query()
    
        try:
            execute, execute_kwargs = self._prepare(context)
    
            response = execute(
                operation, params=seq_of_parameters, **execute_kwargs
            )
    
        except DriverError as orig:
>           raise DatabaseException(orig)
E           clickhouse_sqlalchemy.exceptions.DatabaseException: Orig exception: Code: 50. Unknown type JSON

clickhouse_sqlalchemy/drivers/native/connector.py:176: DatabaseException
============================ short test summary info =============================
FAILED tests/types/test_json.py::JSONTestCaseNative::test_select_insert - clickhouse_sqlalchemy.exceptions.DatabaseException: Orig exception: Code: 50....

self.session.execute(text(self.compile(CreateTable(self.table))))
self.session.execute(self.table.insert(), [{'x': data}])
coltype = inspect(self.session.bind).get_columns('test')[0]['type']
Expand Down