From 8bdf4f5cf981aaa00c1a93a3a01f56a71ce059f1 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Tue, 7 Nov 2023 13:41:38 +0100 Subject: [PATCH] Moved MSSQL connector to open source --- lib/dl_connector_mssql/LICENSE | 201 ++++++++++++++ lib/dl_connector_mssql/Makefile | 12 + lib/dl_connector_mssql/README.md | 1 + .../dl_connector_mssql/__init__.py | 0 .../dl_connector_mssql/api/__init__.py | 0 .../api/api_schema/__init__.py | 0 .../api/api_schema/connection.py | 17 ++ .../api/connection_form/__init__.py | 0 .../api/connection_form/form_config.py | 81 ++++++ .../dl_connector_mssql/api/connection_info.py | 7 + .../dl_connector_mssql/api/connector.py | 53 ++++ .../dl_connector_mssql/api/i18n/__init__.py | 0 .../dl_connector_mssql/api/i18n/localizer.py | 28 ++ .../dl_connector_mssql/core/__init__.py | 0 .../dl_connector_mssql/core/adapters_mssql.py | 254 ++++++++++++++++++ .../core/connection_executors.py | 35 +++ .../dl_connector_mssql/core/connector.py | 63 +++++ .../dl_connector_mssql/core/constants.py | 11 + .../dl_connector_mssql/core/data_source.py | 38 +++ .../core/data_source_migration.py | 13 + .../dl_connector_mssql/core/dto.py | 10 + .../dl_connector_mssql/core/exc.py | 43 +++ .../dl_connector_mssql/core/query_compiler.py | 13 + .../dl_connector_mssql/core/sa_types.py | 37 +++ .../core/storage_schemas/__init__.py | 0 .../core/storage_schemas/connection.py | 7 + .../dl_connector_mssql/core/target_dto.py | 5 + .../core/testing/__init__.py | 0 .../core/testing/connection.py | 40 +++ .../core/type_transformer.py | 62 +++++ .../dl_connector_mssql/core/us_connection.py | 71 +++++ .../dl_connector_mssql/db_testing/__init__.py | 0 .../db_testing/connector.py | 7 + .../db_testing/engine_wrapper.py | 42 +++ .../dl_connector_mssql/formula/__init__.py | 0 .../dl_connector_mssql/formula/connector.py | 20 ++ .../dl_connector_mssql/formula/constants.py | 13 + .../formula/context_processor.py | 13 + .../formula/definitions/__init__.py | 0 .../formula/definitions/all.py | 26 ++ .../formula/definitions/conditional_blocks.py | 22 ++ .../definitions/functions_aggregation.py | 61 +++++ .../formula/definitions/functions_datetime.py | 189 +++++++++++++ .../formula/definitions/functions_logical.py | 33 +++ .../formula/definitions/functions_markup.py | 20 ++ .../formula/definitions/functions_math.py | 223 +++++++++++++++ .../formula/definitions/functions_string.py | 186 +++++++++++++ .../formula/definitions/functions_type.py | 194 +++++++++++++ .../formula/definitions/operators_binary.py | 156 +++++++++++ .../formula/definitions/operators_ternary.py | 11 + .../formula/definitions/operators_unary.py | 50 ++++ .../dl_connector_mssql/formula/literal.py | 17 ++ .../formula/type_constructor.py | 17 ++ .../formula_ref/__init__.py | 0 .../formula_ref/human_dialects.py | 13 + .../dl_connector_mssql/formula_ref/i18n.py | 23 ++ .../dl_connector_mssql/formula_ref/plugin.py | 12 + .../en/LC_MESSAGES/dl_connector_mssql.mo | Bin 0 -> 301 bytes .../en/LC_MESSAGES/dl_connector_mssql.po | 13 + .../dl_formula_ref_dl_connector_mssql.mo | Bin 0 -> 241 bytes .../dl_formula_ref_dl_connector_mssql.po | 13 + .../ru/LC_MESSAGES/dl_connector_mssql.mo | Bin 0 -> 301 bytes .../ru/LC_MESSAGES/dl_connector_mssql.po | 13 + .../dl_formula_ref_dl_connector_mssql.mo | Bin 0 -> 360 bytes .../dl_formula_ref_dl_connector_mssql.po | 13 + .../dl_connector_mssql/py.typed | 0 .../dl_connector_mssql_tests/__init__.py | 0 .../dl_connector_mssql_tests/db/__init__.py | 0 .../db/api/__init__.py | 0 .../dl_connector_mssql_tests/db/api/base.py | 58 ++++ .../db/api/test_complex_queries.py | 12 + .../db/api/test_connection.py | 7 + .../db/api/test_dashsql.py | 116 ++++++++ .../db/api/test_data.py | 34 +++ .../db/api/test_dataset.py | 7 + .../dl_connector_mssql_tests/db/config.py | 90 +++++++ .../dl_connector_mssql_tests/db/conftest.py | 64 +++++ .../db/core/__init__.py | 0 .../dl_connector_mssql_tests/db/core/base.py | 57 ++++ .../db/core/test_connection.py | 34 +++ .../db/core/test_connection_executor.py | 83 ++++++ .../db/core/test_data_source.py | 114 ++++++++ .../db/core/test_dataset.py | 16 ++ .../db/formula/__init__.py | 0 .../db/formula/base.py | 17 ++ .../db/formula/test_conditional_blocks.py | 7 + .../db/formula/test_functions_aggregation.py | 7 + .../db/formula/test_functions_datetime.py | 8 + .../db/formula/test_functions_logical.py | 7 + .../db/formula/test_functions_markup.py | 7 + .../db/formula/test_functions_math.py | 7 + .../db/formula/test_functions_string.py | 21 ++ .../formula/test_functions_type_conversion.py | 78 ++++++ .../db/formula/test_literals.py | 13 + .../db/formula/test_misc_funcs.py | 7 + .../db/formula/test_operators.py | 7 + .../dl_connector_mssql_tests/unit/__init__.py | 0 .../dl_connector_mssql_tests/unit/conftest.py | 0 .../unit/test_connection_form.py | 10 + lib/dl_connector_mssql/docker-compose.yml | 42 +++ .../docker-compose/tests/entrypoint.sh | 3 + lib/dl_connector_mssql/pyproject.toml | 94 +++++++ metapkg/poetry.lock | 30 ++- metapkg/pyproject.toml | 1 + 104 files changed, 3559 insertions(+), 1 deletion(-) create mode 100644 lib/dl_connector_mssql/LICENSE create mode 100644 lib/dl_connector_mssql/Makefile create mode 100644 lib/dl_connector_mssql/README.md create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/connection.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/form_config.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/connection_info.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/connector.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/i18n/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/api/i18n/localizer.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/adapters_mssql.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/connection_executors.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/connector.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/constants.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/data_source.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/data_source_migration.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/dto.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/exc.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/query_compiler.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/sa_types.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/connection.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/target_dto.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/testing/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/testing/connection.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/type_transformer.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/core/us_connection.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/db_testing/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/db_testing/connector.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/db_testing/engine_wrapper.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/connector.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/constants.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/context_processor.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/all.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/conditional_blocks.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_aggregation.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_datetime.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_logical.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_markup.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_math.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_string.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_type.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_binary.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_ternary.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_unary.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/literal.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula/type_constructor.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula_ref/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula_ref/human_dialects.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula_ref/i18n.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/formula_ref/plugin.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_connector_mssql.mo create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_connector_mssql.po create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.mo create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.po create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_connector_mssql.mo create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_connector_mssql.po create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.mo create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.po create mode 100644 lib/dl_connector_mssql/dl_connector_mssql/py.typed create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_complex_queries.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_connection.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dashsql.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_data.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dataset.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/conftest.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/base.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection_executor.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_data_source.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_dataset.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/base.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_conditional_blocks.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_aggregation.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_datetime.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_logical.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_markup.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_math.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_string.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_type_conversion.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_literals.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_misc_funcs.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_operators.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/unit/__init__.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/unit/conftest.py create mode 100644 lib/dl_connector_mssql/dl_connector_mssql_tests/unit/test_connection_form.py create mode 100644 lib/dl_connector_mssql/docker-compose.yml create mode 100644 lib/dl_connector_mssql/docker-compose/tests/entrypoint.sh create mode 100644 lib/dl_connector_mssql/pyproject.toml diff --git a/lib/dl_connector_mssql/LICENSE b/lib/dl_connector_mssql/LICENSE new file mode 100644 index 000000000..74ba5f6c7 --- /dev/null +++ b/lib/dl_connector_mssql/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023 YANDEX LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/dl_connector_mssql/Makefile b/lib/dl_connector_mssql/Makefile new file mode 100644 index 000000000..bbdc142fc --- /dev/null +++ b/lib/dl_connector_mssql/Makefile @@ -0,0 +1,12 @@ +# Include if exists +-include .env + + +include ../../tools/common_makefile.mk + + +PACKAGE_NAME = dl_connector_mssql + + +devenv-d: + docker-compose up --build -d diff --git a/lib/dl_connector_mssql/README.md b/lib/dl_connector_mssql/README.md new file mode 100644 index 000000000..d635f5307 --- /dev/null +++ b/lib/dl_connector_mssql/README.md @@ -0,0 +1 @@ +# dl_connector_mssql diff --git a/lib/dl_connector_mssql/dl_connector_mssql/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/connection.py b/lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/connection.py new file mode 100644 index 000000000..630b57419 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/api/api_schema/connection.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from dl_api_connector.api_schema.connection_base import ConnectionMetaMixin +from dl_api_connector.api_schema.connection_mixins import ( + DataExportForbiddenMixin, + RawSQLLevelMixin, +) +from dl_api_connector.api_schema.connection_sql import ClassicSQLConnectionSchema + +from dl_connector_mssql.core.us_connection import ConnectionMSSQL + + +class MSSQLConnectionSchema( + ConnectionMetaMixin, RawSQLLevelMixin, DataExportForbiddenMixin, ClassicSQLConnectionSchema +): + TARGET_CLS = ConnectionMSSQL + ALLOW_MULTI_HOST = True diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/form_config.py b/lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/form_config.py new file mode 100644 index 000000000..bc5fa2bb5 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/api/connection_form/form_config.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from typing import Optional + +from dl_api_commons.base_models import TenantDef +from dl_api_connector.form_config.models.api_schema import ( + FormActionApiSchema, + FormApiSchema, + FormFieldApiSchema, +) +from dl_api_connector.form_config.models.base import ( + ConnectionForm, + ConnectionFormFactory, + ConnectionFormMode, +) +from dl_api_connector.form_config.models.common import CommonFieldName +import dl_api_connector.form_config.models.rows as C +from dl_api_connector.form_config.models.shortcuts.rows import RowConstructor +from dl_configs.connectors_settings import ConnectorSettingsBase + +from dl_connector_mssql.api.connection_info import MSSQLConnectionInfoProvider + + +class MSSQLConnectionFormFactory(ConnectionFormFactory): + def get_form_config( + self, + connector_settings: Optional[ConnectorSettingsBase], + tenant: Optional[TenantDef], + ) -> ConnectionForm: + rc = RowConstructor(localizer=self._localizer) + + common_api_schema_items: list[FormFieldApiSchema] = [ + FormFieldApiSchema(name=CommonFieldName.host, required=True), + FormFieldApiSchema(name=CommonFieldName.port, required=True), + FormFieldApiSchema(name=CommonFieldName.db_name, required=True), + FormFieldApiSchema(name=CommonFieldName.username, required=True), + FormFieldApiSchema(name=CommonFieldName.password, required=self.mode == ConnectionFormMode.create), + ] + + edit_api_schema = FormActionApiSchema( + items=[ + *common_api_schema_items, + FormFieldApiSchema(name=CommonFieldName.cache_ttl_sec, nullable=True), + FormFieldApiSchema(name=CommonFieldName.raw_sql_level), + FormFieldApiSchema(name=CommonFieldName.data_export_forbidden), + ] + ) + + create_api_schema = FormActionApiSchema( + items=[ + *edit_api_schema.items, + *self._get_top_level_create_api_schema_items(), + ] + ) + + check_api_schema = FormActionApiSchema( + items=[ + *common_api_schema_items, + *self._get_top_level_check_api_schema_items(), + ] + ) + + return ConnectionForm( + title=MSSQLConnectionInfoProvider.get_title(self._localizer), + rows=[ + rc.host_row(), + rc.port_row(default_value="1433"), + rc.db_name_row(), + rc.username_row(), + rc.password_row(self.mode), + C.CacheTTLRow(name=CommonFieldName.cache_ttl_sec), + rc.raw_sql_level_row(), + rc.collapse_advanced_settings_row(), + rc.data_export_forbidden_row(), + ], + api_schema=FormApiSchema( + create=create_api_schema if self.mode == ConnectionFormMode.create else None, + edit=edit_api_schema if self.mode == ConnectionFormMode.edit else None, + check=check_api_schema, + ), + ) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/connection_info.py b/lib/dl_connector_mssql/dl_connector_mssql/api/connection_info.py new file mode 100644 index 000000000..a2266feab --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/api/connection_info.py @@ -0,0 +1,7 @@ +from dl_api_connector.connection_info import ConnectionInfoProvider + +from dl_connector_mssql.api.i18n.localizer import Translatable + + +class MSSQLConnectionInfoProvider(ConnectionInfoProvider): + title_translatable = Translatable("label_connector-mssql") diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py b/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py new file mode 100644 index 000000000..88d5e28e3 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/api/connector.py @@ -0,0 +1,53 @@ +from dl_api_connector.api_schema.source_base import ( + SchematizedSQLDataSourceSchema, + SchematizedSQLDataSourceTemplateSchema, + SubselectDataSourceSchema, + SubselectDataSourceTemplateSchema, +) +from dl_api_connector.connector import ( + ApiConnectionDefinition, + ApiConnector, + ApiSourceDefinition, +) + +from dl_connector_mssql.api.api_schema.connection import MSSQLConnectionSchema +from dl_connector_mssql.api.connection_form.form_config import MSSQLConnectionFormFactory +from dl_connector_mssql.api.connection_info import MSSQLConnectionInfoProvider +from dl_connector_mssql.api.i18n.localizer import CONFIGS +from dl_connector_mssql.core.connector import ( + MSSQLCoreConnectionDefinition, + MSSQLCoreConnector, + MSSQLSubselectCoreSourceDefinition, + MSSQLTableCoreSourceDefinition, +) +from dl_connector_mssql.formula.constants import DIALECT_NAME_MSSQLSRV + + +class MSSQLApiTableSourceDefinition(ApiSourceDefinition): + core_source_def_cls = MSSQLTableCoreSourceDefinition + api_schema_cls = SchematizedSQLDataSourceSchema + template_api_schema_cls = SchematizedSQLDataSourceTemplateSchema + + +class MSSQLApiSubselectSourceDefinition(ApiSourceDefinition): + core_source_def_cls = MSSQLSubselectCoreSourceDefinition + api_schema_cls = SubselectDataSourceSchema + template_api_schema_cls = SubselectDataSourceTemplateSchema + + +class MSSQLApiConnectionDefinition(ApiConnectionDefinition): + core_conn_def_cls = MSSQLCoreConnectionDefinition + api_generic_schema_cls = MSSQLConnectionSchema + info_provider_cls = MSSQLConnectionInfoProvider + form_factory_cls = MSSQLConnectionFormFactory + + +class MSSQLApiConnector(ApiConnector): + core_connector_cls = MSSQLCoreConnector + connection_definitions = (MSSQLApiConnectionDefinition,) + source_definitions = ( + MSSQLApiTableSourceDefinition, + MSSQLApiSubselectSourceDefinition, + ) + formula_dialect_name = DIALECT_NAME_MSSQLSRV + translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/i18n/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/api/i18n/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/api/i18n/localizer.py b/lib/dl_connector_mssql/dl_connector_mssql/api/i18n/localizer.py new file mode 100644 index 000000000..05961b596 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/api/i18n/localizer.py @@ -0,0 +1,28 @@ +import os + +import attr + +from dl_i18n.localizer_base import Translatable as BaseTranslatable +from dl_i18n.localizer_base import TranslationConfig + +import dl_connector_mssql as package + + +DOMAIN = f"{package.__name__}" +CONFIGS = [ + TranslationConfig( + path=os.path.relpath(os.path.join(os.path.dirname(__file__), "../../locales")), + domain=DOMAIN, + locale="en", + ), + TranslationConfig( + path=os.path.relpath(os.path.join(os.path.dirname(__file__), "../../locales")), + domain=DOMAIN, + locale="ru", + ), +] + + +@attr.s +class Translatable(BaseTranslatable): + domain: str = attr.ib(default=DOMAIN) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/adapters_mssql.py b/lib/dl_connector_mssql/dl_connector_mssql/core/adapters_mssql.py new file mode 100644 index 000000000..031ac4256 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/adapters_mssql.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import datetime +import decimal +import logging +import re +from typing import ( + List, + Optional, + Tuple, + Type, +) +from urllib.parse import quote_plus + +import pyodbc +import sqlalchemy as sa +from sqlalchemy import exc as sa_exc +from sqlalchemy.dialects import mssql as ms_types + +from dl_core.connection_executors.adapters.adapters_base_sa_classic import BaseClassicAdapter +from dl_core.connection_executors.models.db_adapter_data import ( + DBAdapterQuery, + RawColumnInfo, + RawSchemaInfo, +) +from dl_core.connection_models import ( + DBIdent, + SATextTableDefinition, + SchemaIdent, + TableIdent, +) +from dl_core.connectors.base.error_transformer import DBExcKWArgs +from dl_core.db.native_type import CommonNativeType +import dl_core.exc as exc + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL +from dl_connector_mssql.core.exc import ( + CommitOrRollbackFailed, + SyncMssqlSourceDoesNotExistError, +) + + +LOGGER = logging.getLogger(__name__) + + +class MSSQLDefaultAdapter(BaseClassicAdapter): + conn_type = CONNECTION_TYPE_MSSQL + + dsn_template = "{dialect}:///?odbc_connect=" + quote_plus(";").join( + [ + quote_plus("DRIVER={FreeTDS}"), + quote_plus("Server=") + "{host}", + quote_plus("Port=") + "{port}", + quote_plus("Database=") + "{db_name}", + quote_plus("UID=") + "{user}", + quote_plus("PWD=") + "{passwd}", + quote_plus("TDS_Version=8.0"), + ] + ) # {...}s are are left unquoted for future formatting in `get_conn_line` + + def _get_db_version(self, db_ident: DBIdent) -> Optional[str]: + return self.execute(DBAdapterQuery("SELECT @@VERSION", db_name=db_ident.db_name)).get_all()[0][0] + + _type_code_to_sa = { + int: ms_types.INTEGER, + float: ms_types.FLOAT, + decimal.Decimal: ms_types.DECIMAL, + bool: ms_types.BIT, + str: ms_types.NTEXT, + datetime.datetime: ms_types.DATETIME, + } + + MSSQL_LIST_SOURCES_ALL_SCHEMAS_SQL = "SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES;" + + def _get_tables(self, schema_ident: SchemaIdent) -> List[TableIdent]: + if schema_ident.schema_name is not None: + # For a single schema, plug into the common SA code. + # (might not be ever used) + return super()._get_tables(schema_ident) + + db_name = schema_ident.db_name + db_engine = self.get_db_engine(db_name) + query = self.MSSQL_LIST_SOURCES_ALL_SCHEMAS_SQL + result = db_engine.execute(sa.text(query)) + return [ + TableIdent( + db_name=db_name, + schema_name=schema_name, + table_name=name, + ) + for schema_name, name in result + ] + + def _get_subselect_table_info(self, subquery: SATextTableDefinition) -> RawSchemaInfo: + """ + For the record, pyodbc's cursor info only contains very approximate type information. + Test-data example (name, type_code, row value): + + [('number', int, 0), + ('num_tinyint', int, 0), + ('num_smallint', int, 0), + ('num_integer', int, 0), + ('num_bigint', int, 0), + ('num_float', float, 0.0), + ('num_real', float, 0.0), + ('num_numeric', decimal.Decimal, Decimal('0')), + ('num_decimal', decimal.Decimal, Decimal('0')), + ('num_bit', bool, False), + ('num_char', str, '0 '), + ('num_varchar', str, '0'), + ('num_text', str, '0'), + ('num_nchar', str, '0 '), + ('num_nvarchar', str, '0'), + ('num_ntext', str, '0'), + ('num_date', str, '2020-01-01'), + ('num_datetime', datetime.datetime, datetime.datetime(1900, 1, 1, 0, 0)), + ('num_datetime2', str, '2020-01-01 00:00:00.0000000'), + ('num_smalldatetime', datetime.datetime, datetime.datetime(1900, 1, 1, 0, 0)), + ('num_datetimeoffset', str, '2020-01-01 00:00:00.0000000 +00:00'), + ('uuid', str, '5F473FA6-E7C3-45E6-9190-A13C7E1558BF')] + + All stuff it returns: name, type, display size (pyodbc does not set + this value), internal size (in bytes), precision, scale, nullable. + + However, there's a default stored procedure for getting a select schema: + https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-describe-first-result-set-transact-sql?view=sql-server-ver15#permissions + + exec sp_describe_first_result_set @tsql = N'select * from {subselect}' + + Note that the entire subquery is passed as a string. + """ + from dl_core.connection_executors.models.db_adapter_data import DBAdapterQuery + + # 'select * from () as source' + select_all = sa.select([sa.literal_column("*")]).select_from(subquery.text) + select_all_str = str(select_all) # Should be straightforward and safe and reliable. Really. + sa_query_text = "exec sp_describe_first_result_set @tsql = :select_all" + sa_query = sa.sql.text(sa_query_text) + sa_query = sa_query.bindparams( + # TODO?: might need a better `type_` value + sa.sql.bindparam("select_all", select_all_str, type_=sa.types.Unicode), + ) + dba_query = DBAdapterQuery(query=sa_query) + query_res = self.execute(dba_query) + data = [row for chunk in query_res.data_chunks for row in chunk] + names = query_res.cursor_info["names"] + if data: + assert len(names) == len(data[0]) + data = [dict(zip(names, row)) for row in data] + + engine = self.get_db_engine(db_name=None) + dialect = engine.dialect + ischema_names = dialect.ischema_names + + columns = [] + for column_info in data: + name = column_info["name"] + if not name: + LOGGER.warning("Empty name in mssql subselect schema: %r", column_info) + continue + + type_name = column_info["system_type_name"] + type_name_base = type_name.split("(", 1)[0] + + sa_type = ischema_names.get(type_name_base) + if sa_type is None: + LOGGER.warning("Unknown/unsupported type in mssql subselect schema: %r", column_info) + sa_type = sa.sql.sqltypes.NullType + + # NOTE: it is possible to instantiate the `sa_type` here; but for + # now, there's no known use for that. + + # Side note: any `cast()` in mssql tends to make the value nullable. + nullable = column_info["is_nullable"] + + native_type = CommonNativeType.normalize_name_and_create( + conn_type=self.conn_type, + name=self.normalize_sa_col_type(sa_type), + nullable=nullable, + ) + + columns.append( + RawColumnInfo( + name=name, + title=name, + nullable=native_type.nullable, + native_type=native_type, + ) + ) + + return RawSchemaInfo(columns=tuple(columns)) + + _EXC_CODE_RE = re.compile( + r"\(\'[0-9A-Z]+\', [\'\"]\[(?P[0-9A-Z]+)\] " r"\[FreeTDS\][^(]+" r"\((?P\d+)\)" + ) + _EXC_CODE_MAP = { + # [42S22] Invalid column name '.+'. (207) + 207: exc.ColumnDoesNotExist, + # [42S02] Invalid object name '.+'. (208) + 208: SyncMssqlSourceDoesNotExistError, + # [22007] Conversion failed when converting date and/or time from character string. (241) + 241: exc.DataParseError, + # [22012] Divide by zero error encountered. (8134) + 8134: exc.DivisionByZero, + # [08S01] Read from the server failed (20004) + 20004: exc.SourceConnectError, + # [08S01] Write to the server failed (20006) + 20006: exc.SourceConnectError, + # [01000] Unexpected EOF from the server (20017) + 20017: exc.SourceClosedPrematurely, + # ? + # [23000] The statement terminated. The maximum recursion 100 + # has been exhausted before statement completion. (530) + # [42000] Snapshot isolation transaction failed in database '.+' + # because the object accessed by the statement has been modified + # by a DDL statement... (3961) + # [42000] Transaction (Process ID \d+) was deadlocked on lock... (1205) + # [42000] The batch could not be analyzed because of compile errors. (11501) + } + + _EXC_STATE_MAP = { + # [08001] Unable to connect to data source (0) + "08001": exc.SourceConnectError, + # [HY000] Could not perform COMMIT or ROLLBACK (0) + "HY000": CommitOrRollbackFailed, + } + + EXTRA_EXC_CLS = (pyodbc.Error, sa_exc.DBAPIError) + + @classmethod + def get_exc_class(cls, err_msg: str) -> Optional[Type[exc.DatabaseQueryError]]: + err_match = cls._EXC_CODE_RE.match(err_msg) + if err_match is not None: + code = int(err_match.group("code")) + state = err_match.group("state").upper() + if code in cls._EXC_CODE_MAP: + return cls._EXC_CODE_MAP[code] + elif state in cls._EXC_STATE_MAP: + return cls._EXC_STATE_MAP[state] + return exc.DatabaseQueryError + + return None + + @classmethod + def make_exc( # TODO: Move to ErrorTransformer + cls, wrapper_exc: Exception, orig_exc: Optional[Exception], debug_compiled_query: Optional[str] + ) -> Tuple[Type[exc.DatabaseQueryError], DBExcKWArgs]: + exc_cls, kw = super().make_exc(wrapper_exc, orig_exc, debug_compiled_query) + + db_msg = kw["db_message"] + specific_exc_cls = cls.get_exc_class(db_msg) # type: ignore # TODO: fix + exc_cls = specific_exc_cls if specific_exc_cls is not None else exc_cls + + return exc_cls, kw diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/connection_executors.py b/lib/dl_connector_mssql/dl_connector_mssql/core/connection_executors.py new file mode 100644 index 000000000..9c34066d0 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/connection_executors.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import List + +import attr + +from dl_core.connection_executors.async_sa_executors import DefaultSqlAlchemyConnExecutor + +from dl_connector_mssql.core.adapters_mssql import MSSQLDefaultAdapter +from dl_connector_mssql.core.dto import MSSQLConnDTO +from dl_connector_mssql.core.target_dto import MSSQLConnTargetDTO + + +@attr.s(cmp=False, hash=False) +class MSSQLConnExecutor(DefaultSqlAlchemyConnExecutor[MSSQLDefaultAdapter]): + TARGET_ADAPTER_CLS = MSSQLDefaultAdapter + + _conn_dto: MSSQLConnDTO = attr.ib() + + async def _make_target_conn_dto_pool(self) -> List[MSSQLConnTargetDTO]: # type: ignore # TODO: fix + dto_pool = [] + for host in self._conn_hosts_pool: + dto_pool.append( + MSSQLConnTargetDTO( + conn_id=self._conn_dto.conn_id, + pass_db_messages_to_user=self._conn_options.pass_db_messages_to_user, + pass_db_query_to_user=self._conn_options.pass_db_query_to_user, + host=host, + port=self._conn_dto.port, + db_name=self._conn_dto.db_name, + username=self._conn_dto.username, + password=self._conn_dto.password, + ) + ) + return dto_pool diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py b/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py new file mode 100644 index 000000000..f03e746ce --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/connector.py @@ -0,0 +1,63 @@ +import pyodbc + +from dl_core.connectors.base.connector import ( + CoreConnectionDefinition, + CoreConnector, +) +from dl_core.connectors.sql_base.connector import ( + SQLSubselectCoreSourceDefinitionBase, + SQLTableCoreSourceDefinitionBase, +) + +from dl_connector_mssql.core.adapters_mssql import MSSQLDefaultAdapter +from dl_connector_mssql.core.connection_executors import MSSQLConnExecutor +from dl_connector_mssql.core.constants import ( + BACKEND_TYPE_MSSQL, + CONNECTION_TYPE_MSSQL, + SOURCE_TYPE_MSSQL_SUBSELECT, + SOURCE_TYPE_MSSQL_TABLE, +) +from dl_connector_mssql.core.data_source import ( + MSSQLDataSource, + MSSQLSubselectDataSource, +) +from dl_connector_mssql.core.data_source_migration import MSSQLDataSourceMigrator +from dl_connector_mssql.core.query_compiler import MSSQLQueryCompiler +from dl_connector_mssql.core.sa_types import SQLALCHEMY_MSSQL_TYPES +from dl_connector_mssql.core.storage_schemas.connection import ConnectionMSSQLDataStorageSchema +from dl_connector_mssql.core.type_transformer import MSSQLServerTypeTransformer +from dl_connector_mssql.core.us_connection import ConnectionMSSQL + + +class MSSQLCoreConnectionDefinition(CoreConnectionDefinition): + conn_type = CONNECTION_TYPE_MSSQL + connection_cls = ConnectionMSSQL + us_storage_schema_cls = ConnectionMSSQLDataStorageSchema + type_transformer_cls = MSSQLServerTypeTransformer + sync_conn_executor_cls = MSSQLConnExecutor + async_conn_executor_cls = MSSQLConnExecutor + dialect_string = "bi_mssql" + data_source_migrator_cls = MSSQLDataSourceMigrator + + +class MSSQLTableCoreSourceDefinition(SQLTableCoreSourceDefinitionBase): + source_type = SOURCE_TYPE_MSSQL_TABLE + source_cls = MSSQLDataSource + + +class MSSQLSubselectCoreSourceDefinition(SQLSubselectCoreSourceDefinitionBase): + source_type = SOURCE_TYPE_MSSQL_SUBSELECT + source_cls = MSSQLSubselectDataSource + + +class MSSQLCoreConnector(CoreConnector): + backend_type = BACKEND_TYPE_MSSQL + connection_definitions = (MSSQLCoreConnectionDefinition,) + source_definitions = ( + MSSQLTableCoreSourceDefinition, + MSSQLSubselectCoreSourceDefinition, + ) + rqe_adapter_classes = frozenset({MSSQLDefaultAdapter}) + sa_types = SQLALCHEMY_MSSQL_TYPES + query_fail_exceptions = frozenset({pyodbc.Error}) + compiler_cls = MSSQLQueryCompiler diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/constants.py b/lib/dl_connector_mssql/dl_connector_mssql/core/constants.py new file mode 100644 index 000000000..3a5291c7a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/constants.py @@ -0,0 +1,11 @@ +from dl_constants.enums import ( + ConnectionType, + DataSourceType, + SourceBackendType, +) + + +BACKEND_TYPE_MSSQL = SourceBackendType.declare("MSSQL") +CONNECTION_TYPE_MSSQL = ConnectionType.declare("mssql") +SOURCE_TYPE_MSSQL_TABLE = DataSourceType.declare("MSSQL_TABLE") +SOURCE_TYPE_MSSQL_SUBSELECT = DataSourceType.declare("MSSQL_SUBSELECT") diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/data_source.py b/lib/dl_connector_mssql/dl_connector_mssql/core/data_source.py new file mode 100644 index 000000000..d3869bff8 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/data_source.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import logging + +from dl_constants.enums import DataSourceType +from dl_core.data_source.sql import ( + BaseSQLDataSource, + StandardSchemaSQLDataSource, + SubselectDataSource, +) + +from dl_connector_mssql.core.constants import ( + CONNECTION_TYPE_MSSQL, + SOURCE_TYPE_MSSQL_SUBSELECT, + SOURCE_TYPE_MSSQL_TABLE, +) +from dl_connector_mssql.core.query_compiler import MSSQLQueryCompiler + + +LOGGER = logging.getLogger(__name__) + + +class MSSQLDataSourceMixin(BaseSQLDataSource): + compiler_cls = MSSQLQueryCompiler + + conn_type = CONNECTION_TYPE_MSSQL + + @classmethod + def is_compatible_with_type(cls, source_type: DataSourceType) -> bool: + return source_type in (SOURCE_TYPE_MSSQL_TABLE, SOURCE_TYPE_MSSQL_SUBSELECT) + + +class MSSQLDataSource(MSSQLDataSourceMixin, StandardSchemaSQLDataSource): # type: ignore # TODO: fix + """MSSQL table""" + + +class MSSQLSubselectDataSource(MSSQLDataSourceMixin, SubselectDataSource): # type: ignore # TODO: fix + """MSSQL table""" diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/data_source_migration.py b/lib/dl_connector_mssql/dl_connector_mssql/core/data_source_migration.py new file mode 100644 index 000000000..3ae0d724c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/data_source_migration.py @@ -0,0 +1,13 @@ +from dl_core.connectors.sql_base.data_source_migration import DefaultSQLDataSourceMigrator +from dl_core.data_source_spec.sql import StandardSchemaSQLDataSourceSpec + +from dl_connector_mssql.core.constants import ( + SOURCE_TYPE_MSSQL_SUBSELECT, + SOURCE_TYPE_MSSQL_TABLE, +) + + +class MSSQLDataSourceMigrator(DefaultSQLDataSourceMigrator): + table_source_type = SOURCE_TYPE_MSSQL_TABLE + table_dsrc_spec_cls = StandardSchemaSQLDataSourceSpec + subselect_source_type = SOURCE_TYPE_MSSQL_SUBSELECT diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/dto.py b/lib/dl_connector_mssql/dl_connector_mssql/core/dto.py new file mode 100644 index 000000000..e8d082434 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/dto.py @@ -0,0 +1,10 @@ +import attr + +from dl_core.connection_models.dto_defs import DefaultSQLDTO + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL + + +@attr.s(frozen=True) +class MSSQLConnDTO(DefaultSQLDTO): + conn_type = CONNECTION_TYPE_MSSQL diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/exc.py b/lib/dl_connector_mssql/dl_connector_mssql/core/exc.py new file mode 100644 index 000000000..11078950e --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/exc.py @@ -0,0 +1,43 @@ +import re +from typing import ( + Any, + Dict, + Optional, +) + +import dl_core.exc as exc + + +class SyncMssqlSourceDoesNotExistError(exc.SourceDoesNotExist): + ERR_RE = re.compile(r".*Invalid\sobject\sname\s'(?P.*)'\.\s\(208\).*") + + def __init__( + self, + db_message: Optional[str] = None, + query: Optional[str] = None, + message: Optional[str] = None, + details: Optional[Dict[str, Any]] = None, + orig: Optional[Exception] = None, + debug_info: Optional[Dict[str, Any]] = None, + params: Optional[Dict[str, Any]] = None, + ): + super(SyncMssqlSourceDoesNotExistError, self).__init__( + db_message=db_message, + query=query, + message=message, + details=details, + orig=orig, + debug_info=debug_info, + params=params, + ) + + if self.orig and self.orig.args and len(self.orig.args) >= 2: + message = self.orig.args[1] + if message and (match := self.ERR_RE.match(message)): + if table := match.group("table"): + self.params["table_definition"] = table + + +class CommitOrRollbackFailed(exc.DatabaseQueryError): + err_code = exc.DatabaseQueryError.err_code + ["COMMIT_OR_ROLLBACK_FAILED"] + default_message = "Failed to COMMIT or ROLLBACK" diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/query_compiler.py b/lib/dl_connector_mssql/dl_connector_mssql/core/query_compiler.py new file mode 100644 index 000000000..768dedf5c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/query_compiler.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dl_core.connectors.base.query_compiler import QueryCompiler +from dl_core.query.bi_query import BIQuery +from dl_core.query.expression import ExpressionCtx + + +class MSSQLQueryCompiler(QueryCompiler): + def should_order_by_alias(self, expr_ctx: ExpressionCtx, bi_query: BIQuery) -> bool: + # No idea, why MSSQL acts this way, but it just does + if bi_query.limit is not None and bi_query.offset is not None: + return False + return super().should_order_by_alias(expr_ctx=expr_ctx, bi_query=bi_query) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/sa_types.py b/lib/dl_connector_mssql/dl_connector_mssql/core/sa_types.py new file mode 100644 index 000000000..9512bb2b8 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/sa_types.py @@ -0,0 +1,37 @@ +from sqlalchemy.dialects import mssql as ms_types + +from dl_core.db.sa_types_base import ( + make_native_type, + simple_instantiator, +) + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL + + +SQLALCHEMY_MSSQL_BASE_TYPES = ( + ms_types.TINYINT, + ms_types.SMALLINT, + ms_types.INTEGER, + ms_types.BIGINT, + ms_types.FLOAT, + ms_types.REAL, + ms_types.NUMERIC, + ms_types.DECIMAL, + ms_types.BIT, + ms_types.CHAR, + ms_types.VARCHAR, + ms_types.TEXT, + ms_types.NCHAR, + ms_types.NVARCHAR, + ms_types.NTEXT, + ms_types.DATE, + ms_types.DATETIME, + ms_types.DATETIME2, + ms_types.SMALLDATETIME, + ms_types.DATETIMEOFFSET, + ms_types.UNIQUEIDENTIFIER, +) +SQLALCHEMY_MSSQL_TYPES = { + make_native_type(CONNECTION_TYPE_MSSQL, typecls): simple_instantiator(typecls) + for typecls in SQLALCHEMY_MSSQL_BASE_TYPES +} diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/connection.py b/lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/connection.py new file mode 100644 index 000000000..9863b31dd --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/storage_schemas/connection.py @@ -0,0 +1,7 @@ +from dl_core.us_manager.storage_schemas.connection import ConnectionSQLDataStorageSchema + +from dl_connector_mssql.core.us_connection import ConnectionMSSQL + + +class ConnectionMSSQLDataStorageSchema(ConnectionSQLDataStorageSchema[ConnectionMSSQL.DataModel]): + TARGET_CLS = ConnectionMSSQL.DataModel diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/target_dto.py b/lib/dl_connector_mssql/dl_connector_mssql/core/target_dto.py new file mode 100644 index 000000000..da090ef2e --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/target_dto.py @@ -0,0 +1,5 @@ +from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO + + +class MSSQLConnTargetDTO(BaseSQLConnTargetDTO): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/testing/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/core/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/testing/connection.py b/lib/dl_connector_mssql/dl_connector_mssql/core/testing/connection.py new file mode 100644 index 000000000..4b866d260 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/testing/connection.py @@ -0,0 +1,40 @@ +from typing import ( + Any, + Optional, +) +import uuid + +from dl_constants.enums import RawSQLLevel +from dl_core.us_manager.us_manager_sync import SyncUSManager + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL +from dl_connector_mssql.core.us_connection import ConnectionMSSQL + + +def make_mssql_saved_connection( + sync_usm: SyncUSManager, + db_name: Optional[str], + host: str, + port: Optional[int], + username: Optional[str], + password: Optional[str], + raw_sql_level: RawSQLLevel = RawSQLLevel.off, + **kwargs: Any, +) -> ConnectionMSSQL: + conn_name = "mssql test conn {}".format(uuid.uuid4()) + conn = ConnectionMSSQL.create_from_dict( + data_dict=ConnectionMSSQL.DataModel( + db_name=db_name, + host=host, + port=port, + username=username, + password=password, + raw_sql_level=raw_sql_level, + ), + ds_key=conn_name, + type_=CONNECTION_TYPE_MSSQL.name, + us_manager=sync_usm, + **kwargs, + ) + sync_usm.save(conn) + return conn diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/type_transformer.py b/lib/dl_connector_mssql/dl_connector_mssql/core/type_transformer.py new file mode 100644 index 000000000..223cef4d6 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/type_transformer.py @@ -0,0 +1,62 @@ +import sqlalchemy as sa +from sqlalchemy.dialects import mssql as ms_types + +from dl_constants.enums import UserDataType +from dl_core.db.conversion_base import ( + LowercaseTypeCaster, + TypeTransformer, + make_native_type, +) + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL + + +class MSSQLServerTypeTransformer(TypeTransformer): + conn_type = CONNECTION_TYPE_MSSQL + native_to_user_map = { + **{ + make_native_type(CONNECTION_TYPE_MSSQL, t): UserDataType.integer # type: ignore # TODO: fix + for t in (ms_types.TINYINT, ms_types.SMALLINT, ms_types.INTEGER, ms_types.BIGINT) + }, + **{ + make_native_type(CONNECTION_TYPE_MSSQL, t): UserDataType.float + for t in (ms_types.FLOAT, ms_types.REAL, ms_types.NUMERIC, ms_types.DECIMAL) + }, + make_native_type(CONNECTION_TYPE_MSSQL, ms_types.BIT): UserDataType.boolean, + **{ + make_native_type(CONNECTION_TYPE_MSSQL, t): UserDataType.string + for t in ( + ms_types.CHAR, + ms_types.VARCHAR, + ms_types.TEXT, + ms_types.NCHAR, + ms_types.NVARCHAR, + ms_types.NTEXT, + ) + }, + make_native_type(CONNECTION_TYPE_MSSQL, ms_types.DATE): UserDataType.date, + **{ + make_native_type(CONNECTION_TYPE_MSSQL, t): UserDataType.genericdatetime + for t in (ms_types.DATETIME, ms_types.DATETIME2, ms_types.SMALLDATETIME, ms_types.DATETIMEOFFSET) + }, + make_native_type(CONNECTION_TYPE_MSSQL, ms_types.UNIQUEIDENTIFIER): UserDataType.uuid, + make_native_type(CONNECTION_TYPE_MSSQL, sa.sql.sqltypes.NullType): UserDataType.unsupported, + } + user_to_native_map = { + UserDataType.integer: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.BIGINT), + UserDataType.float: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.FLOAT), + UserDataType.boolean: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.BIT), + UserDataType.string: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.VARCHAR), + UserDataType.date: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.DATE), + UserDataType.datetime: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.DATETIME), + UserDataType.genericdatetime: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.DATETIME), + UserDataType.geopoint: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.VARCHAR), + UserDataType.geopolygon: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.VARCHAR), + UserDataType.uuid: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.UNIQUEIDENTIFIER), + UserDataType.markup: make_native_type(CONNECTION_TYPE_MSSQL, ms_types.VARCHAR), + UserDataType.unsupported: make_native_type(CONNECTION_TYPE_MSSQL, sa.sql.sqltypes.NullType), + } + casters = { + **TypeTransformer.casters, # type: ignore # TODO: fix + UserDataType.uuid: LowercaseTypeCaster(), + } diff --git a/lib/dl_connector_mssql/dl_connector_mssql/core/us_connection.py b/lib/dl_connector_mssql/dl_connector_mssql/core/us_connection.py new file mode 100644 index 000000000..2d4bf280c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/core/us_connection.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from typing import ( + Callable, + ClassVar, +) + +import attr + +from dl_core.connection_executors.sync_base import SyncConnExecutorBase +from dl_core.us_connection_base import ( + ClassicConnectionSQL, + ConnectionBase, + DataSourceTemplate, +) +from dl_i18n.localizer_base import Localizer + +from dl_connector_mssql.core.constants import ( + SOURCE_TYPE_MSSQL_SUBSELECT, + SOURCE_TYPE_MSSQL_TABLE, +) +from dl_connector_mssql.core.dto import MSSQLConnDTO + + +class ConnectionMSSQL(ClassicConnectionSQL): + has_schema: ClassVar[bool] = True + default_schema_name = "dbo" + source_type = SOURCE_TYPE_MSSQL_TABLE + allowed_source_types = frozenset((SOURCE_TYPE_MSSQL_TABLE, SOURCE_TYPE_MSSQL_SUBSELECT)) + allow_dashsql: ClassVar[bool] = True + allow_cache: ClassVar[bool] = True + is_always_user_source: ClassVar[bool] = True + + @attr.s(kw_only=True) + class DataModel(ClassicConnectionSQL.DataModel): + pass + + def get_conn_dto(self) -> MSSQLConnDTO: + return MSSQLConnDTO( + conn_id=self.uuid, + host=self.data.host, + multihosts=self.parse_multihosts(), # type: ignore # TODO: fix + port=self.data.port, + db_name=self.data.db_name, + username=self.data.username, + password=self.password, + ) + + def get_data_source_template_templates(self, localizer: Localizer) -> list[DataSourceTemplate]: + return self._make_subselect_templates( + source_type=SOURCE_TYPE_MSSQL_SUBSELECT, + field_doc_key="MSSQL_SUBSELECT/subsql", + localizer=localizer, + ) + + def get_parameter_combinations( + self, + conn_executor_factory: Callable[[ConnectionBase], SyncConnExecutorBase], + ) -> list[dict]: + if not self.db_name: + return [] + + assert self.has_schema + return [ + dict(schema_name=tid.schema_name, table_name=tid.table_name) + for tid in self.get_tables(schema_name=None, conn_executor_factory=conn_executor_factory) + ] + + @property + def allow_public_usage(self) -> bool: + return True diff --git a/lib/dl_connector_mssql/dl_connector_mssql/db_testing/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/db_testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/db_testing/connector.py b/lib/dl_connector_mssql/dl_connector_mssql/db_testing/connector.py new file mode 100644 index 000000000..ef1f53d55 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/db_testing/connector.py @@ -0,0 +1,7 @@ +from dl_db_testing.connectors.base.connector import DbTestingConnector + +from dl_connector_mssql.db_testing.engine_wrapper import MSSQLEngineWrapper + + +class MSSQLDbTestingConnector(DbTestingConnector): + engine_wrapper_classes = (MSSQLEngineWrapper,) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/db_testing/engine_wrapper.py b/lib/dl_connector_mssql/dl_connector_mssql/db_testing/engine_wrapper.py new file mode 100644 index 000000000..ed0cee7cf --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/db_testing/engine_wrapper.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from typing import Optional +import urllib.parse + +from sqlalchemy.engine.url import URL + +from dl_db_testing.database.engine_wrapper import EngineWrapperBase + + +class MSSQLEngineWrapper(EngineWrapperBase): + URL_PREFIX = "mssql" # Not using the bi_* version because we only need basic functionality here + + def get_conn_credentials(self, full: bool = False) -> dict: + url = self.url + if isinstance(url, URL): + odbc_dsn = url.query["odbc_connect"] + else: + urldata = urllib.parse.urlparse(str(url)) + query = dict(urllib.parse.parse_qsl(urldata.query)) + odbc_dsn = query["odbc_connect"] + + assert isinstance(odbc_dsn, str) + odbc_props = {pair.split("=")[0]: pair.split("=")[1] for pair in odbc_dsn.split(";")} + return dict( + host=odbc_props["Server"], + port=int(odbc_props["Port"]), + username=odbc_props["UID"], + password=odbc_props["PWD"], + db_name=odbc_props["Database"], + ) + + def count_sql_sessions(self) -> int: + cur = self.execute("exec sp_who") + try: + lines = cur.fetchall() + return len(lines) + finally: + cur.close() + + def get_version(self) -> Optional[str]: + return self.execute("SELECT @@VERSION").scalar() diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/connector.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/connector.py new file mode 100644 index 000000000..fd053f90a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/connector.py @@ -0,0 +1,20 @@ +from sqlalchemy.dialects.mssql.base import MSDialect + +from dl_formula.connectors.base.connector import FormulaConnector + +from dl_connector_mssql.formula.constants import MssqlDialect +from dl_connector_mssql.formula.context_processor import MSSQLContextPostprocessor +from dl_connector_mssql.formula.definitions.all import DEFINITIONS +from dl_connector_mssql.formula.literal import MSSQLLiteralizer +from dl_connector_mssql.formula.type_constructor import MSSQLTypeConstructor + + +class MSSQLFormulaConnector(FormulaConnector): + dialect_ns_cls = MssqlDialect + dialects = MssqlDialect.MSSQLSRV + default_dialect = MssqlDialect.MSSQLSRV_14_0 + op_definitions = DEFINITIONS + literalizer_cls = MSSQLLiteralizer + context_processor_cls = MSSQLContextPostprocessor + type_constructor_cls = MSSQLTypeConstructor + sa_dialect = MSDialect() diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/constants.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/constants.py new file mode 100644 index 000000000..8a2603f9c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/constants.py @@ -0,0 +1,13 @@ +from dl_formula.core.dialect import ( + DialectName, + DialectNamespace, + simple_combo, +) + + +DIALECT_NAME_MSSQLSRV = DialectName.declare("MSSQLSRV") + + +class MssqlDialect(DialectNamespace): + MSSQLSRV_14_0 = simple_combo(name=DIALECT_NAME_MSSQLSRV, version=(14, 0)) + MSSQLSRV = MSSQLSRV_14_0 diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/context_processor.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/context_processor.py new file mode 100644 index 000000000..a0a32da4d --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/context_processor.py @@ -0,0 +1,13 @@ +import sqlalchemy as sa +from sqlalchemy.sql.elements import ClauseElement + +from dl_formula.connectors.base.context_processor import BooleanlessContextPostprocessor +from dl_formula.core.datatype import DataType + + +class MSSQLContextPostprocessor(BooleanlessContextPostprocessor): + def booleanize_expression(self, data_type: DataType, expression: ClauseElement) -> ClauseElement: + return sa.func.CONVERT(sa.text("BIT"), expression) == 1 + + def debooleanize_expression(self, data_type: DataType, expression: ClauseElement) -> ClauseElement: + return sa.func.IIF(expression, 1, 0) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/all.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/all.py new file mode 100644 index 000000000..c32d3ba6b --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/all.py @@ -0,0 +1,26 @@ +from dl_connector_mssql.formula.definitions.conditional_blocks import DEFINITIONS_COND_BLOCKS +from dl_connector_mssql.formula.definitions.functions_aggregation import DEFINITIONS_AGG +from dl_connector_mssql.formula.definitions.functions_datetime import DEFINITIONS_DATETIME +from dl_connector_mssql.formula.definitions.functions_logical import DEFINITIONS_LOGICAL +from dl_connector_mssql.formula.definitions.functions_markup import DEFINITIONS_MARKUP +from dl_connector_mssql.formula.definitions.functions_math import DEFINITIONS_MATH +from dl_connector_mssql.formula.definitions.functions_string import DEFINITIONS_STRING +from dl_connector_mssql.formula.definitions.functions_type import DEFINITIONS_TYPE +from dl_connector_mssql.formula.definitions.operators_binary import DEFINITIONS_BINARY +from dl_connector_mssql.formula.definitions.operators_ternary import DEFINITIONS_TERNARY +from dl_connector_mssql.formula.definitions.operators_unary import DEFINITIONS_UNARY + + +DEFINITIONS = [ + *DEFINITIONS_COND_BLOCKS, + *DEFINITIONS_AGG, + *DEFINITIONS_DATETIME, + *DEFINITIONS_LOGICAL, + *DEFINITIONS_MARKUP, + *DEFINITIONS_MATH, + *DEFINITIONS_STRING, + *DEFINITIONS_TYPE, + *DEFINITIONS_UNARY, + *DEFINITIONS_BINARY, + *DEFINITIONS_TERNARY, +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/conditional_blocks.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/conditional_blocks.py new file mode 100644 index 000000000..cbca54c01 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/conditional_blocks.py @@ -0,0 +1,22 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.conditional_blocks as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_COND_BLOCKS = [ + # _case_block_ + base.CaseBlock.for_dialect(D.MSSQLSRV), + # _if_block_ + base.IfBlock3( + variants=[ + V(D.MSSQLSRV, sa.func.IIF), + ] + ), + base.IfBlockMulti.for_dialect(D.MSSQLSRV), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_aggregation.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_aggregation.py new file mode 100644 index 000000000..d7af391b4 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_aggregation.py @@ -0,0 +1,61 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.functions_aggregation as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_AGG = [ + # avg + base.AggAvgFromNumber.for_dialect(D.MSSQLSRV), + base.AggAvgFromDate.for_dialect(D.MSSQLSRV), + base.AggAvgFromDatetime.for_dialect(D.MSSQLSRV), + base.AggAvgFromDatetimeTZ.for_dialect(D.MSSQLSRV), + # avg_if + base.AggAvgIf.for_dialect(D.MSSQLSRV), + # count + base.AggCount0.for_dialect(D.MSSQLSRV), + base.AggCount1.for_dialect(D.MSSQLSRV), + # count_if + base.AggCountIf.for_dialect(D.MSSQLSRV), + # countd + base.AggCountd.for_dialect(D.MSSQLSRV), + # countd_if + base.AggCountdIf.for_dialect(D.MSSQLSRV), + # max + base.AggMax.for_dialect(D.MSSQLSRV), + # min + base.AggMin.for_dialect(D.MSSQLSRV), + # stdev + base.AggStdev( + variants=[ + V(D.MSSQLSRV, sa.func.STDEV), + ] + ), + # stdevp + base.AggStdevp( + variants=[ + V(D.MSSQLSRV, sa.func.STDEVP), + ] + ), + # sum + base.AggSum.for_dialect(D.MSSQLSRV), + # sum_if + base.AggSumIf.for_dialect(D.MSSQLSRV), + # var + base.AggVar( + variants=[ + V(D.MSSQLSRV, sa.func.VAR), + ] + ), + # varp + base.AggVarp( + variants=[ + V(D.MSSQLSRV, sa.func.VARP), + ] + ), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_datetime.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_datetime.py new file mode 100644 index 000000000..f9bc74dd5 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_datetime.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import sqlalchemy as sa +from sqlalchemy.sql import ClauseElement + +from dl_formula.definitions.base import TranslationVariant +from dl_formula.definitions.common import raw_sql +from dl_formula.definitions.common_datetime import ( + EPOCH_START_DOW, + EPOCH_START_S, +) +import dl_formula.definitions.functions_datetime as base +from dl_formula.definitions.literals import un_literal + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +def make_mssql_datetrunc(date: ClauseElement, unit: str, start_of_year: ClauseElement) -> ClauseElement: + norm_unit = base.norm_datetrunc_unit(unit) + if norm_unit == "week": + return make_mssql_datetrunc_week(date) + return ( + start_of_year + if norm_unit == "year" + else sa.func.DATEADD( + raw_sql(norm_unit), + sa.func.DATEDIFF(raw_sql(norm_unit), start_of_year, date), + start_of_year, + ) + ) + + +def make_mssql_datetrunc_week(date: ClauseElement) -> ClauseElement: + set_day_to_monday = sa.func.DATEADD( + sa.text("day"), + # This monstrosity is used to get @@DATEFIRST independent day of week in MSSQL where Monday = 0 + # In order to understand that consider that for any values of @@DATEFIRST + # (DATEPART(dw, :date) + @@DATEFIRST) % 7 returns the day of week number where + # Saturday = 0, Sunday = 1, Monday = 2 and so on. So an offset of 5 shifts it to Monday = 0 + -((5 + sa.func.DATEPART(sa.text("dw"), date) + sa.text("@@DATEFIRST")) % 7), + date, + ) + return sa.cast(sa.cast(set_day_to_monday, sa.types.DATE), sa.dialects.mssql.DATETIME) + + +DEFINITIONS_DATETIME = [ + # dateadd + base.FuncDateadd1.for_dialect(D.MSSQLSRV), + base.FuncDateadd2Unit.for_dialect(D.MSSQLSRV), + base.FuncDateadd2Number.for_dialect(D.MSSQLSRV), + base.FuncDateadd3Legacy.for_dialect(D.MSSQLSRV), + base.FuncDateadd3DateConstNum( + variants=[ + V( + D.MSSQLSRV, + lambda date, what, num: sa.cast( + sa.func.DATEADD(raw_sql(un_literal(what)), un_literal(num), date), sa.Date + ), + ), + ] + ), + base.FuncDateadd3DatetimeConstNum( + variants=[ + V( + D.MSSQLSRV, + lambda dt, what, num: (sa.cast(sa.func.DATEADD(raw_sql(what.value), num.value, dt), sa.DateTime)), + ), + ] + ), + # datepart + base.FuncDatepart2Legacy.for_dialect(D.MSSQLSRV), + base.FuncDatepart2.for_dialect(D.MSSQLSRV), + base.FuncDatepart3Const.for_dialect(D.MSSQLSRV), + base.FuncDatepart3NonConst.for_dialect(D.MSSQLSRV), + # datetrunc + base.FuncDatetrunc2Date( + variants=[ + V( + D.MSSQLSRV, + lambda date, unit: ( + make_mssql_datetrunc(date, unit, sa.func.DATEFROMPARTS(sa.func.YEAR(date), 1, 1)) + if base.norm_datetrunc_unit(unit) in {"year", "quarter", "month", "week"} + else date + ), + ), + ] + ), + base.FuncDatetrunc2Datetime( + variants=[ + V( + D.MSSQLSRV, + lambda date, unit: make_mssql_datetrunc( + date, unit, sa.func.DATETIMEFROMPARTS(sa.func.YEAR(date), 1, 1, 0, 0, 0, 0) + ), + ), + ] + ), + # day + base.FuncDay( + variants=[ + V(D.MSSQLSRV, sa.func.DAY), + ] + ), + # dayofweek + base.FuncDayofweek1.for_dialect(D.MSSQLSRV), + base.FuncDayofweek2( + variants=[ + V( + D.MSSQLSRV, + lambda date, firstday: ( + ( + ( + sa.func.DATEDIFF(raw_sql("DAY"), EPOCH_START_S, date) + - (EPOCH_START_DOW + base.norm_fd(firstday) - 1) + ) + % 7 + + 1 + ) + ), + ), + ] + ), + # genericnow + base.FuncGenericNow( + variants=[ + V(D.MSSQLSRV, lambda: sa.type_coerce(raw_sql("CURRENT_TIMESTAMP"), sa.DateTime)), + ] + ), + # hour + base.FuncHourDate.for_dialect(D.MSSQLSRV), + base.FuncHourDatetime( + variants=[ + V(D.MSSQLSRV, lambda date: sa.func.DATEPART(raw_sql("hour"), date)), + ] + ), + # minute + base.FuncMinuteDate.for_dialect(D.MSSQLSRV), + base.FuncMinuteDatetime( + variants=[ + V(D.MSSQLSRV, lambda date: sa.func.DATEPART(raw_sql("minute"), date)), + ] + ), + # month + base.FuncMonth( + variants=[ + V(D.MSSQLSRV, sa.func.MONTH), + ] + ), + # now + base.FuncNow( + variants=[ + V(D.MSSQLSRV, lambda: sa.type_coerce(raw_sql("CURRENT_TIMESTAMP"), sa.DateTime)), + ] + ), + # quarter + base.FuncQuarter( + variants=[ + V(D.MSSQLSRV, lambda date: sa.func.DATEPART(raw_sql("QUARTER"), date)), + ] + ), + # second + base.FuncSecondDate.for_dialect(D.MSSQLSRV), + base.FuncSecondDatetime( + variants=[ + V(D.MSSQLSRV, lambda date: sa.func.DATEPART(raw_sql("second"), date)), + ] + ), + # today + base.FuncToday( + variants=[ + V(D.MSSQLSRV, lambda: sa.type_coerce(sa.cast(sa.func.GETDATE(), sa.Date()), sa.Date())), + ] + ), + # week + base.FuncWeek( + variants=[ + V(D.MSSQLSRV, lambda date: sa.func.DATEPART(raw_sql("ISO_WEEK"), date)), + ] + ), + # year + base.FuncYear( + variants=[ + V(D.MSSQLSRV, sa.func.YEAR), + ] + ), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_logical.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_logical.py new file mode 100644 index 000000000..1f9d40346 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_logical.py @@ -0,0 +1,33 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.functions_logical as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_LOGICAL = [ + # case + base.FuncCase.for_dialect(D.MSSQLSRV), + # if + base.FuncIf.for_dialect(D.MSSQLSRV), + # ifnull + base.FuncIfnull( + variants=[ + V(D.MSSQLSRV, sa.func.ISNULL), + ] + ), + # iif + base.FuncIif3Legacy.for_dialect(D.MSSQLSRV), + # isnull + base.FuncIsnull.for_dialect(D.MSSQLSRV), + # zn + base.FuncZn( + variants=[ + V(D.MSSQLSRV, lambda x: sa.func.ISNULL(x, 0)), + ] + ), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_markup.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_markup.py new file mode 100644 index 000000000..75f770417 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_markup.py @@ -0,0 +1,20 @@ +import dl_formula.definitions.functions_markup as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +DEFINITIONS_MARKUP = [ + # + + base.BinaryPlusMarkup.for_dialect(D.MSSQLSRV), + # __str + base.FuncInternalStrConst.for_dialect(D.MSSQLSRV), + base.FuncInternalStr.for_dialect(D.MSSQLSRV), + # bold + base.FuncBold.for_dialect(D.MSSQLSRV), + # italic + base.FuncItalics.for_dialect(D.MSSQLSRV), + # markup + base.ConcatMultiMarkup.for_dialect(D.MSSQLSRV), + # url + base.FuncUrl.for_dialect(D.MSSQLSRV), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_math.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_math.py new file mode 100644 index 000000000..b4ced89fe --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_math.py @@ -0,0 +1,223 @@ +import sqlalchemy as sa +import sqlalchemy.dialects.mssql as sa_mssqlsrv + +from dl_formula.core.datatype import DataType +from dl_formula.definitions.args import ArgTypeSequence +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.functions_math as base +from dl_formula.definitions.type_strategy import ParamsFromArgs + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +class FuncDivMSSQLInt(base.FuncDiv): + variants = [V(D.MSSQLSRV, lambda x, y: x / y)] + argument_types = [ + ArgTypeSequence([DataType.INTEGER, DataType.INTEGER]), + ] + + +class FuncDivMSSQLFloat(base.FuncDiv): + variants = [V(D.MSSQLSRV, lambda x, y: sa.cast(x / y, sa.BIGINT))] + argument_types = [ + ArgTypeSequence([DataType.FLOAT, DataType.FLOAT]), + ] + + +class FuncDivSafe2MSSQLInt(base.FuncDivSafe2): + variants = [V(D.MSSQLSRV, lambda x, y: sa.func.IIF(y != 0, x / y, None))] + argument_types = [ + ArgTypeSequence([DataType.INTEGER, DataType.INTEGER]), + ] + + +class FuncDivSafe3MSSQLInt(base.FuncDivSafe3): + variants = [V(D.MSSQLSRV, lambda x, y, default: sa.func.IIF(y != 0, x / y, default))] + argument_types = [ + ArgTypeSequence([DataType.INTEGER, DataType.INTEGER, DataType.INTEGER]), + ] + + +class FuncDivSafe2MSSQLFloat(base.FuncDivSafe2): + variants = [V(D.MSSQLSRV, lambda x, y: sa.func.IIF(y != 0, sa.cast(x / y, sa.BIGINT), None))] + argument_types = [ + ArgTypeSequence([DataType.FLOAT, DataType.FLOAT]), + ] + + +class FuncDivSafe3MSSQLFloat(base.FuncDivSafe3): + variants = [V(D.MSSQLSRV, lambda x, y, default: sa.func.IIF(y != 0, sa.cast(x / y, sa.BIGINT), default))] + argument_types = [ + ArgTypeSequence([DataType.FLOAT, DataType.FLOAT, DataType.INTEGER]), + ] + + +class FuncGreatestMSSQL(base.FuncGreatestBase): + variants = [V(D.MSSQLSRV, lambda x, y: sa.func.IIF(x >= y, x, y))] + argument_types = [ + ArgTypeSequence([DataType.FLOAT, DataType.FLOAT]), + ArgTypeSequence([DataType.STRING, DataType.STRING]), + ArgTypeSequence([DataType.BOOLEAN, DataType.BOOLEAN]), + ] + + +class FuncGreatestDatesMSSQL(base.FuncGreatestBase): + variants = [ + V(D.MSSQLSRV, lambda x, y: sa.cast(sa.func.IIF(x >= y, x, y), sa.Date)), + ] + argument_types = [ + ArgTypeSequence([DataType.DATE, DataType.DATE]), + ] + + +class FuncGreatestDatetimesMSSQL(base.FuncGreatestBase): + variants = [ + V(D.MSSQLSRV, lambda x, y: sa.cast(sa.func.IIF(x >= y, x, y), sa.DateTime)), + ] + argument_types = [ + # Note: should not mix tz-naive and tz-aware datetimes together. + ArgTypeSequence([DataType.DATETIME, DataType.DATETIME]), + ArgTypeSequence([DataType.DATETIMETZ, DataType.DATETIMETZ]), + ArgTypeSequence([DataType.GENERICDATETIME, DataType.GENERICDATETIME]), + ] + return_type_params = ParamsFromArgs(0) + + +class FuncLeastMSSQL(base.FuncLeastBase): + variants = [V(D.MSSQLSRV, lambda x, y: sa.func.IIF(x <= y, x, y))] + argument_types = [ + ArgTypeSequence([DataType.FLOAT, DataType.FLOAT]), + ArgTypeSequence([DataType.STRING, DataType.STRING]), + ArgTypeSequence([DataType.BOOLEAN, DataType.BOOLEAN]), + ] + + +class FuncLeastDatesMSSQL(base.FuncLeastBase): + variants = [ + V(D.MSSQLSRV, lambda x, y: sa.cast(sa.func.IIF(x <= y, x, y), sa.Date)), + ] + argument_types = [ + ArgTypeSequence([DataType.DATE, DataType.DATE]), + ] + + +class FuncLeastDatetimesMSSQL(base.FuncLeastBase): + variants = [ + V(D.MSSQLSRV, lambda x, y: sa.cast(sa.func.IIF(x <= y, x, y), sa.DateTime)), + ] + argument_types = [ + # Note: should not mix tz-naive and tz-aware datetimes together. + ArgTypeSequence([DataType.DATETIME, DataType.DATETIME]), + ArgTypeSequence([DataType.DATETIMETZ, DataType.DATETIMETZ]), + ArgTypeSequence([DataType.GENERICDATETIME, DataType.GENERICDATETIME]), + ] + return_type_params = ParamsFromArgs(0) + + +DEFINITIONS_MATH = [ + # abs + base.FuncAbs.for_dialect(D.MSSQLSRV), + # acos + base.FuncAcos.for_dialect(D.MSSQLSRV), + # asin + base.FuncAsin.for_dialect(D.MSSQLSRV), + # atan + base.FuncAtan.for_dialect(D.MSSQLSRV), + # atan2 + base.FuncAtan2( + variants=[ + V(D.MSSQLSRV, sa.func.ATN2), + ] + ), + # ceiling + base.FuncCeiling( + variants=[ + V(D.MSSQLSRV, sa.func.CEILING), + ] + ), + # cos + base.FuncCos.for_dialect(D.MSSQLSRV), + # cot + base.FuncCot.for_dialect(D.MSSQLSRV), + # degrees + base.FuncDegrees.for_dialect(D.MSSQLSRV), + # div + FuncDivMSSQLInt(), + FuncDivMSSQLFloat(), + # div_safe + FuncDivSafe2MSSQLInt(), + FuncDivSafe3MSSQLInt(), + FuncDivSafe2MSSQLFloat(), + FuncDivSafe3MSSQLFloat(), + # exp + base.FuncExp.for_dialect(D.MSSQLSRV), + # fdiv_safe + base.FuncFDivSafe2.for_dialect(D.MSSQLSRV), + base.FuncFDivSafe3.for_dialect(D.MSSQLSRV), + # floor + base.FuncFloor.for_dialect(D.MSSQLSRV), + # greatest + base.FuncGreatest1.for_dialect(D.MSSQLSRV), + FuncGreatestMSSQL(), + FuncGreatestDatesMSSQL(), + FuncGreatestDatetimesMSSQL(), + base.GreatestMulti.for_dialect(D.MSSQLSRV), + # least + base.FuncLeast1.for_dialect(D.MSSQLSRV), + FuncLeastMSSQL(), + FuncLeastDatesMSSQL(), + FuncLeastDatetimesMSSQL(), + base.LeastMulti.for_dialect(D.MSSQLSRV), + # ln + base.FuncLn( + variants=[ + V(D.MSSQLSRV, sa.func.LOG), + ] + ), + # log + base.FuncLog( + variants=[ + V(D.MSSQLSRV, lambda x, base: sa.func.LOG(x, base)), + ] + ), + # log10 + base.FuncLog10.for_dialect(D.MSSQLSRV), + # pi + base.FuncPi.for_dialect(D.MSSQLSRV), + # power + base.FuncPower( + variants=[ + V(D.MSSQLSRV, lambda x, y: sa.func.POWER(sa.cast(x, sa_mssqlsrv.FLOAT), sa.cast(y, sa_mssqlsrv.FLOAT))), + ] + ), + # radians + base.FuncRadians( + variants=[ + V(D.MSSQLSRV, lambda x: sa.func.RADIANS(sa.cast(x, sa_mssqlsrv.FLOAT))), + ] + ), + # round + base.FuncRound1( + variants=[ + V(D.MSSQLSRV, lambda x: sa.func.ROUND(x, 0)), + ] + ), + base.FuncRound2.for_dialect(D.MSSQLSRV), + # sign + base.FuncSign.for_dialect(D.MSSQLSRV), + # sin + base.FuncSin.for_dialect(D.MSSQLSRV), + # sqrt + base.FuncSqrt.for_dialect(D.MSSQLSRV), + # square + base.FuncSquare( + variants=[ + V(D.MSSQLSRV, sa.func.SQUARE), + ] + ), + # tan + base.FuncTan.for_dialect(D.MSSQLSRV), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_string.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_string.py new file mode 100644 index 000000000..ef601b9d7 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_string.py @@ -0,0 +1,186 @@ +from typing import Any + +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.functions_string as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +def make_like_pattern_mssql_const(value: Any, left_any: bool = True, right_any: bool = True) -> str: + assert isinstance(value, str) + # XXXX: does not handle `[` / `]` in the value. + # Probably should be + # `value = re.sub(r'([\[%])', r'[\1]', value)` + # https://stackoverflow.com/q/439495 + value = value.replace("%", "[%]") + result = "{}{}{}".format( + "%" if left_any else "", + value, + "%" if right_any else "", + ) + # result = literal(result) + return result + + +def make_like_pattern_mssql_clause(clause, left_any=True, right_any=True): # type: ignore + # e.g. `ColumnClause` + pieces = [] + if left_any: + pieces += ["%"] + # XXXX: need at least the `REPLACE(…, '%', '[%]')` wrapping here. + pieces += [clause] + if right_any: + pieces += ["%"] + return sa.func.CONCAT(*pieces) + + +DEFINITIONS_STRING = [ + # ascii + base.FuncAscii.for_dialect(D.MSSQLSRV), + # char + base.FuncChar.for_dialect(D.MSSQLSRV), + # concat + base.Concat1.for_dialect((D.MSSQLSRV)), + base.ConcatMultiStrConst.for_dialect(D.MSSQLSRV), + base.ConcatMultiStr.for_dialect(D.MSSQLSRV), + base.ConcatMultiAny.for_dialect(D.MSSQLSRV), + # contains + base.FuncContainsConst( + variants=[ + V( + D.MSSQLSRV, + lambda x, y: x.like( + make_like_pattern_mssql_const(y.value, left_any=True, right_any=True), + ), + ), + ] + ), + base.FuncContainsNonConst( + variants=[ + V( + D.MSSQLSRV, + lambda x, y: sa.func.PATINDEX(make_like_pattern_mssql_clause(y, left_any=True, right_any=True), x) > 0, + ), + ] + ), + base.FuncContainsNonString.for_dialect(D.MSSQLSRV), + # notcontains + base.FuncNotContainsConst.for_dialect(D.MSSQLSRV), + base.FuncNotContainsNonConst.for_dialect(D.MSSQLSRV), + base.FuncNotContainsNonString.for_dialect(D.MSSQLSRV), + # endswith + base.FuncEndswithConst( + variants=[ + V( + D.MSSQLSRV, + lambda x, y: x.like( + make_like_pattern_mssql_const(y.value, left_any=True, right_any=False), + ), + ), + ] + ), + base.FuncEndswithNonConst( + variants=[ + V( + D.MSSQLSRV, + lambda x, y: (sa.func.SUBSTRING(x, sa.func.LEN(x) - sa.func.LEN(y) + 1, sa.func.LEN(x)) == y), + ), + ] + ), + base.FuncEndswithNonString.for_dialect(D.MSSQLSRV), + # find + base.FuncFind2( + variants=[ + V( + D.MSSQLSRV, + lambda x, y: sa.func.PATINDEX(make_like_pattern_mssql_clause(y, left_any=True, right_any=True), x), + ), + ] + ), + base.FuncFind3( + variants=[ + V( + D.MSSQLSRV, + lambda x, y, z: sa.func.IIF( + sa.func.PATINDEX( + make_like_pattern_mssql_clause(y, left_any=True, right_any=True), + sa.func.SUBSTRING(x, z, sa.func.DATALENGTH(x)), + ) + > 0, + sa.func.PATINDEX( + make_like_pattern_mssql_clause(y, left_any=True, right_any=True), + sa.func.SUBSTRING(x, z, sa.func.DATALENGTH(x)), + ) + + z + - 1, + 0, + ), + ), + ] + ), + # icontains + base.FuncIContainsNonConst.for_dialect(D.MSSQLSRV), + base.FuncIContainsNonString.for_dialect(D.MSSQLSRV), + # iendswith + base.FuncIEndswithNonConst.for_dialect(D.MSSQLSRV), + base.FuncIEndswithNonString.for_dialect(D.MSSQLSRV), + # istartswith + base.FuncIStartswithNonConst.for_dialect(D.MSSQLSRV), + base.FuncIStartswithNonString.for_dialect(D.MSSQLSRV), + # left + base.FuncLeft.for_dialect(D.MSSQLSRV), + # len + base.FuncLenString( + variants=[ + V(D.MSSQLSRV, sa.func.LEN), + ] + ), + # lower + base.FuncLowerConst.for_dialect(D.MSSQLSRV), + base.FuncLowerNonConst.for_dialect(D.MSSQLSRV), + # ltrim + base.FuncLtrim.for_dialect(D.MSSQLSRV), + # replace + base.FuncReplace.for_dialect(D.MSSQLSRV), + # right + base.FuncRight.for_dialect(D.MSSQLSRV), + # rtrim + base.FuncRtrim.for_dialect(D.MSSQLSRV), + # space + base.FuncSpaceConst.for_dialect(D.MSSQLSRV), + base.FuncSpaceNonConst.for_dialect(D.MSSQLSRV), + # startswith + base.FuncStartswithConst( + variants=[ + V( + D.MSSQLSRV, + lambda x, y: x.like( + make_like_pattern_mssql_const(y.value, left_any=False, right_any=True), + ), + ), + ] + ), + base.FuncStartswithNonConst( + variants=[ + V(D.MSSQLSRV, lambda x, y: sa.func.SUBSTRING(x, 1, sa.func.LEN(y)) == y), + ] + ), + base.FuncStartswithNonString.for_dialect(D.MSSQLSRV), + # substr + base.FuncSubstr2( + variants=[ + V(D.MSSQLSRV, lambda text, start: sa.func.SUBSTRING(text, start, sa.func.DATALENGTH(text))), + ] + ), + base.FuncSubstr3.for_dialect(D.MSSQLSRV), + # trim + base.FuncTrim.for_dialect(D.MSSQLSRV), + # upper + base.FuncUpperConst.for_dialect(D.MSSQLSRV), + base.FuncUpperNonConst.for_dialect(D.MSSQLSRV), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_type.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_type.py new file mode 100644 index 000000000..c07c9c546 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/functions_type.py @@ -0,0 +1,194 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +from dl_formula.definitions.common_datetime import EPOCH_START_S +import dl_formula.definitions.functions_type as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +class MSSQLFuncBoolFromNumber(base.FuncBoolFromNumber): + variants = [ + V( + D.MSSQLSRV, + lambda value: sa.case(whens=[(value.is_(None), sa.null()), (value != sa.literal(0), 1)], else_=0), + ), + ] + return_flags = 0 # type: ignore + + +DEFINITIONS_TYPE = [ + # bool + base.FuncBoolFromNull( + variants=[ + V(D.MSSQLSRV, lambda _: sa.null()), + ] + ), + MSSQLFuncBoolFromNumber(), + base.FuncBoolFromBool.for_dialect(D.MSSQLSRV), + base.FuncBoolFromStrGeo.for_dialect(D.MSSQLSRV), + base.FuncBoolFromDateDatetime.for_dialect(D.MSSQLSRV), + # date + base.FuncDate1FromNull.for_dialect(D.MSSQLSRV), + base.FuncDate1FromDatetime.for_dialect(D.MSSQLSRV), + base.FuncDate1FromString.for_dialect(D.MSSQLSRV), + base.FuncDate1FromNumber( + variants=[ + V(D.MSSQLSRV, lambda expr: sa.cast(sa.func.DATEADD(sa.text("SECOND"), expr, EPOCH_START_S), sa.Date())), + ] + ), + # datetime + base.FuncDatetime1FromNull.for_dialect(D.MSSQLSRV), + base.FuncDatetime1FromDatetime.for_dialect(D.MSSQLSRV), + base.FuncDatetime1FromDate.for_dialect(D.MSSQLSRV), + base.FuncDatetime1FromNumber( + variants=[ + V(D.MSSQLSRV, lambda expr: sa.func.DATEADD(sa.text("SECOND"), expr, EPOCH_START_S)), + ] + ), + base.FuncDatetime1FromString.for_dialect(D.MSSQLSRV), + # datetimetz + base.FuncDatetimeTZConst.for_dialect(D.MSSQLSRV), + # float + base.FuncFloatNumber( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(value, sa.FLOAT)), + ] + ), + base.FuncFloatString( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(value, sa.FLOAT)), + ] + ), + base.FuncFloatFromBool( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(value, sa.FLOAT)), + ] + ), + base.FuncFloatFromDate( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), EPOCH_START_S, value), sa.FLOAT)), + ] + ), + base.FuncFloatFromDatetime( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), EPOCH_START_S, value), sa.FLOAT)), + ] + ), + base.FuncFloatFromGenericDatetime( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), EPOCH_START_S, value), sa.FLOAT)), + ] + ), + # genericdatetime + base.FuncGenericDatetime1FromNull.for_dialect(D.MSSQLSRV), + base.FuncGenericDatetime1FromDatetime.for_dialect(D.MSSQLSRV), + base.FuncGenericDatetime1FromDate.for_dialect(D.MSSQLSRV), + base.FuncGenericDatetime1FromNumber( + variants=[ + V(D.MSSQLSRV, lambda expr: sa.func.DATEADD(sa.text("SECOND"), expr, EPOCH_START_S)), + ] + ), + base.FuncGenericDatetime1FromString.for_dialect(D.MSSQLSRV), + # geopoint + base.FuncGeopointFromStr.for_dialect(D.MSSQLSRV), + base.FuncGeopointFromCoords.for_dialect(D.MSSQLSRV), + # geopolygon + base.FuncGeopolygon.for_dialect(D.MSSQLSRV), + # int + base.FuncIntFromNull( + variants=[ + V(D.MSSQLSRV, lambda _: sa.cast(sa.null(), sa.BIGINT())), + ] + ), + base.FuncIntFromInt.for_dialect(D.MSSQLSRV), + base.FuncIntFromFloat( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(value, sa.BIGINT())), + ] + ), + base.FuncIntFromBool( + variants=[ + V(D.MSSQLSRV, lambda value: value), + ] + ), + base.FuncIntFromStr( + variants=[ + V(D.MSSQLSRV, lambda value: sa.cast(value, sa.BIGINT)), + ] + ), + base.FuncIntFromDate( + variants=[ + V( + D.MSSQLSRV, + lambda value: sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), EPOCH_START_S, value), sa.BIGINT()), + ), + ] + ), + base.FuncIntFromDatetime( + variants=[ + V( + D.MSSQLSRV, + lambda value: sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), EPOCH_START_S, value), sa.BIGINT()), + ), + ] + ), + base.FuncIntFromGenericDatetime( + variants=[ + V( + D.MSSQLSRV, + lambda value: sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), EPOCH_START_S, value), sa.BIGINT()), + ), + ] + ), + # str + base.FuncStrFromUnsupported( + variants=[ + V(D.MSSQLSRV, lambda value: sa.func.TRIM(sa.cast(value, sa.CHAR))), + ] + ), + base.FuncStrFromInteger( + variants=[ + V(D.MSSQLSRV, lambda value: sa.func.TRIM(sa.cast(value, sa.CHAR))), + ] + ), + base.FuncStrFromFloat( + variants=[ + # "128" is deprecated constant value, but it seems, currently it's the only + # universal (for all mssqlserver versions) value to stringify float as is. + # Also there is a little untidiness with float zero: STR(0.0) -> "0.0E0" + # https://stackoverflow.com/questions/3715675/how-to-convert-float-to-varchar-in-sql-server/24909501#24909501 + V(D.MSSQLSRV, lambda value: sa.func.CONVERT(sa.text("VARCHAR"), value, 128)), + ] + ), + base.FuncStrFromBool( + variants=[ + V( + D.MSSQLSRV, + lambda value: sa.case( + whens=[(value.is_(None), sa.null()), (value != sa.literal(0), "True")], else_="False" + ), + ), + ] + ), + base.FuncStrFromStrGeo.for_dialect(D.MSSQLSRV), + base.FuncStrFromDate( + variants=[ + V(D.MSSQLSRV, lambda value: sa.func.CONVERT(sa.text("VARCHAR"), value, 23)), + ] + ), + base.FuncStrFromDatetime( + variants=[ + V(D.MSSQLSRV, lambda value: sa.func.CONVERT(sa.text("VARCHAR"), value, 120)), + ] + ), + base.FuncStrFromString.for_dialect(D.MSSQLSRV), + base.FuncStrFromUUID( + variants=[ + V(D.MSSQLSRV, lambda value: sa.func.CONVERT(sa.text("VARCHAR"), value)), + ] + ), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_binary.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_binary.py new file mode 100644 index 000000000..eb0ce3ecb --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_binary.py @@ -0,0 +1,156 @@ +import sqlalchemy as sa +import sqlalchemy.dialects.mssql as sa_mssqlsrv + +from dl_formula.definitions.base import TranslationVariant +from dl_formula.definitions.common_datetime import DAY_SEC +import dl_formula.definitions.operators_binary as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_BINARY = [ + # != + base.BinaryNotEqual.for_dialect(D.MSSQLSRV), + # % + base.BinaryModInteger.for_dialect(D.MSSQLSRV), + base.BinaryModFloat( + variants=[ + V(D.MSSQLSRV, lambda left, right: left - sa.func.FLOOR(left / right) * right), + ] + ), + # * + base.BinaryMultNumbers.for_dialect(D.MSSQLSRV), + base.BinaryMultStringConst.for_dialect(D.MSSQLSRV), + base.BinaryMultStringNonConst( + variants=[ + V(D.MSSQLSRV, sa.func.REPLICATE), + ] + ), + # + + base.BinaryPlusNumbers.for_dialect(D.MSSQLSRV), + base.BinaryPlusStrings.for_dialect(D.MSSQLSRV), + base.BinaryPlusDateInt( + variants=[ + V(D.MSSQLSRV, lambda date, days: sa.type_coerce(sa.func.DATEADD(sa.text("day"), days, date), sa.Date)), + ] + ), + base.BinaryPlusDateFloat( + variants=[ + V( + D.MSSQLSRV, + lambda date, days: sa.type_coerce(sa.func.DATEADD(sa.text("day"), base.as_bigint(days), date), sa.Date), + ), + ] + ), + base.BinaryPlusDatetimeNumber( + variants=[ + V(D.MSSQLSRV, lambda dt, days: sa.func.DATEADD(sa.text("second"), days * DAY_SEC, dt)), + ] + ), + base.BinaryPlusGenericDatetimeNumber( + variants=[ + V(D.MSSQLSRV, lambda dt, days: sa.func.DATEADD(sa.text("second"), days * DAY_SEC, dt)), + ] + ), + # - + base.BinaryMinusNumbers.for_dialect(D.MSSQLSRV), + base.BinaryMinusDateInt( + variants=[ + V(D.MSSQLSRV, lambda date, days: sa.type_coerce(sa.func.DATEADD(sa.text("day"), -days, date), sa.Date)), + ] + ), + base.BinaryMinusDateFloat( + variants=[ + V( + D.MSSQLSRV, + lambda date, days: sa.type_coerce( + sa.func.DATEADD(sa.text("day"), -base.as_bigint(sa.func.CEILING(days)), date), sa.Date + ), + ), + ] + ), + base.BinaryMinusDatetimeNumber( + variants=[ + V(D.MSSQLSRV, lambda dt, days: sa.func.DATEADD(sa.text("second"), -days * DAY_SEC, dt)), + ] + ), + base.BinaryMinusGenericDatetimeNumber( + variants=[ + V(D.MSSQLSRV, lambda dt, days: sa.func.DATEADD(sa.text("second"), -days * DAY_SEC, dt)), + ] + ), + base.BinaryMinusDates( + variants=[ + V( + D.MSSQLSRV, + lambda left, right: sa.cast( + sa.func.FLOOR( + sa.cast(sa.cast(left, sa.DateTime()), sa_mssqlsrv.FLOAT) + - sa.cast(sa.cast(right, sa.DateTime()), sa_mssqlsrv.FLOAT) + ), + sa.BIGINT(), + ), + ), + ] + ), + base.BinaryMinusDatetimes( + variants=[ + V( + D.MSSQLSRV, + lambda left, right: ( + sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), right, left), sa_mssqlsrv.FLOAT) / DAY_SEC + ), + ), + ] + ), + base.BinaryMinusGenericDatetimes( + variants=[ + V( + D.MSSQLSRV, + lambda left, right: ( + sa.cast(sa.func.DATEDIFF(sa.text("SECOND"), right, left), sa_mssqlsrv.FLOAT) / DAY_SEC + ), + ), + ] + ), + # / + base.BinaryDivInt( + variants=[ + V(D.MSSQLSRV, lambda x, y: sa.cast(x, sa.FLOAT) / y), + ] + ), + base.BinaryDivFloat.for_dialect(D.MSSQLSRV), + # < + base.BinaryLessThan.for_dialect(D.MSSQLSRV), + # <= + base.BinaryLessThanOrEqual.for_dialect(D.MSSQLSRV), + # == + base.BinaryEqual.for_dialect(D.MSSQLSRV), + # > + base.BinaryGreaterThan.for_dialect(D.MSSQLSRV), + # >= + base.BinaryGreaterThanOrEqual.for_dialect(D.MSSQLSRV), + # ^ + base.BinaryPower.for_dialect(D.MSSQLSRV), + # _!= + base.BinaryNotEqualInternal.for_dialect(D.MSSQLSRV), + # _== + base.BinaryEqualInternal.for_dialect(D.MSSQLSRV), + # _dneq + base.BinaryEqualDenullified.for_dialect(D.MSSQLSRV), + # and + base.BinaryAnd.for_dialect(D.MSSQLSRV), + # in + base.BinaryIn.for_dialect(D.MSSQLSRV), + # like + base.BinaryLike.for_dialect(D.MSSQLSRV), + # notin + base.BinaryNotIn.for_dialect(D.MSSQLSRV), + # notlike + base.BinaryNotLike.for_dialect(D.MSSQLSRV), + # or + base.BinaryOr.for_dialect(D.MSSQLSRV), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_ternary.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_ternary.py new file mode 100644 index 000000000..a822324dc --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_ternary.py @@ -0,0 +1,11 @@ +import dl_formula.definitions.operators_ternary as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +DEFINITIONS_TERNARY = [ + # between + base.TernaryBetween.for_dialect(D.MSSQLSRV), + # notbetween + base.TernaryNotBetween.for_dialect(D.MSSQLSRV), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_unary.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_unary.py new file mode 100644 index 000000000..8159d0376 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/definitions/operators_unary.py @@ -0,0 +1,50 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.operators_unary as base + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_UNARY = [ + # isfalse + base.UnaryIsFalseStringGeo.for_dialect(D.MSSQLSRV), + base.UnaryIsFalseNumbers.for_dialect(D.MSSQLSRV), + base.UnaryIsFalseDateTime( + variants=[ + V(D.MSSQLSRV, lambda x: sa.literal(0)), + ] + ), + base.UnaryIsFalseBoolean( + variants=[ + V(D.MSSQLSRV, lambda x: x == 0), + ] + ), + # istrue + base.UnaryIsTrueStringGeo.for_dialect(D.MSSQLSRV), + base.UnaryIsTrueNumbers.for_dialect(D.MSSQLSRV), + base.UnaryIsTrueDateTime( + variants=[ + V(D.MSSQLSRV, lambda x: sa.literal(1)), + ] + ), + base.UnaryIsTrueBoolean( + variants=[ + V(D.MSSQLSRV, lambda x: x != 0), + ] + ), + # neg + base.UnaryNegate.for_dialect(D.MSSQLSRV), + # not + base.UnaryNotBool.for_dialect(D.MSSQLSRV), + base.UnaryNotNumbers.for_dialect(D.MSSQLSRV), + base.UnaryNotStringGeo.for_dialect(D.MSSQLSRV), + base.UnaryNotDateDatetime( + variants=[ + V(D.MSSQLSRV, lambda x: sa.literal(0)), + ] + ), +] diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/literal.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/literal.py new file mode 100644 index 000000000..94640b1b3 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/literal.py @@ -0,0 +1,17 @@ +import datetime + +import sqlalchemy as sa +import sqlalchemy.dialects.mssql.base as sa_mssql + +from dl_formula.connectors.base.literal import ( + Literal, + Literalizer, +) +from dl_formula.core.dialect import DialectCombo + + +class MSSQLLiteralizer(Literalizer): + __slots__ = () + + def literal_datetime(self, value: datetime.datetime, dialect: DialectCombo) -> Literal: + return sa.cast(value.isoformat(), sa_mssql.DATETIMEOFFSET) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula/type_constructor.py b/lib/dl_connector_mssql/dl_connector_mssql/formula/type_constructor.py new file mode 100644 index 000000000..9649d412b --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula/type_constructor.py @@ -0,0 +1,17 @@ +import sqlalchemy.dialects.mssql as sa_mssqlsrv +from sqlalchemy.types import TypeEngine + +from dl_formula.connectors.base.type_constructor import DefaultSATypeConstructor +from dl_formula.core.datatype import DataType + + +class MSSQLTypeConstructor(DefaultSATypeConstructor): + def get_sa_type(self, data_type: DataType) -> TypeEngine: + type_map: dict[DataType, TypeEngine] = { + DataType.BOOLEAN: sa_mssqlsrv.BIT(), + DataType.UUID: sa_mssqlsrv.UNIQUEIDENTIFIER(), + } + if (type_eng := type_map.get(data_type)) is not None: + return type_eng + else: + return super().get_sa_type(data_type) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/human_dialects.py b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/human_dialects.py new file mode 100644 index 000000000..03c83e9e1 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/human_dialects.py @@ -0,0 +1,13 @@ +from dl_formula_ref.texts import StyledDialect + +from dl_connector_mssql.formula.constants import MssqlDialect +from dl_connector_mssql.formula_ref.i18n import Translatable + + +HUMAN_DIALECTS = { + MssqlDialect.MSSQLSRV_14_0: StyledDialect( + "`Microsoft SQL Server 2017 (14.0)`", + "`Microsoft`
`SQL Server 2017`
`(14.0)`", + Translatable("`Microsoft SQL Server` version `2017 (14.0)`"), + ), +} diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/i18n.py b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/i18n.py new file mode 100644 index 000000000..6f448e1df --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/i18n.py @@ -0,0 +1,23 @@ +import os + +import attr + +from dl_i18n.localizer_base import Translatable as BaseTranslatable +from dl_i18n.localizer_base import TranslationConfig + +import dl_connector_mssql as package + + +DOMAIN = f"dl_formula_ref_{package.__name__}" + +_LOCALE_DIR = os.path.join(os.path.dirname(__file__), "..", "locales") + +CONFIGS = [ + TranslationConfig(path=_LOCALE_DIR, domain=DOMAIN, locale="en"), + TranslationConfig(path=_LOCALE_DIR, domain=DOMAIN, locale="ru"), +] + + +@attr.s +class Translatable(BaseTranslatable): + domain: str = attr.ib(default=DOMAIN) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/plugin.py b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/plugin.py new file mode 100644 index 000000000..59123ca15 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/formula_ref/plugin.py @@ -0,0 +1,12 @@ +from dl_formula_ref.plugins.base.plugin import FormulaRefPlugin + +from dl_connector_mssql.formula.constants import MssqlDialect +from dl_connector_mssql.formula_ref.human_dialects import HUMAN_DIALECTS +from dl_connector_mssql.formula_ref.i18n import CONFIGS + + +class MSSQLFormulaRefPlugin(FormulaRefPlugin): + any_dialects = frozenset((*MssqlDialect.MSSQLSRV.to_list(),)) + compeng_support_dialects = frozenset((*MssqlDialect.MSSQLSRV.to_list(),)) + human_dialects = HUMAN_DIALECTS + translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_connector_mssql.mo b/lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_connector_mssql.mo new file mode 100644 index 0000000000000000000000000000000000000000..4e8e46befbd2e3ec2674d314be4310e217917421 GIT binary patch literal 301 zcmY++!AiqG5C-5?^b+t=@Futq;8eYps9-Hog2tvz>{Yg#F-FqaIJ=SdL3|e9#%FPQ zD*ntjFfjbTgM)7+*%v3`OdN`9u}v +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-10-16 14:32+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "label_connector-mssql" +msgstr "MS SQL Server" diff --git a/lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.mo b/lib/dl_connector_mssql/dl_connector_mssql/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.mo new file mode 100644 index 0000000000000000000000000000000000000000..00261bf604624733771edc1595785d8dd496d13e GIT binary patch literal 241 zcmYMsF>b;@5QSk-Qn{o~u|-M-Nar|;1Xu=H$O$42{l+_(MP}C89ZPVKoF%v6EDTNf zrEjXIxeOkEds1IKiD&U3hT>K%{=}=eZT?$(cfg7vH?Tz?tfNG&XX$FT%+xm9eC3}X z(FR-R8H!L_1M{v9P`X4J&ALGf;99w^^A%3dnhc(RQ6ERXi;BF+?r&d4lPGn>NDSq< q#Y8clc-%JH(wnOG+6UzK?{We0edjfW9eT*9a>i1fsQAZn)Vl(gx +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-10-16 14:32+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "`Microsoft SQL Server` version `2017 (14.0)`" +msgstr "" diff --git a/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_connector_mssql.mo b/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_connector_mssql.mo new file mode 100644 index 0000000000000000000000000000000000000000..4e8e46befbd2e3ec2674d314be4310e217917421 GIT binary patch literal 301 zcmY++!AiqG5C-5?^b+t=@Futq;8eYps9-Hog2tvz>{Yg#F-FqaIJ=SdL3|e9#%FPQ zD*ntjFfjbTgM)7+*%v3`OdN`9u}v +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-10-16 14:32+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "label_connector-mssql" +msgstr "MS SQL Server" diff --git a/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.mo b/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.mo new file mode 100644 index 0000000000000000000000000000000000000000..35d015b41cf9aab700c73f34bde04ea382200429 GIT binary patch literal 360 zcmZ|K!AiqG5C-5VddbnVhdnB|F%i)c6|5zdpwYC6J!eg(ai#8t-Ho&t(TDIw1hJ2i zzJSl-M8SgK&wO)OW)|jiW9`+#tfEb{gVxb8Y7?XOZGb+|6Y|i(ie+Jar``Qc7V%t3 zEyso=gG-WtdIXgbY?@0!vOU*3BwOCT<8EhGtmHl9hK2JUeL3;)HIboDVSjs5Sq#gyR20Xr!yuBm{J4m!BMpxg;B8l)^l9DCxroF zXgaIFCkCcwx0*06j!1sTlm>HhllEz6Ij5M=V^A~@xh%LSebTw*#1rhC}o8J literal 0 HcmV?d00001 diff --git a/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.po b/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.po new file mode 100644 index 000000000..c9d470c8d --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_mssql.po @@ -0,0 +1,13 @@ +# Copyright (c) 2023 YANDEX LLC +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-10-16 14:32+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "`Microsoft SQL Server` version `2017 (14.0)`" +msgstr "`Microsoft SQL Server` версии `2017 (14.0)`" diff --git a/lib/dl_connector_mssql/dl_connector_mssql/py.typed b/lib/dl_connector_mssql/dl_connector_mssql/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py new file mode 100644 index 000000000..e4c76262a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/base.py @@ -0,0 +1,58 @@ +import pytest + +from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration +from dl_api_lib_testing.connection_base import ConnectionTestBase +from dl_api_lib_testing.data_api_base import StandardizedDataApiTestBase +from dl_api_lib_testing.dataset_base import DatasetTestBase +from dl_constants.enums import RawSQLLevel + +from dl_connector_mssql.core.constants import ( + CONNECTION_TYPE_MSSQL, + SOURCE_TYPE_MSSQL_TABLE, +) +from dl_connector_mssql_tests.db.config import ( + API_TEST_CONFIG, + CoreConnectionSettings, +) +from dl_connector_mssql_tests.db.core.base import BaseMSSQLTestClass + + +class MSSQLConnectionTestBase(BaseMSSQLTestClass, ConnectionTestBase): + conn_type = CONNECTION_TYPE_MSSQL + bi_compeng_pg_on = False + + @pytest.fixture(scope="class") + def bi_test_config(self) -> ApiTestEnvironmentConfiguration: + return API_TEST_CONFIG + + @pytest.fixture(scope="class") + def connection_params(self) -> dict: + return dict( + db_name=CoreConnectionSettings.DB_NAME, + host=CoreConnectionSettings.HOST, + port=CoreConnectionSettings.PORT, + username=CoreConnectionSettings.USERNAME, + password=CoreConnectionSettings.PASSWORD, + **(dict(raw_sql_level=self.raw_sql_level.value) if self.raw_sql_level is not None else {}), + ) + + +class MSSQLDashSQLConnectionTest(MSSQLConnectionTestBase): + raw_sql_level = RawSQLLevel.dashsql + + +class MSSQLDatasetTestBase(MSSQLConnectionTestBase, DatasetTestBase): + @pytest.fixture(scope="class") + def dataset_params(self, sample_table) -> dict: + return dict( + source_type=SOURCE_TYPE_MSSQL_TABLE.name, + parameters=dict( + db_name=sample_table.db.name, + schema_name=sample_table.schema, + table_name=sample_table.name, + ), + ) + + +class MSSQLDataApiTestBase(MSSQLDatasetTestBase, StandardizedDataApiTestBase): + mutation_caches_on = False diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_complex_queries.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_complex_queries.py new file mode 100644 index 000000000..cb640c225 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_complex_queries.py @@ -0,0 +1,12 @@ +from dl_api_lib_testing.connector.complex_queries import DefaultBasicComplexQueryTestSuite +from dl_testing.regulated_test import RegulatedTestParams + +from dl_connector_mssql_tests.db.api.base import MSSQLDataApiTestBase + + +class TestMSSQLBasicComplexQueries(MSSQLDataApiTestBase, DefaultBasicComplexQueryTestSuite): + test_params = RegulatedTestParams( + mark_features_skipped={ + DefaultBasicComplexQueryTestSuite.feature_window_functions: "Native window functions are not implemented" + } + ) diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_connection.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_connection.py new file mode 100644 index 000000000..f882ad3f6 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_connection.py @@ -0,0 +1,7 @@ +from dl_api_lib_testing.connector.connection_suite import DefaultConnectorConnectionTestSuite + +from dl_connector_mssql_tests.db.api.base import MSSQLConnectionTestBase + + +class TestMSSQLConnection(MSSQLConnectionTestBase, DefaultConnectorConnectionTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dashsql.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dashsql.py new file mode 100644 index 000000000..61200c94c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dashsql.py @@ -0,0 +1,116 @@ +from aiohttp.test_utils import TestClient +import pytest + +from dl_api_lib_testing.connector.dashsql_suite import DefaultDashSQLTestSuite + +from dl_connector_mssql_tests.db.api.base import MSSQLDashSQLConnectionTest +from dl_connector_mssql_tests.db.config import SUBSELECT_QUERY_FULL + + +class TestMSSQLDashSQL(MSSQLDashSQLConnectionTest, DefaultDashSQLTestSuite): + @pytest.mark.asyncio + async def test_result(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str): + resp = await self.get_dashsql_response( + data_api_aio=data_api_lowlevel_aiohttp_client, + conn_id=saved_connection_id, + query=SUBSELECT_QUERY_FULL, + ) + + resp_data = await resp.json() + assert resp_data[0]["event"] == "metadata", resp_data + assert resp_data[0]["data"]["names"] == [ + "number", + "num_tinyint", + "num_smallint", + "num_integer", + "num_bigint", + "num_float", + "num_real", + "num_numeric", + "num_decimal", + "num_bit", + "num_char", + "num_varchar", + "num_text", + "num_nchar", + "num_nvarchar", + "num_ntext", + "num_date", + "num_datetime", + "num_datetime2", + "num_smalldatetime", + "num_datetimeoffset", + "uuid", + ] + assert resp_data[0]["data"]["driver_types"] == [ + "int", + "int", + "int", + "int", + "int", + "float", + "float", + "decimal", + "decimal", + "bool", + "str", + "str", + "str", + "str", + "str", + "str", + "str", + "datetime", + "str", + "datetime", + "str", + "str", + ] + assert resp_data[0]["data"]["db_types"] == [ + "integer", + "integer", + "integer", + "integer", + "integer", + "float", + "float", + "decimal", + "decimal", + "bit", + "ntext", + "ntext", + "ntext", + "ntext", + "ntext", + "ntext", + "ntext", + "datetime", + "ntext", + "datetime", + "ntext", + "ntext", + ] + assert resp_data[0]["data"]["bi_types"] == [ + "integer", + "integer", + "integer", + "integer", + "integer", + "float", + "float", + "float", + "float", + "boolean", + "string", + "string", + "string", + "string", + "string", + "string", + "string", + "genericdatetime", + "string", + "genericdatetime", + "string", + "string", + ] diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_data.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_data.py new file mode 100644 index 000000000..786d8fa0a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_data.py @@ -0,0 +1,34 @@ +from dl_api_lib_testing.connector.data_api_suites import ( + DefaultConnectorDataDistinctTestSuite, + DefaultConnectorDataGroupByFormulaTestSuite, + DefaultConnectorDataPreviewTestSuite, + DefaultConnectorDataRangeTestSuite, + DefaultConnectorDataResultTestSuite, +) +from dl_testing.regulated_test import RegulatedTestParams + +from dl_connector_mssql_tests.db.api.base import MSSQLDataApiTestBase + + +class TestMSSQLDataResult(MSSQLDataApiTestBase, DefaultConnectorDataResultTestSuite): + test_params = RegulatedTestParams( + mark_features_skipped={ + DefaultConnectorDataResultTestSuite.array_support: "MSSQL doesn't support arrays", + } + ) + + +class TestMSSQLDataGroupBy(MSSQLDataApiTestBase, DefaultConnectorDataGroupByFormulaTestSuite): + pass + + +class TestMSSQLDataRange(MSSQLDataApiTestBase, DefaultConnectorDataRangeTestSuite): + pass + + +class TestMSSQLDataDistinct(MSSQLDataApiTestBase, DefaultConnectorDataDistinctTestSuite): + pass + + +class TestMSSQLDataPreview(MSSQLDataApiTestBase, DefaultConnectorDataPreviewTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dataset.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dataset.py new file mode 100644 index 000000000..e4f1ebe1c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/api/test_dataset.py @@ -0,0 +1,7 @@ +from dl_api_lib_testing.connector.dataset_suite import DefaultConnectorDatasetTestSuite + +from dl_connector_mssql_tests.db.api.base import MSSQLDatasetTestBase + + +class TestMSSQLDataset(MSSQLDatasetTestBase, DefaultConnectorDatasetTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py new file mode 100644 index 000000000..d2b9a04e3 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/config.py @@ -0,0 +1,90 @@ +import os +from typing import ClassVar + +from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration +from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_testing.containers import get_test_container_hostport + +from dl_connector_mssql.formula.constants import MssqlDialect as D + + +# Infra settings +CORE_TEST_CONFIG = DefaultCoreTestConfiguration( + host_us_http=get_test_container_hostport("us", fallback_port=52111).host, + port_us_http=get_test_container_hostport("us", fallback_port=52111).port, + host_us_pg=get_test_container_hostport("pg-us", fallback_port=52110).host, + port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=52110).port, + us_master_token="AC1ofiek8coB", + core_connector_ep_names=["mssql"], +) + +COMPOSE_PROJECT_NAME = os.environ.get("COMPOSE_PROJECT_NAME", "dl_connector_mssql") +MSSQL_CONTAINER_LABEL = "db-mssql-14" + + +class CoreConnectionSettings: + DB_NAME: ClassVar[str] = "test_data" + HOST: ClassVar[str] = get_test_container_hostport("db-mssql-14", fallback_port=52100).host + PORT: ClassVar[int] = get_test_container_hostport("db-mssql-14", fallback_port=52100).port + USERNAME: ClassVar[str] = "datalens" + PASSWORD: ClassVar[str] = "qweRTY123" + + +SUBSELECT_QUERY_FULL = r""" +select + number, + cast(number as tinyint) as num_tinyint, + cast(number as smallint) as num_smallint, + cast(number as integer) as num_integer, + cast(number as bigint) as num_bigint, + cast(number as float) as num_float, + cast(number as real) as num_real, + cast(number as numeric) as num_numeric, + cast(number as decimal) as num_decimal, + cast(number as bit) as num_bit, + cast(number as char) as num_char, + cast(number as varchar) as num_varchar, + cast(cast(number as varchar) as text) as num_text, + cast(number as nchar) as num_nchar, + cast(number as nvarchar) as num_nvarchar, + cast(cast(number as nvarchar) as ntext) as num_ntext, + cast(concat('2020-01-0', number + 1) as date) as num_date, + cast(number as datetime) as num_datetime, + cast(concat('2020-01-01T00:00:0', + number) as datetime2) as num_datetime2, + cast(number as smalldatetime) as num_smalldatetime, + cast(concat('2020-01-01T00:00:00+00:0', + number) as datetimeoffset) as num_datetimeoffset, + NEWID() as uuid +from + ( + select 0 as number + union all + select 1 as number + union all + select 6 as number + ) as base +""" + +_DB_URL = ( + f'mssql+pyodbc:///?odbc_connect=DRIVER%3D%7BFreeTDS%7D%3BServer%3D{get_test_container_hostport("db-mssql-14", fallback_port=52100).host}%3B' + f'Port%3D{get_test_container_hostport("db-mssql-14", fallback_port=52100).port}%3BDatabase%3Dtest_data%3BUID%3Ddatalens%3BPWD%3DqweRTY123%3BTDS_Version%3D8.0' +) +ADMIN_URL = ( + f'mssql+pyodbc:///?odbc_connect=DRIVER%3D%7BFreeTDS%7D%3BServer%3D{get_test_container_hostport("db-mssql-14", fallback_port=52100).host}%3B' + f'Port%3D{get_test_container_hostport("db-mssql-14", fallback_port=52100).port}%3BUID%3Dsa%3BPWD%3DqweRTY123%3BTDS_Version%3D8.0' +) +ADMIN_W_DB_URL = ( + f'mssql+pyodbc:///?odbc_connect=DRIVER%3D%7BFreeTDS%7D%3BServer%3D{get_test_container_hostport("db-mssql-14", fallback_port=52100).host}%3B' + f'Port%3D{get_test_container_hostport("db-mssql-14", fallback_port=52100).port}%3BDatabase%3Dtest_data%3BUID%3Dsa%3BPWD%3DqweRTY123%3BTDS_Version%3D8.0' +) +DB_CORE_URL = _DB_URL +DB_URLS = { + D.MSSQLSRV_14_0: _DB_URL, +} + +API_TEST_CONFIG = ApiTestEnvironmentConfiguration( + api_connector_ep_names=["mssql"], + core_test_config=CORE_TEST_CONFIG, + ext_query_executer_secret_key="_some_test_secret_key_", +) diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/conftest.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/conftest.py new file mode 100644 index 000000000..66d59d176 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/conftest.py @@ -0,0 +1,64 @@ +from frozendict import frozendict + +from dl_api_lib_testing.initialization import initialize_api_lib_test +from dl_core_testing.database import ( + CoreDbConfig, + CoreDbDispenser, +) +from dl_db_testing.database.engine_wrapper import DbEngineConfig +from dl_formula_testing.forced_literal import forced_literal_use + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL +from dl_connector_mssql_tests.db.config import ( + ADMIN_URL, + ADMIN_W_DB_URL, + API_TEST_CONFIG, +) + + +pytest_plugins = ("aiohttp.pytest_plugin",) # and it, in turn, includes 'pytest_asyncio.plugin' + + +def pytest_configure(config): # noqa + initialize_api_lib_test(pytest_config=config, api_test_config=API_TEST_CONFIG) + initialize_db() + + +def initialize_db(): + db_dispenser = CoreDbDispenser() + # Use the admin URL since we don't have a database or a user yet + admin_db_config = CoreDbConfig( + engine_config=DbEngineConfig( + url=ADMIN_URL, + engine_params=frozendict({"connect_args": frozendict({"autocommit": True})}), + ), + conn_type=CONNECTION_TYPE_MSSQL, + ) + admin_db = db_dispenser.get_database(db_config=admin_db_config) + admin_db.execute("sp_configure 'CONTAINED DATABASE AUTHENTICATION', 1") + admin_db.execute("RECONFIGURE") + admin_db.execute( + "IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = 'test_data') " + "BEGIN CREATE DATABASE [test_data] CONTAINMENT = PARTIAL END" + ) + + # Switch to the admin URL with the DB name + admin_db_config = CoreDbConfig( + engine_config=DbEngineConfig( + url=ADMIN_W_DB_URL, + engine_params=frozendict({"connect_args": frozendict({"autocommit": True})}), + ), + conn_type=CONNECTION_TYPE_MSSQL, + ) + admin_db = db_dispenser.get_database(db_config=admin_db_config) + admin_db.execute( + "IF NOT EXISTS(SELECT * FROM sys.database_principals where name = 'datalens') " + "CREATE USER datalens WITH PASSWORD = 'qweRTY123'" + ) + admin_db.execute("GRANT CREATE TABLE, ALTER, INSERT, SELECT, UPDATE, DELETE TO datalens") + + +__all__ = ( + # auto-use fixtures: + "forced_literal_use", +) diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/base.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/base.py new file mode 100644 index 000000000..8c683e91c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/base.py @@ -0,0 +1,57 @@ +import asyncio +from typing import Generator + +import pytest + +from dl_core.us_manager.us_manager_sync import SyncUSManager +from dl_core_testing.environment import restart_container_by_label +from dl_core_testing.testcases.connection import BaseConnectionTestClass + +from dl_connector_mssql.core.constants import CONNECTION_TYPE_MSSQL +from dl_connector_mssql.core.testing.connection import make_mssql_saved_connection +from dl_connector_mssql.core.us_connection import ConnectionMSSQL +import dl_connector_mssql_tests.db.config as test_config + + +class BaseMSSQLTestClass(BaseConnectionTestClass[ConnectionMSSQL]): + conn_type = CONNECTION_TYPE_MSSQL + core_test_config = test_config.CORE_TEST_CONFIG + + @pytest.fixture(autouse=True) + # FIXME: This fixture is a temporary solution for failing core tests when they are run together with api tests + def loop(self, event_loop: asyncio.AbstractEventLoop) -> Generator[asyncio.AbstractEventLoop, None, None]: + asyncio.set_event_loop(event_loop) + yield event_loop + # Attempt to cover an old version of pytest-asyncio: + # https://github.com/pytest-dev/pytest-asyncio/commit/51d986cec83fdbc14fa08015424c79397afc7ad9 + asyncio.set_event_loop_policy(None) + + def db_reinit_hook(self) -> None: + restart_container_by_label( + label=test_config.MSSQL_CONTAINER_LABEL, + compose_project=test_config.COMPOSE_PROJECT_NAME, + ) + + @pytest.fixture(scope="class") + def db_url(self) -> str: + return test_config.DB_CORE_URL + + @pytest.fixture(scope="function") + def connection_creation_params(self) -> dict: + return dict( + db_name=test_config.CoreConnectionSettings.DB_NAME, + host=test_config.CoreConnectionSettings.HOST, + port=test_config.CoreConnectionSettings.PORT, + username=test_config.CoreConnectionSettings.USERNAME, + password=test_config.CoreConnectionSettings.PASSWORD, + **(dict(raw_sql_level=self.raw_sql_level) if self.raw_sql_level is not None else {}), + ) + + @pytest.fixture(scope="function") + def saved_connection( + self, + sync_us_manager: SyncUSManager, + connection_creation_params: dict, + ) -> ConnectionMSSQL: + conn = make_mssql_saved_connection(sync_usm=sync_us_manager, **connection_creation_params) + return conn diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection.py new file mode 100644 index 000000000..29eb2d27a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection.py @@ -0,0 +1,34 @@ +from dl_core.us_connection_base import DataSourceTemplate +from dl_core_testing.testcases.connection import DefaultConnectionTestClass + +from dl_connector_mssql.core.us_connection import ConnectionMSSQL +from dl_connector_mssql_tests.db.core.base import BaseMSSQLTestClass + + +class TestMSSQLConnection( + BaseMSSQLTestClass, + DefaultConnectionTestClass[ConnectionMSSQL], +): + do_check_data_export_flag = True + + def check_saved_connection(self, conn: ConnectionMSSQL, params: dict) -> None: + assert conn.uuid is not None + assert conn.data.db_name == params["db_name"] + assert conn.data.host == params["host"] + assert conn.data.port == params["port"] + assert conn.data.username == params["username"] + assert conn.data.password == params["password"] + + def check_data_source_templates( + self, + conn: ConnectionMSSQL, + dsrc_templates: list[DataSourceTemplate], + ) -> None: + assert dsrc_templates + for dsrc_tmpl in dsrc_templates: + assert dsrc_tmpl.title + if dsrc_tmpl.parameters.get("schema_name") is not None: + assert dsrc_tmpl.group == [dsrc_tmpl.parameters["schema_name"]] + + schema_names = {tmpl.parameters.get("schema_name") for tmpl in dsrc_templates} + assert "dbo" in schema_names diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection_executor.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection_executor.py new file mode 100644 index 000000000..f666db0a5 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_connection_executor.py @@ -0,0 +1,83 @@ +from typing import ( + Optional, + Sequence, +) + +import pytest +from sqlalchemy.dialects import mssql as mssql_types + +from dl_constants.enums import UserDataType +from dl_core.connection_models.common_models import DBIdent +from dl_core_testing.testcases.connection_executor import ( + DefaultAsyncConnectionExecutorTestSuite, + DefaultSyncAsyncConnectionExecutorCheckBase, + DefaultSyncConnectionExecutorTestSuite, +) +from dl_testing.regulated_test import RegulatedTestParams + +from dl_connector_mssql.core.us_connection import ConnectionMSSQL +from dl_connector_mssql_tests.db.config import CoreConnectionSettings +from dl_connector_mssql_tests.db.core.base import BaseMSSQLTestClass + + +class MSSQLSyncAsyncConnectionExecutorCheckBase( + BaseMSSQLTestClass, + DefaultSyncAsyncConnectionExecutorCheckBase[ConnectionMSSQL], +): + test_params = RegulatedTestParams( + mark_tests_failed={ + DefaultAsyncConnectionExecutorTestSuite.test_get_table_schema_info_for_nonexistent_table: ( + "Empty schema is returned instead of an error" + ), # FIXME + }, + ) + + @pytest.fixture(scope="function") + def db_ident(self) -> DBIdent: + return DBIdent(db_name=CoreConnectionSettings.DB_NAME) + + def check_db_version(self, db_version: Optional[str]) -> None: + assert db_version is not None + assert "." in db_version + + +class TestMSSQLSyncConnectionExecutor( + MSSQLSyncAsyncConnectionExecutorCheckBase, + DefaultSyncConnectionExecutorTestSuite[ConnectionMSSQL], +): + def get_schemas_for_type_recognition(self) -> dict[str, Sequence[DefaultSyncConnectionExecutorTestSuite.CD]]: + return { + "mssql_types_number": [ + self.CD(mssql_types.TINYINT(), UserDataType.integer), + self.CD(mssql_types.SMALLINT(), UserDataType.integer), + self.CD(mssql_types.INTEGER(), UserDataType.integer), + self.CD(mssql_types.BIGINT(), UserDataType.integer), + self.CD(mssql_types.FLOAT(), UserDataType.float), + self.CD(mssql_types.REAL(), UserDataType.float), + self.CD(mssql_types.NUMERIC(), UserDataType.float), + self.CD(mssql_types.DECIMAL(), UserDataType.float), + self.CD(mssql_types.BIT(), UserDataType.boolean), + ], + "mssql_types_text": [ + self.CD(mssql_types.CHAR(), UserDataType.string), + self.CD(mssql_types.VARCHAR(100), UserDataType.string), + self.CD(mssql_types.TEXT(), UserDataType.string), + self.CD(mssql_types.NCHAR(), UserDataType.string), + self.CD(mssql_types.NVARCHAR(100), UserDataType.string), + self.CD(mssql_types.NTEXT(), UserDataType.string), + ], + "mssql_types_date": [ + self.CD(mssql_types.DATE(), UserDataType.date), + self.CD(mssql_types.DATETIME(), UserDataType.genericdatetime), + self.CD(mssql_types.DATETIME2(), UserDataType.genericdatetime), + self.CD(mssql_types.SMALLDATETIME(), UserDataType.genericdatetime), + self.CD(mssql_types.DATETIMEOFFSET(), UserDataType.genericdatetime), + ], + } + + +class TestMSSQLAsyncConnectionExecutor( + MSSQLSyncAsyncConnectionExecutorCheckBase, + DefaultAsyncConnectionExecutorTestSuite[ConnectionMSSQL], +): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_data_source.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_data_source.py new file mode 100644 index 000000000..25b29460a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_data_source.py @@ -0,0 +1,114 @@ +import pytest + +from dl_constants.enums import ( + RawSQLLevel, + UserDataType, +) +from dl_core.data_source_spec.sql import ( + StandardSchemaSQLDataSourceSpec, + SubselectDataSourceSpec, +) +from dl_core.db import SchemaColumn +from dl_core_testing.fixtures.sample_tables import TABLE_SPEC_SAMPLE_SUPERSTORE +from dl_core_testing.testcases.data_source import ( + DataSourceTestByViewClass, + DefaultDataSourceTestClass, +) + +from dl_connector_mssql.core.constants import ( + SOURCE_TYPE_MSSQL_SUBSELECT, + SOURCE_TYPE_MSSQL_TABLE, +) +from dl_connector_mssql.core.data_source import ( + MSSQLDataSource, + MSSQLSubselectDataSource, +) +from dl_connector_mssql.core.us_connection import ConnectionMSSQL +from dl_connector_mssql_tests.db.config import SUBSELECT_QUERY_FULL +from dl_connector_mssql_tests.db.core.base import BaseMSSQLTestClass + + +class TestMSSQLTableDataSource( + BaseMSSQLTestClass, + DefaultDataSourceTestClass[ + ConnectionMSSQL, + StandardSchemaSQLDataSourceSpec, + MSSQLDataSource, + ], +): + DSRC_CLS = MSSQLDataSource + + @pytest.fixture(scope="class") + def initial_data_source_spec(self, sample_table) -> StandardSchemaSQLDataSourceSpec: + dsrc_spec = StandardSchemaSQLDataSourceSpec( + source_type=SOURCE_TYPE_MSSQL_TABLE, + db_name=sample_table.db.name, + schema_name=sample_table.schema, + table_name=sample_table.name, + ) + return dsrc_spec + + def get_expected_simplified_schema(self) -> list[tuple[str, UserDataType]]: + return list(TABLE_SPEC_SAMPLE_SUPERSTORE.table_schema) + + +class TestMSSQLSubselectDataSource( + BaseMSSQLTestClass, + DefaultDataSourceTestClass[ + ConnectionMSSQL, + SubselectDataSourceSpec, + MSSQLSubselectDataSource, + ], +): + DSRC_CLS = MSSQLSubselectDataSource + + raw_sql_level = RawSQLLevel.subselect + + @pytest.fixture(scope="class") + def initial_data_source_spec(self, sample_table) -> SubselectDataSourceSpec: + dsrc_spec = SubselectDataSourceSpec( + source_type=SOURCE_TYPE_MSSQL_SUBSELECT, + subsql=f'SELECT * FROM "{sample_table.name}"', + ) + return dsrc_spec + + def get_expected_simplified_schema(self) -> list[tuple[str, UserDataType]]: + remapped_types = {UserDataType.date: UserDataType.string} + expected_schema = [ + # MSSQL cannot identify dates in sub-queries correctly + (name, remapped_types.get(user_type, user_type)) + for name, user_type in TABLE_SPEC_SAMPLE_SUPERSTORE.table_schema + ] + return expected_schema + + +class TestMSSQLSubselectByView( + BaseMSSQLTestClass, + DataSourceTestByViewClass[ + ConnectionMSSQL, + SubselectDataSourceSpec, + MSSQLSubselectDataSource, + ], +): + DSRC_CLS = MSSQLSubselectDataSource + + raw_sql_level = RawSQLLevel.subselect + + @pytest.fixture(scope="session") + def initial_data_source_spec(self) -> SubselectDataSourceSpec: + dsrc_spec = SubselectDataSourceSpec( + source_type=SOURCE_TYPE_MSSQL_SUBSELECT, + subsql=SUBSELECT_QUERY_FULL, + ) + return dsrc_spec + + def postprocess_view_schema_column(self, schema_col: SchemaColumn) -> SchemaColumn: + # MSSQL subselect schema does not use a cursor-based types + # (it uses `sp_describe_first_result_set`) + # but it still manages to make some critical failures, + # representing date/datetime type as a string type. + if schema_col.native_type.name in ("date", "datetime2", "datetimeoffset"): + schema_col = schema_col.clone( + user_type=UserDataType.string, native_type=schema_col.native_type.clone(name="nvarchar") + ) + return schema_col diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_dataset.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_dataset.py new file mode 100644 index 000000000..637cd1a19 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/core/test_dataset.py @@ -0,0 +1,16 @@ +from dl_core_testing.testcases.dataset import DefaultDatasetTestSuite +from dl_testing.regulated_test import RegulatedTestParams + +from dl_connector_mssql.core.constants import SOURCE_TYPE_MSSQL_TABLE +from dl_connector_mssql.core.us_connection import ConnectionMSSQL +from dl_connector_mssql_tests.db.core.base import BaseMSSQLTestClass + + +class TestMSSQLDataset(BaseMSSQLTestClass, DefaultDatasetTestSuite[ConnectionMSSQL]): + source_type = SOURCE_TYPE_MSSQL_TABLE + + test_params = RegulatedTestParams( + mark_tests_failed={ + DefaultDatasetTestSuite.test_get_param_hash: "db_name in dsrc", # TODO: FIXME + }, + ) diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/base.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/base.py new file mode 100644 index 000000000..e175772ed --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/base.py @@ -0,0 +1,17 @@ +import pytest + +from dl_formula_testing.testcases.base import FormulaConnectorTestBase + +from dl_connector_mssql.formula.constants import MssqlDialect as D +from dl_connector_mssql_tests.db.config import DB_URLS + + +class MSSQLTestBase(FormulaConnectorTestBase): + dialect = D.MSSQLSRV_14_0 + supports_arrays = False + supports_uuid = True + bool_is_expression = True + + @pytest.fixture(scope="class") + def db_url(self) -> str: + return DB_URLS[self.dialect] diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_conditional_blocks.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_conditional_blocks.py new file mode 100644 index 000000000..23bec56fa --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_conditional_blocks.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.conditional_blocks import DefaultConditionalBlockFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestConditionalBlockMSSQL(MSSQLTestBase, DefaultConditionalBlockFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_aggregation.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_aggregation.py new file mode 100644 index 000000000..70a78cbf1 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_aggregation.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_aggregation import DefaultMainAggFunctionFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestMainAggFunctionMSSQL(MSSQLTestBase, DefaultMainAggFunctionFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_datetime.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_datetime.py new file mode 100644 index 000000000..fe20bb0bb --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_datetime.py @@ -0,0 +1,8 @@ +from dl_formula_testing.testcases.functions_datetime import DefaultDateTimeFunctionFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestDateTimeFunctionMSSQL(MSSQLTestBase, DefaultDateTimeFunctionFormulaConnectorTestSuite): + supports_deprecated_dateadd = True + supports_deprecated_datepart_2 = True diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_logical.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_logical.py new file mode 100644 index 000000000..909f7e8bc --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_logical.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_logical import DefaultLogicalFunctionFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestLogicalFunctionMSSQL(MSSQLTestBase, DefaultLogicalFunctionFormulaConnectorTestSuite): + supports_iif = True diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_markup.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_markup.py new file mode 100644 index 000000000..ee59cb398 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_markup.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_markup import DefaultMarkupFunctionFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestMarkupFunctionMSSQL(MSSQLTestBase, DefaultMarkupFunctionFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_math.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_math.py new file mode 100644 index 000000000..357519979 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_math.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_math import DefaultMathFunctionFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestMathFunctionMSSQL(MSSQLTestBase, DefaultMathFunctionFormulaConnectorTestSuite): + supports_atan_2_in_origin = False diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_string.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_string.py new file mode 100644 index 000000000..49262fd4a --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_string.py @@ -0,0 +1,21 @@ +import pytest + +from dl_formula_testing.testcases.functions_string import DefaultStringFunctionFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestStringFunctionMSSQL(MSSQLTestBase, DefaultStringFunctionFormulaConnectorTestSuite): + datetime_str_ending = " +00:00" + supports_regex_extract = False + supports_regex_extract_nth = False + supports_regex_replace = False + supports_regex_match = False + supports_split_3 = False + supports_non_const_percent_escape = False + + def test_contains_extended(self) -> None: # type: ignore + pytest.skip() # Override base test; default checks not supported + + def test_notcontains_extended(self) -> None: # type: ignore + pytest.skip() # Override base test; default checks not supported diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_type_conversion.py new file mode 100644 index 000000000..1e52492a1 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_functions_type_conversion.py @@ -0,0 +1,78 @@ +from dl_formula_testing.testcases.functions_type_conversion import ( + DefaultBoolTypeFunctionFormulaConnectorTestSuite, + DefaultDateTypeFunctionFormulaConnectorTestSuite, + DefaultFloatTypeFunctionFormulaConnectorTestSuite, + DefaultGenericDatetimeTypeFunctionFormulaConnectorTestSuite, + DefaultGeopointTypeFunctionFormulaConnectorTestSuite, + DefaultGeopolygonTypeFunctionFormulaConnectorTestSuite, + DefaultIntTypeFunctionFormulaConnectorTestSuite, + DefaultStrTypeFunctionFormulaConnectorTestSuite, +) + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +# STR + + +class TestStrTypeFunctionMSSQL(MSSQLTestBase, DefaultStrTypeFunctionFormulaConnectorTestSuite): + zero_float_to_str_value = "0.0E0" + skip_custom_tz = True + + +# FLOAT + + +class TestFloatTypeFunctionMSSQL(MSSQLTestBase, DefaultFloatTypeFunctionFormulaConnectorTestSuite): + pass + + +# BOOL + + +class TestBoolTypeFunctionMSSQL(MSSQLTestBase, DefaultBoolTypeFunctionFormulaConnectorTestSuite): + pass + + +# INT + + +class TestIntTypeFunctionMSSQL(MSSQLTestBase, DefaultIntTypeFunctionFormulaConnectorTestSuite): + pass + + +# DATE + + +class TestDateTypeFunctionMSSQL(MSSQLTestBase, DefaultDateTypeFunctionFormulaConnectorTestSuite): + pass + + +# GENERICDATETIME (& DATETIME) + + +class TestGenericDatetimeTypeFunctionMSSQL( + MSSQLTestBase, + DefaultGenericDatetimeTypeFunctionFormulaConnectorTestSuite, +): + pass + + +# GEOPOINT + + +class TestGeopointTypeFunctionMSSQL( + MSSQLTestBase, + DefaultGeopointTypeFunctionFormulaConnectorTestSuite, +): + pass + + +# GEOPOLYGON + + +class TestGeopolygonTypeFunctionMSSQL( + MSSQLTestBase, + DefaultGeopolygonTypeFunctionFormulaConnectorTestSuite, +): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_literals.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_literals.py new file mode 100644 index 000000000..1ad0e0e53 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_literals.py @@ -0,0 +1,13 @@ +import datetime + +from dl_formula_testing.testcases.literals import DefaultLiteralFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestConditionalBlockMSSQL(MSSQLTestBase, DefaultLiteralFormulaConnectorTestSuite): + recognizes_datetime_type = False # SQLAlchemy does not recognize DATETIMEOFFSET as a datetime.datetime + supports_microseconds = True + supports_utc = True + supports_custom_tz = True + default_tz = datetime.timezone.utc diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_misc_funcs.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_misc_funcs.py new file mode 100644 index 000000000..5c29c9664 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_misc_funcs.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.misc_funcs import DefaultMiscFunctionalityConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestMiscFunctionalityMSSQL(MSSQLTestBase, DefaultMiscFunctionalityConnectorTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_operators.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_operators.py new file mode 100644 index 000000000..10be9402c --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/db/formula/test_operators.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.operators import DefaultOperatorFormulaConnectorTestSuite + +from dl_connector_mssql_tests.db.formula.base import MSSQLTestBase + + +class TestOperatorMSSQL(MSSQLTestBase, DefaultOperatorFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/__init__.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/conftest.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/conftest.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/test_connection_form.py b/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/test_connection_form.py new file mode 100644 index 000000000..9602bde50 --- /dev/null +++ b/lib/dl_connector_mssql/dl_connector_mssql_tests/unit/test_connection_form.py @@ -0,0 +1,10 @@ +from dl_api_connector.i18n.localizer import CONFIGS as BI_API_CONNECTOR_CONFIGS +from dl_api_lib_testing.connection_form_base import ConnectionFormTestBase + +from dl_connector_mssql.api.connection_form.form_config import MSSQLConnectionFormFactory +from dl_connector_mssql.api.i18n.localizer import CONFIGS as BI_CONNECTOR_MSSQL_CONFIGS + + +class TestMSSQLConnectionForm(ConnectionFormTestBase): + CONN_FORM_FACTORY_CLS = MSSQLConnectionFormFactory + TRANSLATION_CONFIGS = BI_API_CONNECTOR_CONFIGS + BI_CONNECTOR_MSSQL_CONFIGS diff --git a/lib/dl_connector_mssql/docker-compose.yml b/lib/dl_connector_mssql/docker-compose.yml new file mode 100644 index 000000000..5d596616e --- /dev/null +++ b/lib/dl_connector_mssql/docker-compose.yml @@ -0,0 +1,42 @@ +version: '3.7' + +x-constants: + US_MASTER_TOKEN: &c-us-master-token "AC1ofiek8coB" + +services: + db-mssql-14: + labels: + datalens.ci.service: db-mssql-14 + + # image: "microsoft/mssql-server-linux:2017-CU12" + image: "mcr.microsoft.com/mssql/server:2017-CU12@sha256:19b9392f035fc9f82b77f6833d1490bca8cb041b445cd451de0d1f1f3efe70e8" + environment: + ACCEPT_EULA: "Y" + SA_PASSWORD: "qweRTY123" + ports: + - "52100:1433" + + # INFRA + pg-us: + build: + context: ../../mainrepo/lib/testenv-common/images + dockerfile: Dockerfile.pg-us + environment: + POSTGRES_DB: us-db-ci_purgeable + POSTGRES_USER: us + POSTGRES_PASSWORD: us + ports: + - "52110:5432" + + us: + build: + context: ../../mainrepo/lib/testenv-common/images + dockerfile: Dockerfile.us + depends_on: + - pg-us + environment: + POSTGRES_DSN_LIST: "postgres://us:us@pg-us:5432/us-db-ci_purgeable" + AUTH_POLICY: "required" + MASTER_TOKEN: *c-us-master-token + ports: + - "52111:80" diff --git a/lib/dl_connector_mssql/docker-compose/tests/entrypoint.sh b/lib/dl_connector_mssql/docker-compose/tests/entrypoint.sh new file mode 100644 index 000000000..5fc44481d --- /dev/null +++ b/lib/dl_connector_mssql/docker-compose/tests/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec "$@" diff --git a/lib/dl_connector_mssql/pyproject.toml b/lib/dl_connector_mssql/pyproject.toml new file mode 100644 index 000000000..dc9818a2b --- /dev/null +++ b/lib/dl_connector_mssql/pyproject.toml @@ -0,0 +1,94 @@ + +[tool.poetry] +name = "datalens-connector-mssql" +version = "0.0.1" +description = "" +authors = ["DataLens Team "] +packages = [{include = "dl_connector_mssql"}] +license = "Apache 2.0" +readme = "README.md" + + +[tool.poetry.dependencies] +attrs = ">=22.2.0" +pyodbc = ">=4.0.35" +python = ">=3.10, <3.12" +sqlalchemy = ">=1.4.46, <2.0" +datalens-api-commons = {path = "../dl_api_commons"} +datalens-constants = {path = "../dl_constants"} +datalens-formula-ref = {path = "../dl_formula_ref"} +datalens-i18n = {path = "../dl_i18n"} +datalens-sqlalchemy-mssql = {path = "../dl_sqlalchemy_mssql"} +datalens-formula = {path = "../dl_formula"} +datalens-configs = {path = "../dl_configs"} +datalens-api-connector = {path = "../dl_api_connector"} +datalens-core = {path = "../dl_core"} +datalens-db-testing = {path = "../dl_db_testing"} + +[tool.poetry.plugins] +[tool.poetry.plugins."dl_api_lib.connectors"] +mssql = "dl_connector_mssql.api.connector:MSSQLApiConnector" + +[tool.poetry.plugins."dl_core.connectors"] +mssql = "dl_connector_mssql.core.connector:MSSQLCoreConnector" + +[tool.poetry.plugins."dl_db_testing.connectors"] +mssql = "dl_connector_mssql.db_testing.connector:MSSQLDbTestingConnector" + +[tool.poetry.plugins."dl_formula.connectors"] +mssql = "dl_connector_mssql.formula.connector:MSSQLFormulaConnector" + +[tool.poetry.plugins."dl_formula_ref.plugins"] +mssql = "dl_connector_mssql.formula_ref.plugin:MSSQLFormulaRefPlugin" + +[tool.poetry.group] +[tool.poetry.group.tests.dependencies] +aiohttp = ">=3.8.1" +pytest = ">=7.2.2" +pytest-asyncio = ">=0.20.3" +pyodbc = ">=4.0.35" +datalens-api-lib-testing = {path = "../dl_api_lib_testing"} +datalens-formula-testing = {path = "../dl_formula_testing"} +datalens-testing = {path = "../dl_testing"} +datalens-core-testing = {path = "../dl_core_testing"} + +[build-system] +build-backend = "poetry.core.masonry.api" +requires = [ + "poetry-core", +] + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra" +testpaths = ["dl_connector_mssql_tests/db", "dl_connector_mssql_tests/unit"] + + + +[datalens.pytest.db] +root_dir = "dl_connector_mssql_tests/" +target_path = "db unit" + +[datalens.pytest.unit] +root_dir = "dl_connector_mssql_tests/" +target_path = "unit" +skip_compose = "true" + +[tool.mypy] +warn_unused_configs = true +disallow_untyped_defs = true +check_untyped_defs = true +strict_optional = true + +[[tool.mypy.overrides]] +module = ["pyodbc.*"] +ignore_missing_imports = true + +[datalens.i18n.domains] +dl_connector_mssql = [ + {path = "dl_connector_mssql/api"}, + {path = "dl_connector_mssql/core"}, +] +dl_formula_ref_dl_connector_mssql = [ + {path = "dl_connector_mssql/formula_ref"}, +] diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index 20090b5d2..686a49a2e 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -1423,6 +1423,34 @@ sqlalchemy = ">=1.4.46, <2.0" type = "directory" url = "../lib/dl_connector_metrica" +[[package]] +name = "datalens-connector-mssql" +version = "0.0.1" +description = "" +optional = false +python-versions = ">=3.10, <3.12" +files = [] +develop = false + +[package.dependencies] +attrs = ">=22.2.0" +datalens-api-commons = {path = "../dl_api_commons"} +datalens-api-connector = {path = "../dl_api_connector"} +datalens-configs = {path = "../dl_configs"} +datalens-constants = {path = "../dl_constants"} +datalens-core = {path = "../dl_core"} +datalens-db-testing = {path = "../dl_db_testing"} +datalens-formula = {path = "../dl_formula"} +datalens-formula-ref = {path = "../dl_formula_ref"} +datalens-i18n = {path = "../dl_i18n"} +datalens-sqlalchemy-mssql = {path = "../dl_sqlalchemy_mssql"} +pyodbc = ">=4.0.35" +sqlalchemy = ">=1.4.46, <2.0" + +[package.source] +type = "directory" +url = "../lib/dl_connector_mssql" + [[package]] name = "datalens-connector-mysql" version = "0.0.1" @@ -6299,4 +6327,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.12" -content-hash = "a3b22cceddc48663da18969232e937b9ead9c954511b658b9b1ed6cf238827cd" +content-hash = "635660602d7db2e63190e4eea32a7ef2cab1cd4d1c0b96651dd9d295a180599d" diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index 5300226ac..01006dbe3 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -146,6 +146,7 @@ datalens-connector-oracle = {path = "../lib/dl_connector_oracle"} datalens-connector-mysql = {path = "../lib/dl_connector_mysql"} datalens-sqlalchemy-mysql = {path = "../lib/dl_sqlalchemy_mysql"} datalens-maintenance = {path = "../lib/dl_maintenance"} +datalens-connector-mssql = {path = "../lib/dl_connector_mssql"} [tool.poetry.group.dev.dependencies] black = "==23.3.0"