From ed5463016ebcc0173f2175c635fbef8eb23054da Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Mon, 13 Nov 2023 18:27:48 +0100 Subject: [PATCH 01/27] Moved dataset capability test to dl_core_testing (#92) --- .../dl_core_testing/testcases/dataset.py | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/lib/dl_core_testing/dl_core_testing/testcases/dataset.py b/lib/dl_core_testing/dl_core_testing/testcases/dataset.py index 5132168e6..c3ed9f167 100644 --- a/lib/dl_core_testing/dl_core_testing/testcases/dataset.py +++ b/lib/dl_core_testing/dl_core_testing/testcases/dataset.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import ( + AbstractSet, ClassVar, Generic, Optional, @@ -11,10 +12,13 @@ import sqlalchemy as sa from dl_constants.enums import ( + ConnectionType, DataSourceRole, DataSourceType, + JoinType, ) from dl_core.data_processing.stream_base import DataStream +from dl_core.dataset_capabilities import DatasetCapabilities from dl_core.query.bi_query import BIQuery from dl_core.query.expression import ExpressionCtx from dl_core.services_registry.top_level import ServicesRegistry @@ -22,8 +26,14 @@ from dl_core.us_dataset import Dataset from dl_core.us_manager.us_manager_sync import SyncUSManager from dl_core_testing.data import DataFetcher -from dl_core_testing.database import DbTable -from dl_core_testing.dataset import make_dataset +from dl_core_testing.database import ( + Db, + DbTable, +) +from dl_core_testing.dataset import ( + get_created_from, + make_dataset, +) from dl_core_testing.dataset_wrappers import ( DatasetTestWrapper, EditableDatasetTestWrapper, @@ -202,3 +212,42 @@ def test_get_param_hash( assert hash_from_dataset == hash_from_template assert found_template + + def _check_compatible_source_types(self, compat_source_types: AbstractSet[DataSourceType]) -> None: + assert self.source_type in compat_source_types + + def _check_compatible_connection_types(self, compat_conn_types: AbstractSet[ConnectionType]) -> None: + assert not compat_conn_types, "Multiple connections are not supported" + + def _check_supported_join_types(self, supp_join_types: AbstractSet[JoinType]) -> None: + assert set(supp_join_types).issuperset({JoinType.inner, JoinType.left}) + + def _allow_adding_sources(self, dataset: Dataset) -> bool: + return True + + def test_compatibility_info( + self, + db: Db, + saved_connection: ConnectionBase, + saved_dataset: Dataset, + sync_us_manager: SyncUSManager, + ) -> None: + dataset = saved_dataset + connection = saved_connection + + ds_wrapper = DatasetTestWrapper(dataset=dataset, us_manager=sync_us_manager) + capabilities = DatasetCapabilities(dataset=dataset, dsrc_coll_factory=ds_wrapper.dsrc_coll_factory) + + compat_source_types = capabilities.get_compatible_source_types() + self._check_compatible_source_types(compat_source_types) + + compat_conn_types = capabilities.get_compatible_connection_types() + self._check_compatible_connection_types(compat_conn_types) + + supp_join_types = capabilities.get_supported_join_types() + self._check_supported_join_types(supp_join_types) + + assert capabilities.get_effective_connection_id() == connection.uuid + + if self._allow_adding_sources(dataset=dataset): + assert capabilities.source_can_be_added(connection_id=connection.uuid, created_from=get_created_from(db=db)) From 163d13b8d063c51525f92392460a952894313fcd Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Mon, 13 Nov 2023 22:02:07 +0100 Subject: [PATCH 02/27] Fixed CI image for MSSQL (#93) --- ci/get_base_img_hash.sh | 2 +- .../Dockerfile | 4 ++++ lib/dl_sqlalchemy_mssql/README.md | 1 + lib/dl_sqlalchemy_mssql/README.rst | 15 --------------- lib/dl_sqlalchemy_mssql/pyproject.toml | 2 +- 5 files changed, 7 insertions(+), 17 deletions(-) create mode 100644 lib/dl_sqlalchemy_mssql/README.md delete mode 100644 lib/dl_sqlalchemy_mssql/README.rst diff --git a/ci/get_base_img_hash.sh b/ci/get_base_img_hash.sh index 74a032539..01050c725 100644 --- a/ci/get_base_img_hash.sh +++ b/ci/get_base_img_hash.sh @@ -5,5 +5,5 @@ set -eu export LC_ALL=C -IMG_HASH_DOCKER_IMAGE_DIR="$(find $ROOT_DIR/metapkg -type f -print0 | sort -z | xargs -0 sha1sum -z | sha1sum | cut -d \ -f1)" +IMG_HASH_DOCKER_IMAGE_DIR="$(find $ROOT_DIR/metapkg $ROOT_DIR/docker_build -type f -print0 | sort -z | xargs -0 sha1sum -z | sha1sum | cut -d \ -f1)" echo "rebuild_flag:8:$IMG_HASH_DOCKER_IMAGE_DIR" | sha1sum | cut -d \ -f1 diff --git a/docker_build/target_dl_base_linux_w_db_bin_dependencies/Dockerfile b/docker_build/target_dl_base_linux_w_db_bin_dependencies/Dockerfile index e220044bc..a0fe3616e 100644 --- a/docker_build/target_dl_base_linux_w_db_bin_dependencies/Dockerfile +++ b/docker_build/target_dl_base_linux_w_db_bin_dependencies/Dockerfile @@ -7,3 +7,7 @@ RUN chmod a+x /tmp/scripts/*.sh && \ --exit-on-error \ /tmp/scripts && \ rm -rf /tmp/scripts + +# Configure libraries for MSSQL +RUN cat /usr/share/tdsodbc/odbcinst.ini >> /etc/odbcinst.ini +ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/odbc diff --git a/lib/dl_sqlalchemy_mssql/README.md b/lib/dl_sqlalchemy_mssql/README.md new file mode 100644 index 000000000..01ca9ba2b --- /dev/null +++ b/lib/dl_sqlalchemy_mssql/README.md @@ -0,0 +1 @@ +# MSSQL connector for DataLens diff --git a/lib/dl_sqlalchemy_mssql/README.rst b/lib/dl_sqlalchemy_mssql/README.rst deleted file mode 100644 index cd1841429..000000000 --- a/lib/dl_sqlalchemy_mssql/README.rst +++ /dev/null @@ -1,15 +0,0 @@ -BI MSSQL SQLAlchemy dialect -================================ - -A subclass of the standard mssql dialect with datalens-backend-specific modifications (e.g. parameterized queries with GROUP BY: https://github.com/level12/sqlalchemy_pyodbc_mssql) - - -Connection Parameters -===================== - -Syntax for the connection string: - - .. code-block:: python - - 'bi_mssql+pyodbc:///DRIVER={FreeTDS};Server=;Port=;Database=;' - 'UID=;PWD=;TDS_Version=8.0' diff --git a/lib/dl_sqlalchemy_mssql/pyproject.toml b/lib/dl_sqlalchemy_mssql/pyproject.toml index 59b5f6aaa..549a8cf92 100644 --- a/lib/dl_sqlalchemy_mssql/pyproject.toml +++ b/lib/dl_sqlalchemy_mssql/pyproject.toml @@ -6,7 +6,7 @@ description = "BI MSSQL SQLAlchemy Dialect" authors = ["DataLens Team "] packages = [{include = "dl_sqlalchemy_mssql"}] license = "Apache 2.0" -readme = "README.rst" +readme = "README.md" [tool.poetry.dependencies] From aa9a2adc197f66ef3b6d7cd7da1c4f28be5fb90c Mon Sep 17 00:00:00 2001 From: Valentin Gologuzov Date: Mon, 13 Nov 2023 23:03:00 +0100 Subject: [PATCH 03/27] Fixed terrarium/bi_ci mypy errors, updated terrarium checks action a bit (#90) * Fixed mypy errors, marked some tomlkit access with ignore * Fixed GHA to run terrarium mypy tests --- .github/workflows/terrarium_check.yaml | 16 +++++++--------- .../bi_ci/bi_ci/detect_affected_packages.py | 10 +++++----- terrarium/bi_ci/bi_ci/execute_mypy_multi.py | 7 ++++--- terrarium/bi_ci/bi_ci/pkg_ref.py | 18 ++++++++++-------- terrarium/bi_ci/bi_ci/split_pytest_tasks.py | 2 +- .../bi_ci_tests/unit/test_detect_affected.py | 2 -- .../unit/test_fix_ports_in_compose.py | 6 +++++- terrarium/bi_ci/pyproject.toml | 13 +++++++++++++ terrarium/dl_repmanager/pyproject.toml | 8 ++++++-- 9 files changed, 51 insertions(+), 31 deletions(-) delete mode 100644 terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py diff --git a/.github/workflows/terrarium_check.yaml b/.github/workflows/terrarium_check.yaml index 0a233e0c6..3c78869eb 100644 --- a/.github/workflows/terrarium_check.yaml +++ b/.github/workflows/terrarium_check.yaml @@ -29,19 +29,18 @@ jobs: rm -rf ./.??* || true - name: Checkout code uses: actions/checkout@v4 - - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - name: Setup common tools - run: pip install --no-input poetry pytest + run: pip install --no-input poetry - name: Install dependencies run: | cd "terrarium/${{ matrix.value }}" - pip install --no-input . + poetry -v install --with pytest --without mypy - name: Pytest run: | cd "terrarium/${{ matrix.value }}" - pytest . + poetry run pytest . - mypu: + mypy: runs-on: [ self-hosted, linux ] container: image: "python:3.11.0" @@ -60,14 +59,13 @@ jobs: rm -rf ./.??* || true - name: Checkout code uses: actions/checkout@v4 - - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - name: Setup common tools - run: pip install --no-input poetry mypy + run: pip install --no-input poetry - name: Install dependencies run: | cd "terrarium/${{ matrix.value }}" - pip install --no-input . + poetry -v install --with mypy --without pytest - name: Mypy run: | cd "terrarium/${{ matrix.value }}" - mypy . + poetry run mypy . diff --git a/terrarium/bi_ci/bi_ci/detect_affected_packages.py b/terrarium/bi_ci/bi_ci/detect_affected_packages.py index 6ccb8721b..3dc927b88 100644 --- a/terrarium/bi_ci/bi_ci/detect_affected_packages.py +++ b/terrarium/bi_ci/bi_ci/detect_affected_packages.py @@ -49,7 +49,7 @@ def gen_pkg_dirs(cfg: Config) -> Iterator[Path]: yield item -def get_reverse_dependencies(direct_dependency: dict[str, list[str]]): +def get_reverse_dependencies(direct_dependency: dict[str, list[str]]) -> dict[str, list[str]]: reverse_ref = defaultdict(list) for pkg in direct_dependency: for dep in direct_dependency[pkg]: @@ -57,11 +57,10 @@ def get_reverse_dependencies(direct_dependency: dict[str, list[str]]): return reverse_ref -def get_leafs(dependencies): +def get_leafs(dependencies: dict[str, list[str]]) -> set[str]: all_values = [] for deps in dependencies.values(): all_values.extend(deps) - all_values = set(all_values) leafs = set(all_values) - set([k for k in dependencies.keys() if len(dependencies[k]) > 0]) return leafs @@ -133,8 +132,9 @@ def process( to_test = set() for pkg in direct_affected: - to_test.add(pkg.self_pkg_name) - to_test.update(affection_map.get(pkg.self_pkg_name, {})) + if pkg.self_pkg_name: + to_test.add(pkg.self_pkg_name) + to_test.update(affection_map.get(pkg.self_pkg_name, {})) return [pkg_by_ref[k] for k in to_test] diff --git a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py index 1284fd710..50c90d2c8 100644 --- a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py +++ b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py @@ -17,7 +17,7 @@ def get_mypy_targets(pkg_dir: Path) -> list[str]: try: with open(pkg_dir / PYPROJECT_TOML) as fh: meta = tomlkit.load(fh) - return meta["datalens"]["meta"]["mypy"]["targets"] + return meta["datalens"]["meta"]["mypy"]["targets"] # type: ignore except NonExistentKey: pass @@ -33,10 +33,11 @@ def get_targets(root: Path) -> Iterable[str]: def main(root: Path, targets_file: Path = None) -> None: # type: ignore # clize can't recognize type annotation "Optional" + paths: Iterable[str] if targets_file is not None: - paths: Iterable[str] = json.load(open(targets_file)) + paths = json.load(open(targets_file)) else: - paths: Iterable[str] = get_targets(root) + paths = get_targets(root) failed_list: list[str] = [] mypy_cache_dir = Path("/tmp/mypy_cache") mypy_cache_dir.mkdir(exist_ok=True) diff --git a/terrarium/bi_ci/bi_ci/pkg_ref.py b/terrarium/bi_ci/bi_ci/pkg_ref.py index 35fd4be2d..d31183c77 100644 --- a/terrarium/bi_ci/bi_ci/pkg_ref.py +++ b/terrarium/bi_ci/bi_ci/pkg_ref.py @@ -3,6 +3,7 @@ import attrs import tomlkit +from tomlkit import TOMLDocument @attrs.define(slots=False) @@ -11,11 +12,11 @@ class PkgRef: full_path: Path @cached_property - def partial_parent_path(self): + def partial_parent_path(self) -> Path: return self.full_path.relative_to(self.root) @cached_property - def self_toml(self): + def self_toml(self) -> TOMLDocument | None: try: return tomlkit.load(open(self.full_path / "pyproject.toml")) except Exception as err: @@ -30,11 +31,12 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) -> result = set() raw = {} if spec: - raw = dict(spec["tool"]["poetry"]["dependencies"]) + # tomlkit & mypy very annoying together, hence a bunch of ignores + raw = dict(spec["tool"]["poetry"]["dependencies"]) # type: ignore for group in include_groups or []: - if "group" not in spec["tool"]["poetry"]: + if "group" not in spec["tool"]["poetry"]: # type: ignore continue - raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {})) + raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {})) # type: ignore for name, specifier in raw.items(): if isinstance(specifier, dict) and "path" in specifier: @@ -43,9 +45,9 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) -> return result @cached_property - def self_pkg_name(self): + def self_pkg_name(self) -> str | None: try: - return self.self_toml["tool"]["poetry"]["name"] + return self.self_toml["tool"]["poetry"]["name"] # type: ignore except (KeyError, TypeError): return None @@ -53,6 +55,6 @@ def self_pkg_name(self): def skip_test(self) -> bool: spec = self.self_toml try: - return spec["datalens_ci"]["skip_test"] + return spec["datalens_ci"]["skip_test"] # type: ignore except (KeyError, TypeError): return False diff --git a/terrarium/bi_ci/bi_ci/split_pytest_tasks.py b/terrarium/bi_ci/bi_ci/split_pytest_tasks.py index 22d31137a..23e7bb590 100644 --- a/terrarium/bi_ci/bi_ci/split_pytest_tasks.py +++ b/terrarium/bi_ci/bi_ci/split_pytest_tasks.py @@ -51,7 +51,7 @@ def split_tests( if pytest_targets: for section in pytest_targets.keys(): - spec = toml_data["datalens"]["pytest"][section] + spec = pytest_targets.get("section", {}) # supporting only a single label for now for category in spec.get("labels", []): split_result[category].append((package_path, section)) diff --git a/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py b/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py deleted file mode 100644 index 10cf3ad0a..000000000 --- a/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_dummy(): - pass diff --git a/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py b/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py index 65e9aa407..9d35db23d 100644 --- a/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py +++ b/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py @@ -4,7 +4,11 @@ from bi_ci.fix_ports_in_compose import remove_ports_from_docker_compose -def test_remove_ports_from_docker_compose(tmpdir, sample_compose_src, sample_compose_expected): +def test_remove_ports_from_docker_compose( + tmpdir: Path, + sample_compose_src: str, + sample_compose_expected: Path, +): tmp_src = Path(tmpdir) / "docker-compose.yml" tmp_dst = Path(tmpdir) / "docker-compose-modified.yml" diff --git a/terrarium/bi_ci/pyproject.toml b/terrarium/bi_ci/pyproject.toml index 93fab7c5e..06d046555 100644 --- a/terrarium/bi_ci/pyproject.toml +++ b/terrarium/bi_ci/pyproject.toml @@ -30,6 +30,13 @@ clize = ">=5.0.0" poetry = ">=1.5.0" pyyaml = ">=6.0.1" +[tool.poetry.group.pytest.dependencies] +pytest = ">=7.4.3" + +[tool.poetry.group.mypy.dependencies] +types_PyYAML = "*" +mypy = ">= 1.7.0" + [tool.poetry.scripts] detect-affected-packages = "bi_ci.detect_affected_packages:main" run-tests = "bi_ci.run_tests:runner_cli" @@ -49,3 +56,9 @@ warn_unused_configs = true disallow_untyped_defs = true check_untyped_defs = true strict_optional = true +exclude = [ + "^bi_ci_tests/", # TOML's double-quoted strings require escaping backslashes +] +[tool.black] +line-length = 120 +target-version = ['py310'] diff --git a/terrarium/dl_repmanager/pyproject.toml b/terrarium/dl_repmanager/pyproject.toml index bbfc17a52..fbb137955 100644 --- a/terrarium/dl_repmanager/pyproject.toml +++ b/terrarium/dl_repmanager/pyproject.toml @@ -17,8 +17,12 @@ tomlkit = "==0.11.8" requests = ">=2.31.0" datalens-cli-tools = {path = "../dl_cli_tools"} -[tool.poetry.group.tests.dependencies] -pytest = ">=7.2.2" +[tool.poetry.group.pytest.dependencies] +pytest = ">=7.4.3" + +[tool.poetry.group.mypy.dependencies] +types_PyYAML = "*" +mypy = ">= 1.7.0" [build-system] requires = ["poetry-core"] From 7d20f8f3edc82d270928f5e85550685957758aee Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:01:04 +0100 Subject: [PATCH 04/27] Add a markup test (#89) --- .../db/data_api/result/test_markup.py | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py new file mode 100644 index 000000000..05ddb073f --- /dev/null +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import uuid + +import pytest + +from dl_api_lib_testing.data_api_base import DataApiTestParams +from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows +from dl_api_lib_tests.db.base import DefaultApiTestBase + + +class TestUMarkup(DefaultApiTestBase): + @pytest.fixture(scope="function") + def data_api_test_params(self, sample_table) -> DataApiTestParams: + # This default is defined for the sample table + return DataApiTestParams( + two_dims=("category", "city"), + summable_field="sales", + range_field="sales", + distinct_field="city", + date_field="order_date", + ) + + def test_markup(self, saved_dataset, data_api, data_api_test_params): + ds = saved_dataset + + field_a, field_b, field_nulled = (str(uuid.uuid4()) for _ in range(3)) + formula_a = """ + markup( + italic(url( + "http://example.com/?city=" + [city] + "&_=1", + [city])), + " (", bold(str([order_date])), ")") + """ + formula_b = """ + url("https://example.com/?text=" + str([sales]) + " usd в рублях", str([sales])) + """ + formula_nulled = """ + url(if([sales] > 0, NULL, "neg"), str([sales])) + """ + + result_resp = data_api.get_result( + dataset=ds, + updates=[ + ds.field( + id=field_a, + title="Field A", + formula=formula_a, + ).add(), + ds.field( + id=field_b, + title="Field B", + formula=formula_b, + ).add(), + ds.field( + id=field_nulled, + title="Field Nulled", + formula=formula_nulled, + ).add(), + ], + fields=[ + ds.field(id=field_a), + ds.field(id=field_b), + ds.field(id=field_nulled), + ], + ) + + assert result_resp.status_code == 200, result_resp.response_errors + data_rows = get_data_rows(result_resp) + assert data_rows + + some_row = data_rows[0] + assert len(some_row) == 3 + res_a, res_b, res_nulled = some_row + assert isinstance(res_a, dict) + assert isinstance(res_b, dict) + assert res_nulled is None + + b_val = res_b["content"]["content"] + assert isinstance(b_val, str) + expected_b = { + "type": "url", + "url": f"https://example.com/?text={b_val} usd в рублях", + "content": {"type": "text", "content": b_val}, + } + assert res_b == expected_b From 532611cc69e59e1d50e3163c306d78612e78d3a3 Mon Sep 17 00:00:00 2001 From: Andrey Snytin Date: Tue, 14 Nov 2023 20:22:35 +0100 Subject: [PATCH 05/27] fixed sentry deps (#97) --- app/dl_control_api/pyproject.toml | 2 -- lib/dl_api_commons/pyproject.toml | 3 +- lib/dl_api_lib/pyproject.toml | 1 - lib/dl_core/pyproject.toml | 2 -- lib/dl_file_uploader_api_lib/pyproject.toml | 1 - metapkg/poetry.lock | 37 +++------------------ metapkg/pyproject.toml | 2 -- 7 files changed, 5 insertions(+), 43 deletions(-) diff --git a/app/dl_control_api/pyproject.toml b/app/dl_control_api/pyproject.toml index 8810e4097..abf1601c9 100644 --- a/app/dl_control_api/pyproject.toml +++ b/app/dl_control_api/pyproject.toml @@ -11,8 +11,6 @@ readme = "README.md" [tool.poetry.dependencies] python = ">=3.10, <3.12" Flask = ">=2.2.5" -blinker = ">=1.5" -raven = ">=6.10.0" datalens-version = {path = "../../lib/dl_version"} datalens-sqlalchemy-postgres = {path = "../../lib/dl_sqlalchemy_postgres"} datalens-utils = {path = "../../lib/dl_utils"} diff --git a/lib/dl_api_commons/pyproject.toml b/lib/dl_api_commons/pyproject.toml index 9f8c2b308..e3ead9c64 100644 --- a/lib/dl_api_commons/pyproject.toml +++ b/lib/dl_api_commons/pyproject.toml @@ -18,8 +18,7 @@ marshmallow = ">=3.19.0" multidict = ">=4.0" opentracing = ">=2.4.0" python = ">=3.10, <3.12" -raven = ">=6.10.0" -sentry-sdk = ">=1.15.0" +sentry-sdk = {version = ">=1.15.0", extras = ["flask"]} typing-extensions = ">=4.5.0" datalens-utils = {path = "../dl_utils"} datalens-constants = {path = "../dl_constants"} diff --git a/lib/dl_api_lib/pyproject.toml b/lib/dl_api_lib/pyproject.toml index 85b6ef834..e4ad498c9 100644 --- a/lib/dl_api_lib/pyproject.toml +++ b/lib/dl_api_lib/pyproject.toml @@ -19,7 +19,6 @@ marshmallow-oneofschema = ">=3.0.1" more-itertools = ">=9.1.0" pandas = ">=1.5.3" python = ">=3.10, <3.12" -sentry-sdk = ">=1.15.0" werkzeug = ">=2.2.3" statcommons = {path = "../statcommons"} datalens-api-commons = {path = "../dl_api_commons"} diff --git a/lib/dl_core/pyproject.toml b/lib/dl_core/pyproject.toml index 0f9b51964..0bf5f6067 100644 --- a/lib/dl_core/pyproject.toml +++ b/lib/dl_core/pyproject.toml @@ -34,7 +34,6 @@ python = ">=3.10, <3.12" python-dateutil = ">=2.8.2" pytz = ">=2022.7.1" pyyaml = ">=6.0.1" -raven = ">=6.10.0" redis = ">=4.5.4" requests = ">=2.28.2" shortuuid = ">=1.0.11" @@ -98,7 +97,6 @@ module = [ "anyascii.*", "lz4.*", "marshmallow_oneofschema.*", - "raven.*", "types_aiobotocore_s3.*", "mypy_boto3_s3.*" ] diff --git a/lib/dl_file_uploader_api_lib/pyproject.toml b/lib/dl_file_uploader_api_lib/pyproject.toml index 94f124267..311ff9c16 100644 --- a/lib/dl_file_uploader_api_lib/pyproject.toml +++ b/lib/dl_file_uploader_api_lib/pyproject.toml @@ -19,7 +19,6 @@ marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" python = ">=3.10, <3.12" redis = ">=4.5.4" -sentry-sdk = ">=1.15.0" datalens-file-uploader-task-interface = {path = "../dl_file_uploader_task_interface"} datalens-api-commons = {path = "../dl_api_commons"} datalens-file-uploader-lib = {path = "../dl_file_uploader_lib"} diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index bb874cfb1..e50e252ae 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -1043,8 +1043,7 @@ ipdb = ">=0.13.13" marshmallow = ">=3.19.0" multidict = ">=4.0" opentracing = ">=2.4.0" -raven = ">=6.10.0" -sentry-sdk = ">=1.15.0" +sentry-sdk = {version = ">=1.15.0", extras = ["flask"]} typing-extensions = ">=4.5.0" [package.source] @@ -1110,7 +1109,6 @@ marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" more-itertools = ">=9.1.0" pandas = ">=1.5.3" -sentry-sdk = ">=1.15.0" statcommons = {path = "../statcommons"} werkzeug = ">=2.2.3" @@ -1665,7 +1663,6 @@ files = [] develop = false [package.dependencies] -blinker = ">=1.5" datalens-api-lib = {path = "../../lib/dl_api_lib"} datalens-app-tools = {path = "../../lib/dl_app_tools"} datalens-connector-chyt = {path = "../../lib/dl_connector_chyt"} @@ -1679,7 +1676,6 @@ datalens-sqlalchemy-postgres = {path = "../../lib/dl_sqlalchemy_postgres"} datalens-utils = {path = "../../lib/dl_utils"} datalens-version = {path = "../../lib/dl_version"} Flask = ">=2.2.5" -raven = ">=6.10.0" [package.source] type = "directory" @@ -1726,7 +1722,6 @@ opentracing = ">=2.4.0" python-dateutil = ">=2.8.2" pytz = ">=2022.7.1" pyyaml = ">=6.0.1" -raven = ">=6.10.0" redis = ">=4.5.4" redis_cache_lock = {path = "../redis-cache-lock"} requests = ">=2.28.2" @@ -1868,7 +1863,6 @@ gunicorn = ">=20.1.0" marshmallow = ">=3.19.0" marshmallow-oneofschema = ">=3.0.1" redis = ">=4.5.4" -sentry-sdk = ">=1.15.0" [package.source] type = "directory" @@ -4906,7 +4900,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4914,15 +4907,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4939,7 +4925,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4947,7 +4932,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -5057,21 +5041,6 @@ files = [ [package.extras] full = ["numpy"] -[[package]] -name = "raven" -version = "6.10.0" -description = "Raven is a client for Sentry (https://getsentry.com)" -optional = false -python-versions = "*" -files = [ - {file = "raven-6.10.0-py2.py3-none-any.whl", hash = "sha256:44a13f87670836e153951af9a3c80405d36b43097db869a36e92809673692ce4"}, - {file = "raven-6.10.0.tar.gz", hash = "sha256:3fa6de6efa2493a7c827472e984ce9b020797d0da16f1db67197bcc23c8fae54"}, -] - -[package.extras] -flask = ["Flask (>=0.8)", "blinker (>=1.1)"] -tests = ["Flask (>=0.8)", "Flask-Login (>=0.2.0)", "ZConfig", "aiohttp", "anyjson", "blinker (>=1.1)", "blinker (>=1.1)", "bottle", "celery (>=2.5)", "coverage (<4)", "exam (>=0.5.2)", "flake8 (==3.5.0)", "logbook", "mock", "nose", "pytest (>=3.2.0,<3.3.0)", "pytest-cov (==2.5.1)", "pytest-flake8 (==1.0.0)", "pytest-pythonpath (==0.7.2)", "pytest-timeout (==1.2.1)", "pytest-xdist (==1.18.2)", "pytz", "requests", "sanic (>=0.7.0)", "tornado (>=4.1,<5.0)", "tox", "webob", "webtest", "wheel"] - [[package]] name = "redis" version = "4.5.4" @@ -5368,7 +5337,9 @@ files = [ ] [package.dependencies] +blinker = {version = ">=1.1", optional = true, markers = "extra == \"flask\""} certifi = "*" +flask = {version = ">=0.11", optional = true, markers = "extra == \"flask\""} urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} [package.extras] @@ -6364,4 +6335,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.12" -content-hash = "08be0db497c668327c51002dc3d5bdbe0c4405cab9d833018c70f16c6c59b6fd" +content-hash = "5072dacb98bd69cb90cf3d061072aeb1a8920ca749f1aa8eca98996b61aa8ed7" diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index 45849c60c..770054f39 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -36,7 +36,6 @@ marshmallow-oneofschema = "==3.0.1" pandas = "==1.5.3" pyopenssl = "==23.2.0" python-dateutil = "==2.8.2" -raven = "==6.10.0" requests = "==2.28.2" tabulate = "==0.9.0" async-timeout = "==4.0.2" @@ -260,7 +259,6 @@ pyodbc = {ignore = true} pytest = {ignore = true} pytest-asyncio = {ignore = true} pyyaml = {ignore = true} -raven = {ignore = true} responses = {ignore = true} ruff = {ignore = true} sentry-sdk = {ignore = true} From d9da83c434cb940816f15cdb49b03292811d60e9 Mon Sep 17 00:00:00 2001 From: Valentin Gologuzov Date: Wed, 15 Nov 2023 11:13:48 +0100 Subject: [PATCH 06/27] Adding top level pyproject.toml to keep black/isort/ruff settings (#95) --- pyproject.toml | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..1ccea9653 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,100 @@ +[tool.poetry] +name = "datalens_backend_root_package" +version = "0.1.0" +description = "Aux pyproject.toml for the common project properties" +authors = ["DataLens Team "] +license = "Apache 2.0" +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.10" + +[build-system] +build-backend = "poetry.core.masonry.api" +requires = [ + "poetry-core" +] + +[tool.black] +line-length = 120 +target-version = ['py310'] +force-exclude= ''' +/( + # The following are specific to Black, you probably don't want those. + lib/redis-cache-lock + | lib/clickhouse-sqlalchemy + | lib/dl_formula/dl_formula/parser/antlr/gen +)/ +''' + +[tool.isort] +line_length = 120 +profile = "black" +skip_glob = [ + "lib/redis-cache-lock/*", + "lib/clickhouse-sqlalchemy/*", + "lib/dl_formula/dl_formula/parser/antlr/gen/*", +] +multi_line_output = 3 +force_grid_wrap = 2 +lines_after_imports = 2 +include_trailing_comma = true +force_sort_within_sections = true +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "FIRSTPARTY", + "CONNECTORS", + "LOCALFOLDER" +] +known_first_party = [ + "bi_*", + "dl_*", + "dc_*" +] +known_connectors = [ + "bi_connector_*", + "dl_connector_*" +] + +[tool.ruff] +line-length = 120 +force-exclude = true +select = [ + # Pyflakes + "F", + # Bugbear + "B" +] +ignore = [ + "E501", # line lengh, should be checked and fixed by black + "F401", # unused imports +] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "tools_venv", + "lib/redis-cache-lock/**", + "lib/clickhouse-sqlalchemy/**", + "lib/dl_formula/dl_formula/parser/antlr/gen/**", +] From e5d555d5304e6ce597c365b2cc90a8f0e57e6573 Mon Sep 17 00:00:00 2001 From: Ovsyannikov Dmitrii Date: Wed, 15 Nov 2023 11:56:54 +0100 Subject: [PATCH 07/27] ci: add code quality workflow (#46) * ci: add code quality workflow * fix: isort test_markup.py --- .github/workflows/main.yml | 57 +++++++++++++++++++ .../db/data_api/result/test_markup.py | 2 +- metapkg/poetry.lock | 29 +++++++++- metapkg/pyproject.toml | 1 + 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 028fd83a4..060309fe7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -219,3 +219,60 @@ jobs: ./report/**/*.xml event_name: ${{ github.event.workflow_run.event }} report_individual_runs: "true" + + codestyle_all_without_ruff: + runs-on: [self-hosted, linux, light] + needs: gh_build_image + container: + # until https://github.com/github/docs/issues/25520 is resolved, using vars + image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: 'Cleanup build folder' + run: | + rm -rf ./* || true + rm -rf ./.??* || true + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + - run: | + task cq:check_dir -- . + task cq:check_dir_strict -- . + env: + VENV_PATH: /venv + SKIP_RUFF: true + + codestyle_changed_without_ruff: + runs-on: [ self-hosted, linux, light ] + needs: gh_build_image + container: + # until https://github.com/github/docs/issues/25520 is resolved, using vars + image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}" + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: 'Cleanup build folder' + run: | + rm -rf ./* || true + rm -rf ./.??* || true + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + # https://github.com/actions/runner-images/issues/6775 + - run: git config --global --add safe.directory . + - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} + - run: | + TARGET=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ github.event.pull_request.base.sha }} --head ${{ github.event.pull_request.head.sha }}) + echo $TARGET + task cq:check_target -- "$TARGET" + task cq:check_target_strict -- "$TARGET" 1>/dev/null 2>/dev/null + env: + VENV_PATH: /venv + SKIP_RUFF: true + TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }} diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py index 05ddb073f..4608d8448 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_markup.py @@ -4,8 +4,8 @@ import pytest -from dl_api_lib_testing.data_api_base import DataApiTestParams from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows +from dl_api_lib_testing.data_api_base import DataApiTestParams from dl_api_lib_tests.db.base import DefaultApiTestBase diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index e50e252ae..b86b70784 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -3440,6 +3440,23 @@ qtconsole = ["qtconsole"] test = ["pytest (<7.1)", "pytest-asyncio", "testpath"] test-extra = ["curio", "matplotlib (!=3.2.0)", "nbformat", "numpy (>=1.21)", "pandas", "pytest (<7.1)", "pytest-asyncio", "testpath", "trio"] +[[package]] +name = "isort" +version = "5.12.0" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, + {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, +] + +[package.extras] +colors = ["colorama (>=0.4.3)"] +pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] +plugins = ["setuptools"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] + [[package]] name = "itsdangerous" version = "2.1.2" @@ -4900,6 +4917,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -4907,8 +4925,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -4925,6 +4950,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -4932,6 +4958,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -6335,4 +6362,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.12" -content-hash = "5072dacb98bd69cb90cf3d061072aeb1a8920ca749f1aa8eca98996b61aa8ed7" +content-hash = "3f5b4176287c0aa8efe9c53345031cba496ae1f93dd6d93c2ce5aef496b5240f" diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index 770054f39..b3f92b982 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -152,6 +152,7 @@ datalens-attrs-model-mapper-doc-tools = {path = "../lib/dl_attrs_model_mapper_do [tool.poetry.group.dev.dependencies] black = "==23.3.0" ruff = "==0.0.267" +isort = "==5.12.0" ipdb = "==0.13.13" pyodbc = "==4.0.35" pytest = "==7.2.2" From 067be297e7c918660a29850203a264f7f34c6fcf Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Wed, 15 Nov 2023 16:12:29 +0100 Subject: [PATCH 08/27] Added rm command to repmanager (#99) --- .../dl_repmanager/dl_repmanager/fs_editor.py | 2 +- .../dl_repmanager/management_plugins.py | 15 ++++++++++++++- .../dl_repmanager/repository_manager.py | 2 +- .../dl_repmanager/scripts/repmanager_cli.py | 11 +++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/terrarium/dl_repmanager/dl_repmanager/fs_editor.py b/terrarium/dl_repmanager/dl_repmanager/fs_editor.py index 5f2700f60..67407c206 100644 --- a/terrarium/dl_repmanager/dl_repmanager/fs_editor.py +++ b/terrarium/dl_repmanager/dl_repmanager/fs_editor.py @@ -241,7 +241,7 @@ def _move_path(self, old_path: Path, new_path: Path) -> None: subprocess.run(f'git add "{rel_old_path}" && git mv "{rel_old_path}" "{rel_new_path}"', shell=True) def _remove_path(self, path: Path) -> None: - result = subprocess.run(f'git rm "{path}"', shell=True) + result = subprocess.run(f'git rm -r "{path}"', shell=True) if result.returncode != 0: super()._remove_path(path) diff --git a/terrarium/dl_repmanager/dl_repmanager/management_plugins.py b/terrarium/dl_repmanager/dl_repmanager/management_plugins.py index 96e8d5f4b..8bb26f1fa 100644 --- a/terrarium/dl_repmanager/dl_repmanager/management_plugins.py +++ b/terrarium/dl_repmanager/dl_repmanager/management_plugins.py @@ -193,7 +193,20 @@ def register_package(self, package_info: PackageInfo) -> None: pass def unregister_package(self, package_info: PackageInfo) -> None: - pass # FIXME: Remove package from dependencies + package_meta_io_factory = PackageMetaIOFactory(fs_editor=self.fs_editor) + + # Scan other packages to see if they are dependent on this one and update these dependency entries + for other_package_info in self.package_index.list_package_infos(): + if other_package_info == package_info: + continue + + for section_name in other_package_info.requirement_lists: + if other_package_info.is_dependent_on(package_info, section_name=section_name): + with package_meta_io_factory.package_meta_writer(other_package_info.toml_path) as pkg_meta_writer: + pkg_meta_writer.remove_requirement_item( + section_name=section_name, + item_name=package_info.package_reg_name, + ) def re_register_package(self, old_package_info: PackageInfo, new_package_info: PackageInfo) -> None: package_meta_io_factory = PackageMetaIOFactory(fs_editor=self.fs_editor) diff --git a/terrarium/dl_repmanager/dl_repmanager/repository_manager.py b/terrarium/dl_repmanager/dl_repmanager/repository_manager.py index ea30226b1..3e085d470 100644 --- a/terrarium/dl_repmanager/dl_repmanager/repository_manager.py +++ b/terrarium/dl_repmanager/dl_repmanager/repository_manager.py @@ -111,7 +111,7 @@ def _register_package(self, package_info: PackageInfo) -> None: def _unregister_package(self, package_info: PackageInfo) -> None: for mng_plugin in self.repository_env.get_plugins(package_index=self.package_index): - mng_plugin.register_package(package_info=package_info) + mng_plugin.unregister_package(package_info=package_info) def _re_register_package(self, old_package_info: PackageInfo, new_package_info: PackageInfo) -> None: for mng_plugin in self.repository_env.get_plugins(package_index=self.package_index): diff --git a/terrarium/dl_repmanager/dl_repmanager/scripts/repmanager_cli.py b/terrarium/dl_repmanager/dl_repmanager/scripts/repmanager_cli.py index 84fe5d959..81bb8b725 100644 --- a/terrarium/dl_repmanager/dl_repmanager/scripts/repmanager_cli.py +++ b/terrarium/dl_repmanager/dl_repmanager/scripts/repmanager_cli.py @@ -120,6 +120,12 @@ def make_parser() -> argparse.ArgumentParser: help="New name of the package", ) + subparsers.add_parser( + "rm", + parents=[package_name_parser], + help="Remove package", + ) + move_code_parser = subparsers.add_parser( "rename-module", parents=[old_new_import_names_parser], @@ -269,6 +275,9 @@ def rename(self, package_name: str, new_package_name: str) -> None: def ch_package_type(self, package_name: str, package_type: str) -> None: self.repository_manager.change_package_type(package_module_name=package_name, new_package_type=package_type) + def rm(self, package_name: str) -> None: + self.repository_manager.remove_package(package_module_name=package_name) + def package_list(self, package_type: str, mask: str, base_path: Path) -> None: for package_info in self.package_index.list_package_infos(package_type=package_type): printable_values = dict( @@ -459,6 +468,8 @@ def run_parsed_args(cls, args: argparse.Namespace) -> None: tool.rename(package_name=args.package_name, new_package_name=args.new_package_name) case "ch-package-type": tool.ch_package_type(package_name=args.package_name, package_type=args.package_type) + case "rm": + tool.rm(package_name=args.package_name) case "package-list": tool.package_list(package_type=args.package_type, mask=args.mask, base_path=Path(args.base_path)) case "rename-module": From 8502239f727b6c84890e6d8b6f99b848d936f8f6 Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Wed, 15 Nov 2023 17:19:45 +0100 Subject: [PATCH 09/27] Unknown field in LODs issue test (#100) --- .../complex_queries/test_ext_agg_basic.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/complex_queries/test_ext_agg_basic.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/complex_queries/test_ext_agg_basic.py index 3e2030151..5769c7e8c 100644 --- a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/complex_queries/test_ext_agg_basic.py +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/complex_queries/test_ext_agg_basic.py @@ -995,3 +995,22 @@ def test_bi_4652_measure_filter_with_total_in_select(self, control_api, data_api data_rows = get_data_rows(result_resp) dim_values = [row[0] for row in data_rows] assert len(dim_values) == len(set(dim_values)), "Dimension values are not unique" + + @pytest.mark.xfail(reason="https://github.com/datalens-tech/datalens-backend/issues/98") # FIXME + def test_fixed_with_unknown_field(self, control_api, data_api, saved_dataset): + ds = add_formulas_to_dataset( + api_v1=control_api, + dataset=saved_dataset, + formulas={ + "sales sum fx unknown": "SUM([sales] FIXED [unknown])", + }, + ) + + result_resp = data_api.get_result( + dataset=ds, + fields=[ + ds.find_field(title="sales sum fx unknown"), + ], + fail_ok=True, + ) + assert result_resp.status_code == HTTPStatus.BAD_REQUEST, result_resp.json From 62751187ffec0105940b8d7b7846a352d9f6f4b1 Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Thu, 16 Nov 2023 13:16:37 +0100 Subject: [PATCH 10/27] Add a ping test (#102) --- .../dl_api_lib_tests/db/data_api/test_ping.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 lib/dl_api_lib/dl_api_lib_tests/db/data_api/test_ping.py diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/test_ping.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/test_ping.py new file mode 100644 index 000000000..35f875a28 --- /dev/null +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/test_ping.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest +import shortuuid + +from dl_api_lib_tests.db.base import DefaultApiTestBase +from dl_configs.enums import RequiredService + + +class TestPing(DefaultApiTestBase): + @pytest.mark.asyncio + async def test_ping(self, data_api_lowlevel_aiohttp_client): + client = data_api_lowlevel_aiohttp_client + req_id = shortuuid.uuid() + + resp = await client.get("/ping", headers={"x-request-id": req_id}) + assert resp.status == 200 + js = await resp.json() + assert js["request_id"].startswith(req_id + "--") + + @pytest.mark.asyncio + async def test_ping_ready(self, data_api_lowlevel_aiohttp_client): + client = data_api_lowlevel_aiohttp_client + req_id = shortuuid.uuid() + + resp = await client.get("/ping_ready", headers={"x-request-id": req_id}) + assert resp.status == 200 + js = await resp.json() + assert js["request_id"].startswith(req_id + "--") + details = js["details"] + assert details[RequiredService.POSTGRES.name] is True + assert details[RequiredService.RQE_INT_SYNC.name] == 200 + assert details[RequiredService.RQE_EXT_SYNC.name] == 200 From bccd1663691d06d994ada62ff1b9c12659fc1f8c Mon Sep 17 00:00:00 2001 From: KonstantAnxiety <58992437+KonstantAnxiety@users.noreply.github.com> Date: Thu, 16 Nov 2023 15:47:15 +0300 Subject: [PATCH 11/27] Add mysql DBA tests (#101) --- .../core/async_adapters_mysql.py | 8 +++++++- .../core/error_transformer.py | 14 ++++++++++++++ .../db/core/test_adapter.py | 19 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_adapter.py diff --git a/lib/dl_connector_mysql/dl_connector_mysql/core/async_adapters_mysql.py b/lib/dl_connector_mysql/dl_connector_mysql/core/async_adapters_mysql.py index 458436520..32eb1168d 100644 --- a/lib/dl_connector_mysql/dl_connector_mysql/core/async_adapters_mysql.py +++ b/lib/dl_connector_mysql/dl_connector_mysql/core/async_adapters_mysql.py @@ -146,11 +146,17 @@ async def _execute_by_steps(self, db_adapter_query: DBAdapterQuery) -> AsyncIter chunk_size = db_adapter_query.get_effective_chunk_size(self._default_chunk_size) query = db_adapter_query.query + debug_compiled_query = db_adapter_query.debug_compiled_query escape_percent = not db_adapter_query.is_dashsql_query # DON'T escape only for dashsql compiled_query, compiled_query_parameters = compile_mysql_query( query, dialect=self._dialect, escape_percent=escape_percent ) - debug_query = query if isinstance(query, str) else compile_query_for_debug(query, self._dialect) + debug_query = None + if self._target_dto.pass_db_query_to_user: + if debug_compiled_query is not None: + debug_query = debug_compiled_query + else: + debug_query = query if isinstance(query, str) else compile_query_for_debug(query, self._dialect) with self.handle_execution_error(debug_query): async with self._get_connection(db_adapter_query.db_name) as conn: diff --git a/lib/dl_connector_mysql/dl_connector_mysql/core/error_transformer.py b/lib/dl_connector_mysql/dl_connector_mysql/core/error_transformer.py index c51d45edd..d23cb333a 100644 --- a/lib/dl_connector_mysql/dl_connector_mysql/core/error_transformer.py +++ b/lib/dl_connector_mysql/dl_connector_mysql/core/error_transformer.py @@ -36,6 +36,16 @@ def _(exc: Exception) -> bool: return _ +def is_source_connect_async_error() -> ExcMatchCondition: + def _(exc: Exception) -> bool: + if isinstance(exc, pymysql.OperationalError): + if len(exc.args) >= 2 and exc.args[0] == 2003: + return True + return False + + return _ + + class AsyncMysqlChainedDbErrorTransformer(error_transformer.ChainedDbErrorTransformer): @staticmethod def _get_error_kw( @@ -53,6 +63,10 @@ def _get_error_kw( async_mysql_db_error_transformer: DbErrorTransformer = AsyncMysqlChainedDbErrorTransformer( ( + Rule( + when=is_source_connect_async_error(), + then_raise=exc.SourceConnectError, + ), Rule( when=is_table_does_not_exist_async_error(), then_raise=MysqlSourceDoesNotExistError, diff --git a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_adapter.py b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_adapter.py new file mode 100644 index 000000000..be0725e0d --- /dev/null +++ b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_adapter.py @@ -0,0 +1,19 @@ +from dl_core_testing.testcases.adapter import BaseAsyncAdapterTestClass +from dl_testing.regulated_test import RegulatedTestParams + +from dl_connector_mysql.core.async_adapters_mysql import AsyncMySQLAdapter +from dl_connector_mysql.core.target_dto import MySQLConnTargetDTO +from dl_connector_mysql_tests.db.core.base import BaseMySQLTestClass + + +class TestAsyncMySQLAdapter( + BaseMySQLTestClass, + BaseAsyncAdapterTestClass[MySQLConnTargetDTO], +): + test_params = RegulatedTestParams( + mark_tests_skipped={ + BaseAsyncAdapterTestClass.test_default_pass_db_query_to_user: "Not relevant", + }, + ) + + ASYNC_ADAPTER_CLS = AsyncMySQLAdapter From caab92a0b779647f2fa7482f261ffcfd95552593 Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:06:12 +0100 Subject: [PATCH 12/27] Add parameters tests (#107) --- .../db/data_api/result/test_parameters.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py new file mode 100644 index 000000000..e95b1fd75 --- /dev/null +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from http import HTTPStatus + +import pytest + +from dl_api_client.dsmaker.primitives import ( + IntegerParameterValue, + RangeParameterValueConstraint, +) +from dl_api_client.dsmaker.shortcuts.dataset import ( + add_formulas_to_dataset, + add_parameters_to_dataset, +) +from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows +from dl_api_lib_tests.db.base import DefaultApiTestBase +from dl_constants.enums import UserDataType + + +class TestParameters(DefaultApiTestBase): + @pytest.mark.parametrize( + ("multiplier", "expected_status_code"), + ( + (None, HTTPStatus.OK), + (2, HTTPStatus.OK), + (5, HTTPStatus.OK), + (-1, HTTPStatus.BAD_REQUEST), + ), + ) + def test_parameter_in_formula(self, control_api, data_api, saved_dataset, multiplier, expected_status_code): + default_multiplier = 1 + ds = add_parameters_to_dataset( + api_v1=control_api, + dataset_id=saved_dataset.id, + parameters={ + "Multiplier": ( + IntegerParameterValue(default_multiplier), + RangeParameterValueConstraint(min=IntegerParameterValue(default_multiplier)), + ), + }, + ) + + integer_field = next(field for field in saved_dataset.result_schema if field.data_type == UserDataType.integer) + ds = add_formulas_to_dataset( + api_v1=control_api, + dataset=ds, + formulas={ + "Multiplied Field": f"[{integer_field.title}] * [Multiplier]", + }, + ) + + result_resp = data_api.get_result( + dataset=ds, + fields=[ + integer_field, + ds.find_field(title="Multiplier"), + ds.find_field(title="Multiplied Field"), + ], + parameters=[ + ds.find_field(title="Multiplier").parameter_value(multiplier), + ], + fail_ok=True, + ) + assert result_resp.status_code == expected_status_code, result_resp.json + + if expected_status_code == HTTPStatus.OK: + data_rows = get_data_rows(result_resp) + assert data_rows + for row in data_rows: + assert int(row[1]) == (multiplier or default_multiplier) + assert int(row[0]) * int(row[1]) == int(row[2]) + + def test_parameter_no_constraint(self, control_api, data_api, dataset_id): + ds = add_parameters_to_dataset( + api_v1=control_api, + dataset_id=dataset_id, + parameters={ + "Param": (IntegerParameterValue(0), None), + }, + ) + ds = add_formulas_to_dataset( + api_v1=control_api, + dataset=ds, + formulas={ + "Value": "[Param]", + }, + ) + + result_resp = data_api.get_result( + dataset=ds, + fields=[ + ds.find_field(title="Value"), + ], + parameters=[ + ds.find_field(title="Param").parameter_value(1), + ], + limit=1, + ) + assert result_resp.status_code == HTTPStatus.OK, result_resp.json + assert int(get_data_rows(result_resp)[0][0]) == 1 From cb5945009b030c3ea1f1a1acdd375e0c039c7726 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 17 Nov 2023 14:23:18 +0100 Subject: [PATCH 13/27] Added description of tool usage to repmanager (#105) --- terrarium/dl_gitmanager/README.md | 8 +- terrarium/dl_repmanager/README.md | 252 +++++++++++++++++++++++++++++- 2 files changed, 255 insertions(+), 5 deletions(-) diff --git a/terrarium/dl_gitmanager/README.md b/terrarium/dl_gitmanager/README.md index 1249c964a..faf30712d 100644 --- a/terrarium/dl_gitmanager/README.md +++ b/terrarium/dl_gitmanager/README.md @@ -14,13 +14,13 @@ pip install -Ue Show the main help message -``` +```bash dl-git --help dl-git --h ``` The `--help` (`-h`) option can also be used for any command: -``` +```bash dl-git --help ``` @@ -43,7 +43,7 @@ If the path is inside a submodule, then the submodule is considered to be the ro List files that have changed between two given revisions including the changes in all submodules -``` +```bash dl-git range-diff-paths --base --head dl-git range-diff-paths --base --head --absolute dl-git range-diff-paths --base --head --only-added-commits @@ -53,7 +53,7 @@ Here `base` and `head` can be a commit ID, branch name, `HEAD~3` or any similar that is usually accepted by git. These arguments are optional. By default `head` is `HEAD` and `base` is `HEAD~1`. Thgis means you can use the following command to see the diff of the last commit in the current branch: -``` +```bash dl-git range-diff-paths ``` diff --git a/terrarium/dl_repmanager/README.md b/terrarium/dl_repmanager/README.md index 1333ed77b..958818c15 100644 --- a/terrarium/dl_repmanager/README.md +++ b/terrarium/dl_repmanager/README.md @@ -1 +1,251 @@ -TODO +# dl_repmanager + +Package containing tools for package/repo management. + + +## Installation + +```bash +pip install -Ue +``` + + +# `dl-package` + +Tool for working with metadata of individual packages. + + +## Commands + + +### list-i18n-domains + +List i18n domains of a package + +```bash +dl-package --package-path list-i18n-domains + +# example: +dl-package --package-path lib/dl_connector_clickhouse list-i18n-domains +``` +Will print something like: +``` +dl_connector_clickhouse=dl_connector_clickhouse/api;dl_connector_clickhouse/core +dl_formula_ref_dl_connector_clickhouse=dl_connector_clickhouse/formula_ref +``` + + +### set-meta-array + +Set array property in package's `pyproject.toml` +```bash +dl-package --package-path set-meta-array --toml-section --toml-key --toml-value + +# example: +dl-package --package-path lib/dl_core set-meta-array --toml-section tool.poetry --toml-key authors --toml-value "Alicia ";James " +``` + + +### set-meta-text + +Set text property in package's `pyproject.toml` +```bash +dl-package --package-path set-meta-text --toml-section --toml-key --toml-value + +# example: +dl-package --package-path lib/dl_core set-meta-text --toml-section tool.poetry --toml-key license --toml-value "Apache 2.0" +``` + + +# `dl-repo` + +Tool for managing and inspecting packages as part of the repository and the repository as a whole. +This includes creation, renaming, moving of packages, etc. with updating all the necessary dependencies and registries. + + +## Common options + + +### --help + +Show the main help message or help for a specific command + +``` +dl-repo --help +dl-repo --h +dl-repo -h +``` + + +### --config + +Optional. Path to the repository configuration file (`dl-repo.yml`). +By default the tool will discover the config automatically going up from the CWD. + +See [dl-repo.yml](../../dl-repo.yml). + +```bash +dl-repo --config /my/custom/repo-config.yml ... +``` + + +### --fs-editor + +Controls how FS operations are performed + +Possible values: +- `default` +- `git` +- `virtual` + +The default is set in the repo config ([dl-repo.yml](../../dl-repo.yml)). If not, the default is `default`. + +```bash +dl-repo --fs-editor git ... +``` + + +## Commands + + +### ch-package-type + +Change the type of (essentially, move) a package. + +```bash +dl-repo ch-package-type --package-name --package-type + +# example: +dl-repo ch-package-type --package-name dl_core --package-type app +``` + +The `--package-name` argument is the name of the package to move. This argument is required. + +The `--package-type` argument specifies the name of the package's new type, which maps to a ceertain folder in the repo. +See the repo config ([dl-repo.yml](../../dl-repo.yml)). This argument is required. + + +### compare-resulting-deps + +*TODO* + + +### copy + +Create a new package as a copy of an existing one. + +```bash +dl-repo copy --package-name --from-package-name + +# example: +dl-repo copy --package-name dl_super_package --from-package-name dl_core +``` + +The `--package-name` argument is the name of the new package's folder and module name. This argument is required. + +The `--from-package-name` argument of the package to copy (to use as the boilerplate). This argument is required. + + +### ensure-mypy-common + +*TODO* + + +### import-list + +*TODO* + + +### init + +Create a new package + +```bash +dl-repo init --package-type --package-name + +# example: +dl-repo init --package-type lib --package-name dl_super_package +``` + +The `--package-type` argument specifies where the package will be created. Package types are configured +in the repo config ([dl-repo.yml](../../dl-repo.yml)). This argument is required. + +The `--package-name` argument is the name of the new package's folder and module name. This argument is required. + + +### package-list + +*TODO* + + +### recurse-packages + +*TODO* + + +### rename + +Rename a package. + +```bash +dl-repo rename --package-name --new-package-name + +# example: +dl-repo rename --package-name dl_core --new-package-name dl_ultra_core +``` + +The `--package-name` argument is the original name of the package to rename. This argument is required. + +The `--new-package-name` argument Is the new name of the package. + + +### rename-module + +*TODO* + + +### req-check + +Check whether the requirements of a package are consistent with imports in the package's code. + +```bash +dl-repo req-check --package-name +dl-repo req-check --package-name --test +``` + +The `--package-name` argument is the original name of the package to check. + +The `--tests` flag tells the tool to check test requirements instead of the main ones. + + +### req-list + +List requirements of a package + +```bash +dl-repo req-list --package-name +``` + + +### resolve + +*TODO* + + +### rm + +Remove a package. + +```bash +dl-repo rm --package-name + +# example: +dl-repo rm --package-name dl_core +``` + +The `--package-name` argument is the name of the package to remove. This argument is required. + + +### search-imports + +*TODO* From dc144799e272ef45c0b1089d48978cb5cb2142fb Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 17 Nov 2023 14:35:02 +0100 Subject: [PATCH 14/27] Knowledge Base Begins (#103) * Knowledge Base Begins * Minor fixes for KB initial commit * Some more minor fixes for KB initial commit --- .gitignore | 1 + README.md | 41 +++++++------------------------- kb/index.md | 6 +++++ kb/test_embeds.md | 15 ++++++++++++ kb/tooling/index.md | 8 +++++++ kb/tooling/task_commands.md | 47 +++++++++++++++++++++++++++++++++++++ kb/using_kb.md | 26 ++++++++++++++++++++ 7 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 kb/index.md create mode 100644 kb/test_embeds.md create mode 100644 kb/tooling/index.md create mode 100644 kb/tooling/task_commands.md create mode 100644 kb/using_kb.md diff --git a/.gitignore b/.gitignore index 1ab2f3221..43c6d713b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ ci_artifacts .DS_Store artifacts Taskfile.yml +.obsidian diff --git a/README.md b/README.md index 966a18acd..f001b2204 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,15 @@ -## Management tasks +# datalens-backend -Running tasks: -``` -task -``` +## About +This is the repository for the back-end implementation of DataLens -### Environment (`env:`) +Head over to the [Knowledge Base](kb/index.md) for documentation on this repo. -- `task env:devenv`: - Create development/testing environment (run it from a package dir) -- `task env:devenv-d`: - Create development/testing environment in detached mode (run it from a package dir) -- `task env:ensure_venv`: Command to create virtual env for the mainrepo tools. - It requires presence of .env in the mainrepo/tools. +[Code of conduct](CODE_OF_CONDUCT.md) +[Contributing](CONTRIBUTING.md) -### Generation (`gen:`) +## License -- `task gen:antlr`: - (Re-)generate ANTLR code files for formula -- `task gen:i18n-po`: - Sync/generate `.po` files for package (run it from a package dir) -- `task gen:i18n-binaries`: - Generate binary `.mo` files from `.po` files for package (run it from a package dir) - - -### Code quality (`cq:`) - -Experimental tasks to check and fix source files. - -- `task cq:fix_changed`: - Apply all auto-fixes -- `task cq:check_changed`: - Check for any non-conformity in code style/format/lint -- `task cq:fix_dir -- {single dir}`: - Apply all auto-fixes to the given dir absolute path -- `task cq:check_dir -- {single dir}`: - Check for any non-conformity in code style/format/lint in the given dir abs path +`datalens-backend` is available under the Apache 2.0 license. diff --git a/kb/index.md b/kb/index.md new file mode 100644 index 000000000..4d64a2fb6 --- /dev/null +++ b/kb/index.md @@ -0,0 +1,6 @@ +# Datalens Backend Knowledge Base + +Welcome to the KB! + +See: +- [Repository tooling](tooling/index.md) diff --git a/kb/test_embeds.md b/kb/test_embeds.md new file mode 100644 index 000000000..32e1f21b9 --- /dev/null +++ b/kb/test_embeds.md @@ -0,0 +1,15 @@ +This is a file to test embedded diagrams. + + +``` plantuml +@startuml +digraph foo { + node [style=rounded] + node1 [shape=box] + node2 [fillcolor=yellow, style="rounded,filled", shape=diamond] + node3 [shape=record, label="{ a | b | c }"] + + node1 -> node2 -> node3 +}; +@enduml +``` diff --git a/kb/tooling/index.md b/kb/tooling/index.md new file mode 100644 index 000000000..b36917a8f --- /dev/null +++ b/kb/tooling/index.md @@ -0,0 +1,8 @@ +# Repository Tooling + +This section is about the custom tooling available in this repo. + +- [task commands](task_commands.md) - a set of (`make`-like) shortcuts for various commands and scripts for repository management, development, testing, etc. +- tools from `terrarium` ([README](../../terrarium/README.md)): + - [dl-git](../../terrarium/dl_gitmanager/README.md) - a wrapper for advanced git commands, for usage mainly in the CI workflow + - [dl-repo / dl-package](../../terrarium/dl_repmanager/README.md) - tools for managing and inspecting packages, their dependencies, meta-packages, etc. diff --git a/kb/tooling/task_commands.md b/kb/tooling/task_commands.md new file mode 100644 index 000000000..bd7697999 --- /dev/null +++ b/kb/tooling/task_commands.md @@ -0,0 +1,47 @@ +## Management tasks + +These commands require the ``taskfile`` tool. See [this page](https://taskfile.dev/installation/) +for installation options. + +Running tasks: +``` +task +``` + + +### Environment (`env:`) + +Working with the testing/development environment. + +- `task env:devenv`: + Create development/testing environment (run it from a package dir) +- `task env:devenv-d`: + Create development/testing environment in detached mode (run it from a package dir) +- `task env:ensure_venv`: Command to create virtual env for the mainrepo tools. + It requires presence of .env in the mainrepo/tools. + + +### Generation (`gen:`) + +Generating files to be used from the code. + +- `task gen:antlr`: + (Re-)generate ANTLR code files for formula +- `task gen:i18n-po`: + Sync/generate `.po` files for package (run it from a package dir) +- `task gen:i18n-binaries`: + Generate binary `.mo` files from `.po` files for package (run it from a package dir) + + +### Code quality (`cq:`) + +Checking and fixing source files. + +- `task cq:fix_changed`: + Apply all auto-fixes +- `task cq:check_changed`: + Check for any non-conformity in code style/format/lint +- `task cq:fix_dir -- {single dir}`: + Apply all auto-fixes to the given dir absolute path +- `task cq:check_dir -- {single dir}`: + Check for any non-conformity in code style/format/lint in the given dir abs path diff --git a/kb/using_kb.md b/kb/using_kb.md new file mode 100644 index 000000000..a160ec945 --- /dev/null +++ b/kb/using_kb.md @@ -0,0 +1,26 @@ +# Working with the KB + +This KB is mostly `Markdown`, but it has embedded charts. +These usually don't work out-of-the-box and might require a little additional configuration. + +To check the correect rendering of embedded charts open [this file](test_embeds.md). +If your editor supports this, and everything is configured correctly, +you should see a rendered charts. + +Here are two options you can use to work with this KB + +## PyCharm + +- Install `graphviz`. +- Install and enable the Markdown plugin in PyCharm. + +In theory this should be enough, but you may find that charts give you a rendering error +about not finding "dot". +In this case find the `dot` executable in your system and copy it to `/opt/local/bin/dot`. +It should work now. + +## Obsidian + +- Install the `Obsidian` app +- Install the PlantUML plugin in Obsidian. + You might need to configure the path to the dot executable. From 15854329cdf27f39107fde071b0e485d0dd88cfc Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 17 Nov 2023 14:46:02 +0100 Subject: [PATCH 15/27] Minor updates for KB (#108) --- kb/index.md | 3 ++- kb/using_kb.md | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/kb/index.md b/kb/index.md index 4d64a2fb6..0cd32f282 100644 --- a/kb/index.md +++ b/kb/index.md @@ -2,5 +2,6 @@ Welcome to the KB! -See: +Topics: +- [Working with this KB](using_kb.md) (editor configuration) - [Repository tooling](tooling/index.md) diff --git a/kb/using_kb.md b/kb/using_kb.md index a160ec945..709cc60bd 100644 --- a/kb/using_kb.md +++ b/kb/using_kb.md @@ -1,6 +1,6 @@ # Working with the KB -This KB is mostly `Markdown`, but it has embedded charts. +This KB is mostly `Markdown`, but it has embedded PlantUML charts. These usually don't work out-of-the-box and might require a little additional configuration. To check the correect rendering of embedded charts open [this file](test_embeds.md). @@ -22,5 +22,6 @@ It should work now. ## Obsidian - Install the `Obsidian` app -- Install the PlantUML plugin in Obsidian. +- Install and enable the PlantUML plugin in Obsidian. You might need to configure the path to the dot executable. +- Open the kb folder as a vault From a87925eaf3d9746202423585c65312ad0ea41c5a Mon Sep 17 00:00:00 2001 From: Ovsyannikov Dmitrii Date: Tue, 21 Nov 2023 12:56:58 +0100 Subject: [PATCH 16/27] docs: clean old readme for dl_formula_ref (#110) --- lib/dl_formula_ref/README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/dl_formula_ref/README.md b/lib/dl_formula_ref/README.md index d65156173..f172d9ca4 100644 --- a/lib/dl_formula_ref/README.md +++ b/lib/dl_formula_ref/README.md @@ -1,13 +1,3 @@ # Formula reference Package to generate BI docs source - -## Development - -### Makefile commands - -- `make init-venv` - Initialize dependencies for doc generation -- `make clean-venv` - Clean dependencies for doc generation -- `make generate-example-data` - Generate example data -- `DOCS_SOURCE_PATH={docs-source repo path} make generate-docs-source` - Generate docs, -alternatively you can leave DOCS_SOURCE_PATH empty and docs will be generated locally to `docs-source` folder From 8d45e962b7613493cc619bdfd98f10b323f08623 Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Tue, 21 Nov 2023 14:43:54 +0100 Subject: [PATCH 17/27] BI-4944: add ydb connector (#106) --- lib/dl_connector_ydb/LICENSE | 201 +++++++++++ lib/dl_connector_ydb/README.md | 1 + .../dl_connector_ydb/__init__.py | 6 + .../dl_connector_ydb/api/__init__.py | 0 .../dl_connector_ydb/api/ydb/__init__.py | 0 .../api/ydb/api_schema/__init__.py | 0 .../api/ydb/api_schema/connection.py | 25 ++ .../api/ydb/connection_form/__init__.py | 0 .../api/ydb/connection_form/form_config.py | 128 +++++++ .../api/ydb/connection_info.py | 7 + .../dl_connector_ydb/api/ydb/connector.py | 53 +++ .../dl_connector_ydb/api/ydb/i18n/__init__.py | 0 .../api/ydb/i18n/localizer.py | 28 ++ .../dl_connector_ydb/core/__init__.py | 0 .../dl_connector_ydb/core/base/__init__.py | 1 + .../dl_connector_ydb/core/base/adapter.py | 126 +++++++ .../dl_connector_ydb/core/base/data_source.py | 11 + .../core/base/query_compiler.py | 12 + .../core/base/type_transformer.py | 84 +++++ .../dl_connector_ydb/core/ydb/__init__.py | 0 .../dl_connector_ydb/core/ydb/adapter.py | 117 ++++++ .../core/ydb/connection_executors.py | 38 ++ .../dl_connector_ydb/core/ydb/connector.py | 73 ++++ .../dl_connector_ydb/core/ydb/constants.py | 13 + .../dl_connector_ydb/core/ydb/data_source.py | 47 +++ .../dl_connector_ydb/core/ydb/dto.py | 12 + .../dl_connector_ydb/core/ydb/settings.py | 24 ++ .../core/ydb/storage_schemas/__init__.py | 0 .../core/ydb/storage_schemas/connection.py | 14 + .../dl_connector_ydb/core/ydb/target_dto.py | 10 + .../core/ydb/type_transformer.py | 15 + .../core/ydb/us_connection.py | 110 ++++++ .../dl_connector_ydb/db_testing/__init__.py | 0 .../dl_connector_ydb/db_testing/connector.py | 7 + .../db_testing/engine_wrapper.py | 144 ++++++++ .../dl_connector_ydb/formula/__init__.py | 0 .../dl_connector_ydb/formula/connector.py | 13 + .../dl_connector_ydb/formula/constants.py | 15 + .../formula/definitions/__init__.py | 0 .../formula/definitions/all.py | 26 ++ .../formula/definitions/conditional_blocks.py | 22 ++ .../definitions/functions_aggregation.py | 173 +++++++++ .../formula/definitions/functions_datetime.py | 188 ++++++++++ .../formula/definitions/functions_logical.py | 34 ++ .../formula/definitions/functions_markup.py | 20 ++ .../formula/definitions/functions_math.py | 204 +++++++++++ .../formula/definitions/functions_string.py | 232 ++++++++++++ .../formula/definitions/functions_type.py | 185 ++++++++++ .../formula/definitions/operators_binary.py | 147 ++++++++ .../formula/definitions/operators_ternary.py | 11 + .../formula/definitions/operators_unary.py | 36 ++ .../dl_connector_ydb/formula_ref/__init__.py | 0 .../formula_ref/human_dialects.py | 22 ++ .../dl_connector_ydb/formula_ref/i18n.py | 23 ++ .../dl_connector_ydb/formula_ref/plugin.py | 22 ++ .../en/LC_MESSAGES/dl_connector_ydb.mo | Bin 0 -> 408 bytes .../en/LC_MESSAGES/dl_connector_ydb.po | 19 + .../dl_formula_ref_dl_connector_ydb.mo | Bin 0 -> 241 bytes .../dl_formula_ref_dl_connector_ydb.po | 13 + .../ru/LC_MESSAGES/dl_connector_ydb.mo | Bin 0 -> 436 bytes .../ru/LC_MESSAGES/dl_connector_ydb.po | 19 + .../dl_formula_ref_bi_connector_ydb.mo | Bin 0 -> 442 bytes .../dl_formula_ref_bi_connector_ydb.po | 13 + .../dl_connector_ydb/py.typed | 0 .../dl_connector_ydb_tests/__init__.py | 0 .../dl_connector_ydb_tests/db/__init__.py | 0 .../dl_connector_ydb_tests/db/api/__init__.py | 0 .../dl_connector_ydb_tests/db/api/base.py | 90 +++++ .../db/api/test_connection.py | 7 + .../db/api/test_dashsql.py | 98 +++++ .../db/api/test_dataset.py | 12 + .../dl_connector_ydb_tests/db/config.py | 340 ++++++++++++++++++ .../dl_connector_ydb_tests/db/conftest.py | 17 + .../db/formula/__init__.py | 0 .../dl_connector_ydb_tests/db/formula/base.py | 41 +++ .../db/formula/test_conditional_blocks.py | 7 + .../db/formula/test_functions_aggregation.py | 7 + .../db/formula/test_functions_datetime.py | 7 + .../db/formula/test_functions_logical.py | 7 + .../db/formula/test_functions_markup.py | 7 + .../db/formula/test_functions_math.py | 7 + .../db/formula/test_functions_string.py | 13 + .../formula/test_functions_type_conversion.py | 77 ++++ .../db/formula/test_literals.py | 9 + .../db/formula/test_misc_funcs.py | 7 + .../db/formula/test_operators.py | 23 ++ .../dl_connector_ydb_tests/unit/__init__.py | 0 .../dl_connector_ydb_tests/unit/conftest.py | 0 .../unit/test_connection_form.py | 20 ++ lib/dl_connector_ydb/docker-compose.yml | 39 ++ lib/dl_connector_ydb/pyproject.toml | 86 +++++ metapkg/poetry.lock | 52 ++- metapkg/pyproject.toml | 1 + 93 files changed, 3717 insertions(+), 1 deletion(-) create mode 100644 lib/dl_connector_ydb/LICENSE create mode 100644 lib/dl_connector_ydb/README.md create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/connection.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/form_config.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_info.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/localizer.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/base/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/base/adapter.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/base/data_source.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/base/query_compiler.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/base/type_transformer.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/adapter.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connection_executors.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/constants.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/data_source.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/dto.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/settings.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/connection.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/target_dto.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/type_transformer.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/core/ydb/us_connection.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/db_testing/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/db_testing/connector.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/db_testing/engine_wrapper.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/connector.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/constants.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/all.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/conditional_blocks.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_aggregation.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_datetime.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_logical.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_markup.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_math.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_string.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_binary.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_ternary.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_unary.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula_ref/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula_ref/human_dialects.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula_ref/i18n.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/formula_ref/plugin.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.mo create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.po create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.mo create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.po create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_connector_ydb.mo create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_connector_ydb.po create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.mo create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.po create mode 100644 lib/dl_connector_ydb/dl_connector_ydb/py.typed create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_connection.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dashsql.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dataset.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/conftest.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/base.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_conditional_blocks.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_aggregation.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_datetime.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_logical.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_markup.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_math.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_string.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_literals.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_misc_funcs.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_operators.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/unit/__init__.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/unit/conftest.py create mode 100644 lib/dl_connector_ydb/dl_connector_ydb_tests/unit/test_connection_form.py create mode 100644 lib/dl_connector_ydb/docker-compose.yml create mode 100644 lib/dl_connector_ydb/pyproject.toml diff --git a/lib/dl_connector_ydb/LICENSE b/lib/dl_connector_ydb/LICENSE new file mode 100644 index 000000000..74ba5f6c7 --- /dev/null +++ b/lib/dl_connector_ydb/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023 YANDEX LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/lib/dl_connector_ydb/README.md b/lib/dl_connector_ydb/README.md new file mode 100644 index 000000000..0b5a93cef --- /dev/null +++ b/lib/dl_connector_ydb/README.md @@ -0,0 +1 @@ +# dl_connector_ydb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/__init__.py new file mode 100644 index 000000000..0bfdfc461 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/__init__.py @@ -0,0 +1,6 @@ +try: + from ydb_proto_stubs_import import init_ydb_stubs + + init_ydb_stubs() +except ImportError: + pass # stubs will be initialized from the ydb package diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/connection.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/connection.py new file mode 100644 index 000000000..7973fe6bc --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/api_schema/connection.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from marshmallow import fields as ma_fields + +from dl_api_connector.api_schema.connection_base import ConnectionSchema +from dl_api_connector.api_schema.connection_base_fields import ( + cache_ttl_field, + secret_string_field, +) +from dl_api_connector.api_schema.connection_mixins import RawSQLLevelMixin +from dl_api_connector.api_schema.connection_sql import DBHostField +from dl_api_connector.api_schema.extras import FieldExtra + +from dl_connector_ydb.core.ydb.us_connection import YDBConnection + + +class YDBConnectionSchema(RawSQLLevelMixin, ConnectionSchema): + TARGET_CLS = YDBConnection + + host = DBHostField(attribute="data.host", required=True, bi_extra=FieldExtra(editable=True)) + port = ma_fields.Integer(attribute="data.port", required=True, bi_extra=FieldExtra(editable=True)) + db_name = ma_fields.String(attribute="data.db_name", required=True, bi_extra=FieldExtra(editable=True)) + + token = secret_string_field(attribute="data.token", required=False, allow_none=True) + cache_ttl_sec = cache_ttl_field(attribute="data.cache_ttl_sec") diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/form_config.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/form_config.py new file mode 100644 index 000000000..653e13bcf --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_form/form_config.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from enum import Enum +from typing import ( + Optional, + Sequence, + Type, +) + +from dl_api_commons.base_models import TenantDef +from dl_api_connector.form_config.models.api_schema import ( + FormActionApiSchema, + FormApiSchema, + FormFieldApiSchema, +) +from dl_api_connector.form_config.models.base import ( + ConnectionForm, + ConnectionFormFactory, + ConnectionFormMode, +) +from dl_api_connector.form_config.models.common import ( + CommonFieldName, + OAuthApplication, +) +import dl_api_connector.form_config.models.rows as C +from dl_api_connector.form_config.models.rows.base import FormRow +from dl_api_connector.form_config.models.shortcuts.rows import RowConstructor +from dl_configs.connectors_settings import ConnectorSettingsBase + +from dl_connector_ydb.api.ydb.connection_info import YDBConnectionInfoProvider +from dl_connector_ydb.core.ydb.settings import YDBConnectorSettings + + +class YDBOAuthApplication(OAuthApplication): + ydb = "ydb" + + +class YDBConnectionFormFactory(ConnectionFormFactory): + def _get_base_common_api_schema_items(self, names_source: Type[Enum]) -> list[FormFieldApiSchema]: + return [ + FormFieldApiSchema(name=names_source.host, required=True), + FormFieldApiSchema(name=names_source.port, required=True), + FormFieldApiSchema(name=names_source.db_name, required=True), + ] + + def _get_default_db_section(self, rc: RowConstructor, connector_settings: YDBConnectorSettings) -> list[FormRow]: + return [ + C.OAuthTokenRow( + name=CommonFieldName.token, + fake_value="******" if self.mode == ConnectionFormMode.edit else None, + application=YDBOAuthApplication.ydb, + ), + rc.host_row(default_value=connector_settings.DEFAULT_HOST_VALUE), + rc.port_row(default_value="2135"), + rc.db_name_row(), + ] + + def _get_base_edit_api_schema(self) -> FormActionApiSchema: + return FormActionApiSchema( + items=[ + FormFieldApiSchema(name=CommonFieldName.cache_ttl_sec, nullable=True), + FormFieldApiSchema(name=CommonFieldName.raw_sql_level), + ] + ) + + def _get_base_create_api_schema(self, edit_api_schema: FormActionApiSchema) -> FormActionApiSchema: + return FormActionApiSchema( + items=[ + *edit_api_schema.items, + *self._get_top_level_create_api_schema_items(), + ] + ) + + def _get_base_check_api_schema(self, common_api_schema_items: list[FormFieldApiSchema]) -> FormActionApiSchema: + return FormActionApiSchema( + items=[ + *common_api_schema_items, + *self._get_top_level_check_api_schema_items(), + ] + ) + + def _get_base_form_config( + self, + db_section_rows: Sequence[FormRow], + create_api_schema: FormActionApiSchema, + edit_api_schema: FormActionApiSchema, + check_api_schema: FormActionApiSchema, + rc: RowConstructor, + ) -> ConnectionForm: + return ConnectionForm( + title=YDBConnectionInfoProvider.get_title(self._localizer), + rows=[ + *db_section_rows, + C.CacheTTLRow(name=CommonFieldName.cache_ttl_sec), + rc.raw_sql_level_row(), + ], + api_schema=FormApiSchema( + create=create_api_schema if self.mode == ConnectionFormMode.create else None, + edit=edit_api_schema if self.mode == ConnectionFormMode.edit else None, + check=check_api_schema, + ), + ) + + def get_form_config( + self, + connector_settings: Optional[ConnectorSettingsBase], + tenant: Optional[TenantDef], + ) -> ConnectionForm: + assert connector_settings is not None and isinstance(connector_settings, YDBConnectorSettings) + rc = RowConstructor(localizer=self._localizer) + + edit_api_schema = self._get_base_edit_api_schema() + common_api_schema_items = self._get_base_common_api_schema_items(names_source=CommonFieldName) + db_section_rows = self._get_default_db_section(rc=rc, connector_settings=connector_settings) + common_api_schema_items.append( + FormFieldApiSchema(name=CommonFieldName.token, required=self.mode == ConnectionFormMode.create) + ) + edit_api_schema.items.extend(common_api_schema_items) + + create_api_schema = self._get_base_create_api_schema(edit_api_schema=edit_api_schema) + check_api_schema = self._get_base_check_api_schema(common_api_schema_items=common_api_schema_items) + return self._get_base_form_config( + db_section_rows=db_section_rows, + create_api_schema=create_api_schema, + edit_api_schema=edit_api_schema, + check_api_schema=check_api_schema, + rc=rc, + ) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_info.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_info.py new file mode 100644 index 000000000..0c6f84a4d --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connection_info.py @@ -0,0 +1,7 @@ +from dl_api_connector.connection_info import ConnectionInfoProvider + +from dl_connector_ydb.api.ydb.i18n.localizer import Translatable + + +class YDBConnectionInfoProvider(ConnectionInfoProvider): + title_translatable = Translatable("label_connector-ydb") diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py new file mode 100644 index 000000000..dd945884e --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/connector.py @@ -0,0 +1,53 @@ +from dl_api_connector.api_schema.source_base import ( + SQLDataSourceSchema, + SQLDataSourceTemplateSchema, + SubselectDataSourceSchema, + SubselectDataSourceTemplateSchema, +) +from dl_api_connector.connector import ( + ApiConnectionDefinition, + ApiConnector, + ApiSourceDefinition, +) + +from dl_connector_ydb.api.ydb.api_schema.connection import YDBConnectionSchema +from dl_connector_ydb.api.ydb.connection_form.form_config import YDBConnectionFormFactory +from dl_connector_ydb.api.ydb.connection_info import YDBConnectionInfoProvider +from dl_connector_ydb.api.ydb.i18n.localizer import CONFIGS +from dl_connector_ydb.core.ydb.connector import ( + YDBCoreConnectionDefinition, + YDBCoreConnector, + YDBCoreSourceDefinition, + YDBCoreSubselectSourceDefinition, +) +from dl_connector_ydb.formula.constants import DIALECT_NAME_YDB + + +class YDBApiTableSourceDefinition(ApiSourceDefinition): + core_source_def_cls = YDBCoreSourceDefinition + api_schema_cls = SQLDataSourceSchema + template_api_schema_cls = SQLDataSourceTemplateSchema + + +class YDBApiSubselectSourceDefinition(ApiSourceDefinition): + core_source_def_cls = YDBCoreSubselectSourceDefinition + api_schema_cls = SubselectDataSourceSchema + template_api_schema_cls = SubselectDataSourceTemplateSchema + + +class YDBApiConnectionDefinition(ApiConnectionDefinition): + core_conn_def_cls = YDBCoreConnectionDefinition + api_generic_schema_cls = YDBConnectionSchema + info_provider_cls = YDBConnectionInfoProvider + form_factory_cls = YDBConnectionFormFactory + + +class YDBApiConnector(ApiConnector): + core_connector_cls = YDBCoreConnector + connection_definitions = (YDBApiConnectionDefinition,) + source_definitions = ( + YDBApiTableSourceDefinition, + YDBApiSubselectSourceDefinition, + ) + formula_dialect_name = DIALECT_NAME_YDB + translation_configs = frozenset(CONFIGS) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/localizer.py b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/localizer.py new file mode 100644 index 000000000..d9dca6626 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/api/ydb/i18n/localizer.py @@ -0,0 +1,28 @@ +import os + +import attr + +from dl_i18n.localizer_base import Translatable as BaseTranslatable +from dl_i18n.localizer_base import TranslationConfig + +import dl_connector_ydb as package + + +DOMAIN = f"{package.__name__}" +CONFIGS = [ + TranslationConfig( + path=os.path.relpath(os.path.join(os.path.dirname(__file__), "../../../locales")), + domain=DOMAIN, + locale="en", + ), + TranslationConfig( + path=os.path.relpath(os.path.join(os.path.dirname(__file__), "../../../locales")), + domain=DOMAIN, + locale="ru", + ), +] + + +@attr.s +class Translatable(BaseTranslatable): + domain: str = attr.ib(default=DOMAIN) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/base/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/core/base/__init__.py new file mode 100644 index 000000000..acd7a25f4 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/base/__init__.py @@ -0,0 +1 @@ +""" Shared connector logic for YDB-YQL-ScanQuery and YQ connectors """ diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/base/adapter.py b/lib/dl_connector_ydb/dl_connector_ydb/core/base/adapter.py new file mode 100644 index 000000000..f82f4c59a --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/base/adapter.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import datetime +import logging +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Optional, + Tuple, + Type, + TypeVar, +) + +import attr +import sqlalchemy as sa + +from dl_core import exc +from dl_core.connection_executors.adapters.adapters_base_sa_classic import BaseClassicAdapter +from dl_core.connection_models import TableIdent + + +if TYPE_CHECKING: + from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO # noqa: F401 + from dl_core.connection_executors.models.db_adapter_data import ExecutionStepCursorInfo + from dl_core.connection_models import DBIdent + from dl_core.connectors.base.error_transformer import DBExcKWArgs + from dl_core.db.native_type import SATypeSpec + + +LOGGER = logging.getLogger(__name__) + +_DBA_YQL_BASE_DTO_TV = TypeVar("_DBA_YQL_BASE_DTO_TV", bound="BaseSQLConnTargetDTO") + + +@attr.s +class YQLAdapterBase(BaseClassicAdapter[_DBA_YQL_BASE_DTO_TV]): + def _get_db_version(self, db_ident: DBIdent) -> Optional[str]: + # Not useful. + return None + + def _is_table_exists(self, table_ident: TableIdent) -> bool: + # TODO?: use get_columns for this. + return True + + _type_code_to_sa = { + None: sa.TEXT, # fallback + "Int8": sa.INTEGER, + "Int16": sa.INTEGER, + "Int32": sa.INTEGER, + "Int64": sa.INTEGER, + "Uint8": sa.INTEGER, + "Uint16": sa.INTEGER, + "Uint32": sa.INTEGER, + "Uint64": sa.INTEGER, + "Float": sa.FLOAT, + "Double": sa.FLOAT, + "String": sa.TEXT, + "Utf8": sa.TEXT, + "Json": sa.TEXT, + "Yson": sa.TEXT, + "Uuid": sa.TEXT, + "Date": sa.DATE, + "Datetime": sa.DATETIME, + "Timestamp": sa.DATETIME, + "Interval": sa.INTEGER, + "Bool": sa.BOOLEAN, + } + _type_code_to_sa = { + **_type_code_to_sa, + # Nullable types: + **{name + "?": sa_type for name, sa_type in _type_code_to_sa.items() if name}, + } + _type_code_to_sa_prefixes = { + "Decimal(": sa.FLOAT, + } + + def _cursor_column_to_sa(self, cursor_col: Tuple[Any, ...], require: bool = True) -> Optional[SATypeSpec]: + result = super()._cursor_column_to_sa(cursor_col) + if result is not None: + return result + # Fallback: prefix + type_code = cursor_col[1] + for type_prefix, sa_type in self._type_code_to_sa_prefixes.items(): + if type_code.startswith(type_prefix): + return sa_type + if require: + raise ValueError(f"Unknown type_code: {type_code!r}") + return None + + _subselect_cursor_info_where_false: ClassVar[bool] = False + + @staticmethod + def _convert_bytes(value: bytes) -> str: + return value.decode("utf-8", errors="replace") + + @staticmethod + def _convert_ts(value: int) -> datetime.datetime: + return datetime.datetime.utcfromtimestamp(value / 1e6).replace(tzinfo=datetime.timezone.utc) + + def _get_row_converters(self, cursor_info: ExecutionStepCursorInfo) -> Tuple[Optional[Callable[[Any], Any]], ...]: + type_names_norm = [col[1].lower().strip("?") for col in cursor_info.raw_cursor_description] + return tuple( + self._convert_bytes + if type_name_norm == "string" + else self._convert_ts + if type_name_norm == "timestamp" + else None + for type_name_norm in type_names_norm + ) + + @classmethod + def make_exc( # TODO: Move to ErrorTransformer + cls, wrapper_exc: Exception, orig_exc: Optional[Exception], debug_compiled_query: Optional[str] + ) -> Tuple[Type[exc.DatabaseQueryError], DBExcKWArgs]: + exc_cls, kw = super().make_exc(wrapper_exc, orig_exc, debug_compiled_query) + + try: + message = wrapper_exc.message + except Exception: + pass + else: + kw["db_message"] = kw.get("db_message") or message + + return exc_cls, kw diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/base/data_source.py b/lib/dl_connector_ydb/dl_connector_ydb/core/base/data_source.py new file mode 100644 index 000000000..93692788a --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/base/data_source.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +from typing import Optional + +from dl_core.data_source.sql import BaseSQLDataSource + + +class YQLDataSourceMixin(BaseSQLDataSource): + @property + def db_version(self) -> Optional[str]: + return None # not expecting anything useful diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/base/query_compiler.py b/lib/dl_connector_ydb/dl_connector_ydb/core/base/query_compiler.py new file mode 100644 index 000000000..d8b94d678 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/base/query_compiler.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +from typing import ClassVar + +from dl_core.connectors.base.query_compiler import ( + QueryCompiler, + SectionAliasMode, +) + + +class YQLQueryCompiler(QueryCompiler): + groupby_alias_mode: ClassVar[SectionAliasMode] = SectionAliasMode.by_alias_in_select diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/base/type_transformer.py b/lib/dl_connector_ydb/dl_connector_ydb/core/base/type_transformer.py new file mode 100644 index 000000000..5fb79f6dd --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/base/type_transformer.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + ClassVar, + Dict, + Tuple, +) + +import sqlalchemy as sa +import ydb.sqlalchemy as ydb_sa + +from dl_constants.enums import ( + ConnectionType, + UserDataType, +) +from dl_core.db.conversion_base import ( + TypeTransformer, + make_native_type, +) + + +if TYPE_CHECKING: + from dl_core.db.native_type import SATypeSpec + + +class YQLTypeTransformerBase(TypeTransformer): + conn_type: ClassVar[ConnectionType] + + _base_type_map: Dict[UserDataType, Tuple[SATypeSpec, ...]] = { + # Note: first SA type is used as the default. + UserDataType.integer: ( + sa.BIGINT, + sa.SMALLINT, + sa.INTEGER, + ydb_sa.types.UInt32, + ydb_sa.types.UInt64, + ydb_sa.types.UInt8, + ), + UserDataType.float: ( + sa.FLOAT, + sa.REAL, + sa.NUMERIC, + # see also: DOUBLE_PRECISION, + ), + UserDataType.boolean: (sa.BOOLEAN,), + UserDataType.string: ( + sa.TEXT, + sa.CHAR, + sa.VARCHAR, + # see also: ENUM, + ), + # see also: UUID + UserDataType.date: (sa.DATE,), + UserDataType.datetime: ( + sa.DATETIME, + sa.TIMESTAMP, + ), + UserDataType.genericdatetime: ( + sa.DATETIME, + sa.TIMESTAMP, + ), + UserDataType.unsupported: (sa.sql.sqltypes.NullType,), # Actually the default, so should not matter much. + } + _extra_type_map: Dict[UserDataType, SATypeSpec] = { # user-to-native only + UserDataType.geopoint: sa.TEXT, + UserDataType.geopolygon: sa.TEXT, + UserDataType.uuid: sa.TEXT, # see also: UUID + UserDataType.markup: sa.TEXT, + } + + native_to_user_map = { + make_native_type(ConnectionType.unknown, sa_type): bi_type + for bi_type, sa_types in _base_type_map.items() + for sa_type in sa_types + if bi_type != UserDataType.datetime + } + user_to_native_map = { + **{ + bi_type: make_native_type(ConnectionType.unknown, sa_types[0]) + for bi_type, sa_types in _base_type_map.items() + }, + **{bi_type: make_native_type(ConnectionType.unknown, sa_type) for bi_type, sa_type in _extra_type_map.items()}, + } diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/adapter.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/adapter.py new file mode 100644 index 000000000..1945c63eb --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/adapter.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import logging +from typing import ( + TYPE_CHECKING, + ClassVar, + Iterable, + TypeVar, +) + +import attr +import grpc +import ydb.dbapi as ydb_dbapi +import ydb.issues as ydb_cli_err + +from dl_constants.enums import ConnectionType +from dl_core import exc +from dl_core.connection_models import TableIdent + +from dl_connector_ydb.core.base.adapter import YQLAdapterBase +from dl_connector_ydb.core.ydb.constants import CONNECTION_TYPE_YDB +from dl_connector_ydb.core.ydb.target_dto import YDBConnTargetDTO + + +if TYPE_CHECKING: + from dl_core.connection_models import SchemaIdent + + +LOGGER = logging.getLogger(__name__) + + +_DBA_YDB_BASE_DTO_TV = TypeVar("_DBA_YDB_BASE_DTO_TV", bound=YDBConnTargetDTO) + + +@attr.s +class YDBAdapterBase(YQLAdapterBase[_DBA_YDB_BASE_DTO_TV]): + conn_type: ClassVar[ConnectionType] = CONNECTION_TYPE_YDB + dsn_template: ClassVar[str] = "{dialect}:///ydb/" # 'yql:///ydb/' + + proto_schema: ClassVar[str] = "grpc" + + def _update_connect_args(self, args: dict) -> None: + args.update(auth_token=self._target_dto.password) + + def get_connect_args(self) -> dict: + target_dto = self._target_dto + args = dict( + endpoint="{}://{}:{}".format( + self.proto_schema, + target_dto.host, + target_dto.port, + ), + database=target_dto.db_name, + ) + self._update_connect_args(args) + return args + + EXTRA_EXC_CLS = (ydb_dbapi.Error, ydb_cli_err.Error, grpc.RpcError) + + def _list_table_names_i(self, db_name: str, show_dot: bool = False) -> Iterable[str]: + assert db_name, "db_name is required here" + db_engine = self.get_db_engine(db_name) + connection = db_engine.connect() + try: + # SA db_engine -> SA connection -> DBAPI connection -> YDB driver + driver = connection.connection.driver + assert driver + + queue = [db_name] + # Relative paths in `select` are also valid (i.e. "... from `some_dir/some_table`"), + # so, for visual convenience, remove the db prefix. + unprefix = db_name.rstrip("/") + "/" + while queue: + path = queue.pop(0) + resp = driver.scheme_client.async_list_directory(path) + res = resp.result() + children = [ + ( + "{}/{}".format(path, child.name), + child, + ) + for child in res.children + if show_dot or not child.name.startswith(".") + ] + children.sort() + for full_path, child in children: + if child.is_any_table(): + yield full_path.removeprefix(unprefix) + elif child.is_directory(): + queue.append(full_path) + finally: + connection.close() + + def _list_table_names(self, db_name: str, show_dot: bool = False) -> Iterable[str]: + driver_excs = self.EXTRA_EXC_CLS + try: + result = self._list_table_names_i(db_name=db_name, show_dot=show_dot) + for item in result: + yield item + except driver_excs as err: + raise exc.DatabaseQueryError(db_message=str(err), query="list_directory()") + + def _get_tables(self, schema_ident: SchemaIdent) -> list[TableIdent]: + db_name = schema_ident.db_name + assert db_name is not None + return [ + TableIdent( + schema_name=None, + db_name=db_name, + table_name=name, + ) + for name in self._list_table_names(db_name) + ] + + +class YDBAdapter(YDBAdapterBase[YDBConnTargetDTO]): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connection_executors.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connection_executors.py new file mode 100644 index 000000000..c47d15682 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connection_executors.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Sequence, +) + +import attr + +from dl_core.connection_executors.async_sa_executors import DefaultSqlAlchemyConnExecutor + +from dl_connector_ydb.core.ydb.adapter import YDBAdapter +from dl_connector_ydb.core.ydb.target_dto import YDBConnTargetDTO + + +if TYPE_CHECKING: + from dl_connector_ydb.core.ydb.dto import YDBConnDTO + + +@attr.s(cmp=False, hash=False) +class YDBAsyncAdapterConnExecutor(DefaultSqlAlchemyConnExecutor[YDBAdapter]): + TARGET_ADAPTER_CLS = YDBAdapter + + _conn_dto: YDBConnDTO = attr.ib() + + async def _make_target_conn_dto_pool(self) -> Sequence[YDBConnTargetDTO]: + return [ + YDBConnTargetDTO( + conn_id=self._conn_dto.conn_id, + pass_db_messages_to_user=self._conn_options.pass_db_messages_to_user, + pass_db_query_to_user=self._conn_options.pass_db_query_to_user, + host=self._conn_dto.host, + port=self._conn_dto.port, + db_name=self._conn_dto.db_name, + username=self._conn_dto.username or "", + password=self._conn_dto.password or "", + ) + ] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py new file mode 100644 index 000000000..12f1c080d --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/connector.py @@ -0,0 +1,73 @@ +from ydb.sqlalchemy import register_dialect as yql_register_dialect + +from dl_core.connectors.base.connector import ( + CoreConnectionDefinition, + CoreConnector, + CoreSourceDefinition, +) +from dl_core.data_source_spec.sql import ( + StandardSQLDataSourceSpec, + SubselectDataSourceSpec, +) +from dl_core.us_manager.storage_schemas.data_source_spec_base import ( + SQLDataSourceSpecStorageSchema, + SubselectDataSourceSpecStorageSchema, +) + +from dl_connector_ydb.core.base.query_compiler import YQLQueryCompiler +from dl_connector_ydb.core.ydb.adapter import YDBAdapter +from dl_connector_ydb.core.ydb.connection_executors import YDBAsyncAdapterConnExecutor +from dl_connector_ydb.core.ydb.constants import ( + BACKEND_TYPE_YDB, + CONNECTION_TYPE_YDB, + SOURCE_TYPE_YDB_SUBSELECT, + SOURCE_TYPE_YDB_TABLE, +) +from dl_connector_ydb.core.ydb.data_source import ( + YDBSubselectDataSource, + YDBTableDataSource, +) +from dl_connector_ydb.core.ydb.settings import YDBSettingDefinition +from dl_connector_ydb.core.ydb.storage_schemas.connection import YDBConnectionDataStorageSchema +from dl_connector_ydb.core.ydb.type_transformer import YDBTypeTransformer +from dl_connector_ydb.core.ydb.us_connection import YDBConnection + + +class YDBCoreConnectionDefinition(CoreConnectionDefinition): + conn_type = CONNECTION_TYPE_YDB + connection_cls = YDBConnection + us_storage_schema_cls = YDBConnectionDataStorageSchema + type_transformer_cls = YDBTypeTransformer + sync_conn_executor_cls = YDBAsyncAdapterConnExecutor + async_conn_executor_cls = YDBAsyncAdapterConnExecutor + dialect_string = "yql" + settings_definition = YDBSettingDefinition + + +class YDBCoreSourceDefinition(CoreSourceDefinition): + source_type = SOURCE_TYPE_YDB_TABLE + source_cls = YDBTableDataSource + source_spec_cls = StandardSQLDataSourceSpec + us_storage_schema_cls = SQLDataSourceSpecStorageSchema + + +class YDBCoreSubselectSourceDefinition(CoreSourceDefinition): + source_type = SOURCE_TYPE_YDB_SUBSELECT + source_cls = YDBSubselectDataSource + source_spec_cls = SubselectDataSourceSpec + us_storage_schema_cls = SubselectDataSourceSpecStorageSchema + + +class YDBCoreConnector(CoreConnector): + backend_type = BACKEND_TYPE_YDB + connection_definitions = (YDBCoreConnectionDefinition,) + source_definitions = ( + YDBCoreSourceDefinition, + YDBCoreSubselectSourceDefinition, + ) + rqe_adapter_classes = frozenset({YDBAdapter}) + compiler_cls = YQLQueryCompiler + + @classmethod + def registration_hook(cls) -> None: + yql_register_dialect() diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/constants.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/constants.py new file mode 100644 index 000000000..85cbbfb00 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/constants.py @@ -0,0 +1,13 @@ +from dl_constants.enums import ( + ConnectionType, + DataSourceType, + SourceBackendType, +) + + +BACKEND_TYPE_YDB = SourceBackendType.declare("YDB") + +CONNECTION_TYPE_YDB = ConnectionType.declare("ydb") + +SOURCE_TYPE_YDB_TABLE = DataSourceType.declare("YDB_TABLE") +SOURCE_TYPE_YDB_SUBSELECT = DataSourceType.declare("YDB_SUBSELECT") diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/data_source.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/data_source.py new file mode 100644 index 000000000..443bac7ac --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/data_source.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from typing import ( + Any, + Optional, +) + +from dl_constants.enums import DataSourceType +from dl_core.data_source.sql import ( + StandardSQLDataSource, + SubselectDataSource, + require_table_name, +) +from dl_core.utils import sa_plain_text + +from dl_connector_ydb.core.base.data_source import YQLDataSourceMixin +from dl_connector_ydb.core.ydb.constants import ( + CONNECTION_TYPE_YDB, + SOURCE_TYPE_YDB_SUBSELECT, + SOURCE_TYPE_YDB_TABLE, +) + + +class YDBDataSourceMixin(YQLDataSourceMixin): + conn_type = CONNECTION_TYPE_YDB + + @classmethod + def is_compatible_with_type(cls, source_type: DataSourceType) -> bool: + return source_type in (SOURCE_TYPE_YDB_TABLE, SOURCE_TYPE_YDB_SUBSELECT) + + +class YDBTableDataSource(YDBDataSourceMixin, StandardSQLDataSource): + """YDB table""" + + @require_table_name + def get_sql_source(self, alias: Optional[str] = None) -> Any: + # cross-db joins are not supported + assert not self.db_name or self.db_name == self.connection.db_name + + # Unlike `super()`, not adding the database name here. + q = self.quote + alias_str = "" if alias is None else f" AS {q(alias)}" + return sa_plain_text(f"{q(self.table_name)}{alias_str}") + + +class YDBSubselectDataSource(YDBDataSourceMixin, SubselectDataSource): + """YDB subselect""" diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/dto.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/dto.py new file mode 100644 index 000000000..c292aa569 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/dto.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import attr + +from dl_core.connection_models.dto_defs import DefaultSQLDTO + +from dl_connector_ydb.core.ydb.constants import CONNECTION_TYPE_YDB + + +@attr.s(frozen=True) +class YDBConnDTO(DefaultSQLDTO): + conn_type = CONNECTION_TYPE_YDB diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/settings.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/settings.py new file mode 100644 index 000000000..7d32efa0a --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/settings.py @@ -0,0 +1,24 @@ +from typing import Optional + +import attr + +from dl_configs.connectors_settings import ( + ConnectorsConfigType, + ConnectorSettingsBase, +) +from dl_configs.settings_loaders.meta_definition import s_attrib +from dl_core.connectors.settings.primitives import ConnectorSettingsDefinition + + +@attr.s(frozen=True) +class YDBConnectorSettings(ConnectorSettingsBase): + DEFAULT_HOST_VALUE: Optional[str] = s_attrib("DEFAULT_HOST_VALUE", missing=None) # type: ignore + + +def ydb_settings_fallback(full_cfg: ConnectorsConfigType) -> dict[str, ConnectorSettingsBase]: + return dict(YDB=YDBConnectorSettings()) + + +class YDBSettingDefinition(ConnectorSettingsDefinition): + settings_class = YDBConnectorSettings + fallback = ydb_settings_fallback diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/connection.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/connection.py new file mode 100644 index 000000000..192a484ac --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/storage_schemas/connection.py @@ -0,0 +1,14 @@ +from marshmallow import fields as ma_fields + +from dl_core.us_manager.storage_schemas.connection import ConnectionSQLDataStorageSchema + +from dl_connector_ydb.core.ydb.us_connection import YDBConnection + + +class YDBConnectionDataStorageSchema(ConnectionSQLDataStorageSchema[YDBConnection.DataModel]): + TARGET_CLS = YDBConnection.DataModel + + token = ma_fields.String(required=False, allow_none=True, dump_default=None, load_default=None) + + username = ma_fields.String(required=False, allow_none=True, dump_default=None, load_default=None) + password = ma_fields.String(required=False, allow_none=True, dump_default=None, load_default=None) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/target_dto.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/target_dto.py new file mode 100644 index 000000000..7bef7e77e --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/target_dto.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +import attr + +from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO + + +@attr.s(frozen=True) +class YDBConnTargetDTO(BaseSQLConnTargetDTO): + """""" diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/type_transformer.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/type_transformer.py new file mode 100644 index 000000000..918d5d205 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/type_transformer.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from dl_connector_ydb.core.base.type_transformer import YQLTypeTransformerBase +from dl_connector_ydb.core.ydb.constants import CONNECTION_TYPE_YDB + + +class YDBTypeTransformer(YQLTypeTransformerBase): + conn_type = CONNECTION_TYPE_YDB + + native_to_user_map = { + nt.clone(conn_type=CONNECTION_TYPE_YDB): bi_t for nt, bi_t in YQLTypeTransformerBase.native_to_user_map.items() + } + user_to_native_map = { + bi_t: nt.clone(conn_type=CONNECTION_TYPE_YDB) for bi_t, nt in YQLTypeTransformerBase.user_to_native_map.items() + } diff --git a/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/us_connection.py b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/us_connection.py new file mode 100644 index 000000000..764bb4170 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/core/ydb/us_connection.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from typing import ( + TYPE_CHECKING, + Callable, + ClassVar, + Optional, +) + +import attr + +from dl_core.connection_executors.sync_base import SyncConnExecutorBase +from dl_core.us_connection_base import ( + ClassicConnectionSQL, + ConnectionBase, + DataSourceTemplate, +) +from dl_core.utils import secrepr +from dl_i18n.localizer_base import Localizer +from dl_utils.utils import DataKey + +from dl_connector_ydb.api.ydb.i18n.localizer import Translatable +from dl_connector_ydb.core.ydb.constants import ( + SOURCE_TYPE_YDB_SUBSELECT, + SOURCE_TYPE_YDB_TABLE, +) +from dl_connector_ydb.core.ydb.dto import YDBConnDTO + + +if TYPE_CHECKING: + from dl_core.connection_models.common_models import TableIdent + + +class YDBConnection(ClassicConnectionSQL): + allow_cache: ClassVar[bool] = True + is_always_user_source: ClassVar[bool] = True + allow_dashsql: ClassVar[bool] = True + + source_type = SOURCE_TYPE_YDB_TABLE + + @attr.s(kw_only=True) + class DataModel(ClassicConnectionSQL.DataModel): + token: Optional[str] = attr.ib(default=None, repr=secrepr) + + username = None # type: ignore # not applicable + password = None # type: ignore # -> 'token' + + @classmethod + def get_secret_keys(cls) -> set[DataKey]: + return { + *super().get_secret_keys(), + DataKey(parts=("token",)), + } + + def get_conn_dto(self) -> YDBConnDTO: + assert self.data.db_name + return YDBConnDTO( + conn_id=self.uuid, + host=self.data.host, + multihosts=(), + port=self.data.port, + db_name=self.data.db_name, + username="", # not applicable + password=self.data.token, + ) + + def get_data_source_template_templates(self, localizer: Localizer) -> list[DataSourceTemplate]: + return [ + DataSourceTemplate( + title="YDB table", + tab_title=localizer.translate(Translatable("source_templates-tab_title-table")), + source_type=SOURCE_TYPE_YDB_TABLE, + parameters=dict(), + form=[ + { + "name": "table_name", + "input_type": "text", + "default": "", + "required": True, + "title": localizer.translate(Translatable("source_templates-label-ydb_table")), + "field_doc_key": "YDB_TABLE/table_name", + }, + ], + group=[], + connection_id=self.uuid, # type: ignore # TODO: fix + ), + ] + self._make_subselect_templates( + title="Subselect over YDB", + source_type=SOURCE_TYPE_YDB_SUBSELECT, + localizer=localizer, + ) + + def get_tables( + self, + conn_executor_factory: Callable[[ConnectionBase], SyncConnExecutorBase], + db_name: Optional[str] = None, + schema_name: Optional[str] = None, + ) -> list[TableIdent]: + if db_name is None: + # Only current-database listing is feasible here. + db_name = self.data.db_name + return super().get_tables( + conn_executor_factory=conn_executor_factory, + db_name=db_name, + schema_name=schema_name, + ) + + @property + def allow_public_usage(self) -> bool: + return True diff --git a/lib/dl_connector_ydb/dl_connector_ydb/db_testing/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/db_testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/db_testing/connector.py b/lib/dl_connector_ydb/dl_connector_ydb/db_testing/connector.py new file mode 100644 index 000000000..940d3f307 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/db_testing/connector.py @@ -0,0 +1,7 @@ +from dl_db_testing.connectors.base.connector import DbTestingConnector + +from dl_connector_ydb.db_testing.engine_wrapper import YQLEngineWrapper + + +class YQLDbTestingConnector(DbTestingConnector): + engine_wrapper_classes = (YQLEngineWrapper,) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/db_testing/engine_wrapper.py b/lib/dl_connector_ydb/dl_connector_ydb/db_testing/engine_wrapper.py new file mode 100644 index 000000000..71fc68a1f --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/db_testing/engine_wrapper.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import os +from typing import ( + Any, + Callable, + NamedTuple, + Optional, + Sequence, + Type, +) + +import shortuuid +import sqlalchemy as sa +from sqlalchemy.types import TypeEngine +import ydb + +from dl_db_testing.database.engine_wrapper import EngineWrapperBase + + +class YdbTypeSpec(NamedTuple): + type: ydb.PrimitiveType + to_sql_str: Callable[[Any], str] + + +SA_TYPE_TO_YDB_TYPE: dict[Type[TypeEngine], YdbTypeSpec] = { + sa.SmallInteger: YdbTypeSpec(type=ydb.PrimitiveType.Uint8, to_sql_str=str), + sa.Integer: YdbTypeSpec(type=ydb.PrimitiveType.Int32, to_sql_str=str), + sa.BigInteger: YdbTypeSpec(type=ydb.PrimitiveType.Int64, to_sql_str=str), + sa.Float: YdbTypeSpec(type=ydb.PrimitiveType.Double, to_sql_str=str), + sa.Boolean: YdbTypeSpec(type=ydb.PrimitiveType.Bool, to_sql_str=lambda x: str(bool(x))), + sa.String: YdbTypeSpec(type=ydb.PrimitiveType.String, to_sql_str=lambda x: f'"{x}"'), + sa.Unicode: YdbTypeSpec(type=ydb.PrimitiveType.Utf8, to_sql_str=lambda x: f'"{x}"'), + sa.Date: YdbTypeSpec(type=ydb.PrimitiveType.Date, to_sql_str=lambda x: f'DateTime::MakeDate($date_parse("{x}"))'), + sa.DateTime: YdbTypeSpec( + ydb.PrimitiveType.Datetime, to_sql_str=lambda x: f'DateTime::MakeDatetime($datetime_parse("{x}"))' + ), + sa.TIMESTAMP: YdbTypeSpec( + ydb.PrimitiveType.Timestamp, to_sql_str=lambda x: f'DateTime::MakeTimestamp($datetime_parse("{x}"))' + ), +} + + +class YQLEngineWrapper(EngineWrapperBase): + URL_PREFIX = "yql" + + def get_conn_credentials(self, full: bool = False) -> dict: + return dict( + endpoint=self.engine.url.query["endpoint"], + db_name=self.engine.url.query["database"], + ) + + def get_version(self) -> Optional[str]: + return None + + def _generate_table_description(self, columns: Sequence[sa.Column]) -> ydb.TableDescription: + table = ydb.TableDescription().with_columns( + *[ydb.Column(col.name, ydb.OptionalType(SA_TYPE_TO_YDB_TYPE[type(col.type)].type)) for col in columns] + ) + primary_keys = [col.name for col in columns if False] # if primary_key] # FIXME + if not primary_keys: + primary_keys = [columns[0].name] + return table.with_primary_keys(*primary_keys) + + def _get_table_path(self, table: sa.Table) -> str: + return os.path.join(self.engine.url.query["database"], table.name) + + def _get_connection_params(self) -> ydb.DriverConfig: + return ydb.DriverConfig( + endpoint=self.engine.url.query["endpoint"], + database=self.engine.url.query["database"], + ) + + def table_from_columns( + self, + columns: Sequence[sa.Column], + *, + schema: Optional[str] = None, + table_name: Optional[str] = None, + ) -> sa.Table: + table_name = table_name or f"test_table_{shortuuid.uuid()[:10]}" + table = sa.Table(table_name, sa.MetaData(), *columns, schema=schema) + return table + + def create_table(self, table: sa.Table) -> None: + table_description = self._generate_table_description(table.columns) + table_path = self._get_table_path(table) + connection_params = self._get_connection_params() + driver = ydb.Driver(connection_params) + driver.wait(timeout=5) + session = driver.table_client.session().create() + session.create_table(table_path, table_description) + driver.stop(timeout=5) + + def insert_into_table(self, table: sa.Table, data: Sequence[dict]) -> None: + connection_params = ydb.DriverConfig( + endpoint=self.engine.url.query["endpoint"], + database=self.engine.url.query["database"], + ) + driver = ydb.Driver(connection_params) + driver.wait(timeout=5) + session = driver.table_client.session().create() + + table_path = self._get_table_path(table) + + upsert_query_prefix = f""" + $date_parse = DateTime::Parse("%Y-%m-%d"); + $datetime_parse = DateTime::Parse("%Y-%m-%d %H:%M:%S"); + UPSERT INTO `{table_path}` ({", ".join([column.name for column in table.columns])}) VALUES + """ + upserts = ( + "({})".format( + ", ".join( + [ + ( + "NULL" + if data[column.name] is None + else SA_TYPE_TO_YDB_TYPE[type(column.type)].to_sql_str(data[column.name]) + ) + for column in table.columns + ] + ) + ) + for data in data + ) + session.transaction().execute(upsert_query_prefix + ",\n".join(upserts) + ";", commit_tx=True) + driver.stop(timeout=5) + + def drop_table(self, db_name: str, table: sa.Table) -> None: + connection_params = self._get_connection_params() + driver = ydb.Driver(connection_params) + driver.wait(timeout=5) + session = driver.table_client.session().create() + table_path = self._get_table_path(table) + + try: + session.drop_table(table_path) + except ydb.issues.SchemeError as err: + if "does not exist" in str(err): + pass # Table does not exist + else: + raise + + driver.stop(timeout=5) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/connector.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/connector.py new file mode 100644 index 000000000..7d87087c4 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/connector.py @@ -0,0 +1,13 @@ +from ydb.sqlalchemy import YqlDialect as SAYqlDialect + +from dl_formula.connectors.base.connector import FormulaConnector + +from dl_connector_ydb.formula.constants import YqlDialect as YqlDialectNS +from dl_connector_ydb.formula.definitions.all import DEFINITIONS + + +class YQLFormulaConnector(FormulaConnector): + dialect_ns_cls = YqlDialectNS + dialects = YqlDialectNS.YQL + op_definitions = DEFINITIONS + sa_dialect = SAYqlDialect() diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/constants.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/constants.py new file mode 100644 index 000000000..5cb951655 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/constants.py @@ -0,0 +1,15 @@ +from dl_formula.core.dialect import ( + DialectName, + DialectNamespace, + simple_combo, +) + + +DIALECT_NAME_YDB = DialectName.declare("YDB") # YDB ScanQuery connection (YQL dialect) +DIALECT_NAME_YQ = DialectName.declare("YQ") # YQ (Yandex Query) (YQL dialect) + + +class YqlDialect(DialectNamespace): + YDB = simple_combo(name=DIALECT_NAME_YDB) + YQ = simple_combo(name=DIALECT_NAME_YQ) + YQL = YDB | YQ diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/all.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/all.py new file mode 100644 index 000000000..4ad0a9540 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/all.py @@ -0,0 +1,26 @@ +from dl_connector_ydb.formula.definitions.conditional_blocks import DEFINITIONS_COND_BLOCKS +from dl_connector_ydb.formula.definitions.functions_aggregation import DEFINITIONS_AGG +from dl_connector_ydb.formula.definitions.functions_datetime import DEFINITIONS_DATETIME +from dl_connector_ydb.formula.definitions.functions_logical import DEFINITIONS_LOGICAL +from dl_connector_ydb.formula.definitions.functions_markup import DEFINITIONS_MARKUP +from dl_connector_ydb.formula.definitions.functions_math import DEFINITIONS_MATH +from dl_connector_ydb.formula.definitions.functions_string import DEFINITIONS_STRING +from dl_connector_ydb.formula.definitions.functions_type import DEFINITIONS_TYPE +from dl_connector_ydb.formula.definitions.operators_binary import DEFINITIONS_BINARY +from dl_connector_ydb.formula.definitions.operators_ternary import DEFINITIONS_TERNARY +from dl_connector_ydb.formula.definitions.operators_unary import DEFINITIONS_UNARY + + +DEFINITIONS = [ + *DEFINITIONS_COND_BLOCKS, + *DEFINITIONS_AGG, + *DEFINITIONS_DATETIME, + *DEFINITIONS_LOGICAL, + *DEFINITIONS_MARKUP, + *DEFINITIONS_MATH, + *DEFINITIONS_STRING, + *DEFINITIONS_TYPE, + *DEFINITIONS_UNARY, + *DEFINITIONS_BINARY, + *DEFINITIONS_TERNARY, +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/conditional_blocks.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/conditional_blocks.py new file mode 100644 index 000000000..7944cb16b --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/conditional_blocks.py @@ -0,0 +1,22 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.conditional_blocks as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_COND_BLOCKS = [ + # _case_block_ + base.CaseBlock.for_dialect(D.YQL), + # _if_block_ + base.IfBlock3( + variants=[ + V(D.YQL, sa.func.IF), + ] + ), + base.IfBlockMulti.for_dialect(D.YQL), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_aggregation.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_aggregation.py new file mode 100644 index 000000000..6abe584da --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_aggregation.py @@ -0,0 +1,173 @@ +import sqlalchemy as sa +import ydb.sqlalchemy as ydb_sa + +from dl_formula.definitions.base import ( + TranslationVariant, + TranslationVariantWrapped, +) +from dl_formula.definitions.common import quantile_value +import dl_formula.definitions.functions_aggregation as base +from dl_formula.definitions.literals import un_literal +from dl_formula.shortcuts import n + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make +VW = TranslationVariantWrapped.make + + +def _all_concat_yql(expr, sep=", "): # type: ignore + res = expr + res = sa.cast(res, sa.TEXT) + res = sa.func.AGGREGATE_LIST_DISTINCT(res) + res = sa.func.ListSortAsc(res) + # Would be nicer to cast List to List at this point. + res = sa.func.String.JoinFromList(res, sep) + return res + + +DEFINITIONS_AGG = [ + # all_concat + base.AggAllConcat1( + variants=[ + V(D.YQL, _all_concat_yql), + ] + ), + base.AggAllConcat2( + variants=[ + V(D.YQL, _all_concat_yql), + ] + ), + # any + base.AggAny( + variants=[ + V(D.YQL, sa.func.SOME), + ] + ), + # arg_max + base.AggArgMax( + variants=[ + V(D.YQL, sa.func.MAX_BY), + ] + ), + # arg_min + base.AggArgMin( + variants=[ + V(D.YQL, sa.func.MIN_BY), + ] + ), + # avg + base.AggAvgFromNumber.for_dialect(D.YQL), + base.AggAvgFromDate( + variants=[ + # in YQL AVG returns Double which can not be casted to Datetime so we have to convert it to INT explicitly + VW(D.YQL, lambda date_val: n.func.DATE(n.func.INT(n.func.AVG(n.func.INT(date_val))))) + ] + ), + base.AggAvgFromDatetime.for_dialect(D.YQL), + # avg_if + base.AggAvgIf( + variants=[ + V(D.YQL, sa.func.avg_if), + ] + ), + # count + base.AggCount0.for_dialect(D.YQL), + base.AggCount1.for_dialect(D.YQL), + # count_if + base.AggCountIf( + variants=[ + V(D.YQL, sa.func.COUNT_IF), + ] + ), + # countd + base.AggCountd.for_dialect(D.YQL), + # countd_approx + base.AggCountdApprox( + variants=[ + V(D.YQL, sa.func.CountDistinctEstimate), + ] + ), + # countd_if + base.AggCountdIf.for_dialect(D.YQL), + # max + base.AggMax.for_dialect(D.YQL), + # median + base.AggMedian( + variants=[ + V(D.YQL, sa.func.MEDIAN), + ] + ), + # min + base.AggMin.for_dialect(D.YQL), + # quantile + base.AggQuantile( + variants=[ + V( + D.YQL, + lambda expr, quant: sa.func.PERCENTILE( + expr, + quantile_value(un_literal(quant)), + ), + ), + ] + ), + # stdev + base.AggStdev( + variants=[ + V(D.YQL, sa.func.STDDEVSAMP), + ] + ), + # stdevp + base.AggStdevp( + variants=[ + V(D.YQL, sa.func.STDDEVPOP), + ] + ), + # sum + base.AggSum.for_dialect(D.YQL), + # sum_if + base.AggSumIf( + variants=[ + V(D.YQL, sa.func.SUM_IF), + ] + ), + # top_concat + base.AggTopConcat1( + variants=[ + # String::JoinFromList(ListMap(TOPFREQ(expr, amount), ($x) -> { RETURN cast($x.Value as Utf8); }), sep) + V( + D.YQL, + lambda expr, amount, sep=", ": sa.func.String.JoinFromList( + sa.func.ListMap(sa.func.TOPFREQ(expr, amount), ydb_sa.types.Lambda(lambda x: sa.cast(x, sa.Text))), + ", ", + ), + ), + ] + ), + base.AggTopConcat2( + variants=[ + # String::JoinFromList(ListMap(TOPFREQ(expr, amount), ($x) -> { RETURN cast($x.Value as Utf8); }), sep) + V( + D.YQL, + lambda expr, amount, sep=", ": sa.func.String.JoinFromList( + sa.func.ListMap(sa.func.TOPFREQ(expr, amount), ydb_sa.types.Lambda(lambda x: sa.cast(x, sa.Text))), + ", ", + ), + ), + ] + ), + # var + base.AggVar( + variants=[ + V(D.YQL, sa.func.VARSAMP), + ] + ), + # varp + base.AggVarp( + variants=[ + V(D.YQL, sa.func.VARPOP), + ] + ), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_datetime.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_datetime.py new file mode 100644 index 000000000..816bf43e9 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_datetime.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import sqlalchemy as sa + +from dl_formula.definitions.base import ( + TranslationVariant, + TranslationVariantWrapped, +) +from dl_formula.definitions.common_datetime import ( + YQL_INTERVAL_FUNCS, + date_add_yql, + datetime_add_yql, +) +import dl_formula.definitions.functions_datetime as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make +VW = TranslationVariantWrapped.make + + +YQL_DATE_DATETRUNC_FUNCS = { + "year": "StartOfYear", + "quarter": "StartOfQuarter", + "month": "StartOfMonth", + "week": "StartOfWeek", +} + + +def _datetrunc2_yql_impl(date_ctx, unit_ctx): # type: ignore + date_expr = date_ctx.expression + unit = base.norm_datetrunc_unit(unit_ctx.expression) + + func_name = YQL_DATE_DATETRUNC_FUNCS.get(unit) + if func_name is not None: + func = getattr(sa.func.DateTime, func_name) + return sa.func.DateTime.MakeDatetime(func(date_expr)) + + amount = 1 + func_name = YQL_INTERVAL_FUNCS.get(unit) + if func_name is not None: + func = getattr(sa.func.DateTime, func_name) + return sa.func.DateTime.MakeDatetime( + sa.func.DateTime.StartOf( + date_expr, + func(amount), + ) + ) + + # This normally should not happen + raise NotImplementedError(f"Unsupported unit {unit}") + + +DEFINITIONS_DATETIME = [ + # dateadd + base.FuncDateadd1.for_dialect(D.YQL), + base.FuncDateadd2Unit.for_dialect(D.YQL), + base.FuncDateadd2Number.for_dialect(D.YQL), + base.FuncDateadd3DateConstNum( + variants=[ + V(D.YQL, lambda date, what, num: date_add_yql(date, what, num, const_mult=True)), + ] + ), + base.FuncDateadd3DateNonConstNum( + variants=[ + V(D.YQL, lambda date, what, num: date_add_yql(date, what, num, const_mult=False)), + ] + ), + base.FuncDateadd3DatetimeConstNum( + variants=[ + V(D.YQL, lambda date, what, num: datetime_add_yql(date, what, num, const_mult=True)), + ] + ), + base.FuncDateadd3DatetimeNonConstNum( + variants=[ + V(D.YQL, lambda date, what, num: datetime_add_yql(date, what, num, const_mult=False)), + ] + ), + base.FuncDateadd3GenericDatetimeNonConstNum( + variants=[ + V(D.YQL, lambda date, what, num: datetime_add_yql(date, what, num, const_mult=False)), + ] + ), + # datepart + base.FuncDatepart2.for_dialect(D.YQL), + base.FuncDatepart3Const.for_dialect(D.YQL), + base.FuncDatepart3NonConst.for_dialect(D.YQL), + # datetrunc + base.FuncDatetrunc2Date( + variants=[ + V( + D.YQL, + lambda date, unit: sa.func.DateTime.MakeDate( + getattr(sa.func.DateTime, YQL_DATE_DATETRUNC_FUNCS[base.norm_datetrunc_unit(unit)])(date) + ) + if base.norm_datetrunc_unit(unit) in YQL_DATE_DATETRUNC_FUNCS + else date, + ), + ] + ), + base.FuncDatetrunc2Datetime( + variants=[ + VW(D.YQL, _datetrunc2_yql_impl), + ] + ), + # day + base.FuncDay( + variants=[ + V(D.YQL, sa.func.DateTime.GetDayOfMonth), + ] + ), + # dayofweek + base.FuncDayofweek1.for_dialect(D.YQL), + base.FuncDayofweek2( + variants=[ + V( + D.YQL, + lambda date_expr, firstday_expr: base.dow_firstday_shift( + sa.func.DateTime.GetDayOfWeek(date_expr), firstday_expr + ), + ), + ] + ), + # genericnow + base.FuncGenericNow( + variants=[ + V(D.YQL, sa.func.CurrentUtcDatetime), + ] + ), + # hour + base.FuncHourDate.for_dialect(D.YQL), + base.FuncHourDatetime( + variants=[ + V(D.YQL, sa.func.DateTime.GetHour), + ] + ), + # minute + base.FuncMinuteDate.for_dialect(D.YQL), + base.FuncMinuteDatetime( + variants=[ + V(D.YQL, sa.func.DateTime.GetMinute), + ] + ), + # month + base.FuncMonth( + variants=[ + V(D.YQL, sa.func.DateTime.GetMonth), + ] + ), + # now + base.FuncNow( + variants=[ + V(D.YQL, sa.func.CurrentUtcDatetime), + ] + ), + # quarter + base.FuncQuarter( + variants=[ + V(D.YQL, lambda date: sa.cast((sa.func.DateTime.GetMonth(date) + 2) / 3, sa.INTEGER)), + ] + ), + # second + base.FuncSecondDate.for_dialect(D.YQL), + base.FuncSecondDatetime( + variants=[ + V(D.YQL, sa.func.DateTime.GetSecond), + ] + ), + # today + base.FuncToday( + variants=[ + V(D.YQL, sa.func.CurrentUtcDate), # https://ydb.tech/en/docs/yql/reference/syntax/not_yet_supported#now + ] + ), + # week + base.FuncWeek( + variants=[ + V(D.YQL, sa.func.DateTime.GetWeekOfYearIso8601), + ] + ), + # year + base.FuncYear( + variants=[ + V(D.YQL, sa.func.DateTime.GetYear), + ] + ), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_logical.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_logical.py new file mode 100644 index 000000000..fdf9fa4fa --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_logical.py @@ -0,0 +1,34 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.functions_logical as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_LOGICAL = [ + # case + base.FuncCase.for_dialect(D.YQL), + # if + base.FuncIf.for_dialect(D.YQL), + # ifnull + base.FuncIfnull( + variants=[ + V(D.YQL, sa.func.coalesce), + ] + ), + # iif + base.FuncIif3Legacy.for_dialect(D.YQL), + # isnull + base.FuncIsnull.for_dialect(D.YQL), + # zn + base.FuncZn( + variants=[ + # See also: `NANVL()` to also replace `NaN`s. + V(D.YQL, lambda x: sa.func.coalesce(x, 0)), + ] + ), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_markup.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_markup.py new file mode 100644 index 000000000..2cf9677ba --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_markup.py @@ -0,0 +1,20 @@ +import dl_formula.definitions.functions_markup as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +DEFINITIONS_MARKUP = [ + # + + base.BinaryPlusMarkup.for_dialect(D.YQL), + # __str + base.FuncInternalStrConst.for_dialect(D.YQL), + base.FuncInternalStr.for_dialect(D.YQL), + # bold + base.FuncBold.for_dialect(D.YQL), + # italic + base.FuncItalics.for_dialect(D.YQL), + # markup + base.ConcatMultiMarkup.for_dialect(D.YQL), + # url + base.FuncUrl.for_dialect(D.YQL), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_math.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_math.py new file mode 100644 index 000000000..751ccb306 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_math.py @@ -0,0 +1,204 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import ( + TranslationVariant, + TranslationVariantWrapped, +) +import dl_formula.definitions.functions_math as base +from dl_formula.shortcuts import n + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make +VW = TranslationVariantWrapped.make + + +DEFINITIONS_MATH = [ + # abs + base.FuncAbs( + variants=[ + V(D.YQL, sa.func.Math.Abs), # `Math::Abs(…)` + ] + ), + # acos + base.FuncAcos( + variants=[ + V(D.YQL, sa.func.Math.Acos), + ] + ), + # asin + base.FuncAsin( + variants=[ + V(D.YQL, sa.func.Math.Asin), + ] + ), + # atan + base.FuncAtan( + variants=[ + V(D.YQL, sa.func.Math.Atan), + ] + ), + # atan2 + base.FuncAtan2( + variants=[ + V(D.YQL, sa.func.Math.Atan2), + ] + ), + # ceiling + base.FuncCeiling( + variants=[ + V(D.YQL, sa.func.Math.Ceil), + ] + ), + # cos + base.FuncCos( + variants=[ + V(D.YQL, sa.func.Math.Cos), + ] + ), + # cot + base.FuncCot( + variants=[ + V(D.YQL, lambda x: sa.func.Math.Cos(x) / sa.func.Math.Sin(x)), + ] + ), + # degrees + base.FuncDegrees( + variants=[ + V(D.YQL, lambda x: x / sa.func.Math.Pi() * 180.0), + ] + ), + # div + base.FuncDivBasic( + variants=[ + V(D.YQL, lambda x, y: sa.cast(x / y, sa.BIGINT)), + ] + ), + # div_safe + base.FuncDivSafe2( + variants=[ + V(D.YQL, lambda x, y: sa.cast(sa.func.IF(y != 0, x / y), sa.BIGINT)), + ] + ), + base.FuncDivSafe3( + variants=[ + V(D.YQL, lambda x, y, default: sa.cast(sa.func.IF(y != 0, x / y, default), sa.BIGINT)), + ] + ), + # exp + base.FuncExp( + variants=[ + V(D.YQL, sa.func.Math.Exp), + ] + ), + # fdiv_safe + base.FuncFDivSafe2( + variants=[ + V(D.YQL, lambda x, y: sa.func.IF(y != 0, x / y)), + ] + ), + base.FuncFDivSafe3( + variants=[ + V(D.YQL, lambda x, y, default: sa.func.IF(y != 0, x / y, default)), + ] + ), + # floor + base.FuncFloor( + variants=[ + V(D.YQL, sa.func.Math.Floor), + ] + ), + # greatest + base.FuncGreatest1.for_dialect(D.YQL), + base.FuncGreatestMain.for_dialect(D.YQL), + base.GreatestMulti.for_dialect(D.YQL), + # least + base.FuncLeast1.for_dialect(D.YQL), + base.FuncLeastMain.for_dialect(D.YQL), + base.LeastMulti.for_dialect(D.YQL), + # ln + base.FuncLn( + variants=[ + V(D.YQL, sa.func.Math.Log), + ] + ), + # log + base.FuncLog( + variants=[ + V(D.YQL, lambda x, y: sa.func.Math.Log(x) / sa.func.Math.Log(y)), + ] + ), + # log10 + base.FuncLog10( + variants=[ + V(D.YQL, sa.func.Math.Log10), + ] + ), + # pi + base.FuncPi( + variants=[ + V(D.YQL, sa.func.Math.Pi), + ] + ), + # power + base.FuncPower( + variants=[ + V(D.YQL, sa.func.Math.Pow), + ] + ), + # radians + base.FuncRadians( + variants=[ + V(D.YQL, lambda x: x * sa.func.Math.Pi() / 180.0), + ] + ), + # round + base.FuncRound1( + variants=[ + V(D.YQL, sa.func.Math.Round), + ] + ), + base.FuncRound2( + variants=[ + # in YQL Math::Round takes power of 10 instead of precision, so we have to invert the `num` value + V(D.YQL, lambda x, num: sa.func.Math.Round(x, -num)), + ] + ), + # sign + base.FuncSign( + variants=[ + V( + D.YQL, + lambda x: n.if_( + n.if_(x < 0).then(-1), # type: ignore # TODO: fix + n.if_(x > 0).then(1), # type: ignore # TODO: fix + ).else_(0), + ), + ] + ), + # sin + base.FuncSin( + variants=[ + V(D.YQL, sa.func.Math.Sin), + ] + ), + # sqrt + base.FuncSqrt( + variants=[ + V(D.YQL, sa.func.Math.Sqrt), + ] + ), + # square + base.FuncSquare( + variants=[ + V(D.YQL, lambda x: sa.func.Math.Pow(x, 2)), + ] + ), + # tan + base.FuncTan( + variants=[ + V(D.YQL, sa.func.Math.Tan), + ] + ), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_string.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_string.py new file mode 100644 index 000000000..74310c529 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_string.py @@ -0,0 +1,232 @@ +import sqlalchemy as sa +import ydb.sqlalchemy as ydb_sa + +from dl_formula.definitions.base import TranslationVariant +from dl_formula.definitions.common import ( + ifnotnull, + make_binary_chain, +) +import dl_formula.definitions.functions_string as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_STRING = [ + # ascii + base.FuncAscii( + variants=[ + # ListHead(Unicode::ToCodePointList(Unicode::Substring('ы', 0, 1))) = 1099 + V( + D.YQL, + lambda x: sa.func.ListHead( + sa.func.Unicode.ToCodePointList(sa.func.Unicode.Substring(sa.cast(x, sa.TEXT), 0, 1)) + ), + ), + ] + ), + # char + base.FuncChar( + variants=[ + # CASE WHEN value IS NULL THEN NULL ELSE + # Unicode::FromCodePointList(AsList(COALESCE(CAST(value AS UInt32), 0))) + # END + V( + D.YQL, + lambda value: ifnotnull( + value, + # int -> List -> utf8 + sa.func.Unicode.FromCodePointList( + sa.func.AsList( + # coalesce is needed to un-Nullable the type. + sa.func.COALESCE(sa.cast(value, ydb_sa.types.UInt32), 0), + ) + ), + ), + ), + ] + ), + # concat + base.Concat1.for_dialect((D.YQL)), + base.ConcatMultiStrConst.for_dialect(D.YQL), + base.ConcatMultiStr( + variants=[ + V( + D.YQL, + lambda *args: make_binary_chain( + (lambda x, y: x.concat(y)), # should result in `x || y` SQL. + *args, # should result in `x || y || z || ...` SQL + wrap_as_nodes=False, + ), + ), + ] + ), + base.ConcatMultiAny.for_dialect(D.YQL), + # contains + base.FuncContainsConst( + variants=[ + # V(D.YQL, + # # # “'%', '_' and '\' are currently not supported in ESCAPE clause,” + # lambda x, y: x.like('%{}%'.format(quote_like(y.value, escape='!')), escape='!')), + # # Allows UTF8; also, notably, does not allow a nullable second argument: + V(D.YQL, sa.func.String.Contains), + ] + ), + base.FuncContainsNonConst( + variants=[ + # `''` shouldn't be ever used due to `ifnotnull`. + V(D.YQL, lambda x, y: ifnotnull(y, sa.func.String.Contains(x, sa.func.COALESCE(y, "")))), + ] + ), + base.FuncContainsNonString.for_dialect(D.YQL), + # notcontains + base.FuncNotContainsConst.for_dialect(D.YQL), + base.FuncNotContainsNonConst.for_dialect(D.YQL), + base.FuncNotContainsNonString.for_dialect(D.YQL), + # endswith + base.FuncEndswithConst( + variants=[ + V(D.YQL, sa.func.String.EndsWith), + ] + ), + base.FuncEndswithNonConst( + variants=[ + # `''` shouldn't ever happen due to `ifnotnull`. + V(D.YQL, lambda x, y: ifnotnull(y, sa.func.String.EndsWith(x, sa.func.COALESCE(y, "")))), + ] + ), + base.FuncEndswithNonString.for_dialect(D.YQL), + # find + base.FuncFind2( + variants=[ + # In YQL indices start from 0, but we count them from 1, so have to do -1/+1 here + V(D.YQL, lambda text, piece: sa.func.COALESCE(sa.func.Unicode.Find(text, piece), -1) + 1), + ] + ), + base.FuncFind3( + variants=[ + # In YQL indices start from 0, but we count them from 1, so have to do -1/+1 here + V( + D.YQL, + lambda text, piece, startpos: sa.func.COALESCE(sa.func.Unicode.Find(text, piece, startpos), -1) + 1, + ), + ] + ), + # icontains + base.FuncIContainsNonConst.for_dialect(D.YQL), + base.FuncIContainsNonString.for_dialect(D.YQL), + # iendswith + base.FuncIEndswithNonConst.for_dialect(D.YQL), + base.FuncIEndswithNonString.for_dialect(D.YQL), + # istartswith + base.FuncIStartswithNonConst.for_dialect(D.YQL), + base.FuncIStartswithNonString.for_dialect(D.YQL), + # left + base.FuncLeft( + variants=[ + V(D.YQL, lambda x, y: sa.func.Unicode.Substring(sa.cast(x, sa.TEXT), 0, y)), + ] + ), + # len + base.FuncLenString( + variants=[ + V(D.YQL, lambda val: sa.func.Unicode.GetLength(sa.cast(val, sa.TEXT))), + ] + ), + # lower + base.FuncLowerConst.for_dialect(D.YQL), + base.FuncLowerNonConst( + variants=[ + V(D.YQL, lambda val: sa.func.Unicode.ToLower(sa.cast(val, sa.TEXT))), + ] + ), + # regexp_extract + # TODO: YQL + # https://ydb.tech/en/docs/yql/reference/udf/list/hyperscan + # Problem: "By default, all functions work in the single-byte mode. + # However, if the regular expression is a valid UTF-8 string but is not a valid ASCII string, + # the UTF-8 mode is enabled automatically." However, we can't use higher-order functions yet. + # replace + base.FuncReplace( + variants=[ + V( + D.YQL, + lambda val, repl_from, repl_with: sa.func.Unicode.ReplaceAll( + sa.cast(val, sa.TEXT), + sa.func.COALESCE(sa.cast(repl_from, sa.TEXT), ""), + sa.func.COALESCE(sa.cast(repl_with, sa.TEXT), ""), + ), + ), + ] + ), + # right + base.FuncRight( + variants=[ + V( + D.YQL, + lambda x, y: sa.func.Unicode.Substring( + sa.cast(x, sa.TEXT), + sa.func.Unicode.GetLength(sa.cast(x, sa.TEXT)) - y, + ), + ), + ] + ), + # space + base.FuncSpaceConst.for_dialect(D.YQL), + base.FuncSpaceNonConst( + variants=[ + # YQL string multiplication: also consider + # sa.func.ListConcat(sa.func.ListReplicate(sa.cast(' ', sa.TEXT), size)) + V(D.YQL, lambda size: sa.cast(sa.func.String.LeftPad("", size, " "), sa.TEXT)), + ] + ), + # split + base.FuncSplit3( + variants=[ + V( + D.YQL, + lambda text, delim, ind: sa.func.ListHead( + sa.func.ListSkip( + sa.func.Unicode.SplitToList(sa.cast(text, sa.TEXT), delim), # must be non-nullable + ind - 1, + ) + ), + ), + ] + ), + # startswith + base.FuncStartswithConst( + variants=[ + V(D.YQL, sa.func.String.StartsWith), + ] + ), + base.FuncStartswithNonConst( + variants=[ + # `''` shouldn't ever happen due to `ifnotnull`. + V(D.YQL, lambda x, y: ifnotnull(y, sa.func.String.StartsWith(x, sa.func.COALESCE(y, "")))), + ] + ), + base.FuncStartswithNonString.for_dialect(D.YQL), + # substr + base.FuncSubstr2( + variants=[ + # In YQL indices start from 0, but we count them from 1, so have to do -1 here + V(D.YQL, lambda val, start: sa.func.Unicode.Substring(sa.cast(val, sa.TEXT), start - 1)), + ] + ), + base.FuncSubstr3( + variants=[ + # In YQL indices start from 0, but we count them from 1, so have to do -1 here + V(D.YQL, lambda val, start, length: sa.func.Unicode.Substring(sa.cast(val, sa.TEXT), start - 1, length)), + ] + ), + # upper + base.FuncUpperConst.for_dialect(D.YQL), + base.FuncUpperNonConst( + variants=[ + V(D.YQL, lambda val: sa.func.Unicode.ToUpper(sa.cast(val, sa.TEXT))), + ] + ), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py new file mode 100644 index 000000000..7402a16b8 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/functions_type.py @@ -0,0 +1,185 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.functions_type as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_TYPE = [ + # bool + base.FuncBoolFromNull.for_dialect(D.YQL), + base.FuncBoolFromNumber.for_dialect(D.YQL), + base.FuncBoolFromBool.for_dialect(D.YQL), + base.FuncBoolFromStrGeo.for_dialect(D.YQL), + base.FuncBoolFromDateDatetime.for_dialect(D.YQL), + # date + base.FuncDate1FromNull.for_dialect(D.YQL), + base.FuncDate1FromDatetime.for_dialect(D.YQL), + base.FuncDate1FromString.for_dialect(D.YQL), + base.FuncDate1FromNumber( + variants=[ + V( + D.YQL, lambda expr: sa.cast(sa.cast(sa.cast(expr, sa.BIGINT), sa.DATETIME), sa.DATE) + ), # number -> dt -> date + ] + ), + # datetime + base.FuncDatetime1FromNull.for_dialect(D.YQL), + base.FuncDatetime1FromDatetime.for_dialect(D.YQL), + base.FuncDatetime1FromDate.for_dialect(D.YQL), + base.FuncDatetime1FromNumber( + variants=[ + V(D.YQL, lambda expr: sa.cast(sa.cast(expr, sa.BIGINT), sa.DateTime())), + ] + ), + base.FuncDatetime1FromString( + variants=[ + # e.g. `DateTime::MakeDatetime(DateTime::ParseIso8601('2021-06-01 18:00:59')) as c` + V(D.YQL, lambda expr: sa.func.DateTime.MakeDatetime(sa.func.DateTime.ParseIso8601(expr))), + ] + ), + # datetimetz + base.FuncDatetimeTZConst.for_dialect(D.YQL), + # float + base.FuncFloatNumber( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.FLOAT)), # TODO: need it to become SQL `CAST(… AS DOUBLE)`. + ] + ), + base.FuncFloatString( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.FLOAT)), + ] + ), + base.FuncFloatFromBool( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.FLOAT)), + ] + ), + base.FuncFloatFromDate( + variants=[ + V(D.YQL, lambda expr: sa.cast(sa.cast(expr, sa.DATETIME), sa.FLOAT)), # date -> dt -> number + ] + ), + base.FuncFloatFromDatetime( + variants=[ + V(D.YQL, lambda expr: sa.cast(expr, sa.FLOAT)), + ] + ), + base.FuncFloatFromGenericDatetime( + variants=[ + V(D.YQL, lambda expr: sa.cast(expr, sa.FLOAT)), + ] + ), + # genericdatetime + base.FuncGenericDatetime1FromNull.for_dialect(D.YQL), + base.FuncGenericDatetime1FromDatetime.for_dialect(D.YQL), + base.FuncGenericDatetime1FromDate.for_dialect(D.YQL), + base.FuncGenericDatetime1FromNumber( + variants=[ + V(D.YQL, lambda expr: sa.cast(sa.cast(expr, sa.BIGINT), sa.DateTime())), + ] + ), + base.FuncGenericDatetime1FromString( + variants=[ + # e.g. `DateTime::MakeDatetime(DateTime::ParseIso8601('2021-06-01 18:00:59')) as c` + V(D.YQL, lambda expr: sa.func.DateTime.MakeDatetime(sa.func.DateTime.ParseIso8601(expr))), + ] + ), + # geopoint + base.FuncGeopointFromStr.for_dialect(D.YQL), + base.FuncGeopointFromCoords.for_dialect(D.YQL), + # geopolygon + base.FuncGeopolygon.for_dialect(D.YQL), + # int + base.FuncIntFromNull( + variants=[ + V(D.YQL, lambda _: sa.cast(sa.null(), sa.BIGINT())), + ] + ), + base.FuncIntFromInt.for_dialect(D.YQL), + base.FuncIntFromFloat( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.BIGINT())), + ] + ), + base.FuncIntFromBool( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.BIGINT)), + ] + ), + base.FuncIntFromStr( + variants=[ + V(D.YQL, lambda expr: sa.func.cast(expr, sa.BIGINT)), + ] + ), + base.FuncIntFromDate( + variants=[ + V(D.YQL, lambda expr: sa.cast(sa.cast(expr, sa.DATETIME), sa.BIGINT)), + ] + ), + base.FuncIntFromDatetime( + variants=[ + V(D.YQL, lambda expr: sa.cast(expr, sa.BIGINT)), + ] + ), + base.FuncIntFromGenericDatetime( + variants=[ + V(D.YQL, lambda expr: sa.cast(expr, sa.BIGINT)), + ] + ), + # str + base.FuncStrFromNull( + variants=[ + V(D.YQL, lambda value: sa.cast(sa.null(), sa.TEXT)), + ] + ), + base.FuncStrFromUnsupported( + variants=[ + # YQL: uncertain. + # Does not work for e.g. arrays: + # V(D.YQL, lambda value: sa.cast(value, sa.TEXT)), + # Does not work for e.g. Decimal: + V( + D.YQL, + lambda value: sa.cast(sa.func.ToBytes(sa.func.Yson.SerializePretty(sa.func.Yson.From(value))), sa.TEXT), + ), + ] + ), + base.FuncStrFromInteger( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.TEXT)), + ] + ), + base.FuncStrFromFloat( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.TEXT)), + ] + ), + base.FuncStrFromBool( + variants=[ + V(D.YQL, lambda value: sa.case(whens=[(value.is_(None), sa.null()), (value, "True")], else_="False")), + ] + ), + base.FuncStrFromStrGeo.for_dialect(D.YQL), + base.FuncStrFromDate( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.TEXT)), + ] + ), + base.FuncStrFromDatetime( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.TEXT)), # results in e.g. "2021-06-01T15:20:24Z" + ] + ), + base.FuncStrFromString.for_dialect(D.YQL), + base.FuncStrFromUUID( + variants=[ + V(D.YQL, lambda value: sa.cast(value, sa.TEXT)), + ] + ), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_binary.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_binary.py new file mode 100644 index 000000000..3a3ab1a96 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_binary.py @@ -0,0 +1,147 @@ +import sqlalchemy as sa + +from dl_formula.definitions.base import TranslationVariant +from dl_formula.definitions.common_datetime import DAY_USEC +import dl_formula.definitions.operators_binary as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_BINARY = [ + # != + base.BinaryNotEqual.for_dialect(D.YQL), + # % + base.BinaryModInteger.for_dialect(D.YQL), + base.BinaryModFloat.for_dialect(D.YQL), + # * + base.BinaryMultNumbers.for_dialect(D.YQL), + base.BinaryMultStringConst.for_dialect(D.YQL), + # + + base.BinaryPlusNumbers.for_dialect(D.YQL), + base.BinaryPlusStrings.for_dialect(D.YQL), + base.BinaryPlusDateInt( + variants=[ + V(D.YQL, lambda date, days: date + sa.func.DateTime.IntervalFromDays(days)), + ] + ), + base.BinaryPlusDateFloat( + variants=[ + V(D.YQL, lambda date, days: date + sa.func.DateTime.IntervalFromDays(sa.cast(days, sa.INTEGER))), + ] + ), + base.BinaryPlusDatetimeNumber( + variants=[ + V( + D.YQL, + lambda date, days: (date + sa.func.DateTime.IntervalFromMicroseconds(base.as_bigint(days * DAY_USEC))), + ), + ] + ), + base.BinaryPlusGenericDatetimeNumber( + variants=[ + V( + D.YQL, + lambda dt, days: (dt + sa.func.DateTime.IntervalFromMicroseconds(base.as_bigint(days * DAY_USEC))), + ), + ] + ), + # - + base.BinaryMinusInts( + variants=[ + V(D.YQL, lambda num1, num2: (sa.cast(num1, sa.INTEGER) - sa.cast(num2, sa.INTEGER))), + ] + ), + base.BinaryMinusNumbers.for_dialect(D.YQL), + base.BinaryMinusDateInt( + variants=[ + V(D.YQL, lambda date, days: date - sa.func.DateTime.IntervalFromDays(days)), + ] + ), + base.BinaryMinusDateFloat( + variants=[ + V( + D.YQL, + lambda date, days: ( + date - sa.func.DateTime.IntervalFromDays(sa.cast(sa.func.Math.Ceil(days), sa.INTEGER)) + ), + ), + ] + ), + base.BinaryMinusDatetimeNumber( + variants=[ + V( + D.YQL, + lambda date, days: (date - sa.func.DateTime.IntervalFromMicroseconds(base.as_bigint(days * DAY_USEC))), + ), + ] + ), + base.BinaryMinusGenericDatetimeNumber( + variants=[ + V( + D.YQL, + lambda dt, days: (dt - sa.func.DateTime.IntervalFromMicroseconds(base.as_bigint(days * DAY_USEC))), + ), + ] + ), + base.BinaryMinusDates( + variants=[ + V(D.YQL, lambda left, right: sa.func.DateTime.ToDays(left - right)), + ] + ), + base.BinaryMinusDatetimes( + variants=[ + V(D.YQL, lambda left, right: sa.func.DateTime.ToMicroseconds(left - right) / float(DAY_USEC)), + ] + ), + base.BinaryMinusGenericDatetimes( + variants=[ + V(D.YQL, lambda left, right: sa.func.DateTime.ToMicroseconds(left - right) / float(DAY_USEC)), + ] + ), + # / + base.BinaryDivInt( + variants=[ + # See also: https://ydb.tech/en/docs/yql/reference/syntax/pragma#classicdivision + V(D.YQL, lambda x, y: sa.cast(x, sa.FLOAT) / y), + ] + ), + base.BinaryDivFloat.for_dialect(D.YQL), + # < + base.BinaryLessThan.for_dialect(D.YQL), + # <= + base.BinaryLessThanOrEqual.for_dialect(D.YQL), + # == + base.BinaryEqual.for_dialect(D.YQL), + # > + base.BinaryGreaterThan.for_dialect(D.YQL), + # >= + base.BinaryGreaterThanOrEqual.for_dialect(D.YQL), + # ^ + base.BinaryPower.for_dialect(D.YQL), + # _!= + base.BinaryNotEqualInternal.for_dialect(D.YQL), + # _== + base.BinaryEqualInternal.for_dialect(D.YQL), + # _dneq + base.BinaryEqualDenullified( + variants=[ + # YQL does not support ISNULL and other complex operations in JOIN conditions + V(D.YQL, lambda left, right: left == right), # type: ignore + ] + ), + # and + base.BinaryAnd.for_dialect(D.YQL), + # in + base.BinaryIn.for_dialect(D.YQL), + # like + base.BinaryLike.for_dialect(D.YQL), + # notin + base.BinaryNotIn.for_dialect(D.YQL), + # notlike + base.BinaryNotLike.for_dialect(D.YQL), + # or + base.BinaryOr.for_dialect(D.YQL), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_ternary.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_ternary.py new file mode 100644 index 000000000..ac01ba047 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_ternary.py @@ -0,0 +1,11 @@ +import dl_formula.definitions.operators_ternary as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +DEFINITIONS_TERNARY = [ + # between + base.TernaryBetween.for_dialect(D.YQL), + # notbetween + base.TernaryNotBetween.for_dialect(D.YQL), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_unary.py b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_unary.py new file mode 100644 index 000000000..ea2090e7b --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula/definitions/operators_unary.py @@ -0,0 +1,36 @@ +from dl_formula.definitions.base import TranslationVariant +import dl_formula.definitions.operators_unary as base + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +V = TranslationVariant.make + + +DEFINITIONS_UNARY = [ + # isfalse + base.UnaryIsFalseStringGeo.for_dialect(D.YQL), + base.UnaryIsFalseNumbers.for_dialect(D.YQL), + base.UnaryIsFalseDateTime.for_dialect(D.YQL), + base.UnaryIsFalseBoolean( + variants=[ + V(D.YQL, lambda x: x == False), # noqa: E712 + ] + ), + # istrue + base.UnaryIsTrueStringGeo.for_dialect(D.YQL), + base.UnaryIsTrueNumbers.for_dialect(D.YQL), + base.UnaryIsTrueDateTime.for_dialect(D.YQL), + base.UnaryIsTrueBoolean( + variants=[ + V(D.YQL, lambda x: x == True), # noqa: E712 + ] + ), + # neg + base.UnaryNegate.for_dialect(D.YQL), + # not + base.UnaryNotBool.for_dialect(D.YQL), + base.UnaryNotNumbers.for_dialect(D.YQL), + base.UnaryNotStringGeo.for_dialect(D.YQL), + base.UnaryNotDateDatetime.for_dialect(D.YQL), +] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/human_dialects.py b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/human_dialects.py new file mode 100644 index 000000000..f3b08485a --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/human_dialects.py @@ -0,0 +1,22 @@ +from dl_formula_ref.texts import StyledDialect + +from dl_connector_ydb.formula.constants import YqlDialect + + +HUMAN_DIALECTS = { + YqlDialect.YDB: StyledDialect( + "`YDB`", + "`YDB`
(`YQL`)", + "`YDB` (`YQL`)", + ), + YqlDialect.YQ: StyledDialect( + "`YQ`", + "`YQ`", + "`YQ`", + ), + YqlDialect.YQL: StyledDialect( + "`YQL`", + "`YQL`", + "`YQL`", + ), +} diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/i18n.py b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/i18n.py new file mode 100644 index 000000000..b8a6d8028 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/i18n.py @@ -0,0 +1,23 @@ +import os + +import attr + +from dl_i18n.localizer_base import Translatable as BaseTranslatable +from dl_i18n.localizer_base import TranslationConfig + +import dl_connector_ydb as package + + +DOMAIN = f"dl_formula_ref_{package.__name__}" + +_LOCALE_DIR = os.path.join(os.path.dirname(__file__), "..", "locales") + +CONFIGS = [ + TranslationConfig(path=_LOCALE_DIR, domain=DOMAIN, locale="en"), + TranslationConfig(path=_LOCALE_DIR, domain=DOMAIN, locale="ru"), +] + + +@attr.s +class Translatable(BaseTranslatable): + domain: str = attr.ib(default=DOMAIN) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/plugin.py b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/plugin.py new file mode 100644 index 000000000..f857c19de --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/formula_ref/plugin.py @@ -0,0 +1,22 @@ +from dl_formula_ref.functions.date import FUNCTION_NOW +from dl_formula_ref.plugins.base.plugin import FormulaRefPlugin +from dl_formula_ref.registry.note import Note + +from dl_connector_ydb.formula.constants import YqlDialect +from dl_connector_ydb.formula_ref.human_dialects import HUMAN_DIALECTS +from dl_connector_ydb.formula_ref.i18n import ( + CONFIGS, + Translatable, +) + + +class YQLFormulaRefPlugin(FormulaRefPlugin): + any_dialects = frozenset((*YqlDialect.YDB.to_list(),)) + human_dialects = HUMAN_DIALECTS + translation_configs = frozenset(CONFIGS) + function_extensions = [ + FUNCTION_NOW.extend( + dialect=YqlDialect.YDB, + notes=(Note(Translatable("On {dialects:YQL}, the function always returns the UTC date and time.")),), + ), + ] diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.mo b/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.mo new file mode 100644 index 0000000000000000000000000000000000000000..5add0edcb1f5633880d87b071baf96ec0d3bba8d GIT binary patch literal 408 zcmZXP!AiqG5Qd}HOAa3N>S14Cs@_Ueu&q%-iIy4*f)F;zm`2jwxI2O7wwZgTjf&Z0o*2Qr{^b%7 literal 0 HcmV?d00001 diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.po b/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.po new file mode 100644 index 000000000..d9968f043 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_connector_ydb.po @@ -0,0 +1,19 @@ +# Copyright (c) 2023 YANDEX LLC +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-09-22 08:16+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "label_connector-ydb" +msgstr "YDB" + +msgid "source_templates-tab_title-table" +msgstr "Table" + +msgid "source_templates-label-ydb_table" +msgstr "YDB table path" diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.mo b/lib/dl_connector_ydb/dl_connector_ydb/locales/en/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.mo new file mode 100644 index 0000000000000000000000000000000000000000..00261bf604624733771edc1595785d8dd496d13e GIT binary patch literal 241 zcmYMsF>b;@5QSk-Qn{o~u|-M-Nar|;1Xu=H$O$42{l+_(MP}C89ZPVKoF%v6EDTNf zrEjXIxeOkEds1IKiD&U3hT>K%{=}=eZT?$(cfg7vH?Tz?tfNG&XX$FT%+xm9eC3}X z(FR-R8H!L_1M{v9P`X4J&ALGf;99w^^A%3dnhc(RQ6ERXi;BF+?r&d4lPGn>NDSq< q#Y8clc-%JH(wnOG+6UzK?{We0edjfW9eT*9a>i1fsQAZn)Vl(gx +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-09-22 08:16+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "On {dialects:YQL}, the function always returns the UTC date and time." +msgstr "" diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_connector_ydb.mo b/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_connector_ydb.mo new file mode 100644 index 0000000000000000000000000000000000000000..2b3674e7756353d545ad775ff4b2d5f0d13018a8 GIT binary patch literal 436 zcmZXO&q@O^5Qn4I%N{)F)kA!M_~%wu1>3qxk!q>6AP7r#cdRShWKB|NPeO~Hym}S{ zJy$^#eE_f77w}n}u0jPvzI>g#$SiV#j8BRzAQuQlu73T9enmc! z8RT+`kR|jGJ%@fk?crH@*L-}YQ$QgbS#BS>!r$GY1gAIuW?vC*moL`N@ZA0x8sPFyKT*y z(xnkMJOQDZO!3VZ+3IrQtmKAPj5vQ*fz)o>{@oW3f)d zGp;n4&7 +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-09-22 08:16+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "label_connector-ydb" +msgstr "YDB" + +msgid "source_templates-tab_title-table" +msgstr "Таблица" + +msgid "source_templates-label-ydb_table" +msgstr "Путь к таблице в YDB" diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.mo b/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.mo new file mode 100644 index 0000000000000000000000000000000000000000..9f0eac8e176ddd70a897e4df7f450f939ebd56b2 GIT binary patch literal 442 zcmZ9I%}N6?6op68%`9EJxDU`&-IY>`Po=|u~4hXCXzwckv$|;1#yuoGDTjI4pLsREL?w- zW>{qoH$h0d%vaL6sGi_Wq>ofpezc8$f6P^2VVVL%R(mLy)}q(OK&mn4d@Mv3s* zK_7$JHBpTAvLdld#ge;>-N|m-g@`N074iBpb3n0cwH0@1$aleSQz4n!uX{%%xAY0h zWs?cwaz6?}-f|#!87kY-&;MQz=H86-xBfH_dS;$LznMEd(eL_0k8z{tdWuH_JvNVe ftS4p&X~zuB2=pv<577O!7zDJ=F=_(%VOi@Ja!H+= literal 0 HcmV?d00001 diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.po b/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.po new file mode 100644 index 000000000..2757f6bf2 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.po @@ -0,0 +1,13 @@ +# Copyright (c) 2023 YANDEX LLC +# This file is distributed under the same license as the DataLens package. +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: datalens-opensource@yandex-team.ru\n" +"POT-Creation-Date: 2023-09-22 08:16+0000\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +msgid "On {dialects:YQL}, the function always returns the UTC date and time." +msgstr "В {dialects:YQL} функция всегда возвращает дату и время в зоне UTC." diff --git a/lib/dl_connector_ydb/dl_connector_ydb/py.typed b/lib/dl_connector_ydb/dl_connector_ydb/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py new file mode 100644 index 000000000..0a5debfbc --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/base.py @@ -0,0 +1,90 @@ +import pytest + +from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration +from dl_api_lib_testing.connection_base import ConnectionTestBase +from dl_api_lib_testing.data_api_base import ( + DataApiTestParams, + StandardizedDataApiTestBase, +) +from dl_api_lib_testing.dataset_base import DatasetTestBase +from dl_constants.enums import RawSQLLevel +from dl_core_testing.database import ( + C, + Db, + DbTable, + make_table, +) + +from dl_connector_ydb.core.ydb.constants import ( + CONNECTION_TYPE_YDB, + SOURCE_TYPE_YDB_TABLE, +) +from dl_connector_ydb_tests.db.config import ( + API_TEST_CONFIG, + CONNECTION_PARAMS, + DB_CORE_URL, + TABLE_DATA, + TABLE_NAME, + TABLE_SCHEMA, +) + + +class YDBConnectionTestBase(ConnectionTestBase): + bi_compeng_pg_on = False + conn_type = CONNECTION_TYPE_YDB + + @pytest.fixture(scope="class") + def db_url(self) -> str: + return DB_CORE_URL + + @pytest.fixture(scope="class") + def bi_test_config(self) -> ApiTestEnvironmentConfiguration: + return API_TEST_CONFIG + + @pytest.fixture(scope="class") + def connection_params(self) -> dict: + return CONNECTION_PARAMS + + @pytest.fixture(scope="class") + def sample_table(self, db: Db) -> DbTable: + db_table = make_table( + db=db, + name=TABLE_NAME, + columns=[C(name=name, user_type=user_type, sa_type=sa_type) for name, user_type, sa_type in TABLE_SCHEMA], + data=[], # to avoid producing a sample data + create_in_db=False, + ) + db.create_table(db_table.table) + db.insert_into_table(db_table.table, TABLE_DATA) + return db_table + + +class YDBDashSQLConnectionTest(YDBConnectionTestBase): + @pytest.fixture(scope="class") + def connection_params(self) -> dict: + return CONNECTION_PARAMS | dict(raw_sql_level=RawSQLLevel.dashsql.value) + + +class YDBDatasetTestBase(YDBConnectionTestBase, DatasetTestBase): + @pytest.fixture(scope="class") + def dataset_params(self, sample_table: DbTable) -> dict: + return dict( + source_type=SOURCE_TYPE_YDB_TABLE.name, + parameters=dict( + table_name=sample_table.name, + ), + ) + + +class YDBDataApiTestBase(YDBDatasetTestBase, StandardizedDataApiTestBase): + mutation_caches_on = False + + @pytest.fixture(scope="class") + def data_api_test_params(self) -> DataApiTestParams: + return DataApiTestParams( + two_dims=("some_string", "some_int32"), + summable_field="some_int32", + range_field="some_int64", + distinct_field="id", + date_field="some_date", + ) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_connection.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_connection.py new file mode 100644 index 000000000..89221454b --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_connection.py @@ -0,0 +1,7 @@ +from dl_api_lib_testing.connector.connection_suite import DefaultConnectorConnectionTestSuite + +from dl_connector_ydb_tests.db.api.base import YDBConnectionTestBase + + +class TestYDBConnection(YDBConnectionTestBase, DefaultConnectorConnectionTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dashsql.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dashsql.py new file mode 100644 index 000000000..5cf29077e --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dashsql.py @@ -0,0 +1,98 @@ +from aiohttp.test_utils import TestClient +import pytest + +from dl_api_lib_testing.connector.dashsql_suite import DefaultDashSQLTestSuite +from dl_core_testing.database import DbTable + +from dl_connector_ydb_tests.db.api.base import YDBDashSQLConnectionTest +from dl_connector_ydb_tests.db.config import DASHSQL_QUERY + + +class TestYDBDashSQL(YDBDashSQLConnectionTest, DefaultDashSQLTestSuite): + @pytest.mark.asyncio + async def test_result( + self, + data_api_lowlevel_aiohttp_client: TestClient, + saved_connection_id: str, + sample_table: DbTable, + ): + resp = await self.get_dashsql_response( + data_api_aio=data_api_lowlevel_aiohttp_client, + conn_id=saved_connection_id, + query=DASHSQL_QUERY.format(table_name=sample_table.name), + ) + + resp_data = await resp.json() + assert resp_data[0]["event"] == "metadata", resp_data + assert resp_data[0]["data"]["names"][:12] == [ + "id", + "some_str", + "some_utf8", + "some_int", + "some_uint8", + "some_int64", + "some_uint64", + "some_double", + "some_bool", + "some_date", + "some_datetime", + "some_timestamp", + ] + assert resp_data[0]["data"]["driver_types"][:12] == [ + "int32?", + "string", + "utf8?", + "int32", + "uint8?", + "int64", + "uint64", + "double", + "bool", + "date", + "datetime", + "timestamp", + ] + assert resp_data[0]["data"]["db_types"][:12] == [ + "integer", + "text", + "text", + "integer", + "integer", + "integer", + "integer", + "float", + "boolean", + "date", + "datetime", + "datetime", + ] + assert resp_data[0]["data"]["bi_types"][:12] == [ + "integer", + "string", + "string", + "integer", + "integer", + "integer", + "integer", + "float", + "boolean", + "date", + "genericdatetime", + "genericdatetime", + ] + + assert resp_data[-1]["event"] == "footer", resp_data[-1] + + @pytest.mark.asyncio + async def test_result_with_error(self, data_api_lowlevel_aiohttp_client: TestClient, saved_connection_id: str): + resp = await self.get_dashsql_response( + data_api_aio=data_api_lowlevel_aiohttp_client, + conn_id=saved_connection_id, + query="select 1/", + fail_ok=True, + ) + + resp_data = await resp.json() + assert resp.status == 400, resp_data + assert resp_data["code"] == "ERR.DS_API.DB", resp_data + assert resp_data.get("details", {}).get("db_message"), resp_data diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dataset.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dataset.py new file mode 100644 index 000000000..7756b270b --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/api/test_dataset.py @@ -0,0 +1,12 @@ +from dl_api_client.dsmaker.primitives import Dataset +from dl_api_lib_testing.connector.dataset_suite import DefaultConnectorDatasetTestSuite + +from dl_connector_ydb_tests.db.api.base import YDBDatasetTestBase +from dl_connector_ydb_tests.db.config import TABLE_SCHEMA + + +class TestYDBDataset(YDBDatasetTestBase, DefaultConnectorDatasetTestSuite): + def check_basic_dataset(self, ds: Dataset) -> None: + assert ds.id + field_names = {field.title for field in ds.result_schema} + assert field_names == {column[0] for column in TABLE_SCHEMA} diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py new file mode 100644 index 000000000..92825577a --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/config.py @@ -0,0 +1,340 @@ +import sqlalchemy as sa + +from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration +from dl_constants.enums import UserDataType +from dl_core_testing.configuration import DefaultCoreTestConfiguration +from dl_testing.containers import get_test_container_hostport + +from dl_connector_ydb.formula.constants import YqlDialect as D + + +# Infra settings +CORE_TEST_CONFIG = DefaultCoreTestConfiguration( + host_us_http=get_test_container_hostport("us", fallback_port=51911).host, + port_us_http=get_test_container_hostport("us", fallback_port=51911).port, + host_us_pg=get_test_container_hostport("pg-us", fallback_port=51910).host, + port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=51910).port, + us_master_token="AC1ofiek8coB", + core_connector_ep_names=["ydb"], +) + +_DB_URL = f'yql:///?endpoint={get_test_container_hostport("db-ydb", fallback_port=51900).host}%3A{get_test_container_hostport("db-ydb", fallback_port=51900).port}&database=%2Flocal' +DB_CORE_URL = _DB_URL +DB_CONFIGURATIONS = { + D.YDB: _DB_URL, +} + +CONNECTION_PARAMS = dict( + host=get_test_container_hostport("db-ydb", fallback_port=51900).host, + port=get_test_container_hostport("db-ydb", fallback_port=51900).port, + db_name="/local", +) +TABLE_SCHEMA = ( + ("id", UserDataType.integer, sa.Integer), + ("some_int32", UserDataType.integer, sa.Integer), + ("some_int64", UserDataType.integer, sa.BigInteger), + ("some_uint8", UserDataType.integer, sa.SmallInteger), + ("some_bool", UserDataType.boolean, sa.Boolean), + ("some_double", UserDataType.float, sa.Float), + ("some_string", UserDataType.string, sa.String), + ("some_utf8", UserDataType.string, sa.Unicode), + ("some_date", UserDataType.date, sa.Date), + ("some_datetime", UserDataType.genericdatetime, sa.DateTime), + ("some_timestamp", UserDataType.unsupported, sa.TIMESTAMP), +) +TABLE_DATA = [ + { + "id": 1, + "some_int32": 1073741824, + "some_int64": 4611686018427387904, + "some_uint8": 254, + "some_bool": True, + "some_double": None, + "some_string": None, + "some_utf8": None, + "some_date": None, + "some_datetime": None, + "some_timestamp": None, + }, + { + "id": 2, + "some_int32": -1, + "some_int64": -2, + "some_uint8": 3, + "some_bool": False, + "some_double": None, + "some_string": None, + "some_utf8": None, + "some_date": None, + "some_datetime": None, + "some_timestamp": None, + }, + { + "id": 3, + "some_int32": None, + "some_int64": None, + "some_uint8": None, + "some_bool": None, + "some_double": 79079710.35104989, + "some_string": None, + "some_utf8": None, + "some_date": None, + "some_datetime": None, + "some_timestamp": None, + }, + { + "id": 4, + "some_int32": None, + "some_int64": None, + "some_uint8": None, + "some_bool": None, + "some_double": 7.8, + "some_string": None, + "some_utf8": None, + "some_date": None, + "some_datetime": None, + "some_timestamp": None, + }, + { + "id": 5, + "some_int32": None, + "some_int64": None, + "some_uint8": None, + "some_bool": None, + "some_double": None, + "some_string": "ff0aff", + "some_utf8": "… «C≝⋯≅M»!", + "some_date": None, + "some_datetime": None, + "some_timestamp": None, + }, + { + "id": 6, + "some_int32": None, + "some_int64": None, + "some_uint8": None, + "some_bool": None, + "some_double": None, + "some_string": None, + "some_utf8": None, + "some_date": "2021-06-07", + "some_datetime": "2021-06-07T18:19:20Z", + "some_timestamp": "2021-06-07T18:19:20Z", + }, + { + "id": 7, + "some_int32": None, + "some_int64": None, + "some_uint8": None, + "some_bool": None, + "some_double": None, + "some_string": None, + "some_utf8": None, + "some_date": "1970-12-31", + "some_datetime": "1970-12-31T23:58:57Z", + "some_timestamp": "1970-12-31T23:58:57Z", + }, +] +TABLE_NAME = "test_table_h" + +DASHSQL_QUERY = r""" +select + id, + MAX('⋯') as some_str, + MAX(CAST('⋯' AS UTF8)) as some_utf8, + MAX(111) as some_int, + MAX(CAST(111 AS UInt8)) as some_uint8, + MAX(4398046511104) as some_int64, + MAX(18446744073709551606) as some_uint64, + MAX(1.11e-11) as some_double, + MAX(true) as some_bool, + MAX(Date('2021-06-09')) as some_date, + MAX(Datetime('2021-06-09T20:50:47Z')) as some_datetime, + MAX(Timestamp('2021-07-10T21:51:48.841512Z')) as some_timestamp, + + MAX(ListHead(ListSkip(Unicode::SplitToList(CAST(some_string AS UTF8), ''), 3))) as str_split, + MAX(ListConcat(ListReplicate(CAST(' ' AS UTF8), 5))) as num_space_by_lst, + MAX(CAST(String::LeftPad('', 5, ' ') AS UTF8)) as num_space, + MAX(Unicode::ReplaceAll(CAST(some_string AS UTF8), COALESCE(CAST('f' AS UTF8), ''), COALESCE(CAST(some_string AS UTF8), ''))) as str_replace, + MAX(Unicode::Substring(CAST(some_utf8 AS UTF8), 3, 3)) as utf8_tst, + MAX(Unicode::Substring(CAST(some_string AS UTF8), Unicode::GetLength(CAST(some_string AS UTF8)) - 3)) as str_right, + MAX(Unicode::Substring(CAST(some_utf8 AS UTF8), Unicode::GetLength(CAST(some_utf8 AS UTF8)) - 3)) as utf8_right, + MAX(Unicode::Substring(CAST(some_string AS UTF8), 0, 3)) as str_left, + MAX(Unicode::Substring(CAST(some_utf8 AS UTF8), 0, 3)) as utf8_left, + MAX(Unicode::Substring(some_utf8, CAST(String::Find(some_utf8, '≝') AS UInt64))) as utf8_find_substring_wrong, + MAX(String::StartsWith(some_utf8, '…')) as utf8_startswith_const, + MAX(String::EndsWith(some_utf8, '!')) as utf8_endswith_const, + MAX(Unicode::ToLower(CAST(some_string AS UTF8))) as str_lower, + MAX(Unicode::ToUpper(CAST(some_utf8 AS UTF8))) as utf8_upper, + MAX(Unicode::ToLower(CAST(some_utf8 AS UTF8))) as utf8_lower, + MAX(CASE WHEN some_string IS NULL THEN NULL ELSE String::Contains(some_utf8, COALESCE(some_string, '')) END) as utf8_contains_nonconst, + MIN(String::Contains(some_string, 'a')) as str_contains_const, + MIN(String::Contains(some_utf8, '!')) as utf8_contains_const, + MIN(some_string LIKE '%a%' ESCAPE '!') as str_contains_like_const, + MIN(some_utf8 LIKE '%!!%' ESCAPE '!') as utf8_contains_like_const, + MIN(some_utf8 || some_utf8) as utf8_concat, + MIN(CASE WHEN some_uint8 IS NULL THEN NULL ELSE Unicode::FromCodePointList(AsList(COALESCE(CAST(some_uint8 AS SMALLINT), 0))) END) as num_char, + MIN(ListHead(Unicode::ToCodePointList(Unicode::Substring(cast(some_utf8 as utf8), 0, 1)))) as utf8_ascii, + + MIN(some_string) as str_straight, + MIN(some_utf8) as utf8_straight, + MIN(some_uint8) as uint8_straight, + MIN(Math::Tan(some_double)) as dbl_tan, + MIN(Math::Pow(some_double, 2)) as dbl_square, + MIN(Math::Sqrt(some_double)) as dbl_sqrt, + MIN(Math::Sin(some_double)) as dbl_sin, + MIN(CASE WHEN some_double < 0 THEN -1 WHEN some_double > 0 THEN 1 ELSE 0 END) as dbl_sign, + MIN(Math::Round(some_double, -2)) as dbl_round2n, + MIN(Math::Round(some_double, 2)) as dbl_round2, + MIN(Math::Round(some_double)) as dbl_round, + MIN(CAST(some_int64 / some_uint8 AS BIGINT)) as int_int_div, + MIN(LEAST(some_uint8, some_int64)) as int_least, + MIN(GREATEST(some_uint8, some_int64)) as int_greatest, + MIN(Math::Log10(some_uint8)) as int_log10, + MIN(Math::Log(some_uint8)) as int_log, + MIN(Math::Exp(some_uint8)) as int_exp, + MIN(some_uint8 / Math::Pi() * 180.0) as int_degrees, + MAX(Math::Cos(some_double)) as dbl_cos, + MAX(Math::Floor(some_double)) as dbl_floor, + MAX(Math::Ceil(some_double)) as dbl_ceil, + MAX(some_double) as dbl_straight, + MIN(Math::Atan2(some_uint8, some_int32)) as int_atan2, + MIN(Math::Atan(some_uint8)) as int_atan, + MIN(Math::Asin(some_uint8)) as int_asin, + MIN(Math::Acos(some_uint8)) as int_acos, + MIN(COALESCE(some_uint8, some_int64)) as int_coalesce, + MIN_BY(some_int64, Math::Abs(some_int64)) as int_min_by, + MAX_BY(some_int64, Math::Abs(some_int64)) as int_max_by, + MAX(some_int64 IS NULL) as int_isnull, + COUNT(DISTINCT IF(some_int64 > -9999, some_int64, NULL)) as some_countd_if, + MAX(IF(some_bool, 1, 0)) as some_if, + MAX(some_datetime not between Datetime('2011-06-07T18:19:20Z') and Datetime('2031-06-07T18:19:20Z')) as dt_notbetween, + MAX(some_datetime between Datetime('2022-06-07T18:19:20Z') and Datetime('2031-06-07T18:19:20Z')) as dt_between_f, + MAX(some_datetime between Datetime('2011-06-07T18:19:20Z') and Datetime('2031-06-07T18:19:20Z')) as dt_between, + MAX(some_double in (1.0, NULL)) as dbl_in_null, + MIN(some_double not in (1.0, 2.2, 3579079710.351049881)) as dbl_notin, + MAX(some_double in (1.0, 2.2, 3579079710.351049881)) as dbl_in_f, + -- IN may produce unexpected result when used with nullable arguments. Consider adding 'PRAGMA AnsiInForEmptyOrNullableItemsCollections;' + MAX(some_double in (1.0, 2.2, 1579079710.351049881)) as dbl_in, + MAX(some_utf8 in ('a', 'b', 'C')) as text_in_f, + MAX(some_utf8 in ('a', 'b', '… «C≝⋯≅M»!')) as text_in, + SOME(some_bool or some_int64 is not null) as some_or, + MAX(some_bool and some_int64 is null) as some_and, + MAX(some_utf8 > some_string) as str_gt, + MAX(some_utf8 <= some_string) as str_lte, + MAX(some_double >= some_double) as dbl_gte, + MAX(some_double <= some_double) as dbl_lte, + MAX(some_double < some_double) as dbl_lt, + MAX(some_double != some_double) as dbl_neq, + MAX(some_double = some_double) as dbl_eq, + MAX(some_utf8 = some_utf8) as text_eq, + MAX(some_utf8 not like 'ы%') as text_not_like, + MAX(some_utf8 like 'ы%') as text_like_false, + MAX(some_utf8 like '%') as text_like, + MAX(some_string like 'ы%') as bytes_like_false, + MAX(some_string like '%') as bytes_like, + MAX(DateTime::ToMicroseconds( + some_datetime + - (some_datetime + DateTime::IntervalFromSeconds(12345)) + ) / 86400000000.0) as datetime_datetime_sub, + MAX(DateTime::ToDays( + some_date + - (some_date + DateTime::IntervalFromSeconds(1234567)) + )) as date_date_sub, + MAX(some_datetime - DateTime::IntervalFromMicroseconds(CAST(Math::Ceil(4.4 * 86400 * 1000000) AS INTEGER))) as datetime_sub, + MAX(some_datetime + DateTime::IntervalFromMicroseconds(CAST(4.4 * 86400 * 1000000 AS INTEGER))) as datetime_add, + MAX(some_date - DateTime::IntervalFromDays(CAST(Math::Ceil(4.4) AS INTEGER))) as date_subtract, + MAX(some_date + DateTime::IntervalFromDays(CAST(4.4 AS INTEGER))) as date_add, + MAX(some_int64 - some_uint8) as int_subtract, + MAX(some_int64 + some_uint8) as int_add, + MAX(some_double % (some_double / 3.456)) as dbl_mod, + MAX(some_int64 % some_uint8) as int_mod, + MAX(CAST(some_int64 / 10000 AS DOUBLE) / some_uint8) as int_div, + MIN(some_int64 * some_double) as num_mult, + MIN(Math::Pow(some_int64, 2)) as num_pow, + MAX(some_bool = FALSE) as bool_isfalse, + MAX(some_bool = TRUE) as bool_istrue, + MIN(some_int64 != 0.0) as num_istrue, + MIN(-some_int64) as num_neg, + MIN(some_utf8 = '') as text_not, + MIN(some_string = '') as bytes_not, + MIN(some_uint8 = 0) as num_not, + MIN(NOT some_bool) as bool_not, + + MIN(DateTime::GetWeekOfYearIso8601(some_datetime)) as datetime_yearweek, + MIN(DateTime::GetDayOfWeek(some_datetime)) as datetime_weekday, -- 1 .. 7 + MIN(DateTime::GetYear(some_datetime)) as datetime_year, + MIN(CAST((DateTime::GetMonth(some_datetime) + 2) / 3 AS INTEGER)) as datetime_quarter, + MIN(DateTime::GetMonth(some_datetime)) as datetime_month, + MIN(DateTime::GetDayOfMonth(some_datetime)) as datetime_day, + MIN(DateTime::GetHour(some_datetime)) as datetime_hour, + MIN(DateTime::GetMinute(some_datetime)) as datetime_minute, + MIN(DateTime::GetSecond(some_datetime)) as datetime_second, + + MIN(DateTime::MakeDatetime(DateTime::StartOfWeek(some_datetime))) as datetime_startofweek, + MIN(DateTime::MakeDatetime(DateTime::StartOfMonth(some_datetime))) as datetime_startofmonth, + MIN(DateTime::MakeDatetime(DateTime::StartOfQuarter(some_datetime))) as datetime_startofquarter, + MIN(DateTime::MakeDatetime(DateTime::StartOfYear(some_datetime))) as datetime_startofyear, + + MIN(DateTime::MakeDate(DateTime::StartOfWeek(some_date))) as date_startofweek, + MIN(DateTime::MakeDate(DateTime::StartOfMonth(some_date))) as date_startofmonth, + MIN(DateTime::MakeDate(DateTime::StartOfQuarter(some_date))) as date_startofquarter, + MIN(DateTime::MakeDate(DateTime::StartOfYear(some_date))) as date_startofyear, + + MIN(DateTime::MakeDate(DateTime::ShiftYears(some_date, coalesce(some_uint8, 0)))) as date_shiftyears, + MIN('[' || CAST(some_double AS UTF8) || ',' || CAST(37.622504 AS UTF8) || ']') as tst_geopoint, + MIN(CAST(CAST('2008e1c9-44a6-4fac-a61d-e42675b77309' AS UUID) AS UTF8)) as text_from_uuid, + -- SOME(CAST('2008e1c9-44a6-4fac-a61d-e42675b77309' AS UUID)) as some_uuid, + MIN(DateTime::Format('%Y-%m-%d %H:%M:%S')(some_datetime)) as text_from_datetime_proper, + MIN(CAST(some_datetime AS UTF8)) as text_from_datetime, + MIN(CAST(some_date AS UTF8)) as text_from_date, + MIN(CASE WHEN true IS NULL THEN NULL WHEN true = true THEN 'True' ELSE 'False' END) as text_from_bool, + MIN(CAST(ToBytes(Yson::SerializePretty(Yson::From(some_string))) AS UTF8)) as text_from_stuff, + MIN(CASE WHEN some_datetime IS NULL THEN NULL ELSE true END) as bool_from_datetime, + MIN(CAST(true as BIGINT)) as int_from_bool, + MIN(CAST(some_double AS UTF8)) as text_from_double, -- XXXXXXXXXX: 1579079710.35105 -> "1579079710" + DateTime::MakeDatetime(DateTime::ParseIso8601('2021-06-01 18:00:59')) as datetime_from_str, + MIN(CAST(NULL AS BOOL)) as null_boolean, + MIN(CAST(NULL AS BIGINT)) as null_bigint, + MIN(CAST(NULL AS DATETIME)) as null_datetime, + MIN(CAST(NULL AS DATE)) as null_date, + MIN(Math::Abs(-1 * some_double)) as dbl_abs, + String::JoinFromList(ListMap(TOPFREQ(some_datetime, 5), ($x) -> {{ RETURN cast($x.Value as Utf8); }}), ', ') as top_concat, + String::JoinFromList(ListSortAsc(AGGREGATE_LIST_DISTINCT(cast(some_date as Utf8))), ', ') as date_all_concat, + SOME(some_string) as bytes_some, + MEDIAN(some_int64) as int_median, + PERCENTILE(some_int64, 0.8) as int_percentile, + VARPOP(some_int64) as int_varpop, + VARSAMP(some_int64) as int_varsamp, + STDDEVPOP(some_int64) as int_stddevpop, + STDDEVSAMP(some_int64) as int_stddevsamp, + CountDistinctEstimate(some_double) as dbl_count_distinct_approx, + COUNT(DISTINCT some_double) as dbl_count_distinct, + COUNT_IF(some_double < 0) as dbl_count_if_empty, + COUNT_IF(some_double > 0) as dbl_count_if, + COUNT(1) as cnt, + MIN(some_date) as date_min, + MAX(some_datetime) as datetime_max, + CAST(CAST(AVG(CAST(some_datetime as DOUBLE)) AS BIGINT) AS DATETIME) as datetime_avg, + -- date -> dt -> bigint -> avg (-> double) -> bigint -> datetime -> date + CAST(CAST(CAST( + AVG( + CAST(CAST(some_date AS DATETIME) AS BIGINT) + ) + AS BIGINT) AS DATETIME) AS DATE) as date_avg, + AVG_IF(some_int64, some_int64 > -1) as int_avg_if, + AVG(some_int64) as int_avg, + SUM_IF(some_int64, some_int64 > 10) as int_sum_if, + SUM(some_int64) as int_sum, + +from `{table_name}` +group by id +order by id +limit 1000 +""" + +API_TEST_CONFIG = ApiTestEnvironmentConfiguration( + api_connector_ep_names=["ydb"], + core_test_config=CORE_TEST_CONFIG, + ext_query_executer_secret_key="_some_test_secret_key_", +) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/conftest.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/conftest.py new file mode 100644 index 000000000..787557269 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/conftest.py @@ -0,0 +1,17 @@ +from dl_api_lib_testing.initialization import initialize_api_lib_test +from dl_formula_testing.forced_literal import forced_literal_use + +from dl_connector_ydb_tests.db.config import API_TEST_CONFIG + + +pytest_plugins = ("aiohttp.pytest_plugin",) # and it, in turn, includes 'pytest_asyncio.plugin' + + +def pytest_configure(config): # noqa + initialize_api_lib_test(pytest_config=config, api_test_config=API_TEST_CONFIG) + + +__all__ = ( + # auto-use fixtures: + "forced_literal_use", +) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/base.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/base.py new file mode 100644 index 000000000..f96e81739 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/base.py @@ -0,0 +1,41 @@ +import pytest +import sqlalchemy as sa + +from dl_formula_testing.database import ( + Db, + FormulaDbDispenser, +) +from dl_formula_testing.testcases.base import FormulaConnectorTestBase + +from dl_connector_ydb.formula.constants import YqlDialect as D +from dl_connector_ydb_tests.db.config import DB_CONFIGURATIONS + + +class YqlDbDispenser(FormulaDbDispenser): + def ensure_db_is_up(self, db: Db) -> tuple[bool, str]: + # first, check that the db is up + status, msg = super().ensure_db_is_up(db) + if not status: + return status, msg + + test_table = db.table_from_columns([sa.Column(name="col1", type_=sa.Integer())]) + + # secondly, try to create a test table: for some reason + # it could be that YDB is up but you still can't do it + try: + db.create_table(test_table) + db.drop_table(test_table) + return True, "" + except Exception as exc: + return False, str(exc) + + +class YQLTestBase(FormulaConnectorTestBase): + dialect = D.YDB + supports_arrays = False + supports_uuid = True + db_dispenser = YqlDbDispenser() + + @pytest.fixture(scope="class") + def db_url(self) -> str: + return DB_CONFIGURATIONS[self.dialect] diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_conditional_blocks.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_conditional_blocks.py new file mode 100644 index 000000000..226f11ef1 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_conditional_blocks.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.conditional_blocks import DefaultConditionalBlockFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestConditionalBlockYQL(YQLTestBase, DefaultConditionalBlockFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_aggregation.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_aggregation.py new file mode 100644 index 000000000..7a77489e2 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_aggregation.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_aggregation import DefaultMainAggFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestMainAggFunctionYQL(YQLTestBase, DefaultMainAggFunctionFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_datetime.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_datetime.py new file mode 100644 index 000000000..c820bc77f --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_datetime.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_datetime import DefaultDateTimeFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestDateTimeFunctionYQL(YQLTestBase, DefaultDateTimeFunctionFormulaConnectorTestSuite): + supports_datepart_2_non_const = False diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_logical.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_logical.py new file mode 100644 index 000000000..3d7c14727 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_logical.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_logical import DefaultLogicalFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestLogicalFunctionYQL(YQLTestBase, DefaultLogicalFunctionFormulaConnectorTestSuite): + supports_iif = True diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_markup.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_markup.py new file mode 100644 index 000000000..9d7708a29 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_markup.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_markup import DefaultMarkupFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestMarkupFunctionYQL(YQLTestBase, DefaultMarkupFunctionFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_math.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_math.py new file mode 100644 index 000000000..799e57200 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_math.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.functions_math import DefaultMathFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestMathFunctionYQL(YQLTestBase, DefaultMathFunctionFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_string.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_string.py new file mode 100644 index 000000000..ceee2622e --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_string.py @@ -0,0 +1,13 @@ +from dl_formula_testing.testcases.functions_string import DefaultStringFunctionFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestStringFunctionYQL(YQLTestBase, DefaultStringFunctionFormulaConnectorTestSuite): + datetime_str_separator = "T" + datetime_str_ending = "Z" + supports_trimming_funcs = False + supports_regex_extract = False + supports_regex_extract_nth = False + supports_regex_replace = False + supports_regex_match = False diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py new file mode 100644 index 000000000..7b7360f5e --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_functions_type_conversion.py @@ -0,0 +1,77 @@ +from dl_formula_testing.evaluator import DbEvaluator +from dl_formula_testing.testcases.functions_type_conversion import ( + DefaultBoolTypeFunctionFormulaConnectorTestSuite, + DefaultDateTypeFunctionFormulaConnectorTestSuite, + DefaultFloatTypeFunctionFormulaConnectorTestSuite, + DefaultGenericDatetimeTypeFunctionFormulaConnectorTestSuite, + DefaultGeopointTypeFunctionFormulaConnectorTestSuite, + DefaultGeopolygonTypeFunctionFormulaConnectorTestSuite, + DefaultIntTypeFunctionFormulaConnectorTestSuite, + DefaultStrTypeFunctionFormulaConnectorTestSuite, +) +from dl_formula_testing.util import to_str + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +# STR + + +class TestStrTypeFunctionYQL(YQLTestBase, DefaultStrTypeFunctionFormulaConnectorTestSuite): + zero_float_to_str_value = "0" + skip_custom_tz = True + + def test_str_from_datetime(self, dbe: DbEvaluator) -> None: + assert to_str(dbe.eval("STR(#2019-01-02 03:04:05#)")) == "2019-01-02T03:04:05Z" + + +# FLOAT + + +class TestFloatTypeFunctionYQL(YQLTestBase, DefaultFloatTypeFunctionFormulaConnectorTestSuite): + pass + + +# BOOL + + +class TestBoolTypeFunctionYQL(YQLTestBase, DefaultBoolTypeFunctionFormulaConnectorTestSuite): + pass + + +# INT + + +class TestIntTypeFunctionYQL(YQLTestBase, DefaultIntTypeFunctionFormulaConnectorTestSuite): + pass + + +# DATE + + +class TestDateTypeFunctionYQL(YQLTestBase, DefaultDateTypeFunctionFormulaConnectorTestSuite): + pass + + +# GENERICDATETIME (& DATETIME) + + +class TestGenericDatetimeTypeFunctionYQL( + YQLTestBase, + DefaultGenericDatetimeTypeFunctionFormulaConnectorTestSuite, +): + pass + + +# GEOPOINT + + +class TestGeopointTypeFunctionYQL(YQLTestBase, DefaultGeopointTypeFunctionFormulaConnectorTestSuite): + pass + + +# GEOPOLYGON + + +class TestGeopolygonTypeFunctionYQL(YQLTestBase, DefaultGeopolygonTypeFunctionFormulaConnectorTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_literals.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_literals.py new file mode 100644 index 000000000..e0af98271 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_literals.py @@ -0,0 +1,9 @@ +from dl_formula_testing.testcases.literals import DefaultLiteralFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestConditionalBlockYQL(YQLTestBase, DefaultLiteralFormulaConnectorTestSuite): + supports_microseconds = False + supports_utc = False + supports_custom_tz = False diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_misc_funcs.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_misc_funcs.py new file mode 100644 index 000000000..9ec771371 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_misc_funcs.py @@ -0,0 +1,7 @@ +from dl_formula_testing.testcases.misc_funcs import DefaultMiscFunctionalityConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestMiscFunctionalityYQL(YQLTestBase, DefaultMiscFunctionalityConnectorTestSuite): + pass diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_operators.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_operators.py new file mode 100644 index 000000000..3410b710b --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/db/formula/test_operators.py @@ -0,0 +1,23 @@ +import sqlalchemy as sa + +from dl_formula_testing.evaluator import DbEvaluator +from dl_formula_testing.testcases.operators import DefaultOperatorFormulaConnectorTestSuite + +from dl_connector_ydb_tests.db.formula.base import YQLTestBase + + +class TestOperatorYQL(YQLTestBase, DefaultOperatorFormulaConnectorTestSuite): + subtraction_round_dt = False + supports_string_int_multiplication = False + + def test_subtraction_unsigned_ints(self, dbe: DbEvaluator) -> None: + assert dbe.eval("SECOND(#2019-01-23 15:07:47#) - SECOND(#2019-01-23 15:07:48#)") == -1 + + def test_in_date(self, dbe: DbEvaluator, data_table: sa.Table) -> None: + # YDB doesn't allow ordering by columns not from SELECT clause, so use WHERE instead + assert dbe.eval( + "[date_value] in (#2014-10-05#)", where="IF([date_value] = #2014-10-05#, TRUE, FALSE)", from_=data_table + ) + assert dbe.eval( + "[date_value] not in (#2014-10-05#)", where="IF([date_value] = #2014-10-06#, TRUE, FALSE)", from_=data_table + ) diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/__init__.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/conftest.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/conftest.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/test_connection_form.py b/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/test_connection_form.py new file mode 100644 index 000000000..df0055f19 --- /dev/null +++ b/lib/dl_connector_ydb/dl_connector_ydb_tests/unit/test_connection_form.py @@ -0,0 +1,20 @@ +from typing import Optional + +import pytest + +from dl_api_connector.i18n.localizer import CONFIGS as DL_API_CONNECTOR_CONFIGS +from dl_api_lib_testing.connection_form_base import ConnectionFormTestBase +from dl_configs.connectors_settings import ConnectorSettingsBase + +from dl_connector_ydb.api.ydb.connection_form.form_config import YDBConnectionFormFactory +from dl_connector_ydb.api.ydb.i18n.localizer import CONFIGS as DL_CONNECTOR_YDB_CONFIGS +from dl_connector_ydb.core.ydb.settings import YDBConnectorSettings + + +class TestYDBConnectionForm(ConnectionFormTestBase): + CONN_FORM_FACTORY_CLS = YDBConnectionFormFactory + TRANSLATION_CONFIGS = DL_API_CONNECTOR_CONFIGS + DL_CONNECTOR_YDB_CONFIGS + + @pytest.fixture + def connectors_settings(self) -> Optional[ConnectorSettingsBase]: + return YDBConnectorSettings() diff --git a/lib/dl_connector_ydb/docker-compose.yml b/lib/dl_connector_ydb/docker-compose.yml new file mode 100644 index 000000000..7535877f3 --- /dev/null +++ b/lib/dl_connector_ydb/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3.7' + +x-constants: + US_MASTER_TOKEN: &c-us-master-token "AC1ofiek8coB" + +services: + db-ydb: + image: "cr.yandex/yc/yandex-docker-local-ydb:latest" + environment: + YDB_LOCAL_SURVIVE_RESTART: "true" + GRPC_PORT: "51900" + # hostname: "localhost" # you might want to uncomment this for local testing + ports: + - "51900:51900" + + # INFRA + pg-us: + build: + context: ../testenv-common/images + dockerfile: Dockerfile.pg-us + environment: + POSTGRES_DB: us-db-ci_purgeable + POSTGRES_USER: us + POSTGRES_PASSWORD: us + ports: + - "51910:5432" + + us: + build: + context: ../testenv-common/images + dockerfile: Dockerfile.us + depends_on: + - pg-us + environment: + POSTGRES_DSN_LIST: "postgres://us:us@pg-us:5432/us-db-ci_purgeable" + AUTH_POLICY: "required" + MASTER_TOKEN: *c-us-master-token + ports: + - "51911:80" diff --git a/lib/dl_connector_ydb/pyproject.toml b/lib/dl_connector_ydb/pyproject.toml new file mode 100644 index 000000000..b1bff4720 --- /dev/null +++ b/lib/dl_connector_ydb/pyproject.toml @@ -0,0 +1,86 @@ + +[tool.poetry] +name = "datalens-connector-ydb" +version = "0.0.1" +description = "" +authors = ["DataLens Team "] +packages = [{include = "dl_connector_ydb"}] +license = "Apache 2.0" +readme = "README.md" + + +[tool.poetry.dependencies] +attrs = ">=22.2.0" +grpcio = ">=1.45.0rc1" +marshmallow = ">=3.19.0" +python = ">=3.10, <3.12" +sqlalchemy = ">=1.4.46, <2.0" +ydb = ">=3.5.1" +datalens-api-commons = {path = "../dl_api_commons"} +datalens-api-connector = {path = "../dl_api_connector"} +datalens-configs = {path = "../dl_configs"} +datalens-constants = {path = "../dl_constants"} +datalens-core = {path = "../dl_core"} +datalens-core-testing = {path = "../dl_core_testing"} +datalens-db-testing = {path = "../dl_db_testing"} +datalens-formula = {path = "../dl_formula"} +datalens-formula-ref = {path = "../dl_formula_ref"} +datalens-i18n = {path = "../dl_i18n"} + +[tool.poetry.plugins] +[tool.poetry.plugins."dl_api_lib.connectors"] +ydb = "dl_connector_ydb.api.ydb.connector:YDBApiConnector" + +[tool.poetry.plugins."dl_core.connectors"] +ydb = "dl_connector_ydb.core.ydb.connector:YDBCoreConnector" + +[tool.poetry.plugins."dl_db_testing.connectors"] +yql = "dl_connector_ydb.db_testing.connector:YQLDbTestingConnector" + +[tool.poetry.plugins."dl_formula.connectors"] +yql = "dl_connector_ydb.formula.connector:YQLFormulaConnector" + +[tool.poetry.plugins."dl_formula_ref.plugins"] +yql = "dl_connector_ydb.formula_ref.plugin:YQLFormulaRefPlugin" + +[tool.poetry.group.tests.dependencies] +pytest = ">=7.2.2" +datalens-formula-testing = {path = "../dl_formula_testing"} + +[build-system] +build-backend = "poetry.core.masonry.api" +requires = [ + "poetry-core", +] + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra" +testpaths = [] + + + +[datalens.pytest.db] +root_dir = "dl_connector_ydb_tests/" +target_path = "db" +compose_file_base = "docker-compose" + +[datalens.pytest.unit] +root_dir = "dl_connector_ydb_tests/" +target_path = "unit" +skip_compose = "true" + +[tool.mypy] +warn_unused_configs = true +disallow_untyped_defs = true +check_untyped_defs = true +strict_optional = true + +[datalens.i18n.domains] +dl_connector_ydb = [ + {path = "dl_connector_ydb/api"}, + {path = "dl_connector_ydb/core"}, +] +dl_formula_ref_dl_connector_ydb = [ + {path = "dl_connector_ydb/formula_ref"}, +] diff --git a/metapkg/poetry.lock b/metapkg/poetry.lock index b86b70784..3a7547e6b 100644 --- a/metapkg/poetry.lock +++ b/metapkg/poetry.lock @@ -1636,6 +1636,36 @@ sqlalchemy = ">=1.4.46, <2.0" type = "directory" url = "../lib/dl_connector_snowflake" +[[package]] +name = "datalens-connector-ydb" +version = "0.0.1" +description = "" +optional = false +python-versions = ">=3.10, <3.12" +files = [] +develop = false + +[package.dependencies] +attrs = ">=22.2.0" +datalens-api-commons = {path = "../dl_api_commons"} +datalens-api-connector = {path = "../dl_api_connector"} +datalens-configs = {path = "../dl_configs"} +datalens-constants = {path = "../dl_constants"} +datalens-core = {path = "../dl_core"} +datalens-core-testing = {path = "../dl_core_testing"} +datalens-db-testing = {path = "../dl_db_testing"} +datalens-formula = {path = "../dl_formula"} +datalens-formula-ref = {path = "../dl_formula_ref"} +datalens-i18n = {path = "../dl_i18n"} +grpcio = ">=1.45.0rc1" +marshmallow = ">=3.19.0" +sqlalchemy = ">=1.4.46, <2.0" +ydb = ">=3.5.1" + +[package.source] +type = "directory" +url = "../lib/dl_connector_ydb" + [[package]] name = "datalens-constants" version = "0.0.1" @@ -6344,6 +6374,26 @@ files = [ idna = ">=2.0" multidict = ">=4.0" +[[package]] +name = "ydb" +version = "3.5.2" +description = "YDB Python SDK" +optional = false +python-versions = "*" +files = [ + {file = "ydb-3.5.2-py2.py3-none-any.whl", hash = "sha256:0ef3ae929c8267e18ce56a6a8979b3712a953fe2254e12909aa422b89637902a"}, + {file = "ydb-3.5.2.tar.gz", hash = "sha256:7e698843468a81976e5dd8b3d4324e02bf41a763a18eab2648d4401c95481c67"}, +] + +[package.dependencies] +aiohttp = "<4" +grpcio = ">=1.42.0" +packaging = "*" +protobuf = ">=3.13.0,<5.0.0" + +[package.extras] +yc = ["yandexcloud"] + [[package]] name = "zipp" version = "3.17.0" @@ -6362,4 +6412,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10, <3.12" -content-hash = "3f5b4176287c0aa8efe9c53345031cba496ae1f93dd6d93c2ce5aef496b5240f" +content-hash = "2dc00c341f5ceda1eff986f55a7448dc6f236ff4ec02efc16a1be9991351fafd" diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index b3f92b982..cfd68cba8 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -148,6 +148,7 @@ datalens-maintenance = {path = "../lib/dl_maintenance"} datalens-connector-mssql = {path = "../lib/dl_connector_mssql"} datalens-attrs-model-mapper = {path = "../lib/dl_attrs_model_mapper"} datalens-attrs-model-mapper-doc-tools = {path = "../lib/dl_attrs_model_mapper_doc_tools"} +datalens-connector-ydb = {path = "../lib/dl_connector_ydb"} [tool.poetry.group.dev.dependencies] black = "==23.3.0" From f78b590799b8d27505b0c164fd6ce5486677a481 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Tue, 21 Nov 2023 17:38:49 +0100 Subject: [PATCH 18/27] Fixed locales in ydb (#114) --- ...or_ydb.mo => dl_formula_ref_dl_connector_ydb.mo} | Bin ...or_ydb.po => dl_formula_ref_dl_connector_ydb.po} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/{dl_formula_ref_bi_connector_ydb.mo => dl_formula_ref_dl_connector_ydb.mo} (100%) rename lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/{dl_formula_ref_bi_connector_ydb.po => dl_formula_ref_dl_connector_ydb.po} (100%) diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.mo b/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.mo similarity index 100% rename from lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.mo rename to lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.mo diff --git a/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.po b/lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.po similarity index 100% rename from lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_bi_connector_ydb.po rename to lib/dl_connector_ydb/dl_connector_ydb/locales/ru/LC_MESSAGES/dl_formula_ref_dl_connector_ydb.po From 94854b80de81eff043384df65787267bd62127de Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Tue, 21 Nov 2023 18:27:04 +0100 Subject: [PATCH 19/27] Add pagination tests (#109) --- .../unit/test_pagination.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 lib/dl_query_processing/dl_query_processing_tests/unit/test_pagination.py diff --git a/lib/dl_query_processing/dl_query_processing_tests/unit/test_pagination.py b/lib/dl_query_processing/dl_query_processing_tests/unit/test_pagination.py new file mode 100644 index 000000000..6867826be --- /dev/null +++ b/lib/dl_query_processing/dl_query_processing_tests/unit/test_pagination.py @@ -0,0 +1,149 @@ +from typing import Optional + +from dl_constants.enums import ( + FieldRole, + FieldType, + UserDataType, +) +from dl_query_processing.legend.block_legend import ( + BlockLegend, + BlockLegendMeta, + BlockSpec, +) +from dl_query_processing.legend.field_legend import ( + FieldObjSpec, + Legend, + LegendItem, + RowRoleSpec, +) +from dl_query_processing.merging.primitives import ( + MergedQueryDataRow, + MergedQueryDataStream, + MergedQueryMetaInfo, +) +from dl_query_processing.pagination.paginator import QueryPaginator + + +def _make_legend() -> Legend: + legend = Legend( + items=[ + LegendItem( + legend_item_id=0, + obj=FieldObjSpec(id="guid", title="title"), + role_spec=RowRoleSpec(role=FieldRole.row), + field_type=FieldType.DIMENSION, + data_type=UserDataType.unsupported, + ), + ] + ) + return legend + + +def _make_block_legend( + block_limit: Optional[int] = None, + block_offset: Optional[int] = None, + glob_limit: Optional[int] = None, + glob_offset: Optional[int] = None, +) -> BlockLegend: + legend = _make_legend() + block_legend = BlockLegend( + blocks=[ + BlockSpec( + block_id=0, + parent_block_id=0, + legend=legend, + limit=block_limit, + offset=block_offset, + ), + ], + meta=BlockLegendMeta( + limit=glob_limit, + offset=glob_offset, + ), + ) + return block_legend + + +def test_pre_paginate_single_block(): + paginator = QueryPaginator() + pre_paginator = paginator.get_pre_paginator() + + block_legend = _make_block_legend(glob_limit=None, glob_offset=None, block_limit=None, block_offset=None) + + # No pagination + block_legend = pre_paginator.pre_paginate(block_legend) + assert block_legend.blocks[0].limit is None + assert block_legend.blocks[0].offset is None + assert block_legend.meta.limit is None + assert block_legend.meta.offset is None + + # block limit + block_legend = _make_block_legend(glob_limit=None, glob_offset=None, block_limit=7, block_offset=None) + block_legend = pre_paginator.pre_paginate(block_legend) + assert block_legend.blocks[0].limit == 7 + assert block_legend.blocks[0].offset is None + assert block_legend.meta.limit is None + assert block_legend.meta.offset is None + + # global limit (pagination is moved from global to block level) + block_legend = _make_block_legend(glob_limit=7, glob_offset=None, block_limit=None, block_offset=None) + block_legend = pre_paginator.pre_paginate(block_legend) + assert block_legend.blocks[0].limit == 7 + assert block_legend.blocks[0].offset is None + assert block_legend.meta.limit is None + assert block_legend.meta.offset is None + + # global and block pagination (nothing happens) + block_legend = _make_block_legend(glob_limit=7, glob_offset=None, block_limit=None, block_offset=3) + block_legend = pre_paginator.pre_paginate(block_legend) + assert block_legend.blocks[0].limit is None + assert block_legend.blocks[0].offset == 3 + assert block_legend.meta.limit == 7 + assert block_legend.meta.offset is None + + +def test_post_paginate_single_block(): + paginator = QueryPaginator() + post_paginator = paginator.get_post_paginator() + + legend = _make_legend() + legend_item_ids = [item.legend_item_id for item in legend.items] + + rows = [MergedQueryDataRow(data=(), legend_item_ids=())] * 10 + assert len(rows) == 10 + + stream = MergedQueryDataStream( + rows=rows, + legend=legend, + legend_item_ids=legend_item_ids, + meta=MergedQueryMetaInfo(blocks=[], limit=None, offset=None), + ) + stream = post_paginator.post_paginate(stream) + assert len(list(stream.rows)) == 10 + + stream = MergedQueryDataStream( + rows=rows, + legend=legend, + legend_item_ids=legend_item_ids, + meta=MergedQueryMetaInfo(blocks=[], limit=7, offset=None), + ) + stream = post_paginator.post_paginate(stream) + assert len(list(stream.rows)) == 7 + + stream = MergedQueryDataStream( + rows=rows, + legend=legend, + legend_item_ids=legend_item_ids, + meta=MergedQueryMetaInfo(blocks=[], limit=None, offset=2), + ) + stream = post_paginator.post_paginate(stream) + assert len(list(stream.rows)) == 8 + + stream = MergedQueryDataStream( + rows=rows, + legend=legend, + legend_item_ids=legend_item_ids, + meta=MergedQueryMetaInfo(blocks=[], limit=5, offset=2), + ) + stream = post_paginator.post_paginate(stream) + assert len(list(stream.rows)) == 5 From b1cbb4bb156ed1ecf6b62a3a9fa18d2a03d915f7 Mon Sep 17 00:00:00 2001 From: Konstantin Chupin <91148200+ya-kc@users.noreply.github.com> Date: Tue, 21 Nov 2023 19:52:40 +0100 Subject: [PATCH 20/27] [DLBACK-49] Actualize signature for EnvParamGetter.get_xxx_value() (#115) --- .../dl_connector_bigquery/testing/secrets.py | 4 +- .../core/testing/secrets.py | 8 +-- .../dl_testing/env_params/generic.py | 4 +- .../dl_testing/env_params/getter.py | 53 ++++++++++++++++--- .../dl_testing/env_params/loader.py | 3 +- lib/dl_testing/dl_testing/env_params/main.py | 6 ++- lib/dl_testing/dl_testing/regulated_test.py | 4 +- 7 files changed, 65 insertions(+), 17 deletions(-) diff --git a/lib/dl_connector_bigquery/dl_connector_bigquery/testing/secrets.py b/lib/dl_connector_bigquery/dl_connector_bigquery/testing/secrets.py index a17ae1fb2..e14c27b5d 100644 --- a/lib/dl_connector_bigquery/dl_connector_bigquery/testing/secrets.py +++ b/lib/dl_connector_bigquery/dl_connector_bigquery/testing/secrets.py @@ -36,11 +36,11 @@ class BigQuerySecretReader(BigQuerySecretReaderBase): @_project_config.default def _make_project_config(self) -> dict: - return self._env_param_getter.get_json_value(self.KEY_CONFIG) + return self._env_param_getter.get_json_value_strict(self.KEY_CONFIG) @property def project_config(self) -> dict: return self._project_config def get_creds(self) -> str: - return self._env_param_getter.get_str_value(self.KEY_CREDS) + return self._env_param_getter.get_str_value_strict(self.KEY_CREDS) diff --git a/lib/dl_connector_snowflake/dl_connector_snowflake/core/testing/secrets.py b/lib/dl_connector_snowflake/dl_connector_snowflake/core/testing/secrets.py index c35b87b50..4de19ca6b 100644 --- a/lib/dl_connector_snowflake/dl_connector_snowflake/core/testing/secrets.py +++ b/lib/dl_connector_snowflake/dl_connector_snowflake/core/testing/secrets.py @@ -64,17 +64,17 @@ class SnowFlakeSecretReader(SnowFlakeSecretReaderBase): @_project_config.default def _make_project_config(self) -> dict: - return self._env_param_getter.get_json_value(self.KEY_CONFIG) + return self._env_param_getter.get_json_value_strict(self.KEY_CONFIG) @property def project_config(self) -> dict: return self._project_config def get_client_secret(self) -> str: - return self._env_param_getter.get_str_value(self.KEY_CLIENT_SECRET) + return self._env_param_getter.get_str_value_strict(self.KEY_CLIENT_SECRET) def get_refresh_token_expired(self) -> str: - return self._env_param_getter.get_str_value(self.KEY_REFRESH_TOKEN_EXPIRED) + return self._env_param_getter.get_str_value_strict(self.KEY_REFRESH_TOKEN_EXPIRED) def get_refresh_token_x(self) -> str: - return self._env_param_getter.get_str_value(self.KEY_REFRESH_TOKEN_X) + return self._env_param_getter.get_str_value_strict(self.KEY_REFRESH_TOKEN_X) diff --git a/lib/dl_testing/dl_testing/env_params/generic.py b/lib/dl_testing/dl_testing/env_params/generic.py index 306555893..16e9c14e1 100644 --- a/lib/dl_testing/dl_testing/env_params/generic.py +++ b/lib/dl_testing/dl_testing/env_params/generic.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Optional + import attr import yaml @@ -12,7 +14,7 @@ class GenericEnvParamGetter(EnvParamGetter): _loader: EnvParamGetterLoader = attr.ib(init=False, factory=EnvParamGetterLoader) _key_mapping: dict[str, tuple[str, str]] = attr.ib(init=False, factory=dict) # key -> (getter_name, remapped_key) - def get_str_value(self, key: str) -> str: + def get_str_value(self, key: str) -> Optional[str]: getter_name, remapped_key = self._key_mapping[key] getter = self._loader.get_getter(getter_name) value = getter.get_str_value(remapped_key) diff --git a/lib/dl_testing/dl_testing/env_params/getter.py b/lib/dl_testing/dl_testing/env_params/getter.py index f97923050..a4d859ef3 100644 --- a/lib/dl_testing/dl_testing/env_params/getter.py +++ b/lib/dl_testing/dl_testing/env_params/getter.py @@ -2,26 +2,65 @@ import abc import json +from typing import ( + NoReturn, + Optional, +) import yaml +def _raise_error_no_key(key: str) -> NoReturn: + raise ValueError(f"Key {key!r} is missing") + + class EnvParamGetter(abc.ABC): @abc.abstractmethod - def get_str_value(self, key: str) -> str: + def get_str_value(self, key: str) -> Optional[str]: raise NotImplementedError - def get_int_value(self, key: str) -> int: + def get_str_value_strict(self, key: str) -> str: + str_value = self.get_str_value(key) + if str_value is None: + _raise_error_no_key(key) + return str_value + + def get_int_value(self, key: str) -> Optional[int]: str_value = self.get_str_value(key) - return int(str_value) + if str_value is not None: + return int(str_value) + return None - def get_json_value(self, key: str) -> dict: + def get_int_value_strict(self, key: str) -> int: + int_value = self.get_int_value(key) + if int_value is None: + _raise_error_no_key(key) + return int_value + + def get_json_value(self, key: str) -> Optional[dict]: str_value = self.get_str_value(key) - return json.loads(str_value) + if str_value is not None: + return json.loads(str_value) + return None + + def get_json_value_strict(self, key: str) -> dict: + json_value = self.get_json_value(key) + if json_value is None: + _raise_error_no_key(key) + return json_value - def get_yaml_value(self, key: str) -> dict: + def get_yaml_value(self, key: str) -> Optional[dict]: str_value = self.get_str_value(key) - return yaml.safe_load(str_value) + if str_value is not None: + return yaml.safe_load(str_value) + return None + def get_yaml_value_strict(self, key: str) -> dict: + yaml_value = self.get_yaml_value(key) + if yaml_value is None: + _raise_error_no_key(key) + return yaml_value + + @abc.abstractmethod def initialize(self, config: dict) -> None: pass diff --git a/lib/dl_testing/dl_testing/env_params/loader.py b/lib/dl_testing/dl_testing/env_params/loader.py index 21e1e02bb..83224cb31 100644 --- a/lib/dl_testing/dl_testing/env_params/loader.py +++ b/lib/dl_testing/dl_testing/env_params/loader.py @@ -1,6 +1,7 @@ from typing import ( ClassVar, Mapping, + Optional, Sequence, ) @@ -40,7 +41,7 @@ def _auto_add_getter(self, name: str) -> None: getter.initialize(config={}) self._getters[name] = getter - def _resolve_setting_item(self, setting: dict, requirement_getter: EnvParamGetter) -> str: + def _resolve_setting_item(self, setting: dict, requirement_getter: EnvParamGetter) -> Optional[str]: if setting["type"] == "value": return setting["value"] if setting["type"] == "param": diff --git a/lib/dl_testing/dl_testing/env_params/main.py b/lib/dl_testing/dl_testing/env_params/main.py index 7d02f2acf..981fc20c7 100644 --- a/lib/dl_testing/dl_testing/env_params/main.py +++ b/lib/dl_testing/dl_testing/env_params/main.py @@ -1,4 +1,5 @@ import os +from typing import Optional import attr from dotenv import ( @@ -15,6 +16,9 @@ class DirectEnvParamGetter(EnvParamGetter): def get_str_value(self, key: str) -> str: return str(key) + def initialize(self, config: dict) -> None: + pass + @attr.s class OsEnvParamGetter(EnvParamGetter): @@ -26,7 +30,7 @@ def initialize(self, config: dict) -> None: env_file = os.environ.get("DL_TESTS_ENV_FILE") or find_dotenv(filename=".env") self._env_from_file = dotenv_values(env_file) - def get_str_value(self, key: str) -> str: + def get_str_value(self, key: str) -> Optional[str]: env_value = os.environ.get(key) if env_value is None: diff --git a/lib/dl_testing/dl_testing/regulated_test.py b/lib/dl_testing/dl_testing/regulated_test.py index 8bece74d4..081e99d78 100644 --- a/lib/dl_testing/dl_testing/regulated_test.py +++ b/lib/dl_testing/dl_testing/regulated_test.py @@ -193,7 +193,9 @@ def regulated_test_case(test_cls: type, /) -> type: @overload -def regulated_test_case(*, test_params: RegulatedTestParams = RegulatedTestParams()) -> Callable[[type], type]: +def regulated_test_case( + *, test_params: RegulatedTestParams = RegulatedTestParams() # noqa B008 +) -> Callable[[type], type]: ... From 9b3fab953c6b630418aac7aa76c64075e241d8cf Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Tue, 21 Nov 2023 20:26:07 +0100 Subject: [PATCH 21/27] Fixed window function examples (#113) --- .../dl_formula_ref/functions/window.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/dl_formula_ref/dl_formula_ref/functions/window.py b/lib/dl_formula_ref/dl_formula_ref/functions/window.py index ebdeecfbf..1792516cd 100644 --- a/lib/dl_formula_ref/dl_formula_ref/functions/window.py +++ b/lib/dl_formula_ref/dl_formula_ref/functions/window.py @@ -1,6 +1,7 @@ from typing import List from dl_formula.core.datatype import DataType +from dl_formula.inspect.function import supports_ordering from dl_formula_ref.categories.window import CATEGORY_WINDOW from dl_formula_ref.examples.config import ( ExampleConfig, @@ -52,6 +53,7 @@ def _make_standard_window_examples(func: str) -> List[DataExample]: + order_by_str = " ORDER BY [City], [Category]" if supports_ordering(name=func.lower(), is_window=True) else "" func = func.upper() examples = [ DataExample( @@ -67,9 +69,9 @@ def _make_standard_window_examples(func: str) -> List[DataExample]: ("City", "[City]"), ("Category", "[Category]"), ("Order Sum", "[Order Sum]"), - (f"{func} 1", f"{func}([Order Sum] TOTAL)"), - (f"{func} 2", f"{func}([Order Sum] WITHIN [City])"), - (f"{func} 3", f"{func}([Order Sum] WITHIN [Category])"), + (f"{func} 1", f"{func}([Order Sum] TOTAL{order_by_str})"), + (f"{func} 2", f"{func}([Order Sum] WITHIN [City]{order_by_str})"), + (f"{func} 3", f"{func}([Order Sum] WITHIN [Category]{order_by_str})"), ], ], override_formula_fields=[ @@ -623,17 +625,17 @@ def _make_mfunc_examples(func: str) -> List[DataExample]: ("City", "[City]"), ("Category", "[Category]"), ("Order Sum", "[Order Sum]"), - (f"{func} 1", f"{func}([Order Sum], 1 TOTAL)"), - (f"{func} 2", f"{func}([Order Sum], 1 WITHIN [City])"), - (f"{func} 3", f"{func}([Order Sum], 1 WITHIN [Category])"), + (f"{func} 1", f"{func}([Order Sum], 1 TOTAL ORDER BY [City], [Category])"), + (f"{func} 2", f"{func}([Order Sum], 1 WITHIN [City] ORDER BY [Category])"), + (f"{func} 3", f"{func}([Order Sum], 1 WITHIN [Category] ORDER BY [City])"), ], ], override_formula_fields=[ ("City", "[City]"), ("Category", "[Category]"), ("Order Sum", "SUM([Orders])"), - (f"{func} 1", f"{func}(SUM([Orders]), 1 TOTAL ORDER BY [City])"), - (f"{func} 2", f"{func}(SUM([Orders]), 1 WITHIN [City] ORDER BY [City])"), + (f"{func} 1", f"{func}(SUM([Orders]), 1 TOTAL ORDER BY [City], [Category])"), + (f"{func} 2", f"{func}(SUM([Orders]), 1 WITHIN [City] ORDER BY [Category])"), (f"{func} 3", f"{func}(SUM([Orders]), 1 AMONG [City] ORDER BY [City])"), ], ), From 4b11450debdb3d53f1a67af7a38f6366a5e94cf3 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Wed, 22 Nov 2023 15:48:04 +0100 Subject: [PATCH 22/27] Added validation of path move in GitFilesystemEditor (#86) --- .../dl_repmanager/dl_repmanager/fs_editor.py | 17 +++++++++++++++++ terrarium/dl_repmanager/pyproject.toml | 1 + 2 files changed, 18 insertions(+) diff --git a/terrarium/dl_repmanager/dl_repmanager/fs_editor.py b/terrarium/dl_repmanager/dl_repmanager/fs_editor.py index 67407c206..3d530dbec 100644 --- a/terrarium/dl_repmanager/dl_repmanager/fs_editor.py +++ b/terrarium/dl_repmanager/dl_repmanager/fs_editor.py @@ -17,6 +17,7 @@ ) import attr +from git import Repo as GitRepo @attr.s(frozen=True) @@ -234,7 +235,23 @@ def _path_exists(self, path: Path) -> bool: class GitFilesystemEditor(DefaultFilesystemEditor): """An FS editor that buses git to move files and directories""" + def _validate_paths_in_same_repo(self, *paths: Path) -> None: + root_paths = {GitRepo(path, search_parent_directories=True).working_tree_dir for path in paths} + if len(root_paths): + raise RuntimeError( + "Cross-repository operations are not supported for GitFilesystemEditor. Use `--fs-editor default`" + ) + + def _find_existing_parent(self, path: Path) -> Path: + assert path.is_absolute() + while not path.exists(): + if path.parent == path: + break + path = path.parent + return path + def _move_path(self, old_path: Path, new_path: Path) -> None: + self._validate_paths_in_same_repo(old_path, self._find_existing_parent(new_path)) cwd = Path.cwd() rel_old_path = Path(os.path.relpath(old_path, cwd)) rel_new_path = Path(os.path.relpath(new_path, cwd)) diff --git a/terrarium/dl_repmanager/pyproject.toml b/terrarium/dl_repmanager/pyproject.toml index fbb137955..9d602b50d 100644 --- a/terrarium/dl_repmanager/pyproject.toml +++ b/terrarium/dl_repmanager/pyproject.toml @@ -11,6 +11,7 @@ readme = "README.md" [tool.poetry.dependencies] attrs = ">=22.2.0" frozendict = ">=2.3.8" +gitpython = ">=3.1.37" python = ">=3.10, <3.12" pyyaml = ">=6.0.1" tomlkit = "==0.11.8" From 78fb4c47638eafb78ab2a71a6b7ef8a13b829ce0 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Wed, 22 Nov 2023 16:12:14 +0100 Subject: [PATCH 23/27] A couple of sa dialect tests for postgresdql and mysql from legacy (#116) --- .../db/core/test_sa_dialect.py | 42 +++++++++++++++++++ .../db/core/test_sa_dialect.py | 40 ++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_sa_dialect.py create mode 100644 lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_sa_dialect.py diff --git a/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_sa_dialect.py b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_sa_dialect.py new file mode 100644 index 000000000..b83cbb35c --- /dev/null +++ b/lib/dl_connector_mysql/dl_connector_mysql_tests/db/core/test_sa_dialect.py @@ -0,0 +1,42 @@ +import datetime + +import pytest +import sqlalchemy as sa +import sqlalchemy.dialects.mysql +import sqlalchemy.sql.sqltypes + +from dl_connector_mysql_tests.db.core.base import BaseMySQLTestClass + + +class TestMySQLSaDialect(BaseMySQLTestClass): + @pytest.mark.parametrize( + ("value", "type_", "expected"), + ( + pytest.param( + datetime.date(2022, 1, 2), sqlalchemy.sql.sqltypes.Date(), datetime.date(2022, 1, 2), id="date-as-date" + ), + pytest.param( + "2022-01-02", sqlalchemy.dialects.mysql.DATE(), datetime.date(2022, 1, 2), id="date-as-string" + ), + pytest.param( + datetime.datetime(2022, 1, 2, 12, 59, 59), + sqlalchemy.sql.sqltypes.DateTime(), + datetime.datetime(2022, 1, 2, 12, 59, 59), + id="datetime-as-datetime", + ), + pytest.param( + "2022-01-02T12:59:59", + sqlalchemy.dialects.mysql.DATETIME(fsp=6), + datetime.datetime(2022, 1, 2, 12, 59, 59), + id="datetime-as-string", + ), + ), + ) + def test_mysql_literal_bind_datetimes(self, value, type_, expected, db): + execute = db.execute + dialect = db._engine_wrapper.dialect + + query = sa.select([sa.literal(value, type_=type_)]) + compiled = str(query.compile(dialect=dialect, compile_kwargs={"literal_binds": True})) + res_literal = list(execute(compiled)) + assert res_literal[0][0] == expected diff --git a/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_sa_dialect.py b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_sa_dialect.py new file mode 100644 index 000000000..96313a9fc --- /dev/null +++ b/lib/dl_connector_postgresql/dl_connector_postgresql_tests/db/core/test_sa_dialect.py @@ -0,0 +1,40 @@ +import datetime + +import pytest +import pytz +import sqlalchemy as sa + +from dl_connector_postgresql_tests.db.core.base import BasePostgreSQLTestClass + + +TEST_VALUES = [datetime.date(2020, 1, 1)] + [ + datetime.datetime(2020, idx1 + 1, idx2 + 1, 3, 4, 5, us).replace(tzinfo=tzinfo) + for idx1, us in enumerate((0, 123356)) + for idx2, tzinfo in enumerate( + ( + None, + datetime.timezone.utc, + pytz.timezone("America/New_York"), + ) + ) +] + + +class TestPostgresqlSaDialect(BasePostgreSQLTestClass): + @pytest.mark.parametrize("value", TEST_VALUES, ids=[val.isoformat() for val in TEST_VALUES]) + def test_pg_literal_bind_datetimes(self, value, db): + """ + Test that query results for literal_binds matches the query results without, + for the custom dialect code. + + This test should be in the bi_postgresql dialect itself, but it doesn't have + a postgres-using test at the moment. + """ + execute = db.execute + dialect = db._engine_wrapper.dialect + + query = sa.select([sa.literal(value)]) + compiled = str(query.compile(dialect=dialect, compile_kwargs={"literal_binds": True})) + res_direct = list(execute(query)) + res_literal = list(execute(compiled)) + assert res_direct == res_literal, dict(literal_query=compiled) From efac0a9e8e1398900697e3062b7f44eeb1dbe0d8 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Thu, 23 Nov 2023 10:52:42 +0100 Subject: [PATCH 24/27] Moved redis settings from api test config to core test config (#112) * Moved redis settings from api test config to core test config * Renamed as_url to as_single_host_url --- lib/dl_api_lib/dl_api_lib/app/data_api/app.py | 15 +---- .../dl_api_lib_testing/app.py | 27 --------- .../dl_api_lib_testing/base.py | 3 +- .../dl_api_lib_testing/configuration.py | 8 --- .../dl_api_lib_testing/data_api_base.py | 7 +-- .../dl_configs/settings_submodels.py | 9 +++ .../db/base/core/base.py | 6 +- .../db/config.py | 6 +- .../dl_connector_chyt/core/adapters.py | 6 +- .../core/clickhouse_base/adapters.py | 2 +- .../dl_connector_promql/core/adapter.py | 2 +- .../adapters/async_adapters_remote.py | 6 +- lib/dl_core/dl_core/utils.py | 14 ----- .../dl_core_testing/configuration.py | 59 +++++++++++++++++++ .../dl_file_uploader_lib/settings_utils.py | 8 +-- lib/dl_utils/dl_utils/utils.py | 12 ++++ 16 files changed, 99 insertions(+), 91 deletions(-) diff --git a/lib/dl_api_lib/dl_api_lib/app/data_api/app.py b/lib/dl_api_lib/dl_api_lib/app/data_api/app.py index 31a18820c..0ec6178fa 100644 --- a/lib/dl_api_lib/dl_api_lib/app/data_api/app.py +++ b/lib/dl_api_lib/dl_api_lib/app/data_api/app.py @@ -71,7 +71,6 @@ RedisSentinelService, SingleHostSimpleRedisService, ) -from dl_core.utils import make_url LOGGER = logging.getLogger(__name__) @@ -248,12 +247,7 @@ def create_app( if self._settings.CACHES_REDIS.MODE == RedisMode.single_host: redis_server_single_host = SingleHostSimpleRedisService( instance_kind=RedisInstanceKind.caches, - url=make_url( - protocol="rediss" if self._settings.CACHES_REDIS.SSL else "redis", - host=self._settings.CACHES_REDIS.HOSTS[0], - port=self._settings.CACHES_REDIS.PORT, - path=str(self._settings.CACHES_REDIS.DB), - ), + url=self._settings.CACHES_REDIS.as_single_host_url(), password=self._settings.CACHES_REDIS.PASSWORD, ssl=self._settings.CACHES_REDIS.SSL, ) @@ -278,12 +272,7 @@ def create_app( if self._settings.MUTATIONS_REDIS.MODE == RedisMode.single_host: mutations_redis_server_single_host = SingleHostSimpleRedisService( instance_kind=RedisInstanceKind.mutations, - url=make_url( - protocol="rediss" if self._settings.MUTATIONS_REDIS.SSL else "redis", - host=self._settings.MUTATIONS_REDIS.HOSTS[0], - port=self._settings.MUTATIONS_REDIS.PORT, - path=str(self._settings.MUTATIONS_REDIS.DB), - ), + url=self._settings.MUTATIONS_REDIS.as_single_host_url(), password=self._settings.MUTATIONS_REDIS.PASSWORD, ssl=self._settings.MUTATIONS_REDIS.SSL, ) diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py index 69da45d86..307c5635a 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py @@ -98,33 +98,6 @@ def rqe_config_subprocess_cm(self) -> Generator[RQEConfig, None, None]: ) -@attr.s -class RedisSettingMaker: - bi_test_config: ApiTestEnvironmentConfiguration = attr.ib(kw_only=True) - - def get_redis_settings(self, db: int) -> RedisSettings: - return RedisSettings( # type: ignore # TODO: fix compatibility of models using `s_attrib` with mypy - MODE=RedisMode.single_host, - CLUSTER_NAME="", - HOSTS=(self.bi_test_config.redis_host,), - PORT=self.bi_test_config.redis_port, - DB=db, - PASSWORD=self.bi_test_config.redis_password, - ) - - def get_redis_settings_default(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_default) - - def get_redis_settings_cache(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_cache) - - def get_redis_settings_mutation(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_mutation) - - def get_redis_settings_arq(self) -> RedisSettings: - return self.get_redis_settings(self.bi_test_config.redis_db_arq) - - class TestingSRFactoryBuilder(SRFactoryBuilder[AppSettings]): def _get_required_services(self, settings: AppSettings) -> set[RequiredService]: return {RequiredService.RQE_INT_SYNC, RequiredService.RQE_EXT_SYNC} diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/base.py index eb85f299a..068b2db29 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/base.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/base.py @@ -26,7 +26,6 @@ from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig from dl_api_lib.loader import preload_api_lib from dl_api_lib_testing.app import ( - RedisSettingMaker, RQEConfigurationMaker, TestingControlApiAppFactory, ) @@ -104,7 +103,7 @@ def create_control_api_settings( core_test_config = bi_test_config.core_test_config us_config = core_test_config.get_us_config() - redis_setting_maker = RedisSettingMaker(bi_test_config=bi_test_config) + redis_setting_maker = core_test_config.get_redis_setting_maker() settings = ControlApiAppSettings( CONNECTOR_AVAILABILITY=ConnectorAvailabilityConfig.from_settings( diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py index 48a9ed3ec..d61dee412 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/configuration.py @@ -25,14 +25,6 @@ class ApiTestEnvironmentConfiguration: file_uploader_api_host: str = attr.ib(default="http://127.0.0.1") file_uploader_api_port: int = attr.ib(default=9999) - redis_host: str = attr.ib(default="") - redis_port: int = attr.ib(default=6379) - redis_password: str = attr.ib(default="") - redis_db_default: int = attr.ib(default=0) - redis_db_cache: int = attr.ib(default=1) - redis_db_mutation: int = attr.ib(default=2) - redis_db_arq: int = attr.ib(default=11) - connector_availability_settings: ConnectorAvailabilityConfigSettings = attr.ib( factory=ConnectorAvailabilityConfigSettings, ) diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py index 8c00f652e..b10309931 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/data_api_base.py @@ -21,10 +21,7 @@ from dl_api_client.dsmaker.primitives import Dataset from dl_api_lib.app.data_api.app import DataApiAppFactory from dl_api_lib.app_settings import DataApiAppSettings -from dl_api_lib_testing.app import ( - RedisSettingMaker, - TestingDataApiAppFactory, -) +from dl_api_lib_testing.app import TestingDataApiAppFactory from dl_api_lib_testing.base import ApiTestBase from dl_api_lib_testing.client import ( TestClientConverterAiohttpToFlask, @@ -67,7 +64,7 @@ def create_data_api_settings( ) -> DataApiAppSettings: core_test_config = bi_test_config.core_test_config us_config = core_test_config.get_us_config() - redis_setting_maker = RedisSettingMaker(bi_test_config=bi_test_config) + redis_setting_maker = core_test_config.get_redis_setting_maker() return DataApiAppSettings( SENTRY_ENABLED=False, diff --git a/lib/dl_configs/dl_configs/settings_submodels.py b/lib/dl_configs/dl_configs/settings_submodels.py index 319371f07..21234c88d 100644 --- a/lib/dl_configs/dl_configs/settings_submodels.py +++ b/lib/dl_configs/dl_configs/settings_submodels.py @@ -6,6 +6,7 @@ from dl_configs.settings_loaders.meta_definition import s_attrib from dl_configs.settings_loaders.settings_obj_base import SettingsBase from dl_configs.utils import split_by_comma +from dl_utils.utils import make_url def redis_mode_env_var_converter(env_value: str) -> RedisMode: @@ -25,6 +26,14 @@ class RedisSettings(SettingsBase): PASSWORD: str = s_attrib("PASSWORD", sensitive=True, missing=None) SSL: Optional[bool] = s_attrib("SSL", missing=None) + def as_single_host_url(self) -> str: + return make_url( + protocol="rediss" if self.SSL else "redis", + host=self.HOSTS[0], + port=self.PORT, + path=str(self.DB), + ) + @attr.s(frozen=True) class CorsSettings(SettingsBase): diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py index db5506f3c..3a662632f 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py @@ -18,7 +18,6 @@ RequestContextInfo, TenantCommon, ) -from dl_api_lib_testing.app import RedisSettingMaker from dl_configs.settings_submodels import S3Settings from dl_constants.enums import DataSourceType from dl_core.db import ( @@ -26,6 +25,7 @@ get_type_transformer, ) from dl_core.services_registry import ServicesRegistry +from dl_core_testing.configuration import RedisSettingMaker from dl_core_testing.database import DbTable from dl_core_testing.fixtures.primitives import FixtureTableSpec from dl_core_testing.fixtures.sample_tables import TABLE_SPEC_SAMPLE_SUPERSTORE @@ -89,8 +89,8 @@ def conn_bi_context(self) -> RequestContextInfo: @pytest.fixture(scope="session") def redis_setting_maker(self) -> RedisSettingMaker: - bi_test_config = test_config.API_TEST_CONFIG - return RedisSettingMaker(bi_test_config=bi_test_config) + core_test_config = test_config.CORE_TEST_CONFIG + return core_test_config.get_redis_setting_maker() @pytest.fixture(scope="session") def s3_settings(self) -> S3Settings: diff --git a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py index 9a52eb713..df42bcf2d 100644 --- a/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py +++ b/lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/config.py @@ -12,6 +12,9 @@ port_us_pg_5432=get_test_container_hostport("pg-us", fallback_port=52610).port, us_master_token="AC1ofiek8coB", core_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"], + redis_host=get_test_container_hostport("redis", fallback_port=52604).host, + redis_port=get_test_container_hostport("redis", fallback_port=52604).port, + redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5", ) SR_CONNECTION_SETTINGS = FileS3ConnectorSettings( @@ -37,7 +40,4 @@ api_connector_ep_names=["clickhouse", "file", "gsheets_v2", "yadocs"], core_test_config=CORE_TEST_CONFIG, ext_query_executer_secret_key="_some_test_secret_key_", - redis_host=get_test_container_hostport("redis", fallback_port=52604).host, - redis_port=get_test_container_hostport("redis", fallback_port=52604).port, - redis_password="AwockEuvavDyinmeakmiRiopanbesBepsensUrdIz5", ) diff --git a/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py b/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py index 05a7bf6d5..0e7d84b76 100644 --- a/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py +++ b/lib/dl_connector_chyt/dl_connector_chyt/core/adapters.py @@ -16,11 +16,9 @@ from dl_core.connection_executors.models.db_adapter_data import RawIndexInfo from dl_core.connection_models import TableIdent from dl_core.connectors.base.error_transformer import DBExcKWArgs -from dl_core.utils import ( - get_current_w3c_tracing_headers, - make_url, -) +from dl_core.utils import get_current_w3c_tracing_headers from dl_utils.aio import await_sync +from dl_utils.utils import make_url from dl_connector_chyt.core.constants import CONNECTION_TYPE_CHYT from dl_connector_chyt.core.target_dto import ( diff --git a/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py b/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py index a9f0a7a71..312400958 100644 --- a/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py +++ b/lib/dl_connector_clickhouse/dl_connector_clickhouse/core/clickhouse_base/adapters.py @@ -63,7 +63,7 @@ GenericNativeType, norm_native_type, ) -from dl_core.utils import make_url +from dl_utils.utils import make_url from dl_connector_clickhouse.core.clickhouse_base.ch_commons import ( ClickHouseBaseUtils, diff --git a/lib/dl_connector_promql/dl_connector_promql/core/adapter.py b/lib/dl_connector_promql/dl_connector_promql/core/adapter.py index 424deff73..e460979a3 100644 --- a/lib/dl_connector_promql/dl_connector_promql/core/adapter.py +++ b/lib/dl_connector_promql/dl_connector_promql/core/adapter.py @@ -29,7 +29,7 @@ from dl_core.connection_executors.adapters.async_adapters_base import AsyncRawExecutionResult from dl_core.db.native_type import GenericNativeType from dl_core.exc import DatabaseQueryError -from dl_core.utils import make_url +from dl_utils.utils import make_url from dl_connector_promql.core.constants import CONNECTION_TYPE_PROMQL diff --git a/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py b/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py index 756e6eb69..cdbdfeea9 100644 --- a/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py +++ b/lib/dl_core/dl_core/connection_executors/adapters/async_adapters_remote.py @@ -35,7 +35,6 @@ generic_profiler_async, ) from dl_core import exc as common_exc -from dl_core import utils from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter from dl_core.connection_executors.adapters.async_adapters_base import ( AsyncDBAdapter, @@ -62,6 +61,7 @@ from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest from dl_core.connection_models.conn_options import ConnectOptions from dl_core.enums import RQEEventType +from dl_utils.utils import make_url if TYPE_CHECKING: @@ -147,14 +147,14 @@ async def _make_request( url: str if self._use_sync_rqe: - url = utils.make_url( + url = make_url( protocol=qe.sync_protocol, host=qe.sync_host, port=qe.sync_port, path=rel_path, ) else: - url = utils.make_url( + url = make_url( protocol=qe.async_protocol, host=qe.async_host, port=qe.async_port, diff --git a/lib/dl_core/dl_core/utils.py b/lib/dl_core/dl_core/utils.py index 5953d1b08..d874688d3 100644 --- a/lib/dl_core/dl_core/utils.py +++ b/lib/dl_core/dl_core/utils.py @@ -2,7 +2,6 @@ import ipaddress import logging -import os import re from typing import ( Any, @@ -42,25 +41,12 @@ stringify_dl_cookies, stringify_dl_headers, ) -from dl_configs.settings_loaders.env_remap import remap_env from dl_constants.api_constants import DLHeadersCommon LOGGER = logging.getLogger(__name__) -def make_url( - protocol: str, - host: str, - port: int, - path: Optional[str] = None, -) -> str: - # TODO FIX: Sanitize/use urllib - if path is None: - path = "" - return f"{protocol}://{host}:{port}/{path.lstrip('/')}" - - def get_requests_session() -> requests.Session: session = requests.Session() ua = "{}, Datalens".format(requests.utils.default_user_agent()) diff --git a/lib/dl_core_testing/dl_core_testing/configuration.py b/lib/dl_core_testing/dl_core_testing/configuration.py index e14085902..0851f1b7c 100644 --- a/lib/dl_core_testing/dl_core_testing/configuration.py +++ b/lib/dl_core_testing/dl_core_testing/configuration.py @@ -13,6 +13,8 @@ CryptoKeysConfig, get_single_key_crypto_keys_config, ) +from dl_configs.enums import RedisMode +from dl_configs.settings_submodels import RedisSettings from dl_core.loader import CoreLibraryConfig @@ -27,6 +29,39 @@ class UnitedStorageConfiguration: force: bool = attr.ib(kw_only=True, default=True) +@attr.s(frozen=True) +class RedisSettingMaker: + redis_host: str = attr.ib(default="") + redis_port: int = attr.ib(default=6379) + redis_password: str = attr.ib(default="") + redis_db_default: int = attr.ib(default=0) + redis_db_cache: int = attr.ib(default=1) + redis_db_mutation: int = attr.ib(default=2) + redis_db_arq: int = attr.ib(default=11) + + def get_redis_settings(self, db: int) -> RedisSettings: + return RedisSettings( # type: ignore # TODO: fix compatibility of models using `s_attrib` with mypy + MODE=RedisMode.single_host, + CLUSTER_NAME="", + HOSTS=(self.redis_host,), + PORT=self.redis_port, + DB=db, + PASSWORD=self.redis_password, + ) + + def get_redis_settings_default(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_default) + + def get_redis_settings_cache(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_cache) + + def get_redis_settings_mutation(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_mutation) + + def get_redis_settings_arq(self) -> RedisSettings: + return self.get_redis_settings(self.redis_db_arq) + + @attr.s(frozen=True) class CoreTestEnvironmentConfigurationBase(abc.ABC): @abc.abstractmethod @@ -41,6 +76,10 @@ def get_crypto_keys_config(self) -> CryptoKeysConfig: def get_core_library_config(self) -> CoreLibraryConfig: raise NotImplementedError + @abc.abstractmethod + def get_redis_setting_maker(self) -> RedisSettingMaker: + raise NotImplementedError + # These are used only for creation of local environments in tests, not actual external ones DEFAULT_FERNET_KEY = "h1ZpilcYLYRdWp7Nk8X1M1kBPiUi8rdjz9oBfHyUKIk=" @@ -54,8 +93,17 @@ class DefaultCoreTestConfiguration(CoreTestEnvironmentConfigurationBase): port_us_pg_5432: int = attr.ib(kw_only=True) us_master_token: str = attr.ib(kw_only=True) fernet_key: str = attr.ib(kw_only=True, default=DEFAULT_FERNET_KEY) + core_connector_ep_names: Optional[Collection[str]] = attr.ib(kw_only=True, default=None) + redis_host: str = attr.ib(default="") + redis_port: int = attr.ib(default=6379) + redis_password: str = attr.ib(default="") + redis_db_default: int = attr.ib(default=0) + redis_db_cache: int = attr.ib(default=1) + redis_db_mutation: int = attr.ib(default=2) + redis_db_arq: int = attr.ib(default=11) + def get_us_config(self) -> UnitedStorageConfiguration: return UnitedStorageConfiguration( us_master_token=self.us_master_token, @@ -66,6 +114,17 @@ def get_us_config(self) -> UnitedStorageConfiguration: def get_crypto_keys_config(self) -> CryptoKeysConfig: return get_single_key_crypto_keys_config(key_id="0", key_value=self.fernet_key) + def get_redis_setting_maker(self) -> RedisSettingMaker: + return RedisSettingMaker( + redis_host=self.redis_host, + redis_port=self.redis_port, + redis_password=self.redis_password, + redis_db_default=self.redis_db_default, + redis_db_cache=self.redis_db_cache, + redis_db_mutation=self.redis_db_mutation, + redis_db_arq=self.redis_db_arq, + ) + def get_core_library_config(self) -> CoreLibraryConfig: return CoreLibraryConfig( core_connector_ep_names=self.core_connector_ep_names, diff --git a/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py b/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py index 2e1f00abc..1fd0b0b6e 100644 --- a/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py +++ b/lib/dl_file_uploader_lib/dl_file_uploader_lib/settings_utils.py @@ -5,7 +5,6 @@ RedisSentinelService, SingleHostSimpleRedisService, ) -from dl_core.utils import make_url from dl_file_uploader_lib.settings import FileUploaderBaseSettings @@ -14,12 +13,7 @@ def init_redis_service(settings: FileUploaderBaseSettings) -> RedisBaseService: if settings.REDIS_APP.MODE == RedisMode.single_host: assert len(settings.REDIS_APP.HOSTS) == 1 redis_service = SingleHostSimpleRedisService( - url=make_url( - protocol="rediss" if settings.REDIS_APP.SSL else "redis", - host=settings.REDIS_APP.HOSTS[0], - port=settings.REDIS_APP.PORT, - path=str(settings.REDIS_APP.DB), - ), + url=settings.REDIS_APP.as_single_host_url(), password=settings.REDIS_APP.PASSWORD, instance_kind=RedisInstanceKind.persistent, ssl=settings.REDIS_APP.SSL, diff --git a/lib/dl_utils/dl_utils/utils.py b/lib/dl_utils/dl_utils/utils.py index f54affdf3..1b45e9ca3 100644 --- a/lib/dl_utils/dl_utils/utils.py +++ b/lib/dl_utils/dl_utils/utils.py @@ -221,3 +221,15 @@ def time_it_cm(label: str) -> Generator[None, None, None]: delta = time() - t0 if delta >= 0.01: print(f"Time elapsed for {label}: {delta}") + + +def make_url( + protocol: str, + host: str, + port: int, + path: Optional[str] = None, +) -> str: + # TODO FIX: Sanitize/use urllib + if path is None: + path = "" + return f"{protocol}://{host}:{port}/{path.lstrip('/')}" From 9a972563280f10d15a9d0b1ebbd7f4ea6cae6826 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Thu, 23 Nov 2023 16:09:39 +0100 Subject: [PATCH 25/27] Fixed resolution of MQM factories (#117) * Fixed resolution of MQM factories * Style fix --- lib/dl_api_lib/dl_api_lib/query/registry.py | 36 ++++++++++++++----- .../multi_query_mutator_factory.py | 20 +++++------ .../dl_connector_bitrix_gds/api/connector.py | 5 +++ 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/lib/dl_api_lib/dl_api_lib/query/registry.py b/lib/dl_api_lib/dl_api_lib/query/registry.py index 0ff165fca..115faeab4 100644 --- a/lib/dl_api_lib/dl_api_lib/query/registry.py +++ b/lib/dl_api_lib/dl_api_lib/query/registry.py @@ -1,6 +1,7 @@ from typing import ( Collection, Optional, + Sequence, Type, ) @@ -87,25 +88,42 @@ class MQMFactorySettingItem: _MQM_FACTORY_REGISTRY: dict[MQMFactoryKey, Type[MultiQueryMutatorFactoryBase]] = {} +def _get_default_mqm_factory_cls() -> Type[MultiQueryMutatorFactoryBase]: + return DefaultMultiQueryMutatorFactory + + def get_multi_query_mutator_factory( query_proc_mode: QueryProcessingMode, backend_type: SourceBackendType, dialect: DialectCombo, result_schema: ResultSchema, -) -> Optional[MultiQueryMutatorFactoryBase]: - factory_cls = _MQM_FACTORY_REGISTRY.get( - # First try with exact dialect +) -> MultiQueryMutatorFactoryBase: + prioritized_keys = ( + # First try with exact dialect and mode (exact match) MQMFactoryKey(query_proc_mode=query_proc_mode, backend_type=backend_type, dialect=dialect), - _MQM_FACTORY_REGISTRY.get( - # Then try without the dialect, just the backend - MQMFactoryKey(query_proc_mode=query_proc_mode, backend_type=backend_type, dialect=None), - DefaultMultiQueryMutatorFactory, # If still nothing, then use the default - ), + # Now the fallbacks begin... + # Try without the dialect (all dialects within backend), just the backend and mode + MQMFactoryKey(query_proc_mode=query_proc_mode, backend_type=backend_type, dialect=None), + # Fall back to `basic` mode (but still within the backend type) + # First try with the specific dialect + MQMFactoryKey(query_proc_mode=QueryProcessingMode.basic, backend_type=backend_type, dialect=dialect), + # If still nothing, try without specifying the dialect (all dialects within backend) + MQMFactoryKey(query_proc_mode=QueryProcessingMode.basic, backend_type=backend_type, dialect=None), ) + # Now iterate over all of these combinations IN THAT VERY ORDER(!) + factory_cls: Optional[Type[MultiQueryMutatorFactoryBase]] = None + for key in prioritized_keys: + factory_cls = _MQM_FACTORY_REGISTRY.get(key) + if factory_cls is not None: + break # found something + if factory_cls is None: - return None + # Not found for any of the combinations + # Use the ultimate default + factory_cls = _get_default_mqm_factory_cls() + assert factory_cls is not None return factory_cls(result_schema=result_schema) diff --git a/lib/dl_api_lib/dl_api_lib/service_registry/multi_query_mutator_factory.py b/lib/dl_api_lib/dl_api_lib/service_registry/multi_query_mutator_factory.py index bc8c0dd9e..63c692dc4 100644 --- a/lib/dl_api_lib/dl_api_lib/service_registry/multi_query_mutator_factory.py +++ b/lib/dl_api_lib/dl_api_lib/service_registry/multi_query_mutator_factory.py @@ -1,4 +1,5 @@ import abc +import logging from typing import Sequence import attr @@ -14,6 +15,9 @@ from dl_query_processing.multi_query.mutators.base import MultiQueryMutatorBase +LOGGER = logging.getLogger(__name__) + + class SRMultiQueryMutatorFactory(abc.ABC): @abc.abstractmethod def get_mqm_factory( @@ -51,14 +55,10 @@ def get_mqm_factory( dialect=dialect, result_schema=dataset.result_schema, ) - if factory is None: - # Try again for the basic mode - factory = get_multi_query_mutator_factory( - query_proc_mode=QueryProcessingMode.basic, - backend_type=backend_type, - dialect=dialect, - result_schema=dataset.result_schema, - ) - - assert factory is not None + LOGGER.info( + f"Resolved MQM factory for backend_type {backend_type.name} " + f"and dialect {dialect.common_name_and_version} " + f"in {self._query_proc_mode.name} mode " + f"to {type(factory).__name__}" + ) return factory diff --git a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py index b5b25257d..67277a734 100644 --- a/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py +++ b/lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/api/connector.py @@ -11,6 +11,7 @@ ) from dl_api_lib.query.registry import MQMFactorySettingItem from dl_constants.enums import QueryProcessingMode +from dl_query_processing.multi_query.factory import NoCompengMultiQueryMutatorFactory from dl_connector_bitrix_gds.api.api_schema.connection import BitrixGDSConnectionSchema from dl_connector_bitrix_gds.api.connection_form.form_config import BitrixGDSConnectionFormFactory @@ -47,6 +48,10 @@ class BitrixGDSApiConnector(ApiConnector): query_proc_mode=QueryProcessingMode.basic, factory_cls=BitrixGDSMultiQueryMutatorFactory, ), + MQMFactorySettingItem( + query_proc_mode=QueryProcessingMode.no_compeng, + factory_cls=NoCompengMultiQueryMutatorFactory, + ), ) connection_definitions = (BitrixGDSApiConnectionDefinition,) source_definitions = (BitrixGDSApiSourceDefinition,) From d0f188c8b47593e16f4a613284e31dca9bba7dc8 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 24 Nov 2023 00:07:37 +0100 Subject: [PATCH 26/27] Fix for compeng caches (#119) --- .../dl_core/data_processing/cache/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/dl_core/dl_core/data_processing/cache/utils.py b/lib/dl_core/dl_core/data_processing/cache/utils.py index 70902ccdf..3cd6adff4 100644 --- a/lib/dl_core/dl_core/data_processing/cache/utils.py +++ b/lib/dl_core/dl_core/data_processing/cache/utils.py @@ -138,6 +138,20 @@ def get_cache_options( refresh_ttl_on_read=ttl_info.refresh_ttl_on_read, ) + def get_data_key( + self, + *, + query_res_info: QueryAndResultInfo, + from_info: Optional[PreparedFromInfo] = None, + base_key: LocalKeyRepresentation = LocalKeyRepresentation(), + ) -> Optional[LocalKeyRepresentation]: + # TODO: Remove after switching to new cache keys + compiled_query = self.get_query_str_for_cache( + query=query_res_info.query, + dialect=from_info.query_compiler.dialect, + ) + return base_key.extend(part_type="query", part_content=compiled_query) + @attr.s class SelectorCacheOptionsBuilder(DatasetOptionsBuilder): @@ -173,6 +187,8 @@ def make_data_select_cache_key( is_bleeding_edge_user: bool, base_key: LocalKeyRepresentation = LocalKeyRepresentation(), ) -> LocalKeyRepresentation: + # TODO: Remove after switching to new cache keys, + # but put the db_name + target_connection.get_cache_key_part() parts somewhere assert from_info.target_connection_ref is not None target_connection = self._us_entry_buffer.get_entry(from_info.target_connection_ref) assert isinstance(target_connection, ConnectionBase) @@ -201,6 +217,7 @@ def get_data_key( from_info: Optional[PreparedFromInfo] = None, base_key: LocalKeyRepresentation = LocalKeyRepresentation(), ) -> Optional[LocalKeyRepresentation]: + # TODO: Remove after switching to new cache keys compiled_query = self.get_query_str_for_cache( query=query_res_info.query, dialect=from_info.query_compiler.dialect, From 218d3e576ba568506c81aaacb60885929d00f781 Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Fri, 24 Nov 2023 09:40:34 +0100 Subject: [PATCH 27/27] BI-4885: add basic RLS tests (#120) --- .../db/control_api/test_rls.py | 58 +++++++++++++++++++ .../dl_api_lib_testing/app.py | 52 +++++++++++++---- .../dl_api_lib_testing/rls.py | 4 ++ .../rls_configs/missing_login_updated | 2 +- .../rls_configs/missing_login_updated.json | 2 +- .../test_data/rls_configs/simple_updated | 2 +- .../test_data/rls_configs/simple_updated.json | 2 +- 7 files changed, 107 insertions(+), 15 deletions(-) create mode 100644 lib/dl_api_lib/dl_api_lib_tests/db/control_api/test_rls.py diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/control_api/test_rls.py b/lib/dl_api_lib/dl_api_lib_tests/db/control_api/test_rls.py new file mode 100644 index 000000000..078144e34 --- /dev/null +++ b/lib/dl_api_lib/dl_api_lib_tests/db/control_api/test_rls.py @@ -0,0 +1,58 @@ +import pytest + +from dl_api_lib_testing.rls import ( + RLS_CONFIG_CASES, + config_to_comparable, + load_rls_config, +) +from dl_api_lib_tests.db.base import DefaultApiTestBase + + +class TestDataset(DefaultApiTestBase): + @staticmethod + def add_rls_to_dataset(control_api, dataset, rls_config): + field_guid = dataset.result_schema[0].id + dataset.rls = {field_guid: rls_config} + resp = control_api.save_dataset(dataset, fail_ok=True) + return field_guid, resp + + @pytest.mark.parametrize("case", RLS_CONFIG_CASES, ids=[c["name"] for c in RLS_CONFIG_CASES]) + def test_create_and_update_rls(self, control_api, saved_dataset, case): + config = case["config"] + ds = saved_dataset + field_guid, rls_resp = self.add_rls_to_dataset(control_api, ds, config) + assert rls_resp.status_code == 200, rls_resp.json + + resp = control_api.load_dataset(ds) + assert resp.status_code == 200, resp.json + ds = resp.dataset + assert config_to_comparable(ds.rls[field_guid]) == config_to_comparable(case["config_to_compare"]) + + config_updated = case.get("config_updated") + if config_updated is None: + return + field_guid, rls_resp = self.add_rls_to_dataset(control_api, ds, config_updated) + assert rls_resp.status_code == 200, rls_resp.json + + resp = control_api.load_dataset(ds) + assert resp.status_code == 200, resp.json + assert config_to_comparable(resp.dataset.rls[field_guid]) == config_to_comparable(config_updated) + + def test_create_rls_for_nonexistent_user(self, control_api, saved_dataset): + config = load_rls_config("bad_login") + ds = saved_dataset + field_guid, rls_resp = self.add_rls_to_dataset(control_api, ds, config) + assert rls_resp.status_code == 200, rls_resp.json + + resp = control_api.load_dataset(ds) + assert resp.status_code == 200, resp.json + assert "!FAILED_robot-user2" in resp.dataset.rls[field_guid] + + def test_create_rls_from_invalid_config(self, control_api, saved_dataset): + config = load_rls_config("bad") + _, rls_resp = self.add_rls_to_dataset(control_api, saved_dataset, config) + + assert rls_resp.status_code == 400 + assert rls_resp.bi_status_code == "ERR.DS_API.RLS.PARSE" + assert rls_resp.json["message"] == "RLS: Parsing failed at line 2" + assert rls_resp.json["details"] == {"description": "Wrong format"} diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py index 307c5635a..9daa920f7 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/app.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/app.py @@ -13,10 +13,7 @@ from dl_api_lib.app.control_api.app import EnvSetupResult as ControlApiEnvSetupResult from dl_api_lib.app.data_api.app import DataApiAppFactory from dl_api_lib.app.data_api.app import EnvSetupResult as DataApiEnvSetupResult -from dl_api_lib.app_common import ( - SRFactoryBuilder, - StandaloneServiceRegistryFactory, -) +from dl_api_lib.app_common import SRFactoryBuilder from dl_api_lib.app_common_settings import ConnOptionsMutatorsFactory from dl_api_lib.app_settings import ( AppSettings, @@ -27,26 +24,33 @@ from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig from dl_api_lib_testing.configuration import ApiTestEnvironmentConfiguration from dl_configs.connectors_settings import ConnectorSettingsBase -from dl_configs.enums import ( - RedisMode, - RequiredService, -) +from dl_configs.enums import RequiredService from dl_configs.rqe import ( RQEBaseURL, RQEConfig, ) -from dl_configs.settings_submodels import RedisSettings from dl_constants.enums import ( ConnectionType, + RLSSubjectType, USAuthMode, ) from dl_core.aio.middlewares.services_registry import services_registry_middleware from dl_core.aio.middlewares.us_manager import service_us_manager_middleware from dl_core.data_processing.cache.primitives import CacheTTLConfig +from dl_core.rls import ( + RLS_FAILED_USER_NAME_PREFIX, + BaseSubjectResolver, + RLSSubject, +) +from dl_core.services_registry import ServicesRegistry from dl_core.services_registry.entity_checker import EntityUsageChecker from dl_core.services_registry.env_manager_factory_base import EnvManagerFactory -from dl_core.services_registry.inst_specific_sr import InstallationSpecificServiceRegistryFactory +from dl_core.services_registry.inst_specific_sr import ( + InstallationSpecificServiceRegistry, + InstallationSpecificServiceRegistryFactory, +) from dl_core.services_registry.rqe_caches import RQECachesSetting +from dl_core.utils import FutureRef from dl_core_testing.app_test_workarounds import TestEnvManagerFactory from dl_core_testing.fixture_server_runner import WSGIRunner @@ -98,6 +102,32 @@ def rqe_config_subprocess_cm(self) -> Generator[RQEConfig, None, None]: ) +@attr.s +class TestingSubjectResolver(BaseSubjectResolver): + def get_subjects_by_names(self, names: list[str]) -> list[RLSSubject]: + """Mock resolver. Considers a user real if his name starts with 'user'""" + return [ + RLSSubject( + subject_id="", + subject_type=RLSSubjectType.user if name.startswith("user") else RLSSubjectType.notfound, + subject_name=name if name.startswith("user") else RLS_FAILED_USER_NAME_PREFIX + name, + ) + for name in names + ] + + +@attr.s +class TestingServiceRegistry(InstallationSpecificServiceRegistry): + async def get_subject_resolver(self) -> BaseSubjectResolver: + return TestingSubjectResolver() + + +@attr.s +class TestingServiceRegistryFactory(InstallationSpecificServiceRegistryFactory): + def get_inst_specific_sr(self, sr_ref: FutureRef[ServicesRegistry]) -> TestingServiceRegistry: + return TestingServiceRegistry(service_registry_ref=sr_ref) + + class TestingSRFactoryBuilder(SRFactoryBuilder[AppSettings]): def _get_required_services(self, settings: AppSettings) -> set[RequiredService]: return {RequiredService.RQE_INT_SYNC, RequiredService.RQE_EXT_SYNC} @@ -109,7 +139,7 @@ def _get_inst_specific_sr_factory( self, settings: AppSettings, ) -> Optional[InstallationSpecificServiceRegistryFactory]: - return StandaloneServiceRegistryFactory() + return TestingServiceRegistryFactory() def _get_entity_usage_checker(self, settings: AppSettings) -> Optional[EntityUsageChecker]: return None diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py b/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py index 15234d0de..00b77206e 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/rls.py @@ -65,6 +65,10 @@ def load_rls(name: str) -> list[RLSEntry]: MAIN_TEST_CASE = RLS_CONFIG_CASES[0] +def config_to_comparable(conf: str): + return set((line.split(": ")[0], ",".join(sorted(line.split(": ")[1]))) for line in conf.strip().split("\n")) + + def check_text_config_to_rls_entries(case: dict, subject_resolver: BaseSubjectResolver) -> None: field_guid, config, expected_rls_entries = case["field_guid"], case["config"], case["rls_entries"] entries = FieldRLSSerializer.from_text_config(config, field_guid, subject_resolver=subject_resolver) diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated index 488798cc4..55501e1e7 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated @@ -1,3 +1,3 @@ 'Москва': user2, user1 'Самара': user3, !FAILED_someuser, user1 -'Омск': user5, pg +'Омск': user5, user7 diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated.json b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated.json index 1f058217f..71674aa91 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated.json +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/missing_login_updated.json @@ -52,7 +52,7 @@ "subject": { "subject_id": "1120000000000251", "subject_type": "user", - "subject_name": "pg" + "subject_name": "user7" } } ] diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated index 7ab13577f..678a18b59 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated @@ -1,3 +1,3 @@ 'Москва': user2 'Самара': user1, user3 -'Омск': user5, pg +'Омск': user5, user7 diff --git a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated.json b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated.json index 2fdf870ec..aa287c3f2 100644 --- a/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated.json +++ b/lib/dl_api_lib_testing/dl_api_lib_testing/test_data/rls_configs/simple_updated.json @@ -36,7 +36,7 @@ "subject": { "subject_id": "1120000000000251", "subject_type": "user", - "subject_name": "pg" + "subject_name": "user7" } } ]