Merge branch 'main' into yadocs_update

.github/workflows/main.yml

@@ -219,3 +219,60 @@ jobs:
             ./report/**/*.xml
           event_name: ${{ github.event.workflow_run.event }}
           report_individual_runs: "true"
+
+  codestyle_all_without_ruff:
+    runs-on: [self-hosted, linux, light]
+    needs: gh_build_image
+    container:
+      # until https://github.com/github/docs/issues/25520 is resolved, using vars
+      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: 'Cleanup build folder'
+        run: |
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - run: |
+          task cq:check_dir -- .
+          task cq:check_dir_strict -- .
+        env:
+          VENV_PATH: /venv
+          SKIP_RUFF: true
+
+  codestyle_changed_without_ruff:
+    runs-on: [ self-hosted, linux, light ]
+    needs: gh_build_image
+    container:
+      # until https://github.com/github/docs/issues/25520 is resolved, using vars
+      image: "ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}/datalens_ci_with_code:${{ github.sha }}"
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: 'Cleanup build folder'
+        run: |
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      # https://github.com/actions/runner-images/issues/6775
+      - run: git config --global --add safe.directory .
+      - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
+      - run: |
+          TARGET=$(. /venv/bin/activate && dl-git range-diff-paths --only-added-commits --base ${{ github.event.pull_request.base.sha }} --head ${{ github.event.pull_request.head.sha }})
+          echo $TARGET
+          task cq:check_target -- "$TARGET"
+          task cq:check_target_strict  -- "$TARGET" 1>/dev/null 2>/dev/null
+        env:
+          VENV_PATH: /venv
+          SKIP_RUFF: true
+          TEST_TARGET_OVERRIDE: ${{ github.event.inputs.test_targets }}

.github/workflows/terrarium_check.yaml

@@ -29,19 +29,18 @@ jobs:
           rm -rf ./.??* || true
       - name: Checkout code
         uses: actions/checkout@v4
-      - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
       - name: Setup common tools
-        run: pip install --no-input poetry pytest
+        run: pip install --no-input poetry
       - name: Install dependencies
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          pip install --no-input .
+          poetry -v install --with pytest --without mypy
       - name: Pytest
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          pytest .
+          poetry run pytest .
 
-  mypu:
+  mypy:
     runs-on: [ self-hosted, linux ]
     container:
       image: "python:3.11.0"
@@ -60,14 +59,13 @@ jobs:
           rm -rf ./.??* || true
       - name: Checkout code
         uses: actions/checkout@v4
-      - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
       - name: Setup common tools
-        run: pip install --no-input poetry mypy
+        run: pip install --no-input poetry
       - name: Install dependencies
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          pip install --no-input .
+          poetry -v install --with mypy --without pytest
       - name: Mypy
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          mypy .
+          poetry run mypy .

.gitignore

@@ -15,3 +15,4 @@ ci_artifacts
 .DS_Store
 artifacts
 Taskfile.yml
+.obsidian

README.md

@@ -1,40 +1,15 @@
-## Management tasks
+# datalens-backend
 
-Running tasks:
-```
-task <task_name>
-```
+## About
 
+This is the repository for the back-end implementation of DataLens
 
-### Environment (`env:`)
+Head over to the [Knowledge Base](kb/index.md) for documentation on this repo.
 
-- `task env:devenv`:
-  Create development/testing environment (run it from a package dir)
-- `task env:devenv-d`:
-  Create development/testing environment in detached mode (run it from a package dir)
-- `task env:ensure_venv`: Command to create virtual env for the mainrepo tools. 
-  It requires presence of .env in the mainrepo/tools.
+[Code of conduct](CODE_OF_CONDUCT.md)
 
+[Contributing](CONTRIBUTING.md)
 
-### Generation (`gen:`)
+## License
 
-- `task gen:antlr`:
-  (Re-)generate ANTLR code files for formula
-- `task gen:i18n-po`:
-  Sync/generate `.po` files for package (run it from a package dir)
-- `task gen:i18n-binaries`:
-  Generate binary `.mo` files from `.po` files for package (run it from a package dir)
-
-
-### Code quality (`cq:`)
-
-Experimental tasks to check and fix source files. 
-
-- `task cq:fix_changed`:
-  Apply all auto-fixes
-- `task cq:check_changed`:
-  Check for any non-conformity in code style/format/lint
-- `task cq:fix_dir -- {single dir}`:
-  Apply all auto-fixes to the given dir absolute path
-- `task cq:check_dir -- {single dir}`:
-  Check for any non-conformity in code style/format/lint in the given dir abs path
+`datalens-backend` is available under the Apache 2.0 license.

app/dl_control_api/pyproject.toml

@@ -11,8 +11,6 @@ readme = "README.md"
 [tool.poetry.dependencies]
 python = ">=3.10, <3.12"
 Flask = ">=2.2.5"
-blinker = ">=1.5"
-raven = ">=6.10.0"
 datalens-version = {path = "../../lib/dl_version"}
 datalens-sqlalchemy-postgres = {path = "../../lib/dl_sqlalchemy_postgres"}
 datalens-utils = {path = "../../lib/dl_utils"}

ci/get_base_img_hash.sh

@@ -5,5 +5,5 @@
 set -eu
 
 export LC_ALL=C
-IMG_HASH_DOCKER_IMAGE_DIR="$(find $ROOT_DIR/metapkg  -type f -print0   | sort -z | xargs -0 sha1sum -z | sha1sum  | cut -d \  -f1)"
+IMG_HASH_DOCKER_IMAGE_DIR="$(find $ROOT_DIR/metapkg $ROOT_DIR/docker_build -type f -print0   | sort -z | xargs -0 sha1sum -z | sha1sum  | cut -d \  -f1)"
 echo "rebuild_flag:8:$IMG_HASH_DOCKER_IMAGE_DIR" | sha1sum | cut -d \  -f1

docker_build/target_dl_base_linux_w_db_bin_dependencies/Dockerfile

@@ -7,3 +7,7 @@ RUN chmod a+x /tmp/scripts/*.sh && \
     --exit-on-error \
     /tmp/scripts && \
     rm -rf /tmp/scripts
+
+# Configure libraries for MSSQL
+RUN cat /usr/share/tdsodbc/odbcinst.ini >> /etc/odbcinst.ini
+ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/odbc

kb/index.md

@@ -0,0 +1,7 @@
+# Datalens Backend Knowledge Base
+
+Welcome to the KB!
+
+Topics:
+- [Working with this KB](using_kb.md) (editor configuration)
+- [Repository tooling](tooling/index.md)

kb/test_embeds.md

@@ -0,0 +1,15 @@
+This is a file to test embedded diagrams. 
+
+
+``` plantuml
+@startuml
+digraph foo {
+  node [style=rounded]
+  node1 [shape=box]
+  node2 [fillcolor=yellow, style="rounded,filled", shape=diamond]
+  node3 [shape=record, label="{ a | b | c }"]
+
+  node1 -> node2 -> node3
+};
+@enduml
+```

kb/tooling/index.md

@@ -0,0 +1,8 @@
+# Repository Tooling
+
+This section is about the custom tooling available in this repo.
+
+- [task commands](task_commands.md) - a set of (`make`-like) shortcuts for various commands and scripts for repository management, development, testing, etc.
+- tools from `terrarium` ([README](../../terrarium/README.md)):
+  - [dl-git](../../terrarium/dl_gitmanager/README.md) - a wrapper for advanced git commands, for usage mainly in the CI workflow
+  - [dl-repo / dl-package](../../terrarium/dl_repmanager/README.md) - tools for managing and inspecting packages, their dependencies, meta-packages, etc.

kb/tooling/task_commands.md

@@ -0,0 +1,47 @@
+## Management tasks
+
+These commands require the ``taskfile`` tool. See [this page](https://taskfile.dev/installation/)
+for installation options.
+
+Running tasks:
+```
+task <task_name>
+```
+
+
+### Environment (`env:`)
+
+Working with the testing/development environment.
+
+- `task env:devenv`:
+  Create development/testing environment (run it from a package dir)
+- `task env:devenv-d`:
+  Create development/testing environment in detached mode (run it from a package dir)
+- `task env:ensure_venv`: Command to create virtual env for the mainrepo tools. 
+  It requires presence of .env in the mainrepo/tools.
+
+
+### Generation (`gen:`)
+
+Generating files to be used from the code.
+
+- `task gen:antlr`:
+  (Re-)generate ANTLR code files for formula
+- `task gen:i18n-po`:
+  Sync/generate `.po` files for package (run it from a package dir)
+- `task gen:i18n-binaries`:
+  Generate binary `.mo` files from `.po` files for package (run it from a package dir)
+
+
+### Code quality (`cq:`)
+
+Checking and fixing source files. 
+
+- `task cq:fix_changed`:
+  Apply all auto-fixes
+- `task cq:check_changed`:
+  Check for any non-conformity in code style/format/lint
+- `task cq:fix_dir -- {single dir}`:
+  Apply all auto-fixes to the given dir absolute path
+- `task cq:check_dir -- {single dir}`:
+  Check for any non-conformity in code style/format/lint in the given dir abs path

kb/using_kb.md

@@ -0,0 +1,27 @@
+# Working with the KB
+
+This KB is mostly `Markdown`, but it has embedded PlantUML charts.
+These usually don't work out-of-the-box and might require a little additional configuration.
+
+To check the correect rendering of embedded charts open [this file](test_embeds.md).
+If your editor supports this, and everything is configured correctly,
+you should see a rendered charts.
+
+Here are two options you can use to work with this KB
+
+## PyCharm
+
+- Install `graphviz`.
+- Install and enable the Markdown plugin in PyCharm.
+
+In theory this should be enough, but you may find that charts give you a rendering error
+about not finding "dot".
+In this case find the `dot` executable in your system and copy it to `/opt/local/bin/dot`.
+It should work now.
+
+## Obsidian
+
+- Install the `Obsidian` app
+- Install and enable the PlantUML plugin in Obsidian.
+  You might need to configure the path to the dot executable.
+- Open the kb folder as a vault

lib/dl_api_commons/pyproject.toml

@@ -18,8 +18,7 @@ marshmallow = ">=3.19.0"
 multidict = ">=4.0"
 opentracing = ">=2.4.0"
 python = ">=3.10, <3.12"
-raven = ">=6.10.0"
-sentry-sdk = ">=1.15.0"
+sentry-sdk = {version = ">=1.15.0", extras = ["flask"]}
 typing-extensions = ">=4.5.0"
 datalens-utils = {path = "../dl_utils"}
 datalens-constants = {path = "../dl_constants"}

lib/dl_api_lib/dl_api_lib/app/data_api/app.py

@@ -71,7 +71,6 @@
     RedisSentinelService,
     SingleHostSimpleRedisService,
 )
-from dl_core.utils import make_url
 
 
 LOGGER = logging.getLogger(__name__)
@@ -248,12 +247,7 @@ def create_app(
             if self._settings.CACHES_REDIS.MODE == RedisMode.single_host:
                 redis_server_single_host = SingleHostSimpleRedisService(
                     instance_kind=RedisInstanceKind.caches,
-                    url=make_url(
-                        protocol="rediss" if self._settings.CACHES_REDIS.SSL else "redis",
-                        host=self._settings.CACHES_REDIS.HOSTS[0],
-                        port=self._settings.CACHES_REDIS.PORT,
-                        path=str(self._settings.CACHES_REDIS.DB),
-                    ),
+                    url=self._settings.CACHES_REDIS.as_single_host_url(),
                     password=self._settings.CACHES_REDIS.PASSWORD,
                     ssl=self._settings.CACHES_REDIS.SSL,
                 )
@@ -278,12 +272,7 @@ def create_app(
             if self._settings.MUTATIONS_REDIS.MODE == RedisMode.single_host:
                 mutations_redis_server_single_host = SingleHostSimpleRedisService(
                     instance_kind=RedisInstanceKind.mutations,
-                    url=make_url(
-                        protocol="rediss" if self._settings.MUTATIONS_REDIS.SSL else "redis",
-                        host=self._settings.MUTATIONS_REDIS.HOSTS[0],
-                        port=self._settings.MUTATIONS_REDIS.PORT,
-                        path=str(self._settings.MUTATIONS_REDIS.DB),
-                    ),
+                    url=self._settings.MUTATIONS_REDIS.as_single_host_url(),
                     password=self._settings.MUTATIONS_REDIS.PASSWORD,
                     ssl=self._settings.MUTATIONS_REDIS.SSL,
                 )

lib/dl_api_lib/dl_api_lib/query/registry.py

@@ -1,6 +1,7 @@
 from typing import (
     Collection,
     Optional,
+    Sequence,
     Type,
 )
 
@@ -87,25 +88,42 @@ class MQMFactorySettingItem:
 _MQM_FACTORY_REGISTRY: dict[MQMFactoryKey, Type[MultiQueryMutatorFactoryBase]] = {}
 
 
+def _get_default_mqm_factory_cls() -> Type[MultiQueryMutatorFactoryBase]:
+    return DefaultMultiQueryMutatorFactory
+
+
 def get_multi_query_mutator_factory(
     query_proc_mode: QueryProcessingMode,
     backend_type: SourceBackendType,
     dialect: DialectCombo,
     result_schema: ResultSchema,
-) -> Optional[MultiQueryMutatorFactoryBase]:
-    factory_cls = _MQM_FACTORY_REGISTRY.get(
-        # First try with exact dialect
+) -> MultiQueryMutatorFactoryBase:
+    prioritized_keys = (
+        # First try with exact dialect and mode (exact match)
         MQMFactoryKey(query_proc_mode=query_proc_mode, backend_type=backend_type, dialect=dialect),
-        _MQM_FACTORY_REGISTRY.get(
-            # Then try without the dialect, just the backend
-            MQMFactoryKey(query_proc_mode=query_proc_mode, backend_type=backend_type, dialect=None),
-            DefaultMultiQueryMutatorFactory,  # If still nothing, then use the default
-        ),
+        # Now the fallbacks begin...
+        # Try without the dialect (all dialects within backend), just the backend and mode
+        MQMFactoryKey(query_proc_mode=query_proc_mode, backend_type=backend_type, dialect=None),
+        # Fall back to `basic` mode (but still within the backend type)
+        # First try with the specific dialect
+        MQMFactoryKey(query_proc_mode=QueryProcessingMode.basic, backend_type=backend_type, dialect=dialect),
+        # If still nothing, try without specifying the dialect (all dialects within backend)
+        MQMFactoryKey(query_proc_mode=QueryProcessingMode.basic, backend_type=backend_type, dialect=None),
     )
 
+    # Now iterate over all of these combinations IN THAT VERY ORDER(!)
+    factory_cls: Optional[Type[MultiQueryMutatorFactoryBase]] = None
+    for key in prioritized_keys:
+        factory_cls = _MQM_FACTORY_REGISTRY.get(key)
+        if factory_cls is not None:
+            break  # found something
+
     if factory_cls is None:
-        return None
+        # Not found for any of the combinations
+        # Use the ultimate default
+        factory_cls = _get_default_mqm_factory_cls()
 
+    assert factory_cls is not None
     return factory_cls(result_schema=result_schema)