From caab92a0b779647f2fa7482f261ffcfd95552593 Mon Sep 17 00:00:00 2001 From: Nick Proskurin <42863572+MCPN@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:06:12 +0100 Subject: [PATCH 1/4] Add parameters tests (#107) --- .../db/data_api/result/test_parameters.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py diff --git a/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py new file mode 100644 index 000000000..e95b1fd75 --- /dev/null +++ b/lib/dl_api_lib/dl_api_lib_tests/db/data_api/result/test_parameters.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from http import HTTPStatus + +import pytest + +from dl_api_client.dsmaker.primitives import ( + IntegerParameterValue, + RangeParameterValueConstraint, +) +from dl_api_client.dsmaker.shortcuts.dataset import ( + add_formulas_to_dataset, + add_parameters_to_dataset, +) +from dl_api_client.dsmaker.shortcuts.result_data import get_data_rows +from dl_api_lib_tests.db.base import DefaultApiTestBase +from dl_constants.enums import UserDataType + + +class TestParameters(DefaultApiTestBase): + @pytest.mark.parametrize( + ("multiplier", "expected_status_code"), + ( + (None, HTTPStatus.OK), + (2, HTTPStatus.OK), + (5, HTTPStatus.OK), + (-1, HTTPStatus.BAD_REQUEST), + ), + ) + def test_parameter_in_formula(self, control_api, data_api, saved_dataset, multiplier, expected_status_code): + default_multiplier = 1 + ds = add_parameters_to_dataset( + api_v1=control_api, + dataset_id=saved_dataset.id, + parameters={ + "Multiplier": ( + IntegerParameterValue(default_multiplier), + RangeParameterValueConstraint(min=IntegerParameterValue(default_multiplier)), + ), + }, + ) + + integer_field = next(field for field in saved_dataset.result_schema if field.data_type == UserDataType.integer) + ds = add_formulas_to_dataset( + api_v1=control_api, + dataset=ds, + formulas={ + "Multiplied Field": f"[{integer_field.title}] * [Multiplier]", + }, + ) + + result_resp = data_api.get_result( + dataset=ds, + fields=[ + integer_field, + ds.find_field(title="Multiplier"), + ds.find_field(title="Multiplied Field"), + ], + parameters=[ + ds.find_field(title="Multiplier").parameter_value(multiplier), + ], + fail_ok=True, + ) + assert result_resp.status_code == expected_status_code, result_resp.json + + if expected_status_code == HTTPStatus.OK: + data_rows = get_data_rows(result_resp) + assert data_rows + for row in data_rows: + assert int(row[1]) == (multiplier or default_multiplier) + assert int(row[0]) * int(row[1]) == int(row[2]) + + def test_parameter_no_constraint(self, control_api, data_api, dataset_id): + ds = add_parameters_to_dataset( + api_v1=control_api, + dataset_id=dataset_id, + parameters={ + "Param": (IntegerParameterValue(0), None), + }, + ) + ds = add_formulas_to_dataset( + api_v1=control_api, + dataset=ds, + formulas={ + "Value": "[Param]", + }, + ) + + result_resp = data_api.get_result( + dataset=ds, + fields=[ + ds.find_field(title="Value"), + ], + parameters=[ + ds.find_field(title="Param").parameter_value(1), + ], + limit=1, + ) + assert result_resp.status_code == HTTPStatus.OK, result_resp.json + assert int(get_data_rows(result_resp)[0][0]) == 1 From cb5945009b030c3ea1f1a1acdd375e0c039c7726 Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 17 Nov 2023 14:23:18 +0100 Subject: [PATCH 2/4] Added description of tool usage to repmanager (#105) --- terrarium/dl_gitmanager/README.md | 8 +- terrarium/dl_repmanager/README.md | 252 +++++++++++++++++++++++++++++- 2 files changed, 255 insertions(+), 5 deletions(-) diff --git a/terrarium/dl_gitmanager/README.md b/terrarium/dl_gitmanager/README.md index 1249c964a..faf30712d 100644 --- a/terrarium/dl_gitmanager/README.md +++ b/terrarium/dl_gitmanager/README.md @@ -14,13 +14,13 @@ pip install -Ue Show the main help message -``` +```bash dl-git --help dl-git --h ``` The `--help` (`-h`) option can also be used for any command: -``` +```bash dl-git --help ``` @@ -43,7 +43,7 @@ If the path is inside a submodule, then the submodule is considered to be the ro List files that have changed between two given revisions including the changes in all submodules -``` +```bash dl-git range-diff-paths --base --head dl-git range-diff-paths --base --head --absolute dl-git range-diff-paths --base --head --only-added-commits @@ -53,7 +53,7 @@ Here `base` and `head` can be a commit ID, branch name, `HEAD~3` or any similar that is usually accepted by git. These arguments are optional. By default `head` is `HEAD` and `base` is `HEAD~1`. Thgis means you can use the following command to see the diff of the last commit in the current branch: -``` +```bash dl-git range-diff-paths ``` diff --git a/terrarium/dl_repmanager/README.md b/terrarium/dl_repmanager/README.md index 1333ed77b..958818c15 100644 --- a/terrarium/dl_repmanager/README.md +++ b/terrarium/dl_repmanager/README.md @@ -1 +1,251 @@ -TODO +# dl_repmanager + +Package containing tools for package/repo management. + + +## Installation + +```bash +pip install -Ue +``` + + +# `dl-package` + +Tool for working with metadata of individual packages. + + +## Commands + + +### list-i18n-domains + +List i18n domains of a package + +```bash +dl-package --package-path list-i18n-domains + +# example: +dl-package --package-path lib/dl_connector_clickhouse list-i18n-domains +``` +Will print something like: +``` +dl_connector_clickhouse=dl_connector_clickhouse/api;dl_connector_clickhouse/core +dl_formula_ref_dl_connector_clickhouse=dl_connector_clickhouse/formula_ref +``` + + +### set-meta-array + +Set array property in package's `pyproject.toml` +```bash +dl-package --package-path set-meta-array --toml-section --toml-key --toml-value + +# example: +dl-package --package-path lib/dl_core set-meta-array --toml-section tool.poetry --toml-key authors --toml-value "Alicia ";James " +``` + + +### set-meta-text + +Set text property in package's `pyproject.toml` +```bash +dl-package --package-path set-meta-text --toml-section --toml-key --toml-value + +# example: +dl-package --package-path lib/dl_core set-meta-text --toml-section tool.poetry --toml-key license --toml-value "Apache 2.0" +``` + + +# `dl-repo` + +Tool for managing and inspecting packages as part of the repository and the repository as a whole. +This includes creation, renaming, moving of packages, etc. with updating all the necessary dependencies and registries. + + +## Common options + + +### --help + +Show the main help message or help for a specific command + +``` +dl-repo --help +dl-repo --h +dl-repo -h +``` + + +### --config + +Optional. Path to the repository configuration file (`dl-repo.yml`). +By default the tool will discover the config automatically going up from the CWD. + +See [dl-repo.yml](../../dl-repo.yml). + +```bash +dl-repo --config /my/custom/repo-config.yml ... +``` + + +### --fs-editor + +Controls how FS operations are performed + +Possible values: +- `default` +- `git` +- `virtual` + +The default is set in the repo config ([dl-repo.yml](../../dl-repo.yml)). If not, the default is `default`. + +```bash +dl-repo --fs-editor git ... +``` + + +## Commands + + +### ch-package-type + +Change the type of (essentially, move) a package. + +```bash +dl-repo ch-package-type --package-name --package-type + +# example: +dl-repo ch-package-type --package-name dl_core --package-type app +``` + +The `--package-name` argument is the name of the package to move. This argument is required. + +The `--package-type` argument specifies the name of the package's new type, which maps to a ceertain folder in the repo. +See the repo config ([dl-repo.yml](../../dl-repo.yml)). This argument is required. + + +### compare-resulting-deps + +*TODO* + + +### copy + +Create a new package as a copy of an existing one. + +```bash +dl-repo copy --package-name --from-package-name + +# example: +dl-repo copy --package-name dl_super_package --from-package-name dl_core +``` + +The `--package-name` argument is the name of the new package's folder and module name. This argument is required. + +The `--from-package-name` argument of the package to copy (to use as the boilerplate). This argument is required. + + +### ensure-mypy-common + +*TODO* + + +### import-list + +*TODO* + + +### init + +Create a new package + +```bash +dl-repo init --package-type --package-name + +# example: +dl-repo init --package-type lib --package-name dl_super_package +``` + +The `--package-type` argument specifies where the package will be created. Package types are configured +in the repo config ([dl-repo.yml](../../dl-repo.yml)). This argument is required. + +The `--package-name` argument is the name of the new package's folder and module name. This argument is required. + + +### package-list + +*TODO* + + +### recurse-packages + +*TODO* + + +### rename + +Rename a package. + +```bash +dl-repo rename --package-name --new-package-name + +# example: +dl-repo rename --package-name dl_core --new-package-name dl_ultra_core +``` + +The `--package-name` argument is the original name of the package to rename. This argument is required. + +The `--new-package-name` argument Is the new name of the package. + + +### rename-module + +*TODO* + + +### req-check + +Check whether the requirements of a package are consistent with imports in the package's code. + +```bash +dl-repo req-check --package-name +dl-repo req-check --package-name --test +``` + +The `--package-name` argument is the original name of the package to check. + +The `--tests` flag tells the tool to check test requirements instead of the main ones. + + +### req-list + +List requirements of a package + +```bash +dl-repo req-list --package-name +``` + + +### resolve + +*TODO* + + +### rm + +Remove a package. + +```bash +dl-repo rm --package-name + +# example: +dl-repo rm --package-name dl_core +``` + +The `--package-name` argument is the name of the package to remove. This argument is required. + + +### search-imports + +*TODO* From dc144799e272ef45c0b1089d48978cb5cb2142fb Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 17 Nov 2023 14:35:02 +0100 Subject: [PATCH 3/4] Knowledge Base Begins (#103) * Knowledge Base Begins * Minor fixes for KB initial commit * Some more minor fixes for KB initial commit --- .gitignore | 1 + README.md | 41 +++++++------------------------- kb/index.md | 6 +++++ kb/test_embeds.md | 15 ++++++++++++ kb/tooling/index.md | 8 +++++++ kb/tooling/task_commands.md | 47 +++++++++++++++++++++++++++++++++++++ kb/using_kb.md | 26 ++++++++++++++++++++ 7 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 kb/index.md create mode 100644 kb/test_embeds.md create mode 100644 kb/tooling/index.md create mode 100644 kb/tooling/task_commands.md create mode 100644 kb/using_kb.md diff --git a/.gitignore b/.gitignore index 1ab2f3221..43c6d713b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ ci_artifacts .DS_Store artifacts Taskfile.yml +.obsidian diff --git a/README.md b/README.md index 966a18acd..f001b2204 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,15 @@ -## Management tasks +# datalens-backend -Running tasks: -``` -task -``` +## About +This is the repository for the back-end implementation of DataLens -### Environment (`env:`) +Head over to the [Knowledge Base](kb/index.md) for documentation on this repo. -- `task env:devenv`: - Create development/testing environment (run it from a package dir) -- `task env:devenv-d`: - Create development/testing environment in detached mode (run it from a package dir) -- `task env:ensure_venv`: Command to create virtual env for the mainrepo tools. - It requires presence of .env in the mainrepo/tools. +[Code of conduct](CODE_OF_CONDUCT.md) +[Contributing](CONTRIBUTING.md) -### Generation (`gen:`) +## License -- `task gen:antlr`: - (Re-)generate ANTLR code files for formula -- `task gen:i18n-po`: - Sync/generate `.po` files for package (run it from a package dir) -- `task gen:i18n-binaries`: - Generate binary `.mo` files from `.po` files for package (run it from a package dir) - - -### Code quality (`cq:`) - -Experimental tasks to check and fix source files. - -- `task cq:fix_changed`: - Apply all auto-fixes -- `task cq:check_changed`: - Check for any non-conformity in code style/format/lint -- `task cq:fix_dir -- {single dir}`: - Apply all auto-fixes to the given dir absolute path -- `task cq:check_dir -- {single dir}`: - Check for any non-conformity in code style/format/lint in the given dir abs path +`datalens-backend` is available under the Apache 2.0 license. diff --git a/kb/index.md b/kb/index.md new file mode 100644 index 000000000..4d64a2fb6 --- /dev/null +++ b/kb/index.md @@ -0,0 +1,6 @@ +# Datalens Backend Knowledge Base + +Welcome to the KB! + +See: +- [Repository tooling](tooling/index.md) diff --git a/kb/test_embeds.md b/kb/test_embeds.md new file mode 100644 index 000000000..32e1f21b9 --- /dev/null +++ b/kb/test_embeds.md @@ -0,0 +1,15 @@ +This is a file to test embedded diagrams. + + +``` plantuml +@startuml +digraph foo { + node [style=rounded] + node1 [shape=box] + node2 [fillcolor=yellow, style="rounded,filled", shape=diamond] + node3 [shape=record, label="{ a | b | c }"] + + node1 -> node2 -> node3 +}; +@enduml +``` diff --git a/kb/tooling/index.md b/kb/tooling/index.md new file mode 100644 index 000000000..b36917a8f --- /dev/null +++ b/kb/tooling/index.md @@ -0,0 +1,8 @@ +# Repository Tooling + +This section is about the custom tooling available in this repo. + +- [task commands](task_commands.md) - a set of (`make`-like) shortcuts for various commands and scripts for repository management, development, testing, etc. +- tools from `terrarium` ([README](../../terrarium/README.md)): + - [dl-git](../../terrarium/dl_gitmanager/README.md) - a wrapper for advanced git commands, for usage mainly in the CI workflow + - [dl-repo / dl-package](../../terrarium/dl_repmanager/README.md) - tools for managing and inspecting packages, their dependencies, meta-packages, etc. diff --git a/kb/tooling/task_commands.md b/kb/tooling/task_commands.md new file mode 100644 index 000000000..bd7697999 --- /dev/null +++ b/kb/tooling/task_commands.md @@ -0,0 +1,47 @@ +## Management tasks + +These commands require the ``taskfile`` tool. See [this page](https://taskfile.dev/installation/) +for installation options. + +Running tasks: +``` +task +``` + + +### Environment (`env:`) + +Working with the testing/development environment. + +- `task env:devenv`: + Create development/testing environment (run it from a package dir) +- `task env:devenv-d`: + Create development/testing environment in detached mode (run it from a package dir) +- `task env:ensure_venv`: Command to create virtual env for the mainrepo tools. + It requires presence of .env in the mainrepo/tools. + + +### Generation (`gen:`) + +Generating files to be used from the code. + +- `task gen:antlr`: + (Re-)generate ANTLR code files for formula +- `task gen:i18n-po`: + Sync/generate `.po` files for package (run it from a package dir) +- `task gen:i18n-binaries`: + Generate binary `.mo` files from `.po` files for package (run it from a package dir) + + +### Code quality (`cq:`) + +Checking and fixing source files. + +- `task cq:fix_changed`: + Apply all auto-fixes +- `task cq:check_changed`: + Check for any non-conformity in code style/format/lint +- `task cq:fix_dir -- {single dir}`: + Apply all auto-fixes to the given dir absolute path +- `task cq:check_dir -- {single dir}`: + Check for any non-conformity in code style/format/lint in the given dir abs path diff --git a/kb/using_kb.md b/kb/using_kb.md new file mode 100644 index 000000000..a160ec945 --- /dev/null +++ b/kb/using_kb.md @@ -0,0 +1,26 @@ +# Working with the KB + +This KB is mostly `Markdown`, but it has embedded charts. +These usually don't work out-of-the-box and might require a little additional configuration. + +To check the correect rendering of embedded charts open [this file](test_embeds.md). +If your editor supports this, and everything is configured correctly, +you should see a rendered charts. + +Here are two options you can use to work with this KB + +## PyCharm + +- Install `graphviz`. +- Install and enable the Markdown plugin in PyCharm. + +In theory this should be enough, but you may find that charts give you a rendering error +about not finding "dot". +In this case find the `dot` executable in your system and copy it to `/opt/local/bin/dot`. +It should work now. + +## Obsidian + +- Install the `Obsidian` app +- Install the PlantUML plugin in Obsidian. + You might need to configure the path to the dot executable. From 15854329cdf27f39107fde071b0e485d0dd88cfc Mon Sep 17 00:00:00 2001 From: Grigory Statsenko Date: Fri, 17 Nov 2023 14:46:02 +0100 Subject: [PATCH 4/4] Minor updates for KB (#108) --- kb/index.md | 3 ++- kb/using_kb.md | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/kb/index.md b/kb/index.md index 4d64a2fb6..0cd32f282 100644 --- a/kb/index.md +++ b/kb/index.md @@ -2,5 +2,6 @@ Welcome to the KB! -See: +Topics: +- [Working with this KB](using_kb.md) (editor configuration) - [Repository tooling](tooling/index.md) diff --git a/kb/using_kb.md b/kb/using_kb.md index a160ec945..709cc60bd 100644 --- a/kb/using_kb.md +++ b/kb/using_kb.md @@ -1,6 +1,6 @@ # Working with the KB -This KB is mostly `Markdown`, but it has embedded charts. +This KB is mostly `Markdown`, but it has embedded PlantUML charts. These usually don't work out-of-the-box and might require a little additional configuration. To check the correect rendering of embedded charts open [this file](test_embeds.md). @@ -22,5 +22,6 @@ It should work now. ## Obsidian - Install the `Obsidian` app -- Install the PlantUML plugin in Obsidian. +- Install and enable the PlantUML plugin in Obsidian. You might need to configure the path to the dot executable. +- Open the kb folder as a vault