diff --git a/dl-repo.yml b/dl-repo.yml index e7fa29852..12f6abbd5 100644 --- a/dl-repo.yml +++ b/dl-repo.yml @@ -5,6 +5,9 @@ dl_repo: default_boilerplate_path: lib/dl_package_boilerplate + default_registration_metapackages: + - main_mpkg + package_types: - type: terrarium # Repository tools root_path: terrarium @@ -35,9 +38,17 @@ dl_repo: sentry_sdk: sentry-sdk snowflake: snowflake-connector-python - metapackages: [] + metapackages: + - name: main_mpkg + toml: metapkg/pyproject.toml plugins: + - type: toml_registration + config: + metapackages: + - name: main_mpkg + package_types: ["lib", "app"] + - type: dependency_registration edit_exclude_masks: # Files at paths that match these patterns (via re.match) will not be edited diff --git a/metapkg/pyproject.toml b/metapkg/pyproject.toml index a8b16a182..cd93f8635 100644 --- a/metapkg/pyproject.toml +++ b/metapkg/pyproject.toml @@ -131,12 +131,6 @@ datalens-file-uploader-worker-lib = {path = "../lib/dl_file_uploader_worker_lib" datalens-connector-bigquery = {path = "../lib/dl_connector_bigquery"} datalens-task-processor = {path = "../lib/dl_task_processor"} -# -# This section defines dependencies groups per app. -# To export/install specific app without other dependencies - use `--only` flag -# Group naming convention: `app_${APP_FOLDER_NAME}` -# - [tool.poetry.group.dev.dependencies] black = "==23.3.0" ruff = "==0.0.267" @@ -163,6 +157,18 @@ platformdirs = "==3.8.0" clize = "==5.0.0" poetry = "==1.5.0" +# +# This section defines dependencies groups per app. +# To export/install specific app without other dependencies - use `--only` flag +# Group naming convention: `app_${APP_FOLDER_NAME}` +# + +[tool.poetry.group.app_dl_os_control_api.dependencies] +datalens-control-api = {path = "../app/dl_control_api"} + +[tool.poetry.group.app_dl_os_data_api.dependencies] +datalens-data-api = {path = "../app/dl_data_api"} + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" @@ -340,9 +346,3 @@ secretstorage = {ignore = true} tzdata = {ignore = true} xattr = {ignore = true} appnope = {ignore = true} - -[tool.poetry.group.app_dl_os_control_api.dependencies] -datalens-control-api = {path = "../app/dl_control_api"} - -[tool.poetry.group.app_dl_os_data_api.dependencies] -datalens-data-api = {path = "../app/dl_data_api"} diff --git a/terrarium/dl_repmanager/dl_repmanager/management_plugins.py b/terrarium/dl_repmanager/dl_repmanager/management_plugins.py index 43bfe2bdf..96e8d5f4b 100644 --- a/terrarium/dl_repmanager/dl_repmanager/management_plugins.py +++ b/terrarium/dl_repmanager/dl_repmanager/management_plugins.py @@ -1,10 +1,12 @@ from __future__ import annotations import abc +import contextlib import os from pathlib import Path from typing import ( TYPE_CHECKING, + Generator, Type, ) @@ -87,68 +89,85 @@ def transform_package_list(old_text: str) -> str: @attr.s class MainTomlRepositoryManagementPlugin(RepositoryManagementPlugin): - _metapackage_name: str = attr.ib(init=False) - - @_metapackage_name.default - def _make_metapackage_name(self) -> str: - return self.config_data["metapackage"] - - @property - def _metapackage_path(self) -> Path: - return self.repository_env.get_metapackage_spec(self._metapackage_name).toml_path - - def _get_path_for_toml(self, package_info: PackageInfo) -> Path: - toml_abs_dir = (self.base_path / self._metapackage_path).parent + _metapackages_by_package_type: dict[str, list[str]] = attr.ib(init=False) + + @_metapackages_by_package_type.default + def _make_metapackages_by_package_type(self) -> dict[str, list[str]]: + result: dict[str, list[str]] = {} + for metapkg_config in self.config_data["metapackages"]: + for package_type in metapkg_config["package_types"]: + if package_type not in result: + result[package_type] = [] + result[package_type].append(metapkg_config["name"]) + return result + + def _get_metapackage_paths(self, package_type: str) -> list[Path]: + metapackage_names = self._metapackages_by_package_type.get(package_type, ()) + return [ + self.repository_env.get_metapackage_spec(metapackage_name).toml_path + for metapackage_name in metapackage_names + ] + + def _get_path_for_toml(self, metapackage_path: Path, package_info: PackageInfo) -> Path: + toml_abs_dir = (self.base_path / metapackage_path).parent return package_info.get_relative_path(toml_abs_dir) - def _register_main(self, toml_writer: TOMLWriter, package_info: PackageInfo) -> None: - package_path_for_toml = self._get_path_for_toml(package_info) + def _register_main(self, metapackage_path: Path, package_info: PackageInfo) -> None: + package_path_for_toml = self._get_path_for_toml(metapackage_path=metapackage_path, package_info=package_info) package_dep_table = tomlkit.inline_table() package_dep_table.add("path", str(package_path_for_toml)) - toml_writer.get_editable_section("tool.poetry.group.ci.dependencies").add( - package_info.package_reg_name, package_dep_table - ) - - def _unregister_main(self, toml_writer: TOMLWriter, package_info: PackageInfo): - with toml_writer.suppress_non_existent_key(): - toml_writer.get_editable_section("tool.poetry.dependencies").remove(package_info.package_reg_name) - with toml_writer.suppress_non_existent_key(): - toml_writer.get_editable_section("tool.poetry.group.dev.dependencies").remove(package_info.package_reg_name) - with toml_writer.suppress_non_existent_key(): - toml_writer.get_editable_section("tool.poetry.group.ci.dependencies").remove(package_info.package_reg_name) - - def _register_app(self, toml_writer: TOMLWriter, package_info: PackageInfo) -> None: - package_path_for_toml = self._get_path_for_toml(package_info) + with self._metapackage_toml_writer(metapackage_path=metapackage_path) as toml_writer: + section = toml_writer.get_editable_section("tool.poetry.group.ci.dependencies") + section[package_info.package_reg_name] = package_dep_table + + def _unregister_main(self, metapackage_path: Path, package_info: PackageInfo): + with self._metapackage_toml_writer(metapackage_path=metapackage_path) as toml_writer: + with toml_writer.suppress_non_existent_key(): + toml_writer.get_editable_section("tool.poetry.dependencies").remove(package_info.package_reg_name) + with toml_writer.suppress_non_existent_key(): + toml_writer.get_editable_section("tool.poetry.group.dev.dependencies").remove( + package_info.package_reg_name + ) + with toml_writer.suppress_non_existent_key(): + toml_writer.get_editable_section("tool.poetry.group.ci.dependencies").remove( + package_info.package_reg_name + ) + + def _register_app(self, metapackage_path: Path, package_info: PackageInfo) -> None: + package_path_for_toml = self._get_path_for_toml(metapackage_path=metapackage_path, package_info=package_info) package_base_name = package_info.abs_path.name package_dep_table = tomlkit.inline_table() package_dep_table.add("path", str(package_path_for_toml)) - section = toml_writer.add_section(f"tool.poetry.group.app_{package_base_name}.dependencies") - section.add(package_info.package_reg_name, package_dep_table) - section.add(tomlkit.nl()) + with self._metapackage_toml_writer(metapackage_path=metapackage_path) as toml_writer: + section = toml_writer.add_section(f"tool.poetry.group.app_{package_base_name}.dependencies") + section.add(package_info.package_reg_name, package_dep_table) + section.add(tomlkit.nl()) - def _unregister_app(self, toml_writer: TOMLWriter, package_info: PackageInfo) -> None: + def _unregister_app(self, metapackage_path: Path, package_info: PackageInfo) -> None: package_base_name = package_info.abs_path.name - toml_writer.delete_section(f"tool.poetry.group.app_{package_base_name}.dependencies") - - def register_package(self, package_info: PackageInfo) -> None: - toml_path = self.base_path / self._metapackage_path + with self._metapackage_toml_writer(metapackage_path=metapackage_path) as toml_writer: + toml_writer.delete_section(f"tool.poetry.group.app_{package_base_name}.dependencies") + @contextlib.contextmanager + def _metapackage_toml_writer(self, metapackage_path: Path) -> Generator[TOMLWriter, None, None]: + toml_path = self.base_path / metapackage_path toml_io_factory = TOMLIOFactory(fs_editor=self.fs_editor) with toml_io_factory.toml_writer(toml_path) as toml_writer: + yield toml_writer + + def register_package(self, package_info: PackageInfo) -> None: + for metapackage_path in self._get_metapackage_paths(package_type=package_info.package_type): if "main_dependency_group" in self.repository_env.get_tags(package_info.package_type): - self._register_main(toml_writer=toml_writer, package_info=package_info) + self._register_main(metapackage_path=metapackage_path, package_info=package_info) if "own_dependency_group" in self.repository_env.get_tags(package_info.package_type): - self._register_app(toml_writer=toml_writer, package_info=package_info) + self._register_app(metapackage_path=metapackage_path, package_info=package_info) def unregister_package(self, package_info: PackageInfo) -> None: - toml_path = self.base_path / self._metapackage_path - - toml_io_factory = TOMLIOFactory(fs_editor=self.fs_editor) - with toml_io_factory.toml_writer(toml_path) as toml_writer: + for metapackage_path in self._get_metapackage_paths(package_type=package_info.package_type): if "main_dependency_group" in self.repository_env.get_tags(package_info.package_type): - self._unregister_main(toml_writer=toml_writer, package_info=package_info) + self._unregister_main(metapackage_path=metapackage_path, package_info=package_info) if "own_dependency_group" in self.repository_env.get_tags(package_info.package_type): - self._unregister_app(toml_writer=toml_writer, package_info=package_info) + self._unregister_app(metapackage_path=metapackage_path, package_info=package_info) @attr.s diff --git a/terrarium/dl_repmanager/dl_repmanager/repository_env.py b/terrarium/dl_repmanager/dl_repmanager/repository_env.py index 2c0306dca..f8a32a658 100644 --- a/terrarium/dl_repmanager/dl_repmanager/repository_env.py +++ b/terrarium/dl_repmanager/dl_repmanager/repository_env.py @@ -144,13 +144,6 @@ def get_metapackage_spec(self, metapackage_name: str) -> MetaPackageSpec: def get_edit_exclude_masks(self) -> frozenset[re.Pattern]: return self.edit_exclude_masks - return tuple( - re.compile(pattern_str) - for pattern_str in ( - r".*\.mo", - r".*\.xlsx", - ) - ) _DEFAULT_FS_EDITOR_TYPE = "default"