From a739c2c06159cc0ca8a87d707c73feaa65162030 Mon Sep 17 00:00:00 2001 From: Valentin Gologuzov Date: Sun, 12 Nov 2023 00:41:21 +0100 Subject: [PATCH] Fixed mypy errors, marked some tomlkit access with ignore --- .../bi_ci/bi_ci/detect_affected_packages.py | 10 +++++----- terrarium/bi_ci/bi_ci/execute_mypy_multi.py | 7 ++++--- terrarium/bi_ci/bi_ci/pkg_ref.py | 18 ++++++++++-------- terrarium/bi_ci/bi_ci/split_pytest_tasks.py | 2 +- .../bi_ci_tests/unit/test_detect_affected.py | 2 -- .../unit/test_fix_ports_in_compose.py | 6 +++++- terrarium/bi_ci/pyproject.toml | 9 +++++++++ 7 files changed, 34 insertions(+), 20 deletions(-) delete mode 100644 terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py diff --git a/terrarium/bi_ci/bi_ci/detect_affected_packages.py b/terrarium/bi_ci/bi_ci/detect_affected_packages.py index 6ccb8721b..3dc927b88 100644 --- a/terrarium/bi_ci/bi_ci/detect_affected_packages.py +++ b/terrarium/bi_ci/bi_ci/detect_affected_packages.py @@ -49,7 +49,7 @@ def gen_pkg_dirs(cfg: Config) -> Iterator[Path]: yield item -def get_reverse_dependencies(direct_dependency: dict[str, list[str]]): +def get_reverse_dependencies(direct_dependency: dict[str, list[str]]) -> dict[str, list[str]]: reverse_ref = defaultdict(list) for pkg in direct_dependency: for dep in direct_dependency[pkg]: @@ -57,11 +57,10 @@ def get_reverse_dependencies(direct_dependency: dict[str, list[str]]): return reverse_ref -def get_leafs(dependencies): +def get_leafs(dependencies: dict[str, list[str]]) -> set[str]: all_values = [] for deps in dependencies.values(): all_values.extend(deps) - all_values = set(all_values) leafs = set(all_values) - set([k for k in dependencies.keys() if len(dependencies[k]) > 0]) return leafs @@ -133,8 +132,9 @@ def process( to_test = set() for pkg in direct_affected: - to_test.add(pkg.self_pkg_name) - to_test.update(affection_map.get(pkg.self_pkg_name, {})) + if pkg.self_pkg_name: + to_test.add(pkg.self_pkg_name) + to_test.update(affection_map.get(pkg.self_pkg_name, {})) return [pkg_by_ref[k] for k in to_test] diff --git a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py index 1284fd710..50c90d2c8 100644 --- a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py +++ b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py @@ -17,7 +17,7 @@ def get_mypy_targets(pkg_dir: Path) -> list[str]: try: with open(pkg_dir / PYPROJECT_TOML) as fh: meta = tomlkit.load(fh) - return meta["datalens"]["meta"]["mypy"]["targets"] + return meta["datalens"]["meta"]["mypy"]["targets"] # type: ignore except NonExistentKey: pass @@ -33,10 +33,11 @@ def get_targets(root: Path) -> Iterable[str]: def main(root: Path, targets_file: Path = None) -> None: # type: ignore # clize can't recognize type annotation "Optional" + paths: Iterable[str] if targets_file is not None: - paths: Iterable[str] = json.load(open(targets_file)) + paths = json.load(open(targets_file)) else: - paths: Iterable[str] = get_targets(root) + paths = get_targets(root) failed_list: list[str] = [] mypy_cache_dir = Path("/tmp/mypy_cache") mypy_cache_dir.mkdir(exist_ok=True) diff --git a/terrarium/bi_ci/bi_ci/pkg_ref.py b/terrarium/bi_ci/bi_ci/pkg_ref.py index 35fd4be2d..d31183c77 100644 --- a/terrarium/bi_ci/bi_ci/pkg_ref.py +++ b/terrarium/bi_ci/bi_ci/pkg_ref.py @@ -3,6 +3,7 @@ import attrs import tomlkit +from tomlkit import TOMLDocument @attrs.define(slots=False) @@ -11,11 +12,11 @@ class PkgRef: full_path: Path @cached_property - def partial_parent_path(self): + def partial_parent_path(self) -> Path: return self.full_path.relative_to(self.root) @cached_property - def self_toml(self): + def self_toml(self) -> TOMLDocument | None: try: return tomlkit.load(open(self.full_path / "pyproject.toml")) except Exception as err: @@ -30,11 +31,12 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) -> result = set() raw = {} if spec: - raw = dict(spec["tool"]["poetry"]["dependencies"]) + # tomlkit & mypy very annoying together, hence a bunch of ignores + raw = dict(spec["tool"]["poetry"]["dependencies"]) # type: ignore for group in include_groups or []: - if "group" not in spec["tool"]["poetry"]: + if "group" not in spec["tool"]["poetry"]: # type: ignore continue - raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {})) + raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {})) # type: ignore for name, specifier in raw.items(): if isinstance(specifier, dict) and "path" in specifier: @@ -43,9 +45,9 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) -> return result @cached_property - def self_pkg_name(self): + def self_pkg_name(self) -> str | None: try: - return self.self_toml["tool"]["poetry"]["name"] + return self.self_toml["tool"]["poetry"]["name"] # type: ignore except (KeyError, TypeError): return None @@ -53,6 +55,6 @@ def self_pkg_name(self): def skip_test(self) -> bool: spec = self.self_toml try: - return spec["datalens_ci"]["skip_test"] + return spec["datalens_ci"]["skip_test"] # type: ignore except (KeyError, TypeError): return False diff --git a/terrarium/bi_ci/bi_ci/split_pytest_tasks.py b/terrarium/bi_ci/bi_ci/split_pytest_tasks.py index 22d31137a..23e7bb590 100644 --- a/terrarium/bi_ci/bi_ci/split_pytest_tasks.py +++ b/terrarium/bi_ci/bi_ci/split_pytest_tasks.py @@ -51,7 +51,7 @@ def split_tests( if pytest_targets: for section in pytest_targets.keys(): - spec = toml_data["datalens"]["pytest"][section] + spec = pytest_targets.get("section", {}) # supporting only a single label for now for category in spec.get("labels", []): split_result[category].append((package_path, section)) diff --git a/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py b/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py deleted file mode 100644 index 10cf3ad0a..000000000 --- a/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_dummy(): - pass diff --git a/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py b/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py index 65e9aa407..9d35db23d 100644 --- a/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py +++ b/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py @@ -4,7 +4,11 @@ from bi_ci.fix_ports_in_compose import remove_ports_from_docker_compose -def test_remove_ports_from_docker_compose(tmpdir, sample_compose_src, sample_compose_expected): +def test_remove_ports_from_docker_compose( + tmpdir: Path, + sample_compose_src: str, + sample_compose_expected: Path, +): tmp_src = Path(tmpdir) / "docker-compose.yml" tmp_dst = Path(tmpdir) / "docker-compose-modified.yml" diff --git a/terrarium/bi_ci/pyproject.toml b/terrarium/bi_ci/pyproject.toml index 93fab7c5e..8ad8abda0 100644 --- a/terrarium/bi_ci/pyproject.toml +++ b/terrarium/bi_ci/pyproject.toml @@ -30,6 +30,9 @@ clize = ">=5.0.0" poetry = ">=1.5.0" pyyaml = ">=6.0.1" +[tool.poetry.group.mypy.dependencies] +types_PyYAML = "*" + [tool.poetry.scripts] detect-affected-packages = "bi_ci.detect_affected_packages:main" run-tests = "bi_ci.run_tests:runner_cli" @@ -49,3 +52,9 @@ warn_unused_configs = true disallow_untyped_defs = true check_untyped_defs = true strict_optional = true +exclude = [ + "^bi_ci_tests/", # TOML's double-quoted strings require escaping backslashes +] +[tool.black] +line-length = 120 +target-version = ['py310']