diff --git a/.github/workflows/terrarium_check.yaml b/.github/workflows/terrarium_check.yaml index 0a233e0c6..3c78869eb 100644 --- a/.github/workflows/terrarium_check.yaml +++ b/.github/workflows/terrarium_check.yaml @@ -29,19 +29,18 @@ jobs: rm -rf ./.??* || true - name: Checkout code uses: actions/checkout@v4 - - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - name: Setup common tools - run: pip install --no-input poetry pytest + run: pip install --no-input poetry - name: Install dependencies run: | cd "terrarium/${{ matrix.value }}" - pip install --no-input . + poetry -v install --with pytest --without mypy - name: Pytest run: | cd "terrarium/${{ matrix.value }}" - pytest . + poetry run pytest . - mypu: + mypy: runs-on: [ self-hosted, linux ] container: image: "python:3.11.0" @@ -60,14 +59,13 @@ jobs: rm -rf ./.??* || true - name: Checkout code uses: actions/checkout@v4 - - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }} - name: Setup common tools - run: pip install --no-input poetry mypy + run: pip install --no-input poetry - name: Install dependencies run: | cd "terrarium/${{ matrix.value }}" - pip install --no-input . + poetry -v install --with mypy --without pytest - name: Mypy run: | cd "terrarium/${{ matrix.value }}" - mypy . + poetry run mypy . diff --git a/terrarium/bi_ci/bi_ci/detect_affected_packages.py b/terrarium/bi_ci/bi_ci/detect_affected_packages.py index 6ccb8721b..3dc927b88 100644 --- a/terrarium/bi_ci/bi_ci/detect_affected_packages.py +++ b/terrarium/bi_ci/bi_ci/detect_affected_packages.py @@ -49,7 +49,7 @@ def gen_pkg_dirs(cfg: Config) -> Iterator[Path]: yield item -def get_reverse_dependencies(direct_dependency: dict[str, list[str]]): +def get_reverse_dependencies(direct_dependency: dict[str, list[str]]) -> dict[str, list[str]]: reverse_ref = defaultdict(list) for pkg in direct_dependency: for dep in direct_dependency[pkg]: @@ -57,11 +57,10 @@ def get_reverse_dependencies(direct_dependency: dict[str, list[str]]): return reverse_ref -def get_leafs(dependencies): +def get_leafs(dependencies: dict[str, list[str]]) -> set[str]: all_values = [] for deps in dependencies.values(): all_values.extend(deps) - all_values = set(all_values) leafs = set(all_values) - set([k for k in dependencies.keys() if len(dependencies[k]) > 0]) return leafs @@ -133,8 +132,9 @@ def process( to_test = set() for pkg in direct_affected: - to_test.add(pkg.self_pkg_name) - to_test.update(affection_map.get(pkg.self_pkg_name, {})) + if pkg.self_pkg_name: + to_test.add(pkg.self_pkg_name) + to_test.update(affection_map.get(pkg.self_pkg_name, {})) return [pkg_by_ref[k] for k in to_test] diff --git a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py index 1284fd710..50c90d2c8 100644 --- a/terrarium/bi_ci/bi_ci/execute_mypy_multi.py +++ b/terrarium/bi_ci/bi_ci/execute_mypy_multi.py @@ -17,7 +17,7 @@ def get_mypy_targets(pkg_dir: Path) -> list[str]: try: with open(pkg_dir / PYPROJECT_TOML) as fh: meta = tomlkit.load(fh) - return meta["datalens"]["meta"]["mypy"]["targets"] + return meta["datalens"]["meta"]["mypy"]["targets"] # type: ignore except NonExistentKey: pass @@ -33,10 +33,11 @@ def get_targets(root: Path) -> Iterable[str]: def main(root: Path, targets_file: Path = None) -> None: # type: ignore # clize can't recognize type annotation "Optional" + paths: Iterable[str] if targets_file is not None: - paths: Iterable[str] = json.load(open(targets_file)) + paths = json.load(open(targets_file)) else: - paths: Iterable[str] = get_targets(root) + paths = get_targets(root) failed_list: list[str] = [] mypy_cache_dir = Path("/tmp/mypy_cache") mypy_cache_dir.mkdir(exist_ok=True) diff --git a/terrarium/bi_ci/bi_ci/pkg_ref.py b/terrarium/bi_ci/bi_ci/pkg_ref.py index 35fd4be2d..d31183c77 100644 --- a/terrarium/bi_ci/bi_ci/pkg_ref.py +++ b/terrarium/bi_ci/bi_ci/pkg_ref.py @@ -3,6 +3,7 @@ import attrs import tomlkit +from tomlkit import TOMLDocument @attrs.define(slots=False) @@ -11,11 +12,11 @@ class PkgRef: full_path: Path @cached_property - def partial_parent_path(self): + def partial_parent_path(self) -> Path: return self.full_path.relative_to(self.root) @cached_property - def self_toml(self): + def self_toml(self) -> TOMLDocument | None: try: return tomlkit.load(open(self.full_path / "pyproject.toml")) except Exception as err: @@ -30,11 +31,12 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) -> result = set() raw = {} if spec: - raw = dict(spec["tool"]["poetry"]["dependencies"]) + # tomlkit & mypy very annoying together, hence a bunch of ignores + raw = dict(spec["tool"]["poetry"]["dependencies"]) # type: ignore for group in include_groups or []: - if "group" not in spec["tool"]["poetry"]: + if "group" not in spec["tool"]["poetry"]: # type: ignore continue - raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {})) + raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {})) # type: ignore for name, specifier in raw.items(): if isinstance(specifier, dict) and "path" in specifier: @@ -43,9 +45,9 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) -> return result @cached_property - def self_pkg_name(self): + def self_pkg_name(self) -> str | None: try: - return self.self_toml["tool"]["poetry"]["name"] + return self.self_toml["tool"]["poetry"]["name"] # type: ignore except (KeyError, TypeError): return None @@ -53,6 +55,6 @@ def self_pkg_name(self): def skip_test(self) -> bool: spec = self.self_toml try: - return spec["datalens_ci"]["skip_test"] + return spec["datalens_ci"]["skip_test"] # type: ignore except (KeyError, TypeError): return False diff --git a/terrarium/bi_ci/bi_ci/split_pytest_tasks.py b/terrarium/bi_ci/bi_ci/split_pytest_tasks.py index 22d31137a..23e7bb590 100644 --- a/terrarium/bi_ci/bi_ci/split_pytest_tasks.py +++ b/terrarium/bi_ci/bi_ci/split_pytest_tasks.py @@ -51,7 +51,7 @@ def split_tests( if pytest_targets: for section in pytest_targets.keys(): - spec = toml_data["datalens"]["pytest"][section] + spec = pytest_targets.get("section", {}) # supporting only a single label for now for category in spec.get("labels", []): split_result[category].append((package_path, section)) diff --git a/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py b/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py deleted file mode 100644 index 10cf3ad0a..000000000 --- a/terrarium/bi_ci/bi_ci_tests/unit/test_detect_affected.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_dummy(): - pass diff --git a/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py b/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py index 65e9aa407..9d35db23d 100644 --- a/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py +++ b/terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py @@ -4,7 +4,11 @@ from bi_ci.fix_ports_in_compose import remove_ports_from_docker_compose -def test_remove_ports_from_docker_compose(tmpdir, sample_compose_src, sample_compose_expected): +def test_remove_ports_from_docker_compose( + tmpdir: Path, + sample_compose_src: str, + sample_compose_expected: Path, +): tmp_src = Path(tmpdir) / "docker-compose.yml" tmp_dst = Path(tmpdir) / "docker-compose-modified.yml" diff --git a/terrarium/bi_ci/pyproject.toml b/terrarium/bi_ci/pyproject.toml index 93fab7c5e..06d046555 100644 --- a/terrarium/bi_ci/pyproject.toml +++ b/terrarium/bi_ci/pyproject.toml @@ -30,6 +30,13 @@ clize = ">=5.0.0" poetry = ">=1.5.0" pyyaml = ">=6.0.1" +[tool.poetry.group.pytest.dependencies] +pytest = ">=7.4.3" + +[tool.poetry.group.mypy.dependencies] +types_PyYAML = "*" +mypy = ">= 1.7.0" + [tool.poetry.scripts] detect-affected-packages = "bi_ci.detect_affected_packages:main" run-tests = "bi_ci.run_tests:runner_cli" @@ -49,3 +56,9 @@ warn_unused_configs = true disallow_untyped_defs = true check_untyped_defs = true strict_optional = true +exclude = [ + "^bi_ci_tests/", # TOML's double-quoted strings require escaping backslashes +] +[tool.black] +line-length = 120 +target-version = ['py310'] diff --git a/terrarium/dl_repmanager/pyproject.toml b/terrarium/dl_repmanager/pyproject.toml index bbfc17a52..fbb137955 100644 --- a/terrarium/dl_repmanager/pyproject.toml +++ b/terrarium/dl_repmanager/pyproject.toml @@ -17,8 +17,12 @@ tomlkit = "==0.11.8" requests = ">=2.31.0" datalens-cli-tools = {path = "../dl_cli_tools"} -[tool.poetry.group.tests.dependencies] -pytest = ">=7.2.2" +[tool.poetry.group.pytest.dependencies] +pytest = ">=7.4.3" + +[tool.poetry.group.mypy.dependencies] +types_PyYAML = "*" +mypy = ">= 1.7.0" [build-system] requires = ["poetry-core"]