Fixed terrarium/bi_ci mypy errors, updated terrarium checks action a …

…bit (#90)

* Fixed mypy errors, marked some tomlkit access with ignore

* Fixed GHA to run terrarium mypy tests

.github/workflows/terrarium_check.yaml

@@ -29,19 +29,18 @@ jobs:
           rm -rf ./.??* || true
       - name: Checkout code
         uses: actions/checkout@v4
-      - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
       - name: Setup common tools
-        run: pip install --no-input poetry pytest
+        run: pip install --no-input poetry
       - name: Install dependencies
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          pip install --no-input .
+          poetry -v install --with pytest --without mypy
       - name: Pytest
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          pytest .
+          poetry run pytest .
 
-  mypu:
+  mypy:
     runs-on: [ self-hosted, linux ]
     container:
       image: "python:3.11.0"
@@ -60,14 +59,13 @@ jobs:
           rm -rf ./.??* || true
       - name: Checkout code
         uses: actions/checkout@v4
-      - run: git config --global --add safe.directory /__w/${{ github.event.repository.name }}/${{ github.event.repository.name }}
       - name: Setup common tools
-        run: pip install --no-input poetry mypy
+        run: pip install --no-input poetry
       - name: Install dependencies
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          pip install --no-input .
+          poetry -v install --with mypy --without pytest
       - name: Mypy
         run: | 
           cd "terrarium/${{ matrix.value }}"
-          mypy .
+          poetry run mypy .

terrarium/bi_ci/bi_ci/detect_affected_packages.py

@@ -49,19 +49,18 @@ def gen_pkg_dirs(cfg: Config) -> Iterator[Path]:
                 yield item
 
 
-def get_reverse_dependencies(direct_dependency: dict[str, list[str]]):
+def get_reverse_dependencies(direct_dependency: dict[str, list[str]]) -> dict[str, list[str]]:
     reverse_ref = defaultdict(list)
     for pkg in direct_dependency:
         for dep in direct_dependency[pkg]:
             reverse_ref[dep].append(pkg)
     return reverse_ref
 
 
-def get_leafs(dependencies):
+def get_leafs(dependencies: dict[str, list[str]]) -> set[str]:
     all_values = []
     for deps in dependencies.values():
         all_values.extend(deps)
-    all_values = set(all_values)
     leafs = set(all_values) - set([k for k in dependencies.keys() if len(dependencies[k]) > 0])
     return leafs
 
@@ -133,8 +132,9 @@ def process(
         to_test = set()
 
         for pkg in direct_affected:
-            to_test.add(pkg.self_pkg_name)
-            to_test.update(affection_map.get(pkg.self_pkg_name, {}))
+            if pkg.self_pkg_name:
+                to_test.add(pkg.self_pkg_name)
+                to_test.update(affection_map.get(pkg.self_pkg_name, {}))
 
     return [pkg_by_ref[k] for k in to_test]
 

terrarium/bi_ci/bi_ci/execute_mypy_multi.py

@@ -17,7 +17,7 @@ def get_mypy_targets(pkg_dir: Path) -> list[str]:
     try:
         with open(pkg_dir / PYPROJECT_TOML) as fh:
             meta = tomlkit.load(fh)
-            return meta["datalens"]["meta"]["mypy"]["targets"]
+            return meta["datalens"]["meta"]["mypy"]["targets"]  # type: ignore
     except NonExistentKey:
         pass
 
@@ -33,10 +33,11 @@ def get_targets(root: Path) -> Iterable[str]:
 
 def main(root: Path, targets_file: Path = None) -> None:  # type: ignore
     # clize can't recognize type annotation "Optional"
+    paths: Iterable[str]
     if targets_file is not None:
-        paths: Iterable[str] = json.load(open(targets_file))
+        paths = json.load(open(targets_file))
     else:
-        paths: Iterable[str] = get_targets(root)
+        paths = get_targets(root)
     failed_list: list[str] = []
     mypy_cache_dir = Path("/tmp/mypy_cache")
     mypy_cache_dir.mkdir(exist_ok=True)

terrarium/bi_ci/bi_ci/pkg_ref.py

@@ -3,6 +3,7 @@
 
 import attrs
 import tomlkit
+from tomlkit import TOMLDocument
 
 
 @attrs.define(slots=False)
@@ -11,11 +12,11 @@ class PkgRef:
     full_path: Path
 
     @cached_property
-    def partial_parent_path(self):
+    def partial_parent_path(self) -> Path:
         return self.full_path.relative_to(self.root)
 
     @cached_property
-    def self_toml(self):
+    def self_toml(self) -> TOMLDocument | None:
         try:
             return tomlkit.load(open(self.full_path / "pyproject.toml"))
         except Exception as err:
@@ -30,11 +31,12 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) ->
         result = set()
         raw = {}
         if spec:
-            raw = dict(spec["tool"]["poetry"]["dependencies"])
+            # tomlkit & mypy very annoying together, hence a bunch of ignores
+            raw = dict(spec["tool"]["poetry"]["dependencies"])  # type: ignore
             for group in include_groups or []:
-                if "group" not in spec["tool"]["poetry"]:
+                if "group" not in spec["tool"]["poetry"]:  # type: ignore
                     continue
-                raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {}))
+                raw.update(spec["tool"]["poetry"]["group"].get(group, {}).get("dependencies", {}))  # type: ignore
 
         for name, specifier in raw.items():
             if isinstance(specifier, dict) and "path" in specifier:
@@ -43,16 +45,16 @@ def extract_local_requirements(self, include_groups: list[str] | None = None) ->
         return result
 
     @cached_property
-    def self_pkg_name(self):
+    def self_pkg_name(self) -> str | None:
         try:
-            return self.self_toml["tool"]["poetry"]["name"]
+            return self.self_toml["tool"]["poetry"]["name"]  # type: ignore
         except (KeyError, TypeError):
             return None
 
     @property
     def skip_test(self) -> bool:
         spec = self.self_toml
         try:
-            return spec["datalens_ci"]["skip_test"]
+            return spec["datalens_ci"]["skip_test"]  # type: ignore
         except (KeyError, TypeError):
             return False

terrarium/bi_ci/bi_ci/split_pytest_tasks.py

@@ -51,7 +51,7 @@ def split_tests(
 
         if pytest_targets:
             for section in pytest_targets.keys():
-                spec = toml_data["datalens"]["pytest"][section]
+                spec = pytest_targets.get("section", {})
                 # supporting only a single label for now
                 for category in spec.get("labels", []):
                     split_result[category].append((package_path, section))

terrarium/bi_ci/bi_ci_tests/unit/test_fix_ports_in_compose.py

@@ -4,7 +4,11 @@
 from bi_ci.fix_ports_in_compose import remove_ports_from_docker_compose
 
 
-def test_remove_ports_from_docker_compose(tmpdir, sample_compose_src, sample_compose_expected):
+def test_remove_ports_from_docker_compose(
+    tmpdir: Path,
+    sample_compose_src: str,
+    sample_compose_expected: Path,
+):
     tmp_src = Path(tmpdir) / "docker-compose.yml"
     tmp_dst = Path(tmpdir) / "docker-compose-modified.yml"
 

terrarium/bi_ci/pyproject.toml

@@ -30,6 +30,13 @@ clize = ">=5.0.0"
 poetry = ">=1.5.0"
 pyyaml = ">=6.0.1"
 
+[tool.poetry.group.pytest.dependencies]
+pytest = ">=7.4.3"
+
+[tool.poetry.group.mypy.dependencies]
+types_PyYAML = "*"
+mypy = ">= 1.7.0"
+
 [tool.poetry.scripts]
 detect-affected-packages = "bi_ci.detect_affected_packages:main"
 run-tests = "bi_ci.run_tests:runner_cli"
@@ -49,3 +56,9 @@ warn_unused_configs = true
 disallow_untyped_defs = true
 check_untyped_defs = true
 strict_optional = true
+exclude = [
+    "^bi_ci_tests/",  # TOML's double-quoted strings require escaping backslashes
+]
+[tool.black]
+line-length = 120
+target-version = ['py310']

terrarium/dl_repmanager/pyproject.toml

@@ -17,8 +17,12 @@ tomlkit = "==0.11.8"
 requests = ">=2.31.0"
 datalens-cli-tools = {path = "../dl_cli_tools"}
 
-[tool.poetry.group.tests.dependencies]
-pytest = ">=7.2.2"
+[tool.poetry.group.pytest.dependencies]
+pytest = ">=7.4.3"
+
+[tool.poetry.group.mypy.dependencies]
+types_PyYAML = "*"
+mypy = ">= 1.7.0"
 
 [build-system]
 requires = ["poetry-core"]