diff --git a/pyproject.toml b/pyproject.toml index 06cdab66..0ee7cb7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "nbformat>=5.10.4,<6.0", "pip-requirements-parser>=32.0.1,<33.1", "tomli>=2.1.0,<3.0.0 ; python_version < '3.11'", + "typing-extensions>=4.12.2", ] [tool.hatch.version] @@ -107,6 +108,3 @@ strict = true [[tool.mypy.overrides]] module = ["dotty_dict.*", "pip_requirements_parser.*", "tomli.*"] ignore_missing_imports = true - -[tool.pytest.ini_options] -addopts = "--ignore=tests/manual" diff --git a/src/creosote/cli.py b/src/creosote/cli.py index 2d7a473e..81deff6a 100644 --- a/src/creosote/cli.py +++ b/src/creosote/cli.py @@ -1,4 +1,6 @@ import sys +from collections.abc import Sequence +from typing import Optional from loguru import logger @@ -7,17 +9,14 @@ from creosote.config import Features, fail_fast, parse_args -def main(args_=None): - args, default_config = parse_args(args_) +def main(args_: Optional[Sequence[str]] = None) -> int: + args = parse_args(args_) if fail_fast(args): return 1 formatters.configure_logger(verbose=args.verbose, format_=args.format) logger.debug(f"Creosote version: {__version__}") logger.debug(f"Command: creosote {' '.join(sys.argv[1:])}") - logger.debug( - f"Default configuration (may have loaded pyproject.toml): {default_config}" - ) logger.debug(f"Arguments: {args}") if args.features: diff --git a/src/creosote/config.py b/src/creosote/config.py index 7979bf4f..78b5aff9 100644 --- a/src/creosote/config.py +++ b/src/creosote/config.py @@ -2,13 +2,14 @@ import os import sys import typing +from collections.abc import Sequence from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import List, Literal +from typing import Literal, Optional, Union if sys.version_info >= (3, 11): - import tomllib + import tomllib # pyright: ignore[reportUnreachable] else: import tomli as tomllib @@ -25,15 +26,16 @@ class Config: the ``dest`` specified in ``add_argument``. """ + verbose: bool = False format: Literal["default", "no-color", "porcelain"] = "default" - paths: List[str] = field(default_factory=lambda: ["src"]) - sections: List[str] = field(default_factory=lambda: ["project.dependencies"]) - exclude_deps: List[str] = field(default_factory=list) + paths: list[str] = field(default_factory=lambda: ["src"]) + sections: list[str] = field(default_factory=lambda: ["project.dependencies"]) + exclude_deps: list[str] = field(default_factory=list) deps_file: str = "pyproject.toml" - venvs: List[str] = field( + venvs: list[str] = field( default_factory=lambda: [os.environ.get("VIRTUAL_ENV", ".venv")] ) - features: List[str] = field(default_factory=list) + features: list[str] = field(default_factory=list) class Features(Enum): @@ -53,24 +55,30 @@ class CustomAppendAction(argparse.Action): value when the argument is specified for the first time. """ - def __init__(self, option_strings, dest, nargs=None, **kwargs): + def __init__( # type: ignore[no-untyped-def] + self, + option_strings: list[str], + dest: str, + nargs: Optional[list[str]] = None, + **kwargs, # pyright: ignore[reportUnknownParameterType, reportMissingParameterType] + ): """Initialize the action.""" - self.called_times = 0 - super().__init__(option_strings, dest, **kwargs) + self.called_times: int = 0 + super().__init__(option_strings, dest, **kwargs) # pyright: ignore[reportUnknownArgumentType] - def __call__(self, parser, namespace, values, option_string=None): + def __call__(self, parser, namespace, values, option_string=None): # type: ignore[no-untyped-def] # pyright: ignore[reportMissingParameterType, reportImplicitOverride] """When the argument is specified on the commandline.""" - current_values = getattr(namespace, self.dest) + current_values = getattr(namespace, self.dest) # pyright: ignore[reportAny] if self.called_times == 0: current_values = [] - current_values.append(values) + _ = current_values.append(values) # pyright: ignore[reportUnknownMemberType] setattr(namespace, self.dest, current_values) self.called_times += 1 -def show_migration_message(): +def show_migration_message() -> None: """Show warning if you are using v2.x args with v3.x code.""" args = sys.argv[1:] @@ -79,13 +87,13 @@ def show_migration_message(): if outdated_arg in args: logger.error( "Creosote was updated to v3.x with breaking changes. " - "You need to update your CLI arguments. " - "See the migration guide at https://github.com/fredrikaverpil/creosote" + + "You need to update your CLI arguments. " + + "See the migration guide at https://github.com/fredrikaverpil/creosote" ) sys.exit(1) -def parse_args(args): +def parse_args(args: Optional[Sequence[str]]) -> Config: show_migration_message() defaults = load_defaults() @@ -97,13 +105,14 @@ def parse_args(args): ), formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument( + _ = parser.add_argument( "--verbose", dest="verbose", action="store_true", + default=defaults.verbose, help="increase output verbosity", ) - parser.add_argument( + _ = parser.add_argument( "-f", "--format", dest="format", @@ -111,7 +120,7 @@ def parse_args(args): default=defaults.format, help="output format", ) - parser.add_argument( + _ = parser.add_argument( "-V", "--version", dest="version", @@ -119,7 +128,7 @@ def parse_args(args): version=__version__, help="show version and exit", ) - parser.add_argument( + _ = parser.add_argument( "-p", "--path", dest="paths", @@ -128,7 +137,7 @@ def parse_args(args): default=defaults.paths, help="path(s) to Python source code to scan for imports", ) - parser.add_argument( + _ = parser.add_argument( "-s", "--section", dest="sections", @@ -137,7 +146,7 @@ def parse_args(args): default=defaults.sections, help="pyproject.toml section(s) to scan for dependencies", ) - parser.add_argument( + _ = parser.add_argument( "--exclude-dep", dest="exclude_deps", metavar="DEPENDENCY", @@ -145,7 +154,7 @@ def parse_args(args): default=defaults.exclude_deps, help="dependency(ies) to exclude from the scan", ) - parser.add_argument( + _ = parser.add_argument( "-d", "--deps-file", dest="deps_file", @@ -153,7 +162,7 @@ def parse_args(args): default=defaults.deps_file, help="path to the pyproject.toml or requirements[.txt|.in] file", ) - parser.add_argument( + _ = parser.add_argument( "-v", "--venv", dest="venvs", @@ -162,7 +171,7 @@ def parse_args(args): default=defaults.venvs, help="path(s) to the virtual environment (or site-packages)", ) - parser.add_argument( + _ = parser.add_argument( "--use-feature", dest="features", metavar="FEATURE", @@ -176,10 +185,10 @@ def parse_args(args): ) parsed_args = parser.parse_args(args) - return parsed_args, defaults + return Config(**vars(parsed_args)) # pyright: ignore[reportAny] -def load_defaults(src: str = "pyproject.toml") -> Config: +def load_defaults(src: Union[str, Path] = "pyproject.toml") -> Config: """Load pyproject.toml defaults from user config. Expects user configuration at ``[tool.creosote]``. @@ -190,13 +199,13 @@ def load_defaults(src: str = "pyproject.toml") -> Config: project_config = tomllib.load(f) except FileNotFoundError: project_config = {} - creosote_config = project_config.get("tool", {}).get("creosote", {}) + creosote_config = project_config.get("tool", {}).get("creosote", {}) # pyright: ignore[reportAny] # Convert all hyphens to underscores - creosote_config = {k.replace("-", "_"): v for k, v in creosote_config.items()} - return Config(**creosote_config) + creosote_config = {k.replace("-", "_"): v for k, v in creosote_config.items()} # pyright: ignore[reportAny] + return Config(**creosote_config) # pyright: ignore[reportAny] -def fail_fast(args: argparse.Namespace) -> bool: +def fail_fast(args: Config) -> bool: """Check if we should fail fast.""" if is_missing_file(args.deps_file): logger.error(f"File not found: {args.deps_file}") diff --git a/src/creosote/formatters.py b/src/creosote/formatters.py index ade899d2..7e65f368 100644 --- a/src/creosote/formatters.py +++ b/src/creosote/formatters.py @@ -1,5 +1,4 @@ import sys -from typing import List from loguru import logger @@ -8,10 +7,10 @@ def configure_logger(verbose: bool, format_: str) -> None: logger.remove() if format_ == "porcelain": - logger.add(sys.stderr, level="CRITICAL") + _ = logger.add(sys.stderr, level="CRITICAL") return if format_ == "no-color": - logger.add( + _ = logger.add( sys.stderr, level="DEBUG" if verbose else "INFO", colorize=False, @@ -19,7 +18,7 @@ def configure_logger(verbose: bool, format_: str) -> None: ) else: # default - logger.add( + _ = logger.add( sys.stderr, level="DEBUG" if verbose else "INFO", colorize=True, @@ -27,14 +26,14 @@ def configure_logger(verbose: bool, format_: str) -> None: ) -def print_results(unused_dependency_names: List[str], format_: str) -> None: +def print_results(unused_dependency_names: list[str], format_: str) -> None: if unused_dependency_names: if format_ == "porcelain": print("\n".join(unused_dependency_names)) else: logger.error( "Oh no, bloated venv! 🤢 🪣\n" - f"Unused dependencies found: {', '.join(unused_dependency_names)}" + + f"Unused dependencies found: {', '.join(unused_dependency_names)}" ) else: logger.info("No unused dependencies found! ✨") diff --git a/src/creosote/models.py b/src/creosote/models.py index a197bf88..a0d0f519 100644 --- a/src/creosote/models.py +++ b/src/creosote/models.py @@ -1,18 +1,18 @@ import dataclasses -from typing import List, Optional +from typing import Optional @dataclasses.dataclass class ImportInfo: - module: List[str] - name: List[str] + module: list[str] + name: list[str] alias: Optional[str] = None @dataclasses.dataclass class DependencyInfo: name: str # as defined in the dependencies specification file - top_level_import_names: Optional[List[str]] = None - record_import_names: Optional[List[str]] = None + top_level_import_names: Optional[list[str]] = None + record_import_names: Optional[list[str]] = None canonicalized_dep_name: Optional[str] = None - associated_imports: List[ImportInfo] = dataclasses.field(default_factory=list) + associated_imports: list[ImportInfo] = dataclasses.field(default_factory=list) diff --git a/src/creosote/parsers.py b/src/creosote/parsers.py index 3c39ecd7..723e9030 100644 --- a/src/creosote/parsers.py +++ b/src/creosote/parsers.py @@ -2,23 +2,71 @@ import re import sys import tempfile +from collections.abc import Generator from pathlib import Path -from typing import Any, Dict, Generator, List, Union, cast +from typing import Union, cast import nbformat +from typing_extensions import TypeGuard if sys.version_info >= (3, 11): - import tomllib + import tomllib # pyright: ignore[reportUnreachable] else: import tomli as tomllib -from dotty_dict import Dotty, dotty +import dotty_dict # pyright: ignore[reportMissingTypeStubs] from loguru import logger from nbconvert import PythonExporter -from pip_requirements_parser import RequirementsFile +from pip_requirements_parser import ( # pyright: ignore[reportMissingTypeStubs] + RequirementsFile, +) from creosote.models import ImportInfo +GroupName = str +PackageName = str +PEP621Type1 = list[PackageName] +PEP621Type2 = dict[GroupName, list[PackageName]] +PEP621Types = Union[PEP621Type1, PEP621Type2] +PEP735Type1 = dict[GroupName, list[PackageName]] +PEP735Type2 = dict[GroupName, list[Union[dict[str, PackageName], str]]] +PEP735Types = Union[PEP735Type1, PEP735Type2] +PoetryType1 = dict[PackageName, str] +PoetryType2 = dict[PackageName, dict[str, str]] +PoetryTypes = Union[PoetryType1, PoetryType2] +PipfileType = dict[PackageName, str] +AllSupportedTypes = Union[ + PEP621Type1, + PEP621Type2, + PEP735Type1, + PEP735Type2, + PoetryType1, + PoetryType2, + PipfileType, +] + + +def is_list_type(value: AllSupportedTypes) -> TypeGuard[PEP621Type1]: + return isinstance(value, list) + + +def is_dict_of_strings( + value: AllSupportedTypes, +) -> TypeGuard[Union[PoetryType1, PipfileType]]: + return isinstance(value, dict) and all(isinstance(v, str) for v in value.values()) + + +def is_dict_of_lists( + value: AllSupportedTypes, +) -> TypeGuard[Union[PEP621Type2, PEP735Type1]]: + return isinstance(value, dict) and all(isinstance(v, list) for v in value.values()) + + +def is_dict_of_dicts( + value: AllSupportedTypes, +) -> TypeGuard[PoetryType2]: + return isinstance(value, dict) and all(isinstance(v, dict) for v in value.values()) + class DependencyReader: """Read dependencies from various dependency file formats.""" @@ -26,24 +74,24 @@ class DependencyReader: def __init__( self, deps_file: str, - sections: List[str], - exclude_deps: List[str], + sections: list[str], + exclude_deps: list[str], ) -> None: always_excluded_deps = ["python"] # occurs in Poetry setup - self.deps_file = deps_file - self.sections = sections - self.exclude_deps = exclude_deps + always_excluded_deps + self.deps_file: str = deps_file + self.sections: list[str] = sections + self.exclude_deps: list[str] = exclude_deps + always_excluded_deps - def read(self) -> List[str]: + def read(self) -> list[str]: logger.debug(f"Parsing {self.deps_file} for dependencies...") if not Path(self.deps_file).exists(): raise Exception(f"File {self.deps_file} does not exist") - dep_names = [] - always_excluded_deps = ["python"] # occurs in Poetry setup - deps_to_exclude = always_excluded_deps + self.exclude_deps + dep_names: list[str] = [] + always_excluded_deps: list[str] = ["python"] # occurs in Poetry setup + deps_to_exclude: list[str] = always_excluded_deps + self.exclude_deps if self.deps_file.endswith(".toml") or self.deps_file.endswith( "Pipfile" @@ -60,32 +108,31 @@ def read(self) -> List[str]: f"Dependency specs file {self.deps_file} is not supported." ) - logger.info( - f"Found dependencies in {self.deps_file}: " f"{', '.join(dep_names)}" - ) + found = ", ".join(dep_names) + logger.info(f"Found dependencies in {self.deps_file}: {found}") return dep_names def get_deps_from_pep621_toml( - self, section_contents: Union[List[str], Dict[str, List[str]]] - ) -> List[str]: + self, section_contents: PEP621Types + ) -> list[PackageName]: """Get dependency names from toml file using the PEP621 spec. The dependency strings are expected to follow PEP508. """ - dep_strings = [] - if isinstance(section_contents, list): + dep_strings: list[str] = [] + if is_list_type(section_contents): for dep_string in section_contents: dep_strings.append(dep_string) - elif isinstance(section_contents, dict): + elif is_dict_of_lists(section_contents): for _, dep_string_list in section_contents.items(): for dep_string in dep_string_list: dep_strings.append(dep_string) else: raise TypeError("Unexpected dependency format, list expected.") - section_deps = [] + section_deps: list[PackageName] = [] for dep_string in dep_strings: parsed_dep = self.parse_dep_string(dep_string) if parsed_dep: @@ -96,25 +143,25 @@ def get_deps_from_pep621_toml( return section_deps def get_deps_from_pep735_toml( - self, section_contents: Union[List[str], Dict[str, List[str]]] - ) -> List[str]: + self, section_contents: PEP735Types + ) -> list[PackageName]: """Get dependency names from toml file using the PEP735 spec. The dependency strings are expected to follow PEP508. """ - dep_strings = [] - if isinstance(section_contents, dict): + dep_strings: list[str] = [] + if is_dict_of_lists(section_contents): for _, dep_string_list in section_contents.items(): for dep_string in dep_string_list: - if type(dep_string) != str: + if type(dep_string) is not str: logger.debug(f"Skipping non-string entry: {dep_string}") continue dep_strings.append(dep_string) else: raise TypeError("Unexpected dependency format, list expected.") - section_deps = [] + section_deps: list[PackageName] = [] for dep_string in dep_strings: parsed_dep = self.parse_dep_string(dep_string) if parsed_dep: @@ -126,48 +173,55 @@ def get_deps_from_pep735_toml( def get_deps_from_toml_section_keys( self, - section_contents: Dict[str, Any], - ): + section_contents: Union[PoetryTypes, PipfileType], + ) -> list[PackageName]: """Get dependency names from toml section's dict keys.""" - if not isinstance(section_contents, dict): - raise TypeError("Unexpected dependency format, dict expected.") - return section_contents.keys() + if is_dict_of_strings(section_contents) or is_dict_of_dicts(section_contents): + return list(section_contents.keys()) + + raise TypeError("Unexpected dependency format, dict expected.") - def read_toml(self, deps_file: str, sections: List[str]) -> List[str]: + def read_toml(self, deps_file: str, sections: list[str]) -> list[str]: """Read dependency names from toml spec file.""" with open(deps_file, "rb") as infile: contents = tomllib.load(infile) - dotty_contents: Dotty = dotty(contents) - dep_names = [] + dotty_contents = dotty_dict.dotty(contents) # pyright: ignore[reportUnknownMemberType] + dep_names: list[str] = [] for section in sections: try: - section_contents = dotty_contents[section] + section_contents = cast(AllSupportedTypes, dotty_contents[section]) except KeyError as err: raise KeyError(f"Could not find toml section {section}.") from err logger.debug(f"{sections}: {section_contents}") - section_dep_names = [] + section_dep_names: list[str] = [] if section.startswith("project."): logger.debug(f"Detected PEP-621 toml section in {deps_file}") - section_dep_names = self.get_deps_from_pep621_toml(section_contents) + section_dep_names = self.get_deps_from_pep621_toml( + cast(PEP621Types, section_contents) + ) elif section.startswith("tool.pdm"): logger.debug(f"Detected PDM toml section in {deps_file}") - section_dep_names = self.get_deps_from_pep621_toml(section_contents) + section_dep_names = self.get_deps_from_pep621_toml( + cast(PEP621Types, section_contents) + ) elif section.startswith("dependency-groups"): logger.debug(f"Detected PEP-735 toml section in {deps_file}") - section_dep_names = self.get_deps_from_pep735_toml(section_contents) + section_dep_names = self.get_deps_from_pep735_toml( + cast(PEP735Types, section_contents) + ) elif section.startswith("tool.poetry"): logger.debug(f"Detected Poetry toml section in {deps_file}") section_dep_names = self.get_deps_from_toml_section_keys( - cast(dict, section_contents) + cast(PoetryTypes, section_contents) ) elif section.startswith("packages") or section.startswith("dev-packages"): logger.debug(f"Detected pipenv/Pipfile toml section in {deps_file}") section_dep_names = self.get_deps_from_toml_section_keys( - cast(dict, section_contents) + cast(PipfileType, section_contents) ) else: raise TypeError("Unsupported dependency format.") @@ -179,7 +233,7 @@ def read_toml(self, deps_file: str, sections: List[str]) -> List[str]: return sorted(dep_names) - def read_requirements(self, deps_file: str) -> List[str]: + def read_requirements(self, deps_file: str) -> list[str]: """Read dependency names from requirements.txt-format file.""" dep_from_req = RequirementsFile.from_file(deps_file).requirements return sorted([dep.name for dep in dep_from_req if dep.name is not None]) @@ -229,14 +283,18 @@ def get_module_info_from_python_file(path: str) -> Generator[ImportInfo, None, N is_notebook = False if Path(path).suffix == ".ipynb": with open(path) as f: - notebook_content = nbformat.read(f, as_version=4) - + notebook_content = nbformat.read( # type: ignore[no-untyped-call] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] + f, + as_version=4, + ) # Convert the notebook to a temporary .py file - body, _ = PythonExporter().from_notebook_node(notebook_content) + body, _ = PythonExporter().from_notebook_node( # type: ignore[no-untyped-call] + notebook_content # pyright: ignore[reportUnknownArgumentType] + ) # delete_on_close parameter only supported in Python 3.12+ with tempfile.NamedTemporaryFile(delete=False, suffix=".py") as temp_file: - temp_file.write(body.encode("utf-8")) + _ = temp_file.write(body.encode("utf-8")) path = temp_file.name is_notebook = True @@ -267,9 +325,9 @@ def get_module_info_from_python_file(path: str) -> Generator[ImportInfo, None, N Path(path).unlink() -def get_module_names_from_code(paths: List[str]) -> List[ImportInfo]: - resolved_paths: List[Path] = [] - imports = [] +def get_module_names_from_code(paths: list[str]) -> list[ImportInfo]: + resolved_paths: list[Path] = [] + imports: list[ImportInfo] = [] for path in paths: if Path(path).is_dir(): @@ -283,7 +341,7 @@ def get_module_names_from_code(paths: List[str]) -> List[ImportInfo]: for import_info in get_module_info_from_python_file(path=str(resolved_path)): imports.append(import_info) - imports_with_dupes_removed = [] + imports_with_dupes_removed: list[ImportInfo] = [] for import_info in imports: if import_info not in imports_with_dupes_removed: imports_with_dupes_removed.append(import_info) @@ -295,17 +353,17 @@ def get_module_names_from_code(paths: List[str]) -> List[ImportInfo]: return imports_with_dupes_removed -def get_installed_dependency_names(venv: str) -> List[str]: - dep_names = [] +def get_installed_dependency_names(venv: str) -> list[str]: + dep_names: list[str] = [] for path in Path(venv).glob("**/*.dist-info"): dep_names.append(path.name.split("-")[0]) return dep_names def get_excluded_deps_which_are_not_installed( - excluded_deps: List[str], venvs: List[str] -) -> List[str]: - dependency_names: List[str] = [] + excluded_deps: list[str], venvs: list[str] +) -> list[str]: + dependency_names: list[str] = [] if not excluded_deps: return dependency_names @@ -323,7 +381,7 @@ def get_excluded_deps_which_are_not_installed( if dependency_names: logger.warning( "Excluded dependencies not found in virtual environment: " - f"{', '.join(dependency_names)}" + + f"{', '.join(dependency_names)}" ) return dependency_names diff --git a/src/creosote/py.typed b/src/creosote/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/creosote/resolvers.py b/src/creosote/resolvers.py index 4715ed15..baab63df 100644 --- a/src/creosote/resolvers.py +++ b/src/creosote/resolvers.py @@ -2,7 +2,6 @@ import pathlib import re from pathlib import Path -from typing import List from loguru import logger @@ -12,24 +11,26 @@ class DepsResolver: def __init__( self, - imports: List[ImportInfo], - dependency_names: List[str], - venvs: List[str], + imports: list[ImportInfo], + dependency_names: list[str], + venvs: list[str], ): - self.imports: List[ImportInfo] = imports - self.dependencies: List[DependencyInfo] = [ + self.imports: list[ImportInfo] = imports + self.dependencies: list[DependencyInfo] = [ DependencyInfo(name=dep) for dep in dependency_names ] - self.venvs: List[str] = venvs + self.venvs: list[str] = venvs - self.top_level_txt_pattern = re.compile( + self.top_level_txt_pattern: re.Pattern[str] = re.compile( r"\/([\w\.]*).[\d\.]*.dist-info\/top_level.txt" ) - self.record_pattern = re.compile(r"\/([\w\.]*).[\d\.]*.dist-info\/RECORD") + self.record_pattern: re.Pattern[str] = re.compile( + r"\/([\w\.]*).[\d\.]*.dist-info\/RECORD" + ) - self.top_level_filepaths: List[pathlib.Path] = [] - self.record_filepaths: List[pathlib.Path] = [] - self.unused_deps: List[DependencyInfo] = [] + self.top_level_filepaths: list[pathlib.Path] = [] + self.record_filepaths: list[pathlib.Path] = [] + self.unused_deps: list[DependencyInfo] = [] @staticmethod def canonicalize_module_name(module_name: str) -> str: @@ -42,7 +43,7 @@ def is_importable(self, module_name: str) -> bool: except ImportError: return False - def gather_filepaths(self, venv: str, glob_str: str) -> List[Path]: + def gather_filepaths(self, venv: str, glob_str: str) -> list[Path]: logger.debug(f"Gathering all top_level.txt files in venv {venv}...") venv_path = pathlib.Path(venv) filepaths = list(venv_path.glob(glob_str)) @@ -82,7 +83,9 @@ def map_dep_to_import_via_top_level_txt_file( for top_level_filepath in self.top_level_filepaths: normalized_top_level_filepath = top_level_filepath.as_posix() - matches = self.top_level_txt_pattern.findall(normalized_top_level_filepath) + matches: list[str] = self.top_level_txt_pattern.findall( + normalized_top_level_filepath + ) for name_from_top_level in matches: if ( self.canonicalize_module_name(name_from_top_level).lower() @@ -96,7 +99,7 @@ def map_dep_to_import_via_top_level_txt_file( import_names = ", ".join(dep_info.top_level_import_names) logger.debug( f"[{dep_info.name}] found import name(s) " - f"via top_level.txt: {import_names} ⭐️" + + f"via top_level.txt: {import_names} ⭐️" ) return True logger.debug(f"[{dep_info.name}] did not find dep in a top_level.txt file") @@ -105,7 +108,7 @@ def map_dep_to_import_via_top_level_txt_file( def map_dep_to_import_via_record_file(self, dep_info: DependencyInfo) -> bool: for record_filepath in self.record_filepaths: normalized_record_filepath = record_filepath.as_posix() - matches = self.record_pattern.findall(normalized_record_filepath) + matches: list[str] = self.record_pattern.findall(normalized_record_filepath) for name_from_record in matches: if ( self.canonicalize_module_name(name_from_record).lower() @@ -116,7 +119,7 @@ def map_dep_to_import_via_record_file(self, dep_info: DependencyInfo) -> bool: ) as infile: lines = infile.readlines() - import_names_found = [] + import_names_found: list[str] = [] for line in lines: candidate, _hash, _size = line.split(",") if candidate.endswith(".py") and "__init__" in candidate: @@ -129,7 +132,7 @@ def map_dep_to_import_via_record_file(self, dep_info: DependencyInfo) -> bool: import_names = ",".join(dep_info.record_import_names) logger.debug( f"[{dep_info.name}] found import name " - f"via RECORD: {import_names} ⭐️" + + f"via RECORD: {import_names} ⭐️" ) return True @@ -139,7 +142,7 @@ def map_dep_to_import_via_record_file(self, dep_info: DependencyInfo) -> bool: def map_dep_to_canonical_name(self, dep_info: DependencyInfo) -> str: return self.canonicalize_module_name(dep_info.name) - def populate_dependency_info(self): + def populate_dependency_info(self) -> None: """Populate DependencyInfo object with import naming info. There are three strategies from where the import name can be @@ -168,10 +171,12 @@ def populate_dependency_info(self): if not found_via_top_level_txt and not found_via_record: logger.debug( f"[{dep_info.name}] relying on canonicalization " - f"fallback: {dep_info.canonicalized_dep_name } 🤞" + + f"fallback: {dep_info.canonicalized_dep_name } 🤞" ) - def associate_dep_with_import(self, dep_info: DependencyInfo, import_name: str): + def associate_dep_with_import( + self, dep_info: DependencyInfo, import_name: str + ) -> None: for imp in self.imports.copy(): if not imp.module and import_name in imp.name: # import @@ -181,7 +186,7 @@ def associate_dep_with_import(self, dep_info: DependencyInfo, import_name: str): # from import ... dep_info.associated_imports.append(imp) - def resolve(self): + def resolve(self) -> None: """Associate dependency name with import (module) name. The AST has found imports from the source code. This function @@ -208,13 +213,13 @@ def get_unused_dependencies(self) -> None: if not dep_info.associated_imports ] - def resolve_unused_dependency_names(self) -> List[str]: + def resolve_unused_dependency_names(self) -> list[str]: for venv in self.venvs: if not Path(venv).exists(): logger.warning( f"Virtual environment(s) '{', '.join(self.venvs)}' does not exist, " - "cannot resolve top-level names. " - "This may lead to incorrect results." + + "cannot resolve top-level names. " + + "This may lead to incorrect results." ) self.gather_top_level_filepaths(venv=venv) self.gather_record_filepaths(venv=venv) @@ -225,7 +230,7 @@ def resolve_unused_dependency_names(self) -> List[str]: logger.debug( "Dependencies with populated 'associated_import' attribute are used in " - "code. End result of resolve:" + + "code. End result of resolve:" ) for dep_info in self.dependencies: logger.debug(f"- {dep_info}") diff --git a/tests/test_config_loading.py b/tests/test_config_loading.py index 120fa73b..a3addcf5 100644 --- a/tests/test_config_loading.py +++ b/tests/test_config_loading.py @@ -1,39 +1,40 @@ import os +from pathlib import Path from creosote import config -def test_load_defaults_no_file(tmp_path): +def test_load_defaults_no_file(tmp_path: Path) -> None: pyproject = tmp_path / "pyproject.toml" configuration = config.load_defaults(pyproject) assert configuration == config.Config() # Should return a default config -def test_load_defaults_no_tool_section(tmp_path): +def test_load_defaults_no_tool_section(tmp_path: Path) -> None: pyproject = tmp_path / "pyproject.toml" - pyproject.write_text("[foo]") + _ = pyproject.write_text("[foo]") configuration = config.load_defaults(pyproject) assert configuration == config.Config() # Should return a default config -def test_load_defaults_no_tool_creosote_section(tmp_path): +def test_load_defaults_no_tool_creosote_section(tmp_path: Path) -> None: pyproject = tmp_path / "pyproject.toml" - pyproject.write_text("[tool.foo]") + _ = pyproject.write_text("[tool.foo]") configuration = config.load_defaults(pyproject) assert configuration == config.Config() # Should return a default config -def test_load_defaults_tool_creosote_section_simple(tmp_path): +def test_load_defaults_tool_creosote_section_simple(tmp_path: Path) -> None: pyproject = tmp_path / "pyproject.toml" - pyproject.write_text('[tool.creosote]\nvenvs=["foo"]') + _ = pyproject.write_text('[tool.creosote]\nvenvs=["foo"]') configuration = config.load_defaults(pyproject) assert configuration == config.Config(venvs=["foo"]) -def test_load_defaults_tool_creosote_section_complex(tmp_path): +def test_load_defaults_tool_creosote_section_complex(tmp_path: Path) -> None: """More close to a real configuration.""" pyproject = tmp_path / "pyproject.toml" - pyproject.write_text( + _ = pyproject.write_text( """ [tool.creosote] venvs=[".virtual_environment"] @@ -57,14 +58,14 @@ def test_load_defaults_tool_creosote_section_complex(tmp_path): ) -def test_load_defaults_no_venv(): +def test_load_defaults_no_venv() -> None: # Unset VIRTUAL_ENV environment variable - os.environ.pop("VIRTUAL_ENV", None) + _ = os.environ.pop("VIRTUAL_ENV", None) configuration = config.Config() assert configuration.venvs == [".venv"] -def test_load_defaults_set_venv(): +def test_load_defaults_set_venv() -> None: os.environ["VIRTUAL_ENV"] = "foo" configuration = config.Config() assert configuration.venvs == ["foo"] diff --git a/tests/test_custom_append_action.py b/tests/test_custom_append_action.py index cc823d9d..2df6bc12 100644 --- a/tests/test_custom_append_action.py +++ b/tests/test_custom_append_action.py @@ -3,42 +3,42 @@ from creosote.config import CustomAppendAction -def test_custom_append_action_none_default(): +def test_custom_append_action_none_default() -> None: parser = argparse.ArgumentParser() - parser.add_argument("-f", "--foo", dest="foo", action=CustomAppendAction) + _ = parser.add_argument("-f", "--foo", dest="foo", action=CustomAppendAction) args = parser.parse_args(["-f", "1", "-f", "2"]) - assert args.foo == ["1", "2"] + assert args.foo == ["1", "2"] # pyright: ignore[reportAny] -def test_custom_append_action_list_default(): +def test_custom_append_action_list_default() -> None: parser = argparse.ArgumentParser() - parser.add_argument( + _ = parser.add_argument( "-f", "--foo", dest="foo", action=CustomAppendAction, default=["3"] ) args = parser.parse_args([]) - assert args.foo == ["3"] + assert args.foo == ["3"] # pyright: ignore[reportAny] -def test_custom_append_action_list_default_override(): +def test_custom_append_action_list_default_override() -> None: parser = argparse.ArgumentParser() - parser.add_argument( + _ = parser.add_argument( "-f", "--foo", dest="foo", action=CustomAppendAction, default=["3"] ) args = parser.parse_args(["-f", "1", "-f", "2"]) - assert args.foo == ["1", "2"] + assert args.foo == ["1", "2"] # pyright: ignore[reportAny] -def test_custom_append_action_list_set_defaults(): +def test_custom_append_action_list_set_defaults() -> None: parser = argparse.ArgumentParser() - parser.add_argument("-f", "--foo", dest="foo", action=CustomAppendAction) + _ = parser.add_argument("-f", "--foo", dest="foo", action=CustomAppendAction) parser.set_defaults(foo=["3"]) args = parser.parse_args([]) - assert args.foo == ["3"] + assert args.foo == ["3"] # pyright: ignore[reportAny] -def test_custom_append_action_list_set_defaults_override(): +def test_custom_append_action_list_set_defaults_override() -> None: parser = argparse.ArgumentParser() - parser.add_argument("-f", "--foo", dest="foo", action=CustomAppendAction) + _ = parser.add_argument("-f", "--foo", dest="foo", action=CustomAppendAction) parser.set_defaults(foo=["3"]) args = parser.parse_args(["-f", "1", "-f", "2"]) - assert args.foo == ["1", "2"] + assert args.foo == ["1", "2"] # pyright: ignore[reportAny] diff --git a/tests/test_dependency_reader.py b/tests/test_dependency_reader.py index 379fb548..70bf8369 100644 --- a/tests/test_dependency_reader.py +++ b/tests/test_dependency_reader.py @@ -1,5 +1,3 @@ -from typing import List - import pytest from creosote.parsers import DependencyReader @@ -27,7 +25,9 @@ ), ], ) -def test_read_toml_pep621(sections: List[str], expected_dependencies: List[str]): +def test_read_toml_pep621( + sections: list[str], expected_dependencies: list[str] +) -> None: reader = DependencyReader( deps_file="tests/deps_files/pyproject.pep621.toml", sections=sections, @@ -54,7 +54,9 @@ def test_read_toml_pep621(sections: List[str], expected_dependencies: List[str]) ), ], ) -def test_read_toml_pep735(sections: List[str], expected_dependencies: List[str]): +def test_read_toml_pep735( + sections: list[str], expected_dependencies: list[str] +) -> None: reader = DependencyReader( deps_file="tests/deps_files/pyproject.pep735.toml", sections=sections, @@ -81,7 +83,9 @@ def test_read_toml_pep735(sections: List[str], expected_dependencies: List[str]) ), ], ) -def test_read_toml_poetry(sections: List[str], expected_dependencies: List[str]): +def test_read_toml_poetry( + sections: list[str], expected_dependencies: list[str] +) -> None: reader = DependencyReader( deps_file="tests/deps_files/pyproject.poetry.toml", sections=sections, @@ -98,7 +102,9 @@ def test_read_toml_poetry(sections: List[str], expected_dependencies: List[str]) (["dev-packages"], ["pytest"]), ], ) -def test_read_toml_pipfile(sections: List[str], expected_dependencies: List[str]): +def test_read_toml_pipfile( + sections: list[str], expected_dependencies: list[str] +) -> None: reader = DependencyReader( deps_file="tests/deps_files/Pipfile", sections=sections, @@ -119,7 +125,9 @@ def test_read_toml_pipfile(sections: List[str], expected_dependencies: List[str] ("Qt_py==1.0.0", "Qt_py"), ], ) -def test_dependency_without_version_constraint(dependency_string, expected_package): +def test_dependency_without_version_constraint( + dependency_string: str, expected_package: str +) -> None: assert expected_package == DependencyReader.dependency_without_version_constraint( dependency_string ) @@ -134,7 +142,9 @@ def test_dependency_without_version_constraint(dependency_string, expected_packa ("pip @ git+ssh://git@github.com/pypa/pip.git", "pip"), ], ) -def test_pyproject_directref_package_name(dependency_string, expected_package): +def test_pyproject_directref_package_name( + dependency_string: str, expected_package: str +) -> None: assert expected_package == DependencyReader.dependency_without_direct_reference( dependency_string ) diff --git a/tests/test_integration.py b/tests/test_integration.py index e4707a6e..6b1154f6 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,5 +1,6 @@ """This file holds all integration tests. + The general idea: - Invoke the test by runing the CLI and provide arguments. - Assert on the output. @@ -12,7 +13,7 @@ """ -from typing import List +from typing import Any import pytest from _pytest.capture import CaptureFixture @@ -23,7 +24,7 @@ def test_creosote_project_success( venv_manager: VenvManager, - capsys: CaptureFixture, + capsys: CaptureFixture[Any], # pyright: ignore[reportExplicitAny] ) -> None: """Test running cresote on its own project, successfully.""" @@ -38,7 +39,7 @@ def test_creosote_project_success( "pip-requirements-parser", "tomli", ]: - venv_manager.create_record( + _ = venv_manager.create_record( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=[ @@ -68,7 +69,8 @@ def test_creosote_project_success( assert actual_output == [ "Found dependencies in pyproject.toml: " - "dotty-dict, loguru, nbconvert, nbformat, pip-requirements-parser", + + "dotty-dict, loguru, nbconvert, nbformat, pip-requirements-parser, " + + "typing-extensions", "No unused dependencies found! ✨", ] assert exit_code == 0 @@ -186,11 +188,11 @@ def test_creosote_project_success( @pytest.mark.parametrize(["scan_type"], [["RECORD"], ["top_level.txt"]]) def test_no_unused_dependencies_found( # noqa: PLR0913 venv_manager: VenvManager, - capsys: CaptureFixture, + capsys: CaptureFixture[Any], # pyright: ignore[reportExplicitAny] scan_type: str, deps_filename: str, toml_section: str, - deps_file_contents: List[str], + deps_file_contents: list[str], ) -> None: """Test simulated setup without any unused dependencies found.""" @@ -212,7 +214,7 @@ def test_no_unused_dependencies_found( # noqa: PLR0913 for dependency_name in installed_dependencies: if scan_type == "RECORD": - venv_manager.create_record( + _ = venv_manager.create_record( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=[ @@ -220,7 +222,7 @@ def test_no_unused_dependencies_found( # noqa: PLR0913 ], ) elif scan_type == "top_level.txt": - venv_manager.create_top_level_txt( + _ = venv_manager.create_top_level_txt( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=["yolo"], @@ -265,7 +267,7 @@ def test_no_unused_dependencies_found( # noqa: PLR0913 assert actual_output == [ f"Found dependencies in {deps_filepath}: " - "dotty-dict, loguru, pip-requirements-parser, toml", + + "dotty-dict, loguru, pip-requirements-parser, toml", "No unused dependencies found! ✨", ] assert exit_code == 0 @@ -392,11 +394,11 @@ def test_no_unused_dependencies_found( # noqa: PLR0913 @pytest.mark.parametrize(["unused_dep_is_installed"], [[False], [True]]) def test_one_unused_dependency_found( # noqa: PLR0913 venv_manager: VenvManager, - capsys: CaptureFixture, + capsys: CaptureFixture[Any], # pyright: ignore[reportExplicitAny] scan_type: str, deps_filename: str, toml_section: str, - deps_file_contents: List[str], + deps_file_contents: list[str], exclude_unused_dep: bool, unused_dep_is_installed: bool, ) -> None: @@ -429,7 +431,7 @@ def test_one_unused_dependency_found( # noqa: PLR0913 for dependency_name in installed_dependencies: if scan_type == "RECORD": - venv_manager.create_record( + _ = venv_manager.create_record( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=[ @@ -437,7 +439,7 @@ def test_one_unused_dependency_found( # noqa: PLR0913 ], ) elif scan_type == "top_level.txt": - venv_manager.create_top_level_txt( + _ = venv_manager.create_top_level_txt( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=["yolo"], @@ -502,7 +504,8 @@ def test_one_unused_dependency_found( # noqa: PLR0913 def test_repeated_arguments_are_accepted( - venv_manager: VenvManager, capsys: CaptureFixture + venv_manager: VenvManager, + capsys: CaptureFixture[Any], # pyright: ignore[reportExplicitAny] ) -> None: """The same argument is passed multiple times, when supported.""" @@ -546,7 +549,7 @@ def test_repeated_arguments_are_accepted( imports = deps_and_imports_map.values() for dependency_name in installed_dependencies: - venv_manager.create_record( + _ = venv_manager.create_record( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=[ @@ -598,8 +601,9 @@ def test_repeated_arguments_are_accepted( assert exit_code == 0 -def test_repeated_arguments_are_accepted( - venv_manager: VenvManager, capsys: CaptureFixture +def test_ruamel( + venv_manager: VenvManager, + capsys: CaptureFixture[Any], # pyright: ignore[reportExplicitAny] ) -> None: """Asserts that ruamel.yaml is supported.""" @@ -627,7 +631,7 @@ def test_repeated_arguments_are_accepted( imports = deps_and_imports_map.values() for dependency_name in installed_dependencies: - venv_manager.create_record( + _ = venv_manager.create_record( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=[ diff --git a/tests/test_integration_notebook.py b/tests/test_integration_notebook.py index 58e47ef3..f3ed5b2c 100644 --- a/tests/test_integration_notebook.py +++ b/tests/test_integration_notebook.py @@ -1,3 +1,5 @@ +from typing import Any + from _pytest.capture import CaptureFixture from creosote import cli @@ -6,8 +8,8 @@ def test_jupyter_ok( venv_manager: VenvManager, - capsys: CaptureFixture, -): + capsys: CaptureFixture[Any], # pyright: ignore[reportExplicitAny] +) -> None: venv_path, site_packages_path = venv_manager.create_venv() deps_filename = "pyproject.toml" @@ -26,7 +28,7 @@ def test_jupyter_ok( ] for dependency_name in installed_dependencies: - venv_manager.create_record( + _ = venv_manager.create_record( site_packages_path=site_packages_path, dependency_name=dependency_name, contents=[