diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3625bc..2de6080 100755 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,6 @@ default_install_hook_types: [pre-commit, commit-msg] default_stages: [commit, manual] +exclude: ^\{\{cookiecutter.repo_name\}\}/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -58,4 +59,3 @@ repos: stages: - commit - manual - exclude: \{\{cookiecutter.repo_name\}\}/ diff --git a/cookiecutter.json b/cookiecutter.json index 99da881..4aae013 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -5,5 +5,6 @@ "repo_url": "URL to repository", "env_name": "{{ cookiecutter.project_name.lower().replace(' ', '-') + '-env' }}", "install_jupyter": ["yes", "no"], + "linter_name": ["pylint", "ruff"], "cicd_configuration": ["none", "gitlab"] } diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 5249e0e..1a25930 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -106,25 +106,33 @@ def copy_chosen_files(self) -> None: shutil.copy2(src, dst) -def get_ci_cd_file_manager(ci_cd_options: str) -> ConditionalFileManager: +def get_conditional_file_manager(ci_cd_option: str, linter_option: str) -> ConditionalFileManager: template_root_dir = pathlib.Path.cwd() - temp_files_dir = template_root_dir.joinpath(".temp_ci_cd") - if ci_cd_options == "none": - manager = ConditionalFileManager( - temp_files_dir=temp_files_dir, - template_root_dir=template_root_dir, - relevant_paths_list=[], - ) - elif ci_cd_options == "gitlab": - manager = ConditionalFileManager( - temp_files_dir=temp_files_dir, - template_root_dir=template_root_dir, - relevant_paths_list=[".gitlab-ci.yml"], - ) - else: - raise NotImplementedError( - f"Option {ci_cd_options} is not implemented as ci_cd_file_manager" - ) + temp_files_dir = template_root_dir.joinpath(".conditional_files") + relevant_paths = [] + + match ci_cd_option: + case "gitlab": + relevant_paths.append(".gitlab-ci.yml") + case "none": + pass + case _: + raise NotImplementedError(f"Option {ci_cd_option} is not implemented as CI/CD pipeline") + + match linter_option: + case "pylint": + relevant_paths.append(".pylintrc") + case "ruff": + relevant_paths.append("ruff.toml") + case _: + raise NotImplementedError(f"Option {linter_option} is not implemented as Linter") + + manager = ConditionalFileManager( + temp_files_dir=temp_files_dir, + template_root_dir=template_root_dir, + relevant_paths_list=relevant_paths, + ) + return manager @@ -143,7 +151,10 @@ def get_ci_cd_file_manager(ci_cd_options: str) -> ConditionalFileManager: print("before") # setup ci/cd related files (if any) print("initializing") - CICD_FILE_MANAGER = get_ci_cd_file_manager(ci_cd_options="{{cookiecutter.cicd_configuration}}") + CICD_FILE_MANAGER = get_conditional_file_manager( + ci_cd_option="{{cookiecutter.cicd_configuration}}", + linter_option="{{cookiecutter.linter_name}}", + ) print("copying") CICD_FILE_MANAGER.copy_chosen_files() print("cleaning") diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5578bc8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +log_cli = true +log_cli_level = DEBUG +log_cli_format = %(asctime)s %(levelname)s %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S diff --git a/tests/test_cookiecutter_template.py b/tests/test_cookiecutter_template.py index 43503b9..0f8fbba 100644 --- a/tests/test_cookiecutter_template.py +++ b/tests/test_cookiecutter_template.py @@ -1,11 +1,15 @@ # pylint: disable=redefined-outer-name # standard library imports +import itertools +import json import os import pathlib +import random import uuid # third party imports import pytest +import yaml from cookiecutter.main import cookiecutter # local imports @@ -28,7 +32,33 @@ def template_environment(tmp_path, request): PACKAGE_MANAGER.remove_env(env_name=env_name) -def validate_base_project_files(env_dir): +def get_all_possible_configuration_permutations(n_samples: int | None = 5): + """ + Generates all possible configurations from cookiecutter.json for all elements where + the value is a list (i.e., user has >1 pre-defined option). By default, 5 random + permutations will be selected from all permutations and then tested afterwards. + :param number_of_envs: How many environments should be built/tested. Defaults to 5. Set to None to test exhaustively + :return: list of number_of_envs permutations that were randomly selected from all possible permutations. + """ + with open(f"{TEMPLATE_DIRECTORY}/cookiecutter.json", "r", encoding="utf-8") as f: + cookiecutter_config = json.load(f) + + option_fields = {key: val for key, val in cookiecutter_config.items() if isinstance(val, list)} + option_keys, option_vals = zip(*option_fields.items()) + all_permutations = [ + dict(zip(option_keys, permutation)) for permutation in itertools.product(*option_vals) + ] + + if n_samples is not None: + if n_samples > len(all_permutations): + return all_permutations + + return random.sample(all_permutations, n_samples) + + return all_permutations + + +def validate_base_project_files(env_dir: pathlib.Path): """ Validates that the environment directory was created and contains the expected files """ @@ -37,29 +67,96 @@ def validate_base_project_files(env_dir): expected_dir_path = env_dir.joinpath(expected_dir) assert expected_dir_path.is_dir(), f"Did not find dir: {expected_dir_path}" + # Linter & CI files checked separately + expected_files = [ ".commitlintrc.yaml", ".gitattributes", ".gitignore", ".pre-commit-config.yaml", ".prettierrc", - ".pylintrc", "check_commit_msgs.sh", "environment.yaml", "pyproject.toml", "README.md", ] + for expected_file in expected_files: expected_file_path = env_dir.joinpath(expected_file) assert expected_file_path.is_file(), f"Did not find file: {expected_file_path}" -def validate_gitlab_configuration(env_dir, expect_present=True): - file_path = env_dir.joinpath(".gitlab-ci.yml") - if expect_present: - assert file_path.is_file(), f"Did not find file: {file_path}" +def validate_python_environment(env_dir: pathlib.Path) -> list[str]: + with open(env_dir.joinpath("environment.yaml"), "r", encoding="utf-8") as f: + python_deps: list[str] = yaml.safe_load(f)["dependencies"] + + assert "python=3.10.9" in python_deps, "Did not find python=3.10.9 in environment.yaml" + + python_deps_noversion = [i.split("=")[0] for i in python_deps] + + return python_deps, python_deps_noversion + + +def validate_cicd_configuration(env_dir: pathlib.Path, cicd_configuration: str): + all_possible_cicd_configs = {"gitlab": ".gitlab-ci.yml"} + + if cicd_configuration == "none": + for fname in all_possible_cicd_configs.values(): + config_path = env_dir.joinpath(fname) + assert ( + not config_path.is_file() + ), f"Expected not to find cicd config {config_path} for {cicd_configuration}" else: - assert not file_path.is_file(), f"Expected not to find file: {file_path}" + try: + fname = all_possible_cicd_configs[cicd_configuration] + except KeyError: + raise NotImplementedError( # pylint: disable=W0707 + f"No test implemented for cicd for {cicd_configuration}" + ) + + config_path = env_dir.joinpath(fname) + assert ( + config_path.is_file() + ), f"Did not find cicd config {config_path} for {cicd_configuration}" + + +def validate_linter_configuration( + env_dir: pathlib.Path, python_packages: list[str], linter_name: str +): + match linter_name: + case "pylint": + config_name = ".pylintrc" + case "ruff": + config_name = "ruff.toml" + case _: + raise NotImplementedError(f"No test implemented for linter {linter_name}") + + file_path = env_dir.joinpath(config_name) + + assert ( + linter_name in python_packages + ), f"Did not find {linter_name} in environment.yaml but specified as linter" + assert file_path.is_file(), f"Did not find linter config: {file_path} for {linter_name}" + + +def validate_jupyter_configuration(python_packages: list[str], install_jupyter: str): + match install_jupyter: + case "yes": + assert ( + "jupyter" in python_packages + ), "install_jupyter == yes but jupyter not in environment.yaml" + assert ( + "nbqa" in python_packages + ), "install_jupyter == yes but nbqa not in environment.yaml" + case "no": + assert ( + not "jupyter" in python_packages + ), "install_jupyter == no but jupyter in environment.yaml" + assert ( + not "nbqa" in python_packages + ), "install_jupyter == no but nbqa in environment.yaml" + case _: + raise ValueError(f"{install_jupyter} is not an option for install_jupyter") def validate_pre_commit(env_dir, env_name): @@ -77,16 +174,23 @@ def validate_pre_commit(env_dir, env_name): @pytest.mark.parametrize( "template_environment", - [ - {}, - {"cicd_configuration": "gitlab"}, - ], + get_all_possible_configuration_permutations(n_samples=5), indirect=["template_environment"], ) def test_template(template_environment): env_dir, env_name, env_config = template_environment validate_base_project_files(env_dir) - validate_gitlab_configuration( - env_dir, expect_present=env_config.get("cicd_configuration") == "gitlab" + + _, python_packages = validate_python_environment(env_dir) + + validate_cicd_configuration(env_dir, cicd_configuration=env_config.get("cicd_configuration")) + + validate_linter_configuration( + env_dir, python_packages, linter_name=env_config.get("linter_name") ) + + validate_jupyter_configuration( + python_packages, install_jupyter=env_config.get("install_jupyter") + ) + validate_pre_commit(env_dir, env_name) diff --git a/{{cookiecutter.repo_name}}/.temp_ci_cd/.gitlab-ci.yml b/{{cookiecutter.repo_name}}/.conditional_files/.gitlab-ci.yml similarity index 100% rename from {{cookiecutter.repo_name}}/.temp_ci_cd/.gitlab-ci.yml rename to {{cookiecutter.repo_name}}/.conditional_files/.gitlab-ci.yml diff --git a/{{cookiecutter.repo_name}}/.pylintrc b/{{cookiecutter.repo_name}}/.conditional_files/.pylintrc similarity index 100% rename from {{cookiecutter.repo_name}}/.pylintrc rename to {{cookiecutter.repo_name}}/.conditional_files/.pylintrc diff --git a/{{cookiecutter.repo_name}}/.conditional_files/ruff.toml b/{{cookiecutter.repo_name}}/.conditional_files/ruff.toml new file mode 100644 index 0000000..4795d03 --- /dev/null +++ b/{{cookiecutter.repo_name}}/.conditional_files/ruff.toml @@ -0,0 +1,33 @@ +# sync with black +line-length = 100 + +# No need for NBQA as ruff has native Jupyter support +extend-include = ["*.ipynb"] + +# https://docs.astral.sh/ruff/rules/ +ignore = [ +# "E501", # Line too long, handled by black +# "W291", # Trailing whitespace, handled by black +# "W292", # Missing final newline, handled by black +# "PLR0904", # Too many public methods +# "PLR0911", # Too many return statements +# "PLR0913", # Too many arguments +# "PLC0415", # Import outside toplevel + # NA as checks yet but mentioned in github comments + # "C0305", # Trailing newlines, handled by black + # "C0114", # Missing module docstring + # "C0115", # Missing class docstring + # "C0116", # Missing function docstring + # "R0902", # Too many instance attributes + # "R0903", # Too few public methods + # "R0914", # Too many locals + # "W0124", # Confusing with statement + # "C0413", # Wrong import position + # "C0410", # Multiple imports + # "R1705", # No else return + # "W0201", # Attribute defined outside init + # "E1123", # Unexpected keyword arg + # "C0401", # Wrong spelling in comment + # "C0402", # Wrong spelling in docstring + # "C0403" # Invalid character in docstring +] diff --git a/{{cookiecutter.repo_name}}/.pre-commit-config.yaml b/{{cookiecutter.repo_name}}/.pre-commit-config.yaml index 6d42f1c..cd9e67a 100644 --- a/{{cookiecutter.repo_name}}/.pre-commit-config.yaml +++ b/{{cookiecutter.repo_name}}/.pre-commit-config.yaml @@ -32,6 +32,7 @@ repos: pass_filenames: true stages: - commit-msg + {%- if cookiecutter.linter_name == "pylint" %} - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: @@ -40,6 +41,12 @@ repos: stages: - commit - manual + {%- elif cookiecutter.linter_name == "ruff" %} + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.7 + hooks: + - id: ruff + {%- endif %} - repo: local hooks: - id: black @@ -48,11 +55,15 @@ repos: args: [--config=./pyproject.toml] language: system types: [python] + {%- if cookiecutter.linter_name == "pylint" %} - id: pylint name: pylint entry: pylint language: system types: [python] + {%- endif %} + {%- if cookiecutter.install_jupyter == "yes" %} + {%- if cookiecutter.linter_name == "pylint" %} - id: nbqa-pylint name: nbqa-pylint entry: nbqa pylint @@ -61,6 +72,14 @@ repos: args: - --disable=pointless-statement,duplicate-code,expression-not-assigned - --const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|([a-z_][a-z0-9_]{0,50}))$ + {%- elif cookiecutter.linter_name == "ruff" %} + - id: nbqa-ruff + name: nbqa-ruff + entry: nbqa ruff + language: system + files: \.ipynb + {%- endif %} + {%- endif %} - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.1.0 hooks: diff --git a/{{cookiecutter.repo_name}}/environment.yaml b/{{cookiecutter.repo_name}}/environment.yaml index 9c1b46c..80d1350 100644 --- a/{{cookiecutter.repo_name}}/environment.yaml +++ b/{{cookiecutter.repo_name}}/environment.yaml @@ -3,10 +3,14 @@ channels: - conda-forge dependencies: - black=24.4.2 - - nbqa=1.8.5 {%- if cookiecutter.install_jupyter == 'yes' %} - jupyter=1.0.0 + - nbqa=1.8.5 + {%- endif %} + {%- if cookiecutter.linter_name == "pylint" %} + - pylint=3.2.5 + {%- elif cookiecutter.linter_name == "ruff" %} + - ruff=0.6.7 {%- endif %} - python=3.10.9 - pre-commit=3.7.1 - - pylint=3.2.5 diff --git a/{{cookiecutter.repo_name}}/pyproject.toml b/{{cookiecutter.repo_name}}/pyproject.toml index 7a08402..32bc85c 100644 --- a/{{cookiecutter.repo_name}}/pyproject.toml +++ b/{{cookiecutter.repo_name}}/pyproject.toml @@ -17,6 +17,8 @@ exclude = ''' ) ''' +{% if cookiecutter.linter_name == "pylint" %} [tool.isort] -profile='black' +profile = 'black' line_length = 100 +{% endif %}