Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rewrite it in Rust: Ruff as alternative Linter #28

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
default_install_hook_types: [pre-commit, commit-msg]
default_stages: [commit, manual]
exclude: ^\{\{cookiecutter.repo_name\}\}/
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.6.0
Expand Down Expand Up @@ -58,4 +59,3 @@ repos:
stages:
- commit
- manual
exclude: \{\{cookiecutter.repo_name\}\}/
1 change: 1 addition & 0 deletions cookiecutter.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,6 @@
"repo_url": "URL to repository",
"env_name": "{{ cookiecutter.project_name.lower().replace(' ', '-') + '-env' }}",
"install_jupyter": ["yes", "no"],
"linter_name": ["pylint", "ruff"],
"cicd_configuration": ["none", "gitlab"]
}
49 changes: 30 additions & 19 deletions hooks/post_gen_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,25 +106,33 @@ def copy_chosen_files(self) -> None:
shutil.copy2(src, dst)


def get_ci_cd_file_manager(ci_cd_options: str) -> ConditionalFileManager:
def get_conditional_file_manager(ci_cd_option: str, linter_option: str) -> ConditionalFileManager:
template_root_dir = pathlib.Path.cwd()
temp_files_dir = template_root_dir.joinpath(".temp_ci_cd")
if ci_cd_options == "none":
manager = ConditionalFileManager(
temp_files_dir=temp_files_dir,
template_root_dir=template_root_dir,
relevant_paths_list=[],
)
elif ci_cd_options == "gitlab":
manager = ConditionalFileManager(
temp_files_dir=temp_files_dir,
template_root_dir=template_root_dir,
relevant_paths_list=[".gitlab-ci.yml"],
)
else:
raise NotImplementedError(
f"Option {ci_cd_options} is not implemented as ci_cd_file_manager"
)
temp_files_dir = template_root_dir.joinpath(".conditional_files")
relevant_paths = []

match ci_cd_option:
case "gitlab":
relevant_paths.append(".gitlab-ci.yml")
case "none":
pass
case _:
raise NotImplementedError(f"Option {ci_cd_option} is not implemented as CI/CD pipeline")

match linter_option:
case "pylint":
relevant_paths.append(".pylintrc")
case "ruff":
relevant_paths.append("ruff.toml")
case _:
raise NotImplementedError(f"Option {linter_option} is not implemented as Linter")

manager = ConditionalFileManager(
temp_files_dir=temp_files_dir,
template_root_dir=template_root_dir,
relevant_paths_list=relevant_paths,
)

return manager


Expand All @@ -143,7 +151,10 @@ def get_ci_cd_file_manager(ci_cd_options: str) -> ConditionalFileManager:
print("before")
# setup ci/cd related files (if any)
print("initializing")
CICD_FILE_MANAGER = get_ci_cd_file_manager(ci_cd_options="{{cookiecutter.cicd_configuration}}")
CICD_FILE_MANAGER = get_conditional_file_manager(
ci_cd_option="{{cookiecutter.cicd_configuration}}",
linter_option="{{cookiecutter.linter_name}}",
)
print("copying")
CICD_FILE_MANAGER.copy_chosen_files()
print("cleaning")
Expand Down
5 changes: 5 additions & 0 deletions pytest.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[pytest]
log_cli = true
log_cli_level = DEBUG
log_cli_format = %(asctime)s %(levelname)s %(message)s
log_cli_date_format = %Y-%m-%d %H:%M:%S
130 changes: 117 additions & 13 deletions tests/test_cookiecutter_template.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
# pylint: disable=redefined-outer-name
# standard library imports
import itertools
import json
import os
import pathlib
import random
import uuid

# third party imports
import pytest
import yaml
from cookiecutter.main import cookiecutter

# local imports
Expand All @@ -28,7 +32,33 @@ def template_environment(tmp_path, request):
PACKAGE_MANAGER.remove_env(env_name=env_name)


def validate_base_project_files(env_dir):
def get_all_possible_configuration_permutations(n_samples: int | None = 5):
"""
Generates all possible configurations from cookiecutter.json for all elements where
the value is a list (i.e., user has >1 pre-defined option). By default, 5 random
permutations will be selected from all permutations and then tested afterwards.
:param number_of_envs: How many environments should be built/tested. Defaults to 5. Set to None to test exhaustively
:return: list of number_of_envs permutations that were randomly selected from all possible permutations.
"""
with open(f"{TEMPLATE_DIRECTORY}/cookiecutter.json", "r", encoding="utf-8") as f:
cookiecutter_config = json.load(f)

option_fields = {key: val for key, val in cookiecutter_config.items() if isinstance(val, list)}
option_keys, option_vals = zip(*option_fields.items())
all_permutations = [
dict(zip(option_keys, permutation)) for permutation in itertools.product(*option_vals)
]

if n_samples is not None:
if n_samples > len(all_permutations):
return all_permutations

return random.sample(all_permutations, n_samples)

return all_permutations


def validate_base_project_files(env_dir: pathlib.Path):
"""
Validates that the environment directory was created and contains the expected files
"""
Expand All @@ -37,29 +67,96 @@ def validate_base_project_files(env_dir):
expected_dir_path = env_dir.joinpath(expected_dir)
assert expected_dir_path.is_dir(), f"Did not find dir: {expected_dir_path}"

# Linter & CI files checked separately

expected_files = [
".commitlintrc.yaml",
".gitattributes",
".gitignore",
".pre-commit-config.yaml",
".prettierrc",
".pylintrc",
"check_commit_msgs.sh",
"environment.yaml",
"pyproject.toml",
"README.md",
]

for expected_file in expected_files:
expected_file_path = env_dir.joinpath(expected_file)
assert expected_file_path.is_file(), f"Did not find file: {expected_file_path}"


def validate_gitlab_configuration(env_dir, expect_present=True):
file_path = env_dir.joinpath(".gitlab-ci.yml")
if expect_present:
assert file_path.is_file(), f"Did not find file: {file_path}"
def validate_python_environment(env_dir: pathlib.Path) -> list[str]:
with open(env_dir.joinpath("environment.yaml"), "r", encoding="utf-8") as f:
python_deps: list[str] = yaml.safe_load(f)["dependencies"]

assert "python=3.10.9" in python_deps, "Did not find python=3.10.9 in environment.yaml"

python_deps_noversion = [i.split("=")[0] for i in python_deps]

return python_deps, python_deps_noversion


def validate_cicd_configuration(env_dir: pathlib.Path, cicd_configuration: str):
all_possible_cicd_configs = {"gitlab": ".gitlab-ci.yml"}

if cicd_configuration == "none":
for fname in all_possible_cicd_configs.values():
config_path = env_dir.joinpath(fname)
assert (
not config_path.is_file()
), f"Expected not to find cicd config {config_path} for {cicd_configuration}"
else:
assert not file_path.is_file(), f"Expected not to find file: {file_path}"
try:
fname = all_possible_cicd_configs[cicd_configuration]
except KeyError:
raise NotImplementedError( # pylint: disable=W0707
f"No test implemented for cicd for {cicd_configuration}"
)

config_path = env_dir.joinpath(fname)
assert (
config_path.is_file()
), f"Did not find cicd config {config_path} for {cicd_configuration}"


def validate_linter_configuration(
env_dir: pathlib.Path, python_packages: list[str], linter_name: str
):
match linter_name:
case "pylint":
config_name = ".pylintrc"
case "ruff":
config_name = "ruff.toml"
case _:
raise NotImplementedError(f"No test implemented for linter {linter_name}")

file_path = env_dir.joinpath(config_name)

assert (
linter_name in python_packages
), f"Did not find {linter_name} in environment.yaml but specified as linter"
assert file_path.is_file(), f"Did not find linter config: {file_path} for {linter_name}"


def validate_jupyter_configuration(python_packages: list[str], install_jupyter: str):
match install_jupyter:
case "yes":
assert (
"jupyter" in python_packages
), "install_jupyter == yes but jupyter not in environment.yaml"
assert (
"nbqa" in python_packages
), "install_jupyter == yes but nbqa not in environment.yaml"
case "no":
assert (
not "jupyter" in python_packages
), "install_jupyter == no but jupyter in environment.yaml"
assert (
not "nbqa" in python_packages
), "install_jupyter == no but nbqa in environment.yaml"
case _:
raise ValueError(f"{install_jupyter} is not an option for install_jupyter")


def validate_pre_commit(env_dir, env_name):
Expand All @@ -77,16 +174,23 @@ def validate_pre_commit(env_dir, env_name):

@pytest.mark.parametrize(
"template_environment",
[
{},
{"cicd_configuration": "gitlab"},
],
get_all_possible_configuration_permutations(n_samples=5),
indirect=["template_environment"],
)
def test_template(template_environment):
env_dir, env_name, env_config = template_environment
validate_base_project_files(env_dir)
validate_gitlab_configuration(
env_dir, expect_present=env_config.get("cicd_configuration") == "gitlab"

_, python_packages = validate_python_environment(env_dir)

validate_cicd_configuration(env_dir, cicd_configuration=env_config.get("cicd_configuration"))

validate_linter_configuration(
env_dir, python_packages, linter_name=env_config.get("linter_name")
)

validate_jupyter_configuration(
python_packages, install_jupyter=env_config.get("install_jupyter")
)

validate_pre_commit(env_dir, env_name)
33 changes: 33 additions & 0 deletions {{cookiecutter.repo_name}}/.conditional_files/ruff.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# sync with black
line-length = 100

# No need for NBQA as ruff has native Jupyter support
extend-include = ["*.ipynb"]

# https://docs.astral.sh/ruff/rules/
ignore = [
# "E501", # Line too long, handled by black
# "W291", # Trailing whitespace, handled by black
# "W292", # Missing final newline, handled by black
# "PLR0904", # Too many public methods
# "PLR0911", # Too many return statements
# "PLR0913", # Too many arguments
# "PLC0415", # Import outside toplevel
# NA as checks yet but mentioned in github comments
# "C0305", # Trailing newlines, handled by black
# "C0114", # Missing module docstring
# "C0115", # Missing class docstring
# "C0116", # Missing function docstring
# "R0902", # Too many instance attributes
# "R0903", # Too few public methods
# "R0914", # Too many locals
# "W0124", # Confusing with statement
# "C0413", # Wrong import position
# "C0410", # Multiple imports
# "R1705", # No else return
# "W0201", # Attribute defined outside init
# "E1123", # Unexpected keyword arg
# "C0401", # Wrong spelling in comment
# "C0402", # Wrong spelling in docstring
# "C0403" # Invalid character in docstring
]
19 changes: 19 additions & 0 deletions {{cookiecutter.repo_name}}/.pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ repos:
pass_filenames: true
stages:
- commit-msg
{%- if cookiecutter.linter_name == "pylint" %}
- repo: https://github.com/pycqa/isort
rev: 5.13.2
hooks:
Expand All @@ -40,6 +41,12 @@ repos:
stages:
- commit
- manual
{%- elif cookiecutter.linter_name == "ruff" %}
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.6.7
hooks:
- id: ruff
{%- endif %}
- repo: local
hooks:
- id: black
Expand All @@ -48,11 +55,15 @@ repos:
args: [--config=./pyproject.toml]
language: system
types: [python]
{%- if cookiecutter.linter_name == "pylint" %}
- id: pylint
name: pylint
entry: pylint
language: system
types: [python]
{%- endif %}
{%- if cookiecutter.install_jupyter == "yes" %}
{%- if cookiecutter.linter_name == "pylint" %}
- id: nbqa-pylint
name: nbqa-pylint
entry: nbqa pylint
Expand All @@ -61,6 +72,14 @@ repos:
args:
- --disable=pointless-statement,duplicate-code,expression-not-assigned
- --const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|([a-z_][a-z0-9_]{0,50}))$
{%- elif cookiecutter.linter_name == "ruff" %}
- id: nbqa-ruff
name: nbqa-ruff
entry: nbqa ruff
language: system
files: \.ipynb
{%- endif %}
{%- endif %}
- repo: https://github.com/pre-commit/mirrors-prettier
rev: v3.1.0
hooks:
Expand Down
8 changes: 6 additions & 2 deletions {{cookiecutter.repo_name}}/environment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ channels:
- conda-forge
dependencies:
- black=24.4.2
- nbqa=1.8.5
{%- if cookiecutter.install_jupyter == 'yes' %}
- jupyter=1.0.0
- nbqa=1.8.5
{%- endif %}
{%- if cookiecutter.linter_name == "pylint" %}
- pylint=3.2.5
{%- elif cookiecutter.linter_name == "ruff" %}
- ruff=0.6.7
{%- endif %}
- python=3.10.9
- pre-commit=3.7.1
- pylint=3.2.5
4 changes: 3 additions & 1 deletion {{cookiecutter.repo_name}}/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ exclude = '''
)
'''

{% if cookiecutter.linter_name == "pylint" %}
[tool.isort]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think isort is running in a separate pre-commit check and is not related to pylint.

profile='black'
profile = 'black'
line_length = 100
{% endif %}
Loading