diff --git a/cookiecutter.json b/cookiecutter.json index 497e68f..beeac2e 100644 --- a/cookiecutter.json +++ b/cookiecutter.json @@ -8,6 +8,10 @@ "GitLab", "None" ], + "jupytext": [ + "No", + "Yes" + ], "python_package_name": "{{ cookiecutter.__project_name_slug.replace('-', '_') }}", "__package_name": "{{ cookiecutter.python_package_name }}" } \ No newline at end of file diff --git a/docs/tmp_content/overview.md b/docs/tmp_content/overview.md index 165c93b..0a9d13d 100644 --- a/docs/tmp_content/overview.md +++ b/docs/tmp_content/overview.md @@ -11,6 +11,7 @@ Generated project consists of: * a very minimal python code + example test 1. pre-commit hooks: * `black`, `flake8` - enforce code style + * `jupytext` (optional) - sync jupyter notebooks to plain python files * `pycln` - cleanups unused imports * `mypy` - checks type errors * `isort` - sorts imports diff --git a/docs/tmp_content/precommit.md b/docs/tmp_content/precommit.md index 2f96dcd..c1bef99 100644 --- a/docs/tmp_content/precommit.md +++ b/docs/tmp_content/precommit.md @@ -85,4 +85,16 @@ It is highly recommended to do that, as it allows mypy to catch more errors. For Sometimes a package do not have any type information nor additional package with it - in that case you can suppress mypy error by manually marking an exact library to be ignored as mentioned before. -Please refer to [official mypy documentation](https://mypy.readthedocs.io/en/stable/index.html) for more information. \ No newline at end of file +Please refer to [official mypy documentation](https://mypy.readthedocs.io/en/stable/index.html) for more information. + +## jupytext - additional information + +Jupyter notebooks are hard to review - to make it easier it is possible to use `jupytext` to convert them to `.py` files (py:percent format) automatically on commit. + +If you have not used it before, please read [jupytext documentation](https://jupytext.readthedocs.io/en/latest/). + +```{warning} +Do not edit generated files - they will be overwritten by jupytext due to detected differences. + +It is also important to ensure no other linting tool modifies them, as it might lead to endless loop of changes. +``` \ No newline at end of file diff --git a/hooks/post_gen_project.py b/hooks/post_gen_project.py index 4c7f692..4a9635b 100644 --- a/hooks/post_gen_project.py +++ b/hooks/post_gen_project.py @@ -14,6 +14,10 @@ REMOVE_PATHS.extend(gitlab_files) {% endif %} +{% if cookiecutter.jupytext != "Yes" %} +REMOVE_PATHS.extend(["notebooks/example.py"]) +{% endif %} + print("Cleaning files... 🌀") for path in REMOVE_PATHS: path = Path(path) diff --git a/tests/test_template.py b/tests/test_template.py index 2e10594..5c7757d 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -48,7 +48,7 @@ def run_command(command: str, dir: Path): def assert_jinja_resolved(files: Sequence[Path]) -> None: """Asserts to make sure no curly braces appear in a file name nor in it's content. """ - text_files = ['.txt', '.py', '.rst', '.md', '.cfg', '.toml', '.json', '.yaml', '.yml', '.ini', '.sh'] + text_files = ['.txt', '.py', '.rst', '.md', '.cfg', '.toml', '.json', '.yaml', '.yml', '.ini', '.sh', '.ipynb'] validator = JinjaSolvedValidator() for file in files: assert validator.has_unresolved(file.name) == False @@ -111,3 +111,33 @@ def test_template_project_with_gitlab(cookies): rpath: Path = result.project_path assert (rpath / ".gitlab-ci.yml").exists() is True + + +def test_template_project_with_jupytext(cookies): + result = cookies.bake(extra_context={ + "client_name": "no", + "project_name": "jupytext", + "jupytext": "Yes" + }) + assert result.exit_code == 0 + assert result.exception is None + + rpath: Path = result.project_path + jupytext_pos = (rpath / ".pre-commit-config.yaml").read_text(encoding="utf-8").find("jupytext") + assert jupytext_pos != -1 + assert len(list((rpath / "notebooks").glob("*.py"))) > 0, "Notebook should have py:percent file" + + +def test_template_project_no_jupytext(cookies): + result = cookies.bake(extra_context={ + "client_name": "no", + "project_name": "jupytext", + "jupytext": "No" + }) + assert result.exit_code == 0 + assert result.exception is None + + rpath: Path = result.project_path + jupytext_pos = (rpath / ".pre-commit-config.yaml").read_text(encoding="utf-8").find("jupytext") + assert jupytext_pos == -1 + assert len(list((rpath / "notebooks").glob("*.py"))) == 0, "Notebook should not have a py:percent file" diff --git a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml index 139ea76..5cefa17 100644 --- a/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml +++ b/{{ cookiecutter.repo_name }}/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: check-case-conflict - id: check-merge-conflict - id: trailing-whitespace - exclude: ".bumpversion.cfg" + exclude: .bumpversion.cfg|notebooks/.*\.py - id: check-ast - id: check-added-large-files - id: check-toml @@ -17,9 +17,23 @@ repos: rev: 23.10.1 hooks: - id: black - exclude: (docs/) + exclude: (docs/|notebooks/){% if cookiecutter.jupytext == "No" %} - id: black-jupyter - exclude: (docs/) + files: \.ipynb$ + {%- endif -%} + + {% if cookiecutter.jupytext == "Yes" %} + + # Save .ipynb to .py:percent format + - repo: https://github.com/mwouts/jupytext + rev: v1.15.2 + hooks: + - id: jupytext + args: ["--from", ".ipynb", "--to", "py:percent", "--pipe-fmt", "black"] + files: "\\.ipynb$" + additional_dependencies: + - black==23.10.1 + {%- endif %} # Cleaning unused imports. - repo: https://github.com/hadialqattan/pycln @@ -27,7 +41,7 @@ repos: hooks: - id: pycln args: ["-a"] - exclude: (docs/) + exclude: (docs/|notebooks/) # Modernizes python code and upgrade syntax for newer versions of the language - repo: https://github.com/asottile/pyupgrade @@ -53,7 +67,7 @@ repos: hooks: - id: isort args: ["--profile", "black"] - exclude: (docs/) + exclude: (docs/|notebooks/) # Checks Python source files for errors. - repo: https://github.com/PyCQA/flake8 @@ -90,3 +104,4 @@ repos: - id: bandit args: [-c, pyproject.toml, --recursive, src] additional_dependencies: [".[toml]"] # required for pyproject.toml support + exclude: (notebooks/) diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index 887cf45..3fd2cb9 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -76,6 +76,19 @@ Please read more about it [here](https://docs.gitlab.com/ee/user/project/pages/i {% endif %} +{% if cookiecutter.jupytext == "Yes" %} + +# Jupyter notebooks and jupytext + +To make notebooks more friendly for code review and version control we use `jupytext` to sync notebooks with python files. If you have not used it before, please read [jupytext documentation](https://jupytext.readthedocs.io/en/latest/). + +There is pre-commit hook which automatically generates and syncs notebooks with python files on each commit. + +Please ensure you do not edit/modify manually or by other means generated py:percent files as they will conflict with jupytext change detection and lead to endless loop. +Treat them as read-only files and edit only notebooks. + +{% endif %} + # Semantic version bump To bump version of the library please use `bump2version` which will update all version strings. diff --git a/{{ cookiecutter.repo_name }}/notebooks/example.ipynb b/{{ cookiecutter.repo_name }}/notebooks/example.ipynb new file mode 100644 index 0000000..280338b --- /dev/null +++ b/{{ cookiecutter.repo_name }}/notebooks/example.ipynb @@ -0,0 +1,33 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# IPython extension to reload modules before executing user code.\n", + "# For more info check https://ipython.org/ipython-doc/3/config/extensions/autoreload.html\n", + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import {{ cookiecutter.__package_name }}" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/{{ cookiecutter.repo_name }}/notebooks/example.py b/{{ cookiecutter.repo_name }}/notebooks/example.py new file mode 100644 index 0000000..ef9bbdb --- /dev/null +++ b/{{ cookiecutter.repo_name }}/notebooks/example.py @@ -0,0 +1,18 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.15.2 +# --- + +# %% +# IPython extension to reload modules before executing user code. +# For more info check https://ipython.org/ipython-doc/3/config/extensions/autoreload.html +# %load_ext autoreload +# %autoreload 2 + +# %% +import {{ cookiecutter.__package_name }} diff --git a/{{ cookiecutter.repo_name }}/pyproject.toml b/{{ cookiecutter.repo_name }}/pyproject.toml index 4f99fd5..10da5a8 100644 --- a/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/{{ cookiecutter.repo_name }}/pyproject.toml @@ -96,7 +96,7 @@ ignore_missing_imports = false disallow_untyped_defs = true [tool.pylint.basic] -good-names="""i,j,x,y,z,x1,y1,z1,x2,y2,z2,cv,df,dx,dy,dz,w,h,c,b,g,qa,q,a"""" +good-names="i,j,x,y,z,x1,y1,z1,x2,y2,z2,cv,df,dx,dy,dz,w,h,c,b,g,qa,q,a" max-args=8 [tool.pylint.main]