From 3d6ee2ad233fb4ce6a88e713170228c1dce2fab3 Mon Sep 17 00:00:00 2001 From: Sahcim Date: Wed, 24 Jul 2024 20:55:41 +0200 Subject: [PATCH 1/3] wip --- {{ cookiecutter.repo_name }}/Makefile | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 {{ cookiecutter.repo_name }}/Makefile diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile new file mode 100644 index 0000000..6583249 --- /dev/null +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -0,0 +1,32 @@ +.ONESHELL: + +PYTHON_VERSION := 3.11 +WHEEL_VERSION := 0.43.0 + +.DEFAULT_GOAL := venv/bin/activate + +venv/bin/activate: requirements-dev.txt + python$(PYTHON_VERSION) -m venv venv + . venv/bin/activate + pip install --upgrade pip + pip install wheel==$(WHEEL_VERSION) + pip install -r requirements-dev.txt + pre-commit install + +activate: venv/bin/activate + . venv/bin/activate + +lint: activate + pre-commit run --all-files + +clean: + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -delete + rm -rf venv + rm -rf .pytest_cache + rm -rf .coverage + rm -rf build/ + rm -rf dist/ + rm -rf *.egg-info/ + +.PHONY: update, lint, clean, activate \ No newline at end of file From 0d2e0d9d7d948d4cfe71aebd030127ed55ff181f Mon Sep 17 00:00:00 2001 From: Sahcim Date: Mon, 29 Jul 2024 13:09:21 +0200 Subject: [PATCH 2/3] Replace scripts with makefile --- .../.github/workflows/ci.yml | 42 ++++--------------- .../.github/workflows/documentation.yml | 4 +- {{ cookiecutter.repo_name }}/.gitlab-ci.yml | 21 +++++----- .../.libraries-whitelist.txt | 1 + {{ cookiecutter.repo_name }}/Makefile | 36 +++++++++++++--- {{ cookiecutter.repo_name }}/README.md | 18 ++++---- {{ cookiecutter.repo_name }}/build_docs.sh | 16 ------- {{ cookiecutter.repo_name }}/bump_version.sh | 3 -- .../check_licenses.sh | 18 -------- {{ cookiecutter.repo_name }}/setup_dev_env.sh | 11 ----- 10 files changed, 60 insertions(+), 110 deletions(-) delete mode 100755 {{ cookiecutter.repo_name }}/build_docs.sh delete mode 100755 {{ cookiecutter.repo_name }}/bump_version.sh delete mode 100755 {{ cookiecutter.repo_name }}/check_licenses.sh delete mode 100755 {{ cookiecutter.repo_name }}/setup_dev_env.sh diff --git a/{{ cookiecutter.repo_name }}/.github/workflows/ci.yml b/{{ cookiecutter.repo_name }}/.github/workflows/ci.yml index 63e99f0..532bffe 100644 --- a/{{ cookiecutter.repo_name }}/.github/workflows/ci.yml +++ b/{{ cookiecutter.repo_name }}/.github/workflows/ci.yml @@ -20,17 +20,11 @@ jobs: with: python-version: "3.11" - - name: Cache pre-commit - uses: actions/cache@v3 - with: - path: ~/.cache/pre-commit - key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - - name: Install pre-commit - run: pip3 install pre-commit + - name: Install venv and pre-commit + run: make - name: Run pre-commit checks - run: pre-commit run --all-files --show-diff-on-failure --color always + run: make lint - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master @@ -57,16 +51,11 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} comment_tag: trivy - - name: Create venv - run: . ./setup_dev_env.sh - - name: Check licenses - run: ./check_licenses.sh + run: make lic-check - name: Generate pip freeze - run: | - source venv/bin/activate - pip freeze > requirements-freeze.txt + run: make freeze - name: Publish Artefacts uses: actions/upload-artifact@v3 @@ -90,10 +79,7 @@ jobs: retention-days: 10 - name: Validate package build - run: | - source venv/bin/activate - python -m pip install -U build - python -m build + run: make build - name: Publish Package uses: actions/upload-artifact@v3 @@ -124,23 +110,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Cache Dependencies - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}-${{ hashFiles('**/setup.cfg') }}-${{ hashFiles('**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip- - - name: Install Dependencies - run: pip install -r requirements-dev.txt + run: make - name: Run Tests With Coverage - run: | - # run with coverage to not execute tests twice - coverage run -m pytest -v -p no:warnings --junitxml=report.xml tests/ - coverage report - coverage xml + run: make coverage - name: Test Report uses: mikepenz/action-junit-report@v4 diff --git a/{{ cookiecutter.repo_name }}/.github/workflows/documentation.yml b/{{ cookiecutter.repo_name }}/.github/workflows/documentation.yml index 298ce31..7941c97 100644 --- a/{{ cookiecutter.repo_name }}/.github/workflows/documentation.yml +++ b/{{ cookiecutter.repo_name }}/.github/workflows/documentation.yml @@ -24,10 +24,10 @@ jobs: # for best results, it is better to generate # documentation within development environment - name: Create venv - run: . ./setup_dev_env.sh + run: make - name: Build docs - run: ./build_docs.sh + run: make docs - name: Deploy uses: peaceiris/actions-gh-pages@v3 diff --git a/{{ cookiecutter.repo_name }}/.gitlab-ci.yml b/{{ cookiecutter.repo_name }}/.gitlab-ci.yml index d8ef467..b7e3fbb 100644 --- a/{{ cookiecutter.repo_name }}/.gitlab-ci.yml +++ b/{{ cookiecutter.repo_name }}/.gitlab-ci.yml @@ -96,7 +96,7 @@ lint: stage: lint interruptible: true script: - - pre-commit run --all-files + - make lint rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" || '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' changes: @@ -113,7 +113,7 @@ lint-precommit-changed: stage: lint interruptible: true script: - - pre-commit run --all-files + - make lint rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" changes: @@ -145,21 +145,20 @@ tests: stage: tests interruptible: true before_script: - - . setup_dev_env.sh - - pip install build twine bump2version + - make + - . venv/bin/activate + - pip install build twine bump2version #??? script: # fused check license here as it would take more time to do it in different place. - - ./check_licenses.sh + - make lic-check # test package building - echo ${CI_PIPELINE_IID} >> src/{{ cookiecutter.__package_name }}/VERSION - python -m build # generate pip freeze - - pip freeze > requirements-freeze.txt + - make freeze - cat requirements-freeze.txt # run with coverage to not execute tests twice - - coverage run -m pytest -v -p no:warnings --junitxml=report.xml tests/ - - coverage report - - coverage xml + - make coverage coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' artifacts: when: always @@ -244,9 +243,9 @@ pages: stage: pages interruptible: true before_script: - - . setup_dev_env.sh + - make script: - - ./build_docs.sh + - make docs artifacts: paths: - public diff --git a/{{ cookiecutter.repo_name }}/.libraries-whitelist.txt b/{{ cookiecutter.repo_name }}/.libraries-whitelist.txt index 60b6186..69c0d1e 100644 --- a/{{ cookiecutter.repo_name }}/.libraries-whitelist.txt +++ b/{{ cookiecutter.repo_name }}/.libraries-whitelist.txt @@ -1 +1,2 @@ +{{ cookiecutter.__package_name }} pkg_resources \ No newline at end of file diff --git a/{{ cookiecutter.repo_name }}/Makefile b/{{ cookiecutter.repo_name }}/Makefile index 6583249..c0a0d46 100644 --- a/{{ cookiecutter.repo_name }}/Makefile +++ b/{{ cookiecutter.repo_name }}/Makefile @@ -1,5 +1,3 @@ -.ONESHELL: - PYTHON_VERSION := 3.11 WHEEL_VERSION := 0.43.0 @@ -13,10 +11,8 @@ venv/bin/activate: requirements-dev.txt pip install -r requirements-dev.txt pre-commit install -activate: venv/bin/activate +lint: venv/bin/activate . venv/bin/activate - -lint: activate pre-commit run --all-files clean: @@ -29,4 +25,32 @@ clean: rm -rf dist/ rm -rf *.egg-info/ -.PHONY: update, lint, clean, activate \ No newline at end of file +build: venv/bin/activate + . venv/bin/activate + python -m pip install -U build + python -m build + +lic-check: venv/bin/activate + . venv/bin/activate + pip-licenses --from=mixed --ignore-packages `cat .libraries-whitelist.txt`> licenses.txt --allow-only="$(paste -sd ";" .license-whitelist.txt)" + +docs: venv/bin/activate + . venv/bin/activate + pip-licenses --from=mixed --format rst --with-urls --with-description --output-file=docs/licenses_table.rst + sphinx-build -d docs/_build/doctrees docs/ public/ + +bump-version: venv/bin/activate + . venv/bin/activate + bump2version --verbose --commit $(args) + +freeze: venv/bin/activate + . venv/bin/activate + pip freeze > requirements-freeze.txt + +coverage: venv/bin/activate + . venv/bin/activate + coverage run -m pytest -v -p no:warnings --junitxml=report.xml tests/ + coverage report + coverage xml + +.PHONY: lint, clean, build, lic-check, docs, bump-version, freeze, coverage \ No newline at end of file diff --git a/{{ cookiecutter.repo_name }}/README.md b/{{ cookiecutter.repo_name }}/README.md index 2aaa1db..28b41d3 100644 --- a/{{ cookiecutter.repo_name }}/README.md +++ b/{{ cookiecutter.repo_name }}/README.md @@ -13,7 +13,7 @@ To start, you need to setup your local machine. You need to setup virtual environment, simplest way is to run from project root directory: ```bash -$ . ./setup_dev_env.sh +$ make $ source venv/bin/activate ``` This will create a new venv and run `pip install -r requirements-dev.txt`. @@ -31,7 +31,7 @@ All updated files will be reformatted and linted before the commit. To reformat and lint all files in the project, use: -`pre-commit run --all-files` +`make lint` The used linters are configured in `.pre-commit-config.yaml`. You can use `pre-commit autoupdate` to bump tools to the latest versions. @@ -53,10 +53,10 @@ See also [Cookiecutter Data Science opinion](https://drivendata.github.io/cookie In `docs/` directory are Sphinx RST/Markdown files. -To build documentation locally, in your configured environment, you can use `build_docs.sh` script: +To build documentation locally, in your configured environment, you can use `make docs` command: ```bash -$ ./build_docs.sh +$ make docs ``` Then open `public/index.html` file. @@ -110,15 +110,15 @@ To bump version of the library please use `bump2version` which will update all v NOTE: Configuration is in `.bumpversion.cfg` and **this is a main file defining version which should be updated only with bump2version**. -For convenience there is bash script which will create commit, to use it call: +For convenience there is make command which will create commit, to use it call: ```bash # to create a new commit by increasing one semvar: -$ ./bump_version.sh minor -$ ./bump_version.sh major -$ ./bump_version.sh patch +$ make bump-version args=minor +$ make bump-version args=patch +$ make bump-version args=major # to see what is going to change run: -$ ./bump_version.sh --dry-run major +$ ./bump_version.sh args="--dry-run major" ``` Script updates **VERSION** file and setup.cfg automatically uses that version. diff --git a/{{ cookiecutter.repo_name }}/build_docs.sh b/{{ cookiecutter.repo_name }}/build_docs.sh deleted file mode 100755 index 091c6c5..0000000 --- a/{{ cookiecutter.repo_name }}/build_docs.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Force Sphinx to rebuild documentation - otherwise it generates incosistencies. -rm -rf public/ docs/_build - -# Exit on error for the next commands -set -e -x - -# Activate venv for licenses and also it simplifies sphinx code documentation generation. -. venv/bin/activate - -# Generate a table with all installed libraries, licenses etc -pip-licenses --from=mixed --format rst --with-urls --with-description --output-file=docs/licenses_table.rst - -# Build sphinx docs to public/ directory -sphinx-build -d docs/_build/doctrees docs/ public/ diff --git a/{{ cookiecutter.repo_name }}/bump_version.sh b/{{ cookiecutter.repo_name }}/bump_version.sh deleted file mode 100755 index d527d5f..0000000 --- a/{{ cookiecutter.repo_name }}/bump_version.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -e -bump2version --verbose --commit $@ \ No newline at end of file diff --git a/{{ cookiecutter.repo_name }}/check_licenses.sh b/{{ cookiecutter.repo_name }}/check_licenses.sh deleted file mode 100755 index cd213c7..0000000 --- a/{{ cookiecutter.repo_name }}/check_licenses.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e - -. venv/bin/activate - -pip-licenses --from=mixed --ignore-packages `cat .libraries-whitelist.txt`> licenses.txt -cat licenses.txt - -FOUND=$(tail -n +2 licenses.txt | grep -v -f .license-whitelist.txt | wc -l) - -if [[ $FOUND -gt 0 ]]; then - echo "Detected license/s not on the whitelist ($FOUND found)." - tail -n +2 licenses.txt | grep -v -f .license-whitelist.txt - exit 1 -else - echo "Okay." - exit 0 -fi diff --git a/{{ cookiecutter.repo_name }}/setup_dev_env.sh b/{{ cookiecutter.repo_name }}/setup_dev_env.sh deleted file mode 100755 index 8752f58..0000000 --- a/{{ cookiecutter.repo_name }}/setup_dev_env.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ ! -d venv ]; then - python3 -m venv venv - . venv/bin/activate - pip install --upgrade pip - pip install --quiet wheel==0.41.3 - pip install --quiet -r requirements-dev.txt -fi - -. venv/bin/activate From bfd1e2d00129976933c2412de951bc5112a5f869 Mon Sep 17 00:00:00 2001 From: Sahcim Date: Mon, 29 Jul 2024 13:16:30 +0200 Subject: [PATCH 3/3] Fix typos --- {{ cookiecutter.repo_name }}/.gitlab-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/{{ cookiecutter.repo_name }}/.gitlab-ci.yml b/{{ cookiecutter.repo_name }}/.gitlab-ci.yml index b7e3fbb..10bdd78 100644 --- a/{{ cookiecutter.repo_name }}/.gitlab-ci.yml +++ b/{{ cookiecutter.repo_name }}/.gitlab-ci.yml @@ -95,6 +95,8 @@ lint: image: $PRECOMMIT_IMAGE:latest stage: lint interruptible: true + before_script: + - make script: - make lint rules: @@ -112,6 +114,8 @@ lint-precommit-changed: image: $PRECOMMIT_IMAGE:dev-$CI_COMMIT_SHA stage: lint interruptible: true + before_script: + - make script: - make lint rules: @@ -146,7 +150,6 @@ tests: interruptible: true before_script: - make - - . venv/bin/activate - pip install build twine bump2version #??? script: # fused check license here as it would take more time to do it in different place.