diff --git a/.github/citools/python/python-lint-bandit b/.github/citools/python/python-lint-bandit new file mode 100755 index 00000000..030c48a5 --- /dev/null +++ b/.github/citools/python/python-lint-bandit @@ -0,0 +1,31 @@ +#!/bin/bash +# +# .github/citools/python/python-lint-bandit +# + +# shellcheck disable=SC1091 +source ../../.github/citools/includes/wrapper-library || exit + +declare -i retval=0 + +main() { + printf "\nRunning Python Lint - Bandit\n\n" + + show_tool_versions_python_short + + print_ruler + + run_command bandit --version + + print_ruler + + # shellcheck disable=SC2048,SC2068,SC2086 + run_command bandit ${@:---verbose --recursive ./src} + + print_ruler + + echo Exit code: "${retval}" + return "${retval}" +} + +time main "${@}" diff --git a/.github/citools/python/python-lint-formatter b/.github/citools/python/python-lint-pylint similarity index 56% rename from .github/citools/python/python-lint-formatter rename to .github/citools/python/python-lint-pylint index 5d6f3e26..65f637cf 100755 --- a/.github/citools/python/python-lint-formatter +++ b/.github/citools/python/python-lint-pylint @@ -1,6 +1,6 @@ #!/bin/bash # -# .github/citools/python/python-lint-formatter +# .github/citools/python/python-lint-pylint # # shellcheck disable=SC1091 @@ -9,13 +9,18 @@ source ../../.github/citools/includes/wrapper-library || exit declare -i retval=0 main() { - printf "\nRunning Python Formatter\n\n" + printf "\nRunning Python Lint - PyLint\n\n" show_tool_versions_python_short print_ruler - run_command ruff . + run_command pylint --version + + print_ruler + + # shellcheck disable=SC2048,SC2068,SC2086 + run_command pylint ${@:-./src} print_ruler diff --git a/.github/citools/python/python-lint-pyright b/.github/citools/python/python-lint-pyright new file mode 100755 index 00000000..1b5ce2af --- /dev/null +++ b/.github/citools/python/python-lint-pyright @@ -0,0 +1,31 @@ +#!/bin/bash +# +# .github/citools/python/python-lint-pyright +# + +# shellcheck disable=SC1091 +source ../../.github/citools/includes/wrapper-library || exit + +declare -i retval=0 + +main() { + printf "\nRunning Python Lint - PyRight\n\n" + + show_tool_versions_python_short + + print_ruler + + run_command pyright --version + + print_ruler + + # shellcheck disable=SC2048,SC2068,SC2086 + run_command pyright ${@:---stats ./src} + + print_ruler + + echo Exit code: "${retval}" + return "${retval}" +} + +time main "${@}" diff --git a/.github/citools/python/python-lint-refurb b/.github/citools/python/python-lint-refurb new file mode 100755 index 00000000..08aa20d9 --- /dev/null +++ b/.github/citools/python/python-lint-refurb @@ -0,0 +1,31 @@ +#!/bin/bash +# +# .github/citools/python/python-lint-refurb +# + +# shellcheck disable=SC1091 +source ../../.github/citools/includes/wrapper-library || exit + +declare -i retval=0 + +main() { + printf "\nRunning Python Lint - Refurb\n\n" + + show_tool_versions_python_short + + print_ruler + + run_command refurb --version + + print_ruler + + # shellcheck disable=SC2048,SC2068,SC2086 + run_command refurb ${@:-./src} + + print_ruler + + echo Exit code: "${retval}" + return "${retval}" +} + +time main "${@}" diff --git a/.github/citools/python/python-lint-ruff b/.github/citools/python/python-lint-ruff new file mode 100755 index 00000000..4546da01 --- /dev/null +++ b/.github/citools/python/python-lint-ruff @@ -0,0 +1,31 @@ +#!/bin/bash +# +# .github/citools/python/python-lint-ruff +# + +# shellcheck disable=SC1091 +source ../../.github/citools/includes/wrapper-library || exit + +declare -i retval=0 + +main() { + printf "\nRunning Python Lint - Ruff\n\n" + + show_tool_versions_python_short + + print_ruler + + run_command ruff --version + + print_ruler + + # shellcheck disable=SC2048,SC2068,SC2086 + run_command ruff ${@:-check --ignore E501 ./src} + + print_ruler + + echo Exit code: "${retval}" + return "${retval}" +} + +time main "${@}" diff --git a/.github/citools/python/python-test-with-coverage b/.github/citools/python/python-test-with-coverage index 51c09c0c..6bb65cc5 100755 --- a/.github/citools/python/python-test-with-coverage +++ b/.github/citools/python/python-test-with-coverage @@ -19,17 +19,24 @@ main() { print_ruler - run_command pytest --verbose --cov=. --cov-branch --cov-report={term-missing,xml:.coverage.xml} -p no:randomly || ((retval++)) + run_command pytest --version print_ruler - echo Running: coverage report --show-missing - coverage report --show-missing + export PYTHONPATH="./src" + printf "PYTHONPATH=\"%s\"\n" "${PYTHONPATH}" print_ruler - echo Running: coverage annotate - coverage annotate + run_command pytest --verbose --cov=. --cov-branch --cov-report={term-missing,xml:.coverage.xml} -p no:randomly ./test || ((retval++)) + + print_ruler + + run_command coverage report --show-missing + + print_ruler + + run_command coverage annotate print_ruler diff --git a/.github/citools/python/python-test-with-doctests b/.github/citools/python/python-test-with-doctests new file mode 100755 index 00000000..17ee6562 --- /dev/null +++ b/.github/citools/python/python-test-with-doctests @@ -0,0 +1,32 @@ +#!/bin/bash +# +# .github/citools/python/python-test-doctest +# + +# shellcheck disable=SC1091 +source ../../.github/citools/includes/wrapper-library || exit + +declare -i retval=0 + +main() { + printf "\nRunning Python DocTests\n\n" + + show_tool_versions_python_short + + print_ruler + + export PYTHONPATH="./src" + printf "PYTHONPATH=\"%s\"\n" "${PYTHONPATH}" + + print_ruler + + # shellcheck disable=SC2048,SC2068,SC2086 + run_command python -m doctest ${@:--v ./src/*/*.py} + + print_ruler + + echo Exit code: "${retval}" + return "${retval}" +} + +time main "${@}" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0ed77165..e44cb9e2 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -2,9 +2,10 @@ # # .github/workflows/python.yml # -name: Python Workflow +name: Python Workflow 4.0 on: # yamllint disable-line rule:truthy pull_request: + workflow_dispatch: defaults: run: @@ -13,10 +14,15 @@ defaults: jobs: stage1: name: Change Check - runs-on: ubuntu-latest + runs-on: 'ubuntu-latest' outputs: docs_changed: ${{ steps.check_file_changed.outputs.docs_changed }} + matrix_exercise: ${{ steps.check_file_changed.outputs.matrix_exercise }} + exercise_count: ${{ steps.check_file_changed.outputs.exercise_count }} steps: + - name: Check GitHub Vars + id: github-vars + run: set | grep -e ^CI -e ^GITHUB_ -e ^RUNNER_ - name: Checkout Repo id: checkout-repo uses: actions/checkout@v3 @@ -27,17 +33,56 @@ jobs: - name: Get Change List id: check_file_changed run: | + { + printf "Workflow: %s\n\n" "${GITHUB_WORKFLOW}" + printf "Runner: name[%s] arch[%s]\n" "${RUNNER_NAME}" "${RUNNER_ARCH}" + printf "Repo: %s\n" "${GITHUB_REPOSITORY}" + printf "User: %s\n" "${GITHUB_TRIGGERING_ACTOR}" + printf "\n" + } | tee -a "${GITHUB_STEP_SUMMARY}" + # Diff HEAD with the previous commit then output to stdout. - printf "=== Which files changed? ===\n" - GIT_DIFF="$(git diff --name-only HEAD^ HEAD)" - printf "%s\n" "${GIT_DIFF}" + GIT_DIFF="$(git diff --name-only HEAD^ HEAD | tee /tmp/changed_files.txt)" + { + printf "=== Which files changed? ===\n" + printf "\`\`\`text\n" + printf "%s\n" "${GIT_DIFF}" + printf "\`\`\`\n" + } | tee -a "${GITHUB_STEP_SUMMARY}" + HAS_WF_DIFF=false + HAS_EX_DIFF=false + if printf "%s\n" "${GIT_DIFF}" | grep -E '^(.github/workflows/python.yml|.github/citools/python/)$'; then + HAS_WF_DIFF=true + fi + if printf "%s\n" "${GIT_DIFF}" | grep -E '^python/.*[.]py$'; then + HAS_EX_DIFF=true + fi + printf "\n" + printf "=== Did WF/CI change without exercise changes? ===\n" + CI_FORCE_FULL=false + if ${HAS_WF_DIFF} && ! ${HAS_EX_DIFF}; then + CI_FORCE_FULL=true + printf "%s\n" "${CI_FORCE_FULL}" + fi + printf "\n" + + # Get changed exercise list. + if [[ ${GITHUB_EVENT_NAME} == pull_request ]] && ! ${CI_FORCE_FULL}; then + printf "Generating pull request changed exercise list.\n" + grep -E '^python/[-a-z0-9]+./.*$' /tmp/changed_files.txt | cut -f1,2 -d/ | sort -Vu | tee /tmp/exercises.txt || true + else + printf "Generating complete exercise list.\n" + find ./python/ -type d -print | grep -v -E '^[.]/python/(|[.].*)$' | sed -r -e 's:[.]/(python/[-a-z0-9]+)/?.*$:\1:g' | sort -Vu | tee /tmp/exercises.txt || true + HAS_DIFF=true + fi printf "\n" # Check if the files are present in the changed file list (added, modified, deleted) then output to stdout. - HAS_DIFF=false + HAS_DIFF="${HAS_DIFF:-false}" printf "=== Which Python files changed? ===\n" if printf "%s\n" "${GIT_DIFF}" | grep -E '^(python/.*[.]py|.github/workflows/python.yml)'; then HAS_DIFF=true + printf "%s\n" "${HAS_DIFF}" fi printf "\n" @@ -46,71 +91,124 @@ jobs: printf "%s\n" "${HAS_DIFF}" printf "\n" + # Generate exercise job matrix from changed files list. + printf "=== Generating matrix exercise list. ===\n" + declare -i last=0 + declare -i count=0 + last=$(wc -l /tmp/exercises.txt | cut -f1 -d\ ) + ((last -= 1)) || true + entries="" + while read -r line; do + if [[ ! ${count} -lt ${last} ]]; then + comma="" + else + comma="," + fi + printf -v entry "\"%s\"%s\n" "${line}" "${comma}" + ((count += 1)) + entries+="${entry}" + done < <(sort -V /tmp/exercises.txt) + jq --sort-keys . >/tmp/exercises.json <> "${GITHUB_OUTPUT}" + check-matrix: + runs-on: ubuntu-latest + needs: stage1 + steps: + - name: Install json2yaml + run: sudo npm install -g json2yaml + - name: Check matrix::exercises + run: | + exercise_count='${{ needs.stage1.outputs.exercise_count }}' + matrix_exercise='${{ needs.stage1.outputs.matrix_exercise }}' + { + printf "\`\`\`text\n" + printf "exercise_count=%s\n" "${exercise_count}" + printf "matrix_exercise=%s\n" "${matrix_exercise}" + printf "\`\`\`\n" + printf "json:\n" + printf "\`\`\`text\n" + printf "%s" "${matrix_exercise}" | jq . + printf "\`\`\`\n" + } | tee -a "${GITHUB_STEP_SUMMARY}" + { + printf "yaml:\n" + printf "\`\`\`text\n" + printf "{ \"matrix\": { \"exercise\": %s } }" "${matrix_exercise}" | json2yaml + printf "\`\`\`\n" + } | tee -a "${GITHUB_STEP_SUMMARY}" stage2: name: Python Checks strategy: matrix: - python-version: ['3.11', '3.12'] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + os: ["ubuntu-latest", "windows-latest", "macos-latest"] + exclude: + - os: "macos-latest" + - os: "windows-latest" + exercise: ${{fromJson(needs.stage1.outputs.matrix_exercise)}} + runs-on: "${{ matrix.os }}" + container: + image: vpayno/ci-generic-debian:latest needs: [stage1] - if: needs.stage1.outputs.docs_changed == 'True' + if: needs.stage1.outputs.docs_changed == 'true' steps: - - name: Checkout Repo + - name: Checkout Repo [${{ matrix.exercise }}] id: checkout-repo uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Show Python version + - name: Config Python Tools [${{ matrix.exercise }}] + id: config-python-tools + run: |- + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-setup-config + - name: Verify Python Tools [${{ matrix.exercise }}] + id: verify-python-tools + run: |- + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-setup-verify + - name: Show Python version [${{ matrix.exercise }}] id: python-version run: | python --version - - name: Install Python Tools - id: install-python-tools - run: | - python -m pip install --upgrade pip - - name: CD to Python Dir - id: cd-to-python-dir - run: | - pwd - ls - cd ./python - pwd - ls - - name: Analysing the code with pylint + pyenv --version + pip --version + - name: Analysing the code with pylint [${{ matrix.exercise }}] id: pylint run: | - pip install --upgrade pylint - cd ./python - ./for_each pylint ./src - - name: Analysing the code with ruff + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-lint-pylint + - name: Analysing the code with ruff [${{ matrix.exercise }}] id: ruff run: | - pip install --upgrade ruff - cd ./python - ./for_each ruff check --ignore E501 ./src - - name: Analysing the code with pyright + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-lint-ruff + - name: Analysing the code with pyright [${{ matrix.exercise }}] id: pyright run: | - pip install --upgrade pyright - cd ./python - ./for_each pyright --stats ./src || true - - name: Analysing the code with bandit + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-lint-pyright + - name: Analysing the code with bandit [${{ matrix.exercise }}] id: bandit run: | - pip install --upgrade bandit[toml] - cd ./python - ./for_each bandit --verbose --recursive ./src || true - - name: Testing with pytest - id: pytest-test-run + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-lint-bandit + - name: Analysing the code with refurb [${{ matrix.exercise }}] + id: refurb + run: | + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-lint-refurb + - name: Testing with doctests [${{ matrix.exercise }}] + id: pytest-doctest-run + run: |- + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-test-with-doctests + - name: Testing with pytest [${{ matrix.exercise }}] + id: pytest-pytest-run run: |- - sudo apt install libsqlite3-dev - pip install --upgrade db-sqlite3 pysqlite3 - pip install --upgrade pytest coverage pytest-cov - cd ./python - PYTHONPATH="./src" ./for_each pytest --verbose --cov=. --cov-branch --cov-report={term-missing,xml:.coverage.xml} ./test + ./.github/citools/common/run_wrapper_script "./${{ matrix.exercise }}" ../../.github/citools/python/python-test-with-coverage diff --git a/python/run-tests b/python/run-tests index fe89f3d8..a00bef36 100755 --- a/python/run-tests +++ b/python/run-tests @@ -16,64 +16,51 @@ main() { # https://github.com/pylint-dev/pylint - echo Running: pylint ./src - time pylint ./src + echo Running: ../../.github/citools/python/python-lint-pylint + time ../../.github/citools/python/python-lint-pylint print_ruler # https://github.com/charliermarsh/ruff - echo Running: ruff check --ignore E501 ./src - time ruff check --ignore E501 ./src + echo Running: ../../.github/citools/python/python-lint-ruff + time ../../.github/citools/python/python-lint-ruff print_ruler # https://github.com/microsoft/pyright - echo Running: pyright --stats ./src - time pyright --stats ./src + echo Running: ../../.github/citools/python/python-lint-pyright + time ../../.github/citools/python/python-lint-pyright print_ruler # https://github.com/PyCQA/bandit - echo Running: bandit --verbose --recursive ./src - time bandit --verbose --recursive ./src + echo Running: ../../.github/citools/python/python-lint-bandit + time ../../.github/citools/python/python-lint-bandit print_ruler # https://github.com/dosisod/refurb - echo Running: refurb ./src - time refurb ./src + echo Running: ../../.github/citools/python/python-lint-refurb + time ../../.github/citools/python/python-lint-refurb print_ruler - echo python -m doctest -v ./src/*/*py - time python -m doctest -v ./src/*/*py - printf "\n" + echo Running: ../../.github/citools/python/python-test-doctest + time ../../.github/citools/python/python-test-doctest print_ruler # https://github.com/pytest-dev/pytest - tail -v -n 1000 .coveragerc - printf "\n" - - echo Running: PYTHONPATH="./src" pytest --verbose --cov=. --cov-branch --cov-report={term-missing,xml:.coverage.xml} -p no:randomly ./test - time { PYTHONPATH="./src" pytest --verbose --cov=. --cov-branch --cov-report={term-missing,xml:.coverage.xml} -p no:randomly ./test; } - - print_ruler - - echo Running: coverage report --show-missing - time coverage report --show-missing + echo Running: ../../.github/citools/python/python-test-with-coverage + time ../../.github/citools/python/python-test-with-coverage print_ruler - echo coverage annotate - time coverage annotate - printf "\n" - echo tail -n 10000 './*,cover' '|' grep -E -C 3 "'^> def |^! '" tail -n 10000 ./*,cover | grep -E -C 1 '^> def |^! '