From 83184539de4dbf7e664afefa9f67bd17ecc8d62d Mon Sep 17 00:00:00 2001 From: konstantinos Date: Sat, 16 Dec 2023 12:37:29 +0200 Subject: [PATCH 1/3] feat(static-code-analysis): add new Static Code Analysis Reusable Workflow fix syntax refactor(static-code-analysis)): clean code of lint.yml workflow style(docker): clean docker.yml workflow config file fix(static-code-analysis): check diff with previous commit only if signal has not been set yet fix(sqa): step if condition syntax/templating/substitution temp(sqa): remove pylint step fix(sqa): checkout code! fix syntax improve information printed to console by Job fix{sqa): fix trigger conditoin computing debug sqa Job add name to Job steps fix trigger condition put ! for not gh action expresion inside doubel {{ debug debug debug refactor: alternative sytntax clean code activate pylint run on SQA Resuable Workflow feat: fully enable Pylint Job Score Checking at runtime --- .github/workflows/docker.yml | 7 +- .github/workflows/lint.yml | 157 +++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7b29cf7..85f1833 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -103,12 +103,7 @@ jobs: inputs.tests_pass && inputs.tests_run ) || ( inputs.acceptance_policy == 3 && inputs.tests_pass == false && inputs.tests_run == false ) ) - # if: always() - # if: always() && inputs.acceptance_policy != 0 && ( inputs.acceptance_policy == 1 || ( - # needs.*.result == 'success' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') ) || ( - # inputs.acceptance_policy == 3 && !contains(needs.*.result, 'failure') && ( !contains(needs.*.result, 'skipped') || contains(needs.*.result, 'cancelled') ) ) - # ) - # inputs.acceptance_policy == 3 && !contains(needs.*.result, 'failure') && ( contains(needs.*.result, 'skipped') || contains(needs.*.result, 'cancelled') ) ) + runs-on: ubuntu-latest env: IMAGE_SLUG: ${{ inputs.image_slug }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..bd9dc23 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,157 @@ +################################# +## PYTHON STATIC CODE ANALYSIS ## +## Reusable Workflow ## +################################# + +# Static Code Analysis (SCA) is a set of techniques for examining source code +# without executing it. + +## AUTONOMOUS JOB ## + +# 0. Never run +# 1. Always run +# 2. Run SQA on conditions +# - Triggered on a Long-living branch (ie main) +# - Triggered on a v* tag (ie v1.0.0) +# - Source Code changed, compared to previous commit +# 3. Run SQA, if Source Code changed, compared to previous commit + +on: + workflow_call: + inputs: + # Defaults to Policy 2 (CI/CD) + run_policy: + required: false + type: string + default: '2' + dedicated_branches: + required: false + type: string + default: 'master, main, dev' + source_code_targets: + required: false + type: string + default: 'src' + ## Parametrizing Runtime Environment (ie py version) + python_version: + required: false + type: string + default: '3.10' + ## Parametrizing Code Analysis Acceptance Criteria + pylint_threshold: + required: false + type: string + default: '8.0' # out of 10 + # secrets + # outputs + +### STATIC CODE ANALYSIS & LINTING ### +jobs: + # Decide wehther to run Lint Job, given incoming Signal + decide_run_lint: + name: "Decide whether to run Lint Job" + runs-on: ubuntu-latest + if: always() && inputs.run_policy != 0 + steps: + - if: ${{ !contains('1, 2, 3', inputs.run_policy) }} + run: 'echo "Invalid run_policy: ${{ inputs.run_policy }}. Must be >0 and <4" && exit 1' + + - if: inputs.run_policy == 1 + name: 'POLICY: 1 -> Trigger' + run: echo "SHOULD_RUN_SQA=true" >> $GITHUB_ENV + + - if: inputs.run_policy == 2 && contains(inputs.dedicated_branches, github.ref_name) + name: 'POLICY: 2 & Branch: ${{ github.ref_name }} -> Trigger' + run: echo "SHOULD_RUN_SQA=true" >> $GITHUB_ENV + + - if: inputs.run_policy == 2 && startsWith(github.ref, 'refs/tags/v') + name: 'POLICY: 2 & Tag: ${{ github.ref_name }} -> Trigger' + run: echo "SHOULD_RUN_SQA=true" >> $GITHUB_ENV + + - if: ${{ env.SHOULD_RUN_SQA != 'true' && contains('2, 3', inputs.run_policy) }} + name: 'POLICY: 2, 3 -> Derive from DIFF' + run: echo "SHOULD_DERIVE_FROM_DIFF=true" >> $GITHUB_ENV + + - if: ${{ env.SHOULD_DERIVE_FROM_DIFF }} + name: 'POLICY: 2, 3 -> Checkout Code to compute DIFF' + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - if: ${{ env.SHOULD_DERIVE_FROM_DIFF }} + name: 'POLICY: 2, 3 -> Check Source Code DIFF' + run: | + echo "============ List Modified Files ============" + git diff --name-only HEAD^ HEAD + CHANGED_FILES=$(git diff --name-only HEAD^ HEAD) + + # Read Folders we 'Watch' for changes + TARGETS=$(echo "${{ inputs.source_code_targets }}" | tr ',' '\n') + + # Loop through the Watched Folders + for TARGET in $TARGETS; do + # if rel path of changed file matches glob pattern + if [[ $CHANGED_FILES == *"$TARGET"* ]]; then + echo "SHOULD_RUN_SQA=true" >> $GITHUB_ENV + echo "QA Target: $TARGET found in CHANGED_FILES!" + break + fi + done + + ### OUTPUT of JOB ### + - name: "Set 'Run Static Code Analysis' Signal to ${{ env.SHOULD_RUN_SQA }}" + id: set_sqa_signal + run: echo "RUN_SQA=${{ env.SHOULD_RUN_SQA }}" >> $GITHUB_OUTPUT + outputs: + RUN_SQA: ${{ steps.set_sqa_signal.outputs.RUN_SQA }} + + lint: + name: "Static Code Analysis" + runs-on: ubuntu-latest + needs: decide_run_lint + if: always() && needs.decide_run_lint.outputs.RUN_SQA == 'true' + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ inputs.python_version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox==3.28 + + ## Isort ## + - name: 'Isort: Require Semantic and Alphabetic order of the Python Imports' + if: ${{ matrix.platform != 'windows-latest' }} + run: tox -e isort -vv -s false + + ## Black ## + - name: 'Black: Require Project Style to be followed by the Python Code' + if: ${{ matrix.platform != 'windows-latest' }} + run: tox -e black -vv -s false + + # TODO: Retire Pylint and MIGRATE to RUFF + ## Pylint (todo replace with ruff## + - name: Run Pylint tool on Python Code Base + run: tox -e pylint -vv -s false | tee pylint-result.txt + + - run: cat pylint-result.txt + + - name: 'Check Pylint Score > ${{ inputs.pylint_threshold }}/10' + if: ${{ matrix.platform != 'windows-latest' }} + run: | + SCORE=`sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' pylint-result.txt` + echo "SCORE -> $SCORE" + # threshold check + if awk "BEGIN {exit !($SCORE >= ${{ inputs.pylint_threshold }})}"; then + echo "PyLint Passed! | Score: ${SCORE} out of 10 | Threshold: ${{ inputs.pylint_threshold }}" + else + echo "PyLint Failed! | Score: ${SCORE} out of 10 | Threshold: ${{ inputs.pylint_threshold }}" + exit 1 + fi + + ## Pyflakes, Pyroma, McCabe, DodgyRun, Profile Validator ## + - name: Run tox -e prospector + if: ${{ matrix.platform != 'windows-latest' }} + run: tox -e prospector -vv -s false From 5dc98acf027a20518088cf9a91747b2729e8b3ba Mon Sep 17 00:00:00 2001 From: konstantinos Date: Tue, 19 Dec 2023 19:53:33 +0200 Subject: [PATCH 2/3] docs(website): integrate new feature into docs site Refs, and CI Build in RTD --- .readthedocs.yaml | 1 + docs/ref_lint.md | 52 +++++++++++++++++++++++++++++++++++++ docs/tests_index.md | 9 ++++++- mkdocs.yml | 1 + scripts/gen-workflow-ref.py | 16 +++++++++++- 5 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 docs/ref_lint.md diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 93a552b..39f5377 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -30,6 +30,7 @@ build: - chmod +x ./scripts/gen-workflow-ref.py - ./scripts/gen-workflow-ref.py ./.github/workflows/docker.yml > ./docs/ref_docker.md - ./scripts/gen-workflow-ref.py ./.github/workflows/pypi_env.yml > ./docs/ref_pypi_env.md + - ./scripts/gen-workflow-ref.py ./.github/workflows/lint.yml > ./docs/ref_lint.md # Build documentation in the "docs/" directory with mkdocs diff --git a/docs/ref_lint.md b/docs/ref_lint.md new file mode 100644 index 0000000..9c804bf --- /dev/null +++ b/docs/ref_lint.md @@ -0,0 +1,52 @@ +# Workflow lint.yml + +### Trigger Events + +If any of the below events occur, the `lint.yml` workflow will be triggered. + +- workflow_call + +Since there is a `workflow_call` _trigger_event_, this workflow can be triggered (called) by another (caller) workflow. +> Thus, it is a `Reusable Workflow`. + + +## Reusable Workflow + +Event Trigger: `workflow_call` + +### Inputs + +#### Required Inputs + +None + +#### Optional Inputs + +- `dedicated_branches` + - type: _string_ + - Description: + - Default: `master, main, dev` +- `pylint_threshold` + - type: _string_ + - Description: + - Default: `8.0` +- `python_version` + - type: _string_ + - Description: + - Default: `3.10` +- `run_policy` + - type: _string_ + - Description: + - Default: `2` +- `source_code_targets` + - type: _string_ + - Description: + - Default: `src` + +### Secrets + +None + +### Outputs + +None diff --git a/docs/tests_index.md b/docs/tests_index.md index f9e8917..d9c5a93 100644 --- a/docs/tests_index.md +++ b/docs/tests_index.md @@ -10,7 +10,7 @@ There is a dedicated [`cicd-test`](https://github.com/boromir674/cicd-test) Gith The `Test Suite` consists of -- a set of [**GA Workflows**](https://github.com/boromir674/cicd-test/tree/main/.github/workflows) that call our [**Docker**](https://github.com/boromir674/automated-workflows/tree/main/.github/workflows/docker.yml) and [**PyPI**](https://github.com/boromir674/automated-workflows/tree/main/.github/workflows/pypi_env.yml) Workflows in various +- a set of [**GA Workflows**](https://github.com/boromir674/cicd-test/tree/main/.github/workflows) that call our [**Docker**](https://github.com/boromir674/automated-workflows/tree/main/.github/workflows/docker.yml), [**PyPI**](https://github.com/boromir674/automated-workflows/tree/main/.github/workflows/pypi_env.yml), and [**Lint**](https://github.com/boromir674/automated-workflows/tree/main/.github/workflows/lint_env.yml) Workflows in various *Scenarios* and *Situations* - a set of [**Automated Tests**](https://github.com/boromir674/cicd-test/tree/main/tests), implemented with `Pytest` (Python) - a *Test Runner* with a CLI (`pytest -n auto`) @@ -35,3 +35,10 @@ make the necessary assertions. | expected green pass | expected red (because scenario involves upload python dist to existing index, without allow_existing set to True) | | --- | --- | | [![gg](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/pypi_test.yaml/badge.svg)](https://github.com/boromir674/cicd-test/actions/workflows/pypi_test.yaml) | ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/pypi_test_red.yaml/badge.svg) | + + +## Lint Workflow Automated Tests + +| expected green pass | expected red | +| --- | --- | +| [![gg](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/static_code_green.yaml/badge.svg)](https://github.com/boromir674/cicd-test/actions/workflows/static_code_green.yaml) | | diff --git a/mkdocs.yml b/mkdocs.yml index 87e898e..870d5dc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -63,6 +63,7 @@ nav: - References: - "Docker": "ref_docker.md" - "PyPI": "ref_pypi_env.md" + - "Lint": "ref_lint.md" - Topics: - "Test Suite": tests_index.md - tags: tags.md diff --git a/scripts/gen-workflow-ref.py b/scripts/gen-workflow-ref.py index 28a091c..991617c 100644 --- a/scripts/gen-workflow-ref.py +++ b/scripts/gen-workflow-ref.py @@ -15,6 +15,7 @@ class InputArgument(TypedDict): required: bool # allowed values {True, False} type: str # common values {'string', 'boolean', 'number'} + default: t.Optional[str] # default value for the input argument ## Resuable Workflow 'secrets' ## class SecretArgument(TypedDict): @@ -133,7 +134,11 @@ def generate_markdown( ## Workflow Inputs ## markdown_content += f"{'#' * (max_mk_level + 1)} Inputs\n\n" required_inputs: t.Set[str] = {x for x in inputs.keys() if inputs[x].get("required", False)} + + # Required Inputs markdown_content += f"{'#' * (max_mk_level + 2)} Required Inputs\n\n" + if not required_inputs: + markdown_content += f"None\n" for input_name in sorted(required_inputs): markdown_content += f"- `{input_name}`\n" markdown_content += f" - type: _{inputs[input_name].get('type', 'string')}_\n" @@ -142,14 +147,20 @@ def generate_markdown( # Optional Inputs optional_inputs: t.Set[str] = {x for x in inputs.keys() if x not in required_inputs} markdown_content += f"{'#' * (max_mk_level + 2)} Optional Inputs\n\n" + if not optional_inputs: + markdown_content += f"None\n" for input_name in sorted(optional_inputs): markdown_content += f"- `{input_name}`\n" markdown_content += f" - type: _{inputs[input_name].get('type', 'string')}_\n" markdown_content += f" - Description: {inputs[input_name].get('description', '')}\n" + if inputs[input_name].get("default", None): + markdown_content += f" - Default: `{inputs[input_name]['default']}`\n" markdown_content += "\n" ## Workflow Secrets ## markdown_content += f"{'#' * (max_mk_level + 1)} Secrets\n\n" + if not secrets: + markdown_content += f"None\n" for secret_name, secret_details in sorted(secrets.items(), key=lambda x: x[0]): markdown_content += f"- `{secret_name}`\n" markdown_content += f" - type: _{secret_details.get('type', 'string')}_\n" @@ -159,12 +170,15 @@ def generate_markdown( ## Workflow Outputs ## markdown_content += f"{'#' * (max_mk_level + 1)} Outputs\n\n" + if not outputs: + markdown_content += f"None\n" for output_name, output_details in sorted(outputs.items(), key=lambda x: x[0]): markdown_content += f"- `{output_name}`\n" markdown_content += f" - type: _{output_details.get('type', 'string')}_\n" markdown_content += f" - Value: {output_details.get('value', '')}\n" markdown_content += f" - Description: {output_details.get('description', '')}\n" - markdown_content += "\n" + # omit last \n + # markdown_content += "\n" # markdown_content += "### Environments\n\n" # for environment_name, environment_value in repository_details["environments"].items(): From 4a27f5e7c710c67690c9d8079c73e7e0d1fadd37 Mon Sep 17 00:00:00 2001 From: konstantinos Date: Tue, 19 Dec 2023 20:58:34 +0200 Subject: [PATCH 3/3] docs(changelog): start mainting a CHANGELOG, and add all entries --- CHANGELOG.md | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ac902b9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,84 @@ +# Changelog + + +## 1.2.0 (2023-12-19) + +### Changes + +#### features + +- add new **Static Code Analysis** Workflow, leveraging `tox`, as `front-end` + +#### docs + +- website: add **Ref Page** for `Static Code Analysis` Reusable Workflow + + +## 1.1.4 (2023-12-08) + +We add `Continuous Integration`, implemented in `Github Actions`. +So, on `push` to `main`, a Workflow triggers, which runs the **[cicd-test](https://github.com/boromir674/cicd-test)** `Test Suite` (maintained in separate repo), +against our `Automated Workflows`, in various Scenarios. + +The `Test Suite`, includes `Python` **Tests**, which are responsible for triggering dedicated [Test Workflows](https://github.com/boromir674/cicd-test/blob/main/.github/workflows/) of **cicd-test**, in various Scenarios and Configurations, and automatically make `Assertions`. + +### Changes + +#### docs + +- rtd: generate Reference Pages from workflow yaml files on CI +- mkdocs: new 'References' left-side Section (lss), and nest 'Test Suite' under new 'Topics' lss +- reference-pages: add reference pages for docker.yml and pypi_env.yml reusable workflows, in Website +- readme: add ci code badges ! + +#### ci + +- add Job, which runs the CPU-distributed tests, which trigger our Automate Workflows, in diverse Test Scenarios + + +## 1.1.3 (2023-12-04) + +### Changes + +#### docs + +- add guide for CI/CD with PiPY upload in [Documentation Website](https://automated-workflows.readthedocs.io) + + +## 1.1.2 (2023-12-04) + +### Changes + +#### docs + +- add simple Guide for seting up CI/CD with Dockerhub, in [Documentation Website](https://automated-workflows.readthedocs.io) + + +## 1.1.1 (2023-12-04) + +**New Documentation Website** for Automated Workflows! +- Available at: https://automated-workflows.readthedocs.io + +### Changes + +#### docs + +- add the 'GNU Affero General Public License v3.0' LICENSE +- update `README` with URL links to CI, Docs, and Source Code & Code Badges + + +## 1.1.0 (2023-11-13) + +### Changes + +#### features + +allow controlling the `--skip-existing` *twine upload* flag, from Caller Job, when calling the **PyPI Resuable Workflow** + + +## 1.0.0 (2023-11-13) + +**CI/CD Pipelines** for **Docker** and **PyPI**. implemented as Github Actions Resuable Workflows + +- [**Docker**](.github/workflows/docker.yml): Build Docker image and Push to Dockerhub. +- [**PyPI**](.github/workflows/pypi_env.yml): Upload Python distribution to PyPI \ No newline at end of file