diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 0000000..962b617 --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,42 @@ +name: 'Automated Workflows - CI Pipeline' + +on: + push: + branches: + - main + - dev + pull_request: + branches: + - main + +jobs: + test_workflows: + runs-on: ubuntu-latest + steps: + - name: Checkout the Automated Test Suite + uses: actions/checkout@v4 + with: + repository: 'boromir674/cicd-test' + ref: 'main' + path: 'cicd-test' + - name: Setup Python ${{ env.py_ver }} + env: + py_ver: '3.10' + uses: actions/setup-python@v4 + with: + python-version: '${{ env.py_ver }}' + - name: Install tox + run: pip install 'tox<4.0' + + - name: Pin Python Dependencies + run: | + cd cicd-test + tox -e pin-deps + + - name: Run Automated Test Suite + env: + GH_TOKEN: ${{ github.token }} + CICD_TEST_GH_TOKEN: ${{ secrets.CICD_TEST_GH_TOKEN }} + run: | + cd cicd-test + tox -e test -vv diff --git a/.gitignore b/.gitignore index 18cf84c..828eaa9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ package\.json site/ req-docs\.txt reqs\.txt +\.tox/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 6c4a0c5..93a552b 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -26,9 +26,10 @@ build: - python -m poetry export -o req-docs.txt -E docs # post_install: # - python -m pip install -e . - # pre_build: - # - python ./scripts/visualize-ga-workflow.py > ./docs/cicd_mermaid.md - + pre_build: + - chmod +x ./scripts/gen-workflow-ref.py + - ./scripts/gen-workflow-ref.py ./.github/workflows/docker.yml > ./docs/ref_docker.md + - ./scripts/gen-workflow-ref.py ./.github/workflows/pypi_env.yml > ./docs/ref_pypi_env.md # Build documentation in the "docs/" directory with mkdocs diff --git a/README.md b/README.md index 6d12cab..79932cc 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,13 @@ `Reusable Workflows` for **CI/CD Pipelines**, implemented as `Github Actions Workflows`. +[![Workflows Tests](https://img.shields.io/github/actions/workflow/status/boromir674/automated-workflows/cicd.yml?style=plastic&logo=github-actions&logoColor=lightblue&label=Tests&color=lightgreen&link=https%3A%2F%2Fgithub.com%2Fboromir674%2Fautomated-workflows%2Factions%2Fworkflows%2Fcicd.yml)](https://github.com/boromir674/automated-workflows/actions/workflows/cicd.yml) +[![Read the Docs](https://img.shields.io/readthedocs/automated-workflows?style=plastic&logo=readthedocs&logoColor=lightblue&label=Docs&color=lightgreen&link=https%3A%2F%2Fautomated-workflows.readthedocs.io%2F)](https://automated-workflows.readthedocs.io) +[![license](https://img.shields.io/github/license/boromir674/automated-workflows?style=plastic&)](https://github.com/boromir674/automated-workflows/blob/main/LICENSE) + - Source: https://github.com/boromir674/automated-workflows -- Docs: https://automated-workflows.readthedocs.io - CI: https://github.com/boromir674/cicd-test/actions - -| expected green | expected red | -| --- | --- | -| [![gg](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol0_green_0000_1100.yaml/badge.svg)](https://github.com/boromir674/cicd-test/actions/workflows/docker_pol0_green_0000_1100.yaml) ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol1_green_0001_1101.yaml/badge.svg) ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol2_green_1110_0010.yaml/badge.svg) ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol3_green_1111_0011.yaml/badge.svg) | ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol0_red_0100.yaml/badge.svg) ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol1_red_0101.yaml.yaml/badge.svg) ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol2_red_0110.yaml/badge.svg) ![](https://github.com/boromir674/cicd-test/actions/workflows/.github/workflows/docker_pol3_red_0111.yaml/badge.svg) | - -[![docs](https://readthedocs.org/projects/automated-workflows/badge/?version=main)](https://automated-workflows.readthedocs.io/en/main/?badge=main) - -[![license](https://img.shields.io/github/license/boromir674/automated-workflows)](https://github.com/boromir674/automated-workflows/blob/main/LICENSE) +- Docs: https://automated-workflows.readthedocs.io ## Workflows Overview diff --git a/docs/ref_docker.md b/docs/ref_docker.md new file mode 100644 index 0000000..58c72f2 --- /dev/null +++ b/docs/ref_docker.md @@ -0,0 +1,60 @@ +# Workflow docker.yml + +### Trigger Events + +If any of the below events occur, the `docker.yml` workflow will be triggered. + +- workflow_call + +Since there is a `workflow_call` _trigger_event_, this workflow can be triggered (called) by another (caller) workflow. +> Thus, it is a `Reusable Workflow`. + + +## Reusable Workflow + +Event Trigger: `workflow_call` + +### Inputs + +#### Required Inputs + +- `DOCKER_USER` + - type: _string_ + - Description: +- `image_slug` + - type: _string_ + - Description: + +#### Optional Inputs + +- `acceptance_policy` + - type: _string_ + - Description: +- `image_tag` + - type: _string_ + - Description: +- `target_stage` + - type: _string_ + - Description: +- `tests_pass` + - type: _boolean_ + - Description: +- `tests_run` + - type: _boolean_ + - Description: + +### Secrets + +- `DOCKER_PASSWORD` + - type: _string_ + - Required: True + - Description: + +### Outputs + +- `IMAGE_REF` + - type: _string_ + - Value: ${{ jobs.docker_build.outputs.IMAGE_REF }} + - Description: Docker Image reference + + diff --git a/docs/ref_pypi_env.md b/docs/ref_pypi_env.md new file mode 100644 index 0000000..ec97c1e --- /dev/null +++ b/docs/ref_pypi_env.md @@ -0,0 +1,59 @@ +# Workflow pypi_env.yml + +### Trigger Events + +If any of the below events occur, the `pypi_env.yml` workflow will be triggered. + +- workflow_call + +Since there is a `workflow_call` _trigger_event_, this workflow can be triggered (called) by another (caller) workflow. +> Thus, it is a `Reusable Workflow`. + + +## Reusable Workflow + +Event Trigger: `workflow_call` + +### Inputs + +#### Required Inputs + +- `artifacts_path` + - type: _string_ + - Description: +- `distro_name` + - type: _string_ + - Description: +- `distro_version` + - type: _string_ + - Description: +- `pypi_env` + - type: _string_ + - Description: +- `should_trigger` + - type: _boolean_ + - Description: + +#### Optional Inputs + +- `allow_existing` + - type: _boolean_ + - Description: +- `dist_folder` + - type: _string_ + - Description: +- `require_wheel` + - type: _boolean_ + - Description: + +### Secrets + +- `TWINE_PASSWORD` + - type: _string_ + - Required: False + - Description: + +### Outputs + + + diff --git a/mkdocs.yml b/mkdocs.yml index 7374470..87e898e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -54,12 +54,17 @@ markdown_extensions: - pymdownx.highlight nav: + # each item in the list, renders a left side navigation item/page - Home: - "Quick Start": index.md - Guides: - "CI/CD: Dockerhub": "guide_setup_cicd.md" - "CI/CD: PyPI": "guide_setup_pypi.md" - - Test Suite: tests_index.md + - References: + - "Docker": "ref_docker.md" + - "PyPI": "ref_pypi_env.md" + - Topics: + - "Test Suite": tests_index.md - tags: tags.md diff --git a/scripts/gen-workflow-ref.py b/scripts/gen-workflow-ref.py new file mode 100644 index 0000000..28a091c --- /dev/null +++ b/scripts/gen-workflow-ref.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 + +import typing as t + +from typing import TypedDict + +import argparse +from pathlib import Path +import yaml +from yaml.resolver import Resolver + +#### TYPES #### + +## Resuable Workflow 'inputs' ## +class InputArgument(TypedDict): + required: bool # allowed values {True, False} + type: str # common values {'string', 'boolean', 'number'} + +## Resuable Workflow 'secrets' ## +class SecretArgument(TypedDict): + required: bool # allowed values {True, False} + +## Resuable Workflow 'outputs' ## +class OutputArgument(TypedDict): + description: str + value: t.Union[str, int, float, bool, t.List] + +## Resuable Workflow; data under 'workflow_call' (inside 'on' section) ## +class ResuableWorkflowInterface(TypedDict): + """Keys found under the 'workflow_call' key (which is under the 'on' key)""" + inputs: t.Dict[str, InputArgument] + secrets: t.Dict[str, SecretArgument] + outputs: t.Dict[str, OutputArgument] + +class Events(TypedDict): + """Keys found under the 'on' key""" + workflow_call: ResuableWorkflowInterface + +## Job item in 'jobs' list, in document level ## +JobDataInterface = TypedDict('JobDataInterface', { + 'runs-on': str, + 'if': str, + 'name': str, + 'needs':t.List[str], + 'steps': t.List[t.Dict[str, t.Any]], +}) + +class JobData(JobDataInterface, total=False): + pass + +### Resuable Workflow ### +WorkflowData = TypedDict('WorkflowData', { + 'on': Events, + # 'on': { + # 'workflow_call': ResuableWorkflowInterface, + # }, + 'jobs': t.Dict[str, JobData], + 'if': str, +}) + + +def parse_workflow( + workflow_data: WorkflowData +) -> t.Tuple[str, t.Dict[str, t.Optional[t.Any]], ResuableWorkflowInterface, t.Dict[str, t.Any]]: + + # pipeline_jobs: t.Dict[str, JobData] = workflow_data["jobs"] + + # pipeline_jobs: t.List[JobData] = workflow_data.get("jobs", {}).get("docker_build", {}) + + ## Read the job name, if Reusable Workflow declares only 1 job ## + # job_name: t.Optional[str] = None + # if len(pipeline_jobs) == 1: + # for jobname, job_data in pipeline_jobs.items(): + # job_name = job_data.get("name", jobname) + + ## Read Trigger Events; they trigger the workflow, whenever they occur ## + events: t.Dict[str, t.Optional[t.Any]] = workflow_data.get('on', {}) + + # anticipate, that we read a Resuable Github Actions Workflow + # so we expect the 'workflow_call' key to be present, under 'on' key + workflow_interface: ResuableWorkflowInterface = events['workflow_call'] + + # Extract workflow top-level env vars + env_section: t.Dict[str, t.Any] = workflow_data.get("env", {}) + + return events, workflow_interface, env_section + + +def generate_markdown( + workflow_name: str, + events: t.Dict[str, t.Optional[t.Any]], + workflow_interface: ResuableWorkflowInterface, + env_section: t.Dict[str, t.Any], + max_mk_level: int = 2 # max markdown heading level to write +): + # string buffer to hold markdown content + # Write Markdown Top Level Header (#) + markdown_content: str = '' + # markdown_content: str = f"# {workflow_name}\n\n" + + # destructure workflow_interface + inputs: t.Dict[str, InputArgument] = workflow_interface.get("inputs", {}) + secrets: t.Dict[str, SecretArgument] = workflow_interface.get("secrets", {}) + outputs: t.Dict[str, OutputArgument] = workflow_interface.get("outputs", {}) + + # Workflow Events - Section # + markdown_content += f"{'#' * (max_mk_level + 1)} Trigger Events\n\n" + markdown_content += ( + f"If any of the below events occur, the `{workflow_name}` workflow " + "will be triggered.\n\n" + ) + # events List + for event in sorted(events.keys()): + markdown_content += f"- {event}\n" + markdown_content += "\n" + markdown_content += ( + f"Since there is a `workflow_call` _trigger_event_, this workflow can " + "be triggered (called) by another (caller) workflow.\n" + "> Thus, it is a `Reusable Workflow`.\n\n" + ) + # Workflow env vars - Section # + markdown_content += ( + f"{'#' * max_mk_level} Environment Variables\n\n" + f"Environment variables set in the workflow's `env` section:\n" + '\n'.join([f"- {env_var_name}: {env_var_value}" for env_var_name, env_var_value in sorted(env_section.items(), key=lambda x: x[0])]) + \ + "\n" + ) + # Resuable Workflow - Section # + markdown_content += f"{'#' * max_mk_level} Reusable Workflow\n\n" + markdown_content += ( + f"Event Trigger: `workflow_call`\n\n" + ) + ## Workflow Inputs ## + markdown_content += f"{'#' * (max_mk_level + 1)} Inputs\n\n" + required_inputs: t.Set[str] = {x for x in inputs.keys() if inputs[x].get("required", False)} + markdown_content += f"{'#' * (max_mk_level + 2)} Required Inputs\n\n" + for input_name in sorted(required_inputs): + markdown_content += f"- `{input_name}`\n" + markdown_content += f" - type: _{inputs[input_name].get('type', 'string')}_\n" + markdown_content += f" - Description: {inputs[input_name].get('description', '')}\n" + markdown_content += "\n" + # Optional Inputs + optional_inputs: t.Set[str] = {x for x in inputs.keys() if x not in required_inputs} + markdown_content += f"{'#' * (max_mk_level + 2)} Optional Inputs\n\n" + for input_name in sorted(optional_inputs): + markdown_content += f"- `{input_name}`\n" + markdown_content += f" - type: _{inputs[input_name].get('type', 'string')}_\n" + markdown_content += f" - Description: {inputs[input_name].get('description', '')}\n" + markdown_content += "\n" + + ## Workflow Secrets ## + markdown_content += f"{'#' * (max_mk_level + 1)} Secrets\n\n" + for secret_name, secret_details in sorted(secrets.items(), key=lambda x: x[0]): + markdown_content += f"- `{secret_name}`\n" + markdown_content += f" - type: _{secret_details.get('type', 'string')}_\n" + markdown_content += f" - Required: {secret_details.get('required', False)}\n" + markdown_content += f" - Description: {secret_details.get('description', '')}\n" + markdown_content += "\n" + + ## Workflow Outputs ## + markdown_content += f"{'#' * (max_mk_level + 1)} Outputs\n\n" + for output_name, output_details in sorted(outputs.items(), key=lambda x: x[0]): + markdown_content += f"- `{output_name}`\n" + markdown_content += f" - type: _{output_details.get('type', 'string')}_\n" + markdown_content += f" - Value: {output_details.get('value', '')}\n" + markdown_content += f" - Description: {output_details.get('description', '')}\n" + markdown_content += "\n" + + # markdown_content += "### Environments\n\n" + # for environment_name, environment_value in repository_details["environments"].items(): + # markdown_content += f"- {environment_name}: {environment_value}\n" + # markdown_content += "\n" + + return markdown_content + + +##### YAML Workflow PARSER ##### +def parse_workflow_file(workflow_file: Path) -> WorkflowData: + # the 'on' yaml section, which is used for declaring what events trigger the workflow, + # is parsed by default as an (inner) dict mapped not by a 'on' string but by the True boolean value! + # About this behaviour see: + # - https://stackoverflow.com/questions/36463531/pyyaml-automatically-converting-certain-keys-to-boolean-values + # - https://docs.saltstack.com/en/latest/topics/troubleshooting/yaml_idiosyncrasies.html + + # we explicitly define the YAML behaviour so we parse sections with this behaviour + # as strings instead of the True boolean value + + # remove resolver entries for On/Off/Yes/No + for ch in "OoYyNn": + if len(Resolver.yaml_implicit_resolvers[ch]) == 1: + del Resolver.yaml_implicit_resolvers[ch] + else: + Resolver.yaml_implicit_resolvers[ch] = [x for x in + Resolver.yaml_implicit_resolvers[ch] if x[0] != 'tag:yaml.org,2002:bool'] + file_content: str = workflow_file.read_text() + return yaml.safe_load(file_content) + + +##### CLI Args READ ##### +def parse_cli_args() -> t.Tuple[Path, t.Optional[str]]: + """ + Parse command line arguments for processing a .github/workflows/*.yaml file. + + Returns: + Tuple[Path, Optional[str]]: Tuple containing the Path to the workflow YAML file + and an optional output path. If not specified, the result is printed to stdout. + """ + parser = argparse.ArgumentParser(description='Process a .github/workflows/*.yaml file') + parser.add_argument( + 'workflow_path', + help='Path to the workflow YAML file. Example: my_workflow.yaml', + metavar='WORKFLOW_PATH' + ) + parser.add_argument( + '-o', '--output', + help='Output path. If not specified, the result will be printed to stdout.', + metavar='OUTPUT_PATH' + ) + args = parser.parse_args() + + # Convert the provided path to a Path object + workflow_file: Path = Path(args.workflow_path) + + # If the provided file path doesn't exist, try finding it in the current working directory + if not workflow_file.exists(): + workflow_file = Path.cwd() / args.workflow_path + + return workflow_file, args.output + + +######## MAIN ######## +def main(): + workflow_file, output_dest = parse_cli_args() + + # Read the GitHub Actions workflow file in a dict + workflow_data = parse_workflow_file(workflow_file) + + # Parse workflow data + events, workflow_interface, env_section = parse_workflow(workflow_data) + + # Generate Markdown content + markdown_content = generate_markdown( + workflow_file.name, + events, + workflow_interface, + env_section, + max_mk_level=2 + ) + + # Add Top Level Header with Document Title + markdown_content = f"# Workflow {workflow_file.name}\n\n" + markdown_content + + # Print Markdown content to stdout if no output path is specified + if output_dest is None: + print(markdown_content) + return + # Write Markdown content to a file + with open(output_dest, 'w') as f: + f.write(markdown_content) + + +if __name__ == "__main__": + main() + +# Usage example: ./scripts/gen-workflow-ref.py ./.github/workflows/docker.yml > ./docs/ref_docker.md \ No newline at end of file