Skip to content

Commit

Permalink
[NEW] CI/CD and Git Ops v1.5.0
Browse files Browse the repository at this point in the history
Release v1.5.0
  • Loading branch information
boromir674 authored Jul 28, 2024
2 parents 96baa25 + 1818f66 commit d47d430
Show file tree
Hide file tree
Showing 44 changed files with 1,017 additions and 103 deletions.
9 changes: 9 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,21 @@ ci:
- changed-files:
- any-glob-to-any-file:
- '.github/workflows/_*'

# exclude the .github/workflows/_labeler.yml file
- all:
- changed-files:
- all-globs-to-all-files:
- '!*.github/workflows/_labeler.yml'

# OPTION 2
# ci:
# - all:
# - changed-files:
# - any-glob-to-any-file:
# - '.github/workflows/_*'
# - all-globs-to-all-files: '!.github/workflows/_labeler.yml'

pr_labeler:
- any:
- changed-files:
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/_auto-prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

- run: git branch --track "${{ env.RELEASE_BR }}" "origin/${{ env.RELEASE_BR }}"

Expand Down Expand Up @@ -120,4 +120,4 @@ jobs:
- name: "Merge PR 'head': ${{ env.RELEASE_BR }} --> 'base': ${{ env.MAIN_BR }}"
run: gh pr merge "${{ env.RELEASE_BR }}" --auto --delete-branch --merge --subject "${{ env.CM_MSG }}"
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
4 changes: 2 additions & 2 deletions .github/workflows/_merge-rt-in-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

# track remote Main branch, since this Workflow triggers on push tag or pull request activity
- name: Track Remote Main branch
Expand Down Expand Up @@ -109,4 +109,4 @@ jobs:
- name: 'Merge PR ${{ env.TRAIN }} --> ${{ env.RELEASE_BR }}'
run: gh pr merge "${{ env.TRAIN }}" --auto --delete-branch --merge
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} # allow event to trigger "listener" Workflows
GITHUB_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }} # allow event to trigger "listener" Workflows
12 changes: 6 additions & 6 deletions .github/workflows/_merge-to-boarding.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

- run: echo "PR_ID=${{ github.event.pull_request.number }}" >> $GITHUB_ENV

Expand All @@ -45,7 +45,7 @@ jobs:
## this will eclipse the need to maintain the Required Checks on github repo settings

env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
TOPICAL_BR: ${{ github.event.pull_request.head.ref }}

# Recognized labels:
Expand All @@ -65,7 +65,7 @@ jobs:
# Check if PR labels are in the list of recognized labels
for label in $PR_LABELS; do
case $label in
business_logic|template_logic|test|stubs)
rw_docker|rw_test_python|rw_gh_release|rw_go_pr_to_boarding|rw_docs|rw_lint|rw_pydeps)
if [ "$detected_need_for_distro_tests" = false ]; then
buffer_string="Distro${buffer_string:+ AND $buffer_string}"
branch_name_buffer="distro-${branch_name_buffer}"
Expand Down Expand Up @@ -114,7 +114,7 @@ jobs:
## MERGE PR 'User/Topic Branch' --> 'boarding-auto', with Custom Commit Message
- name: 'Merge PR ${{ env._USER_BR }} --> ${{ env._BOARDING_BR }}'
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
BOARDING_MSG: ${{ steps.merge_msg.outputs.MERGE_MSG }}
run: "gh pr merge \"${{ env.PR_ID }}\" --merge --subject \"${{ env.BOARDING_MSG }}\""

Expand All @@ -135,7 +135,7 @@ jobs:
### Open PR 'boarding-auto' --> Branch with `Required Checks`; github Protection Rules
- name: 'Open PR ${{ env._BOARDING_BR }} --> ${{ env.TARGET_BRANCH }}'
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
run: |
gh pr create --head "${{ env._BOARDING_BR }}" --base "${{ env.TARGET_BRANCH }}" \
--title "Dynamic Automated Checks: Merging '${{ env._BOARDING_BR }}' into '${{ env.TARGET_BRANCH }}' Branch" \
Expand All @@ -162,7 +162,7 @@ jobs:
- name: 'Enable Auto Merge'
## The CI Tests derived from commit message, must match the `Required Checks`
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
# DETERMINES what Job Matrix our CI/CD Pipeline, will run
BOARDING_MSG: ${{ steps.merge_msg.outputs.MERGE_MSG }}
run: "gh pr merge \"${{ env._BOARDING_BR }}\" --auto --delete-branch --merge"
6 changes: 3 additions & 3 deletions .github/workflows/_merge-to-train.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

- name: "Ensure Upstream '${{ env.MAIN_BR }}' is tracked (has local counterpart)"
run: git branch --track "${{ env.MAIN_BR }}" "origin/${{ env.MAIN_BR }}" || echo "[WARNING] Branch '${{ env.MAIN_BR }}' already exists locally"
Expand Down Expand Up @@ -66,7 +66,7 @@ jobs:
############## PR ##############
- name: "Create PR Head: '${{ env.INCOMING_BR }}' --> Base: '${{ env.TRAIN }}'"
env:
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
run: |
gh pr create --head "${{ env.INCOMING_BR }}" --base "${{ env.TRAIN }}" \
--title "Merging '${{ env.INCOMING_BR }}', into '${{ env.TRAIN }}'" \
Expand Down Expand Up @@ -100,5 +100,5 @@ jobs:
run: gh pr merge "${{ env.INCOMING_BR }}" --auto --merge --delete-branch
# --delete-branch --squash
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
GH_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
# HEAD is now at ${{ env.TRAIN }}
2 changes: 1 addition & 1 deletion .github/workflows/_pr-to-boarding.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ jobs:
board_tag: "${{ github.ref_name }}"
main_branch: ${{ vars.MAIN_BRANCH || 'main' }}
secrets:
github_pat: ${{ secrets.GH_TOKEN }}
github_pat: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
4 changes: 2 additions & 2 deletions .github/workflows/_pr-to-master.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

##### Open PR: Release --> Main/Master #####
- name: "Open PR 'head': ${{ env.RELEASE }} --> 'base': ${{ env.MAIN_BR }}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_TOKEN: ${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}
run: |
gh pr create --head "${{ env.RELEASE }}" --base "${{ env.MAIN_BR }}" \
--title "Release Version '${{ steps.sem_ver.outputs.SEMVER }}' into '${{ env.MAIN_BR }}' Branch" \
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_start-rt.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

# CHECKOUT 'Release' branch
- name: "Checkout local branch '${{ env.RELEASE_BR }}'"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/_tag-prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
with:
fetch-depth: 0 # 0 indicates all history for all branches and tags.
set-safe-directory: '' # `git config --global --add safe.directory <path>`
token: '${{ secrets.GH_TOKEN }}'
token: '${{ secrets.GA_WORKFLOWS_CI_PR_RW_AND_ACTIONS_RW }}'

- name: Ensure we are on '${{ env.MAIN_BR }}' branch
run: git checkout "${{ env.MAIN_BR }}"
Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/cicd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:
- run-ci

env:
JOB_MATRIX: "{\"tests_filter\": [\"docker_pol0 or docker_pol1\", \"docker_pol2 or docker_pol3\", \"code_viz or docker_test_when_tag_not_given\", \"pypi\", \"go_pr_ or static_code\"]}"
JOB_MATRIX: "{\"tests_filter\": [\"docker_pol0 or docker_pol1\", \"docker_pol2 or docker_pol3\", \"code_viz or docker_test_when_tag_not_given\", \"pypi\", \"go_pr_ or static_code or go_single_status\"]}"

##### Pipeline - 1st level overrides #####
TESTS_PIPE_ON: "true"
Expand Down Expand Up @@ -50,6 +50,8 @@ jobs:
if: ${{ env.AFFECTED_COMPONENTS != '' }}
id: pipe_config_from_gitops
run: |
echo PIPE_TESTS_GITOPS_POLICY='0' >> $GITHUB_ENV
echo PIPE_DOCS_GITOPS_POLICY='0' >> $GITHUB_ENV
if [[ "${{ env.AFFECTED_COMPONENTS }}" =~ "Source" ]]; then
echo PIPE_TESTS_GITOPS_POLICY='1' >> $GITHUB_ENV
fi
Expand All @@ -61,8 +63,8 @@ jobs:
- name: Configure Pipeline
id: set_pipe_config
run: |
echo PIPE_TESTS_POLICY="${{ (env.TESTS_PIPE_ON != 'true' && '0') || (env.TESTS_ALWAYS == 'true' && '1') || (env.PIPE_TESTS_GITOPS_POLICY == '1' && '1') || env.TESTS_JOB_POLICY }}" >> $GITHUB_OUTPUT
echo PIPE_DOCS_POLICY="${{ (env.DOCS_PIPE_ON != 'true' && '0') || (env.DOCS_ALWAYS == 'true' && '1') || (env.PIPE_DOCS_GITOPS_POLICY == '1' && '1') || env.DOCS_JOB_POLICY }}" >> $GITHUB_OUTPUT
echo PIPE_TESTS_POLICY="${{ (env.TESTS_PIPE_ON != 'true' && '0') || (env.TESTS_ALWAYS == 'true' && '1') || (env.AFFECTED_COMPONENTS != '' && env.PIPE_TESTS_GITOPS_POLICY) || env.TESTS_JOB_POLICY }}" >> $GITHUB_OUTPUT
echo PIPE_DOCS_POLICY="${{ (env.DOCS_PIPE_ON != 'true' && '0') || (env.DOCS_ALWAYS == 'true' && '1') || (env.AFFECTED_COMPONENTS != '' && env.PIPE_DOCS_GITOPS_POLICY) || env.DOCS_JOB_POLICY }}" >> $GITHUB_OUTPUT
echo PIPE_DOCS_PY="${{ env.DOCS_PY_VERSION }}" >> $GITHUB_OUTPUT
echo JOB_MATRIX="${JOB_MATRIX}" >> $GITHUB_OUTPUT
outputs:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/go-pr-to-boarding.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
if: steps.user_branch.outputs.USER_BRANCH == ''
run: echo "[ERROR] User Branch not found. Searched for an Upstream branch, but was not found on the remote. Exiting.." && exit 1

- name: Create name for Boarding Branch
- name: 'Derive name for Boarding target PR (aka base) branch: ${{ inputs.board_tag }}-${{ steps.user_branch.outputs.USER_BRANCH }}'
run: echo "BOARDING_BR=${{ inputs.board_tag }}-${{ steps.user_branch.outputs.USER_BRANCH }}" >> $GITHUB_ENV

# Ensure Upstream 'Boarding' Branch exists
Expand Down
62 changes: 62 additions & 0 deletions .github/workflows/go-single-status.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
###########################
## Required Jobs Success ##
###########################

### Features
# single Status to simplify populating and maintaining Github Required Checks

# To simplify adding "Job Statuses" to Github Required Checks, provide just this
# Job to Github Required Checks and declare your "Job Statuses" as job.needs of
# a caller of this.

# To require ALL "Job Statuses" GREEN, from caller's job.needs, supply
# only the 'needs_json' input, with '${{ toJSON(needs) }}' as value

# To have the "logic" of the 'Status Signal' be configurable at runtime for
# each of your CI/CD Pipeline runs, add the maximal set of available/implemented
# QA CI Jobs (ie unit-test, lint, e2e-test, integration-tests, audit, etc) in
# this caller's job.needs section and then control "severity" using the
# 'allowed-failures' and 'allowed-skips' Workflow inputs.

# If you have separate CI and CD Workflows, then add this to the CI Workflow.

# Useful when populating and maintaining Github Required Checks, which involve many
# "Job Statuses", as for example if you do Git Ops involving Github Auto Merge

# Useful to dynamically control the Acceptance Criteria of PR Auto Merge

on:
workflow_call:
inputs:
needs_json:
type: string
description: "Always supply \\$\\{\\{ toJSON(needs) \\}\\} as value. It's a JSON array of caller job.needs."
required: true
## OPTIONAL INPUTS
allowed-failures:
description: 'Job names that are allowed to fail and not affect the outcome, as a comma-separated list or serialized as a JSON string (ie with toJSON)'
default: >-
[]
type: string
required: false
allowed-skips:
description: >-
Job names that are allowed to be skipped and not affect the outcome,
as a comma-separated list or serialized as a JSON string
default: >-
[]
type: string
required: false

jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Emit Acceptance Signal as Job Status
# uses: re-actors/alls-green@cf9edfcf932a0ed6b431433fa183829c68b30e3f
uses: boromir674/ga-acceptance@dev
with:
# only jobs, means ALL Jobs Green
allowed-failures: ${{ inputs.allowed-failures }}
allowed-skips: ${{ inputs.allowed-skips }}
jobs: ${{ inputs.needs_json }}
14 changes: 8 additions & 6 deletions .readthedocs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,14 @@ build:
- python -m poetry export -o req-docs.txt -E docs
# post_install:
# - python -m pip install -e .
pre_build:
- chmod +x ./scripts/gen-workflow-ref.py
- ./scripts/gen-workflow-ref.py ./.github/workflows/docker.yml > ./docs/ref_docker.md
- ./scripts/gen-workflow-ref.py ./.github/workflows/pypi_env.yml > ./docs/ref_pypi_env.md
- ./scripts/gen-workflow-ref.py ./.github/workflows/lint.yml > ./docs/ref_lint.md
- ./scripts/gen-workflow-ref.py ./.github/workflows/python_imports.yml > ./docs/ref_pydeps.md

# Ref Pages are kept up-to-date with scripts/gen-docs-refs.sh, so we do not pre_build
# pre_build:
# - chmod +x ./scripts/gen-workflow-ref.py
# - ./scripts/gen-workflow-ref.py ./.github/workflows/docker.yml > ./docs/ref_docker.md
# - ./scripts/gen-workflow-ref.py ./.github/workflows/pypi_env.yml > ./docs/ref_pypi_env.md
# - ./scripts/gen-workflow-ref.py ./.github/workflows/lint.yml > ./docs/ref_lint.md
# - ./scripts/gen-workflow-ref.py ./.github/workflows/python_imports.yml > ./docs/ref_pydeps.md


# Build documentation in the "docs/" directory with mkdocs
Expand Down
24 changes: 24 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,30 @@
# Changelog


## 1.5.0

New **`Single Status`** Workflow for **Git Ops**,

### Changes

##### feature
- improve step.name content
- add 'Single Status' Workflow for Git Ops

##### build
- add docker and compose files for docs_dev server

##### git ops
- pass in PAT with PR r/w and Actions r/w; change logic of Pipe Config based on Gitops Signal

##### docs
- add all current User Facing Worklflows in Site Navigation
- redesign README.md with better sections and collapsible content
- add 'topic' about go-single-status.yml workflow idea
- automatically generate API Ref Pages and Add them to Site Navigation, if missing
- do not generate Ref Pages during Read The Docs Builds


## 1.4.0 (2024-03-15)

**2 New Reusable Workflows:**
Expand Down
54 changes: 54 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
### BASE ENVS ###
# Base Env 1: For copying (ie from host)
FROM scratch AS strach_env
WORKDIR /workspace

# Base Env 2: For providing app runtime
FROM python:3.12.4-slim-bullseye AS python_env

### BUILDER ENVS ###
# Builder 1: Tooling for Docs development
FROM python_env AS install_tox
RUN pip install --user tox==3.27.1

# Builder 2: Pinned dependency versions, satisfying constraints, in /workspace
FROM strach_env AS dependencies
COPY pyproject.toml .
COPY poetry.lock .


FROM python_env AS install_docs
# Normally, we do a `poetry install` here, but we're using `tox` to manage the environment
# Instead copy the tox distribution from the install_tox image
COPY --from=install_tox /root/.local /root/.local
WORKDIR /workspace
COPY --from=dependencies /workspace .
COPY tox.ini .
RUN /root/.local/bin/tox -e pin-deps -- -E docs
RUN /root/.local/bin/tox -e docs-live --notest


### RUNTIME ENVS ###
FROM python_env AS docs
COPY --from=install_docs /workspace .
COPY --from=install_tox /root/.local /root/.local

FROM docs AS docs_gen
# requires run with volumes
# apt get install some py yaml
RUN apt-get update && \
apt-get install -y --no-install-recommends \
python3-yaml && \
apt-get clean all

# RUN /.tox/docs-live/bin/python -m pip install pyyaml
RUN pip install pyyaml

FROM docs AS docs_live
# requires run with volumes:
# - ./docs/:/workspace/docs/
# - ./mkdocs.yml:/workspace/mkdocs.yml
CMD [ "/root/.local/bin/tox", "-e", "docs-live", "--", "-w", "docs", "-w", "mkdocs.yml", "-a", "0.0.0.0:8020"]


# Runtime Env 1: Docs
Loading

0 comments on commit d47d430

Please sign in to comment.