[NEW] CICD Pipelines v1.3.0

Release v1.3.0

.github/workflows/cicd.yml

@@ -4,10 +4,16 @@ on:
   push:
     branches:
       - main
-      - dev
+    tags:
+      - v*
+      - run-ci
   pull_request:
     branches:
       - main
+env:
+  # Groups Test Cases into sets of 4 and dedicate a Job, to run each set
+  # Job runs on a machine with 4 cpu threads; see pytest-xdist
+  JOB_MATRIX: "{\"tests_filter\": [\"docker_pol0 or docker_pol1\", \"docker_pol2 or docker_pol3\", \"code_viz or docker_test_when_tag_not_given\", \"pypi_test or static_code\"]}"
 
 jobs:
   test_workflows:

.github/workflows/policy_docs.yml

@@ -0,0 +1,133 @@
+#################################
+## DOCS STATIC SITE Build/Test ##
+##      Reusable Workflow      ##
+#################################
+
+### TOOLS ###
+# - mkdocs
+# - sphinx
+
+## AUTONOMOUS JOB ##
+
+#  0. Never run
+#  1. Always run
+#  2. Run Docs on conditions
+#     - Triggered on a Long-living branch (ie main)
+#     - Triggered on a v* tag (ie v1.0.0)
+#     - Docs Source Files (ie docs/) changed, compared to previous commit
+#  3. Run Docs, if Source Files changed, compared to previous commit
+
+on:
+  workflow_call:
+    inputs:
+      # Defaults to Policy 2 (CI/CD)
+      run_policy:
+        default: '2'
+        type: string
+        description: 'Policy for when to run Docs Build'
+        required: false
+      dedicated_branches:
+        default: 'main, master, dev'
+        type: string
+        description: "Branches to always want to run Docs Build. Has effect only on Policy '2'"
+        required: false
+      source_code_targets:
+        default: 'docs/'
+        type: string
+        required: false
+        description: "Comma separated list of folders to watch for changes. Has effect only on Policy '2' and '3'"
+      ## BUILD COMMAND
+      command:
+        default: 'tox -e docs --sitepackages -vv -s false'
+        type: string
+        description: 'Docs Build Command to run'
+        required: false
+      ## Parametrizing Runtime Environment (ie py version)
+      python_version:
+        default: '3.10'
+        type: string
+        description: 'Python Interpreter Version to use for Docs Build Environment'
+        required: false
+
+jobs:
+  docs_policy:
+    name: "Run Docs Workflow/Job?"
+    runs-on: ubuntu-latest
+    if: always() && inputs.run_policy != 0
+    steps:
+      - if: ${{ !contains('1, 2, 3', inputs.run_policy) }}
+        run: 'echo "Invalid run_policy: ${{ inputs.run_policy }}. Must be >0 and <4" && exit 1'
+
+      - if: inputs.run_policy == 1
+        name: 'POLICY: 1 -> Trigger'
+        run: echo "SHOULD_RUN_DOCS=true" >> $GITHUB_ENV
+
+      - if: inputs.run_policy == 2 && contains(inputs.dedicated_branches, github.ref_name)
+        name: 'POLICY: 2 & Branch: ${{ github.ref_name }} -> Trigger'
+        run: echo "SHOULD_RUN_DOCS=true" >> $GITHUB_ENV
+
+      - if: inputs.run_policy == 2 && startsWith(github.ref, 'refs/tags/v')
+        name: 'POLICY: 2 & Tag: ${{ github.ref_name }} -> Trigger'
+        run: echo "SHOULD_RUN_DOCS=true" >> $GITHUB_ENV
+
+      - if: ${{ env.SHOULD_RUN_DOCS != 'true' && contains('2, 3', inputs.run_policy) }}
+        name: 'POLICY: 2, 3 -> Derive from DIFF'
+        run: echo "SHOULD_DERIVE_FROM_DIFF=true" >> $GITHUB_ENV
+
+      - if: ${{ env.SHOULD_DERIVE_FROM_DIFF }}
+        name: 'POLICY: 2, 3 -> Checkout Code to compute DIFF'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - if: ${{ env.SHOULD_DERIVE_FROM_DIFF }}
+        name: 'POLICY: 2, 3 -> Check Docs Source Files DIFF'
+        run: |
+          echo "============ List Modified Files ============"
+          git diff --name-only HEAD^ HEAD
+          CHANGED_FILES=$(git diff --name-only HEAD^ HEAD)
+
+          # Read Folders we 'Watch' for changes
+          TARGETS=$(echo "${{ inputs.source_code_targets }}" | tr ',' '\n')
+
+          # Loop through the Watched Folders
+          for TARGET in $TARGETS; do
+            # if rel path of changed file matches glob pattern
+            if [[ $CHANGED_FILES == *"$TARGET"* ]]; then
+              echo "SHOULD_RUN_DOCS=true" >> $GITHUB_ENV
+              echo " --> DOCS Change, from previous commit, found: $TARGET"
+              break
+            fi
+          done
+
+      ### OUTPUT of JOB ###
+      - name: "Set 'Run Docs' Signal to ${{ env.SHOULD_RUN_DOCS }}"
+        id: set_docs_signal
+        run: echo "RUN_DOCS=${{ env.SHOULD_RUN_DOCS }}" >> $GITHUB_OUTPUT
+    outputs:
+      RUN_DOCS: ${{ steps.set_docs_signal.outputs.RUN_DOCS }}
+
+  docs:
+    name: "Docs: Build & Test"
+    runs-on: ubuntu-latest
+    needs: docs_policy
+    if: always() && needs.docs_policy.outputs.RUN_DOCS == 'true'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ inputs.python_version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python_version }}
+      - name: Install Tox
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox==3.28
+
+      - name: Install Build dependencies
+        if: ${{ matrix.platform == 'macos-latest' && matrix.python-version != '3.6' }}
+        run: brew install enchant
+
+      - name: Build Documentation and Test
+        if: ${{ matrix.platform == 'ubuntu-latest' || matrix.python-version  != '3.6' }}
+        run: ${{ inputs.command }}
+        # run: tox -e docs --sitepackages -vv -s false

.github/workflows/pypi_env.yml

@@ -4,12 +4,6 @@ on:
   workflow_call:
     inputs:
       # Success Policy
-      require_wheel:
-        required: false
-        type: boolean
-      allow_existing:
-        required: false
-        type: boolean
       distro_name:
         required: true
         type: string
@@ -23,10 +17,24 @@ on:
       pypi_env:
         required: true
         type: string
+      # Optional Inputs
+      allow_existing:
+      # Continue uploading files, if one already exists. (Only valid when
+      # uploading to PyPI. Other implementations may not support this.)
+        type: boolean
+        default: true
+        description: 'Continue uploading files, if one already exists, instead of Erroring'
+        required: false
+      require_wheel:
+        type: boolean
+        default: true
+        description: 'Require at least one Wheel Distribution to be uploaded'
+        required: false
       artifacts_path:
-        required: true
         type: string
-      # Optional
+        default: 'downloaded-artifacts'
+        description: 'Where to download CI Artifacts into'
+        required: false
       dist_folder:
         required: false
         type: string
@@ -47,25 +55,41 @@ jobs:
     env:
       DIST_DIR: ${{ inputs.dist_folder || 'dist' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
+    # TODO only Download Build Artifacts
+    # For python, Builds typically are files: *.tar.gz, *.whl, cython builds, etc.
     - name: Download Source & Wheel distributions
       uses: actions/download-artifact@v3
       with:
         path: ${{ inputs.artifacts_path }}
-    - run: ls -la ${{ inputs.artifacts_path }}
 
+    - run: ls -laR ${{ inputs.artifacts_path }}
+
+    # Create DIST Dir folder, to Upload to PyPI
     - run: mkdir ${DIST_DIR}
-    ## PUT TAR.GZ file in DIST FOLDER ##
-    - name: "Copy .tar.gz Source Distribution into '${{ env.DIST_DIR }}' folder"
+
+    ## PUT TAR.GZ file in DIST Dir ##
+    # Every Python Release must a Source Distro Build
+    - name: "Discover .tar.gz Source Distribution, from CI Artifacts"
       run: |
         targz_files=$(find ${{ inputs.artifacts_path }} -type f -name ${{ inputs.distro_name }}*.tar.gz)
         targz_files_array=($targz_files)
 
         SOURCE_DISTRO=${targz_files_array[0]}  # a *.tar.gz file path
+        echo "[INFO] Sdist Build: ${SOURCE_DISTRO}"
+
+        DISTRO_DIR=$(dirname "$SOURCE_DISTRO")
         DISTRO_FILE_NAME=$(basename "$SOURCE_DISTRO")
 
+        echo
+        echo "---> Selected Sdist Build: ${DISTRO_FILE_NAME} <---"
+        echo
+        echo "---> Taken from directory: ${DISTRO_DIR} <---"
+        echo
+
         # Check if all source distribution (.tar.gz) files have the same name
         for file in "${targz_files_array[@]}"; do
+          echo "[INFO] Checking file ${file}"
           if [ "$DISTRO_FILE_NAME" != "$(basename "$file")" ]; then
             echo "Error: Not all Source Distribution .tar.gz files have the same name!"
             exit 1
@@ -74,29 +98,33 @@ jobs:
         echo "SOURCE_DISTRO=$SOURCE_DISTRO" >> $GITHUB_ENV
         echo "DISTRO_FILE_NAME=$DISTRO_FILE_NAME" >> $GITHUB_ENV
 
-    - name: "SOURCE DISTRO: ${{ env.DISTRO_FILE_NAME }}"
-      run: cp "${{ env.SOURCE_DISTRO }}" "${DIST_DIR}"
+    - name: "Copy .tar.gz SOURCE Distro '${{ env.DISTRO_FILE_NAME }}' file, into '${{ env.DIST_DIR }}' folder"
+      run: cp "${{ env.SOURCE_DISTRO }}" "${{ env.DIST_DIR }}"
 
-    ## PUT WHEEL file(s) in DIST FOLDER ##
-    - run: echo wheel_files="$(find ${{ inputs.artifacts_path }} -type f -name ${{ inputs.distro_name }}*.whl)" >> $GITHUB_ENV
-    - run: echo wheel_files_array="(${{ env.wheel_files }})" >> $GITHUB_ENV
-    - name: "Verify at least one Wheel is Distributed"
-      if: inputs.require_wheel
+    ## PUT WHEEL file(s) in DIST Dir ##
+    # A Python Release could have one or more Wheel Builds
+    - name: "Copy .whl WHEEL Distro(s) files, into '${{ env.DIST_DIR }}' folder"
       run: |
-        if [[ ${#wheel_files_array[@]} -eq 0 ]]; then
-          echo "Error: No Wheel Distribution .whl files found!"
-          exit 1
-        fi
+        echo " -> Downloaded Wheel(s):"
+        find ${{ inputs.artifacts_path }} -type f -name "${{ inputs.distro_name }}*.whl"
+        echo
 
-    - name: "WHEEL DISTRO(s): ${{ env.wheel_files }}"
-      run: |
-        wheel_files_array=(${{ env.wheel_files }})
-        for f in "${wheel_files_array[@]}"; do
-          echo "WHEEL DISTRO: $f";
+        res=$(find ${{ inputs.artifacts_path }} -type f -name "${{ inputs.distro_name }}*.whl")
+
+        for f in $res; do
+          echo "[INFO] Copying Wheel Distribution: $f";
           cp $f ${DIST_DIR}
         done
 
     - run: ls -la ${DIST_DIR}
+
+    - name: Verify at least 1 Wheel Distribution, to be uploaded to PyPI
+      if: inputs.require_wheel
+      run: |
+        if [[ $(find ${DIST_DIR} -type f -name "${{ inputs.distro_name }}*.whl" | wc -l) -eq 0 ]]; then
+          echo "Error: No Wheel Distribution .whl files found!"
+          exit 1
+        fi
     ##
 
     ## PyPI Upload Command