Chore: Update DevOps tooling from central repository

.github/workflows/bootstrap.yaml

@@ -31,33 +31,122 @@ jobs:
           path: ".devops"
 
       - name: "Update repository workflows and create PR"
+        id: update-repository
         env:
           GH_TOKEN: ${{ github.token }}
+        # yamllint disable rule:line-length
         run: |
-          # Remove update-devops-tooling branch if it exists
-          git branch -d update-devops-tooling || true
-          git push origin --delete update-devops-tooling || true
-          git config user.name "github-actions[bot]"
-          git config user.email \
-           "41898282+github-actions[bot]@users.noreply.github.com"
-          git checkout -b "update-devops-tooling"
+          ### SHELL CODE START ###
+
+          REPO_DIR=$(git rev-parse --show-toplevel)
+          # Ensure working from top-level of GIT repository
+          CURRENT_DIR=$(pwd)
+          if [ "$REPO_DIR" != "$CURRENT_DIR" ]; then
+              echo "Changing directory to: $REPO_DIR"
+              if ! (cd "$REPO_DIR"); then
+                  echo "Error: unable to change directory"; exit 1
+              fi
+          fi
+
+          # Define a function to allow selective opt-out of devops tooling
+          OPT_OUT=".devops-exclusions"
+          perform_operation() {
+            ELEMENT="$1"
+            if [ ! -f "$OPT_OUT" ]; then
+              # Opt-out file does not exist; all operations will be performed
+              return 1
+            else
+              if grep -Fxq "$ELEMENT" "$OPT_OUT"
+              then
+                # Element is excluded from processing
+                return 0
+              else
+                # Element should be processed
+                return 1
+              fi
+            fi
+          }
+
+          echo "Removing remote branch if it exists: update-devops-tooling"
+          git push origin --delete update-devops-tooling || :
+          STRING=$(dd if=/dev/urandom bs=1k count=1 2>/dev/null | tr -dc 'a-zA-Z0-9' | head -c 10)
+          git checkout -b "update-$STRING"
+
+          # Configure GIT
+          TEST=$(git config -l)
+          if [ -n "$TEST" ]; then
+            git config user.name "github-actions[bot]"
+            git config user.email \
+              "41898282+github-actions[bot]@users.noreply.github.com"
+          fi
+
           FOLDERS=".github .github/workflows scripts"
-          FILES=".pre-commit-config.yaml .prettierignore .gitignore"
           for FOLDER in ${FOLDERS}; do
+            # Check to see if operation should be skipped
+            if (perform_operation "$FOLDER"); then
+              echo "Opted out of DevOps folder: $FOLDER"
+              continue
+            else
               # If necessary, create target folder
               if [ ! -d "$FOLDER" ]; then
-                  mkdir "$FOLDER"
+                echo "Creating target folder: $FOLDER"
+                mkdir "$FOLDER"
               fi
               # Update folder contents
+              echo "Updating folder contents: $FOLDER"
               cp -a .devops/"$FOLDER"/. "$FOLDER"
+            fi
           done
+
           # Copy specified files into repository root
+          FILES=".pre-commit-config.yaml .prettierignore .gitignore"
           for FILE in ${FILES}; do
+            if (perform_operation "$FILE"); then
+              echo "Opted out of DevOps file: $FILE"
+            else
+              echo "Copying file: $FILE"
               cp .devops/"$FILE" "$FILE"
+            fi
           done
-          git add .
-          git commit -m "Chore: Update DevOps tooling from central repository"
-          git push --set-upstream origin update-devops-tooling
-          gh pr create --title \
-            "Chore: Pull DevOps tooling from upstream repository" \
-            --body 'This process automated by a GitHub workflow: bootstrap.yaml'
+
+          # If no changes required, do not throw an error
+          if [ -z "$(git status --porcelain)" ]; then
+            echo "No updates/changes to commit"; exit 0
+          else
+            # Set a flag for use by the next action/step
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+          if [ -n "$GITHUB_TOKEN" ]; then
+            git add .
+            if ! (git commit -as -S -m "Chore: Update DevOps tooling from central repository [skip-ci]" \
+              -m "This commit created by automation/scripting" --no-verify); then
+              echo "Commit failed; aborting"; exit 1
+            else
+              git push --set-upstream origin update-devops-tooling
+              # ToDo: need to verify if we are running in a GHA
+              gh pr create --title \
+                "Chore: Pull DevOps tooling from upstream repository" \
+                --body 'Automated by a GitHub workflow: bootstrap.yaml'
+            fi
+          else
+            echo "Script running in GitHub Actions workflow; proceeding to next step"
+          fi
+          ### SHELL CODE END ###
+
+      - name: Create Pull Request
+        if: steps.update-repository.outputs.changed == 'true'
+        uses: peter-evans/create-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          token: ${{ github.token }}
+          commit-message: "Chore: Update DevOps tooling from central repository [skip-ci]"
+          signoff: "true"
+          branch: update-devops-tooling
+          delete-branch: true
+          title: "Chore: Update DevOps tooling from central repository [skip-ci]"
+          body: |
+            Update repository with content from upstream: os-climate/devops-toolkit
+          labels: |
+            automated pr
+          draft: false

.github/workflows/builds.yaml

@@ -1,5 +1,5 @@
 ---
-name: "🧪 Test builds (matrix)"
+name: "🧪 Test builds (Matrix)"
 
 # yamllint disable-line rule:truthy
 on:
@@ -12,14 +12,15 @@ on:
 
 jobs:
   pre-release:
-    # Don't run if pull request is NOT merged
-    # if: github.event.pull_request.merged == true
     runs-on: "ubuntu-latest"
     continue-on-error: true
+    # Don't run when pull request is merged
+    if: github.event.pull_request.merged == false
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
+
     steps:
       - name: "Populate environment variables"
         id: setenv

.github/workflows/dependencies.yaml

@@ -1,15 +1,15 @@
 ---
-name: "⛔️ Update dependencies"
+name: "♻️ Update dependencies"
 
 # yamllint disable-line rule:truthy
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 8 * * FRI"
+    - cron: "0 8 1 * *"
 
 jobs:
   update-dependencies:
-    name: "Update Python modules"
+    name: "Update dependencies"
     runs-on: ubuntu-latest
     permissions:
       # IMPORTANT: mandatory to raise the PR
@@ -22,15 +22,27 @@ jobs:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
     steps:
-      - uses: actions/checkout@v4
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
 
-      - name: Update dependencies
-        uses: ModeSevenIndustrialSolutions/update-deps-action@v1
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
+
+      - name: "Update Python dependencies"
+        uses: pdm-project/update-deps-action@v1
         with:
           sign-off-commit: "true"
           token: ${{ secrets.GH_TOKEN }}
-          commit-message: "Chore: Update dependencies and pdm.lock"
+          commit-message: "Chore: Update dependencies and pdm.lock [skip ci]"
           pr-title: "Update Python module dependencies"
           update-strategy: eager
           # Whether to install PDM plugins before update
           install-plugins: "false"
+
+      - name: "Export dependencies"
+        run: |
+          pdm export --without-hashes -o requirements.txt
+
+      # Ideally, we should export requirements.txt then amend the earlier PR
+      # update-deps-action could be modified to export PR number as as output
+      # Or we add the option to export the requirements.txt in that action

.github/workflows/documentation.yaml

@@ -1,5 +1,5 @@
 ---
-name: "🗒️ Build documentation"
+name: "🗒️ Build documentation (Matrix)"
 
 # yamllint disable-line rule:truthy
 on:

.github/workflows/linting.yaml

@@ -0,0 +1,56 @@
+---
+name: "⛔️ Standalone linting checks"
+
+# yamllint disable-line rule:truthy
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, reopened, edited, synchronize]
+    branches:
+      - "*"
+      - "!update-devops-tooling"
+
+jobs:
+  linting:
+
+    name: "Unsupported by pre-commit.ci"
+    runs-on: "ubuntu-latest"
+    # Don't run when pull request is merged
+    if: github.event.pull_request.merged == false
+
+    steps:
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+
+      - name: "Install linting tools"
+        run: |
+          pip install --upgrade pip
+          pip install pre-commit mypy
+
+      - name: "Run mypy using pre-commit"
+        run: pre-commit run mypy -a
+
+        # yamllint disable rule:line-length
+        # yamllint disable rule:comments-indentation
+        # yamllint disable rule:comments
+
+        # Provided below as an example, in case needed in future
+        # - name: "Install dependencies"
+        #   run: |
+        #     SOURCE=".pre-commit-config.yaml"
+        #     echo "Install Python dependencies from: $SOURCE"
+        #     echo "With: pip install $PKGS"
+        #     PKGS=$(yq '.repos[] | select (.repo == "https://github.com/pre-commit/mirrors-mypy")' .pre-commit-config.yaml | \
+        #       grep additional_dependencies | \
+        #       awk -F: '{print $2}' | \
+        #       sed "s/\[//g" | \
+        #       sed "s/\]//g" | \
+        #       sed "s/,//g" | \
+        #       sed 's/"//g')
+        #       pip install $PKGS

.github/workflows/release.yaml

@@ -76,7 +76,7 @@ jobs:
       contents: write
     steps:
       - name: "⬇ Download build artefacts"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ github.ref_name }}
           path: dist/
@@ -111,7 +111,7 @@ jobs:
       id-token: write
     steps:
       - name: "⬇ Download build artefacts"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ github.ref_name }}
           path: dist/
@@ -145,7 +145,7 @@ jobs:
       id-token: write
     steps:
       - name: "⬇ Download build artefacts"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ github.ref_name }}
           path: dist/

.github/workflows/security.yaml

@@ -4,7 +4,7 @@
 # For more information see:
 # https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: "⛔️ Security auditing"
+name: "⛔️ Security auditing (Matrix)"
 
 # yamllint disable-line rule:truthy
 on:
@@ -19,10 +19,13 @@ jobs:
   build:
     name: "Audit Python dependencies"
     runs-on: ubuntu-latest
+    # Don't run when pull request is merged
+    if: github.event.pull_request.merged == false
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
+
     steps:
       - name: "Checkout repository"
         uses: actions/checkout@v4
@@ -48,10 +51,3 @@ jobs:
 
       - name: "Run: pip-audit"
         uses: pypa/[email protected]
-        with:
-          ignore-vulns: |
-            PYSEC-2023-163
-
-# Name | Version | ID |
-# --- | --- | --- | --- | ---
-# numexpr | 2.8.7 | PYSEC-2023-163 |

.github/workflows/test-release.yaml

@@ -86,7 +86,7 @@ jobs:
       contents: write
     steps:
       - name: "⬇ Download build artefacts"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: Development
           path: dist/
@@ -132,7 +132,7 @@ jobs:
       id-token: write
     steps:
       - name: "⬇ Download build artefacts"
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: Development
           path: dist/

.github/workflows/testing.yaml

@@ -1,5 +1,5 @@
 ---
-name: "🧪 Unit tests"
+name: "🧪 Unit tests (Matrix)"
 
 # yamllint disable-line rule:truthy
 on:
@@ -14,10 +14,13 @@ jobs:
   build:
     name: "Run unit tests"
     runs-on: ubuntu-latest
+    # Don't run when pull request is merged
+    if: github.event.pull_request.merged == false
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
+
     steps:
       - name: "Checkout repository"
         uses: actions/checkout@v4