Refactor liboqs CI and update Ubuntu images (#1909)

* Refactor liboqs CI to utilize reusable workflows
* Add CI.md documentation file
* Update all Focal jobs to Noble (the latest Ubuntu LTS)
* Minor fixes to address issues related to the update (CT files / syntax / static analysis)

Signed-off-by: Spencer Wilson <[email protected]>

.github/CODEOWNERS

@@ -1,7 +1,7 @@
 # https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners
 
 * @dstebila
-/.circleci @baentsch
+/.github/workflows @SWilson4
 /scripts/copy_from_upstream @baentsch @bhess @alexrow
 /src/common @dstebila
 /src/kem/bike @crockeea

.github/workflows/android.yml

@@ -3,7 +3,7 @@ name: android build
 permissions:
   contents: read
 
-on: [ push, pull_request ]
+on: [workflow_call, workflow_dispatch]
 
 jobs:
 

.github/workflows/apple.yml

@@ -3,7 +3,7 @@ name: apple build
 permissions:
   contents: read
 
-on: [ push, pull_request ]
+on: [workflow_call, workflow_dispatch]
 
 jobs:
 

.github/workflows/basic.yml

@@ -0,0 +1,75 @@
+name: Basic checks
+
+permissions:
+  contents: read
+
+on: [workflow_call, workflow_dispatch]
+
+jobs:
+
+  stylecheck:
+    name: Check code formatting
+    runs-on: ubuntu-latest
+    container: openquantumsafe/ci-ubuntu-latest:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
+      - name: Ensure code conventions are upheld
+        run: python3 -m pytest --verbose tests/test_code_conventions.py
+      - name: Check that doxygen can parse the documentation
+        run: mkdir build && ./scripts/run_doxygen.sh $(which doxygen) ./docs/.Doxyfile ./build
+      - name: Validate CBOM
+        run: scripts/validate_cbom.sh
+
+  upstreamcheck:
+    name: Check upstream code is properly integrated
+    runs-on: ubuntu-latest
+    container: openquantumsafe/ci-ubuntu-latest:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
+      - name: Configure
+        run: |
+          git config --global user.name "ciuser" && \
+          git config --global user.email "[email protected]" && \
+          git config --global --add safe.directory "$PWD" && \
+          echo "LIBOQS_DIR=$PWD" >> "$GITHUB_ENV"
+      - name: Verify copy_from_upstream state after copy
+        working-directory: 'scripts/copy_from_upstream'
+        run: |
+          python3 copy_from_upstream.py copy && \
+          ! git status | grep -i modified
+      - name: Verify copy_from_upstream state after libjade
+        working-directory: 'scripts/copy_from_upstream'
+        run: |
+          python3 copy_from_upstream.py libjade && \
+          ! git status | grep -i modified
+
+  buildcheck:
+    name: Check that code passes a basic build
+    needs: [ stylecheck, upstreamcheck ]
+    runs-on: ubuntu-latest
+    container: openquantumsafe/ci-ubuntu-latest:latest
+    env:
+      KEM_NAME: ml_kem_768
+      SIG_NAME: ml_dsa_65
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
+      - name: Configure
+        run: |
+          mkdir build && \
+          cd build && \
+          cmake -GNinja -DOQS_STRICT_WARNINGS=ON \
+            -GNinja \
+            -DOQS_MINIMAL_BUILD="KEM_$KEM_NAME;SIG_$SIG_NAME" \
+            --warn-uninitialized .. > config.log 2>&1 && \
+          cat config.log && \
+          cmake -LA -N .. && \
+          ! (grep -i "uninitialized variable" config.log)
+      - name: Build code
+        run: ninja
+        working-directory: build
+      - name: Build documentation
+        run: ninja gen_docs
+        working-directory: build

.github/workflows/commit-to-main.yml

@@ -1,88 +1,39 @@
-name: Trigger basic downstream CI
+name: Main branch tests
 
 permissions:
   contents: read
 
 on:
   push:
-    branches: [ "main" ]
+    branches: ['main']
 
 jobs:
-  trigger-downstream-ci:
-    strategy:
-      fail-fast: false
-    runs-on: ubuntu-latest
-    steps:
-      # TODO: missing projects?
-      - name: Trigger OQS-OpenSSL CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --user ${{ secrets.BUILD_TRIGGER_TOKEN }}: \
-               --request POST \
-               --header "Content-Type: application/json" \
-               --data '{ "branch": "OQS-OpenSSL_1_1_1-stable", "parameters": { "run_downstream_tests": true } }' \
-               https://circleci.com/api/v2/project/gh/open-quantum-safe/openssl/pipeline | tee curl_out \
-          && grep -q "201" curl_out
-      - name: Trigger OQS-BoringSSL CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --user ${{ secrets.BUILD_TRIGGER_TOKEN }}: \
-               --request POST \
-               --header "Content-Type: application/json" \
-               --data '{ "branch": "master", "parameters": { "run_downstream_tests": true } }' \
-               https://circleci.com/api/v2/project/gh/open-quantum-safe/boringssl/pipeline | tee curl_out \
-          && grep -q "201" curl_out
-      - name: Trigger OQS-OpenSSH CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --request POST \
-               --header "Accept: application/vnd.github+json" \
-               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
-               --header "X-GitHub-Api-Version: 2022-11-28" \
-               --data '{"ref":"OQS-v9"}' \
-               https://api.github.com/repos/open-quantum-safe/openssh/actions/workflows/ubuntu.yaml/dispatches | tee curl_out \
-          && grep -q "204" curl_out
-      - name: Trigger oqs-provider CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --user ${{ secrets.BUILD_TRIGGER_TOKEN }}: \
-               --request POST \
-               --header "Content-Type: application/json" \
-               --data '{ "branch": "main" }' \
-               https://circleci.com/api/v2/project/gh/open-quantum-safe/oqs-provider/pipeline | tee curl_out \
-          && grep -q "201" curl_out
-      - name: Trigger liboqs-dotnet CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --user ${{ secrets.BUILD_TRIGGER_TOKEN }}: \
-               --request POST \
-               --header "Content-Type: application/json" \
-               --data '{ "branch": "master" }' \
-               https://circleci.com/api/v2/project/gh/open-quantum-safe/liboqs-dotnet/pipeline | tee curl_out \
-          && grep -q "201" curl_out
-      - name: Trigger liboqs-java CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --user ${{ secrets.BUILD_TRIGGER_TOKEN }}: \
-               --request POST \
-               --header "Content-Type: application/json" \
-               --data '{ "branch": "master" }' \
-               https://circleci.com/api/v2/project/gh/open-quantum-safe/liboqs-java/pipeline | tee curl_out \
-          && grep -q "201" curl_out
-      - name: Trigger liboqs-python CI
-        run: |
-          curl --silent \
-               --write-out "\n%{response_code}\n" \
-               --request POST \
-               --header "Accept: application/vnd.github+json" \
-               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
-               --header "X-GitHub-Api-Version: 2022-11-28" \
-               --data '{"event_type":"liboqs-upstream-trigger"}' \
-               https://api.github.com/repos/open-quantum-safe/liboqs-python/dispatches | tee curl_out \
-          && grep -q "204" curl_out
+
+  platform-tests:
+    uses: ./.github/workflows/platforms.yml
+
+  scorecard:
+    uses: ./.github/workflows/scorecard.yml
+    secrets: inherit
+    # complete list of permissions keys as per
+    # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/controlling-permissions-for-github_token#defining-access-for-the-github_token-permissions
+    # accessed September 4, 2024
+    permissions:
+      actions: read
+      attestations: read
+      checks: read
+      contents: read
+      deployments: read
+      id-token: write
+      issues: read
+      discussions: read
+      packages: read
+      pages: read
+      pull-requests: read
+      repository-projects: read
+      security-events: write
+      statuses: read
+
+  basic-downstream:
+    uses: ./.github/workflows/downstream-basic.yml
+    secrets: inherit

.github/workflows/downstream-basic.yml

@@ -0,0 +1,83 @@
+name: Trigger basic downstream CI
+
+permissions:
+  contents: read
+
+on: [workflow_call, workflow_dispatch]
+
+jobs:
+
+  trigger-downstream-ci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger OQS-BoringSSL CI
+        if: ${{ !cancelled() }} # run all steps independent of failures
+        run: |
+          curl --silent \
+               --write-out "\n%{response_code}\n" \
+               --request POST \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               --data '{"event_type":"liboqs-upstream-trigger"}' \
+               https://api.github.com/repos/open-quantum-safe/boringssl/dispatches | tee curl_out \
+          && grep -q "204" curl_out
+      - name: Trigger OQS-OpenSSH CI
+        if: ${{ !cancelled() }} # run all steps independent of failures
+        run: |
+          curl --silent \
+               --write-out "\n%{response_code}\n" \
+               --request POST \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               --data '{"ref":"OQS-v9"}' \
+               https://api.github.com/repos/open-quantum-safe/openssh/actions/workflows/ubuntu.yaml/dispatches | tee curl_out \
+          && grep -q "204" curl_out
+      - name: Trigger oqs-provider CI
+        if: ${{ !cancelled() }} # run all steps independent of failures
+        run: |
+          curl --silent \
+               --write-out "\n%{response_code}\n" \
+               --user ${{ secrets.BUILD_TRIGGER_TOKEN }}: \
+               --request POST \
+               --header "Content-Type: application/json" \
+               --data '{ "branch": "main" }' \
+               https://circleci.com/api/v2/project/gh/open-quantum-safe/oqs-provider/pipeline | tee curl_out \
+          && grep -q "201" curl_out
+      - name: Trigger liboqs-cpp CI
+        if: ${{ !cancelled() }} # run all steps independent of failures
+        run: |
+          curl --silent \
+               --write-out "\n%{response_code}\n" \
+               --request POST \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               --data '{"event_type":"liboqs-upstream-trigger"}' \
+               https://api.github.com/repos/open-quantum-safe/liboqs-cpp/dispatches | tee curl_out \
+          && grep -q "204" curl_out
+      - name: Trigger liboqs-go CI
+        if: ${{ !cancelled() }} # run all steps independent of failures
+        run: |
+          curl --silent \
+               --write-out "\n%{response_code}\n" \
+               --request POST \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               --data '{"event_type":"liboqs-upstream-trigger"}' \
+               https://api.github.com/repos/open-quantum-safe/liboqs-go/dispatches | tee curl_out \
+          && grep -q "204" curl_out
+      - name: Trigger liboqs-python CI
+        if: ${{ !cancelled() }} # run all steps independent of failures
+        run: |
+          curl --silent \
+               --write-out "\n%{response_code}\n" \
+               --request POST \
+               --header "Accept: application/vnd.github+json" \
+               --header "Authorization: Bearer ${{ secrets.OQSBOT_GITHUB_ACTIONS }}" \
+               --header "X-GitHub-Api-Version: 2022-11-28" \
+               --data '{"event_type":"liboqs-upstream-trigger"}' \
+               https://api.github.com/repos/open-quantum-safe/liboqs-python/dispatches | tee curl_out \
+          && grep -q "204" curl_out

.github/workflows/release-test.yml → .github/workflows/downstream-release.yml

@@ -1,23 +1,17 @@
-name: Release tests
+name: Downstream release tests
 
 permissions:
   contents: read
 
+on: [workflow_call, workflow_dispatch]
+
 # Trigger oqs-provider release tests.
-# Runs whenever a release is published, or when a commit message ends with "[trigger downstream]"
-# When triggered by a release, the liboqs release tag and the provider "<release tag>-tracker" branch are used.
-# When triggered by a commit message, the triggering liboqs branch and the provider "<liboqs branch>-tracker" branch are used.
+# When triggered by a release (see release.yml), the liboqs release tag and the provider "<release tag>-tracker" branch are used.
+# When triggered by a commit message (see filter.yml), the triggering liboqs branch and the provider "<liboqs branch>-tracker" branch are used.
 # If the tracker branch does not exist, the downstream pipeline should detect it and run on the main branch instead.
 
-on:
-  push:
-  workflow_dispatch:
-  release:
-    types: [ published ]
-
 jobs:
   oqs-provider-release-test:
-    if: github.event_name == 'release' || contains( github.event.head_commit.message, '[trigger downstream]' )
     runs-on: ubuntu-latest
     steps:
       - name: Checkout release tests script