Skip to content

Commit

Permalink
Refactor test matrix: most logic at top level, call reusable workflow (
Browse files Browse the repository at this point in the history
…#348)

Refactor unit-testing matrix

The main purpose of this is to exclude macos-13 (i.e. the slower Intel-based runner) from the tests that run automatically on pull-request updates. Python 3.11 is still excluded from these runs as well; a workflow dispatch still triggers the full (3-python x 4-platform) test battery.
  • Loading branch information
ajjackson authored Dec 20, 2024
1 parent 10c2639 commit 62be22d
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 77 deletions.
101 changes: 24 additions & 77 deletions .github/workflows/run_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,86 +17,33 @@ jobs:
(github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'no_ci'))
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
config:
- os: ubuntu-latest
skip: false
all-python-versions: ${{(github.event_name == 'workflow_dispatch')}}
coverage: true
- os: windows-latest
skip: false
all-python-versions: ${{(github.event_name == 'workflow_dispatch')}}
coverage: false
- os: macos-latest
skip: false
all-python-versions: ${{(github.event_name == 'workflow_dispatch')}}
coverage: false
- os: macos-13
skip: ${{ github.event_name != 'workflow_dispatch' }}
all-python-versions: true
coverage: false

fail-fast: false

runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Ensure tags are fetched for versioning
fetch-tags: true
- uses: actions/setup-python@v5
with:
python-version: |
3.10
3.11
3.12
- name: Install llvm on MacOS
if: startsWith(matrix.os, 'macos')
shell: bash -l {0}
env:
# Homebrew location is different on Intel Mac
LLVM_DIR: ${{ (matrix.os == 'macos-13') && '/usr/local/opt/llvm' || '/opt/homebrew/opt/llvm' }}
run: |
brew install llvm
echo CC="${LLVM_DIR}/bin/clang" >> $GITHUB_ENV
echo LDFLAGS="-L${LLVM_DIR}/lib $LDFLAGS" >> $GITHUB_ENV
echo CPPFLAGS="-I${LLVM_DIR}/include $CPPFLAGS" >> $GITHUB_ENV
- name: Windows - find MSVC and set environment variables
if: startsWith(matrix.os, 'windows')
shell: bash -l {0}
env:
MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC'
run: |
echo "Available MSVC installations:"
ls "$MSVC_PREFIX"
MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64
CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe"
echo "CC: $CC"
echo "CC=$CC" >> $GITHUB_ENV
CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe"
echo "CC_LD: $CC_LD"
echo "CC_LD=$CC_LD" >> $GITHUB_ENV
- name: Update pip and install dependencies
shell: bash -l {0}
run: |
python -m pip install --upgrade pip
python -m pip install -r tests_and_analysis/ci_requirements.txt
- name: Run tests, skip Python 3.11 unless workflow dispatch
if: github.event_name != 'workflow_dispatch'
env:
TOX_SKIP_ENV: '.*?(py311).*?'
shell: bash -l {0}
run: python -m tox run-parallel

- name: Run tests, workflow dispatch so test all Python versions
if: github.event_name == 'workflow_dispatch'
shell: bash -l {0}
run: python -m tox run-parallel

- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: Unit test results ${{ matrix.os }}
path: tests_and_analysis/test/reports/junit_report*.xml
- name: Publish Codacy coverage
uses: codacy/codacy-coverage-reporter-action@v1
if: startsWith(matrix.os, 'ubuntu')
with:
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
coverage-reports: tests_and_analysis/test/reports/coverage*.xml
- uses: codecov/codecov-action@v3
if: startsWith(matrix.os, 'ubuntu')
with:
files: tests_and_analysis/test/reports/coverage*.xml
uses: ./.github/workflows/test_checkout_one_os.yml
with:
os: ${{ matrix.config.os }}
skip: ${{ matrix.config.skip }}
all-python-versions: ${{ matrix.config.all-python-versions }}
coverage: ${{ matrix.config.coverage }}
secrets: inherit

test-docs:
runs-on: ubuntu-latest
Expand Down
106 changes: 106 additions & 0 deletions .github/workflows/test_checkout_one_os.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
name: Run tests from source on one platform
on:
workflow_call:
inputs:
skip:
description: 'Skip this job'
required: false
default: false
type: boolean
os:
description: 'Target platform'
required: false
default: ubuntu-latest
type: string
all-python-versions:
description: 'Test all Python versions'
required: false
default: false
type: boolean
coverage:
description: 'Publish coverage'
required: false
default: false
type: boolean

jobs:
test:
if: ${{ ! inputs.skip }}
runs-on: ${{ inputs.os }}
env:
MIN_MAX_PYTHON_VERSIONS: |
3.10
3.12
ALL_PYTHON_VERSIONS: |
3.10
3.11
3.12
TOX_SKIP_ENV: ${{ inputs.all-python-versions && 'no-match' || '.*?(py311).*?' }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # Ensure tags are fetched for versioning
fetch-tags: true
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: ${{ inputs.all-python-versions && env.ALL_PYTHON_VERSIONS || env.MIN_MAX_PYTHON_VERSIONS}}

- name: Install llvm on MacOS
if: startsWith( inputs.os , 'macos' )
shell: bash -l {0}
env:
# Homebrew location is different on Intel Mac
LLVM_DIR: ${{ ( inputs.os == 'macos-13') && '/usr/local/opt/llvm' || '/opt/homebrew/opt/llvm' }}
run: |
brew install llvm
echo CC="${LLVM_DIR}/bin/clang" >> $GITHUB_ENV
echo LDFLAGS="-L${LLVM_DIR}/lib $LDFLAGS" >> $GITHUB_ENV
echo CPPFLAGS="-I${LLVM_DIR}/include $CPPFLAGS" >> $GITHUB_ENV
- name: Windows - find MSVC and set environment variables
if: startsWith( inputs.os , 'windows' )
shell: bash -l {0}
env:
MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC'
run: |
echo "Available MSVC installations:"
ls "$MSVC_PREFIX"
MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64
CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe"
echo "CC: $CC"
echo "CC=$CC" >> $GITHUB_ENV
CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe"
echo "CC_LD: $CC_LD"
echo "CC_LD=$CC_LD" >> $GITHUB_ENV
- name: Update pip and install dependencies
shell: bash -l {0}
run: |
python -m pip install --upgrade pip
python -m pip install -r tests_and_analysis/ci_requirements.txt
- name: Run tests
shell: bash -l {0}
run: python -m tox run-parallel

- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: Unit test results ${{ inputs.os }}
path: tests_and_analysis/test/reports/junit_report*.xml

- name: Publish Codacy coverage
uses: codacy/codacy-coverage-reporter-action@v1
if: inputs.coverage
with:
project-token: ${{ secrets.codacy_project_token }}
coverage-reports: tests_and_analysis/test/reports/coverage*.xml

- uses: codecov/codecov-action@v3
if: inputs.coverage
with:
files: tests_and_analysis/test/reports/coverage*.xml

0 comments on commit 62be22d

Please sign in to comment.