benchmarks.yml

## Run benchmarks on the three OSs
## We run only on one python version (a recent one)
## I think that different python versions might share hardware resources so results might have a much larger fluctuation

name: Performance benchmarks

on:
  push:
    branches:
      - main

jobs:

  benchmarks:

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]

    # Do not run on forks
    if: github.repository_owner == 'aiidateam'

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v1
      with:
        python-version: 3.8
    - name: Install dependencies (including dev dependencies at frozen version)
      # I'm using pip install -e to make sure that the coverage properly traces the runs
      # also of the concurrent tests (maybe we can achieve this differently)
      run: |
        python -m pip install --upgrade pip
        pip install -e .[optionaltests]
        pip install -r requirements.lock
    - name: Run benchmarks
      run: pytest --benchmark-only --benchmark-json output.json
    - name: Store benchmark result
      ## Run only on push on main! Otherwise people (or other branches) might access to the github-actions branch
      ## This is currently disabled because we are in a workflow that has the correct 'on' settings (only push,
      ## and only 'main'). Otherwise, enable this
      #if: "github.event_name == 'push' && github.ref == 'refs/heads/main'"
      uses: rhysd/github-action-benchmark@v1
      with:
        name: "Benchmark on ${{ matrix.os }}"
        tool: "pytest"
        output-file-path: output.json
        github-token: ${{ secrets.GITHUB_TOKEN }}
        auto-push: true
        # Show alert with commit comment on detecting possible performance regression
        alert-threshold: '200%'
        comment-on-alert: true
        # For now I don't fail on alert since there is quite some oscillation on the results
        # fail-on-alert: true