ci.yml

name: CI with dynamic parallel matrix

on:
  workflow_dispatch:
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  TIMEOUT: '-1'    # nbconvert timeout
  EXTRA_ARGS: ''   # Extra arguments for nbconvert
  PYTHON_VER: '3.11'
  JULIA_CPU_TARGET: 'generic;haswell,clone_all'
  JULIA_NUM_THREADS: 'auto'
  JULIA_CONDAPKG_BACKEND: 'Null'

jobs:
  setup:
    container:
      image: julia:1.9.3
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VER }}
      - name: Cache Julia deps
        uses: actions/cache@v3
        with:
          path: |
            ~/.julia/artifacts
            ~/.julia/compiled
            ~/.julia/packages
          key: ${{ runner.os }}-juliacontainer-${{ hashFiles('Manifest.toml')}}
          restore-keys: |
            ${{ runner.os }}-juliacontainer-
      - name: Install Julia packages
        env:
          PYTHON: ${{ env.pythonLocation }}/python
          JULIA_PYTHONCALL_EXE: ${{ env.pythonLocation }}/python
        run: julia --color=yes setup.jl
      - name: List notebooks as a JSON array
        id: set-matrix
        working-directory: docs
        run: echo "matrix=$(python -c 'import glob, json; print(json.dumps(glob.glob("**/*.ipynb", recursive=True)))')" >> "$GITHUB_OUTPUT"

  execute:
    needs: setup
    strategy:
      max-parallel: 20
      fail-fast: false
      matrix:
        # Notebooks need to be executed
        notebook: ${{ fromJSON(needs.setup.outputs.matrix) }}
    runs-on: ubuntu-latest
    container:
      image: julia:1.9.3
      env:
        JULIA_CPU_TARGET: 'generic;haswell,clone_all'
        JULIA_NUM_THREADS: 'auto'
        JULIA_CONDAPKG_BACKEND: 'Null'
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Get notebook path
        id: file
        run: echo "name=docs/${{ matrix.notebook }}" >> "$GITHUB_OUTPUT"
      - name: Restore notebook if present
        uses: actions/cache/restore@v3
        id: cache
        with:
          path: ${{ steps.file.outputs.name }}
          key: ${{ runner.os }}-notebook-${{ hashFiles(steps.file.outputs.name, 'requirements.txt', 'Manifest.toml', 'src/**') }}
      - name: Setup Python
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VER }}
      - name: Install Python deps
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          python -m pip install --no-cache-dir nbconvert
          python -m pip install --no-cache-dir -r requirements.txt
      - name: Cache Julia deps
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: actions/cache@v3
        with:
          path: |
            ~/.julia/artifacts
            ~/.julia/compiled
            ~/.julia/packages
          key: ${{ runner.os }}-juliacontainer-${{ hashFiles('Manifest.toml')}}
          restore-keys: |
            ${{ runner.os }}-juliacontainer-
      - name: Install Julia packages
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        env:
          PYTHON: ${{ env.pythonLocation }}/python
          JULIA_PYTHONCALL_EXE: ${{ env.pythonLocation }}/python
        run: julia --color=yes setup.jl
      - name: Execute Notebook
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: >
          jupyter nbconvert --to notebook --execute --inplace ${{ env.EXTRA_ARGS }}
          --ExecutePreprocessor.timeout=${{ env.TIMEOUT }}
          --ExecutePreprocessor.kernel_name=julia-1.$(julia -e 'print(VERSION.minor)')
          docs/${{ matrix.notebook }}
      - name: Cache notebook
        uses: actions/cache/save@v3
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        with:
          path: ${{ steps.file.outputs.name }}
          key: ${{ steps.cache.outputs.cache-primary-key }}
      - name: Upload Notebook
        uses: actions/upload-artifact@v3
        with:
          name: notebooks
          path: docs*/${{ matrix.notebook }}  # keep folder structure
          retention-days: 1

  jupyter-book:
    needs: execute
    runs-on: ubuntu-latest
    # store success output flag for the ci job
    outputs:
      success: ${{ steps.setoutput.outputs.success }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download notebooks
        uses: actions/download-artifact@v3
        with:
          name: notebooks
          path: out/
      - name: Display structure of downloaded files
        run: ls -R
        working-directory: out
      - name: Copy back built notebooks
        run: cp --verbose -rf out/docs/* docs/
      - name: Setup micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: jupyterbook.yml
          init-shell: bash
          cache-environment: true
          post-cleanup: all
      - name: Build website
        shell: micromamba-shell {0}
        run: jupyter-book build docs/
      - name: Upload pages artifact
        if: ${{ github.ref == 'refs/heads/main' }}
        uses: actions/upload-pages-artifact@v2
        with:
          path: docs/_build/html
      - name: Set output flag
        id: setoutput
        run: echo "success=true" >> $GITHUB_OUTPUT

  # CI conclusion for GitHub status check
  # https://brunoscheufler.com/blog/2022-04-09-the-required-github-status-check-that-wasnt
  CI:
    needs: jupyter-book
    if: always()
    runs-on: ubuntu-latest
    steps:
      # pass step only when output of previous jupyter-book job is set
      # in case at least one of the execution fails, jupyter-book is skipped
      # and the output will not be set, which will then cause the ci job to fail
      - run: |
          passed="${{ needs.jupyter-book.outputs.success }}"
          if [[ $passed == "true" ]]; then
            echo "Tests passed"
            exit 0
          else
            echo "Tests failed"
            exit 1
          fi

  # Deployment job
  deploy:
    name: Deploy to GitHub pages
    needs: jupyter-book
    if: ${{ github.ref == 'refs/heads/main' }}
    # Grant GITHUB_TOKEN the permissions required to make a Pages deployment
    permissions:
      pages: write # to deploy to Pages
      id-token: write # to verify the deployment originates from an appropriate source
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2