python-build-test.yaml

name: 'Python: Build and Test'

on:
  push:
    tags:
      - python-*
  workflow_dispatch:
  pull_request:
    paths:
      - 'lace/**'
      - 'pylace/**'
      - 'book/**'
      - '.github/workflows/python-build-test.yaml'
      - '.github/scripts/run_code_in_mdfile.py'
      - '.github/scripts/find_compatible_wheel.py'

jobs:
  lint-python:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: pylace
    
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
          cache: 'pip'
          cache-dependency-path: "pylace/requirements-lint.txt"
      
      - name: Install Python dependencies
        run: |
          pip install --upgrade pip
          pip install -r requirements-lint.txt

      - name: Check format (black)
        run: black --check --diff .

      - name: Lint Check (ruff)
        run: ruff --diff .

      - name: Check Minimum Supported Python Version (vermin)
        run: vermin --target=3.8- --no-tips --violations .

  lint-rust:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: pylace

    steps:
      - uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy

      - name: Cache Rust
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: . -> pylace/target

      - name: Run rustfmt
        run: cargo fmt --all -- --check

      - name: Run clippy
        env:
          RUSTFLAGS: -C debuginfo=0  # Do not produce debug symbols to keep memory usage down
        run : |
          cargo clippy
      - name: Install audit
        run: cargo install cargo-audit
      - name: Run audit
        working-directory: lace
        # Note: Both `polars` and `arrow2` trigger this security violation
        #       due to their reliance on `chrono`, which is the ultimate source of the violation
        #       as of 2/21/23, no version of `chrono` has been published that fixes the issue
        #       and thus neither `polars` or `arrow2` can pass `audit` checks
        run: cargo audit -f Cargo.lock --ignore RUSTSEC-2020-0071

  linux:
    runs-on: ubuntu-latest
    needs: ["lint-python", "lint-rust"]
    strategy:
      matrix:
        target: [x86_64]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: |
            3.8
            3.9
            3.10
            3.11
            3.12

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --manifest-path pylace/Cargo.toml
          manylinux: auto
    
      - name: Install dev dependencies
        run: |
          pip install --upgrade pip
          pip install -r pylace/requirements-dev.txt

      - name: Install pylace
        run: |
          ls -l ./dist
          WHEEL_FILE=$(python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
          echo "Installing $WHEEL_FILE"
          pip install $WHEEL_FILE

      - name: Run Tests
        run: pytest pylace/tests

      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-linux-${{ matrix.target }}
          path: dist

  windows:
    runs-on: windows-latest
    needs: ["lint-python", "lint-rust"]
    strategy:
      matrix:
        target: [x64]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: |
            3.8
            3.9
            3.10
            3.11
            3.12
          architecture: ${{ matrix.target }}
      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --manifest-path pylace/Cargo.toml
    
      - name: Install dev dependencies
        run: |
          pip install --upgrade pip
          pip install -r pylace/requirements-dev.txt

      - name: Install pylace
        run: |
          ls -l ./dist
          $WHEEL_FILE = (python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
          echo "Installing $WHEEL_FILE"
          pip install $WHEEL_FILE

      - name: Run Tests
        run: pytest pylace/tests

      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-windows-${{ matrix.target }}
          path: dist

  macos:
    runs-on: macos-latest
    needs: ["lint-python", "lint-rust"]
    strategy:
      matrix:
        target: [x86_64, aarch64]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: |
            3.8
            3.9
            3.10
            3.11
            3.12
      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --manifest-path pylace/Cargo.toml

      - name: Install dev dependencies
        if: ${{ matrix.target != 'aarch64' }}
        run: |
          pip install --upgrade pip
          pip install -r pylace/requirements-dev.txt

      - name: Install pylace
        if: ${{ matrix.target != 'aarch64' }}
        run: |
          WHEEL_FILE=$(python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
          echo "Installing $WHEEL_FILE"
          pip install $WHEEL_FILE

      - name: Run Tests
        if: ${{ matrix.target != 'aarch64' }}
        run: pytest pylace/tests

      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels-macos-${{ matrix.target }}
          path: dist

  merge:
    runs-on: ubuntu-latest
    needs: [ macos, windows, linux ]
    steps:
      - name: Merge Artifacts
        uses: actions/upload-artifact/merge@v4
        with:
          name: wheels
          pattern: wheels-*

  test-mdbook-python:
    name: Test MDBook Python Snippets
    runs-on: ubuntu-latest
    needs: [merge]
    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: wheels
          path: dist

      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Install codedown
        run: npm install -g codedown

      - name: Install Wheel
        run: |
          pip install packaging
          WHEEL_FILE=$(python3 .github/scripts/find_compatible_wheel.py pylace ./dist)
          echo "Installing $WHEEL_FILE"
          pip install $WHEEL_FILE

      - name: Test MDBook Code Samples (Python)
        env:
          FORCE_COLOR: 1
        run: |
          pip install termcolor yq
          NEW_VERSION=$(tomlq -r .package.version < pylace/Cargo.toml)
          python3 .github/scripts/run_code_in_mdfile.py directory python book $NEW_VERSION --exclusion-file .github/resources/mdbook_exclusions.txt

  test-mdbook-build:
    name: Test MDBook Building
    runs-on: ubuntu-latest
    needs: [merge]
    steps:
      - uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy

      - name: Cache Rust
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: . -> pylace/target

      - name: Install preprocessor
        working-directory: book/lace_preprocess_mdbook_yaml
        run: |
          cargo install --bins --locked --path .

      - name: Install mdBook
        run: |
          cargo install mdbook

      - name: Configure the book in test mode and build
        working-directory: book
        run: |
          cat book.toml book.test.toml > book.toml.tmp
          mv book.toml.tmp book.toml
          mdbook build . -d ./html

  release:
    name: Release
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/python-')
    needs: [merge, test-mdbook-python, test-mdbook-build]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Check Branch
        env:
          NEW_VERSION: ${{github.ref_name}}
        run: |
          git fetch origin master
          git tag --merged origin/master | grep $NEW_VERSION
      - name: Check Semver
        working-directory: pylace
        env:
          NEW_VERSION: ${{github.ref_name}}
        run: |
          pip install yq
          DEPLOYING_VERSION=$(echo "$NEW_VERSION" | perl -lpe 's/^python-//')
          find . -name Cargo.toml -exec tomlq -r .package.version {} \; | xargs -n 1 test "$DEPLOYING_VERSION" =
          find . -name pyproject.toml   -exec tomlq -r .project.version {} \; | xargs -n 1 test "$DEPLOYING_VERSION" =
      - uses: actions/download-artifact@v4
        id: download-wheels
        with:
          name: wheels
          path: wheels
      - name: Publish to PyPI
        uses: PyO3/maturin-action@v1.40.2
        env:
          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
        with:
          working-directory: ${{steps.download-wheels.outputs.download-path}}
          command: upload
          args: --skip-existing *