Skip to content
CI

Fixing Build Config Issues #388

Sign in to view logs

Fixing Build Config Issues

Fixing Build Config Issues #388

Workflow file for this run

.github/workflows/CI.yml at 267893d
name: CI
on:
push:
branches:
- main
- master
tags:
- '*'
pull_request:
branches:
- main
- master
workflow_dispatch:
permissions:
contents: read
env:
DOLMA_TESTS_SKIP_AWS: ${{ secrets.AWS_ACCESS_KEY_ID == '' && 'true' || 'false' }}
DOLMA_TEST_S3_PREFIX: s3://dolma-tests
RUST_CHANNEL: stable
jobs:
info:
runs-on: ubuntu-latest
steps:
- name: Echo environment variables
run: |
echo "reference: ${{ github.ref }}"
echo "event name: ${{ github.event_name }}"
echo "run tests: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}"
echo "is main: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' }}"
echo "is release: ${{ startsWith(github.ref, 'refs/tags/') }}"
echo "skip AWS: ${{ env.DOLMA_TESTS_SKIP_AWS }}"
echo "commit: ${{ github.sha }}"
echo "PR base repo: ${{ github.event.pull_request.base.repo.full_name }}/tree/${{ github.event.pull_request.base.ref }}"
echo "PR head repo: ${{ github.event.pull_request.head.repo.full_name }}/tree/${{ github.event.pull_request.head.ref }}"
prepare-venv:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Cache Virtual Env
uses: actions/cache@v3
# name for referring later
id: cache-venv
with:
# what we cache: the virtualenv
path: ./.venv/
# The cache key depends on pyproject.toml and Cargo.toml
key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml', '**/Cargo.toml, **/Cargo.lock') }}--${{ hashFiles('python/**', 'src/**') }}
- name: Setup system libraries
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
sudo apt-get update
sudo apt-get install --yes --upgrade build-essential cmake protobuf-compiler libssl-dev glibc-source
- name: Install Rust toolchain
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
rustup update ${{ env.RUST_CHANNEL }}
rustup component add --toolchain ${{ env.RUST_CHANNEL }} rustfmt rust-src
rustup default ${{ env.RUST_CHANNEL }}
- name: Install Python
if: steps.cache-venv.outputs.cache-hit != 'true'
uses: actions/setup-python@v4
with:
python-version: '3.8'
architecture: "x64"
- name: Create a new Python environment & install maturin
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
python -m venv .venv
source .venv/bin/activate
pip install -U pip
pip install maturin
- name: Install dolma wheels
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
source .venv/bin/activate
maturin build --release -i $(which python) --out dist
wheel_path=$(ls dist/*.whl)
pip install "${wheel_path}[all]"
tests:
runs-on: ubuntu-latest
needs: prepare-venv
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
name: "${{ matrix.task.name }}"
strategy:
fail-fast: true
matrix:
task:
- name: Check Python style
run: |
isort --check tests/python/ && isort --check python/
black --check tests/python/ && black --check python/
- name: Check Rust style
run: |
rustfmt --edition 2021 src/*.rs --check
- name: Lint Python
run: |
flake8 tests/python/ && flake8 python/
- name: Types Python
run: |
mypy tests/python/ && mypy python/
- name: Run Python tests
run: |
pytest -vs --color=yes tests/python/
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Cache Virtual Env
uses: actions/cache@v3
# name for referring later
id: cache-venv
with:
# what we cache: the virtualenv
path: ./.venv/
# The cache key depends on pyproject.toml and Cargo.toml
key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml', '**/Cargo.toml, **/Cargo.lock') }}--${{ hashFiles('python/**', 'src/**') }}
- name: ${{ matrix.task.name }}
run: |
source .venv/bin/activate
${{ matrix.task.run }}
build-linux:
# if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
runs-on: ubuntu-latest
env:
CC: gcc-11
CXX: g++-11
strategy:
matrix:
# target: [x86_64-unknown-linux-gnu, i686-unknown-linux-gnu]
target: [x86_64-unknown-linux-gnu, i686-unknown-linux-gnu, aarch64-unknown-linux-gnu]
# target: [armv7-unknown-linux-gnueabihf]
# target: [x86_64-unknown-linux-gnu]
# python: ['3.8', '3.9', '3.10', '3.11', '3.12']
python: ['3.10']
name: "Build Linux (python${{ matrix.python }}, ${{ matrix.target }})"
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
cache: 'pip'
- name: Setup environment
run: |
sudo apt-get update
sudo apt-get install --yes --upgrade build-essential cmake protobuf-compiler libssl-dev glibc-source
- name: Install 32bit version of libc
if: ${{ contains(matrix.target, 'i686') }}
run: |
sudo apt-get install --yes --upgrade libc6-dev-i386
- name: Setup cross-compilation to ARM
if: ${{ contains(matrix.target, 'aarch64') }}
run: |
sudo apt-get install --yes --upgrade gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross
export CROSS_TOOLCHAIN_PREFIX=aarch64-linux-gnu-
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="$CROSS_TARGET_RUNNER"
export AR_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"ar
export CC_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc
export CXX_aarch64_unknown_linux_gnu="$CROSS_TOOLCHAIN_PREFIX"g++
export CMAKE_TOOLCHAIN_FILE_aarch64_unknown_linux_gnu=/opt/toolchain.cmake
export RUST_TEST_THREADS=1
export PKG_CONFIG_ALLOW_CROSS=1
export RUST_BACKTRACE=1
# export CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++
# export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
# export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc
- name: Add Rust Target
run: |
rustup target add ${{ matrix.target }}
- name: Install Maturin
run: pip install 'maturin[patchelf]>=1.1,<2.0'
- name: Building wheels
run: |
echo "rust backtrace: $RUST_BACKTRACE"
maturin build \
--release \
--out dist \
--target ${{ matrix.target }} \
--interpreter 'python${{ matrix.python }}'
# --manylinux ${{ matrix.target == 'x86' && 'auto' || '2_28' }}
# - name: Build wheels
# uses: PyO3/maturin-action@v1
# with:
# target: ${{ matrix.target }}
# args: --release --out dist --find-interpreter
# sccache: ${{ matrix.target == 'x86' && 'false' || 'true' }}
# manylinux: ${{ matrix.target == 'x86' && 'auto' || '2_28' }}
- name: Upload wheels
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
build-windows:
if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
runs-on: windows-latest
strategy:
matrix:
target: [x64, x86]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10'
architecture: ${{ matrix.target }}
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
args: --release --out dist --find-interpreter
sccache: true
- name: Upload wheels
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
build-macos:
if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
runs-on: macos-latest
strategy:
matrix:
target: [x86_64, aarch64]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Build wheels
uses: PyO3/maturin-action@v1
with:
target: ${{ matrix.target }}
args: --release --out dist --find-interpreter
sccache: true
- name: Upload wheels
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
sdist:
runs-on: ubuntu-latest
if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
steps:
- uses: actions/checkout@v3
- name: Build sdist
uses: PyO3/maturin-action@v1
with:
command: sdist
args: --out dist
- name: Upload sdist
uses: actions/upload-artifact@v3
with:
name: wheels
path: dist
release:
name: Release
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
needs: [build-linux, build-windows, build-macos, sdist]
steps:
- uses: actions/download-artifact@v3
with:
name: wheels
- name: Publish to PyPI
uses: PyO3/maturin-action@v1
env:
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
with:
command: upload
args: --skip-existing *