From 4427ae4b8efadcd086464a4991ee279562d217f9 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 14 Nov 2023 23:10:53 +1100 Subject: [PATCH] Use cibuildwheel --- .github/workflows/wheels-build.sh | 40 ------ .../workflows/wheels-dependencies.sh | 116 +++++++----------- .github/workflows/wheels-linux.yml | 69 ----------- .github/workflows/wheels-macos.yml | 57 --------- .github/workflows/wheels-test.sh | 44 +++++++ .github/workflows/wheels.yml | 52 ++++++-- .travis.yml | 113 ++--------------- pyproject.toml | 8 ++ 8 files changed, 146 insertions(+), 353 deletions(-) delete mode 100755 .github/workflows/wheels-build.sh rename wheels/config.sh => .github/workflows/wheels-dependencies.sh (54%) mode change 100644 => 100755 delete mode 100644 .github/workflows/wheels-linux.yml delete mode 100644 .github/workflows/wheels-macos.yml create mode 100755 .github/workflows/wheels-test.sh diff --git a/.github/workflows/wheels-build.sh b/.github/workflows/wheels-build.sh deleted file mode 100755 index 0aeec6b967b..00000000000 --- a/.github/workflows/wheels-build.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - # webp, zstd, xz, libtiff, libxcb cause a conflict with building webp, libtiff, libxcb - # libxdmcp causes an issue on macOS < 11 - # curl from brew requires zstd, use system curl - # if php is installed, brew tries to reinstall these after installing openblas - # remove lcms2 and libpng to fix building openjpeg on arm64 - brew remove --ignore-dependencies webp zstd xz libpng libtiff libxcb libxdmcp curl php lcms2 ghostscript - - brew install pkg-config - - if [[ "$PLAT" == "arm64" ]]; then - export MACOSX_DEPLOYMENT_TARGET="11.0" - else - export MACOSX_DEPLOYMENT_TARGET="10.10" - fi -fi - -if [[ "$MB_PYTHON_VERSION" == pypy3* ]]; then - MB_PYTHON_OSX_VER="10.9" -fi - -echo "::group::Install a virtualenv" - source wheels/multibuild/common_utils.sh - source wheels/multibuild/travis_steps.sh - python3 -m pip install virtualenv - before_install -echo "::endgroup::" - -echo "::group::Build wheel" - build_wheel - ls -l "${GITHUB_WORKSPACE}/${WHEEL_SDIR}/" -echo "::endgroup::" - -if [[ $MACOSX_DEPLOYMENT_TARGET != "11.0" ]]; then - echo "::group::Test wheel" - install_run - echo "::endgroup::" -fi diff --git a/wheels/config.sh b/.github/workflows/wheels-dependencies.sh old mode 100644 new mode 100755 similarity index 54% rename from wheels/config.sh rename to .github/workflows/wheels-dependencies.sh index 655ea295b84..3cd454ba251 --- a/wheels/config.sh +++ b/.github/workflows/wheels-dependencies.sh @@ -1,5 +1,16 @@ +#!/bin/bash # Define custom utilities # Test for macOS with [ -n "$IS_MACOS" ] +if [ -z "$IS_MACOS" ]; then + export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} + export MB_ML_VER=${AUDITWHEEL_POLICY:9} +fi +export PLAT=$CIBW_ARCHS +source wheels/multibuild/common_utils.sh +source wheels/multibuild/library_builders.sh +if [ -z "$IS_MACOS" ]; then + source wheels/multibuild/manylinux_utils.sh +fi ARCHIVE_SDIR=pillow-depends-main @@ -27,9 +38,9 @@ BZIP2_VERSION=1.0.8 LIBXCB_VERSION=1.16 BROTLI_VERSION=1.1.0 -if [[ -n "$IS_MACOS" ]] && [[ "$PLAT" == "x86_64" ]]; then +if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "x86_64" ]]; then function build_openjpeg { - local out_dir=$(fetch_unpack https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz) + local out_dir=$(fetch_unpack https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz openjpeg-2.5.0.tar.gz) (cd $out_dir \ && cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ && make install) @@ -39,7 +50,7 @@ fi function build_brotli { local cmake=$(get_modern_cmake) - local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz) + local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-1.1.0.tar.gz) (cd $out_dir \ && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \ && make install) @@ -49,12 +60,7 @@ function build_brotli { fi } -function pre_build { - # Any stuff that you need to do before you start building the wheels - # Runs in the root directory of this repository. - curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip - untar pillow-depends-main.zip - +function build { build_xz if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then yum remove -y zlib-devel @@ -114,74 +120,36 @@ function pre_build { fi build_simple harfbuzz $HARFBUZZ_VERSION https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION tar.xz --with-freetype=yes --with-glib=no if [ -z "$IS_MACOS" ]; then - export FREETYPE_LIBS='' - export FREETYPE_CFLAGS='' + export FREETYPE_LIBS="" + export FREETYPE_CFLAGS="" fi - - # Append licenses - for filename in wheels/dependency_licenses/*; do - echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE - cat $filename >> LICENSE - done } -function pip_wheel_cmd { - local abs_wheelhouse=$1 - if [ -z "$IS_MACOS" ]; then - CFLAGS="$CFLAGS --std=c99" # for Raqm - fi - python3 -m pip wheel $(pip_opts) \ - -C raqm=enable -C raqm=vendor -C fribidi=vendor \ - -w $abs_wheelhouse --no-deps . -} +# Any stuff that you need to do before you start building the wheels +# Runs in the root directory of this repository. +curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip +untar pillow-depends-main.zip -function run_tests_in_repo { - # Run Pillow tests from within source repo - python3 selftest.py - python3 -m pytest -} +if [[ -n "$IS_MACOS" ]]; then + # webp, zstd, xz, libtiff, libxcb cause a conflict with building webp, libtiff, libxcb + # libxdmcp causes an issue on macOS < 11 + # curl from brew requires zstd, use system curl + # if php is installed, brew tries to reinstall these after installing openblas + # remove cairo to fix building harfbuzz on arm64 + # remove lcms2 and libpng to fix building openjpeg on arm64 + brew remove --ignore-dependencies webp zstd xz libpng libtiff libxcb libxdmcp curl php cairo lcms2 ghostscript + + brew install pkg-config + + if [[ "$CIBW_ARCHS" != "arm64" ]]; then + export MACOSX_DEPLOYMENT_TARGET="10.10" + fi +fi -EXP_CODECS="jpg jpg_2000 libtiff zlib" -EXP_MODULES="freetype2 littlecms2 pil tkinter webp" -EXP_FEATURES="fribidi harfbuzz libjpeg_turbo raqm transp_webp webp_anim webp_mux xcb" +wrap_wheel_builder build -function run_tests { - if [ -n "$IS_MACOS" ]; then - brew install fribidi - export PKG_CONFIG_PATH="/usr/local/opt/openblas/lib/pkgconfig" - elif [ -n "$IS_ALPINE" ]; then - apk add curl fribidi - else - apt-get update - apt-get install -y curl libfribidi0 libopenblas-dev pkg-config unzip - fi - if [ -z "$IS_ALPINE" ]; then - python3 -m pip install numpy - fi - python3 -m pip install defusedxml olefile pyroma - - curl -fsSL -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip - untar pillow-test-images.zip - mv test-images-main/* ../Tests/images - - # Runs tests on installed distribution from an empty directory - (cd .. && run_tests_in_repo) - # Test against expected codecs, modules and features - local ret=0 - local codecs=$(python3 -c 'from PIL.features import *; print(" ".join(sorted(get_supported_codecs())))') - if [ "$codecs" != "$EXP_CODECS" ]; then - echo "Codecs should be: '$EXP_CODECS'; but are '$codecs'" - ret=1 - fi - local modules=$(python3 -c 'from PIL.features import *; print(" ".join(sorted(get_supported_modules())))') - if [ "$modules" != "$EXP_MODULES" ]; then - echo "Modules should be: '$EXP_MODULES'; but are '$modules'" - ret=1 - fi - local features=$(python3 -c 'from PIL.features import *; print(" ".join(sorted(get_supported_features())))') - if [ "$features" != "$EXP_FEATURES" ]; then - echo "Features should be: '$EXP_FEATURES'; but are '$features'" - ret=1 - fi - return $ret -} +# Append licenses +for filename in wheels/dependency_licenses/*; do + echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE + cat $filename >> LICENSE +done diff --git a/.github/workflows/wheels-linux.yml b/.github/workflows/wheels-linux.yml deleted file mode 100644 index 8b2d9d451fb..00000000000 --- a/.github/workflows/wheels-linux.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Build Linux wheels - -on: - workflow_call: - inputs: - artifacts-name: - required: true - type: string - -env: - CONFIG_PATH: "wheels/config.sh" - REPO_DIR: "." - TEST_DEPENDS: "pytest pytest-timeout" - -jobs: - build: - name: ${{ matrix.python }} ${{ matrix.mb-ml-libc }}${{ matrix.mb-ml-ver }} - runs-on: "ubuntu-latest" - strategy: - fail-fast: false - matrix: - python: [ - "pypy3.9-7.3.13", - "pypy3.10-7.3.13", - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", - ] - mb-ml-libc: [ "manylinux" ] - mb-ml-ver: [ 2014, "_2_28" ] - include: - - python: "3.8" - mb-ml-libc: "musllinux" - mb-ml-ver: "_1_1" - - python: "3.9" - mb-ml-libc: "musllinux" - mb-ml-ver: "_1_1" - - python: "3.10" - mb-ml-libc: "musllinux" - mb-ml-ver: "_1_1" - - python: "3.11" - mb-ml-libc: "musllinux" - mb-ml-ver: "_1_1" - - python: "3.12" - mb-ml-libc: "musllinux" - mb-ml-ver: "_1_1" - env: - MB_PYTHON_VERSION: ${{ matrix.python }} - MB_ML_LIBC: ${{ matrix.mb-ml-libc }} - MB_ML_VER: ${{ matrix.mb-ml-ver }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Build Wheel - run: .github/workflows/wheels-build.sh - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.artifacts-name }} - path: wheelhouse/*.whl - # Uncomment to get SSH access for testing - # - name: Setup tmate session - # if: failure() - # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/wheels-macos.yml b/.github/workflows/wheels-macos.yml deleted file mode 100644 index c51abf39a11..00000000000 --- a/.github/workflows/wheels-macos.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Build macOS wheels - -on: - workflow_call: - inputs: - artifacts-name: - required: true - type: string - -env: - CONFIG_PATH: "wheels/config.sh" - REPO_DIR: "." - TEST_DEPENDS: "pytest pytest-timeout" - -jobs: - build: - name: ${{ matrix.python }} ${{ matrix.platform }} - runs-on: "macos-latest" - strategy: - fail-fast: false - matrix: - python: [ - "pypy3.9-7.3.13", - "pypy3.10-7.3.13", - "3.8", - "3.9", - "3.10", - "3.11", - "3.12", - ] - platform: [ "x86_64", "arm64" ] - exclude: - - python: "pypy3.9-7.3.13" - platform: "arm64" - - python: "pypy3.10-7.3.13" - platform: "arm64" - env: - PLAT: ${{ matrix.platform }} - MB_PYTHON_VERSION: ${{ matrix.python }} - TRAVIS_OS_NAME: "osx" - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - name: Build Wheel - run: .github/workflows/wheels-build.sh - - uses: actions/upload-artifact@v3 - with: - name: ${{ inputs.artifacts-name }} - path: wheelhouse/*.whl - # Uncomment to get SSH access for testing - # - name: Setup tmate session - # if: failure() - # uses: mxschmitt/action-tmate@v3 diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh new file mode 100755 index 00000000000..fbc692ce729 --- /dev/null +++ b/.github/workflows/wheels-test.sh @@ -0,0 +1,44 @@ +set -e + +EXP_CODECS="jpg jpg_2000 libtiff zlib" +EXP_MODULES="freetype2 littlecms2 pil tkinter webp" +EXP_FEATURES="fribidi harfbuzz libjpeg_turbo raqm transp_webp webp_anim webp_mux xcb" + +if [[ "$OSTYPE" == "darwin"* ]]; then + brew install fribidi + export PKG_CONFIG_PATH="/usr/local/opt/openblas/lib/pkgconfig" +elif [ "${AUDITWHEEL_POLICY:9}" == "musllinux" ]; then + apk add curl fribidi +else + yum install -y fribidi openblas-devel pkgconfig +fi +if [ "${AUDITWHEEL_POLICY:9}" != "musllinux" ]; then + python3 -m pip install numpy +fi + +if [ ! -d "test-images-main" ]; then + curl -fsSL -o pillow-test-images.zip https://github.com/python-pillow/test-images/archive/main.zip + unzip pillow-test-images.zip + mv test-images-main/* Tests/images +fi + +# Runs tests +python3 selftest.py +python3 -m pytest + +# Test against expected codecs, modules and features +codecs=$(python3 -c 'from PIL.features import *; print(" ".join(sorted(get_supported_codecs())))') +if [ "$codecs" != "$EXP_CODECS" ]; then + echo "Codecs should be: '$EXP_CODECS'; but are '$codecs'" + exit 1 +fi +modules=$(python3 -c 'from PIL.features import *; print(" ".join(sorted(get_supported_modules())))') +if [ "$modules" != "$EXP_MODULES" ]; then + echo "Modules should be: '$EXP_MODULES'; but are '$modules'" + exit 1 +fi +features=$(python3 -c 'from PIL.features import *; print(" ".join(sorted(get_supported_features())))') +if [ "$features" != "$EXP_FEATURES" ]; then + echo "Features should be: '$EXP_FEATURES'; but are '$features'" + exit 1 +fi diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index bcffe6839c0..475019c12c3 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -24,15 +24,47 @@ env: FORCE_COLOR: 1 jobs: - macos: - uses: ./.github/workflows/wheels-macos.yml - with: - artifacts-name: "dist" - - linux: - uses: ./.github/workflows/wheels-linux.yml - with: - artifacts-name: "dist" + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: "macOS x86_64" + os: macos-latest + archs: x86_64 + - name: "macOS arm64" + os: macos-latest + archs: arm64 + - name: "manylinux2014 and musllinux x86_64" + os: ubuntu-latest + archs: x86_64 + - name: "manylinux_2_28 x86_64" + os: ubuntu-latest + archs: x86_64 + build: "*manylinux*" + manylinux: "manylinux_2_28" + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Build wheels + run: | + python3 -m pip install cibuildwheel + python3 -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.archs }} + CIBW_BUILD: ${{ matrix.build }} + CIBW_CONFIG_SETTINGS: raqm=enable raqm=vendor fribidi=vendor + CIBW_MANYLINUX_PYPY_X86_64_IMAGE: ${{ matrix.manylinux }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }} + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl sdist: runs-on: ubuntu-latest @@ -56,7 +88,7 @@ jobs: success: permissions: contents: none - needs: [macos, linux, sdist] + needs: [build, sdist] runs-on: ubuntu-latest name: Wheels Successful steps: diff --git a/.travis.yml b/.travis.yml index 503f243e65a..d8add1b6fec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,7 @@ if: tag IS present OR type = api env: global: - - CONFIG_PATH=wheels/config.sh - - REPO_DIR=. - - PLAT=aarch64 - - TEST_DEPENDS=pytest-timeout + - CIBW_ARCHS=aarch64 language: python # Default Python version is usually 3.6 @@ -15,120 +12,30 @@ services: docker jobs: include: - - name: "3.8 Focal manylinux2014 aarch64" + - name: "manylinux2014 and musllinux aarch64" os: linux arch: arm64 - env: - - MB_ML_VER=2014 - - MB_PYTHON_VERSION=3.8 - - name: "3.8 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_2_28" - - MB_PYTHON_VERSION=3.8 - - name: "3.8 musllinux_1_1 aarch64" + - name: "manylinux_2_28 aarch64" os: linux arch: arm64 env: - - MB_ML_VER="_1_1" - - MB_ML_LIBC="musllinux" - - MB_PYTHON_VERSION=3.8 - - name: "3.9 Focal manylinux2014 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER=2014 - - MB_PYTHON_VERSION=3.9 - - name: "3.9 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_2_28" - - MB_PYTHON_VERSION=3.9 - - name: "3.9 musllinux_1_1 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_1_1" - - MB_ML_LIBC="musllinux" - - MB_PYTHON_VERSION=3.9 - - name: "3.10 Focal manylinux2014 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER=2014 - - MB_PYTHON_VERSION=3.10 - - name: "3.10 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_2_28" - - MB_PYTHON_VERSION=3.10 - - name: "3.10 musllinux_1_1 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_1_1" - - MB_ML_LIBC="musllinux" - - MB_PYTHON_VERSION=3.10 - - name: "3.11 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER=2014 - - MB_PYTHON_VERSION=3.11 - - name: "3.11 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_2_28" - - MB_PYTHON_VERSION=3.11 - - name: "3.11 musllinux_1_1 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_1_1" - - MB_ML_LIBC="musllinux" - - MB_PYTHON_VERSION=3.11 - - name: "3.12 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER=2014 - - MB_PYTHON_VERSION=3.12 - - name: "3.12 Focal manylinux_2_28 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_2_28" - - MB_PYTHON_VERSION=3.12 - - name: "3.12 musllinux_1_1 aarch64" - os: linux - arch: arm64 - env: - - MB_ML_VER="_1_1" - - MB_ML_LIBC="musllinux" - - MB_PYTHON_VERSION=3.12 - -before_install: - - source wheels/multibuild/common_utils.sh - - source wheels/multibuild/travis_steps.sh - - before_install + - CIBW_BUILD: "*manylinux*" + - CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + - CIBW_MANYLINUX_PYPY_X86_64_IMAGE: "manylinux_2_28" install: - - build_multilinux aarch64 build_wheel - - ls -l "${TRAVIS_BUILD_DIR}/${WHEEL_SDIR}/" + - python3 -m pip install cibuildwheel script: - - install_run + - python3 -m cibuildwheel --output-dir wheelhouse + - ls -l "${TRAVIS_BUILD_DIR}/wheelhouse/" # Upload wheels to GitHub Releases deploy: provider: releases api_key: $GITHUB_RELEASE_TOKEN file_glob: true - file: "${TRAVIS_BUILD_DIR}/${WHEEL_SDIR}/*.whl" + file: "${TRAVIS_BUILD_DIR}/wheelhouse/*.whl" on: repo: python-pillow/Pillow tags: true diff --git a/pyproject.toml b/pyproject.toml index a49179a3734..c8627c01218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,14 @@ Mastodon = "https://fosstodon.org/@pillow" Source = "https://github.com/python-pillow/Pillow" Twitter = "https://twitter.com/PythonPillow" +[tool.cibuildwheel] +before-all = ".github/workflows/wheels-dependencies.sh" +test-command = "cd {project} && .github/workflows/wheels-test.sh" +test-extras = "tests" + +[tool.cibuildwheel.linux] +environment = { CFLAGS="--std=c99" } + [tool.setuptools] packages = ["PIL"] include-package-data = true