From 60556f820345ced65375d835be9f7a09df010ee0 Mon Sep 17 00:00:00 2001 From: Frankie Dintino Date: Fri, 8 Sep 2023 09:31:58 -0400 Subject: [PATCH] chore: Update dependencies, including libavif to 1.0.1 --- .github/workflows/macos-install.sh | 3 +- .github/workflows/test.yml | 80 +++++++---------- .github/workflows/wheels.yml | 23 +---- ...aom-3.5.0-monochrome-realtime-encode.patch | 43 ---------- wheelbuild/aom-fix-stack-size.patch | 8 -- wheelbuild/config.sh | 85 ++++++++++++++----- 6 files changed, 101 insertions(+), 141 deletions(-) delete mode 100644 wheelbuild/aom-3.5.0-monochrome-realtime-encode.patch diff --git a/.github/workflows/macos-install.sh b/.github/workflows/macos-install.sh index b05060c..6ceab1e 100755 --- a/.github/workflows/macos-install.sh +++ b/.github/workflows/macos-install.sh @@ -2,7 +2,8 @@ set -e -brew install dav1d aom rav1e cmake +brew reinstall cmake +brew install dav1d aom rav1e if [ "$GHA_PYTHON_VERSION" == "2.7" ]; then python2 -m pip install -U tox tox-gh-actions diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d330e92..0f79396 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,6 @@ jobs: "3.9", "3.8", "3.7", - "2.7", ] include: - python-version: "3.7" @@ -95,34 +94,16 @@ jobs: CODECOV_NAME: ${{ matrix.os }} Python ${{ matrix.python-version }} msys: - runs-on: windows-2019 + runs-on: windows-latest - strategy: - fail-fast: false - matrix: - msystem: [ "mingw32", "mingw64" ] - python-version: ["2", "3"] - include: - - msystem: "mingw32" - name: "MSYS2 MinGW 32-bit" - package: "mingw-w64-i686" - env: i686 - - msystem: "mingw64" - name: "MSYS2 MinGW 64-bit" - package: "mingw-w64-x86_64" - env: x86_64 - exclude: - - msystem: "mingw32" - python-version: "2" - - name: ${{ matrix.name }} python ${{ matrix.python-version }} + name: MinGW defaults: run: - shell: msys2 {0} + shell: bash.exe --login -eo pipefail "{0}" env: - MSYSTEM: ${{ matrix.msystem }} + MSYSTEM: MINGW64 CHERE_INVOKING: 1 timeout-minutes: 30 @@ -130,43 +111,46 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: msys2/setup-msys2@v2 - with: - msystem: ${{ matrix.msystem }} - update: true - install: >- - base-devel - git - ${{ matrix.package }}-toolchain - ${{ matrix.package }}-python${{ matrix.python-version }}-pip - ${{ matrix.package }}-python${{ matrix.python-version }}-setuptools - ${{ matrix.package }}-libjpeg-turbo - ${{ matrix.package }}-libtiff - ${{ matrix.package }}-libpng - ${{ matrix.package }}-openjpeg2 - ${{ matrix.package }}-libavif + - name: Set up shell + run: echo "C:\msys64\usr\bin\" >> $env:GITHUB_PATH + shell: pwsh + + - name: Install dependencies + run: | + pacman -S --noconfirm \ + base-devel \ + git \ + mingw-w64-x86_64-gcc \ + mingw-w64-x86_64-toolchain \ + mingw-w64-x86_64-python3-pip \ + mingw-w64-x86_64-python3-setuptools \ + mingw-w64-x86_64-libjpeg-turbo \ + mingw-w64-x86_64-libtiff \ + mingw-w64-x86_64-libpng \ + mingw-w64-x86_64-openjpeg2 \ + mingw-w64-x86_64-zlib \ + mingw-w64-x86_64-libavif - name: Install Dependencies run: | - python${{ matrix.python-version }} -m pip install pytest pytest-cov pillow mock + python3 -m pip install pytest pytest-cov pillow mock - name: Build pillow-avif-plugin - run: CFLAGS="-coverage" python${{ matrix.python-version }} setup.py build_ext install + run: SETUPTOOLS_USE_DISTUTILS="stdlib" CFLAGS="-coverage" python3 -m pip install . - name: Test pillow-avif-plugin run: | - python${{ matrix.python-version }} -m pytest -vx --cov pillow_avif --cov tests --cov-report term --cov-report xml tests + python3 -m pytest -vx --cov pillow_avif --cov tests --cov-report term --cov-report xml tests - name: Upload coverage - run: | - python${{ matrix.python-version }} -m pip install codecov - bash <(curl -s https://codecov.io/bash) -F GHA_Windows - env: - CODECOV_NAME: ${{ matrix.name }} - + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: GHA_Windows + name: "MSYS2 MinGW" success: needs: [build, msys] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest name: Test Successful steps: - name: Success diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 28a3e5a..e4b17fa 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -20,20 +20,12 @@ jobs: matrix: os: [ "ubuntu-20.04", "macos-11" ] python: [ "2.7", "3.7", "3.8", "3.9", "3.10", "3.11" ] - platform: [ "x86_64", "i686" ] - manylinux-version: [ "1", "2010" ] + platform: [ "x86_64" ] + manylinux-version: [ "2010" ] macos-target: [ "10.10" ] mb-ml-libc: [ "manylinux" ] multibuild-sha: [ "34e970c4bc448b73af0127615fc4583b4f247369" ] exclude: - - os: "macos-11" - platform: "i686" - - os: "macos-11" - manylinux-version: "1" - - python: "3.10" - manylinux-version: "1" - - python: "3.11" - manylinux-version: "1" - python: "3.11" manylinux-version: "2010" - python: "3.11" @@ -41,9 +33,6 @@ jobs: include: - os: "macos-11" os-name: "osx" - - os: "ubuntu-20.04" - manylinux-version: "1" - os-name: "manylinux1" - os: "ubuntu-20.04" manylinux-version: "2010" os-name: "manylinux2010" @@ -136,12 +125,6 @@ jobs: os-name: "manylinux2014" manylinux-version: "2014" multibuild-sha: "bb32cfec4f755cb146332a0490abcf3187ce61d1" - - python: "3.11" - platform: "i686" - os: "ubuntu-20.04" - os-name: "manylinux2014" - manylinux-version: "2014" - multibuild-sha: "bb32cfec4f755cb146332a0490abcf3187ce61d1" env: BUILD_COMMIT: HEAD PLAT: ${{ matrix.platform }} @@ -222,7 +205,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] architecture: ["x64"] include: - architecture: "x64" diff --git a/wheelbuild/aom-3.5.0-monochrome-realtime-encode.patch b/wheelbuild/aom-3.5.0-monochrome-realtime-encode.patch deleted file mode 100644 index 07dcd90..0000000 --- a/wheelbuild/aom-3.5.0-monochrome-realtime-encode.patch +++ /dev/null @@ -1,43 +0,0 @@ -From bf9d1f8d2b649ee8a02e6e8792075330bde185d4 Mon Sep 17 00:00:00 2001 -From: James Zern -Date: Thu, 27 Oct 2022 18:17:47 -0700 -Subject: [PATCH] fix monochrome encoding in realtime mode - -this is non-exhaustive, but covers the file in the issue report and the -newly added MonochromeRealtimeTest. - -Bug: https://github.com/AOMediaCodec/libavif/issues/1190 -Change-Id: I947a5d926daaae9c0a667114996ddd7bf9ff2013 ---- - av1/encoder/nonrd_pickmode.c | 3 ++- - av1/encoder/var_based_part.c | 2 +- - test/monochrome_test.cc | 47 +++++++++++++++++++++++++++++------- - 3 files changed, 41 insertions(+), 11 deletions(-) - -diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c -index 7267f9e39b..009b3c360d 100644 ---- a/av1/encoder/nonrd_pickmode.c -+++ b/av1/encoder/nonrd_pickmode.c -@@ -2777,7 +2777,8 @@ void set_color_sensitivity(AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize, - x->color_sensitivity[1] = 0; - return; - } -- for (int i = 1; i <= 2; ++i) { -+ const int num_planes = av1_num_planes(&cpi->common); -+ for (int i = 1; i < num_planes; ++i) { - if (x->color_sensitivity[i - 1] == 2 || source_variance < 50) { - struct macroblock_plane *const p = &x->plane[i]; - const BLOCK_SIZE bs = -diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c -index efcb5a71f0..63f807733b 100644 ---- a/av1/encoder/var_based_part.c -+++ b/av1/encoder/var_based_part.c -@@ -1237,7 +1237,7 @@ static void setup_planes(AV1_COMP *cpi, MACROBLOCK *x, unsigned int *y_sad, - set_ref_ptrs(cm, xd, mi->ref_frame[0], mi->ref_frame[1]); - av1_enc_build_inter_predictor(cm, xd, mi_row, mi_col, NULL, - cm->seq_params->sb_size, AOM_PLANE_Y, -- AOM_PLANE_V); -+ num_planes - 1); - } - } - diff --git a/wheelbuild/aom-fix-stack-size.patch b/wheelbuild/aom-fix-stack-size.patch index d8206e8..beaca04 100644 --- a/wheelbuild/aom-fix-stack-size.patch +++ b/wheelbuild/aom-fix-stack-size.patch @@ -68,11 +68,3 @@ index 8d04312..efbed78 100644 typedef CRITICAL_SECTION pthread_mutex_t; #if _WIN32_WINNT < 0x0600 -@@ -147,6 +148,7 @@ - #include // NOLINT - - #define pthread_t TID -+#define pthread_attr_t int - #define pthread_mutex_t HMTX - - typedef struct { diff --git a/wheelbuild/config.sh b/wheelbuild/config.sh index d4ed5b6..8e23afc 100644 --- a/wheelbuild/config.sh +++ b/wheelbuild/config.sh @@ -4,13 +4,13 @@ set -eo pipefail CONFIG_DIR=$(abspath $(dirname "${BASH_SOURCE[0]}")) ARCHIVE_SDIR=pillow-avif-plugin-depends -LIBAVIF_VERSION=0.11.0 -AOM_VERSION=3.5.0 -DAV1D_VERSION=1.0.0 -SVT_AV1_VERSION=1.3.0 -RAV1E_VERSION=p20230417 -LIBWEBP_SHA=15a91ab179b0b605727d16fb751c12674da9dfec -LIBYUV_SHA=f9fda6e7 +LIBAVIF_VERSION=1.0.1 +AOM_VERSION=3.7.0 +DAV1D_VERSION=1.2.1 +SVT_AV1_VERSION=1.7.0 +RAV1E_VERSION=p20230911 +LIBWEBP_SHA=e2c85878f6a33f29948b43d3492d9cdaf801aa54 +LIBYUV_SHA=464c51a0 CCACHE_VERSION=4.7.1 SCCACHE_VERSION=0.3.0 export PERLBREWURL=https://raw.githubusercontent.com/gugod/App-perlbrew/release-0.92/perlbrew @@ -66,12 +66,26 @@ if [[ -n "${GITHUB_ACTIONS:-}" ]]; then exec 2>&1 fi +function require_package { + local pkg=$1 + local pkgconfig=${PKGCONFIG:-pkg-config} + if ! $pkgconfig --exists $pkg; then + echo "$pkg failed to build" + exit 1 + fi +} + function install_ccache { + if [[ $(type -P ccache) ]]; then + return + fi mkdir -p $PWD/ccache if [ -e /parent-home ]; then ln -s $PWD/ccache /parent-home/.ccache fi - ln -s $PWD/ccache $HOME/.ccache + if [ ! -e $HOME/.ccache ]; then + ln -s $PWD/ccache $HOME/.ccache + fi group_start "Install ccache" if [ -n "$IS_MACOS" ]; then @@ -104,6 +118,9 @@ function install_ccache { } function install_sccache { + if [[ $(type -P sccache) ]]; then + return + fi group_start "Install sccache" if [ -n "$IS_MACOS" ]; then brew install sccache @@ -174,10 +191,6 @@ function build_aom { local cmake_flags=() - if [ -n "$IS_MACOS" ]; then - brew remove --ignore-dependencies aom php imagemagick libavif libheif gd composer - fi - fetch_unpack \ https://storage.googleapis.com/aom-releases/libaom-$AOM_VERSION.tar.gz @@ -205,10 +218,6 @@ function build_aom { extra_cmake_flags+=("-DCMAKE_EXE_LINKER_FLAGS=-Wl,-z,stack-size=2097152") fi - # Fix for https://github.com/AOMediaCodec/libavif/issues/1190 - (cd libaom-$AOM_VERSION \ - && patch -p1 -i $CONFIG_DIR/aom-3.5.0-monochrome-realtime-encode.patch) - mkdir libaom-$AOM_VERSION/build/work (cd libaom-$AOM_VERSION/build/work \ && cmake \ @@ -224,6 +233,9 @@ function build_aom { "${cmake_flags[@]}" \ ../.. \ && make install) + + require_package aom + group_end touch aom-stamp } @@ -310,6 +322,8 @@ function build_svt_av1 { && make install \ && cp SvtAv1Enc.pc $BUILD_PREFIX/lib/pkgconfig) + require_package SvtAv1Enc + group_end touch svt-av1-stamp @@ -341,6 +355,8 @@ function build_rav1e { rm -rf $BUILD_PREFIX/lib/librav1e*.dylib fi + require_package rav1e + group_end } @@ -364,8 +380,24 @@ function build_libsharpyuv { -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_LIBDIR=lib \ "${cmake_flags[@]}" \ && ninja sharpyuv) + + if [ -n "$IS_MACOS" ]; then + CP="sudo cp" + MKDIR="sudo mkdir" + else + CP="cp" + MKDIR="mkdir" + fi + + $CP libwebp-$LIBWEBP_SHA/build/libsharpyuv.a $BUILD_PREFIX/lib + $CP libwebp-$LIBWEBP_SHA/build/sharpyuv/libsharpyuv.pc $BUILD_PREFIX/lib/pkgconfig + $MKDIR -p $BUILD_PREFIX/include/webp/sharpyuv + $CP libwebp-$LIBWEBP_SHA/sharpyuv/*.h $BUILD_PREFIX/include/webp/sharpyuv + + require_package libsharpyuv group_end touch libsharpyuv-stamp } @@ -390,10 +422,20 @@ function build_libyuv { fi (cd libyuv-$LIBYUV_SHA/build \ && cmake -G Ninja .. \ - -DBUILD_SHARED_LIBS=0 \ + -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release \ "${cmake_flags[@]}" .. \ && ninja yuv) + + if [ -n "$IS_MACOS" ]; then + CP="sudo cp" + else + CP="cp" + fi + + $CP libyuv-$LIBYUV_SHA/build/libyuv.a $BUILD_PREFIX/lib + $CP -a libyuv-$LIBYUV_SHA/include/* $BUILD_PREFIX/include + group_end touch libyuv-stamp } @@ -401,6 +443,10 @@ function build_libyuv { function build_libavif { LIBAVIF_CMAKE_FLAGS=() + if [ -n "$IS_MACOS" ]; then + brew remove --ignore-dependencies webp jpeg-xl aom composer gd imagemagick libavif libheif php + fi + build_aom LIBAVIF_CMAKE_FLAGS+=(-DAVIF_CODEC_AOM=ON) @@ -450,12 +496,8 @@ function build_libavif { group_end build_libsharpyuv - mv libwebp-$LIBWEBP_SHA libavif-$LIBAVIF_VERSION/ext/libwebp - LIBAVIF_CMAKE_FLAGS+=(-DAVIF_LOCAL_LIBSHARPYUV=ON) build_libyuv - mv libyuv-$LIBYUV_SHA libavif-$LIBAVIF_VERSION/ext/libyuv - LIBAVIF_CMAKE_FLAGS+=(-DAVIF_LOCAL_LIBYUV=ON) group_start "Build libavif" @@ -465,6 +507,7 @@ function build_libavif { && cmake .. \ -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX \ -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ "${LIBAVIF_CMAKE_FLAGS[@]}" \ && make install)