diff --git a/.ci/requirements-cibw.txt b/.ci/requirements-cibw.txt new file mode 100644 index 0000000..a1424ec --- /dev/null +++ b/.ci/requirements-cibw.txt @@ -0,0 +1 @@ +cibuildwheel==2.21.2 diff --git a/.github/workflows/wheels-dependencies.sh b/.github/workflows/wheels-dependencies.sh new file mode 100755 index 0000000..9774390 --- /dev/null +++ b/.github/workflows/wheels-dependencies.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Define custom utilities +# Test for macOS with [ -n "$IS_MACOS" ] +if [ -z "$IS_MACOS" ]; then + export MB_ML_LIBC=${AUDITWHEEL_POLICY::9} + export MB_ML_VER=${AUDITWHEEL_POLICY:9} +fi +export PLAT=$CIBW_ARCHS +source multibuild/common_utils.sh +source multibuild/library_builders.sh +if [ -z "$IS_MACOS" ]; then + source multibuild/manylinux_utils.sh +fi + +source wheelbuild/config.sh + +function build { + if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "arm64" ]]; then + sudo chown -R runner /usr/local + fi + pre_build +} + +wrap_wheel_builder build diff --git a/.github/workflows/wheels-test.ps1 b/.github/workflows/wheels-test.ps1 new file mode 100644 index 0000000..097b38c --- /dev/null +++ b/.github/workflows/wheels-test.ps1 @@ -0,0 +1,18 @@ +param ([string]$venv, [string]$pillow_avif_plugin="C:\pillow_avif_plugin") +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +Set-PSDebug -Trace 1 +if ("$venv" -like "*\cibw-run-*\pp*-win_amd64\*") { + # unlike CPython, PyPy requires Visual C++ Redistributable to be installed + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + Invoke-WebRequest -Uri 'https://aka.ms/vs/15/release/vc_redist.x64.exe' -OutFile 'vc_redist.x64.exe' + C:\vc_redist.x64.exe /install /quiet /norestart | Out-Null +} +$env:path += ";$pillow_avif_plugin\winbuild\build\bin\" +& "$venv\Scripts\activate.ps1" +& reg add "HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\python.exe" /v "GlobalFlag" /t REG_SZ /d "0x02000000" /f +cd $pillow_avif_plugin +& python -VV +if (!$?) { exit $LASTEXITCODE } +& python -m pytest -vx tests +if (!$?) { exit $LASTEXITCODE } diff --git a/.github/workflows/wheels-test.sh b/.github/workflows/wheels-test.sh new file mode 100755 index 0000000..02c2da5 --- /dev/null +++ b/.github/workflows/wheels-test.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e + +python3 -m pytest diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9e860a1..9a0503c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -13,14 +13,14 @@ env: LIBAVIF_VERSION: 88d3dccda111f6ccbcccd925179f67e7d6fdf4ff jobs: - build: + legacy: name: ${{ matrix.python }} ${{ matrix.os-name }} ${{ matrix.arch == 'x86' && 'x86_64' || matrix.os-type == 'macos' && 'arm64' || 'aarch64' }} runs-on: ${{ matrix.os-type == 'ubuntu' && 'ubuntu-latest' || (matrix.os-type == 'macos' && (matrix.arch == 'x86' || matrix.python == '2.7' || matrix.python == '3.7')) && 'macos-13' || 'macos-latest' }} strategy: fail-fast: false matrix: os-type: [ "ubuntu", "macos" ] - python: [ "2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12" ] + python: [ "2.7", "3.7", "3.8" ] arch: [ "x86", "arm" ] manylinux-version: [ "2014" ] mb-ml-libc: [ "manylinux", "musllinux" ] @@ -88,7 +88,7 @@ jobs: with: repository: multi-build/multibuild path: multibuild - ref: ${{ (matrix.python == '3.11' || matrix.python == '3.12' || matrix.os == 'macos-latest' || (env.PLAT == 'arm64' && matrix.python != '2.7')) && '88146e74ebc86baf97b6fec448ef766d64326582' || '34e970c4bc448b73af0127615fc4583b4f247369' }} + ref: 34e970c4bc448b73af0127615fc4583b4f247369 - uses: actions/setup-python@v4 with: @@ -139,175 +139,219 @@ jobs: name: wheels path: wheelhouse/*.whl - windows: - runs-on: windows-2019 + build-1-QEMU-emulated-wheels: + name: aarch64 ${{ matrix.python-version }} ${{ matrix.spec }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - pp310 + - cp3{9,10,11} + - cp3{12,13} + spec: + - manylinux2014 + - manylinux_2_28 + - musllinux + exclude: + - { python-version: pp310, spec: musllinux } + + steps: + - uses: actions/checkout@v4 + + - name: Checkout multibuild + uses: actions/checkout@v4 + with: + repository: multi-build/multibuild + path: multibuild + ref: 452dd2d1705f6b2375369a6570c415beb3163f70 + + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Install cibuildwheel + run: | + python3 -m pip install -r .ci/requirements-cibw.txt + + - name: Build wheels + run: | + python3 -m cibuildwheel --output-dir wheelhouse + env: + # Build only the currently selected Linux architecture (so we can + # parallelise for speed). + CIBW_ARCHS: "aarch64" + # Likewise, select only one Python version per job to speed this up. + CIBW_BUILD: "${{ matrix.python-version }}-${{ matrix.spec == 'musllinux' && 'musllinux' || 'manylinux' }}*" + CIBW_PRERELEASE_PYTHONS: True + # Extra options for manylinux. + CIBW_MANYLINUX_AARCH64_IMAGE: ${{ matrix.spec }} + CIBW_MANYLINUX_PYPY_AARCH64_IMAGE: ${{ matrix.spec }} + + - uses: actions/upload-artifact@v4 + with: + name: dist-qemu-${{ matrix.python-version }}-${{ matrix.spec }} + path: ./wheelhouse/*.whl + + build-2-native-wheels: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - architecture: ["x64"] include: - - architecture: "x64" - platform-vcvars: "x86_amd64" - platform-msbuild: "x64" - timeout-minutes: 300 + - name: "macOS 10.10 x86_64" + os: macos-13 + cibw_arch: x86_64 + build: "cp3{9,10,11}*" + macosx_deployment_target: "10.10" + - name: "macOS 10.13 x86_64" + os: macos-13 + cibw_arch: x86_64 + build: "cp3{12,13}*" + macosx_deployment_target: "10.13" + - name: "macOS 10.15 x86_64" + os: macos-13 + cibw_arch: x86_64 + build: "pp310*" + macosx_deployment_target: "10.15" + - name: "macOS arm64" + os: macos-latest + cibw_arch: arm64 + macosx_deployment_target: "11.0" + - name: "manylinux2014 and musllinux x86_64" + os: ubuntu-latest + cibw_arch: x86_64 + - name: "manylinux_2_28 x86_64" + os: ubuntu-latest + cibw_arch: x86_64 + build: "*manylinux*" + manylinux: "manylinux_2_28" + steps: + - uses: actions/checkout@v4 - name: ${{ matrix.python-version }} windows ${{ matrix.architecture }} + - name: Checkout multibuild + uses: actions/checkout@v4 + with: + repository: multi-build/multibuild + path: multibuild + ref: 452dd2d1705f6b2375369a6570c415beb3163f70 + - uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install cibuildwheel + run: | + python3 -m pip install -r .ci/requirements-cibw.txt + + - name: Build wheels + run: | + python3 -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.cibw_arch }} + CIBW_BUILD: ${{ matrix.build }} + CIBW_FREE_THREADED_SUPPORT: True + CIBW_MANYLINUX_PYPY_X86_64_IMAGE: ${{ matrix.manylinux }} + CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux }} + CIBW_PRERELEASE_PYTHONS: True + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macosx_deployment_target }} + + - uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.os }}${{ matrix.macosx_deployment_target && format('-{0}', matrix.macosx_deployment_target) }}-${{ matrix.cibw_arch }}${{ matrix.manylinux && format('-{0}', matrix.manylinux) }} + path: ./wheelhouse/*.whl + + windows: + name: Windows ${{ matrix.cibw_arch }} + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + include: + - cibw_arch: x86 + - cibw_arch: AMD64 + - cibw_arch: ARM64 steps: - - name: Checkout pillow-avif-plugin - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - name: Checkout cached dependencies - uses: actions/checkout@v3 - with: - repository: python-pillow/pillow-depends - path: winbuild\depends + - uses: actions/setup-python@v5 + with: + python-version: "3.x" - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~\AppData\Local\pip\Cache - key: - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.architecture }}-${{ hashFiles('**/.github/workflows/test-windows.yml') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.architecture }}- - ${{ runner.os }}-${{ matrix.python-version }}- - - # sets env: pythonLocation - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} + - name: Install cibuildwheel + run: | + python.exe -m pip install -r .ci/requirements-cibw.txt - - name: python -m pip install wheel pytest pytest-cov mock - run: python -m pip install wheel pytest pytest-cov mock + - name: Prepare for build + run: | + choco install nasm --no-progress + echo "C:\Program Files\NASM" >> $env:GITHUB_PATH - - name: Install dependencies - id: install - run: | - 7z x winbuild\depends\nasm-2.16.01-win64.zip "-o$env:RUNNER_WORKSPACE\" - echo "$env:RUNNER_WORKSPACE\nasm-2.16.01" >> $env:GITHUB_PATH + python.exe -m pip install meson + + & python.exe winbuild\build_prepare.py -v --no-imagequant --architecture=${{ matrix.cibw_arch }} + shell: pwsh - python -m pip install meson + - name: Build wheels + run: | + setlocal EnableDelayedExpansion + for %%f in (winbuild\build\license\*) do ( + set x=%%~nf + echo. >> LICENSE + echo ===== %%~nf ===== >> LICENSE + echo. >> LICENSE + type %%f >> LICENSE + ) + call winbuild\\build\\build_env.cmd + %pythonLocation%\python.exe -m cibuildwheel . --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.cibw_arch }} + CIBW_BEFORE_ALL: "{package}\\winbuild\\build\\build_dep_all.cmd" + CIBW_CACHE_PATH: "C:\\cibw" + CIBW_FREE_THREADED_SUPPORT: True + CIBW_PRERELEASE_PYTHONS: True + CIBW_TEST_SKIP: "*-win_arm64" + CIBW_TEST_COMMAND: 'docker run --rm + -v {project}:C:\pillow + -v C:\cibw:C:\cibw + -v %CD%\..\venv-test:%CD%\..\venv-test + -e CI -e GITHUB_ACTIONS + mcr.microsoft.com/windows/servercore:ltsc2022 + powershell C:\pillow\.github\workflows\wheels-test.ps1 %CD%\..\venv-test' + shell: cmd + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: dist-windows-${{ matrix.cibw_arch }} + path: ./wheelhouse/*.whl - # make cache key depend on VS version - & "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" ` - | find """catalog_buildVersion""" ` - | ForEach-Object { $a = $_.split(" ")[1]; echo "vs=$a" >> $env:GITHUB_OUTPUT } - shell: pwsh + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 - - name: Cache build - id: build-cache - uses: actions/cache@v3 - with: - path: winbuild\build - key: - ${{ hashFiles('winbuild\build_prepare.py') }}-${{ hashFiles('.github\workflows\test-windows.yml') }}-${{ env.pythonLocation }}-${{ steps.install.outputs.vs }} - - - name: Prepare build - if: steps.build-cache.outputs.cache-hit != 'true' - run: | - & python.exe winbuild\build_prepare.py -v - shell: pwsh - - - name: Build dependencies / libjpeg-turbo - if: steps.build-cache.outputs.cache-hit != 'true' - run: "& winbuild\\build\\build_dep_libjpeg.cmd" - - - name: Build dependencies / zlib - if: steps.build-cache.outputs.cache-hit != 'true' - run: "& winbuild\\build\\build_dep_zlib.cmd" - - - name: Build dependencies / libpng - if: steps.build-cache.outputs.cache-hit != 'true' - run: "& winbuild\\build\\build_dep_libpng.cmd" - - - name: Build dependencies / meson (python 2.7) - if: steps.build-cache.outputs.cache-hit != 'true' && matrix.python-version == '2.7' - run: "& winbuild\\build\\install_meson.cmd" - - - name: Build dependencies / meson (python 3.x) - if: steps.build-cache.outputs.cache-hit != 'true' && matrix.python-version != '2.7' - run: python -m pip install meson - shell: cmd - - - name: Build dependencies / rav1e - if: steps.build-cache.outputs.cache-hit != 'true' - run: "& winbuild\\build\\build_dep_rav1e.cmd" - - - name: Build dependencies / libavif - if: steps.build-cache.outputs.cache-hit != 'true' - run: "& winbuild\\build\\build_dep_libavif.cmd" - - # trim ~150MB x 9 - - name: Optimize build cache - if: steps.build-cache.outputs.cache-hit != 'true' - run: rmdir /S /Q winbuild\build\src - shell: cmd - - - name: Install dependencies / Pillow - run: | - cmd /c "winbuild\build\build_env.cmd && $env:pythonLocation\python.exe -m pip install -v Pillow" - shell: pwsh - - - name: Build pillow-avif-plugin - run: | - cmd /c "winbuild\build\build_env.cmd && $env:pythonLocation\python.exe -m pip install -v ." - shell: pwsh - - # failing with PyPy3 - - name: Enable heap verification - if: "!contains(matrix.python-version, 'pypy')" - run: "& 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x86\\gflags.exe' /p /enable $env:pythonLocation\\python.exe" - - - name: Test pillow-avif-plugin - run: | - path %GITHUB_WORKSPACE%\\winbuild\\build\\bin;%PATH% - python.exe -m pytest -v -W always --cov pillow_avif --cov tests --cov-report term --cov-report xml tests - shell: cmd - - - name: Prepare to upload errors - if: failure() - run: | - mkdir -p tests/errors - shell: pwsh - - - name: Upload errors - uses: actions/upload-artifact@v3 - if: failure() + - name: Set up Python + uses: actions/setup-python@v5 with: - name: errors - path: tests/errors + python-version: "3.x" + cache: pip + cache-dependency-path: "Makefile" - - name: After success - run: | - coverage xml - shell: pwsh + - run: make sdist - - name: Upload coverage - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: GHA_Windows - name: ${{ runner.os }} Python ${{ matrix.python-version }} ${{ matrix.architecture }} - - - name: Build wheel - id: wheel - if: "github.event_name == 'push'" - run: | - cmd /c "winbuild\build\build_env.cmd && $env:pythonLocation\python.exe -m pip wheel -v ." - shell: pwsh - - - uses: actions/upload-artifact@v3 - if: "github.event_name == 'push'" + - uses: actions/upload-artifact@v4 with: - name: wheels - path: "*.whl" + name: dist-sdist + path: dist/*.tar.gz success: - needs: [build, windows] + needs: [legacy, windows, build-1-QEMU-emulated-wheels, build-2-native-wheels, sdist] runs-on: ubuntu-latest name: Build Successful steps: @@ -317,8 +361,8 @@ jobs: release: name: Create Release runs-on: ubuntu-latest - if: "startsWith(github.ref, 'refs/tags/')" - needs: [build, windows] + if: startsWith(github.ref, 'refs/tags/') + needs: [success] steps: - uses: actions/download-artifact@v3 with: diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..926a6d5 --- /dev/null +++ b/Makefile @@ -0,0 +1,61 @@ +.DEFAULT_GOAL := help + +.PHONY: clean +clean: + rm src/pillow_avif/*.so || true + rm -r build || true + find . -name __pycache__ | xargs rm -r || true + +.PHONY: coverage +coverage: + python3 -c "import pytest" > /dev/null 2>&1 || python3 -m pip install pytest + python3 -m pytest -qq + rm -r htmlcov || true + python3 -c "import coverage" > /dev/null 2>&1 || python3 -m pip install coverage + python3 -m coverage report + +.PHONY: install +install: + python3 -m pip -v install . + +.PHONY: install-coverage +install-coverage: + CFLAGS="-coverage -Werror=implicit-function-declaration" python3 -m pip -v install . + +.PHONY: debug +debug: +# make a debug version if we don't have a -dbg python. Leaves in symbols +# for our stuff, kills optimization, and redirects to dev null so we +# see any build failures. + make clean > /dev/null + CFLAGS='-g -O0' python3 -m pip -v install . > /dev/null + +.PHONY: release-test +release-test: + python3 Tests/check_release_notes.py + python3 -m pip install -e .[tests] + python3 -m pytest tests + python3 -m pip install . + python3 -m pytest -qq + +.PHONY: sdist +sdist: + python3 -m build --help > /dev/null 2>&1 || python3 -m pip install build + python3 -m build --sdist + python3 -m twine --help > /dev/null 2>&1 || python3 -m pip install twine + python3 -m twine check --strict dist/* + +.PHONY: test +test: + python3 -c "import pytest" > /dev/null 2>&1 || python3 -m pip install pytest + python3 -m pytest -qq + +.PHONY: lint +lint: + python3 -c "import tox" > /dev/null 2>&1 || python3 -m pip install tox + python3 -m tox -e lint + +.PHONY: lint-fix +lint-fix: + python3 -c "import black" > /dev/null 2>&1 || python3 -m pip install black + python3 -m black . diff --git a/pyproject.toml b/pyproject.toml index fed528d..badf462 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,9 @@ [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" + +[tool.cibuildwheel] +before-all = ".github/workflows/wheels-dependencies.sh" +build-verbosity = 1 +test-command = "cd {project} && .github/workflows/wheels-test.sh" +test-extras = "tests" diff --git a/setup.py b/setup.py index 098fab6..0237695 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,16 @@ def readme(): url="https://github.com/fdintino/pillow-avif-plugin/", download_url="https://github.com/fdintino/pillow-avif-plugin/releases", install_requires=[], + extras_require={ + "test": [ + "gcovr", + "pytest", + "packaging", + "pytest-cov", + "test-image-results", + "pillow", + ] + }, classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", diff --git a/wheelbuild/config.sh b/wheelbuild/config.sh index b54b4fe..3323ef6 100644 --- a/wheelbuild/config.sh +++ b/wheelbuild/config.sh @@ -4,7 +4,7 @@ set -eo pipefail CONFIG_DIR=$(abspath $(dirname "${BASH_SOURCE[0]}")) ARCHIVE_SDIR=pillow-avif-plugin-depends -LIBAVIF_VERSION=1a1c778f8e0b7ecdf3af9e59a6f33eb4d7d3900e +LIBAVIF_VERSION=1.1.1 RAV1E_VERSION=0.7.1 CCACHE_VERSION=4.7.1 SCCACHE_VERSION=0.3.0 @@ -277,7 +277,7 @@ EOF group_start "Download libavif source" fetch_unpack \ - "https://github.com/AOMediaCodec/libavif/archive/$LIBAVIF_VERSION.tar.gz" \ + "https://github.com/AOMediaCodec/libavif/archive/refs/tags/v$LIBAVIF_VERSION.tar.gz" \ "libavif-$LIBAVIF_VERSION.tar.gz" group_end @@ -451,13 +451,20 @@ function pre_build { install_ninja install_meson + if [[ -n "$IS_MACOS" ]]; then + # clear bash path cache for curl + hash -d curl + fi + if [ -e $HOME/.cargo/env ]; then source $HOME/.cargo/env fi build_libavif - echo "::group::Build wheel" + if [ -z "$CIBW_ARCHS" ]; then + echo "::group::Build wheel" + fi } function run_tests { diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 45c958d..bef7b6e 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -7,6 +7,7 @@ import shutil import struct import subprocess +from typing import Any def cmd_cd(path: str) -> str: @@ -38,30 +39,25 @@ def cmd_rmdir(path: str) -> str: return f'rmdir /S /Q "{path}"' -def cmd_lib_combine(outfile: str, *libfiles) -> str: - params = " ".join(['"%s"' % f for f in libfiles]) - return "LIB.EXE /OUT:{outfile} {params}".format(outfile=outfile, params=params) - - def cmd_nmake( makefile: str | None = None, target: str = "", params: list[str] | None = None, ) -> str: - params = "" if params is None else " ".join(params) - return " ".join( [ "{nmake}", "-nologo", f'-f "{makefile}"' if makefile is not None else "", - f"{params}", + f'{" ".join(params)}' if params is not None else "", f'"{target}"', ] ) -def cmds_cmake(target: str | tuple[str, ...] | list[str], *params) -> list[str]: +def cmds_cmake( + target: str | tuple[str, ...] | list[str], *params: str, build_dir: str = "." +) -> list[str]: if not isinstance(target, str): target = " ".join(target) @@ -78,10 +74,11 @@ def cmds_cmake(target: str | tuple[str, ...] | list[str], *params) -> list[str]: "-DCMAKE_CXX_FLAGS=-nologo", *params, '-G "{cmake_generator}"', - ".", + f'-B "{build_dir}"', + "-S .", ] ), - f"{{cmake}} --build . --clean-first --parallel --target {target}", + f'{{cmake}} --build "{build_dir}" --clean-first --parallel --target {target}', ] @@ -89,7 +86,7 @@ def cmd_msbuild( file: str, configuration: str = "Release", target: str = "Build", - platform: str = "{msbuild_arch}", + plat: str = "{msbuild_arch}", ) -> str: return " ".join( [ @@ -97,7 +94,7 @@ def cmd_msbuild( f"{file}", f'/t:"{target}"', f'/p:Configuration="{configuration}"', - f"/p:Platform={platform}", + f"/p:Platform={plat}", "/m", ] ) @@ -107,69 +104,25 @@ def cmd_msbuild( ARCHITECTURES = { "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, - "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, + "AMD64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, "ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"}, } +V = { + "MESON": "1.5.1", + "LIBAVIF": "1.1.1", + "RAV1E": "0.7.1", +} + + # dependencies, listed in order of compilation -DEPS = { - "libjpeg": { - "url": SF_PROJECTS - + "/libjpeg-turbo/files/3.0.0/libjpeg-turbo-3.0.0.tar.gz/download", - "filename": "libjpeg-turbo-3.0.0.tar.gz", - "dir": "libjpeg-turbo-3.0.0", - "license": ["README.ijg", "LICENSE.md"], - "license_pattern": ( - "(LEGAL ISSUES\n============\n\n.+?)\n\nREFERENCES\n==========" - ".+(libjpeg-turbo Licenses\n======================\n\n.+)$" - ), - "build": [ - *cmds_cmake( - ("jpeg-static", "cjpeg-static", "djpeg-static"), - "-DENABLE_SHARED:BOOL=FALSE", - "-DWITH_JPEG8:BOOL=TRUE", - "-DWITH_CRT_DLL:BOOL=TRUE", - ), - cmd_copy("jpeg-static.lib", "libjpeg.lib"), - cmd_copy("cjpeg-static.exe", "cjpeg.exe"), - cmd_copy("djpeg-static.exe", "djpeg.exe"), - ], - "headers": ["j*.h"], - "libs": ["libjpeg.lib"], - "bins": ["cjpeg.exe", "djpeg.exe"], - }, - "zlib": { - "url": "https://zlib.net/zlib13.zip", - "filename": "zlib13.zip", - "dir": "zlib-1.3", - "license": "README", - "license_pattern": "Copyright notice:\n\n(.+)$", - "build": [ - cmd_nmake(r"win32\Makefile.msc", "clean"), - cmd_nmake(r"win32\Makefile.msc", "zlib.lib"), - cmd_copy("zlib.lib", "z.lib"), - ], - "headers": [r"z*.h"], - "libs": [r"*.lib"], - }, - "libpng": { - "url": SF_PROJECTS + "/libpng/files/libpng16/1.6.39/lpng1639.zip/download", - "filename": "lpng1639.zip", - "dir": "lpng1639", - "license": "LICENSE", - "build": [ - *cmds_cmake("png_static", "-DPNG_SHARED:BOOL=OFF", "-DPNG_TESTS:BOOL=OFF"), - cmd_copy("libpng16_static.lib", "libpng16.lib"), - ], - "headers": [r"png*.h"], - "libs": [r"libpng16.lib"], - }, +DEPS: dict[str, dict[str, Any]] = { "rav1e": { "url": ( - "https://github.com/xiph/rav1e/releases/download/v0.7.1/" - "rav1e-0.7.1-windows-msvc-generic.zip" + f"https://github.com/xiph/rav1e/releases/download/v{V['RAV1E']}/" + f"rav1e-{V['RAV1E']}-windows-msvc-generic.zip" ), - "filename": "rav1e-0.7.1-windows-msvc-generic.zip", + "filename": f"rav1e-{V['RAV1E']}-windows-msvc-generic.zip", "dir": "rav1e-windows-msvc-sdk", "license": "LICENSE", "build": [ @@ -179,12 +132,9 @@ def cmd_msbuild( "libs": [r"lib\*.*"], }, "libavif": { - "url": ( - "https://github.com/AOMediaCodec/libavif/archive/" - "1a1c778f8e0b7ecdf3af9e59a6f33eb4d7d3900e.zip" - ), - "filename": "libavif-1a1c778f8e0b7ecdf3af9e59a6f33eb4d7d3900e.zip", - "dir": "libavif-1a1c778f8e0b7ecdf3af9e59a6f33eb4d7d3900e", + "url": f"https://github.com/AOMediaCodec/libavif/archive/v{V['LIBAVIF']}.zip", + "filename": f"libavif-{V['LIBAVIF']}.zip", + "dir": f"libavif-{V['LIBAVIF']}", "license": "LICENSE", "build": [ cmd_mkdir("build.pillow"), @@ -194,23 +144,20 @@ def cmd_msbuild( "{cmake}", "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_VERBOSE_MAKEFILE=ON", - "-DCMAKE_RULE_MESSAGES:BOOL=OFF", # for NMake - "-DCMAKE_C_COMPILER=cl.exe", # for Ninja - "-DCMAKE_CXX_COMPILER=cl.exe", # for Ninja + "-DCMAKE_RULE_MESSAGES:BOOL=OFF", + "-DCMAKE_C_COMPILER=cl.exe", + "-DCMAKE_CXX_COMPILER=cl.exe", "-DCMAKE_C_FLAGS=-nologo", "-DCMAKE_CXX_FLAGS=-nologo", "-DBUILD_SHARED_LIBS=OFF", - "-DAVIF_CODEC_AOM=ON", - "-DAVIF_LOCAL_AOM=ON", - "-DAVIF_LOCAL_LIBYUV=ON", - "-DAVIF_LOCAL_LIBSHARPYUV=ON", - "-DAVIF_CODEC_RAV1E=ON", + "-DAVIF_CODEC_AOM=LOCAL", + "-DAVIF_LIBYUV=LOCAL", + "-DAVIF_LIBSHARPYUV=LOCAL", + "-DAVIF_CODEC_RAV1E=SYSTEM", "-DAVIF_RAV1E_ROOT={build_dir}", "-DCMAKE_MODULE_PATH={winbuild_dir_cmake}", - "-DAVIF_CODEC_DAV1D=ON", - "-DAVIF_LOCAL_DAV1D=ON", - "-DAVIF_CODEC_SVT=ON", - "-DAVIF_LOCAL_SVT=ON", + "-DAVIF_CODEC_DAV1D=LOCAL", + "-DAVIF_CODEC_SVT=LOCAL", '-G "Ninja"', "..", ] @@ -225,12 +172,16 @@ def cmd_msbuild( # based on distutils._msvccompiler from CPython 3.7.4 -def find_msvs() -> dict[str, str] | None: +def find_msvs(architecture: str) -> dict[str, str] | None: root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") if not root: print("Program Files not found") return None + requires = ["-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64"] + if architecture == "ARM64": + requires += ["-requires", "Microsoft.VisualStudio.Component.VC.Tools.ARM64"] + try: vspath = ( subprocess.check_output( @@ -240,8 +191,7 @@ def find_msvs() -> dict[str, str] | None: ), "-latest", "-prerelease", - "-requires", - "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + *requires, "-property", "installationPath", "-products", @@ -282,6 +232,7 @@ def find_msvs() -> dict[str, str] | None: def download_dep(url: str, file: str) -> None: + import urllib.error import urllib.request ex = None @@ -298,23 +249,16 @@ def download_dep(url: str, file: str) -> None: raise RuntimeError(ex) -def extract_dep(url: str, filename: str) -> None: +def extract_dep(url: str, filename: str, prefs: dict[str, str]) -> None: import tarfile import zipfile - file = os.path.join(args.depends_dir, filename) + depends_dir = prefs["depends_dir"] + sources_dir = prefs["src_dir"] + + file = os.path.join(depends_dir, filename) if not os.path.exists(file): - # First try our mirror - mirror_url = ( - f"https://raw.githubusercontent.com/" - f"python-pillow/pillow-depends/main/{filename}" - ) - try: - download_dep(mirror_url, file) - except RuntimeError as exc: - # Otherwise try upstream - print(exc) - download_dep(url, file) + download_dep(url, file) print("Extracting " + filename) sources_dir_abs = os.path.abspath(sources_dir) @@ -327,7 +271,7 @@ def extract_dep(url: str, filename: str) -> None: msg = "Attempted Path Traversal in Zip File" raise RuntimeError(msg) zf.extractall(sources_dir) - elif filename.endswith(".tar.gz") or filename.endswith(".tgz"): + elif filename.endswith((".tar.gz", ".tgz")): with tarfile.open(file, "r:gz") as tgz: for member in tgz.getnames(): member_abspath = os.path.abspath(os.path.join(sources_dir, member)) @@ -341,18 +285,20 @@ def extract_dep(url: str, filename: str) -> None: raise RuntimeError(msg) -def write_script(name: str, lines: list[str]) -> None: - name = os.path.join(args.build_dir, name) +def write_script( + name: str, lines: list[str], prefs: dict[str, str], verbose: bool +) -> None: + name = os.path.join(prefs["build_dir"], name) lines = [line.format(**prefs) for line in lines] print("Writing " + name) with open(name, "w", newline="") as f: f.write(os.linesep.join(lines)) - if args.verbose: + if verbose: for line in lines: print(" " + line) -def get_footer(dep: dict) -> list[str]: +def get_footer(dep: dict[str, Any]) -> list[str]: lines = [] for out in dep.get("headers", []): lines.append(cmd_copy(out, "{inc_dir}")) @@ -363,7 +309,7 @@ def get_footer(dep: dict) -> list[str]: return lines -def build_env() -> None: +def build_env(prefs: dict[str, str], verbose: bool) -> None: lines = [ "if defined DISTUTILS_USE_SDK goto end", cmd_set("INCLUDE", "{inc_dir}"), @@ -376,34 +322,36 @@ def build_env() -> None: ":end", "@echo on", ] - write_script("build_env.cmd", lines) + write_script("build_env.cmd", lines, prefs, verbose) -def build_dep(name: str) -> str: +def build_dep(name: str, prefs: dict[str, str], verbose: bool) -> str: dep = DEPS[name] - dir = dep["dir"] + directory = dep["dir"] file = f"build_dep_{name}.cmd" + license_dir = prefs["license_dir"] + sources_dir = prefs["src_dir"] - extract_dep(dep["url"], dep["filename"]) + extract_dep(dep["url"], dep["filename"], prefs) licenses = dep["license"] if isinstance(licenses, str): licenses = [licenses] license_text = "" for license_file in licenses: - with open(os.path.join(sources_dir, dir, license_file)) as f: + with open(os.path.join(sources_dir, directory, license_file)) as f: license_text += f.read() if "license_pattern" in dep: match = re.search(dep["license_pattern"], license_text, re.DOTALL) + assert match is not None license_text = "\n".join(match.groups()) - if licenses: - assert len(license_text) > 50 - with open(os.path.join(license_dir, f"{dir}.txt"), "w") as f: - print(f"Writing license {dir}.txt") - f.write(license_text) + assert len(license_text) > 50 + with open(os.path.join(license_dir, f"{directory}.txt"), "w") as f: + print(f"Writing license {directory}.txt") + f.write(license_text) for patch_file, patch_list in dep.get("patch", {}).items(): - patch_file = os.path.join(sources_dir, dir, patch_file.format(**prefs)) + patch_file = os.path.join(sources_dir, directory, patch_file.format(**prefs)) with open(patch_file) as f: text = f.read() for patch_from, patch_to in patch_list.items(): @@ -415,60 +363,50 @@ def build_dep(name: str) -> str: print(f"Patching {patch_file}") f.write(text) - banner = f"Building {name} ({dir})" + banner = f"Building {name} ({directory})" lines = [ r'call "{build_dir}\build_env.cmd"', "@echo " + ("=" * 70), f"@echo ==== {banner:<60} ====", "@echo " + ("=" * 70), - cmd_cd(os.path.join(sources_dir, dir)), + cmd_cd(os.path.join(sources_dir, directory)), *dep.get("build", []), *get_footer(dep), ] - write_script(file, lines) + write_script(file, lines, prefs, verbose) return file -def build_dep_all() -> None: +def build_dep_all(disabled: list[str], prefs: dict[str, str], verbose: bool) -> None: lines = [r'call "{build_dir}\build_env.cmd"'] + gha_groups = "GITHUB_ACTIONS" in os.environ + scripts = ["install_meson.cmd"] for dep_name in DEPS: print() if dep_name in disabled: print(f"Skipping disabled dependency {dep_name}") continue - script = build_dep(dep_name) + scripts.append(build_dep(dep_name, prefs, verbose)) + + for script in scripts: + if gha_groups: + lines.append(f"@echo ::group::Running {script}") lines.append(rf'cmd.exe /c "{{build_dir}}\{script}"') lines.append("if errorlevel 1 echo Build failed! && exit /B 1") + if gha_groups: + lines.append("@echo ::endgroup::") print() lines.append("@echo All Pillow dependencies built successfully!") - write_script("build_dep_all.cmd", lines) - - -def install_meson(): - msi_url = "https://github.com/mesonbuild/meson/releases/download/0.56.2/meson-0.56.2-64.msi" # noqa: E501 - msi_file = os.path.join(args.depends_dir, "meson-0.56.2-64.msi") - download_dep(msi_url, msi_file) - - lines = [ - "@echo on", - "@echo ---- Installing meson ----", - "msiexec /q /i %s" % msi_file, - "@echo meson installed successfully", - ] - write_script("install_meson.cmd", lines) + write_script("build_dep_all.cmd", lines, prefs, verbose) -if __name__ == "__main__": +def main() -> None: winbuild_dir = os.path.dirname(os.path.realpath(__file__)) - pillow_dir = os.path.realpath(os.path.join(winbuild_dir, "..")) parser = argparse.ArgumentParser( prog="winbuild\\build_prepare.py", - description="Download and generate build scripts for Pillow dependencies.", - epilog="""Arguments can also be supplied using the environment variables - PILLOW_BUILD, PILLOW_DEPS, ARCHITECTURE. See winbuild\\build.rst - for more information.""", + description="Download and generate build scripts for pillow-avif-plugin dependencies.", ) parser.add_argument( "-v", "--verbose", action="store_true", help="print generated scripts" @@ -478,15 +416,19 @@ def install_meson(): "--dir", "--build-dir", dest="build_dir", - metavar="PILLOW_BUILD", - default=os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")), + metavar="PILLOW_AVIF_PLUGIN_BUILD", + default=os.environ.get( + "PILLOW_AVIF_PLUGIN_BUILD", os.path.join(winbuild_dir, "build") + ), help="build directory (default: 'winbuild\\build')", ) parser.add_argument( "--depends", dest="depends_dir", - metavar="PILLOW_DEPS", - default=os.environ.get("PILLOW_DEPS", os.path.join(winbuild_dir, "depends")), + metavar="PILLOW_AVIF_PLUGIN_DEPS", + default=os.environ.get( + "PILLOW_AVIF_PLUGIN_DEPS", os.path.join(winbuild_dir, "depends") + ), help="directory used to store cached dependencies " "(default: 'winbuild\\depends')", ) @@ -498,7 +440,7 @@ def install_meson(): ( "ARM64" if platform.machine() == "ARM64" - else ("x86" if struct.calcsize("P") == 4 else "x64") + else ("x86" if struct.calcsize("P") == 4 else "AMD64") ), ), help="build architecture (default: same as host Python)", @@ -511,12 +453,13 @@ def install_meson(): default="Ninja", help="build dependencies using NMake instead of Ninja", ) + args = parser.parse_args() arch_prefs = ARCHITECTURES[args.architecture] print("Target architecture:", args.architecture) - msvs = find_msvs() + msvs = find_msvs(args.architecture) if msvs is None: msg = "Visual Studio not found. Please install Visual Studio 2017 or newer." raise RuntimeError(msg) @@ -552,16 +495,16 @@ def install_meson(): "architecture": args.architecture, **arch_prefs, # Pillow paths - "pillow_dir": pillow_dir, "winbuild_dir": winbuild_dir, "winbuild_dir_cmake": winbuild_dir.replace("\\", "/"), # Build paths + "bin_dir": bin_dir, "build_dir": args.build_dir, + "depends_dir": args.depends_dir, "inc_dir": inc_dir, "lib_dir": lib_dir, - "bin_dir": bin_dir, - "src_dir": sources_dir, "license_dir": license_dir, + "src_dir": sources_dir, # Compilers / Tools **msvs, "cmake": "cmake.exe", # TODO find CMAKE automatically @@ -574,7 +517,22 @@ def install_meson(): print() - write_script(".gitignore", ["*"]) - build_env() - install_meson() - build_dep_all() + write_script(".gitignore", ["*"], prefs, args.verbose) + write_script( + "install_meson.cmd", + [ + r'call "{build_dir}\build_env.cmd"', + "@echo " + ("=" * 70), + f"@echo ==== {'Building meson':<60} ====", + "@echo " + ("=" * 70), + f"python -mpip install meson=={V['MESON']}", + ], + prefs, + args.verbose, + ) + build_env(prefs, args.verbose) + build_dep_all(disabled, prefs, args.verbose) + + +if __name__ == "__main__": + main()