Merge branch 'main' into libavif-plugin

.github/workflows/test.yml

@@ -116,7 +116,7 @@ jobs:
         GHA_PYTHON_VERSION: ${{ matrix.python-version }}
 
     - name: Register gcc problem matcher
-      if: "matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'"
+      if: "matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'"
       run: echo "::add-matcher::.github/problem-matchers/gcc.json"
 
     - name: Build

.github/workflows/wheels-dependencies.sh

@@ -1,11 +1,33 @@
 #!/bin/bash
-# Define custom utilities
-# Test for macOS with [ -n "$IS_MACOS" ]
-if [ -z "$IS_MACOS" ]; then
-    export MB_ML_LIBC=${AUDITWHEEL_POLICY::9}
-    export MB_ML_VER=${AUDITWHEEL_POLICY:9}
+
+# Setup that needs to be done before multibuild utils are invoked
+PROJECTDIR=$(pwd)
+if [[ "$(uname -s)" == "Darwin" ]]; then
+    # Safety check - macOS builds require that CIBW_ARCHS is set, and that it
+    # only contains a single value (even though cibuildwheel allows multiple
+    # values in CIBW_ARCHS).
+    if [[ -z "$CIBW_ARCHS" ]]; then
+        echo "ERROR: Pillow macOS builds require CIBW_ARCHS be defined."
+        exit 1
+    fi
+    if [[ "$CIBW_ARCHS" == *" "* ]]; then
+        echo "ERROR: Pillow macOS builds only support a single architecture in CIBW_ARCHS."
+        exit 1
+    fi
+
+    # Build macOS dependencies in `build/darwin`
+    # Install them into `build/deps/darwin`
+    WORKDIR=$(pwd)/build/darwin
+    BUILD_PREFIX=$(pwd)/build/deps/darwin
+else
+    # Build prefix will default to /usr/local
+    WORKDIR=$(pwd)/build
+    MB_ML_LIBC=${AUDITWHEEL_POLICY::9}
+    MB_ML_VER=${AUDITWHEEL_POLICY:9}
 fi
-export PLAT="${CIBW_ARCHS:-$AUDITWHEEL_ARCH}"
+PLAT="${CIBW_ARCHS:-$AUDITWHEEL_ARCH}"
+
+# Define custom utilities
 source wheels/multibuild/common_utils.sh
 source wheels/multibuild/library_builders.sh
 if [ -z "$IS_MACOS" ]; then
@@ -40,29 +62,40 @@ BROTLI_VERSION=1.1.0
 LIBAVIF_VERSION=1.1.1
 RAV1E_VERSION=0.7.1
 
+function build_pkg_config {
+    if [ -e pkg-config-stamp ]; then return; fi
+    # This essentially duplicates the Homebrew recipe
+    ORIGINAL_CFLAGS=$CFLAGS
+    CFLAGS="$CFLAGS -Wno-int-conversion"
+    build_simple pkg-config 0.29.2 https://pkg-config.freedesktop.org/releases tar.gz \
+        --disable-debug --disable-host-tool --with-internal-glib \
+        --with-pc-path=$BUILD_PREFIX/share/pkgconfig:$BUILD_PREFIX/lib/pkgconfig \
+        --with-system-include-path=$(xcrun --show-sdk-path --sdk macosx)/usr/include
+    CFLAGS=$ORIGINAL_CFLAGS
+    export PKG_CONFIG=$BUILD_PREFIX/bin/pkg-config
+    touch pkg-config-stamp
+}
+
 function build_brotli {
+    if [ -e brotli-stamp ]; then return; fi
     local cmake=$(get_modern_cmake)
     local out_dir=$(fetch_unpack https://github.com/google/brotli/archive/v$BROTLI_VERSION.tar.gz brotli-$BROTLI_VERSION.tar.gz)
     (cd $out_dir \
-        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \
+        && $cmake -DCMAKE_INSTALL_PREFIX=$BUILD_PREFIX -DCMAKE_INSTALL_LIBDIR=$BUILD_PREFIX/lib -DCMAKE_INSTALL_NAME_DIR=$BUILD_PREFIX/lib . \
         && make install)
-    if [[ "$MB_ML_LIBC" == "manylinux" ]]; then
-        cp /usr/local/lib64/libbrotli* /usr/local/lib
-        cp /usr/local/lib64/pkgconfig/libbrotli* /usr/local/lib/pkgconfig
-    fi
+    touch brotli-stamp
 }
 
 function build_harfbuzz {
+    if [ -e harfbuzz-stamp ]; then return; fi
     python3 -m pip install meson ninja
 
     local out_dir=$(fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/$HARFBUZZ_VERSION/$HARFBUZZ_VERSION.tar.xz harfbuzz-$HARFBUZZ_VERSION.tar.xz)
     (cd $out_dir \
-        && meson setup build --buildtype=release -Dfreetype=enabled -Dglib=disabled)
+        && meson setup build --prefix=$BUILD_PREFIX --libdir=$BUILD_PREFIX/lib --buildtype=release -Dfreetype=enabled -Dglib=disabled)
     (cd $out_dir/build \
         && meson install)
-    if [[ "$MB_ML_LIBC" == "manylinux" ]]; then
-        cp /usr/local/lib64/libharfbuzz* /usr/local/lib
-    fi
+    touch harfbuzz-stamp
 }
 
 function install_rav1e {
@@ -135,9 +168,6 @@ function build_libavif {
 }
 
 function build {
-    if [[ -n "$IS_MACOS" ]] && [[ "$CIBW_ARCHS" == "arm64" ]]; then
-        sudo chown -R runner /usr/local
-    fi
     build_xz
     if [ -z "$IS_ALPINE" ] && [ -z "$IS_MACOS" ]; then
         yum remove -y zlib-devel
@@ -156,16 +186,24 @@ function build {
         build_simple xorgproto 2024.1 https://www.x.org/pub/individual/proto
         build_simple libXau 1.0.11 https://www.x.org/pub/individual/lib
         build_simple libpthread-stubs 0.5 https://xcb.freedesktop.org/dist
-        if [[ "$CIBW_ARCHS" == "arm64" ]]; then
-            cp /usr/local/share/pkgconfig/xcb-proto.pc /usr/local/lib/pkgconfig
-        fi
     else
-        sed s/\${pc_sysrootdir\}// /usr/local/share/pkgconfig/xcb-proto.pc > /usr/local/lib/pkgconfig/xcb-proto.pc
+        sed s/\${pc_sysrootdir\}// $BUILD_PREFIX/share/pkgconfig/xcb-proto.pc > $BUILD_PREFIX/lib/pkgconfig/xcb-proto.pc
     fi
     build_simple libxcb $LIBXCB_VERSION https://www.x.org/releases/individual/lib
 
     build_libjpeg_turbo
-    build_tiff
+    if [ -n "$IS_MACOS" ]; then
+        # Custom tiff build to include jpeg; by default, configure won't include
+        # headers/libs in the custom macOS prefix. Explicitly disable webp,
+        # libdeflate and zstd, because on x86_64 macs, it will pick up the
+        # Homebrew versions of those libraries from /usr/local.
+        build_simple tiff $TIFF_VERSION https://download.osgeo.org/libtiff tar.gz \
+            --with-jpeg-include-dir=$BUILD_PREFIX/include --with-jpeg-lib-dir=$BUILD_PREFIX/lib \
+            --disable-webp --disable-libdeflate --disable-zstd
+    else
+        build_tiff
+    fi
+
     build_libpng
     build_lcms2
     build_openjpeg
@@ -190,36 +228,47 @@ function build {
     build_harfbuzz
 }
 
+# Perform all dependency builds in the build subfolder.
+mkdir -p $WORKDIR
+pushd $WORKDIR > /dev/null
+
 # Any stuff that you need to do before you start building the wheels
 # Runs in the root directory of this repository.
-curl -fsSL -o pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip
-untar pillow-depends-main.zip
+if [[ ! -d $WORKDIR/pillow-depends-main ]]; then
+  if [[ ! -f $PROJECTDIR/pillow-depends-main.zip ]]; then
+    echo "Download pillow dependency sources..."
+    curl -fSL -o $PROJECTDIR/pillow-depends-main.zip https://github.com/python-pillow/pillow-depends/archive/main.zip
+  fi
+  echo "Unpacking pillow dependency sources..."
+  untar $PROJECTDIR/pillow-depends-main.zip
+fi
 
 if [[ -n "$IS_MACOS" ]]; then
-  # libdeflate may cause a minimum target error when repairing the wheel
-  # libtiff and libxcb cause a conflict with building libtiff and libxcb
-  # libxau and libxdmcp cause an issue on macOS < 11
-  # remove cairo to fix building harfbuzz on arm64
-  # remove lcms2 and libpng to fix building openjpeg on arm64
-  # remove jpeg-turbo to avoid inclusion on arm64
-  # remove webp and zstd to avoid inclusion on x86_64
-  # remove aom and libavif to fix building on arm64
-  # curl from brew requires zstd, use system curl
-  brew remove --ignore-dependencies libpng libtiff libxcb libxau libxdmcp curl cairo lcms2 zstd
-  if [[ "$CIBW_ARCHS" == "arm64" ]]; then
-    brew remove --ignore-dependencies jpeg-turbo
-  else
-    brew remove --ignore-dependencies libdeflate webp aom libavif
-  fi
+    # Homebrew (or similar packaging environments) install can contain some of
+    # the libraries that we're going to build. However, they may be compiled
+    # with a MACOSX_DEPLOYMENT_TARGET that doesn't match what we want to use,
+    # and they may bring in other dependencies that we don't want. The same will
+    # be true of any other locations on the path. To avoid conflicts, strip the
+    # path down to the bare minimum (which, on macOS, won't include any
+    # development dependencies).
+    export PATH="$BUILD_PREFIX/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin"
+    export CMAKE_PREFIX_PATH=$BUILD_PREFIX
 
-  brew install pkg-config
+    # Ensure the basic structure of the build prefix directory exists.
+    mkdir -p "$BUILD_PREFIX/bin"
+    mkdir -p "$BUILD_PREFIX/lib"
 
-  # clear bash path cache for curl
-  hash -d curl
+    # Ensure pkg-config is available
+    build_pkg_config
+    # Ensure cmake is available
+    python3 -m pip install cmake
 fi
 
 wrap_wheel_builder build
 
+# Return to the project root to finish the build
+popd > /dev/null
+
 # Append licenses
 for filename in wheels/dependency_licenses/*; do
   echo -e "\n\n----\n\n$(basename $filename | cut -f 1 -d '.')\n" | cat >> LICENSE

.github/workflows/wheels-test.sh

@@ -1,12 +1,24 @@
 #!/bin/bash
 set -e
 
+# Ensure fribidi is installed by the system.
 if [[ "$OSTYPE" == "darwin"* ]]; then
-    brew install fribidi
-    export PKG_CONFIG_PATH="/usr/local/opt/openblas/lib/pkgconfig"
-    if [ -f /opt/homebrew/lib/libfribidi.dylib ]; then
-        sudo cp /opt/homebrew/lib/libfribidi.dylib /usr/local/lib
+    # If Homebrew is on the path during the build, it may leak into the wheels.
+    # However, we *do* need Homebrew to provide a copy of fribidi for
+    # testing purposes so that we can verify the fribidi shim works as expected.
+    if [[ "$(uname -m)" == "x86_64" ]]; then
+        HOMEBREW_PREFIX=/usr/local
+    else
+        HOMEBREW_PREFIX=/opt/homebrew
     fi
+    $HOMEBREW_PREFIX/bin/brew install fribidi
+
+    # Add the lib folder for fribidi so that the vendored library can be found.
+    # Don't use $HOMEWBREW_PREFIX/lib directly - use the lib folder where the
+    # installed copy of fribidi is cellared. This ensures we don't pick up the
+    # Homebrew version of any other library that we're dependent on (most notably,
+    # freetype).
+    export DYLD_LIBRARY_PATH=$(dirname $(realpath $HOMEBREW_PREFIX/lib/libfribidi.dylib))
 elif [ "${AUDITWHEEL_POLICY::9}" == "musllinux" ]; then
     apk add curl fribidi
 else

.gitignore

@@ -19,6 +19,7 @@ lib64/
 parts/
 sdist/
 var/
+wheelhouse/
 *.egg-info/
 .installed.cfg
 *.egg
@@ -90,5 +91,9 @@ Tests/images/msp
 Tests/images/picins
 Tests/images/sunraster
 
+# Test and dependency downloads
+pillow-depends-main.zip
+pillow-test-images.zip
+
 # pyinstaller
 *.spec

CHANGES.rst

@@ -5,6 +5,12 @@ Changelog (Pillow)
 11.1.0 (unreleased)
 -------------------
 
+- Removed use of os.path.realpath #8545
+  [radarhere]
+
+- Allow linking to zlib import library on Windows #8519
+  [cubanpit, nulano]
+
 - Detach PyQt6 QPixmap instance before returning #8509
   [radarhere]
 

pyproject.toml

@@ -94,10 +94,17 @@ version = { attr = "PIL.__version__" }
 [tool.cibuildwheel]
 before-all = ".github/workflows/wheels-dependencies.sh"
 build-verbosity = 1
+
 config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable"
+# Disable platform guessing on macOS
+macos.config-settings = "raqm=enable raqm=vendor fribidi=vendor imagequant=disable platform-guessing=disable"
+
 test-command = "cd {project} && .github/workflows/wheels-test.sh"
 test-extras = "tests"
 
+[tool.cibuildwheel.macos.environment]
+PATH = "$(pwd)/build/deps/darwin/bin:$(dirname $(which python3)):/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin"
+
 [tool.black]
 exclude = "wheels/multibuild"
 

setup.py

@@ -449,7 +449,7 @@ def _remove_extension(self, name: str) -> None:
     def get_macos_sdk_path(self) -> str | None:
         try:
             sdk_path = (
-                subprocess.check_output(["xcrun", "--show-sdk-path"])
+                subprocess.check_output(["xcrun", "--show-sdk-path", "--sdk", "macosx"])
                 .strip()
                 .decode("latin1")
             )
@@ -607,6 +607,7 @@ def build_extensions(self) -> None:
                 _add_directory(library_dirs, "/usr/X11/lib")
                 _add_directory(include_dirs, "/usr/X11/include")
 
+            # Add the macOS SDK path.
             sdk_path = self.get_macos_sdk_path()
             if sdk_path:
                 _add_directory(library_dirs, os.path.join(sdk_path, "usr", "lib"))
@@ -691,6 +692,8 @@ def build_extensions(self) -> None:
                     feature.set("zlib", "z")
                 elif sys.platform == "win32" and _find_library_file(self, "zlib"):
                     feature.set("zlib", "zlib")  # alternative name
+                elif sys.platform == "win32" and _find_library_file(self, "zdll"):
+                    feature.set("zlib", "zdll")  # dll import library
 
         if feature.want("jpeg"):
             _dbg("Looking for jpeg")

src/PIL/Image.py

@@ -2552,7 +2552,7 @@ def save(
         filename: str | bytes = ""
         open_fp = False
         if is_path(fp):
-            filename = os.path.realpath(os.fspath(fp))
+            filename = os.fspath(fp)
             open_fp = True
         elif fp == sys.stdout:
             try:
@@ -2561,7 +2561,7 @@ def save(
                 pass
         if not filename and hasattr(fp, "name") and is_path(fp.name):
             # only set the name for metadata purposes
-            filename = os.path.realpath(os.fspath(fp.name))
+            filename = os.fspath(fp.name)
 
         # may mutate self!
         self._ensure_mutable()
@@ -3465,7 +3465,7 @@ def open(
     exclusive_fp = False
     filename: str | bytes = ""
     if is_path(fp):
-        filename = os.path.realpath(os.fspath(fp))
+        filename = os.fspath(fp)
 
     if filename:
         fp = builtins.open(filename, "rb")

src/PIL/ImageFile.py

@@ -130,7 +130,7 @@ def __init__(
         if is_path(fp):
             # filename
             self.fp = open(fp, "rb")
-            self.filename = os.path.realpath(os.fspath(fp))
+            self.filename = os.fspath(fp)
             self._exclusive_fp = True
         else:
             # stream

src/PIL/ImageFont.py

@@ -270,7 +270,7 @@ def load_from_bytes(f: IO[bytes]) -> None:
             )
 
         if is_path(font):
-            font = os.path.realpath(os.fspath(font))
+            font = os.fspath(font)
             if sys.platform == "win32":
                 font_bytes_path = font if isinstance(font, bytes) else font.encode()
                 try:

src/PIL/_typing.py

@@ -47,7 +47,7 @@ class SupportsRead(Protocol[_T_co]):
     def read(self, __length: int = ...) -> _T_co: ...
 
 
-StrOrBytesPath = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
+StrOrBytesPath = Union[str, bytes, os.PathLike[str], os.PathLike[bytes]]
 
 
 __all__ = ["Buffer", "IntegralLike", "StrOrBytesPath", "SupportsRead", "TypeGuard"]