CI & CD: refactoring & speeding up GitHub actions

.github/workflows/analysis-coverage.yml

@@ -35,12 +35,15 @@ jobs:
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-tools
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.py') }}
+          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/linux_build_tools.py') }}
 
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-stuff
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.*') }}
+          key: ${{ env.KEY_HEAD }}-${{ env.KEY_CFFI_BUILD }}-${{ env.KEY_LINUX_LIBS }}
+        env:
+          KEY_LINUX_LIBS: ${{ hashFiles('libheif/linux_*.py') }}-${{ hashFiles('libheif/linux/**') }}
+          KEY_CFFI_BUILD: ${{ hashFiles('libheif/build.py') }}-${{ hashFiles('libheif/heif.h') }}
 
       - name: Install from source
         run: |

.github/workflows/build-cache-deps.yml

@@ -1,10 +1,12 @@
-name: Build cache for dependencies
+name: Build Linux dependencies
 
 on:
   push:
     branches: [master]
     paths:
-      - 'libheif/*.py'
+      - 'libheif/linux/**'
+      - 'libheif/linux_*.py'
+      - 'libheif/build.py'
       - 'libheif/heif.h'
   schedule:
     - cron: '59 23 * * 5' # Every Friday at 23:59 (GMT+2)
@@ -53,12 +55,15 @@ jobs:
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-tools
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.py') }}
+          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/linux_build_tools.py') }}
 
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-stuff
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.*') }}
+          key: ${{ env.KEY_HEAD }}-${{ env.KEY_CFFI_BUILD }}-${{ env.KEY_LINUX_LIBS }}
+        env:
+          KEY_LINUX_LIBS: ${{ hashFiles('libheif/linux_*.py') }}-${{ hashFiles('libheif/linux/**') }}
+          KEY_CFFI_BUILD: ${{ hashFiles('libheif/build.py') }}-${{ hashFiles('libheif/heif.h') }}
 
       - name: Start test build
         run: cibuildwheel

.github/workflows/build-ph-lite.yml

@@ -47,7 +47,7 @@ jobs:
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-tools
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.py') }}
+          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/linux_build_tools.py') }}
 
       - name: Run cibuildwheel
         run: cibuildwheel

.github/workflows/create-release-draft.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Build libheif and dependencies
         shell: msys2 {0}
         run: |
-            cd libheif/mingw-w64-libheif
+            cd libheif/windows/mingw-w64-libheif
             makepkg-mingw --syncdeps --noconfirm -f
             pacman -U mingw-w64-x86_64-libheif-1.12.0-9-any.pkg.tar.zst --noconfirm
 
@@ -145,12 +145,15 @@ jobs:
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-tools
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.py') }}
+          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/linux_build_tools.py') }}
 
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-stuff
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.*') }}
+          key: ${{ env.KEY_HEAD }}-${{ env.KEY_CFFI_BUILD }}-${{ env.KEY_LINUX_LIBS }}
+        env:
+          KEY_LINUX_LIBS: ${{ hashFiles('libheif/linux_*.py') }}-${{ hashFiles('libheif/linux/**') }}
+          KEY_CFFI_BUILD: ${{ hashFiles('libheif/build.py') }}-${{ hashFiles('libheif/heif.h') }}
 
       - name: Run cibuildwheel
         run: cibuildwheel
@@ -204,12 +207,15 @@ jobs:
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-tools
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.py') }}
+          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/linux_build_tools.py') }}
 
       - uses: actions/cache@v3
         with:
           path: ${{ env.BUILD_DIR_PREFIX }}/build-stuff
-          key: ${{ env.KEY_HEAD }}-${{ hashFiles('libheif/*.*') }}
+          key: ${{ env.KEY_HEAD }}-${{ env.KEY_CFFI_BUILD }}-${{ env.KEY_LINUX_LIBS }}
+        env:
+          KEY_LINUX_LIBS: ${{ hashFiles('libheif/linux_*.py') }}-${{ hashFiles('libheif/linux/**') }}
+          KEY_CFFI_BUILD: ${{ hashFiles('libheif/build.py') }}-${{ hashFiles('libheif/heif.h') }}
 
       - name: Run cibuildwheel
         run: cibuildwheel

.github/workflows/test-src-build-linux.yml

@@ -0,0 +1,92 @@
+name: Build from source(Linux)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [master]
+    types: [opened, reopened, edited]
+    paths:
+      - 'libheif/linux/**'
+      - 'libheif/linux_*.py'
+      - 'libheif/build.py'
+      - 'libheif/heif.h'
+      - 'setup.*'
+      - 'pyproject.toml'
+  push:
+    branches: [master]
+    paths:
+      - 'libheif/linux/**'
+      - 'libheif/linux_*.py'
+      - 'libheif/build.py'
+      - 'libheif/heif.h'
+      - 'setup.*'
+      - 'pyproject.toml'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  alpine_3_14_build:
+    name: Alpine:3.14-${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[docs]')"
+    strategy:
+      matrix:
+        arch: ["linux/amd64", "linux/arm64", "linux/arm/v7"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up qemu
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build pillow_heif
+        run: docker buildx build --platform ${{ matrix.arch }} -f docker/src_alpine_3_14.Dockerfile .
+
+  alpine_3_16_build:
+    name: Alpine:3.16-${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[docs]')"
+    strategy:
+      matrix:
+        arch: ["linux/amd64", "linux/arm64", "linux/arm/v7"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up qemu
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build pillow_heif
+        run: docker buildx build --platform ${{ matrix.arch }} -f docker/src_alpine_3_16.Dockerfile .
+
+  ubuntu_20_04_build:
+    name: Ubuntu:Focal-${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[docs]')"
+    strategy:
+      matrix:
+        arch: ["linux/amd64", "linux/arm64", "linux/arm/v7"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up qemu
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build pillow_heif
+        run: docker buildx build --platform ${{ matrix.arch }} -f docker/src_ubuntu_20_04.Dockerfile .
+
+  ubuntu_22_04_build:
+    name: Ubuntu:Jammy-${{ matrix.arch }}
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, '[docs]')"
+    strategy:
+      matrix:
+        arch: ["linux/amd64", "linux/arm64", "linux/arm/v7"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up qemu
+        run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      - name: Build pillow_heif
+        run: docker buildx build --platform ${{ matrix.arch }} -f docker/src_ubuntu_22_04.Dockerfile .

.pre-commit-config.yaml

@@ -4,9 +4,9 @@ repos:
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
-        exclude: libheif/(mingw-w64-libheif|aom-musl)/.*
+        exclude: libheif/(linux|macos|windows)/
     -   id: trailing-whitespace
-        exclude: libheif/(mingw-w64-libheif|aom-musl)/.*
+        exclude: libheif/(linux|macos|windows)/
     -   id: check-toml
     -   id: mixed-line-ending
 

libheif/build.py

@@ -1,13 +1,13 @@
+import sys
 from os import getenv, path
 from pathlib import Path
 from shutil import copy
 from subprocess import DEVNULL, PIPE, run
-from sys import platform
 from warnings import warn
 
 from cffi import FFI
 
-from libheif import build_libs
+from libheif import linux_build_libs
 
 ffi = FFI()
 with open("libheif/heif.h", "r", encoding="utf-8") as f:
@@ -32,29 +32,29 @@
 
 include_path_prefix = ""
 insert = False
-if platform.lower() == "darwin":
+if sys.platform.lower() == "darwin":
     include_path_prefix = getenv("HOMEBREW_PREFIX")
     if not include_path_prefix:
         _result = run(["brew", "--prefix"], stderr=DEVNULL, stdout=PIPE, check=False)
         if not _result.returncode and _result.stdout is not None:
             include_path_prefix = _result.stdout.decode("utf-8").rstrip("\n")
     if not include_path_prefix:
         include_path_prefix = "/opt/local"
-elif platform.lower() == "win32":
+elif sys.platform.lower() == "win32":
     include_path_prefix = getenv("VCPKG_PREFIX")
     if include_path_prefix is None:
         include_path_prefix = "C:\\msys64\\mingw64"
         warn(f"VCPKG_PREFIX environment variable is not set. Assuming `VCPKG_PREFIX={include_path_prefix}`")
 else:
-    include_path_prefix = build_libs.build_libs_linux()
+    include_path_prefix = linux_build_libs.build_libs()
 
 # Need to include "lib" directory to find "heif" library.
 include_path_prefix_lib = path.join(include_path_prefix, "lib")
 if include_path_prefix_lib not in library_dirs:
     library_dirs.append(include_path_prefix_lib)
 
 # MSYS2: rename "libheif.dll.a" to "libheif.lib"
-if platform.lower() == "win32":
+if sys.platform.lower() == "win32":
     lib_export_file = Path(path.join(include_path_prefix_lib, "libheif.dll.a"))
     if lib_export_file.is_file():
         copy(lib_export_file, path.join(include_path_prefix_lib, "libheif.lib"))
@@ -72,8 +72,8 @@
     """,
     include_dirs=include_dirs,
     library_dirs=library_dirs,
-    libraries=["libheif"] if platform.lower() == "win32" else ["heif"],
-    extra_compile_args=["/d2FH4-"] if platform.lower() == "win32" else [],
+    libraries=["libheif"] if sys.platform.lower() == "win32" else ["heif"],
+    extra_compile_args=["/d2FH4-"] if sys.platform.lower() == "win32" else [],
 )
 
 if __name__ == "__main__":