New GitHub Actions for building with cache using for linux binary whe…

…els.
bigcat88 · Nov 5, 2021 · b6ff992 · b6ff992
1 parent fdf029a
commit b6ff992
Show file tree

Hide file tree

Showing 17 changed files with 461 additions and 181 deletions.
diff --git a/.github/workflows/build-cache-deps.yml b/.github/workflows/build-cache-deps.yml
@@ -0,0 +1,143 @@
+name: Build cache for dependecies
+
+on:
+  workflow_dispatch:
+
+jobs:
+  dependencies_linux:
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - runs-on: ubuntu-latest
+            cibw-arch: manylinux_x86_64
+          - runs-on: ubuntu-latest
+            cibw-arch: manylinux_aarch64
+    name: Dependecies • ${{ matrix.cibw-arch }}
+    runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILD_STUFF: "/build-stuff"
+      SCRIPTS: "./build_scripts/linux/"
+      BUILD_ORDER: "pkg-config,cmake,autoconf,automake,libressl,nasm,libde265,aom,libheif,libffi"
+      OS_PACKAGES: "wget libtool git-all"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Make preparations
+        run: |
+          sudo mkdir "$BUILD_STUFF" && sudo chmod 777 "$BUILD_STUFF"
+          BUILD_STUFF_CMD="&& sh $(echo "$BUILD_ORDER" | sed 's/,/.sh \&\& sh /g').sh"
+          echo BUILD_COMMAND="yum install -y -q $OS_PACKAGES && set -ex && cd $SCRIPTS $BUILD_STUFF_CMD" >> $GITHUB_ENV
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: pkg-config
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: cmake
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: autoconf
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: automake
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libressl
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: nasm
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libde265
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: aom
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libheif
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libffi
+
+      - name: Set up QEMU
+        if: matrix.cibw-arch == 'manylinux_aarch64'
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - name: x64 linux pre configure enviroment
+        if: matrix.cibw-arch == 'manylinux_x86_64'
+        run: |
+          echo "CIBW_ARCHS=x86_64" >> $GITHUB_ENV
+          echo "CIBW_MANYLINUX_X86_64_IMAGE=manylinux2014" >> $GITHUB_ENV
+          echo "CIBW_BUILD=cp38-manylinux_x86_64" >> $GITHUB_ENV
+
+      - name: Arm linux pre configure enviroment
+        if: matrix.cibw-arch == 'manylinux_aarch64'
+        run: |
+          echo "CIBW_ARCHS=aarch64" >> $GITHUB_ENV
+          echo "CIBW_BUILD=cp38-manylinux_aarch64" >> $GITHUB_ENV
+
+      - name: Start test build
+        run: |
+          python -m pip install cibuildwheel twine
+          cibuildwheel
+        env:
+          CIBW_TEST_SKIP: "*"
+          CIBW_BEFORE_ALL: ${{ env.BUILD_COMMAND }}
+          CIBW_ENVIRONMENT: BUILD_STUFF=${{ env.BUILD_STUFF }}
+
+      - name: Check builded wheel
+        run: |
+          twine check wheelhouse/*
+
+      - name: Upload builded wheel
+        uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: wheelhouse/*.whl
+          if-no-files-found: error
diff --git a/.github/workflows/create-release-draft.yml b/.github/workflows/create-release-draft.yml
@@ -82,27 +82,21 @@ jobs:
           python-version: '3.8'
           architecture: ${{ matrix.python-arch }}
 
-      - name: Install requirements
-        run: |
-          python -m pip install cibuildwheel twine
-
       - name: macOS x64 pre configure enviroment
         if: matrix.cibw-arch == 'macosx_x86_64'
-        shell: bash
         run: |
           echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
 
-      - name: Configure cibuildwheel
-        shell: bash
-        run: |
-          echo "CIBW_BUILD=*-${{ matrix.cibw-arch }}" >> $GITHUB_ENV
-          echo "CIBW_SKIP=pp*" >> $GITHUB_ENV
-          echo "CIBW_TEST_REQUIRES=pytest piexif" >> $GITHUB_ENV
-          echo "CIBW_TEST_COMMAND=pytest {project}" >> $GITHUB_ENV
-
       - name: Run cibuildwheel
         run: |
+          python -m pip install cibuildwheel twine
           cibuildwheel
+        env:
+          CIBW_BUILD: ${{ format('*-{0}', matrix.cibw-arch) }}
+          CIBW_SKIP: "pp*"
+          CIBW_TEST_COMMAND: "pytest {project}"
+          CIBW_TEST_REQUIRES: "pytest piexif"
+          CIBW_BUILD_VERBOSITY: 1
 
       - name: Check built wheels
         run: |
@@ -126,52 +120,129 @@ jobs:
             cibw-arch: manylinux_aarch64
     name: Wheels • ${{ matrix.cibw-arch }}
     runs-on: ${{ matrix.runs-on }}
+    env:
+      BUILD_STUFF: "/build-stuff"
+      SCRIPTS: "./build_scripts/linux/"
+      BUILD_ORDER: "pkg-config,cmake,autoconf,automake,libressl,nasm,libde265,aom,libheif,libffi"
+      OS_PACKAGES: "wget libtool git-all"
 
     steps:
-      - name: Set up QEMU
-        if: matrix.cibw-arch == 'manylinux_aarch64'
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64
-
       - name: Checkout code
         uses: actions/checkout@v2
         with:
           submodules: recursive
 
-      - name: Install requirements
+      - name: Make preparations
         run: |
-          python -m pip install cibuildwheel twine
+          sudo mkdir "$BUILD_STUFF" && sudo chmod 777 "$BUILD_STUFF"
+          BUILD_STUFF_CMD="&& sh $(echo "$BUILD_ORDER" | sed 's/,/.sh \&\& sh /g').sh"
+          echo BUILD_COMMAND="yum install -y -q $OS_PACKAGES && set -ex && cd $SCRIPTS $BUILD_STUFF_CMD" >> $GITHUB_ENV
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: pkg-config
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: cmake
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: autoconf
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: automake
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libressl
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: nasm
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libde265
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: aom
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libheif
+
+      - uses: actions/cache@v2
+        with:
+          path: ${{ env.BUILD_STUFF }}/${{ env.NAME }}
+          key: ${{ matrix.runs-on }}-${{ matrix.cibw-arch }}-${{ hashFiles(format('{0}{1}.sh', env.SCRIPTS, env.NAME)) }}
+        env:
+          NAME: libffi
+
+      - name: Set up QEMU
+        if: matrix.cibw-arch == 'manylinux_aarch64'
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
 
       - name: x64 linux pre configure enviroment
         if: matrix.cibw-arch == 'manylinux_x86_64'
         run: |
           echo "CIBW_ARCHS=x86_64" >> $GITHUB_ENV
-          echo "CIBW_TEST_COMMAND=pytest {project}" >> $GITHUB_ENV
           echo "CIBW_MANYLINUX_X86_64_IMAGE=manylinux2014" >> $GITHUB_ENV
 
       - name: Arm linux pre configure enviroment
         if: matrix.cibw-arch == 'manylinux_aarch64'
         run: |
           echo "CIBW_ARCHS=aarch64" >> $GITHUB_ENV
-          echo "CIBW_TEST_COMMAND=pytest -k '__everywhere' {project}" >> $GITHUB_ENV
-
-      - name: Configure cibuildwheel
-        shell: bash
-        run: |
-          echo "CIBW_BUILD=*-${{ matrix.cibw-arch }}" >> $GITHUB_ENV
-          echo "CIBW_SKIP=pp*" >> $GITHUB_ENV
-          echo "CIBW_TEST_REQUIRES=pytest piexif" >> $GITHUB_ENV
 
       - name: Run cibuildwheel
         run: |
+          python -m pip install cibuildwheel twine
           cibuildwheel
-
-      - name: Check built wheels
+        env:
+          CIBW_BEFORE_ALL: ${{ env.BUILD_COMMAND }}
+          CIBW_ENVIRONMENT: BUILD_STUFF=${{ env.BUILD_STUFF }}
+          CIBW_BUILD: ${{ format('*-{0}', matrix.cibw-arch) }}
+          CIBW_SKIP: "pp*"
+          CIBW_TEST_COMMAND: "pytest {project}"
+          CIBW_TEST_REQUIRES: "pytest piexif"
+          CIBW_BUILD_VERBOSITY: 1
+
+      - name: Check builded wheels
         run: |
           twine check wheelhouse/*
 
-      - name: Upload built wheels
+      - name: Upload builded wheels
         uses: actions/upload-artifact@v2
         with:
           name: wheels

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,7 +2,13 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.1.4 - 2021-11-01]
+## [0.1.4 - 2021-11-05]
+
+### Added
+
+- GitHub build Actions now use cache.
+- Added `libaom` library to linux build.
+- More tests. 
 
 ### Changed
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -3,7 +3,10 @@ include README.md
 include CHANGELOG.md
 include pytest.ini
 include requirements.txt
-include requirements-dev.txt
+include requirements_dev.txt
 
-graft scripts
 graft tests
+
+exclude .pylintrc
+
+prune build_scripts
diff --git a/build_scripts/linux/aom.sh b/build_scripts/linux/aom.sh
@@ -0,0 +1,18 @@
+VERSION="bb35ba9148543f22ba7d8642e4fbd29ae301f5dc"
+NAME=$(basename "$BASH_SOURCE" | cut -f 1 -d '.')
+URL="https://aomedia.googlesource.com/aom/+archive/$VERSION.tar.gz"
+cd "/host/$BUILD_STUFF" || exit 2
+if [[ -d "$NAME" ]]; then
+  echo "Cache found for lib$NAME, install it..."
+  cd "$NAME/build" || exit 102
+else
+  echo "No cache found for lib$NAME, build it..."
+  mkdir "$NAME" "$NAME/build" "$NAME/$NAME" && cd "$NAME" || exit 104
+  wget -q --no-check-certificate -O "$NAME.tar.gz" "$URL" \
+  && tar xf "$NAME.tar.gz" -C "$NAME" \
+  && rm -f "$NAME.tar.gz" \
+  && cd "./build" \
+  && cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=1 "../$NAME" \
+  && make -j4
+fi
+make install && ldconfig
diff --git a/build_scripts/linux/autoconf.sh b/build_scripts/linux/autoconf.sh
@@ -0,0 +1,19 @@
+VERSION="2.71"
+NAME=$(basename "$BASH_SOURCE" | cut -f 1 -d '.')
+URL="https://ftp.gnu.org/gnu/autoconf/$NAME-$VERSION.tar.gz"
+cd "/host/$BUILD_STUFF" || exit 2
+if [[ -d "$NAME" ]]; then
+  echo "Cache found for $NAME, install it..."
+  cd "$NAME" || exit 102
+else
+  echo "No cache found for $NAME, build it..."
+  mkdir "$NAME"
+  wget -q --no-check-certificate -O "$NAME.tar.gz" "$URL" \
+  && tar xf "$NAME.tar.gz" -C "$NAME" --strip-components 1 \
+  && rm -f "$NAME.tar.gz" \
+  && cd "$NAME" \
+  && ./configure \
+  && make -j4
+fi
+make install \
+&& autoconf --version