From 262520fe86838e8a32cec027ff27216152c574c2 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 2 Nov 2021 16:38:40 -0500 Subject: [PATCH 01/58] Workaround Issue 115 to compile dav1d on arm64 https://github.com/markus-perl/ffmpeg-build-script/issues/115 --- build-ffmpeg | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build-ffmpeg b/build-ffmpeg index af8875d0..1b6ab16d 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -412,9 +412,20 @@ if command_exists "python3"; then if build "dav1d" "0.9.2"; then download "https://code.videolan.org/videolan/dav1d/-/archive/0.9.2/dav1d-0.9.2.tar.gz" make_dir build + # [ Descript - cvanwink ] + # Set extra CFLAGS for arm64 + # https://github.com/markus-perl/ffmpeg-build-script/issues/115 + if $MACOS_M1; then + export CFLAGS="-arch arm64" + fi execute meson build --prefix="${WORKSPACE}" --buildtype=release --default-library=static --libdir="${WORKSPACE}"/lib execute ninja -C build execute ninja -C build install + # [ Descript - cvanwink ] + # Restore default CFLAGS from line 12 + if $MACOS_M1; then + CFLAGS="-I$WORKSPACE/include" + fi build_done "dav1d" "0.9.2" fi CONFIGURE_OPTIONS+=("--enable-libdav1d") From d9e77be1518520eaea619805c741f933c90aab41 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 2 Nov 2021 16:44:57 -0500 Subject: [PATCH 02/58] Disable libtiff on arm64 --- build-ffmpeg | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 1b6ab16d..6f3ea96f 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -750,12 +750,19 @@ fi ## image library ## -if build "libtiff" "4.3.0"; then - download "https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static - execute make -j $MJOBS - execute make install - build_done "libtiff" "4.3.0" +# [ Descript - cvanwink ] +# Disable libtiff on arm64 as it doesn't compile +# ../version:1:1: error: expected unqualified-id +# 4.3.0 +# ^ +if ! $MACOS_M1; then + if build "libtiff" "4.3.0"; then + download "https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + execute make -j $MJOBS + execute make install + build_done "libtiff" "4.3.0" + fi fi if build "libpng" "1.6.37"; then download "https://sourceforge.net/projects/libpng/files/libpng16/1.6.37/libpng-1.6.37.tar.gz/download?use_mirror=gigenet" "libpng-1.6.37.tar.gz" From cce3de268d43ba63777ee775179c7d4cff549ab0 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 2 Nov 2021 20:45:46 -0500 Subject: [PATCH 03/58] Disable libtiff on x86_64 as it also doesn't compile with the same error --- build-ffmpeg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 6f3ea96f..4148cc72 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -751,11 +751,11 @@ fi ## # [ Descript - cvanwink ] -# Disable libtiff on arm64 as it doesn't compile +# Disable libtiff on arm64 and x86_64 as it doesn't compile # ../version:1:1: error: expected unqualified-id # 4.3.0 # ^ -if ! $MACOS_M1; then +if false; then if build "libtiff" "4.3.0"; then download "https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz" execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static From 00d6dfa5a7fc6dc159b6aac2790dc1db3c02a89c Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 2 Nov 2021 20:46:25 -0500 Subject: [PATCH 04/58] Disable libwebp as it doesn't compile on x86_64 --- build-ffmpeg | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 4148cc72..f752ec68 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -776,18 +776,25 @@ fi # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS CPPFLAGS= -if build "libwebp" "1.2.0"; then - download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.0.tar.gz" "libwebp-1.2.0.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static - make_dir build - cd build || exit - execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ - execute make -j $MJOBS - execute make install +# [ Descript - cvanwink ] +# Disable libwebp on arm64 and x86_64 as it doesn't compile on x86_64 +# Undefined symbols for architecture x86_64: +# "_PrintGifError", referenced from: +# _GIFDisplayError in gifdec.c.o +if false; then + if build "libwebp" "1.2.0"; then + download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.0.tar.gz" "libwebp-1.2.0.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + make_dir build + cd build || exit + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ + execute make -j $MJOBS + execute make install - build_done "libwebp" "1.2.0" + build_done "libwebp" "1.2.0" + fi + CONFIGURE_OPTIONS+=("--enable-libwebp") fi -CONFIGURE_OPTIONS+=("--enable-libwebp") ## ## other library From 7540cf479a832840416ef79de9acc2a7c40cccee Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 2 Nov 2021 20:50:09 -0500 Subject: [PATCH 05/58] Add DS_Store to gitignore Cherry Pick 8b606fa7e7f168571bd55d87a1f5f0ed16ee4b57 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4725dcff..e2053e2a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ workspace .idea linux .artifacts +.DS_Store From d2ab2832722786ce6e4b981092bf0466f45c751a Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 3 Nov 2021 08:02:57 -0500 Subject: [PATCH 06/58] Add optional --full-shared argument to build-ffmpeg to build shared libraries --- build-ffmpeg | 245 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 210 insertions(+), 35 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index f752ec68..217ab3aa 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -17,6 +17,7 @@ MACOS_M1=false CONFIGURE_OPTIONS=() NONFREE_AND_GPL=false LATEST=false +SHARED_LIBRARIES=false # Check for Apple Silicon if [[ ("$(uname -m)" == "arm64") && ("$OSTYPE" == "darwin"*) ]]; then @@ -203,6 +204,7 @@ usage() { echo " --latest Build latest version of dependencies if newer available" echo " --full-static Build a full static FFmpeg binary (eg. glibc, pthreads etc...) **only Linux**" echo " Note: Because of the NSS (Name Service Switch), glibc does not recommend static links." + echo " --full-shared Build all shared libraries. Cannot be used with --full-static" echo "" } @@ -243,6 +245,9 @@ while (($# > 0)); do if [[ "$1" == "--latest" ]]; then LATEST=true fi + if [[ "$1" == "--full-shared" ]]; then + SHARED_LIBRARIES=true + fi shift ;; *) @@ -260,6 +265,11 @@ if [ -z "$bflag" ]; then exit 0 fi +if [[ ($SHARED_LIBRARIES == true) && ($LDEXEFLAGS == true) ]]; then + usage + exit 1 +fi + echo "Using $MJOBS make jobs simultaneously." if $NONFREE_AND_GPL; then @@ -270,6 +280,10 @@ if [ -n "$LDEXEFLAGS" ]; then echo "Start the build in full static mode." fi +if $SHARED_LIBRARIES; then + echo "Building shared libraries." +fi + mkdir -p "$PACKAGES" mkdir -p "$WORKSPACE" @@ -323,7 +337,11 @@ fi if build "nasm" "2.15.05"; then download "https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.xz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install build_done "nasm" "2.15.05" @@ -331,7 +349,11 @@ fi if build "zlib" "1.2.11"; then download "https://www.zlib.net/zlib-1.2.11.tar.gz" - execute ./configure --static --prefix="${WORKSPACE}" + if $SHARED_LIBRARIES; then + execute ./configure --shared --prefix="${WORKSPACE}" + else + execute ./configure --static --prefix="${WORKSPACE}" + fi execute make -j $MJOBS execute make install build_done "zlib" "1.2.11" @@ -364,7 +386,11 @@ fi if build "libtool" "2.4.6"; then download "https://ftpmirror.gnu.org/libtool/libtool-2.4.6.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --enable-static --disable-shared + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --disable-static --enable-shared + else + execute ./configure --prefix="${WORKSPACE}" --enable-static --disable-shared + fi execute make -j $MJOBS execute make install build_done "libtool" "2.4.6" @@ -418,7 +444,11 @@ if command_exists "python3"; then if $MACOS_M1; then export CFLAGS="-arch arm64" fi - execute meson build --prefix="${WORKSPACE}" --buildtype=release --default-library=static --libdir="${WORKSPACE}"/lib + if $SHARED_LIBRARIES; then + execute meson build --prefix="${WORKSPACE}" --buildtype=release --default-library=shared --libdir="${WORKSPACE}"/lib + else + execute meson build --prefix="${WORKSPACE}" --buildtype=release --default-library=static --libdir="${WORKSPACE}"/lib + fi execute ninja -C build execute ninja -C build install # [ Descript - cvanwink ] @@ -437,7 +467,11 @@ if ! $MACOS_M1; then # Last known working commit which passed CI Tests from HEAD branch download "https://gitlab.com/AOMediaCodec/SVT-AV1/-/archive/1a3e32b8fdc4abf5c093ee01dfa82803afc75fb4/SVT-AV1-1a3e32b8fdc4abf5c093ee01dfa82803afc75fb4.tar.gz" "svtav1-1a3e32b.tar.gz" cd "${PACKAGES}"/svtav1-1a3e32b/Build/linux || exit - execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=off -DBUILD_SHARED_LIBS=OFF ../.. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release + if $SHARED_LIBRARIES; then + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=off -DBUILD_SHARED_LIBS=ON ../.. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release + else + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=off -DBUILD_SHARED_LIBS=OFF ../.. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release + fi execute make -j $MJOBS execute make install execute cp SvtAv1Enc.pc "${WORKSPACE}/lib/pkgconfig/" @@ -464,14 +498,26 @@ if $NONFREE_AND_GPL; then cd "${PACKAGES}"/x264-5db6aa6 || exit if [[ "$OSTYPE" == "linux-gnu" ]]; then - execute ./configure --prefix="${WORKSPACE}" --enable-static --enable-pic CXXFLAGS="-fPIC" + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --enable-pic CXXFLAGS="-fPIC" + else + execute ./configure --prefix="${WORKSPACE}" --enable-static --enable-pic CXXFLAGS="-fPIC" + fi else - execute ./configure --prefix="${WORKSPACE}" --enable-static --enable-pic + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --enable-pic + else + execute ./configure --prefix="${WORKSPACE}" --enable-static --enable-pic + fi fi execute make -j $MJOBS execute make install - execute make install-lib-static + if $SHARED_LIBRARIES; then + execute make install-lib-shared + else + execute make install-lib-static + fi build_done "x264" "5db6aa6" fi @@ -485,15 +531,27 @@ if $NONFREE_AND_GPL; then rm -rf 8bit 10bit 12bit 2>/dev/null mkdir -p 8bit 10bit 12bit cd 12bit || exit - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON + if $SHARED_LIBRARIES; then + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=ON -DBUILD_SHARED_LIBS=ON -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON + else + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON + fi execute make -j $MJOBS cd ../10bit || exit - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF + if $SHARED_LIBRARIES; then + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=ON -DBUILD_SHARED_LIBS=ON -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF + else + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF + fi execute make -j $MJOBS cd ../8bit || exit ln -sf ../10bit/libx265.a libx265_main10.a ln -sf ../12bit/libx265.a libx265_main12.a - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON + if $SHARED_LIBRARIES; then + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=ON -DBUILD_SHARED_LIBS=ON -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON + else + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON + fi execute make -j $MJOBS mv libx265.a libx265_main.a @@ -531,7 +589,11 @@ if build "libvpx" "1.10.0"; then sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched >build/make/Makefile fi - execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --disable-shared --as=yasm --enable-vp9-highbitdepth + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --enable-shared --disable-static --as=yasm --enable-vp9-highbitdepth + else + execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --disable-shared --as=yasm --enable-vp9-highbitdepth + fi execute make -j $MJOBS execute make install @@ -543,7 +605,11 @@ if $NONFREE_AND_GPL; then if build "xvidcore" "1.3.7"; then download "https://downloads.xvid.com/downloads/xvidcore-1.3.7.tar.gz" cd build/generic || exit - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install @@ -569,7 +635,11 @@ if $NONFREE_AND_GPL; then patch -p1 include/libmp3lame.sym.patched + mv include/libmp3lame.sym.patched include/libmp3lame.sym + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install @@ -695,7 +807,11 @@ CONFIGURE_OPTIONS+=("--enable-libmp3lame") if build "opus" "1.3.1"; then download "https://archive.mozilla.org/pub/opus/opus-1.3.1.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install @@ -705,7 +821,11 @@ CONFIGURE_OPTIONS+=("--enable-libopus") if build "libogg" "1.3.3"; then download "https://ftp.osuosl.org/pub/xiph/releases/ogg/libogg-1.3.3.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install build_done "libogg" "1.3.3" @@ -713,7 +833,11 @@ fi if build "libvorbis" "1.3.6"; then download "https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --enable-static --disable-shared --disable-oggtest + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --disable-static --enable-shared --disable-oggtest + else + execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --enable-static --disable-shared --disable-oggtest + fi execute make -j $MJOBS execute make install @@ -726,7 +850,11 @@ if build "libtheora" "1.1.1"; then sed "s/-fforce-addr//g" configure >configure.patched chmod +x configure.patched mv configure.patched configure - execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --with-vorbis-libraries="${WORKSPACE}"/lib --with-vorbis-includes="${WORKSPACE}"/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm --disable-spec + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --with-vorbis-libraries="${WORKSPACE}"/lib --with-vorbis-includes="${WORKSPACE}"/include/ --disable-static --enable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm --disable-spec + else + execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --with-vorbis-libraries="${WORKSPACE}"/lib --with-vorbis-includes="${WORKSPACE}"/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm --disable-spec + fi execute make -j $MJOBS execute make install @@ -737,7 +865,11 @@ CONFIGURE_OPTIONS+=("--enable-libtheora") if $NONFREE_AND_GPL; then if build "fdk_aac" "2.0.2"; then download "https://sourceforge.net/projects/opencore-amr/files/fdk-aac/fdk-aac-2.0.2.tar.gz/download?use_mirror=gigenet" "fdk-aac-2.0.2.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --enable-pic + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static --enable-pic + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --enable-pic + fi execute make -j $MJOBS execute make install @@ -758,7 +890,11 @@ fi if false; then if build "libtiff" "4.3.0"; then download "https://download.osgeo.org/libtiff/tiff-4.3.0.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install build_done "libtiff" "4.3.0" @@ -768,7 +904,11 @@ if build "libpng" "1.6.37"; then download "https://sourceforge.net/projects/libpng/files/libpng16/1.6.37/libpng-1.6.37.tar.gz/download?use_mirror=gigenet" "libpng-1.6.37.tar.gz" export LDFLAGS="${LDFLAGS}" export CPPFLAGS="${CFLAGS}" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install build_done "libpng" "1.6.37" @@ -784,10 +924,18 @@ CPPFLAGS= if false; then if build "libwebp" "1.2.0"; then download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.0.tar.gz" "libwebp-1.2.0.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi make_dir build cd build || exit - execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ + if $SHARED_LIBRARIES; then + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=ON -DENABLE_STATIC=OFF ../ + else + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ + fi execute make -j $MJOBS execute make install @@ -802,7 +950,11 @@ fi if build "libsdl" "2.0.14"; then download "https://www.libsdl.org/release/SDL2-2.0.14.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi execute make -j $MJOBS execute make install @@ -815,7 +967,11 @@ if $NONFREE_AND_GPL; then export OPENSSL_ROOT_DIR="${WORKSPACE}" export OPENSSL_LIB_DIR="${WORKSPACE}"/lib export OPENSSL_INCLUDE_DIR="${WORKSPACE}"/include/ - execute cmake . -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON -DENABLE_APPS=OFF -DUSE_STATIC_LIBSTDCXX=ON + if $SHARED_LIBRARIES; then + execute cmake . -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=ON -DENABLE_STATIC=OFF -DENABLE_APPS=OFF -DUSE_STATIC_LIBSTDCXX=ON + else + execute cmake . -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON -DENABLE_APPS=OFF -DUSE_STATIC_LIBSTDCXX=ON + fi execute make install if [ -n "$LDEXEFLAGS" ]; then @@ -884,6 +1040,24 @@ fi build "ffmpeg" "$FFMPEG_VERSION" download "https://github.com/FFmpeg/FFmpeg/archive/refs/heads/release/$FFMPEG_VERSION.tar.gz" "FFmpeg-release-$FFMPEG_VERSION.tar.gz" # shellcheck disable=SC2086 +if $SHARED_LIBRARIES; then +./configure "${CONFIGURE_OPTIONS[@]}" \ + --disable-debug \ + --disable-doc \ + --disable-static \ + --enable-pthreads \ + --enable-shared\ + --enable-small \ + --enable-version3 \ + --extra-cflags="${CFLAGS}" \ + --extra-ldexeflags="${LDEXEFLAGS}" \ + --extra-ldflags="${LDFLAGS}" \ + --extra-libs="${EXTRALIBS}" \ + --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ + --pkg-config-flags="--static" \ + --prefix="${WORKSPACE}" \ + --extra-version="${EXTRA_VERSION}" +else ./configure "${CONFIGURE_OPTIONS[@]}" \ --disable-debug \ --disable-doc \ @@ -900,6 +1074,7 @@ download "https://github.com/FFmpeg/FFmpeg/archive/refs/heads/release/$FFMPEG_VE --pkg-config-flags="--static" \ --prefix="${WORKSPACE}" \ --extra-version="${EXTRA_VERSION}" +fi execute make -j $MJOBS execute make install From 44da9e84c7dca79b51b04d52f93c539090a72913 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Wed, 3 Nov 2021 21:00:09 -0500 Subject: [PATCH 07/58] Fix libtheora for arm64 https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63610 https://trac.macports.org/ticket/61096 --- build-ffmpeg | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/build-ffmpeg b/build-ffmpeg index 217ab3aa..e80138ee 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -851,6 +851,10 @@ if build "libtheora" "1.1.1"; then chmod +x configure.patched mv configure.patched configure if $SHARED_LIBRARIES; then + if $MACOSX_M1; then + # fix Libtool bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63610 + export MACOSX_DEPLOYMENT_TARGET=10.11 + fi execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --with-vorbis-libraries="${WORKSPACE}"/lib --with-vorbis-includes="${WORKSPACE}"/include/ --disable-static --enable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm --disable-spec else execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --with-vorbis-libraries="${WORKSPACE}"/lib --with-vorbis-includes="${WORKSPACE}"/include/ --enable-static --disable-shared --disable-oggtest --disable-vorbistest --disable-examples --disable-asm --disable-spec @@ -858,6 +862,13 @@ if build "libtheora" "1.1.1"; then execute make -j $MJOBS execute make install + if $SHARED_LIBRARIES; then + if $MACOSX_M1; then + # restore from Line 26 + export MACOSX_DEPLOYMENT_TARGET=11.0 + fi + fi + build_done "libtheora" "1.1.1" fi CONFIGURE_OPTIONS+=("--enable-libtheora") From bd55b01efb22129a855e3d784813898b0c773e6a Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Mon, 8 Nov 2021 23:12:30 -0600 Subject: [PATCH 08/58] Add initial python runner --- build-ffmpeg-descript.py | 292 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100755 build-ffmpeg-descript.py diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py new file mode 100755 index 00000000..9749d43d --- /dev/null +++ b/build-ffmpeg-descript.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 + +''' +This file is created by Descript to document and augment +the FFmpeg building process, for use in Descript's environment. + +(1) Call build-ffmpeg with the build command +(2) Copy or generate dSYM symbol files to the output folder +''' + +import glob +import os +import pathlib +import platform +import re +import shutil +import subprocess + + +cwd = os.path.dirname(os.path.realpath(__file__)) +packages_dir = os.path.join(cwd, 'packages') +workspace_dir = os.path.join(cwd, 'workspace') +workspace_bin_dir = os.path.join(workspace_dir, 'bin') +workspace_lib_dir = os.path.join(workspace_dir, 'lib') + +skipped_libs = set() +copied_libs = set() +missing_libs = set() +names_without_version = set() + +# +# builds FFmpeg and logs output to build-ffmpeg.log.txt +# +def buildFFmpeg(cwd, workspace_dir): + # create a log file for the build-ffmpeg command for build archival purposes + build_ffmpeg_log_filename = os.path.join(workspace_dir, 'build-ffmpeg.log.txt') + os.makedirs(os.path.dirname(build_ffmpeg_log_filename), exist_ok=True) + build_ffmpeg_log_file = open('./workspace/build-ffmpeg.log.txt', 'w') + + # set environment variables + env = os.environ + env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc + env['VERBOSE'] = 'yes' + + # call main build script + build_ffmpeg_path = os.path.join(cwd, 'build-ffmpeg') + subprocess.call([build_ffmpeg_path, '-b', '--full-shared'], env=env, stdout=build_ffmpeg_log_file) + + # close log file + build_ffmpeg_log_file.close() + +# +# Copies symbol file to the workspace destination +# skips symlinks to avoid duplication +# Copies entire dSYM packages for dylib files already within .dSYM packages +# +def copyOrGenerateSymbolFile(file, dest): + fileref = pathlib.Path(file) + if not fileref.is_symlink(): + symbolFileName = fileref.name + '.dSYM' + destPath = os.path.join(dest, symbolFileName) + + # See if there's a matching pre-existing symbol file. + # If so, copy it, replacing the destination + # example: + # ./packages/libtheora-1.1.1/lib/.libs/libtheoraenc.1.dylib.dSYM/Contents/Resources/DWARF/libtheoraenc.1.dylib + try: + + allParts = fileref.parts + symbolDirIndex = allParts.index(symbolFileName) # throws ValueError if not in allParts + symbolDirParts = allParts[:symbolDirIndex + 1] + symbolDir = os.path.join(*symbolDirParts) + if os.path.exists(destPath): + shutil.rmtree(destPath) + shutil.copytree(symbolDir, destPath) + + # Otherwise, generate a symbol file and place it at the destination + # example: + # ./packages/libtheora-1.1.1/lib/.libs/libtheora.dylib + except ValueError as e: + subprocess.call(['/usr/bin/dsymutil', str(fileref), '-o', destPath]) + +# +# Copies symbol files to the workspace destination +# skips symlinks to avoid duplication +# Copies entire dSYM packages for dylib files already within .dSYM packages +# +def copyOrGenerateSymbolFiles(source, dest): + for fileref in pathlib.Path(source + '/').glob('**/*.dylib'): + copyOrGenerateSymbolFile(str(fileref), dest) + +''' +const { resolve, basename } = require('path'); +const { mkdirSync, readlinkSync, readdirSync } = require('fs'); +const { execSync } = require('child_process'); + +function logAndExec(cmd) { + console.log(`EXEC ${cmd}`); + execSync(cmd); +} + +const baseIncludesDir = resolve(__dirname, 'workspace/include'); +const baseBinDir = resolve(__dirname, 'workspace/bin'); +const baseLibDir = resolve(__dirname, 'workspace/lib'); +const destDir = resolve(__dirname, 'workspace/mac'); + +try { + logAndExec(`rm -r -f ${destDir}`); +} catch (err) { + // +} +logAndExec(`mkdir -p ${destDir}`); + +const skippedLibs = new Set(); +const copiedLibs = new Set(); +const missingLibs = new Set(); +const namesWithoutVersion = new Set(); + +function copyDylibs(binaryName, base = baseBinDir) { + const origPath = resolve(base, binaryName); + const binaryPath = resolve(destDir, binaryName); + + logAndExec(`cp -a ${origPath} ${binaryPath}`); + + const lines = execSync(`otool -L ${binaryPath}`).toString('utf8').split('\n'); + const libsToRewrite = []; + for (const line of lines) { + const match = /[^\s:]+/.exec(line); + if (!match) { + continue; + } + const [path] = match; + if (path.startsWith('/usr/local')) { + missingLibs.add(path); + } else if (path.startsWith('/Users')) { + const filename = basename(path); + const newFilename = resolve(destDir, filename); + if (!copiedLibs.has(path)) { + copiedLibs.add(path); + copiedLibs.add(newFilename); + if (path !== newFilename) { + // copy sym-linked libraries as well + let nameWithoutVersion = filename.split('.')[0]; + // libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) + if (filename.includes('libSDL2')) { + nameWithoutVersion = 'libSDL2'; + } + namesWithoutVersion.add(nameWithoutVersion); + const nameWithoutVersionLib = `${nameWithoutVersion}.dylib`; + logAndExec(`cp -a ${resolve(baseLibDir, nameWithoutVersion)}*.dylib ${destDir}/.`); + + copyDylibs(filename, baseLibDir); + } + } + libsToRewrite.push({path, filename}); + } else { + skippedLibs.add(path); + } + } + + // find the non-sym-linked version of this library + let actualBinaryPath = binaryPath; + try { + const actualBinaryName = readlinkSync(binaryPath); + actualBinaryPath = resolve(destDir, actualBinaryName); + } catch (err) { + // + } + + if (libsToRewrite.length > 0) { + logAndExec(`install_name_tool -id @loader_path/${binaryName} ${libsToRewrite.map(({path, filename}) => `-change ${path} @loader_path/${filename}`).join(' ')} ${actualBinaryPath}`); + } +} + +copyDylibs('ffmpeg'); +copyDylibs('ffprobe'); + +console.log('Copying includes'); +logAndExec(`cp -r ${baseIncludesDir} ${destDir}/.`); + +for (const lib of Array.from(skippedLibs).sort()) { + console.log(`[NOTE] skipped ${lib}`); +} +for (const lib of Array.from(copiedLibs).sort()) { + if (!lib.startsWith(destDir)) { + console.log(`Copied ${lib}`); + } +} +for (const lib of Array.from(missingLibs).sort()) { + console.log(`[WARNING] missing ${lib}`); +} +''' + + + + +# +# +# +def copyLibraryAndDependencies(src_file, dest): + + dest_file = os.path.join(dest, os.path.basename(src_file)) + + # copy file + if os.path.exists(dest_file): + os.remove(dest_file) + shutil.copy2(src_file, dest_file) + + # copy symbol file + src_symbol_package = src_file + '.dSYM' + if os.path.exists(src_symbol_package): + dest_symbol_package = os.path.join(dest, os.path.basename(src_symbol_package)) + if os.path.exists(dest_symbol_package): + shutil.rmtree(dest_symbol_package) + shutil.copytree(src_symbol_package, dest_symbol_package) + + # recursively copy dependencies + otool_proc = subprocess.Popen(['/usr/bin/otool', '-L', src_file], stdout=subprocess.PIPE) + libs_to_rewrite = [] + for line in otool_proc.stdout: + ln = line.decode('utf-8').strip() + match = re.match('[^\s:]+', ln) + if not match: + continue + match_path = match[0] #ln[match.start(0):match.end(0)] + if match_path.startswith('/usr/local'): + missing_libs.add(match_path) + elif match_path.startswith(cwd): + filename = os.path.basename(match_path) + new_filename = os.path.realpath(os.path.join(dest, filename)) + if not match_path in copied_libs: + copied_libs.add(match_path) + copied_libs.add(new_filename) + if match_path != new_filename: + # copy sym-linked libraries as well + name_without_version = filename.split('.')[0] + # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) + if 'libSDL2' in filename: + name_without_version = 'libSDL2' + names_without_version.add(name_without_version) + + dependency_file = os.path.join(workspace_lib_dir, name_without_version) + + for file in glob.glob(r'' + dependency_file + '*.dylib'): + shutil.copy2(os.path.realpath(file), os.path.join(dest, os.path.basename(file))) + dependency_symbol_file = file + '.dSYM' + dest_dependency_symbol_file = os.path.join(dest, os.path.basename(dependency_symbol_file)) + if os.path.exists(dependency_symbol_file) and not os.path.exists(dest_dependency_symbol_file): + shutil.copytree(dependency_symbol_file, dest_dependency_symbol_file) + + if (os.path.exists(dependency_file + '.dylib')): + # RECURSION + copyLibraryAndDependencies(dependency_file + '.dylib', dest) + + libs_to_rewrite.append({'path': match_path, 'filename': filename}) + else: + skipped_libs.add(match_path) + + # find the non-sym-linked version of this library + actual_binary_path = os.path.realpath(dest_file) + + if len(libs_to_rewrite) > 0: + for lib in libs_to_rewrite: + print(' '.join(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['path'], '@loader_path/' + os.path.basename(lib['filename']), actual_binary_path])) + install_name_tool_proc = subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['path'], '@loader_path/' + os.path.basename(lib['filename']), actual_binary_path]) + +# +# +# +def main(): + output_dir = os.path.join(cwd, 'mac', platform.machine()) + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir) + + #buildFFmpeg(cwd, workspace_dir) + + # Generate dSYM files for each built library + #copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) + + # Generate dSYM files for each executable + executables = ['ffmpeg', 'ffprobe'] + for executable in executables: + executable_path = os.path.join(workspace_bin_dir, executable) + copyOrGenerateSymbolFile(executable_path, workspace_bin_dir) + copyLibraryAndDependencies(executable_path, output_dir) + +# +# entry +# +if __name__ == '__main__': + main() From 83e50db235a2ea88700a2d5ec162d229561d6970 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 08:03:47 -0600 Subject: [PATCH 09/58] Script cleanup --- build-ffmpeg-descript.py | 100 +++++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 9749d43d..4bb5f3ae 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -31,7 +31,7 @@ # # builds FFmpeg and logs output to build-ffmpeg.log.txt # -def buildFFmpeg(cwd, workspace_dir): +def buildFFmpeg(script_dir, workspace_dir): # create a log file for the build-ffmpeg command for build archival purposes build_ffmpeg_log_filename = os.path.join(workspace_dir, 'build-ffmpeg.log.txt') os.makedirs(os.path.dirname(build_ffmpeg_log_filename), exist_ok=True) @@ -43,7 +43,7 @@ def buildFFmpeg(cwd, workspace_dir): env['VERBOSE'] = 'yes' # call main build script - build_ffmpeg_path = os.path.join(cwd, 'build-ffmpeg') + build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') subprocess.call([build_ffmpeg_path, '-b', '--full-shared'], env=env, stdout=build_ffmpeg_log_file) # close log file @@ -191,84 +191,90 @@ def copyOrGenerateSymbolFiles(source, dest): } ''' - - - # # # -def copyLibraryAndDependencies(src_file, dest): +def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): + dest_file = os.path.join(dest_folder, os.path.basename(src_file)) - dest_file = os.path.join(dest, os.path.basename(src_file)) - # copy file - if os.path.exists(dest_file): + if overwrite and os.path.exists(dest_file): os.remove(dest_file) shutil.copy2(src_file, dest_file) # copy symbol file src_symbol_package = src_file + '.dSYM' if os.path.exists(src_symbol_package): - dest_symbol_package = os.path.join(dest, os.path.basename(src_symbol_package)) - if os.path.exists(dest_symbol_package): - shutil.rmtree(dest_symbol_package) - shutil.copytree(src_symbol_package, dest_symbol_package) + dest_symbol_package = os.path.join(dest_folder, os.path.basename(src_symbol_package)) + if overwrite and os.path.exists(dest_symbol_package): + shutil.rmtree(dest_symbol_package) + if not os.path.exists(dest_symbol_package): + shutil.copytree(src_symbol_package, dest_symbol_package) + + +# +# Recursive function to copy a library and its (non-system) dependencies +# also fixes loader paths for each library +# +def copyLibraryAndDependencies(src_file, dest_folder): + + dest_file = os.path.join(dest_folder, os.path.basename(src_file)) + + # copy file + copyLibraryAndSymbolPackage(src_file, dest_folder, True) # recursively copy dependencies otool_proc = subprocess.Popen(['/usr/bin/otool', '-L', src_file], stdout=subprocess.PIPE) - libs_to_rewrite = [] + loader_paths_to_rewrite = [] for line in otool_proc.stdout: ln = line.decode('utf-8').strip() match = re.match('[^\s:]+', ln) if not match: continue - match_path = match[0] #ln[match.start(0):match.end(0)] - if match_path.startswith('/usr/local'): - missing_libs.add(match_path) - elif match_path.startswith(cwd): - filename = os.path.basename(match_path) - new_filename = os.path.realpath(os.path.join(dest, filename)) - if not match_path in copied_libs: - copied_libs.add(match_path) - copied_libs.add(new_filename) - if match_path != new_filename: + src_dependency_file = match[0] + if src_dependency_file.startswith('/usr/local'): + missing_libs.add(src_dependency_file) + elif src_dependency_file.startswith(workspace_dir): + dependency_name = os.path.basename(src_dependency_file) + dest_dependency_path = os.path.join(dest_folder, dependency_name) + if not src_dependency_file in copied_libs: + copied_libs.add(src_dependency_file) + copied_libs.add(dest_dependency_path) + if src_dependency_file != dest_dependency_path: # copy sym-linked libraries as well - name_without_version = filename.split('.')[0] + dependency_name_without_version = dependency_name.split('.')[0] # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if 'libSDL2' in filename: - name_without_version = 'libSDL2' - names_without_version.add(name_without_version) + if 'libSDL2' in dependency_name: + dependency_name_without_version = 'libSDL2' + names_without_version.add(dependency_name_without_version) - dependency_file = os.path.join(workspace_lib_dir, name_without_version) - - for file in glob.glob(r'' + dependency_file + '*.dylib'): - shutil.copy2(os.path.realpath(file), os.path.join(dest, os.path.basename(file))) - dependency_symbol_file = file + '.dSYM' - dest_dependency_symbol_file = os.path.join(dest, os.path.basename(dependency_symbol_file)) - if os.path.exists(dependency_symbol_file) and not os.path.exists(dest_dependency_symbol_file): - shutil.copytree(dependency_symbol_file, dest_dependency_symbol_file) - - if (os.path.exists(dependency_file + '.dylib')): - # RECURSION - copyLibraryAndDependencies(dependency_file + '.dylib', dest) + unversioned_dependency_base_name = os.path.join(workspace_lib_dir, dependency_name_without_version) + + # Copy each version variant file + for variant_file in glob.glob(unversioned_dependency_base_name + r'*.dylib'): + copyLibraryAndSymbolPackage(variant_file, dest_folder, False) + + # RECURSIVELY copy dependencies + if (os.path.exists(unversioned_dependency_base_name + '.dylib')): + copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder) - libs_to_rewrite.append({'path': match_path, 'filename': filename}) + loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: - skipped_libs.add(match_path) + skipped_libs.add(src_dependency_file) # find the non-sym-linked version of this library actual_binary_path = os.path.realpath(dest_file) - if len(libs_to_rewrite) > 0: - for lib in libs_to_rewrite: - print(' '.join(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['path'], '@loader_path/' + os.path.basename(lib['filename']), actual_binary_path])) - install_name_tool_proc = subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['path'], '@loader_path/' + os.path.basename(lib['filename']), actual_binary_path]) + if len(loader_paths_to_rewrite) > 0: + for lib in loader_paths_to_rewrite: + print(' '.join(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['old_path'], '@loader_path/' + os.path.basename(lib['new_path']), actual_binary_path])) + install_name_tool_proc = subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['old_path'], '@loader_path/' + os.path.basename(lib['new_path']), actual_binary_path]) # # # def main(): - output_dir = os.path.join(cwd, 'mac', platform.machine()) + output_dir = os.path.join(workspace_dir, 'mac', platform.machine()) if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) From 1a438ab6eaa715265a8a1ed0eb717f735dec931a Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 11:24:23 -0600 Subject: [PATCH 10/58] Fix loader path issues --- build-ffmpeg-descript.py | 54 +++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 4bb5f3ae..d66c9c85 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -26,7 +26,6 @@ skipped_libs = set() copied_libs = set() missing_libs = set() -names_without_version = set() # # builds FFmpeg and logs output to build-ffmpeg.log.txt @@ -200,7 +199,7 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): # copy file if overwrite and os.path.exists(dest_file): os.remove(dest_file) - shutil.copy2(src_file, dest_file) + shutil.copy2(src_file, dest_file, follow_symlinks=False) # copy symbol file src_symbol_package = src_file + '.dSYM' @@ -211,17 +210,32 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): if not os.path.exists(dest_symbol_package): shutil.copytree(src_symbol_package, dest_symbol_package) +# +# +# +def getFilenameWithoutVersion(file_name) -> str: + result = file_name.split('.')[0] + # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) + if 'libSDL2' in result: + result = 'libSDL2' + return result # # Recursive function to copy a library and its (non-system) dependencies # also fixes loader paths for each library # def copyLibraryAndDependencies(src_file, dest_folder): - + dest_file = os.path.join(dest_folder, os.path.basename(src_file)) + print(dest_file) + # copy file copyLibraryAndSymbolPackage(src_file, dest_folder, True) + copied_libs.add(src_file) + copied_libs.add(dest_file) + + this_id = '' # recursively copy dependencies otool_proc = subprocess.Popen(['/usr/bin/otool', '-L', src_file], stdout=subprocess.PIPE) @@ -236,23 +250,19 @@ def copyLibraryAndDependencies(src_file, dest_folder): missing_libs.add(src_dependency_file) elif src_dependency_file.startswith(workspace_dir): dependency_name = os.path.basename(src_dependency_file) + if not len(this_id): + this_id = dependency_name dest_dependency_path = os.path.join(dest_folder, dependency_name) if not src_dependency_file in copied_libs: - copied_libs.add(src_dependency_file) - copied_libs.add(dest_dependency_path) if src_dependency_file != dest_dependency_path: - # copy sym-linked libraries as well - dependency_name_without_version = dependency_name.split('.')[0] - # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if 'libSDL2' in dependency_name: - dependency_name_without_version = 'libSDL2' - names_without_version.add(dependency_name_without_version) - - unversioned_dependency_base_name = os.path.join(workspace_lib_dir, dependency_name_without_version) - - # Copy each version variant file - for variant_file in glob.glob(unversioned_dependency_base_name + r'*.dylib'): - copyLibraryAndSymbolPackage(variant_file, dest_folder, False) + # Copy each version variant file (often symlinks) + dependency_name_without_version = getFilenameWithoutVersion(src_dependency_file) + unversioned_dependency_base_name = os.path.join(os.path.dirname(src_dependency_file), dependency_name_without_version) + for variant_src_file in glob.glob(unversioned_dependency_base_name + r'*.dylib'): + copyLibraryAndSymbolPackage(variant_src_file, dest_folder, False) + variant_dest_file = os.path.join(dest_folder, os.path.basename(variant_src_file)) + copied_libs.add(variant_src_file) + copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies if (os.path.exists(unversioned_dependency_base_name + '.dylib')): @@ -265,10 +275,14 @@ def copyLibraryAndDependencies(src_file, dest_folder): # find the non-sym-linked version of this library actual_binary_path = os.path.realpath(dest_file) + if len(this_id): + print(' '.join(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path])) + subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path]) + if len(loader_paths_to_rewrite) > 0: - for lib in loader_paths_to_rewrite: - print(' '.join(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['old_path'], '@loader_path/' + os.path.basename(lib['new_path']), actual_binary_path])) - install_name_tool_proc = subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + dest_file, '-change', lib['old_path'], '@loader_path/' + os.path.basename(lib['new_path']), actual_binary_path]) + for loader_path in loader_paths_to_rewrite: + print(' '.join(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path])) + subprocess.call(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path]) # # From 7c5d588cfcb2b4d5b2eb1f29eef9ae295ecd4649 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 11:26:02 -0600 Subject: [PATCH 11/58] remove some debugging traces --- build-ffmpeg-descript.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index d66c9c85..566f6dab 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -228,8 +228,6 @@ def copyLibraryAndDependencies(src_file, dest_folder): dest_file = os.path.join(dest_folder, os.path.basename(src_file)) - print(dest_file) - # copy file copyLibraryAndSymbolPackage(src_file, dest_folder, True) copied_libs.add(src_file) @@ -276,12 +274,10 @@ def copyLibraryAndDependencies(src_file, dest_folder): actual_binary_path = os.path.realpath(dest_file) if len(this_id): - print(' '.join(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path])) subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path]) if len(loader_paths_to_rewrite) > 0: for loader_path in loader_paths_to_rewrite: - print(' '.join(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path])) subprocess.call(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path]) # From b948a2def3e9956d9370267c060f94dc5f1e5ef8 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 11:46:18 -0600 Subject: [PATCH 12/58] more cleanup from node script --- build-ffmpeg-descript.py | 120 ++++++--------------------------------- 1 file changed, 17 insertions(+), 103 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 566f6dab..6fb4fe82 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -88,108 +88,6 @@ def copyOrGenerateSymbolFiles(source, dest): for fileref in pathlib.Path(source + '/').glob('**/*.dylib'): copyOrGenerateSymbolFile(str(fileref), dest) -''' -const { resolve, basename } = require('path'); -const { mkdirSync, readlinkSync, readdirSync } = require('fs'); -const { execSync } = require('child_process'); - -function logAndExec(cmd) { - console.log(`EXEC ${cmd}`); - execSync(cmd); -} - -const baseIncludesDir = resolve(__dirname, 'workspace/include'); -const baseBinDir = resolve(__dirname, 'workspace/bin'); -const baseLibDir = resolve(__dirname, 'workspace/lib'); -const destDir = resolve(__dirname, 'workspace/mac'); - -try { - logAndExec(`rm -r -f ${destDir}`); -} catch (err) { - // -} -logAndExec(`mkdir -p ${destDir}`); - -const skippedLibs = new Set(); -const copiedLibs = new Set(); -const missingLibs = new Set(); -const namesWithoutVersion = new Set(); - -function copyDylibs(binaryName, base = baseBinDir) { - const origPath = resolve(base, binaryName); - const binaryPath = resolve(destDir, binaryName); - - logAndExec(`cp -a ${origPath} ${binaryPath}`); - - const lines = execSync(`otool -L ${binaryPath}`).toString('utf8').split('\n'); - const libsToRewrite = []; - for (const line of lines) { - const match = /[^\s:]+/.exec(line); - if (!match) { - continue; - } - const [path] = match; - if (path.startsWith('/usr/local')) { - missingLibs.add(path); - } else if (path.startsWith('/Users')) { - const filename = basename(path); - const newFilename = resolve(destDir, filename); - if (!copiedLibs.has(path)) { - copiedLibs.add(path); - copiedLibs.add(newFilename); - if (path !== newFilename) { - // copy sym-linked libraries as well - let nameWithoutVersion = filename.split('.')[0]; - // libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if (filename.includes('libSDL2')) { - nameWithoutVersion = 'libSDL2'; - } - namesWithoutVersion.add(nameWithoutVersion); - const nameWithoutVersionLib = `${nameWithoutVersion}.dylib`; - logAndExec(`cp -a ${resolve(baseLibDir, nameWithoutVersion)}*.dylib ${destDir}/.`); - - copyDylibs(filename, baseLibDir); - } - } - libsToRewrite.push({path, filename}); - } else { - skippedLibs.add(path); - } - } - - // find the non-sym-linked version of this library - let actualBinaryPath = binaryPath; - try { - const actualBinaryName = readlinkSync(binaryPath); - actualBinaryPath = resolve(destDir, actualBinaryName); - } catch (err) { - // - } - - if (libsToRewrite.length > 0) { - logAndExec(`install_name_tool -id @loader_path/${binaryName} ${libsToRewrite.map(({path, filename}) => `-change ${path} @loader_path/${filename}`).join(' ')} ${actualBinaryPath}`); - } -} - -copyDylibs('ffmpeg'); -copyDylibs('ffprobe'); - -console.log('Copying includes'); -logAndExec(`cp -r ${baseIncludesDir} ${destDir}/.`); - -for (const lib of Array.from(skippedLibs).sort()) { - console.log(`[NOTE] skipped ${lib}`); -} -for (const lib of Array.from(copiedLibs).sort()) { - if (!lib.startsWith(destDir)) { - console.log(`Copied ${lib}`); - } -} -for (const lib of Array.from(missingLibs).sort()) { - console.log(`[WARNING] missing ${lib}`); -} -''' - # # # @@ -244,6 +142,8 @@ def copyLibraryAndDependencies(src_file, dest_folder): if not match: continue src_dependency_file = match[0] + if (src_dependency_file == 'libvpx.so.6'): + breakpoint; if src_dependency_file.startswith('/usr/local'): missing_libs.add(src_dependency_file) elif src_dependency_file.startswith(workspace_dir): @@ -292,7 +192,7 @@ def main(): #buildFFmpeg(cwd, workspace_dir) # Generate dSYM files for each built library - #copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) + copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) # Generate dSYM files for each executable executables = ['ffmpeg', 'ffprobe'] @@ -301,6 +201,20 @@ def main(): copyOrGenerateSymbolFile(executable_path, workspace_bin_dir) copyLibraryAndDependencies(executable_path, output_dir) + # Copy Includes + shutil.copytree( + os.path.join(workspace_dir, 'include'), + os.path.join(output_dir, 'include')) + + for lib in sorted(skipped_libs): + print ('[NOTE] skipped ' + lib) + + for lib in sorted(copied_libs): + print ('Copied ' + lib) + + for lib in sorted(missing_libs): + print ('[WARNING] missing ' + lib) + # # entry # From 04f8245347caace89fa482293be33036041343ca Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 11:46:31 -0600 Subject: [PATCH 13/58] remove --static flag for pkg-config when using shared --- build-ffmpeg | 1 - 1 file changed, 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index e80138ee..660cd99b 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -1065,7 +1065,6 @@ if $SHARED_LIBRARIES; then --extra-ldflags="${LDFLAGS}" \ --extra-libs="${EXTRALIBS}" \ --pkgconfigdir="$WORKSPACE/lib/pkgconfig" \ - --pkg-config-flags="--static" \ --prefix="${WORKSPACE}" \ --extra-version="${EXTRA_VERSION}" else From d7856657fb9dc40604054e1b6ee9c524f0dcba63 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 11:56:38 -0600 Subject: [PATCH 14/58] partial fix for .so files --- build-ffmpeg-descript.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 6fb4fe82..066fcd16 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -64,7 +64,6 @@ def copyOrGenerateSymbolFile(file, dest): # example: # ./packages/libtheora-1.1.1/lib/.libs/libtheoraenc.1.dylib.dSYM/Contents/Resources/DWARF/libtheoraenc.1.dylib try: - allParts = fileref.parts symbolDirIndex = allParts.index(symbolFileName) # throws ValueError if not in allParts symbolDirParts = allParts[:symbolDirIndex + 1] @@ -86,7 +85,9 @@ def copyOrGenerateSymbolFile(file, dest): # def copyOrGenerateSymbolFiles(source, dest): for fileref in pathlib.Path(source + '/').glob('**/*.dylib'): - copyOrGenerateSymbolFile(str(fileref), dest) + copyOrGenerateSymbolFile(str(fileref), dest) + for fileref in pathlib.Path(source + '/').glob('**/*.so*'): + copyOrGenerateSymbolFile(str(fileref), dest) # # @@ -142,8 +143,6 @@ def copyLibraryAndDependencies(src_file, dest_folder): if not match: continue src_dependency_file = match[0] - if (src_dependency_file == 'libvpx.so.6'): - breakpoint; if src_dependency_file.startswith('/usr/local'): missing_libs.add(src_dependency_file) elif src_dependency_file.startswith(workspace_dir): @@ -167,6 +166,8 @@ def copyLibraryAndDependencies(src_file, dest_folder): copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) + elif not src_dependency_file.startswith('/'): + breakpoint else: skipped_libs.add(src_dependency_file) From 35c78a51391b8c1c662b2dc4ff497264d1162f55 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 14:36:28 -0600 Subject: [PATCH 15/58] disable libvpx until linking issues are resolved --- build-ffmpeg | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 660cd99b..a9e1dfc2 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -580,26 +580,31 @@ EOF CONFIGURE_OPTIONS+=("--enable-libx265") fi -if build "libvpx" "1.10.0"; then - download "https://github.com/webmproject/libvpx/archive/refs/tags/v1.10.0.tar.gz" "libvpx-1.10.0.tar.gz" +# [ Descript - cvanwink ] +# Disable libvpx on arm64 and x86_64 as it doesn't link properly +# with the .so files - will revisit after end-to-end is working +if false; then + if build "libvpx" "1.10.0"; then + download "https://github.com/webmproject/libvpx/archive/refs/tags/v1.10.0.tar.gz" "libvpx-1.10.0.tar.gz" - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "Applying Darwin patch" - sed "s/,--version-script//g" build/make/Makefile >build/make/Makefile.patched - sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched >build/make/Makefile - fi + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "Applying Darwin patch" + sed "s/,--version-script//g" build/make/Makefile >build/make/Makefile.patched + sed "s/-Wl,--no-undefined -Wl,-soname/-Wl,-undefined,error -Wl,-install_name/g" build/make/Makefile.patched >build/make/Makefile + fi - if $SHARED_LIBRARIES; then - execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --enable-shared --disable-static --as=yasm --enable-vp9-highbitdepth - else - execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --disable-shared --as=yasm --enable-vp9-highbitdepth - fi - execute make -j $MJOBS - execute make install + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --enable-shared --disable-static --as=yasm --enable-vp9-highbitdepth + else + execute ./configure --prefix="${WORKSPACE}" --disable-unit-tests --disable-shared --as=yasm --enable-vp9-highbitdepth + fi + execute make -j $MJOBS + execute make install - build_done "libvpx" "1.10.0" + build_done "libvpx" "1.10.0" + fi + CONFIGURE_OPTIONS+=("--enable-libvpx") fi -CONFIGURE_OPTIONS+=("--enable-libvpx") if $NONFREE_AND_GPL; then if build "xvidcore" "1.3.7"; then From 9b1eeb63e07997469cdfb112c9bdaf91fa16d596 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 14:49:11 -0600 Subject: [PATCH 16/58] Check that binary is runnable --- build-ffmpeg-descript.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 066fcd16..da4b47b7 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -196,12 +196,22 @@ def main(): copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) # Generate dSYM files for each executable + # and copy their dependencies executables = ['ffmpeg', 'ffprobe'] for executable in executables: executable_path = os.path.join(workspace_bin_dir, executable) + + # check that the build library is runnable + subprocess.check_output([executable_path, '-version']) + copyOrGenerateSymbolFile(executable_path, workspace_bin_dir) copyLibraryAndDependencies(executable_path, output_dir) + # check that the copied file is runnable + print(subprocess.check_output([os.path.join(output_dir, executable), '-version']).decode('utf-8')) + + + # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), From 13267292a42ef160f52555c2d48aad7b60ca60aa Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 15:22:21 -0600 Subject: [PATCH 17/58] add more safety --- build-ffmpeg-descript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index da4b47b7..a26828f0 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -175,11 +175,11 @@ def copyLibraryAndDependencies(src_file, dest_folder): actual_binary_path = os.path.realpath(dest_file) if len(this_id): - subprocess.call(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path]) + subprocess.check_output(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path]) if len(loader_paths_to_rewrite) > 0: for loader_path in loader_paths_to_rewrite: - subprocess.call(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path]) + subprocess.check_output(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path]) # # From 1912a6c7e7aaa2b20d31ee5c343993b95fafc950 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 9 Nov 2021 16:14:19 -0600 Subject: [PATCH 18/58] Workaround for libraries with @rpath --> @loader_path --- build-ffmpeg | 2 +- build-ffmpeg-descript.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index a9e1dfc2..3a33ed45 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -468,7 +468,7 @@ if ! $MACOS_M1; then download "https://gitlab.com/AOMediaCodec/SVT-AV1/-/archive/1a3e32b8fdc4abf5c093ee01dfa82803afc75fb4/SVT-AV1-1a3e32b8fdc4abf5c093ee01dfa82803afc75fb4.tar.gz" "svtav1-1a3e32b.tar.gz" cd "${PACKAGES}"/svtav1-1a3e32b/Build/linux || exit if $SHARED_LIBRARIES; then - execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=off -DBUILD_SHARED_LIBS=ON ../.. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=on -DBUILD_SHARED_LIBS=ON ../.. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release else execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=off -DBUILD_SHARED_LIBS=OFF ../.. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release fi diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index a26828f0..50962a96 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -143,6 +143,13 @@ def copyLibraryAndDependencies(src_file, dest_folder): if not match: continue src_dependency_file = match[0] + + # fix incorrect usage of @rpath + if src_dependency_file.startswith('@rpath/'): + fixed_path = os.path.join(workspace_lib_dir, src_dependency_file[7:]) + loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': fixed_path}) + src_dependency_file = fixed_path + if src_dependency_file.startswith('/usr/local'): missing_libs.add(src_dependency_file) elif src_dependency_file.startswith(workspace_dir): @@ -190,7 +197,7 @@ def main(): shutil.rmtree(output_dir) os.makedirs(output_dir) - #buildFFmpeg(cwd, workspace_dir) + buildFFmpeg(cwd, workspace_dir) # Generate dSYM files for each built library copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) @@ -200,10 +207,6 @@ def main(): executables = ['ffmpeg', 'ffprobe'] for executable in executables: executable_path = os.path.join(workspace_bin_dir, executable) - - # check that the build library is runnable - subprocess.check_output([executable_path, '-version']) - copyOrGenerateSymbolFile(executable_path, workspace_bin_dir) copyLibraryAndDependencies(executable_path, output_dir) From fc7b7916453af998d8da7664b3aaa71c1161fafb Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 9 Nov 2021 16:24:05 -0600 Subject: [PATCH 19/58] comments and cleanup --- build-ffmpeg-descript.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 50962a96..e1971d17 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -5,7 +5,9 @@ the FFmpeg building process, for use in Descript's environment. (1) Call build-ffmpeg with the build command -(2) Copy or generate dSYM symbol files to the output folder +(2) Copy or generate dSYM symbol files to the workspace folder +(3) Copy executables from the workspace folder and all built dependencies to platform outputfolder +(4) Fix dyld ids and loader paths for all built libraries ''' import glob @@ -16,7 +18,6 @@ import shutil import subprocess - cwd = os.path.dirname(os.path.realpath(__file__)) packages_dir = os.path.join(cwd, 'packages') workspace_dir = os.path.join(cwd, 'workspace') @@ -213,8 +214,6 @@ def main(): # check that the copied file is runnable print(subprocess.check_output([os.path.join(output_dir, executable), '-version']).decode('utf-8')) - - # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), From 8c621427c12f1659a389ba29434589a43e7dda30 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 9 Nov 2021 20:17:21 -0600 Subject: [PATCH 20/58] clean up and comments --- build-ffmpeg-descript.py | 47 +++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index e1971d17..cc0025b9 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -8,6 +8,7 @@ (2) Copy or generate dSYM symbol files to the workspace folder (3) Copy executables from the workspace folder and all built dependencies to platform outputfolder (4) Fix dyld ids and loader paths for all built libraries +(5) Zip up the build artifacts ''' import glob @@ -17,13 +18,20 @@ import re import shutil import subprocess +import sys +# +# Constants +# cwd = os.path.dirname(os.path.realpath(__file__)) packages_dir = os.path.join(cwd, 'packages') workspace_dir = os.path.join(cwd, 'workspace') workspace_bin_dir = os.path.join(workspace_dir, 'bin') workspace_lib_dir = os.path.join(workspace_dir, 'lib') +# +# Keep track of which libraries are copied, skipped, or missing +# skipped_libs = set() copied_libs = set() missing_libs = set() @@ -31,9 +39,9 @@ # # builds FFmpeg and logs output to build-ffmpeg.log.txt # -def buildFFmpeg(script_dir, workspace_dir): +def buildFFmpeg(script_dir, log_dir): # create a log file for the build-ffmpeg command for build archival purposes - build_ffmpeg_log_filename = os.path.join(workspace_dir, 'build-ffmpeg.log.txt') + build_ffmpeg_log_filename = os.path.join(log_dir, 'build-ffmpeg.log.txt') os.makedirs(os.path.dirname(build_ffmpeg_log_filename), exist_ok=True) build_ffmpeg_log_file = open('./workspace/build-ffmpeg.log.txt', 'w') @@ -91,7 +99,8 @@ def copyOrGenerateSymbolFiles(source, dest): copyOrGenerateSymbolFile(str(fileref), dest) # -# +# Copies a library and its corresponding .dSYM bundle +# (if present) # def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): dest_file = os.path.join(dest_folder, os.path.basename(src_file)) @@ -111,7 +120,8 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): shutil.copytree(src_symbol_package, dest_symbol_package) # -# +# Helper function to get a base name of a library +# without version numbers # def getFilenameWithoutVersion(file_name) -> str: result = file_name.split('.')[0] @@ -133,6 +143,7 @@ def copyLibraryAndDependencies(src_file, dest_folder): copied_libs.add(src_file) copied_libs.add(dest_file) + # identifier for _this_ library this_id = '' # recursively copy dependencies @@ -152,11 +163,14 @@ def copyLibraryAndDependencies(src_file, dest_folder): src_dependency_file = fixed_path if src_dependency_file.startswith('/usr/local'): + # the build grabbed libraries installed on this machine + # which might not be available on other machines missing_libs.add(src_dependency_file) elif src_dependency_file.startswith(workspace_dir): dependency_name = os.path.basename(src_dependency_file) if not len(this_id): - this_id = dependency_name + # first dependency is the identifier for this library + this_id = dependency_name dest_dependency_path = os.path.join(dest_folder, dependency_name) if not src_dependency_file in copied_libs: if src_dependency_file != dest_dependency_path: @@ -174,21 +188,34 @@ def copyLibraryAndDependencies(src_file, dest_folder): copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) - elif not src_dependency_file.startswith('/'): - breakpoint else: skipped_libs.add(src_dependency_file) # find the non-sym-linked version of this library actual_binary_path = os.path.realpath(dest_file) + # correct the loader path for this library if len(this_id): subprocess.check_output(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path]) + # correct the loader paths for all dependencies if len(loader_paths_to_rewrite) > 0: for loader_path in loader_paths_to_rewrite: subprocess.check_output(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path]) +# +# Read the version string from ./build-ffmpeg +# +def readVersion() -> str: + result = '' + with open(os.path.join(cwd, 'build-ffmpeg')) as f: + lines = f.readlines() + for line in lines: + if line.startswith('SCRIPT_VERSION='): + result = line[15:].strip() + return result + + # # # @@ -198,7 +225,7 @@ def main(): shutil.rmtree(output_dir) os.makedirs(output_dir) - buildFFmpeg(cwd, workspace_dir) + buildFFmpeg(cwd, output_dir) # Generate dSYM files for each built library copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) @@ -228,6 +255,10 @@ def main(): for lib in sorted(missing_libs): print ('[WARNING] missing ' + lib) + # bundle up the build artifacts + os.chdir(output_dir) + subprocess.check_output(['/usr/bin/zip', '--symlinks', '-r', '../ffmpeg-ffprobe-shared-' + sys.platform + '-' + platform.machine() + '.' + readVersion() + '.zip', '.']) + # # entry # From aae01db25959a4bb28bbf73e44b595103af7feff Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 9 Nov 2021 20:17:40 -0600 Subject: [PATCH 21/58] bump version number --- build-ffmpeg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index 3a33ed45..6a156086 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.31 +SCRIPT_VERSION=1.31rc1 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" From e23b1ee2f7e50e7b98435704a5048618157e8d3d Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 9 Nov 2021 20:23:42 -0600 Subject: [PATCH 22/58] fix log folder path --- build-ffmpeg-descript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index cc0025b9..44114a51 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -43,7 +43,7 @@ def buildFFmpeg(script_dir, log_dir): # create a log file for the build-ffmpeg command for build archival purposes build_ffmpeg_log_filename = os.path.join(log_dir, 'build-ffmpeg.log.txt') os.makedirs(os.path.dirname(build_ffmpeg_log_filename), exist_ok=True) - build_ffmpeg_log_file = open('./workspace/build-ffmpeg.log.txt', 'w') + build_ffmpeg_log_file = open(build_ffmpeg_log_filename, 'w') # set environment variables env = os.environ From cff8ef09d89306f874907a44c1d4aab86e2097f7 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 9 Nov 2021 21:03:58 -0600 Subject: [PATCH 23/58] Use consistent logging --- build-ffmpeg-descript.py | 76 +++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 44114a51..4e31f3bc 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -39,12 +39,7 @@ # # builds FFmpeg and logs output to build-ffmpeg.log.txt # -def buildFFmpeg(script_dir, log_dir): - # create a log file for the build-ffmpeg command for build archival purposes - build_ffmpeg_log_filename = os.path.join(log_dir, 'build-ffmpeg.log.txt') - os.makedirs(os.path.dirname(build_ffmpeg_log_filename), exist_ok=True) - build_ffmpeg_log_file = open(build_ffmpeg_log_filename, 'w') - +def buildFFmpeg(script_dir, log_file): # set environment variables env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc @@ -52,17 +47,14 @@ def buildFFmpeg(script_dir, log_dir): # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') - subprocess.call([build_ffmpeg_path, '-b', '--full-shared'], env=env, stdout=build_ffmpeg_log_file) - - # close log file - build_ffmpeg_log_file.close() + subprocess.call([build_ffmpeg_path, '-b', '--full-shared'], env=env, stdout=log_file) # # Copies symbol file to the workspace destination # skips symlinks to avoid duplication # Copies entire dSYM packages for dylib files already within .dSYM packages # -def copyOrGenerateSymbolFile(file, dest): +def copyOrGenerateSymbolFile(file, dest, log_file): fileref = pathlib.Path(file) if not fileref.is_symlink(): symbolFileName = fileref.name + '.dSYM' @@ -85,18 +77,20 @@ def copyOrGenerateSymbolFile(file, dest): # example: # ./packages/libtheora-1.1.1/lib/.libs/libtheora.dylib except ValueError as e: - subprocess.call(['/usr/bin/dsymutil', str(fileref), '-o', destPath]) + args = ['/usr/bin/dsymutil', str(fileref), '-o', destPath] + log_file.write(' '.join(args) + '\n') + subprocess.call(args, stdout=log_file) # # Copies symbol files to the workspace destination # skips symlinks to avoid duplication # Copies entire dSYM packages for dylib files already within .dSYM packages # -def copyOrGenerateSymbolFiles(source, dest): +def copyOrGenerateSymbolFiles(source, dest, log_file): for fileref in pathlib.Path(source + '/').glob('**/*.dylib'): - copyOrGenerateSymbolFile(str(fileref), dest) + copyOrGenerateSymbolFile(str(fileref), dest, log_file) for fileref in pathlib.Path(source + '/').glob('**/*.so*'): - copyOrGenerateSymbolFile(str(fileref), dest) + copyOrGenerateSymbolFile(str(fileref), dest, log_file) # # Copies a library and its corresponding .dSYM bundle @@ -134,7 +128,7 @@ def getFilenameWithoutVersion(file_name) -> str: # Recursive function to copy a library and its (non-system) dependencies # also fixes loader paths for each library # -def copyLibraryAndDependencies(src_file, dest_folder): +def copyLibraryAndDependencies(src_file, dest_folder, log_file): dest_file = os.path.join(dest_folder, os.path.basename(src_file)) @@ -147,7 +141,8 @@ def copyLibraryAndDependencies(src_file, dest_folder): this_id = '' # recursively copy dependencies - otool_proc = subprocess.Popen(['/usr/bin/otool', '-L', src_file], stdout=subprocess.PIPE) + args = ['/usr/bin/otool', '-L', src_file] + otool_proc = subprocess.Popen(args, stdout=subprocess.PIPE) loader_paths_to_rewrite = [] for line in otool_proc.stdout: ln = line.decode('utf-8').strip() @@ -185,7 +180,7 @@ def copyLibraryAndDependencies(src_file, dest_folder): # RECURSIVELY copy dependencies if (os.path.exists(unversioned_dependency_base_name + '.dylib')): - copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder) + copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder, log_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: @@ -196,12 +191,16 @@ def copyLibraryAndDependencies(src_file, dest_folder): # correct the loader path for this library if len(this_id): - subprocess.check_output(['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path]) + args = ['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path] + log_file.write(' '.join(args) + '\n') + subprocess.check_output(args) # correct the loader paths for all dependencies if len(loader_paths_to_rewrite) > 0: for loader_path in loader_paths_to_rewrite: - subprocess.check_output(['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path]) + args = ['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path] + log_file.write(' '.join(args) + '\n') + subprocess.check_output(args) # # Read the version string from ./build-ffmpeg @@ -225,35 +224,54 @@ def main(): shutil.rmtree(output_dir) os.makedirs(output_dir) - buildFFmpeg(cwd, output_dir) + # create a log file for the build-ffmpeg command for build archival purposes + build_ffmpeg_log_file_path = os.path.join(output_dir, 'build-ffmpeg.log.txt') + build_ffmpeg_log_file = open(build_ffmpeg_log_file_path, 'w') + + # Run the script + buildFFmpeg(cwd, build_ffmpeg_log_file) # Generate dSYM files for each built library - copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir) + build_ffmpeg_log_file.write('\nGenerating Symbols\n') + build_ffmpeg_log_file.write('=======================\n') + copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir, build_ffmpeg_log_file) # Generate dSYM files for each executable # and copy their dependencies executables = ['ffmpeg', 'ffprobe'] for executable in executables: + build_ffmpeg_log_file.write('\nCopying & Linking ' + executable + '\n') + build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) - copyOrGenerateSymbolFile(executable_path, workspace_bin_dir) - copyLibraryAndDependencies(executable_path, output_dir) + copyOrGenerateSymbolFile(executable_path, workspace_bin_dir, build_ffmpeg_log_file) + copyLibraryAndDependencies(executable_path, output_dir, build_ffmpeg_log_file) # check that the copied file is runnable - print(subprocess.check_output([os.path.join(output_dir, executable), '-version']).decode('utf-8')) + build_ffmpeg_log_file.write('\nChecking ' + executable + '\n') + build_ffmpeg_log_file.write('=======================\n') + args = [os.path.join(output_dir, executable), '-version'] + build_ffmpeg_log_file.write(' '.join(args) + '\n') + output = subprocess.check_output(args) + build_ffmpeg_log_file.write(output.decode('utf-8')) # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), os.path.join(output_dir, 'include')) + build_ffmpeg_log_file.write('\nLibrary Info\n') + build_ffmpeg_log_file.write('=======================\n') + + for lib in sorted(missing_libs): + build_ffmpeg_log_file.write('[WARNING] missing ' + lib + '\n') + for lib in sorted(skipped_libs): - print ('[NOTE] skipped ' + lib) + build_ffmpeg_log_file.write('[NOTE] skipped ' + lib + '\n') for lib in sorted(copied_libs): - print ('Copied ' + lib) + build_ffmpeg_log_file.write('Copied ' + lib + '\n') - for lib in sorted(missing_libs): - print ('[WARNING] missing ' + lib) + build_ffmpeg_log_file.close() # bundle up the build artifacts os.chdir(output_dir) From d588fc088f39f9acef0aad63bfb8cbd919efa696 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Tue, 9 Nov 2021 21:42:06 -0600 Subject: [PATCH 24/58] disable pcre - ftp is offline --- build-ffmpeg | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 6a156086..8ed60008 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -724,18 +724,24 @@ if command_exists "python3"; then execute ./waf install build_done "serd" "0.30.10" fi - if build "pcre" "8.44"; then - download "https://ftp.pcre.org/pub/pcre/pcre-8.44.tar.gz" "pcre-8.44.tar.gz" - if $SHARED_LIBRARIES; then - execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static - else - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static - fi - execute make -j $MJOBS - execute make install - build_done "pcre" "8.44" + # [ Descript - cvanwink ] + # FTP for pcre looks to be down + if false; then + if build "pcre" "8.44"; then + download "https://ftp.pcre.org/pub/pcre/pcre-8.44.tar.gz" "pcre-8.44.tar.gz" + if $SHARED_LIBRARIES; then + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + else + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static + fi + execute make -j $MJOBS + execute make install + + build_done "pcre" "8.44" + fi fi + if build "sord" "0.16.8"; then download "https://gitlab.com/drobilla/sord/-/archive/v0.16.8/sord-v0.16.8.tar.gz" "sord-v0.16.8.tar.gz" execute cp -r "${PACKAGES}"/autowaf/* "${PACKAGES}/sord-v0.16.8/waflib/" From 358b3ddf89720ee1adf4978fc79342e1fe6e99fc Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 10 Nov 2021 12:29:40 -0600 Subject: [PATCH 25/58] Add GPL-only option and includes some libraries beamcoder needs libpostproc --- build-ffmpeg | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 8ed60008..b046c32a 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -18,6 +18,7 @@ CONFIGURE_OPTIONS=() NONFREE_AND_GPL=false LATEST=false SHARED_LIBRARIES=false +FREE_AND_GPL=false # Check for Apple Silicon if [[ ("$(uname -m)" == "arm64") && ("$OSTYPE" == "darwin"*) ]]; then @@ -205,6 +206,7 @@ usage() { echo " --full-static Build a full static FFmpeg binary (eg. glibc, pthreads etc...) **only Linux**" echo " Note: Because of the NSS (Name Service Switch), glibc does not recommend static links." echo " --full-shared Build all shared libraries. Cannot be used with --full-static" + echo " --enable-gpl-and-free Enable GPL but not non-free codecs. Cannot be used with --enable-gpl-and-non-free - https://ffmpeg.org/legal.html" echo "" } @@ -248,6 +250,10 @@ while (($# > 0)); do if [[ "$1" == "--full-shared" ]]; then SHARED_LIBRARIES=true fi + if [[ "$1" == "--enable-gpl-and-free" ]]; then + CONFIGURE_OPTIONS+=("--enable-gpl") + FREE_AND_GPL=true + fi shift ;; *) @@ -270,6 +276,11 @@ if [[ ($SHARED_LIBRARIES == true) && ($LDEXEFLAGS == true) ]]; then exit 1 fi +if [[ ($NONFREE_AND_GPL == true) && ($FREE_AND_GPL == true) ]]; then + usage + exit 1 +fi + echo "Using $MJOBS make jobs simultaneously." if $NONFREE_AND_GPL; then @@ -284,6 +295,10 @@ if $SHARED_LIBRARIES; then echo "Building shared libraries." fi +if $FREE_AND_GPL; then + echo "GPL and free codecs" +fi + mkdir -p "$PACKAGES" mkdir -p "$WORKSPACE" @@ -491,7 +506,7 @@ if command_exists "cargo"; then CONFIGURE_OPTIONS+=("--enable-librav1e") fi -if $NONFREE_AND_GPL; then +if $NONFREE_AND_GPL || $FREE_AND_GPL; then if build "x264" "5db6aa6"; then download "https://code.videolan.org/videolan/x264/-/archive/5db6aa6cab1b146e07b60cc1736a01f21da01154/x264-5db6aa6cab1b146e07b60cc1736a01f21da01154.tar.gz" "x264-5db6aa6.tar.gz" @@ -524,7 +539,7 @@ if $NONFREE_AND_GPL; then CONFIGURE_OPTIONS+=("--enable-libx264") fi -if $NONFREE_AND_GPL; then +if $NONFREE_AND_GPL || $FREE_AND_GPL; then if build "x265" "3.5"; then download "https://github.com/videolan/x265/archive/Release_3.5.tar.gz" "x265-3.5.tar.gz" # This is actually 3.4 if looking at x265Version.txt cd build/linux || exit @@ -606,7 +621,7 @@ if false; then CONFIGURE_OPTIONS+=("--enable-libvpx") fi -if $NONFREE_AND_GPL; then +if $NONFREE_AND_GPL || $FREE_AND_GPL; then if build "xvidcore" "1.3.7"; then download "https://downloads.xvid.com/downloads/xvidcore-1.3.7.tar.gz" cd build/generic || exit From 7554b2df528fb4e06c638322e62283ea42cec12c Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 10 Nov 2021 12:30:15 -0600 Subject: [PATCH 26/58] Archive Source and keep separate --- build-ffmpeg-descript.py | 44 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 4e31f3bc..8f5a465e 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -19,6 +19,7 @@ import shutil import subprocess import sys +from zipfile import ZipFile # # Constants @@ -47,7 +48,13 @@ def buildFFmpeg(script_dir, log_file): # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') - subprocess.call([build_ffmpeg_path, '-b', '--full-shared'], env=env, stdout=log_file) + args = [ + build_ffmpeg_path, + '-b', # build + '--full-shared', # custom Descript shim to build shared libraries instead of static + '--enable-gpl-and-free'] # custom Descript shim to build GPL but not non-free (libpostproc is needed by Beamcoder and requires GPL) + log_file.write(' '.join(args) + '\n\n') + subprocess.call(args, env=env, stdout=log_file) # # Copies symbol file to the workspace destination @@ -214,6 +221,11 @@ def readVersion() -> str: result = line[15:].strip() return result +# +# Returns a string like darwin-x86_64.1.31rc2 +# +def getPlatformMachineVersion() -> str: + return sys.platform + '-' + platform.machine() + '.' + readVersion() # # @@ -225,9 +237,13 @@ def main(): os.makedirs(output_dir) # create a log file for the build-ffmpeg command for build archival purposes - build_ffmpeg_log_file_path = os.path.join(output_dir, 'build-ffmpeg.log.txt') + log_file_name = 'build-ffmpeg-' + getPlatformMachineVersion() + '.log.txt' + build_ffmpeg_log_file_path = os.path.join(os.path.dirname(output_dir), log_file_name) build_ffmpeg_log_file = open(build_ffmpeg_log_file_path, 'w') + build_ffmpeg_log_file.write('Begin build-ffmpeg-descript.py\n') + build_ffmpeg_log_file.write('=======================\n') + # Run the script buildFFmpeg(cwd, build_ffmpeg_log_file) @@ -271,11 +287,31 @@ def main(): for lib in sorted(copied_libs): build_ffmpeg_log_file.write('Copied ' + lib + '\n') - build_ffmpeg_log_file.close() + build_ffmpeg_log_file.write('\nArchiving third-party source\n') + build_ffmpeg_log_file.write('=======================\n') + + # bundle up the third-party source + # grab each .tar.* from the packages folder + packages_zip_name = '-'.join(executables) + '-packages-' + getPlatformMachineVersion() + '.zip' + with ZipFile(os.path.join(os.path.dirname(output_dir), packages_zip_name), 'w') as myzip: + archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') + for archive in sorted(archives, key=lambda s: str(s).lower()): + build_ffmpeg_log_file.write(os.path.join('packages', archive.name) + '\n') + myzip.write(str(archive.absolute()), archive.name) + + build_ffmpeg_log_file.write('\nArchiving libraries\n') + build_ffmpeg_log_file.write('=======================\n') # bundle up the build artifacts os.chdir(output_dir) - subprocess.check_output(['/usr/bin/zip', '--symlinks', '-r', '../ffmpeg-ffprobe-shared-' + sys.platform + '-' + platform.machine() + '.' + readVersion() + '.zip', '.']) + shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' + args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] + build_ffmpeg_log_file.write(' '.join(args)) + subprocess.check_output(args) + + build_ffmpeg_log_file.write('\nEnd of build-ffmpeg-descript.py\n') + build_ffmpeg_log_file.write('=======================\n') + build_ffmpeg_log_file.close() # # entry From e6d6923b40aa8dfd307df02d0edfba9604a34a94 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 10 Nov 2021 13:58:49 -0600 Subject: [PATCH 27/58] Revert changes to x265, no known way to link these as a shared library --- build-ffmpeg | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index b046c32a..b0994794 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -539,34 +539,22 @@ if $NONFREE_AND_GPL || $FREE_AND_GPL; then CONFIGURE_OPTIONS+=("--enable-libx264") fi -if $NONFREE_AND_GPL || $FREE_AND_GPL; then +if $NONFREE_AND_GPL && !SHARED_LIBRARIES; then if build "x265" "3.5"; then download "https://github.com/videolan/x265/archive/Release_3.5.tar.gz" "x265-3.5.tar.gz" # This is actually 3.4 if looking at x265Version.txt cd build/linux || exit rm -rf 8bit 10bit 12bit 2>/dev/null mkdir -p 8bit 10bit 12bit cd 12bit || exit - if $SHARED_LIBRARIES; then - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=ON -DBUILD_SHARED_LIBS=ON -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON - else - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON - fi + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON execute make -j $MJOBS cd ../10bit || exit - if $SHARED_LIBRARIES; then - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=ON -DBUILD_SHARED_LIBS=ON -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF - else - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF - fi + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF execute make -j $MJOBS cd ../8bit || exit ln -sf ../10bit/libx265.a libx265_main10.a ln -sf ../12bit/libx265.a libx265_main12.a - if $SHARED_LIBRARIES; then - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=ON -DBUILD_SHARED_LIBS=ON -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON - else - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON - fi + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON execute make -j $MJOBS mv libx265.a libx265_main.a From d13a52476a6f320929c440d81fb6b3d9b05bc92d Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Wed, 10 Nov 2021 14:34:19 -0600 Subject: [PATCH 28/58] add one more newline --- build-ffmpeg-descript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 8f5a465e..ebbb9d4b 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -306,7 +306,7 @@ def main(): os.chdir(output_dir) shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] - build_ffmpeg_log_file.write(' '.join(args)) + build_ffmpeg_log_file.write(' '.join(args) + '\n') subprocess.check_output(args) build_ffmpeg_log_file.write('\nEnd of build-ffmpeg-descript.py\n') From a307443f63f99bcdc6d8fcac7a490be9c32326a8 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 10 Nov 2021 14:42:21 -0600 Subject: [PATCH 29/58] cherry-pick fix for x265 shared --- build-ffmpeg | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index b0994794..33548668 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -539,30 +539,34 @@ if $NONFREE_AND_GPL || $FREE_AND_GPL; then CONFIGURE_OPTIONS+=("--enable-libx264") fi -if $NONFREE_AND_GPL && !SHARED_LIBRARIES; then +if $NONFREE_AND_GPL || $FREE_AND_GPL; then if build "x265" "3.5"; then download "https://github.com/videolan/x265/archive/Release_3.5.tar.gz" "x265-3.5.tar.gz" # This is actually 3.4 if looking at x265Version.txt cd build/linux || exit - rm -rf 8bit 10bit 12bit 2>/dev/null - mkdir -p 8bit 10bit 12bit - cd 12bit || exit - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON - execute make -j $MJOBS - cd ../10bit || exit - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF - execute make -j $MJOBS - cd ../8bit || exit - ln -sf ../10bit/libx265.a libx265_main10.a - ln -sf ../12bit/libx265.a libx265_main12.a - execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON - execute make -j $MJOBS + if $SHARED_LIBRARIES; then + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=on -DBUILD_SHARED_LIBS=on ../../source + execute make -j $MJOBS + else + rm -rf 8bit 10bit 12bit 2>/dev/null + mkdir -p 8bit 10bit 12bit + cd 12bit || exit + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF -DMAIN12=ON + execute make -j $MJOBS + cd ../10bit || exit + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DHIGH_BIT_DEPTH=ON -DENABLE_HDR10_PLUS=ON -DEXPORT_C_API=OFF -DENABLE_CLI=OFF + execute make -j $MJOBS + cd ../8bit || exit + ln -sf ../10bit/libx265.a libx265_main10.a + ln -sf ../12bit/libx265.a libx265_main12.a + execute cmake ../../../source -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DENABLE_SHARED=OFF -DBUILD_SHARED_LIBS=OFF -DEXTRA_LIB="x265_main10.a;x265_main12.a;-ldl" -DEXTRA_LINK_FLAGS=-L. -DLINKED_10BIT=ON -DLINKED_12BIT=ON + execute make -j $MJOBS - mv libx265.a libx265_main.a + mv libx265.a libx265_main.a - if [[ "$OSTYPE" == "darwin"* ]]; then - execute "${MACOS_LIBTOOL}" -static -o libx265.a libx265_main.a libx265_main10.a libx265_main12.a 2>/dev/null - else - execute ar -M </dev/null + else + execute ar -M < Date: Fri, 12 Nov 2021 09:55:43 -0600 Subject: [PATCH 30/58] Set 10.11 deployment target --- build-ffmpeg-descript.py | 1 + 1 file changed, 1 insertion(+) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index ebbb9d4b..2ad34988 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -45,6 +45,7 @@ def buildFFmpeg(script_dir, log_file): env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc env['VERBOSE'] = 'yes' + env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') From 5af67732eed781b88c37819c14d2e19b7c1b8813 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 12 Nov 2021 11:11:23 -0600 Subject: [PATCH 31/58] conditionalize 10.11 target to x86_64 --- build-ffmpeg-descript.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 2ad34988..05fdeb81 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -45,7 +45,8 @@ def buildFFmpeg(script_dir, log_file): env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc env['VERBOSE'] = 'yes' - env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' + if platform.machine() == 'x86_64': + env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') From 5210332bda02988e05779f6afad8337750d49b56 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Sun, 14 Nov 2021 22:11:47 -0600 Subject: [PATCH 32/58] Fix deployment target --- build-ffmpeg | 4 ++-- build-ffmpeg-descript.py | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 33548668..ced284fb 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -869,7 +869,7 @@ if build "libtheora" "1.1.1"; then chmod +x configure.patched mv configure.patched configure if $SHARED_LIBRARIES; then - if $MACOSX_M1; then + if $MACOS_M1; then # fix Libtool bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63610 export MACOSX_DEPLOYMENT_TARGET=10.11 fi @@ -881,7 +881,7 @@ if build "libtheora" "1.1.1"; then execute make install if $SHARED_LIBRARIES; then - if $MACOSX_M1; then + if $MACOS_M1; then # restore from Line 26 export MACOSX_DEPLOYMENT_TARGET=11.0 fi diff --git a/build-ffmpeg-descript.py b/build-ffmpeg-descript.py index 05fdeb81..42d20616 100755 --- a/build-ffmpeg-descript.py +++ b/build-ffmpeg-descript.py @@ -29,6 +29,7 @@ workspace_dir = os.path.join(cwd, 'workspace') workspace_bin_dir = os.path.join(workspace_dir, 'bin') workspace_lib_dir = os.path.join(workspace_dir, 'lib') +deployment_target = '11.0' if platform.machine() == 'arm64' else '10.11' # # Keep track of which libraries are copied, skipped, or missing @@ -45,8 +46,7 @@ def buildFFmpeg(script_dir, log_file): env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc env['VERBOSE'] = 'yes' - if platform.machine() == 'x86_64': - env['MACOSX_DEPLOYMENT_TARGET'] = '10.11' + env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target # call main build script build_ffmpeg_path = os.path.join(script_dir, 'build-ffmpeg') @@ -101,11 +101,31 @@ def copyOrGenerateSymbolFiles(source, dest, log_file): for fileref in pathlib.Path(source + '/').glob('**/*.so*'): copyOrGenerateSymbolFile(str(fileref), dest, log_file) +def readDeploymentTarget(src_file) -> str: + args = ['/usr/bin/otool', '-l', src_file] + otool_proc = subprocess.Popen(args, stdout=subprocess.PIPE) + inLoaderCommand = False + for line in otool_proc.stdout: + ln = line.decode('utf-8').strip() + if inLoaderCommand: + if ln.startswith('minos') or ln.startswith('version'): + return ln.split(' ')[1] + if ln.startswith('sdk'): + continue + elif 'LC_VERSION_MIN_MACOSX' in ln or 'LC_BUILD_VERSION' in ln: + inLoaderCommand = True + + return '' + + # # Copies a library and its corresponding .dSYM bundle # (if present) # def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): + this_deployment_target = readDeploymentTarget(src_file) + assert this_deployment_target == deployment_target, '{0} wrong deployment target {1}'.format(src_file, this_deployment_target) + dest_file = os.path.join(dest_folder, os.path.basename(src_file)) # copy file From e7a117a2ca31999b3cbe65d96dd388ef55060214 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Thu, 2 Dec 2021 23:05:58 -0600 Subject: [PATCH 33/58] Move `build-ffmpeg-descript.py` to a `descript` subfolder --- descript/.gitignore | 1 + .../build-ffmpeg-descript.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 descript/.gitignore rename build-ffmpeg-descript.py => descript/build-ffmpeg-descript.py (97%) diff --git a/descript/.gitignore b/descript/.gitignore new file mode 100644 index 00000000..28757cf4 --- /dev/null +++ b/descript/.gitignore @@ -0,0 +1 @@ +mac \ No newline at end of file diff --git a/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py similarity index 97% rename from build-ffmpeg-descript.py rename to descript/build-ffmpeg-descript.py index 42d20616..f70ca163 100755 --- a/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -25,8 +25,9 @@ # Constants # cwd = os.path.dirname(os.path.realpath(__file__)) -packages_dir = os.path.join(cwd, 'packages') -workspace_dir = os.path.join(cwd, 'workspace') +base_dir = pathlib.Path(cwd).parent.absolute() +packages_dir = os.path.join(base_dir, 'packages') +workspace_dir = os.path.join(base_dir, 'workspace') workspace_bin_dir = os.path.join(workspace_dir, 'bin') workspace_lib_dir = os.path.join(workspace_dir, 'lib') deployment_target = '11.0' if platform.machine() == 'arm64' else '10.11' @@ -236,7 +237,7 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): # def readVersion() -> str: result = '' - with open(os.path.join(cwd, 'build-ffmpeg')) as f: + with open(os.path.join(base_dir, 'build-ffmpeg')) as f: lines = f.readlines() for line in lines: if line.startswith('SCRIPT_VERSION='): @@ -253,7 +254,7 @@ def getPlatformMachineVersion() -> str: # # def main(): - output_dir = os.path.join(workspace_dir, 'mac', platform.machine()) + output_dir = os.path.join(cwd, 'mac', platform.machine()) if os.path.exists(output_dir): shutil.rmtree(output_dir) os.makedirs(output_dir) @@ -267,7 +268,7 @@ def main(): build_ffmpeg_log_file.write('=======================\n') # Run the script - buildFFmpeg(cwd, build_ffmpeg_log_file) + buildFFmpeg(base_dir, build_ffmpeg_log_file) # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') From 1d395f8957bfc97927df35964b42ba5d5dc688ec Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Thu, 2 Dec 2021 23:45:44 -0600 Subject: [PATCH 34/58] Improve bundling to match new features from Descript electron build script --- descript/build-ffmpeg-descript.py | 69 +++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index f70ca163..96bc634c 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -19,7 +19,7 @@ import shutil import subprocess import sys -from zipfile import ZipFile +import zipfile # # Constants @@ -250,25 +250,58 @@ def readVersion() -> str: def getPlatformMachineVersion() -> str: return sys.platform + '-' + platform.machine() + '.' + readVersion() + +# +# +# +def generateChecksum(output_folder): + """ + Calculates checksums for every file in `output_folder` + """ + + checksums = set() + + # calculate checksums for all files + for (dirpath, dirnames, filenames) in os.walk(output_folder): + for file in filenames: + args = ['shasum', '-a', '256', os.path.join(dirpath, file)] + output = subprocess.check_output(args) + checksum = output.decode('utf-8').strip() + + # replace absolute path to just filename + # From: '0a88d3f97f356c6a42449fd548f9b586f565899144849019014e36c7683b745e /Users/cvanwink/Source/git/electron/src/out/Testing/dist.zip' + # To: '0a88d3f97f356c6a42449fd548f9b586f565899144849019014e36c7683b745e *electron-v13.1.6-darwin-x64.zip' + checksum = checksum.replace(os.path.join(dirpath, ''), '*') + checksums.add(checksum) + break + + # Write Checksums to file + checksum_file_path = os.path.join(output_folder, 'SHAMSUM256.txt') + checksum_file = open(checksum_file_path, 'w') + for checksum in checksums: + checksum_file.write(f'{checksum}\n') + checksum_file.close() + # # # def main(): - output_dir = os.path.join(cwd, 'mac', platform.machine()) + output_dir = os.path.join(cwd, 'mac') if os.path.exists(output_dir): shutil.rmtree(output_dir) - os.makedirs(output_dir) + temp_dir = os.path.join(output_dir, platform.machine()) + os.makedirs(temp_dir) # create a log file for the build-ffmpeg command for build archival purposes log_file_name = 'build-ffmpeg-' + getPlatformMachineVersion() + '.log.txt' - build_ffmpeg_log_file_path = os.path.join(os.path.dirname(output_dir), log_file_name) - build_ffmpeg_log_file = open(build_ffmpeg_log_file_path, 'w') + log_file_path = os.path.join(output_dir, log_file_name) + build_ffmpeg_log_file = open(log_file_path, 'w') build_ffmpeg_log_file.write('Begin build-ffmpeg-descript.py\n') build_ffmpeg_log_file.write('=======================\n') # Run the script - buildFFmpeg(base_dir, build_ffmpeg_log_file) + #buildFFmpeg(base_dir, build_ffmpeg_log_file) # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') @@ -283,12 +316,12 @@ def main(): build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) copyOrGenerateSymbolFile(executable_path, workspace_bin_dir, build_ffmpeg_log_file) - copyLibraryAndDependencies(executable_path, output_dir, build_ffmpeg_log_file) + copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file) # check that the copied file is runnable build_ffmpeg_log_file.write('\nChecking ' + executable + '\n') build_ffmpeg_log_file.write('=======================\n') - args = [os.path.join(output_dir, executable), '-version'] + args = [os.path.join(temp_dir, executable), '-version'] build_ffmpeg_log_file.write(' '.join(args) + '\n') output = subprocess.check_output(args) build_ffmpeg_log_file.write(output.decode('utf-8')) @@ -296,7 +329,7 @@ def main(): # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), - os.path.join(output_dir, 'include')) + os.path.join(temp_dir, 'include')) build_ffmpeg_log_file.write('\nLibrary Info\n') build_ffmpeg_log_file.write('=======================\n') @@ -315,8 +348,9 @@ def main(): # bundle up the third-party source # grab each .tar.* from the packages folder - packages_zip_name = '-'.join(executables) + '-packages-' + getPlatformMachineVersion() + '.zip' - with ZipFile(os.path.join(os.path.dirname(output_dir), packages_zip_name), 'w') as myzip: + shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' + packages_zip_name = f'{pathlib.Path(shared_zip_name).stem}-packages.zip' + with zipfile.ZipFile(os.path.join(output_dir, packages_zip_name), 'w', zipfile.ZIP_DEFLATED) as myzip: archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') for archive in sorted(archives, key=lambda s: str(s).lower()): build_ffmpeg_log_file.write(os.path.join('packages', archive.name) + '\n') @@ -326,16 +360,25 @@ def main(): build_ffmpeg_log_file.write('=======================\n') # bundle up the build artifacts - os.chdir(output_dir) - shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' + os.chdir(temp_dir) + dest_file = os.path.join(output_dir, shared_zip_name) args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] build_ffmpeg_log_file.write(' '.join(args) + '\n') subprocess.check_output(args) + + shutil.rmtree(temp_dir) build_ffmpeg_log_file.write('\nEnd of build-ffmpeg-descript.py\n') build_ffmpeg_log_file.write('=======================\n') build_ffmpeg_log_file.close() + # zip up log file + with zipfile.ZipFile(os.path.splitext(log_file_path)[0] + '.zip', 'w', zipfile.ZIP_DEFLATED) as myzip: + myzip.write(log_file_path, os.path.basename(log_file_path)) + os.remove(log_file_path) + + generateChecksum(output_dir) + # # entry # From 13ee124567252a9b98f21207e82346ce8440e20c Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Thu, 2 Dec 2021 23:48:24 -0600 Subject: [PATCH 35/58] Remove change which was mistakenly staged and committed --- descript/build-ffmpeg-descript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 96bc634c..669b9858 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -6,7 +6,7 @@ (1) Call build-ffmpeg with the build command (2) Copy or generate dSYM symbol files to the workspace folder -(3) Copy executables from the workspace folder and all built dependencies to platform outputfolder +(3) Copy executables from the workspace folder and all built dependencies to platform output folder (4) Fix dyld ids and loader paths for all built libraries (5) Zip up the build artifacts ''' @@ -301,7 +301,7 @@ def main(): build_ffmpeg_log_file.write('=======================\n') # Run the script - #buildFFmpeg(base_dir, build_ffmpeg_log_file) + buildFFmpeg(base_dir, build_ffmpeg_log_file) # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') From f2bc24f77b4b8f3b0cdfde5019daf8a5d388dbae Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 10:31:18 -0600 Subject: [PATCH 36/58] Separate symbols into their own zip file --- descript/build-ffmpeg-descript.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 669b9858..f59df371 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -291,9 +291,13 @@ def main(): shutil.rmtree(output_dir) temp_dir = os.path.join(output_dir, platform.machine()) os.makedirs(temp_dir) + symbol_temp_dir = os.path.join(output_dir, platform.machine() + '-symbols') + + executables = ['ffmpeg', 'ffprobe'] + base_artifact_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() # create a log file for the build-ffmpeg command for build archival purposes - log_file_name = 'build-ffmpeg-' + getPlatformMachineVersion() + '.log.txt' + log_file_name = base_artifact_name + '-log.txt' log_file_path = os.path.join(output_dir, log_file_name) build_ffmpeg_log_file = open(log_file_path, 'w') @@ -306,11 +310,13 @@ def main(): # Generate dSYM files for each built library build_ffmpeg_log_file.write('\nGenerating Symbols\n') build_ffmpeg_log_file.write('=======================\n') - copyOrGenerateSymbolFiles(packages_dir, workspace_lib_dir, build_ffmpeg_log_file) + copyOrGenerateSymbolFiles(packages_dir, symbol_temp_dir, build_ffmpeg_log_file) + symbol_file_name = base_artifact_name + '-symbols' + shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) + shutil.rmtree(symbol_temp_dir) # Generate dSYM files for each executable # and copy their dependencies - executables = ['ffmpeg', 'ffprobe'] for executable in executables: build_ffmpeg_log_file.write('\nCopying & Linking ' + executable + '\n') build_ffmpeg_log_file.write('=======================\n') @@ -348,8 +354,7 @@ def main(): # bundle up the third-party source # grab each .tar.* from the packages folder - shared_zip_name = '-'.join(executables) + '-shared-' + getPlatformMachineVersion() + '.zip' - packages_zip_name = f'{pathlib.Path(shared_zip_name).stem}-packages.zip' + packages_zip_name = base_artifact_name + '-packages.zip' with zipfile.ZipFile(os.path.join(output_dir, packages_zip_name), 'w', zipfile.ZIP_DEFLATED) as myzip: archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') for archive in sorted(archives, key=lambda s: str(s).lower()): @@ -361,6 +366,7 @@ def main(): # bundle up the build artifacts os.chdir(temp_dir) + shared_zip_name = base_artifact_name + '.zip' dest_file = os.path.join(output_dir, shared_zip_name) args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] build_ffmpeg_log_file.write(' '.join(args) + '\n') From 606d18db1d65f1fd5bd05044b93c4e6e49b17baa Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 10:41:20 -0600 Subject: [PATCH 37/58] Add basic README file for Descript folder --- descript/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 descript/README.md diff --git a/descript/README.md b/descript/README.md new file mode 100644 index 00000000..70c884e1 --- /dev/null +++ b/descript/README.md @@ -0,0 +1,30 @@ +# Build FFmpeg for Descript + +## Author / Contact: + - [Charles Van Winkle](https://github.com/cvanwinkle) + - [Steve Rubin](https://github.com/srubin) + +## Instructions +- Run `build-ffmpeg-descript.py` + + +## Build Overview +The build script automates the following basic operations. +- Creates a log file to archive the compiler/linker and packaging steps +- Runs the modified `buildFFmpeg` shell script, outputs to log file +- Recursively generates or copies `.dSYM` symbol files for each dependency into a `.zip` file + - Also fixes `dlyd` loader paths for each dependency +- Checks that each executable (i.e. `ffmpeg` & `ffprobe`) are runnable +- Copies `includes` header folder +- Checks for any linked dependencies which are linked to locations on the build machine and not present in the archive bundle +- Archives the upstream tar bundles for each `ffmpeg` component + - This is important in case upstream FTP or source servers go offline in the future +- Generates checksum for each created artifact (build, symbols, packages, log) + +## Development +Known issues: +- . + +## Patches + + From 58238849b64d90807946eee4db96700d435c446e Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 10:54:05 -0600 Subject: [PATCH 38/58] Switch comments style to match docstring format --- descript/build-ffmpeg-descript.py | 73 ++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index f59df371..48daab1e 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -40,9 +40,12 @@ missing_libs = set() # -# builds FFmpeg and logs output to build-ffmpeg.log.txt +# # def buildFFmpeg(script_dir, log_file): + """ + builds FFmpeg and logs output to `log_file` + """ # set environment variables env = os.environ env['SKIPINSTALL'] = 'yes' # append 'SKIPINSTALL=yes' to skip prompt for installing FFmpeg to /usr/local/bin/etc @@ -60,11 +63,14 @@ def buildFFmpeg(script_dir, log_file): subprocess.call(args, env=env, stdout=log_file) # -# Copies symbol file to the workspace destination -# skips symlinks to avoid duplication -# Copies entire dSYM packages for dylib files already within .dSYM packages +# # def copyOrGenerateSymbolFile(file, dest, log_file): + """ + Copies a single symbol file to the workspace destination + skips symlinks to avoid duplication + Copies entire `dSYM` packages for `dylib` files already within `.dSYM` packages + """ fileref = pathlib.Path(file) if not fileref.is_symlink(): symbolFileName = fileref.name + '.dSYM' @@ -92,17 +98,27 @@ def copyOrGenerateSymbolFile(file, dest, log_file): subprocess.call(args, stdout=log_file) # -# Copies symbol files to the workspace destination -# skips symlinks to avoid duplication -# Copies entire dSYM packages for dylib files already within .dSYM packages +# # def copyOrGenerateSymbolFiles(source, dest, log_file): + """ + Recursively copies symbol files to the workspace destination + skips symlinks to avoid duplication + Copies entire `dSYM` packages for `dylib` files already within `.dSYM` packages + """ for fileref in pathlib.Path(source + '/').glob('**/*.dylib'): copyOrGenerateSymbolFile(str(fileref), dest, log_file) for fileref in pathlib.Path(source + '/').glob('**/*.so*'): copyOrGenerateSymbolFile(str(fileref), dest, log_file) +# +# +# def readDeploymentTarget(src_file) -> str: + """ + Reads the deployment target of a binary + :return: something like `'10.11'` or an empty string + """ args = ['/usr/bin/otool', '-l', src_file] otool_proc = subprocess.Popen(args, stdout=subprocess.PIPE) inLoaderCommand = False @@ -120,10 +136,13 @@ def readDeploymentTarget(src_file) -> str: # -# Copies a library and its corresponding .dSYM bundle -# (if present) +# # def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): + """ + Copies a library and its corresponding `.dSYM` bundle + (if present) + """ this_deployment_target = readDeploymentTarget(src_file) assert this_deployment_target == deployment_target, '{0} wrong deployment target {1}'.format(src_file, this_deployment_target) @@ -144,21 +163,26 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): shutil.copytree(src_symbol_package, dest_symbol_package) # -# Helper function to get a base name of a library -# without version numbers +# # def getFilenameWithoutVersion(file_name) -> str: - result = file_name.split('.')[0] - # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if 'libSDL2' in result: - result = 'libSDL2' - return result + """ + :return: `'libSDL2'` for something like `'libSDL2-2.0.0.dylib'` + """ + result = file_name.split('.')[0] + # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) + if 'libSDL2' in result: + result = 'libSDL2' + return result # -# Recursive function to copy a library and its (non-system) dependencies -# also fixes loader paths for each library +# # def copyLibraryAndDependencies(src_file, dest_folder, log_file): + """ + Recursive function to copy a library and its (non-system) dependencies + also fixes loader paths for each library to be `@loader_path` + """ dest_file = os.path.join(dest_folder, os.path.basename(src_file)) @@ -233,9 +257,13 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): subprocess.check_output(args) # -# Read the version string from ./build-ffmpeg +# # def readVersion() -> str: + """ + Reads the version string from ../build-ffmpeg + :return: something like `'1.31rc1'` + """ result = '' with open(os.path.join(base_dir, 'build-ffmpeg')) as f: lines = f.readlines() @@ -245,9 +273,12 @@ def readVersion() -> str: return result # -# Returns a string like darwin-x86_64.1.31rc2 +# # def getPlatformMachineVersion() -> str: + """ + :return: a string like `'darwin-x86_64.1.31rc2'` + """ return sys.platform + '-' + platform.machine() + '.' + readVersion() @@ -257,8 +288,8 @@ def getPlatformMachineVersion() -> str: def generateChecksum(output_folder): """ Calculates checksums for every file in `output_folder` + and puts it in a `SHAMSUM256.txt` file """ - checksums = set() # calculate checksums for all files From ccfaa1652ce21e0d65b6fed4d0e0302c6338279e Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 20:34:58 -0600 Subject: [PATCH 39/58] More copy fixes --- descript/build-ffmpeg-descript.py | 60 ++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 48daab1e..80bd0118 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -165,15 +165,42 @@ def copyLibraryAndSymbolPackage(src_file, dest_folder, overwrite): # # # -def getFilenameWithoutVersion(file_name) -> str: +def getFileBaseNameWithoutVersion(file_path) -> str: """ - :return: `'libSDL2'` for something like `'libSDL2-2.0.0.dylib'` + :return: `'libpostproc'` for something like `'/foo/bar/libpostproc.55.9.100.dylib'` """ - result = file_name.split('.')[0] + base_name = os.path.basename(file_path) + base_name = base_name.split('.')[0] # keep everything before first '.' # libSDL2 weirdly has hypthen after then name (i.e., libSDL2-2.0.0.dylib) - if 'libSDL2' in result: - result = 'libSDL2' - return result + if base_name.startswith('libSDL2'): + base_name = 'libSDL2' + return base_name + +# +# +# +def getVersionVariantsForFile(file_path) -> list[str]: + """ + Returns the following three files for any one of the file paths provided: + `'.../ffmpeg-build-script/workspace/lib/libavcodec.58.134.100.dylib'` + `'.../ffmpeg-build-script/workspace/lib/libavcodec.58.dylib'` + `'.../ffmpeg-build-script/workspace/lib/libavcodec.dylib'` + + """ + result = set() + result.add(file_path) + if (pathlib.Path(file_path).is_symlink()): + result.add(os.path.realpath(file_path)) + + dependency_name_without_version = getFileBaseNameWithoutVersion(file_path) + unversioned_dependency_base_name = os.path.join( + os.path.dirname(file_path), + dependency_name_without_version) + + for variant in glob.glob(unversioned_dependency_base_name + r'.*dylib'): + result.add(variant) + + return list(result) # # @@ -206,8 +233,9 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): src_dependency_file = match[0] # fix incorrect usage of @rpath - if src_dependency_file.startswith('@rpath/'): - fixed_path = os.path.join(workspace_lib_dir, src_dependency_file[7:]) + rpath_token = '@rpath/' + if src_dependency_file.startswith(rpath_token): + fixed_path = os.path.join(workspace_lib_dir, src_dependency_file[len(rpath_token):]) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': fixed_path}) src_dependency_file = fixed_path @@ -224,17 +252,14 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): if not src_dependency_file in copied_libs: if src_dependency_file != dest_dependency_path: # Copy each version variant file (often symlinks) - dependency_name_without_version = getFilenameWithoutVersion(src_dependency_file) - unversioned_dependency_base_name = os.path.join(os.path.dirname(src_dependency_file), dependency_name_without_version) - for variant_src_file in glob.glob(unversioned_dependency_base_name + r'*.dylib'): + for variant_src_file in getVersionVariantsForFile(src_dependency_file): copyLibraryAndSymbolPackage(variant_src_file, dest_folder, False) variant_dest_file = os.path.join(dest_folder, os.path.basename(variant_src_file)) copied_libs.add(variant_src_file) copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies - if (os.path.exists(unversioned_dependency_base_name + '.dylib')): - copyLibraryAndDependencies(unversioned_dependency_base_name + '.dylib', dest_folder, log_file) + copyLibraryAndDependencies(os.path.relpath(src_dependency_file), dest_folder, log_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: @@ -342,9 +367,6 @@ def main(): build_ffmpeg_log_file.write('\nGenerating Symbols\n') build_ffmpeg_log_file.write('=======================\n') copyOrGenerateSymbolFiles(packages_dir, symbol_temp_dir, build_ffmpeg_log_file) - symbol_file_name = base_artifact_name + '-symbols' - shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) - shutil.rmtree(symbol_temp_dir) # Generate dSYM files for each executable # and copy their dependencies @@ -352,7 +374,7 @@ def main(): build_ffmpeg_log_file.write('\nCopying & Linking ' + executable + '\n') build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) - copyOrGenerateSymbolFile(executable_path, workspace_bin_dir, build_ffmpeg_log_file) + copyOrGenerateSymbolFile(executable_path, symbol_temp_dir, build_ffmpeg_log_file) copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file) # check that the copied file is runnable @@ -363,6 +385,10 @@ def main(): output = subprocess.check_output(args) build_ffmpeg_log_file.write(output.decode('utf-8')) + symbol_file_name = base_artifact_name + '-symbols' + shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) + shutil.rmtree(symbol_temp_dir) + # Copy Includes shutil.copytree( os.path.join(workspace_dir, 'include'), From 95ae6378fdaeedc85a9b8ab1c404e3daeb0bd954 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Fri, 3 Dec 2021 22:11:39 -0600 Subject: [PATCH 40/58] Remove `--enable_small` for beamcoder --- build-ffmpeg | 1 - 1 file changed, 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index ced284fb..77903aee 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -1076,7 +1076,6 @@ if $SHARED_LIBRARIES; then --disable-static \ --enable-pthreads \ --enable-shared\ - --enable-small \ --enable-version3 \ --extra-cflags="${CFLAGS}" \ --extra-ldexeflags="${LDEXEFLAGS}" \ From c48d8e0b52fd452acfb3cfe92d19bebe4c00577b Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 15:16:42 -0600 Subject: [PATCH 41/58] Don't compile giflib in parallel It fails with too many cores --- build-ffmpeg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index 86ad1b3e..045c7e45 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -344,7 +344,10 @@ if build "giflib" "5.2.1"; then download "https://sourceforge.net/p/giflib/bugs/_discuss/thread/4e811ad29b/c323/attachment/Makefile.patch" execute patch "${PACKAGES}/giflib-5.2.1/Makefile" ${PACKAGES}/Makefile.patch"" fi - execute make -j $MJOBS + # Descript: compilation will fail with more than 8 jobs, so + # force to not run in parallel + #execute make -j $MJOBS + execute make execute make PREFIX="${WORKSPACE}" install build_done "giflib" "5.2.1" fi From ba0110e8a4fa90cad9d27d6d0176d0e65b33f50a Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 15:17:15 -0600 Subject: [PATCH 42/58] fail if build-ffmpeg fails --- descript/build-ffmpeg-descript.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 80bd0118..9a42c5da 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -59,8 +59,9 @@ def buildFFmpeg(script_dir, log_file): '-b', # build '--full-shared', # custom Descript shim to build shared libraries instead of static '--enable-gpl-and-free'] # custom Descript shim to build GPL but not non-free (libpostproc is needed by Beamcoder and requires GPL) - log_file.write(' '.join(args) + '\n\n') - subprocess.call(args, env=env, stdout=log_file) + log_file.write(' '.join(args) + '\n\n') + log_file.flush() + subprocess.run(args, env=env, stdout=log_file, stderr=log_file, check=True) # # From 797fbe7470d92d2d5b0aa894a8cd6f4aaa60657f Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 15:44:20 -0600 Subject: [PATCH 43/58] Fix relpath -> realpath typo --- descript/build-ffmpeg-descript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 9a42c5da..62fd3251 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -260,7 +260,7 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies - copyLibraryAndDependencies(os.path.relpath(src_dependency_file), dest_folder, log_file) + copyLibraryAndDependencies(os.path.realpath(src_dependency_file), dest_folder, log_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: From 499a002de8e8773d065d96e2943f9000bf0fb67c Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Tue, 7 Dec 2021 17:00:53 -0600 Subject: [PATCH 44/58] Fix python 3.8 usage and better logging for missing libs --- descript/build-ffmpeg-descript.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 62fd3251..29770960 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -180,7 +180,7 @@ def getFileBaseNameWithoutVersion(file_path) -> str: # # # -def getVersionVariantsForFile(file_path) -> list[str]: +def getVersionVariantsForFile(file_path): """ Returns the following three files for any one of the file paths provided: `'.../ffmpeg-build-script/workspace/lib/libavcodec.58.134.100.dylib'` @@ -206,10 +206,11 @@ def getVersionVariantsForFile(file_path) -> list[str]: # # # -def copyLibraryAndDependencies(src_file, dest_folder, log_file): +def copyLibraryAndDependencies(src_file, dest_folder, log_file, parent_path = ''): """ Recursive function to copy a library and its (non-system) dependencies also fixes loader paths for each library to be `@loader_path` + :param: `parent_path` - optional argument to show which parent is linking against `src_file` """ dest_file = os.path.join(dest_folder, os.path.basename(src_file)) @@ -243,7 +244,7 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): if src_dependency_file.startswith('/usr/local'): # the build grabbed libraries installed on this machine # which might not be available on other machines - missing_libs.add(src_dependency_file) + missing_libs.add(f'{src_dependency_file} (dependency of {os.path.basename(parent_path)})') elif src_dependency_file.startswith(workspace_dir): dependency_name = os.path.basename(src_dependency_file) if not len(this_id): @@ -260,7 +261,11 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file): copied_libs.add(variant_dest_file) # RECURSIVELY copy dependencies - copyLibraryAndDependencies(os.path.realpath(src_dependency_file), dest_folder, log_file) + copyLibraryAndDependencies( + os.path.realpath(src_dependency_file), + dest_folder, + log_file, + src_file) loader_paths_to_rewrite.append({'old_path': src_dependency_file, 'new_path': dest_dependency_path}) else: @@ -376,7 +381,7 @@ def main(): build_ffmpeg_log_file.write('=======================\n') executable_path = os.path.join(workspace_bin_dir, executable) copyOrGenerateSymbolFile(executable_path, symbol_temp_dir, build_ffmpeg_log_file) - copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file) + copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file, executable_path) # check that the copied file is runnable build_ffmpeg_log_file.write('\nChecking ' + executable + '\n') From c160d3688088584e98e988474811ac6d4113f48c Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 12:22:26 -0600 Subject: [PATCH 45/58] Try to add CI for build-ffmpeg-descript.py Need to log to STDOUT as well as log file --- .github/workflows/descript-build.yml | 22 +++++++ build-ffmpeg | 2 +- descript/build-ffmpeg-descript.py | 88 ++++++++++++++++------------ 3 files changed, 75 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/descript-build.yml diff --git a/.github/workflows/descript-build.yml b/.github/workflows/descript-build.yml new file mode 100644 index 00000000..4c1f3611 --- /dev/null +++ b/.github/workflows/descript-build.yml @@ -0,0 +1,22 @@ +name: descript build test + +on: + push: + paths-ignore: + - '*.md' + pull_request: + branches: [ charles/arm64 ] + +jobs: + build-macos-shared: + name: build in native macOS + runs-on: macos-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: build-ffmpeg-descript.py + run: | + while sleep 300; do echo "=====[ $SECONDS seconds still running ]====="; done & + python3 ./descript/build-ffmpeg-descript.py + kill %1 \ No newline at end of file diff --git a/build-ffmpeg b/build-ffmpeg index 045c7e45..e4231498 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.33rc1 +SCRIPT_VERSION=1.33rc2 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 29770960..1ea491cb 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -39,6 +39,21 @@ copied_libs = set() missing_libs = set() +# +# Global Logging File +# +log_file = None + +# +# +# +def log(str): + """ + Logs to stdout and to log_file + """ + log_file.write(str + '\n') + print(str) + # # # @@ -59,9 +74,9 @@ def buildFFmpeg(script_dir, log_file): '-b', # build '--full-shared', # custom Descript shim to build shared libraries instead of static '--enable-gpl-and-free'] # custom Descript shim to build GPL but not non-free (libpostproc is needed by Beamcoder and requires GPL) - log_file.write(' '.join(args) + '\n\n') - log_file.flush() - subprocess.run(args, env=env, stdout=log_file, stderr=log_file, check=True) + log(' '.join(args) + '\n') + process_result = subprocess.run(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) + log(process_result.stdout.decode('utf-8')) # # @@ -95,8 +110,9 @@ def copyOrGenerateSymbolFile(file, dest, log_file): # ./packages/libtheora-1.1.1/lib/.libs/libtheora.dylib except ValueError as e: args = ['/usr/bin/dsymutil', str(fileref), '-o', destPath] - log_file.write(' '.join(args) + '\n') - subprocess.call(args, stdout=log_file) + log(' '.join(args) + '\n') + process_result = subprocess.run(args, stdout=subprocess.PIPE) + log(process_result.stdout.decode('utf-8')) # # @@ -277,14 +293,14 @@ def copyLibraryAndDependencies(src_file, dest_folder, log_file, parent_path = '' # correct the loader path for this library if len(this_id): args = ['/usr/bin/install_name_tool', '-id', '@loader_path/' + this_id, actual_binary_path] - log_file.write(' '.join(args) + '\n') + log(' '.join(args)) subprocess.check_output(args) # correct the loader paths for all dependencies if len(loader_paths_to_rewrite) > 0: for loader_path in loader_paths_to_rewrite: args = ['/usr/bin/install_name_tool', '-change', loader_path['old_path'], '@loader_path/' + os.path.basename(loader_path['new_path']), actual_binary_path] - log_file.write(' '.join(args) + '\n') + log(' '.join(args)) subprocess.check_output(args) # @@ -361,35 +377,35 @@ def main(): # create a log file for the build-ffmpeg command for build archival purposes log_file_name = base_artifact_name + '-log.txt' log_file_path = os.path.join(output_dir, log_file_name) - build_ffmpeg_log_file = open(log_file_path, 'w') + globals()['log_file'] = open(log_file_path, 'w') - build_ffmpeg_log_file.write('Begin build-ffmpeg-descript.py\n') - build_ffmpeg_log_file.write('=======================\n') + log('Begin build-ffmpeg-descript.py') + log('=======================') # Run the script - buildFFmpeg(base_dir, build_ffmpeg_log_file) + buildFFmpeg(base_dir, log_file) # Generate dSYM files for each built library - build_ffmpeg_log_file.write('\nGenerating Symbols\n') - build_ffmpeg_log_file.write('=======================\n') - copyOrGenerateSymbolFiles(packages_dir, symbol_temp_dir, build_ffmpeg_log_file) + log('\nGenerating Symbols') + log('=======================') + copyOrGenerateSymbolFiles(packages_dir, symbol_temp_dir, log_file) # Generate dSYM files for each executable # and copy their dependencies for executable in executables: - build_ffmpeg_log_file.write('\nCopying & Linking ' + executable + '\n') - build_ffmpeg_log_file.write('=======================\n') + log('\nCopying & Linking ' + executable) + log('=======================') executable_path = os.path.join(workspace_bin_dir, executable) - copyOrGenerateSymbolFile(executable_path, symbol_temp_dir, build_ffmpeg_log_file) - copyLibraryAndDependencies(executable_path, temp_dir, build_ffmpeg_log_file, executable_path) + copyOrGenerateSymbolFile(executable_path, symbol_temp_dir, log_file) + copyLibraryAndDependencies(executable_path, temp_dir, log_file, executable_path) # check that the copied file is runnable - build_ffmpeg_log_file.write('\nChecking ' + executable + '\n') - build_ffmpeg_log_file.write('=======================\n') + log('\nChecking ' + executable) + log('=======================') args = [os.path.join(temp_dir, executable), '-version'] - build_ffmpeg_log_file.write(' '.join(args) + '\n') + log(' '.join(args)) output = subprocess.check_output(args) - build_ffmpeg_log_file.write(output.decode('utf-8')) + log(output.decode('utf-8')) symbol_file_name = base_artifact_name + '-symbols' shutil.make_archive(os.path.join(output_dir, symbol_file_name), 'zip', symbol_temp_dir) @@ -400,20 +416,20 @@ def main(): os.path.join(workspace_dir, 'include'), os.path.join(temp_dir, 'include')) - build_ffmpeg_log_file.write('\nLibrary Info\n') - build_ffmpeg_log_file.write('=======================\n') + log('\nLibrary Info') + log('=======================') for lib in sorted(missing_libs): - build_ffmpeg_log_file.write('[WARNING] missing ' + lib + '\n') + log('[WARNING] missing ' + lib) for lib in sorted(skipped_libs): - build_ffmpeg_log_file.write('[NOTE] skipped ' + lib + '\n') + log('[NOTE] skipped ' + lib) for lib in sorted(copied_libs): - build_ffmpeg_log_file.write('Copied ' + lib + '\n') + log('Copied ' + lib) - build_ffmpeg_log_file.write('\nArchiving third-party source\n') - build_ffmpeg_log_file.write('=======================\n') + log('\nArchiving third-party source') + log('=======================') # bundle up the third-party source # grab each .tar.* from the packages folder @@ -421,25 +437,25 @@ def main(): with zipfile.ZipFile(os.path.join(output_dir, packages_zip_name), 'w', zipfile.ZIP_DEFLATED) as myzip: archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') for archive in sorted(archives, key=lambda s: str(s).lower()): - build_ffmpeg_log_file.write(os.path.join('packages', archive.name) + '\n') + log(os.path.join('packages', archive.name)) myzip.write(str(archive.absolute()), archive.name) - build_ffmpeg_log_file.write('\nArchiving libraries\n') - build_ffmpeg_log_file.write('=======================\n') + log('\nArchiving libraries') + log('=======================') # bundle up the build artifacts os.chdir(temp_dir) shared_zip_name = base_artifact_name + '.zip' dest_file = os.path.join(output_dir, shared_zip_name) args = ['/usr/bin/zip', '--symlinks', '-r', os.path.join('..', shared_zip_name), '.'] - build_ffmpeg_log_file.write(' '.join(args) + '\n') + log(' '.join(args)) subprocess.check_output(args) shutil.rmtree(temp_dir) - build_ffmpeg_log_file.write('\nEnd of build-ffmpeg-descript.py\n') - build_ffmpeg_log_file.write('=======================\n') - build_ffmpeg_log_file.close() + log('\nEnd of build-ffmpeg-descript.py') + log('=======================') + log_file.close() # zip up log file with zipfile.ZipFile(os.path.splitext(log_file_path)[0] + '.zip', 'w', zipfile.ZIP_DEFLATED) as myzip: From 6e920b99850aa7d4bb3daf48576de891a7631d44 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 12:43:26 -0600 Subject: [PATCH 46/58] Better piping of log output --- descript/build-ffmpeg-descript.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 1ea491cb..05507c3a 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -54,6 +54,16 @@ def log(str): log_file.write(str + '\n') print(str) +# +# +# +def log_pipe(pipe): + """ + logs from a pipe by claling log() + """ + for line in iter(pipe.readline, b''): # b'\n'-separated lines + log(line.decode('utf-8').strip()) + # # # @@ -75,8 +85,13 @@ def buildFFmpeg(script_dir, log_file): '--full-shared', # custom Descript shim to build shared libraries instead of static '--enable-gpl-and-free'] # custom Descript shim to build GPL but not non-free (libpostproc is needed by Beamcoder and requires GPL) log(' '.join(args) + '\n') - process_result = subprocess.run(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True) - log(process_result.stdout.decode('utf-8')) + log_file.flush() + shell_proc = subprocess.Popen(args, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + with shell_proc.stdout: + log_pipe(shell_proc.stdout) + exitcode = shell_proc.wait() # 0 means success + if (exitcode != 0): + raise exitcode # # From 2490da300a850731a031ca180c34d061bd9e024a Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 12:46:06 -0600 Subject: [PATCH 47/58] Try better output of python script --- .github/workflows/descript-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/descript-build.yml b/.github/workflows/descript-build.yml index 4c1f3611..2fe7e7ae 100644 --- a/.github/workflows/descript-build.yml +++ b/.github/workflows/descript-build.yml @@ -18,5 +18,5 @@ jobs: - name: build-ffmpeg-descript.py run: | while sleep 300; do echo "=====[ $SECONDS seconds still running ]====="; done & - python3 ./descript/build-ffmpeg-descript.py + output=$(python3 ./descript/build-ffmpeg-descript.py) kill %1 \ No newline at end of file From 0c9133184d16f6f2e67128a306198c584f292465 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 12:52:08 -0600 Subject: [PATCH 48/58] Try different way of running python script --- .github/workflows/descript-build.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/descript-build.yml b/.github/workflows/descript-build.yml index 2fe7e7ae..698ff451 100644 --- a/.github/workflows/descript-build.yml +++ b/.github/workflows/descript-build.yml @@ -15,8 +15,11 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: build-ffmpeg-descript.py + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: 3.8 #install the python needed + + - name: execute py script # run the run.py to get the latest data run: | - while sleep 300; do echo "=====[ $SECONDS seconds still running ]====="; done & - output=$(python3 ./descript/build-ffmpeg-descript.py) - kill %1 \ No newline at end of file + python ./descript/build-ffmpeg-descript.py From a150884d06cae94fa3c6ac0599f6e132071ba173 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 12:58:13 -0600 Subject: [PATCH 49/58] try logging to stderr to see if that helps github --- descript/build-ffmpeg-descript.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 05507c3a..b28d9706 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -49,17 +49,17 @@ # def log(str): """ - Logs to stdout and to log_file + Logs to stderr and to log_file """ log_file.write(str + '\n') - print(str) + print(str, file=sys.stderr) # # # def log_pipe(pipe): """ - logs from a pipe by claling log() + logs from a pipe by calling log() """ for line in iter(pipe.readline, b''): # b'\n'-separated lines log(line.decode('utf-8').strip()) @@ -398,7 +398,7 @@ def main(): log('=======================') # Run the script - buildFFmpeg(base_dir, log_file) + #buildFFmpeg(base_dir, log_file) # Generate dSYM files for each built library log('\nGenerating Symbols') From 8269e96dc52f5b3d6a718bc7472668bd82b76824 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 13:00:10 -0600 Subject: [PATCH 50/58] try stdout again with flush --- descript/build-ffmpeg-descript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index b28d9706..52a547eb 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -49,10 +49,10 @@ # def log(str): """ - Logs to stderr and to log_file + Logs to stdout and to log_file """ log_file.write(str + '\n') - print(str, file=sys.stderr) + print(str, flush=True) # # From ffe9a8725cdb330529e6fa9e18744b0d455e190d Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 13:01:44 -0600 Subject: [PATCH 51/58] re-enable build-ffmpeg --- descript/build-ffmpeg-descript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index 52a547eb..c96b699a 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -398,7 +398,7 @@ def main(): log('=======================') # Run the script - #buildFFmpeg(base_dir, log_file) + buildFFmpeg(base_dir, log_file) # Generate dSYM files for each built library log('\nGenerating Symbols') From e87b40ccaa3120e011e61f9134c0d13e7c9a1135 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 14:45:22 -0600 Subject: [PATCH 52/58] Add more documentation and refine github action --- .github/workflows/descript-build.yml | 6 +++--- descript/README.md | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/descript-build.yml b/.github/workflows/descript-build.yml index 698ff451..6497c6e1 100644 --- a/.github/workflows/descript-build.yml +++ b/.github/workflows/descript-build.yml @@ -1,4 +1,4 @@ -name: descript build test +name: build shared-libs for Descript on: push: @@ -9,7 +9,7 @@ on: jobs: build-macos-shared: - name: build in native macOS + name: ffmpeg-ffprobe-shared-darwin runs-on: macos-latest steps: - name: Checkout code @@ -20,6 +20,6 @@ jobs: with: python-version: 3.8 #install the python needed - - name: execute py script # run the run.py to get the latest data + - name: build-ffmpeg-descript.py run: | python ./descript/build-ffmpeg-descript.py diff --git a/descript/README.md b/descript/README.md index 70c884e1..77958a2e 100644 --- a/descript/README.md +++ b/descript/README.md @@ -1,3 +1,5 @@ +![](../ffmpeg-build-script.png) + # Build FFmpeg for Descript ## Author / Contact: @@ -23,8 +25,22 @@ The build script automates the following basic operations. ## Development Known issues: -- . +- CI can only build `x86_64` and doesn't yet cross-compile to `arm64` +- When running on CI, there's libraries linked from `/usr/local/opt/...` which are non-portable + - Watch the script output at the end for warnings about this. These could turn into errors later. ## Patches +- The build-ffmpeg shell script is modified directly to allow for + - building shared libraries + - disable non-free codecs + - add additional codecs that Descript uses +## Deployment / Releases +- Currently, the script is run manually on a developer's machine + - once for each platform (`x86_64` and `arm64`) +- Build artifacts (`*.zip` files) are manually uploaded to a GitHub release and tagged +- `SHAMSUM256.txt` files need to be merged between the two platforms when adding to a release. +## Clients +- This build is consumed by Descript's Beamcoder fork + - https://github.com/descriptinc/beamcoder \ No newline at end of file From 3ac5fad4d7387f20b0143e723044d3abf1ab63c9 Mon Sep 17 00:00:00 2001 From: Charles Van Winkle Date: Wed, 8 Dec 2021 15:17:33 -0600 Subject: [PATCH 53/58] Add the additional libraries we want for Descript based off of change 6ceca48 --- build-ffmpeg | 95 ++++++++++++++++++++++++++++++++++++++++++++++ descript/README.md | 52 ++++++++++++++++++++++++- 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/build-ffmpeg b/build-ffmpeg index e4231498..df0bd8d8 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -1050,6 +1050,101 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then CONFIGURE_OPTIONS+=("--enable-amf") fi + +## +## Descript Additions +## + +if $SHARED_LIBRARIES; then + if build "twolame" "0.4.0"; then + download "https://downloads.sourceforge.net/twolame/twolame-0.4.0.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + execute make -j $MJOBS + execute make install + + build_done "twolame" "0.4.0" + fi + CONFIGURE_OPTIONS+=("--enable-libtwolame") + + + if build "shine" "3.1.1"; then + download "https://github.com/toots/shine/archive/3.1.1.tar.gz" "shine-3.1.1.tar.gz" + execute autoreconf -i + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + execute make -j $MJOBS + execute make install + + build_done "shine" "3.1.1" + fi + CONFIGURE_OPTIONS+=("--enable-libshine") + + + if build "soxr" "0.1.3"; then + download "http://ftp.debian.org/debian/pool/main/libs/libsoxr/libsoxr_0.1.3.orig.tar.xz" "soxr-0.1.3.tar.xz" + make_dir build + cd build || exit + execute cmake .. -DWITH_OPENMP:BOOL=OFF -Wno-dev -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=ON + execute make -j $MJOBS + execute make install + + build_done "soxr" "0.1.3" + fi + CONFIGURE_OPTIONS+=("--enable-libsoxr") + + + if build "speex" "1.2.0"; then + download "http://downloads.us.xiph.org/releases/speex/speex-1.2.0.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + execute make -j $MJOBS + execute make install + + build_done "speex" "1.2.0" + fi + CONFIGURE_OPTIONS+=("--enable-libspeex") + + + if build "openjpeg" "2.3.1"; then + download "https://github.com/uclouvain/openjpeg/archive/v2.3.1.tar.gz" "openjpeg-v2.3.1.tar.gz" + make_dir build + cd build || exit + execute cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DBUILD_SHARED_LIBS:bool=on + execute make -j $MJOBS + execute make install + + build_done "openjpeg" "2.3.1" + fi + CONFIGURE_OPTIONS+=("--enable-libopenjpeg") + + + if build "snappy" "1.1.8"; then + download "https://github.com/google/snappy/archive/1.1.8.tar.gz" + make_dir build + cd build || exit + execute cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DBUILD_SHARED_LIBS:bool=on + execute make -j $MJOBS + + #if [[ "$OSTYPE" == "darwin"* ]]; then + # execute install_name_tool -id "${WORKSPACE}/lib/libsnappy.1.dylib" libsnappy.1.dylib + #fi + + execute make install + + build_done "snappy" "1.1.8" + fi + CONFIGURE_OPTIONS+=("--enable-libsnappy") + + + if build "xz" "5.2.5"; then + download "https://tukaani.org/xz/xz-5.2.5.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static + execute make -j $MJOBS + execute make install + + build_done "xz" "5.2.5" + fi + +fi + ## ## FFmpeg ## diff --git a/descript/README.md b/descript/README.md index 77958a2e..66f919f7 100644 --- a/descript/README.md +++ b/descript/README.md @@ -43,4 +43,54 @@ Known issues: ## Clients - This build is consumed by Descript's Beamcoder fork - - https://github.com/descriptinc/beamcoder \ No newline at end of file + - https://github.com/descriptinc/beamcoder + +## Previous Documentation of targeted differences (prior to `arm64` port) +- ❌ don't need +- ✅ already in ffmpeg-build-script build +- 🕒 need to add to ffmpeg-build-script + +Target (`evermeet.cx` static build) +- `--cc=/usr/bin/clang` +- `--prefix=/opt/ffmpeg` +- `--extra-version=tessus` +- ❌ `--enable-avisynth` - non-linear editing +- ❌ `--enable-fontconfig` +- ✅ `--enable-gpl` +- ✅ `--enable-libaom` +- ❌ `--enable-libass` +- ❌ `--enable-libbluray` - bluray playback +- 🕒✅ `--enable-libdav1d` +- ❌ `--enable-libfreetype` - text rendering +- ❌ `--enable-libgsm` - GSM audio +- ❌ `--enable-libmodplug` - midi/instrument support (https://github.com/Konstanty/libmodplug) +- ✅ `--enable-libmp3lame` +- ❌ `--enable-libmysofa` - spatial audio +- ✅ `--enable-libopencore-amrnb` +- ✅ `--enable-libopencore-amrwb` +- 🕒✅ `--enable-libopenh264` +- 🕒✅ `--enable-libopenjpeg` +- ✅ `--enable-libopus` +- ❌ `--enable-librubberband` - time stretching +- 🕒✅ `--enable-libshine` - mp3 encoder +- 🕒✅ `--enable-libsnappy` - compression/decompression +- 🕒✅ `--enable-libsoxr` - resampling +- 🕒✅ `--enable-libspeex` - speex audio file format +- ✅ `--enable-libtheora` +- 🕒✅ `--enable-libtwolame` - mpeg2 +- ✅ `--enable-libvidstab` +- ❌ `--enable-libvmaf` - perceptual video quality metric +- ❌ `--enable-libvo-amrwbenc` - VisualOn AMR-WB encoder library +- ✅ `--enable-libvorbis` +- ✅ `--enable-libvpx` +- ✅ `--enable-libwebp` +- ✅ `--enable-libx264` +- ✅ `--enable-libx265` +- ❌ `--enable-libxavs` - AV standard of China +- ✅ `--enable-libxvid` +- 🕒✅ `--enable-libzimg` - Scaling, colorspace conversion, and dithering library +- ❌ `--enable-libzmq` - ZeroMQ Support To Let Multiple Clients Connect To A Single Instance (streaming) +- ❌ `--enable-libzvbi` - capture and decode VBI (vertical blanking interval) data +- ✅ `--enable-version3` +- `--pkg-config-flags=--static` +- `--disable-ffplay` From c3897f4ad60ffb9e8862a8919c95c127d50c2c41 Mon Sep 17 00:00:00 2001 From: C Van Winkle Date: Wed, 8 Dec 2021 16:19:24 -0600 Subject: [PATCH 54/58] Fix soxr on arm64 --- build-ffmpeg | 6 +++++- descript/build-ffmpeg-descript.py | 12 +++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index df0bd8d8..3ec81483 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.33rc2 +SCRIPT_VERSION=1.33rc3 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" @@ -1083,6 +1083,10 @@ if $SHARED_LIBRARIES; then download "http://ftp.debian.org/debian/pool/main/libs/libsoxr/libsoxr_0.1.3.orig.tar.xz" "soxr-0.1.3.tar.xz" make_dir build cd build || exit + if [[ "$OSTYPE" == "darwin"* ]]; then + download "https://raw.githubusercontent.com/macports/macports-ports/master/audio/soxr/files/patch-pffft.c.diff" "patch-pffft.c.diff" + execute patch "${PACKAGES}/soxr-0.1.3/src/pffft.c" ${PACKAGES}/patch-pffft.c.diff"" + fi execute cmake .. -DWITH_OPENMP:BOOL=OFF -Wno-dev -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS:BOOL=ON execute make -j $MJOBS execute make install diff --git a/descript/build-ffmpeg-descript.py b/descript/build-ffmpeg-descript.py index c96b699a..5c792ede 100755 --- a/descript/build-ffmpeg-descript.py +++ b/descript/build-ffmpeg-descript.py @@ -447,13 +447,15 @@ def main(): log('=======================') # bundle up the third-party source - # grab each .tar.* from the packages folder + # grab each .tar.* and any downloaded patches from the packages folder packages_zip_name = base_artifact_name + '-packages.zip' with zipfile.ZipFile(os.path.join(output_dir, packages_zip_name), 'w', zipfile.ZIP_DEFLATED) as myzip: - archives = pathlib.Path(packages_dir + '/').glob('*.tar.*') - for archive in sorted(archives, key=lambda s: str(s).lower()): - log(os.path.join('packages', archive.name)) - myzip.write(str(archive.absolute()), archive.name) + types = ['*.tar.*', '*.patch', '*.diff'] + for file_type in types: + archives = pathlib.Path(packages_dir + '/').glob(file_type) + for archive in sorted(archives, key=lambda s: str(s).lower()): + log(os.path.join('packages', archive.name)) + myzip.write(str(archive.absolute()), archive.name) log('\nArchiving libraries') log('=======================') From 838d28b0d660f214e6143f902c2ce54db0e144a9 Mon Sep 17 00:00:00 2001 From: builder Date: Mon, 18 Dec 2023 11:16:56 -0800 Subject: [PATCH 55/58] Updated to build on OS X 14.1 --- build-ffmpeg | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 3ec81483..3c342628 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -862,8 +862,11 @@ if build "libogg" "1.3.3"; then build_done "libogg" "1.3.3" fi -if build "libvorbis" "1.3.6"; then - download "https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.6.tar.gz" +if build "libvorbis" "1.3.7"; then + download "https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.7.tar.gz" + echo $PWD + patch configure.ac ../../vorbis.config.patch + ./autogen.sh if $SHARED_LIBRARIES; then execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --disable-static --enable-shared --disable-oggtest else @@ -872,7 +875,7 @@ if build "libvorbis" "1.3.6"; then execute make -j $MJOBS execute make install - build_done "libvorbis" "1.3.6" + build_done "libvorbis" "1.3.7" fi CONFIGURE_OPTIONS+=("--enable-libvorbis") @@ -970,8 +973,8 @@ fi ## other library ## -if build "libsdl" "2.0.14"; then - download "https://www.libsdl.org/release/SDL2-2.0.14.tar.gz" +if build "libsdl" "2.28.5"; then + download "https://www.libsdl.org/release/SDL2-2.28.5.tar.gz" if $SHARED_LIBRARIES; then execute ./configure --prefix="${WORKSPACE}" --enable-shared --disable-static else @@ -980,7 +983,7 @@ if build "libsdl" "2.0.14"; then execute make -j $MJOBS execute make install - build_done "libsdl" "2.0.14" + build_done "libsdl" "2.28.5" fi if $NONFREE_AND_GPL; then From 8e24098fbaacb736e2abb15c444fde36dcdaa7bf Mon Sep 17 00:00:00 2001 From: Srikanth K Date: Tue, 19 Dec 2023 11:15:44 -0800 Subject: [PATCH 56/58] Updating build-ffmpeg zlib version and patch file --- build-ffmpeg | 9 +++++---- vorbis.config.patch | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 vorbis.config.patch diff --git a/build-ffmpeg b/build-ffmpeg index 3c342628..901e5d85 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -380,8 +380,8 @@ if build "nasm" "2.15.05"; then build_done "nasm" "2.15.05" fi -if build "zlib" "1.2.11"; then - download "https://www.zlib.net/zlib-1.2.11.tar.gz" +if build "zlib" "1.3"; then + download "https://www.zlib.net/zlib-1.3.tar.gz" if $SHARED_LIBRARIES; then execute ./configure --shared --prefix="${WORKSPACE}" else @@ -389,7 +389,7 @@ if build "zlib" "1.2.11"; then fi execute make -j $MJOBS execute make install - build_done "zlib" "1.2.11" + build_done "zlib" "1.3" fi if build "m4" "1.4.19"; then @@ -865,7 +865,8 @@ fi if build "libvorbis" "1.3.7"; then download "https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.7.tar.gz" echo $PWD - patch configure.ac ../../vorbis.config.patch + + patch configure.ac ${CWD%/descript}vorbis.config.patch ./autogen.sh if $SHARED_LIBRARIES; then execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --disable-static --enable-shared --disable-oggtest diff --git a/vorbis.config.patch b/vorbis.config.patch new file mode 100644 index 00000000..67fcd67b --- /dev/null +++ b/vorbis.config.patch @@ -0,0 +1,15 @@ +--- libvorbis-1.3.7/configure.ac 2020-07-03 18:52:28 ++++ configure.ac 2023-12-18 10:48:08 +@@ -202,9 +202,9 @@ + CFLAGS="-O3 -Wall -Wextra -ffast-math -D__NO_MATH_INLINES -fsigned-char $sparc_cpu" + PROFILE="-pg -g -O3 -D__NO_MATH_INLINES -fsigned-char $sparc_cpu" ;; + *-*-darwin*) +- DEBUG="-DDARWIN -fno-common -force_cpusubtype_ALL -Wall -g -O0 -fsigned-char" +- CFLAGS="-DDARWIN -fno-common -force_cpusubtype_ALL -Wall -g -O3 -ffast-math -fsigned-char" +- PROFILE="-DDARWIN -fno-common -force_cpusubtype_ALL -Wall -g -pg -O3 -ffast-math -fsigned-char";; ++ DEBUG="-DDARWIN -fno-common -Wall -g -O0 -fsigned-char" ++ CFLAGS="-DDARWIN -fno-common -Wall -g -O3 -ffast-math -fsigned-char" ++ PROFILE="-DDARWIN -fno-common -Wall -g -pg -O3 -ffast-math -fsigned-char";; + *-*-os2*) + # Use -W instead of -Wextra because gcc on OS/2 is an old version. + DEBUG="-g -Wall -W -D_REENTRANT -D__NO_MATH_INLINES -fsigned-char" From 2a8b4a5956841a4cb69f95f62f140ef81b83807c Mon Sep 17 00:00:00 2001 From: Srikanth K Date: Tue, 19 Dec 2023 12:01:30 -0800 Subject: [PATCH 57/58] Updating ffmpeg build script to 1.40.rc.1 --- build-ffmpeg | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/build-ffmpeg b/build-ffmpeg index 901e5d85..b3e86363 100755 --- a/build-ffmpeg +++ b/build-ffmpeg @@ -5,7 +5,7 @@ PROGNAME=$(basename "$0") FFMPEG_VERSION=4.4 -SCRIPT_VERSION=1.33rc3 +SCRIPT_VERSION=1.40.rc.1 CWD=$(pwd) PACKAGES="$CWD/packages" WORKSPACE="$CWD/workspace" @@ -515,11 +515,11 @@ if ! $MACOS_M1; then fi if command_exists "cargo"; then - if build "rav1e" "0.5.0-beta"; then + if build "rav1e" "0.6.6"; then execute cargo install cargo-c - download "https://github.com/xiph/rav1e/archive/refs/tags/v0.5.0-beta.tar.gz" + download "https://github.com/xiph/rav1e/archive/refs/tags/v0.6.6.tar.gz" execute cargo cinstall --prefix="${WORKSPACE}" --library-type=staticlib --crt-static --release - build_done "rav1e" "0.5.0-beta" + build_done "rav1e" "0.6.6" fi CONFIGURE_OPTIONS+=("--enable-librav1e") fi @@ -866,7 +866,7 @@ if build "libvorbis" "1.3.7"; then download "https://ftp.osuosl.org/pub/xiph/releases/vorbis/libvorbis-1.3.7.tar.gz" echo $PWD - patch configure.ac ${CWD%/descript}vorbis.config.patch + patch configure.ac ${CWD%/descript}/vorbis.config.patch ./autogen.sh if $SHARED_LIBRARIES; then execute ./configure --prefix="${WORKSPACE}" --with-ogg-libraries="${WORKSPACE}"/lib --with-ogg-includes="${WORKSPACE}"/include/ --disable-static --enable-shared --disable-oggtest @@ -953,23 +953,20 @@ if build "libpng" "1.6.37"; then build_done "libpng" "1.6.37" fi -## does not compile on monterey -> _PrintGifError -if [[ "$OSTYPE" != "darwin"* ]]; then - if build "libwebp" "1.2.1"; then - # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS - CPPFLAGS= - download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" - execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --disable-dependency-tracking --disable-gl --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib - make_dir build - cd build || exit - execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ - execute make -j $MJOBS - execute make install +if build "libwebp" "1.2.1"; then + # libwebp can fail to compile on Ubuntu if these flags were left set to CFLAGS + CPPFLAGS= + download "https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz" "libwebp-1.2.1.tar.gz" + execute ./configure --prefix="${WORKSPACE}" --disable-shared --enable-static --disable-dependency-tracking --disable-gl --with-zlib-include="${WORKSPACE}"/include/ --with-zlib-lib="${WORKSPACE}"/lib + make_dir build + cd build || exit + execute cmake -DCMAKE_INSTALL_PREFIX="${WORKSPACE}" -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_BINDIR=bin -DCMAKE_INSTALL_INCLUDEDIR=include -DENABLE_SHARED=OFF -DENABLE_STATIC=ON ../ + execute make -j $MJOBS + execute make install - build_done "libwebp" "1.2.1" - fi - CONFIGURE_OPTIONS+=("--enable-libwebp") + build_done "libwebp" "1.2.1" fi +CONFIGURE_OPTIONS+=("--enable-libwebp") ## ## other library ## From 705369d778a501f7a59732b377728708b883158a Mon Sep 17 00:00:00 2001 From: Srikanth K Date: Tue, 19 Dec 2023 12:05:07 -0800 Subject: [PATCH 58/58] Adding new auto-release --- .github/workflows/descript-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/descript-build.yml b/.github/workflows/descript-build.yml index 6497c6e1..b2375742 100644 --- a/.github/workflows/descript-build.yml +++ b/.github/workflows/descript-build.yml @@ -5,7 +5,9 @@ on: paths-ignore: - '*.md' pull_request: - branches: [ charles/arm64 ] + branches: + - charles/arm64 + - sk/ffmpeg-4.4-arm64 jobs: build-macos-shared: