From f17f533ca5b1915df89fc433a3e161508c893e95 Mon Sep 17 00:00:00 2001 From: Brett Jia Date: Sun, 21 Jan 2024 15:19:35 -0500 Subject: [PATCH] Revert "Conform builds to manylinux2014 glibc standards (#14)" This reverts commit d4d6c3d0b7f154aedbd48ad8ed973687bcf66ecc. --- .github/workflows/build-and-test-make.yml | 300 ++++----------------- .github/workflows/build-win.yaml | 1 - Makefile.in | 13 +- chrome/patches/boringssl-old-ciphers.patch | 31 --- firefox/patches/curl-impersonate.patch | 2 +- tests/test_impersonate.py | 2 +- 6 files changed, 54 insertions(+), 295 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 27eb0dee..49f3f209 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -14,162 +14,45 @@ permissions: contents: write env: - NSS_VERSION: nss-3.92 + NSS_VERSION: nss-3.77 BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767 jobs: - build-and-test-linux: - name: (Linux ${{ matrix.arch }}) Build curl-impersonate and run the tests - runs-on: ubuntu-latest + build-and-test: + name: Build curl-impersonate and run the tests + runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: + os: [ubuntu-20.04, macos-12] arch: [x86_64] include: - - arch: x86_64 - go_arch: amd64 - docker_arch: linux/amd64 + - os: ubuntu-20.04 + arch: x86_64 host: x86_64-linux-gnu capture_interface: eth0 - image: quay.io/pypa/manylinux2014_x86_64:latest - - arch: aarch64 - go_arch: arm64 - docker_arch: linux/arm/v8 + make: make + - os: ubuntu-20.04 + arch: aarch64 host: aarch64-linux-gnu capture_interface: eth0 - image: quay.io/pypa/manylinux2014_aarch64:latest - - arch: arm - go_arch: armv6l - docker_arch: linux/arm/v7 - host: arm-linux-gnu + make: make + - os: ubuntu-20.04 + arch: arm + host: arm-linux-gnueabihf capture_interface: eth0 - image: ghcr.io/bjia56/armv7l-wheel-builder:main + make: make + - os: macos-12 + arch: x86_64 + host: x86_64-macos + capture_interface: en0 + make: gmake steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Install dependencies - run: | - python_bin_dir=$(docker run ${{ matrix.image }} python3.10 -c 'import sys; import os; print(os.path.dirname(os.path.realpath(sys.executable)))') - echo "runner_uid=$(id -u)" >> $GITHUB_ENV - echo "runner_gid=$(id -g)" >> $GITHUB_ENV - echo "runner_home=$HOME" >> $GITHUB_ENV - docker build -t curl-impersonate-builder -f - . < - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - mkdir ${{ runner.temp }}/install - ./configure --prefix=${{ runner.temp }}/install \ - --with-ca-path=/etc/ssl/certs \ - --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ - --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss - - # Cache the build of BoringSSL, which is the longest part of the build - # We must cache the .zip as well, otherwise the Makefile will - # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to - # reduce the insanely long build times. - - name: Cache BoringSSL source - uses: actions/cache@v3 - with: - path: boringssl.zip - key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }} - - - name: Cache BoringSSL build - id: cache-boringssl - uses: actions/cache@v3 - with: - path: boringssl/build - key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }} - - # Trick the Makefile into skipping the BoringSSL build step - # if it was found in the cache. See Makefile.in - - name: Post BoringSSL cache restore - if: ${{ steps.cache-boringssl.outputs.cache-hit != false }} - run: | - touch boringssl.zip - touch boringssl/.patched - find boringssl/build -type f | xargs touch - - - name: Build the Chrome version of curl-impersonate - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make chrome-build - make chrome-checkbuild - make chrome-install - - # Cache the build of NSS, which is the longest part of the build - # We must cache the .tar.gz as well, otherwise the Makefile will - # rebuild NSS. - - name: Cache NSS source - uses: actions/cache@v3 - with: - path: ${{ env.NSS_VERSION }}.tar.gz - key: ${{ runner.os }}-${{ matrix.arch }}-nss-source-${{ env.NSS_VERSION }} - - - name: Cache NSS build - id: cache-nss - uses: actions/cache@v3 - with: - path: ${{ env.NSS_VERSION }}/dist - key: ${{ runner.os }}-${{ matrix.arch }}-nss-build-${{ env.NSS_VERSION }} - - # Trick the Makefile into skipping the NSS build step - # if it was found in the cache. - - name: Post NSS cache restore - if: ${{ steps.cache-nss.outputs.cache-hit != false }} - run: | - touch ${{ env.NSS_VERSION }}.tar.gz - find ${{ env.NSS_VERSION }}/dist -type f | xargs touch - - - name: Build the Firefox version of curl-impersonate - uses: addnab/docker-run-action@v3 + - uses: actions/setup-python@v4 with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make firefox-build - make firefox-checkbuild - make firefox-install + python-version: '3.10' - - name: Install test dependencies - if: matrix.arch == 'x86_64' + - name: Install Ubuntu dependencies + if: matrix.os == 'ubuntu-20.04' run: | sudo apt-get update sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool @@ -180,122 +63,13 @@ jobs: # More dependencies for the tests sudo apt-get install tcpdump nghttp2-server libnss3 - - name: Prepare the tests - if: matrix.arch == 'x86_64' - run: | - # Compile 'minicurl' which is used by the tests - gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs` - - - uses: actions/setup-python@v4 - if: matrix.arch == 'x86_64' - with: - python-version: '3.10' - - - name: Install dependencies for the tests script - if: matrix.arch == 'x86_64' - run: | - pip3 install -r tests/requirements.txt - - # For now we can only run the tests when native - # tests run the curl-impersonate binary locally. - - name: Run the tests - if: matrix.arch == 'x86_64' - run: | - cd tests - # sudo is needed for capturing packets - python_bin=$(which python3) - sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }} - - # Upload pre-compiled binaries to GitHub releases page. - - name: Create tar release files for libcurl-impersonate - if: startsWith(github.ref, 'refs/tags/') + - name: Install Ubuntu cross-compile dependencies (${{ matrix.arch }}) + if: matrix.os == 'ubuntu-20.04' && matrix.arch != 'x86_64' run: | - cd ${{ runner.temp }}/install/lib - tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate* - echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV - - - name: Clean build - if: startsWith(github.ref, 'refs/tags/') - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make chrome-clean - make firefox-clean - rm -Rf ${{ runner.temp }}/install - mkdir ${{ runner.temp }}/install - - # Recompile curl-impersonate statically when doing a release. - - name: Reconfigure statically - if: startsWith(github.ref, 'refs/tags/') && matrix.arch == 'x86_64' - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - ./configure --prefix=${{ runner.temp }}/install --enable-static - - - name: Rebuild statically - if: startsWith(github.ref, 'refs/tags/') - uses: addnab/docker-run-action@v3 - with: - image: curl-impersonate-builder - options: > - -v ${{ env.runner_home }}:${{ env.runner_home }} - --workdir ${{ github.workspace }} - --user ${{ env.runner_uid }}:${{ env.runner_gid }} - run: | - set -e - make chrome-build - make chrome-checkbuild - make chrome-install-strip - make firefox-build - make firefox-checkbuild - make firefox-install-strip - - - name: Create tar release files for curl-impersonate - if: startsWith(github.ref, 'refs/tags/') - run: | - cd ${{ runner.temp }}/install/bin - tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-ff curl-impersonate-chrome curl_* - echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV - - - name: Upload release files - uses: softprops/action-gh-release@v1 - if: startsWith(github.ref, 'refs/tags/') - with: - files: | - ${{ env.release_file_lib }} - ${{ env.release_file_bin }} - - build-and-test-macos: - name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macos-12] - arch: [x86_64] - include: - - os: macos-12 - arch: x86_64 - host: x86_64-macos - capture_interface: en0 - make: gmake - steps: - - uses: actions/setup-python@v4 - with: - python-version: '3.10' + sudo apt-get install gcc-${{ matrix.host }} g++-${{ matrix.host }} - name: Install macOS dependencies + if: matrix.os == 'macos-12' run: | brew install pkg-config make cmake ninja autoconf automake libtool # Chrome version dependencies @@ -305,6 +79,9 @@ jobs: brew install curl # More dependencies for the tests brew install tcpdump nghttp2 nss + + - name: Install common dependencies + run: | # Firefox version dependencies pip3 install gyp-next @@ -315,6 +92,7 @@ jobs: run: | pip3 install -r tests/requirements.txt + # When cross compiling we need to build zlib first. - name: Build zlib run: | curl -LO https://zlib.net/zlib-1.3.tar.gz @@ -332,6 +110,20 @@ jobs: mkdir ${{ runner.temp }}/install ./configure --prefix=${{ runner.temp }}/install + # When cross compiling a more complicated configuration is needed, since + # curl's configure script can't figure out where some files and libraries + # are located. The locations used here are the ones used by Ubuntu. + - name: Run configure script (cross compiling) + if: matrix.arch != 'x86_64' + run: | + mkdir ${{ runner.temp }}/install + ./configure --prefix=${{ runner.temp }}/install \ + --host=${{ matrix.host }} \ + --with-zlib=${{ runner.temp }}/zlib \ + --with-ca-path=/etc/ssl/certs \ + --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ + --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss + # Cache the build of BoringSSL, which is the longest part of the build # We must cache the .zip as well, otherwise the Makefile will # rebuild BoringSSL. This whole thing is a bit hacky, but necessary to diff --git a/.github/workflows/build-win.yaml b/.github/workflows/build-win.yaml index e8f644ca..1c386a5e 100644 --- a/.github/workflows/build-win.yaml +++ b/.github/workflows/build-win.yaml @@ -75,7 +75,6 @@ jobs: run: ./win/dll2lib.bat ${{ matrix.env }} build\dist\libcurl.dll - name: Build tarball - if: startsWith(github.ref, 'refs/tags/') shell: msys2 {0} run: tar cvzf libcurl-impersonate-${{ github.head_ref || github.ref_name }}.${{ matrix.env }}-win32.tar.gz -C ./build/dist . diff --git a/Makefile.in b/Makefile.in index 95ba5182..3190fabe 100644 --- a/Makefile.in +++ b/Makefile.in @@ -7,7 +7,6 @@ SHELL := bash .DELETE_ON_ERROR: # MAKEFLAGS += --warn-undefined-variables # MAKEFLAGS += --no-builtin-rules -SUBJOBS := 4 BROTLI_VERSION := 1.0.9 # In case this is changed, update build-and-test-make.yml as well @@ -68,7 +67,7 @@ help: ## Show this help message firefox-build: $(CURL_VERSION)/.firefox ## Build the Firefox version of curl-impersonate cd $(CURL_VERSION) # Don't pass this Makefile's MAKEFLAGS - $(MAKE) MAKEFLAGS=-j$(SUBJOBS) + $(MAKE) MAKEFLAGS= .PHONY: firefox-build firefox-checkbuild: ## Run basic checks on the built binary @@ -117,7 +116,7 @@ firefox-clean: ## Clean build artifacts of the Firefox version. Use after re-run chrome-build: $(CURL_VERSION)/.chrome ## Build the Chrome version of curl-impersonate cd $(CURL_VERSION) # Don't pass this Makefile's MAKEFLAGS - $(MAKE) MAKEFLAGS=-j$(SUBJOBS) + $(MAKE) MAKEFLAGS= .PHONY: chrome-build chrome-checkbuild: ## Run basic checks on the built binary @@ -202,7 +201,7 @@ $(brotli_static_libs): brotli-$(BROTLI_VERSION).tar.gz -DCMAKE_SYSTEM_PROCESSOR=$(host_cpu) \ .. - @cmake@ --build . --config Release --target install --parallel $(SUBJOBS) + @cmake@ --build . --config Release --target install $(NSS_VERSION).tar.gz: @@ -214,7 +213,7 @@ $(nss_static_libs): $(NSS_VERSION).tar.gz ifeq ($(host),$(build)) # Native build, use NSS' build script. cd $(NSS_VERSION)/nss - ./build.sh -o --disable-tests --static --python=python3 -j $(SUBJOBS) + ./build.sh -o --disable-tests --static --python=python3 else # We are cross compiling. # Cross compiling NSS is not supported by its build script and is poorly @@ -295,7 +294,7 @@ $(boringssl_static_libs): boringssl.zip boringssl/.patched -DCMAKE_SYSTEM_PROCESSOR=$(host_cpu) \ -GNinja \ .. - @ninja@ -j$(SUBJOBS) + @ninja@ # Fix the directory structure so that curl can compile against it. # See https://everything.curl.dev/source/build/tls/boringssl mkdir -p lib @@ -324,7 +323,7 @@ $(nghttp2_static_libs): $(NGHTTP2_VERSION).tar.bz2 } ./configure $$config_flags - $(MAKE) MAKEFLAGS=-j$(SUBJOBS) + $(MAKE) MAKEFLAGS= $(MAKE) install MAKEFLAGS= $(CURL_VERSION).tar.xz: diff --git a/chrome/patches/boringssl-old-ciphers.patch b/chrome/patches/boringssl-old-ciphers.patch index b3821999..2b278299 100644 --- a/chrome/patches/boringssl-old-ciphers.patch +++ b/chrome/patches/boringssl-old-ciphers.patch @@ -207,34 +207,3 @@ index 57116cd6c..fa1652832 100644 } // Check for invalid algorithms, and filter out |SSL_SIGN_RSA_PKCS1_MD5_SHA1|. -diff --git a/crypto/fipsmodule/rand/urandom_test.cc b/crypto/fipsmodule/rand/urandom_test.cc -index 08e4183..d486c01 100644 ---- a/crypto/fipsmodule/rand/urandom_test.cc -+++ b/crypto/fipsmodule/rand/urandom_test.cc -@@ -38,10 +38,26 @@ - #include "fork_detect.h" - #include "getrandom_fillin.h" - -+#if !defined(NT_ARM_SYSTEM_CALL) -+// https://elixir.bootlin.com/linux/v6.6.8/source/include/uapi/linux/elf.h#L433 -+#define NT_ARM_SYSTEM_CALL 0x404 /* ARM system call number */ -+#endif -+ - #if !defined(PTRACE_O_EXITKILL) - #define PTRACE_O_EXITKILL (1 << 20) - #endif - -+#if defined(OPENSSL_AARCH64) && defined(__linux__) -+// https://elixir.bootlin.com/glibc/glibc-2.38/source/sysdeps/unix/sysv/linux/aarch64/sys/user.h#L22 -+struct user_regs_struct -+{ -+ unsigned long long regs[31]; -+ unsigned long long sp; -+ unsigned long long pc; -+ unsigned long long pstate; -+}; -+#endif -+ - #if defined(OPENSSL_ANDROID) - static const bool kIsAndroid = true; - #else diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index c0a71d04..fe461274 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -1452,7 +1452,7 @@ index 2916c3613..f07fa30ac 100644 + esac + + case $host_cpu in -+ arm|armv7l) ++ arm) + addlib="$addlib -larmv8_c_lib -lgcm-aes-arm32-neon_c_lib" + ;; + aarch64) diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index cb407f20..cb600b84 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -160,7 +160,7 @@ async def nghttpd(): # Otherwise fail. started = await asyncio.wait_for(_wait_nghttpd(proc), timeout=3) if not started: - raise Exception("nghttpd failed to start") + raise Exception("nghttpd failed to start on time") except asyncio.TimeoutError: raise Exception("nghttpd failed to start on time")