Skip to content

Updating docs (#90) #326

Updating docs (#90)

Updating docs (#90) #326

name: Build and test
on:
push:
branches:
- main
- feature/*
- bugfix/*
tags:
- "v*.*.*"
pull_request:
branches:
- main
permissions:
contents: write
env:
BORING_SSL_COMMIT: d24a38200fef19150eef00cad35b138936c08767
jobs:
build-and-test-linux:
name: (${{ matrix.host }}) Build curl-impersonate and run the tests
runs-on: ubuntu-latest
env:
CC: ${{ github.workspace }}/zigshim/cc
CXX: ${{ github.workspace }}/zigshim/cxx
AR: ${{ github.workspace }}/zigshim/ar
ZIG_FLAGS: ${{ matrix.zigflags }}
strategy:
fail-fast: false
matrix:
include:
- arch: x86_64
host: x86_64-linux-gnu
capture_interface: eth0
zigflags: -target x86_64-linux-gnu.2.17
- arch: x86_64
host: x86_64-linux-musl
capture_interface: eth0
zigflags: -target x86_64-linux-musl -fPIC
- arch: i386
host: i386-linux-gnu
capture_interface: eth0
zigflags: -target x86-linux-gnu.2.17
- arch: aarch64
host: aarch64-linux-gnu
capture_interface: eth0
zigflags: -target aarch64-linux-gnu.2.17
- arch: aarch64
host: aarch64-linux-musl
capture_interface: eth0
zigflags: -target aarch64-linux-musl -fPIC -mno-outline-atomics
# - arch: i386
# host: i386-linux-musl
# capture_interface: eth0
# zigflags: -target i386-linux-musl -fPIC -mno-outline-atomics
- arch: arm
host: arm-linux-gnueabihf
capture_interface: eth0
zigflags: -target arm-linux-gnueabihf.2.17 -mcpu=arm1176jzf_s
steps:
- name: Check out the repo
uses: actions/checkout@v2
- name: Set up zig
uses: goto-bus-stop/setup-zig@v2
with:
version: 0.11.0
- name: Install Ubuntu dependencies
run: |
sudo apt-get update
sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool
sudo apt-get install gcc-aarch64-linux-gnu gcc-arm-linux-gnueabihf
# Chrome version dependencies
sudo apt-get install golang-go
# Needed to compile 'minicurl'
sudo apt-get install libcurl4-openssl-dev
# More dependencies for the tests
sudo apt-get install tcpdump nghttp2-server
- name: Build zlib
run: |
curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz
tar xf zlib-1.3.tar.gz
cd zlib-1.3
CHOST=${{ matrix.host }} ./configure --prefix=${{ runner.temp }}/zlib
make
make install
# Make sure curl will link with libz.so.1 and not libz.so
rm -f ${{ runner.temp }}/zlib/lib/libz.so
- name: Build libzstd
run: |
curl -LO https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz
tar xf zstd-1.5.6.tar.gz
cd zstd-1.5.6
make
mkdir -p ${{ runner.temp }}/zstd/lib
mkdir -p ${{ runner.temp }}/zstd/include
# move libzstd to runner.temp
cp lib/libzstd.a lib/libzstd.so.1.5.6 lib/libzstd.mk lib/libzstd.pc lib/libzstd.pc.in ${{ runner.temp }}/zstd/lib/
cp lib/zstd.h ${{ runner.temp }}/zstd/include/
- name: Run configure script
run: |
mkdir ${{ runner.temp }}/install
./configure --prefix=${{ runner.temp }}/install \
--host=${{ matrix.host }} \
--with-zlib=${{ runner.temp }}/zlib \
--with-zstd=${{ runner.temp }}/zstd \
--with-ca-path=/etc/ssl/certs \
--with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \
# Cache the build of BoringSSL, which is the longest part of the build
# We must cache the .zip as well, otherwise the Makefile will
# rebuild BoringSSL. This whole thing is a bit hacky, but necessary to
# reduce the insanely long build times.
- name: Cache BoringSSL source
uses: actions/cache@v4
with:
path: boringssl.zip
key: ${{ runner.os }}-${{ matrix.host }}-boring-source-${{ env.BORING_SSL_COMMIT }}
- name: Cache BoringSSL build
id: cache-boringssl
uses: actions/cache@v4
with:
path: boringssl/build
key: ${{ runner.os }}-${{ matrix.host }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }}
# Trick the Makefile into skipping the BoringSSL build step
# if it was found in the cache. See Makefile.in
- name: Post BoringSSL cache restore
if: ${{ steps.cache-boringssl.outputs.cache-hit != false }}
run: |
touch boringssl.zip
touch boringssl/.patched
find boringssl/build -type f | xargs touch
- name: Build the Chrome version of curl-impersonate
run: |
make chrome-build
make chrome-checkbuild
make chrome-install
- name: Prepare the tests
if: matrix.host == 'x86_64-linux-gnu'
run: |
# Compile 'minicurl' which is used by the tests
gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs`
- uses: actions/setup-python@v4
if: matrix.host == 'x86_64-linux-gnu'
with:
python-version: '3.10'
- name: Install dependencies for the tests script
if: matrix.host == 'x86_64-linux-gnu'
run: |
pip3 install -r tests/requirements.txt
# For now we can only run the tests when native
# tests run the curl-impersonate binary locally.
- name: Run the tests
if: matrix.host == 'x86_64-linux-gnu'
run: |
cd tests
# sudo is needed for capturing packets
python_bin=$(which python3)
sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }}
# Upload pre-compiled binaries to GitHub releases page and Github actions archives.
- name: Create tar release files for libcurl-impersonate
run: |
cp ${{ runner.temp }}/zlib/lib/lib*.a ${{ runner.temp }}/install/lib
cp ${{ runner.temp }}/zstd/lib/lib*.a ${{ runner.temp }}/install/lib
cp nghttp2*/installed/lib/lib*.a ${{ runner.temp }}/install/lib
cp brotli*/out/installed/lib/lib*.a ${{ runner.temp }}/install/lib
cp boringssl*/lib/lib*.a ${{ runner.temp }}/install/lib
cd ${{ runner.temp }}/install/lib
ls -lah .
if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz lib*
else
tar -c -z -f ${{ runner.temp }}/libcurl-impersonate.${{ matrix.host }}.tar.gz lib*
fi
echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: libcurl-impersonate.${{ matrix.host }}.tar.gz
path: ${{ runner.temp }}/libcurl-impersonate*.tar.gz
- name: Clean build
run: |
set -e
make chrome-clean
rm -Rf ${{ runner.temp }}/install
mkdir ${{ runner.temp }}/install
# Recompile curl-impersonate statically when doing a release.
- name: Reconfigure statically
run: |
set -e
./configure --prefix=${{ runner.temp }}/install --enable-static \
--host=${{ matrix.host }} \
--with-zlib=${{ runner.temp }}/zlib \
--with-zstd=${{ runner.temp }}/zstd \
--with-ca-path=/etc/ssl/certs \
--with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \
- name: Rebuild statically
run: |
set -e
make chrome-build
make chrome-checkbuild
if [[ "${{ endsWith(matrix.host, 'musl') }}" == "true" ]]; then
make chrome-install
else
make chrome-install-strip
fi
- name: Create tar release files for curl-impersonate
run: |
cd ${{ runner.temp }}/install/bin
if [[ "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_*
else
tar -c -z -f ${{ runner.temp }}/curl-impersonate.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_*
fi
echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: curl-impersonate.${{ matrix.host }}.tar.gz
path: ${{ runner.temp }}/curl-impersonate*.tar.gz
- name: Upload release files
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
files: |
${{ env.release_file_lib }}
${{ env.release_file_bin }}
build-and-test-macos:
name: (macOS ${{ matrix.arch }}) Build curl-impersonate and run the tests
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-12, macos-14]
include:
- os: macos-12
arch: x86_64
host: x86_64-macos
capture_interface: en0
make: gmake
- os: macos-14
arch: arm64
host: arm64-macos
capture_interface: en0
make: gmake
steps:
- uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install macOS dependencies
run: |
brew install pkg-config make cmake ninja autoconf automake libtool gpatch
# Chrome version dependencies, install only not installed.
which go || brew install go
# Needed to compile 'minicurl'
brew install curl
# More dependencies for the tests
brew install tcpdump nghttp2
# zstd
brew install zstd
- name: Check out the repo
uses: actions/checkout@v2
- name: Install dependencies for the tests script
run: |
pip3 install -r tests/requirements.txt
- name: Build zlib
run: |
curl -LO https://zlib.net/fossils/zlib-1.3.tar.gz
tar xf zlib-1.3.tar.gz
cd zlib-1.3
CHOST=${{ matrix.host }} ./configure --prefix=${{ runner.temp }}/zlib
make
make install
# Make sure curl will link with libz.so.1 and not libz.so
rm -f ${{ runner.temp }}/zlib/lib/libz.so
- name: Run configure script
run: |
mkdir ${{ runner.temp }}/install
./configure LDFLAGS="-L$(brew --prefix zstd)/lib" CPPFLAGS="-I$(brew --prefix zstd)/include" --prefix=${{ runner.temp }}/install
# Cache the build of BoringSSL, which is the longest part of the build
# We must cache the .zip as well, otherwise the Makefile will
# rebuild BoringSSL. This whole thing is a bit hacky, but necessary to
# reduce the insanely long build times.
- name: Cache BoringSSL source
uses: actions/cache@v3
with:
path: boringssl.zip
key: ${{ runner.os }}-${{ matrix.arch }}-boring-source-${{ env.BORING_SSL_COMMIT }}
- name: Cache BoringSSL build
id: cache-boringssl
uses: actions/cache@v3
with:
path: boringssl/build
key: ${{ runner.os }}-${{ matrix.arch }}-boring-build-${{ env.BORING_SSL_COMMIT }}-${{ hashFiles('chrome/patches/boringssl*.patch') }}
# Trick the Makefile into skipping the BoringSSL build step
# if it was found in the cache. See Makefile.in
- name: Post BoringSSL cache restore
if: ${{ steps.cache-boringssl.outputs.cache-hit != false }}
run: |
touch boringssl.zip
touch boringssl/.patched
find boringssl/build -type f | xargs touch
- name: Build the Chrome version of curl-impersonate
run: |
${{ matrix.make }} chrome-build
${{ matrix.make }} chrome-checkbuild
${{ matrix.make }} chrome-install
- name: Prepare the tests
run: |
# Compile 'minicurl' which is used by the tests
gcc -Wall -Werror -o ${{ runner.temp }}/install/bin/minicurl tests/minicurl.c `curl-config --libs`
# For now we can only run the tests when not cross compiling, since the
# tests run the curl-impersonate binary locally.
# - name: Run the tests
# run: |
# cd tests
# # sudo is needed for capturing packets
# python_bin=$(which python3)
# sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }}
# Upload pre-compiled binaries to GitHub releases page.
- name: Create tar release files for libcurl-impersonate
if: startsWith(github.ref, 'refs/tags/')
run: |
cd ${{ runner.temp }}/install/lib
tar -c -z -f ${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz libcurl-impersonate*
echo "release_file_lib=${{ runner.temp }}/libcurl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV
- name: Clean build
if: startsWith(github.ref, 'refs/tags/')
run: |
${{ matrix.make }} chrome-clean
rm -Rf ${{ runner.temp }}/install
mkdir ${{ runner.temp }}/install
# Recompile curl-impersonate statically when doing a release.
- name: Reconfigure statically
if: startsWith(github.ref, 'refs/tags/')
run: |
./configure LDFLAGS="-L$(brew --prefix zstd)/lib" CPPFLAGS="-I$(brew --prefix zstd)/include" --prefix=${{ runner.temp }}/install --enable-static
- name: Rebuild statically
if: startsWith(github.ref, 'refs/tags/')
run: |
${{ matrix.make }} chrome-build
${{ matrix.make }} chrome-checkbuild
${{ matrix.make }} chrome-install-strip
- name: Create tar release files for curl-impersonate
if: startsWith(github.ref, 'refs/tags/')
run: |
cd ${{ runner.temp }}/install/bin
tar -c -z -f ${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz curl-impersonate-chrome curl_*
echo "release_file_bin=${{ runner.temp }}/curl-impersonate-${{ github.ref_name }}.${{ matrix.host }}.tar.gz" >> $GITHUB_ENV
- name: Upload release files
uses: softprops/action-gh-release@v1
if: startsWith(github.ref, 'refs/tags/')
with:
files: |
${{ env.release_file_lib }}
${{ env.release_file_bin }}