diff --git a/.github/workflows/build-and-test-docker.yml b/.github/workflows/build-and-test-docker.yml index 365c63b4..4f3faedb 100644 --- a/.github/workflows/build-and-test-docker.yml +++ b/.github/workflows/build-and-test-docker.yml @@ -3,14 +3,17 @@ name: Docker images on: push: branches: - - main - - feature/* - - bugfix/* + - disabled + # - main + # - feature/* + # - bugfix/* tags: - - "v*.*.*" + - disabled + # - "v*.*.*" pull_request: branches: - - main + - disabled + # - main jobs: build-docker-and-test: diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 6e2e23eb..8b4debdb 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -75,7 +75,7 @@ jobs: # Needed to compile 'minicurl' sudo apt-get install libcurl4-openssl-dev # More dependencies for the tests - sudo apt-get install tcpdump nghttp2-server libnss3 + sudo apt-get install tcpdump nghttp2-server - name: Build zlib run: | @@ -88,15 +88,27 @@ jobs: # Make sure curl will link with libz.so.1 and not libz.so rm -f ${{ runner.temp }}/zlib/lib/libz.so + - name: Build libzstd + run: | + curl -LO https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz + tar xf zstd-1.5.6.tar.gz + cd zstd-1.5.6 + make + mkdir -p ${{ runner.temp }}/zstd/lib + mkdir -p ${{ runner.temp }}/zstd/include + # move libzstd to runner.temp + cp lib/libzstd.a lib/libzstd.so.1.5.6 lib/libzstd.mk lib/libzstd.pc lib/libzstd.pc.in ${{ runner.temp }}/zstd/lib/ + cp lib/zstd.h ${{ runner.temp }}/zstd/include/ + - name: Run configure script run: | mkdir ${{ runner.temp }}/install ./configure --prefix=${{ runner.temp }}/install \ --host=${{ matrix.host }} \ --with-zlib=${{ runner.temp }}/zlib \ + --with-zstd=${{ runner.temp }}/zstd \ --with-ca-path=/etc/ssl/certs \ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ - --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss # Cache the build of BoringSSL, which is the longest part of the build # We must cache the .zip as well, otherwise the Makefile will @@ -160,6 +172,7 @@ jobs: - name: Create tar release files for libcurl-impersonate run: | cp ${{ runner.temp }}/zlib/lib/lib*.a ${{ runner.temp }}/install/lib + cp ${{ runner.temp }}/zstd/lib/lib*.a ${{ runner.temp }}/install/lib cp nghttp2*/installed/lib/lib*.a ${{ runner.temp }}/install/lib cp brotli*/out/installed/lib/lib*.a ${{ runner.temp }}/install/lib cp boringssl/build/lib/lib*.a ${{ runner.temp }}/install/lib @@ -193,9 +206,9 @@ jobs: ./configure --prefix=${{ runner.temp }}/install --enable-static \ --host=${{ matrix.host }} \ --with-zlib=${{ runner.temp }}/zlib \ + --with-zstd=${{ runner.temp }}/zstd \ --with-ca-path=/etc/ssl/certs \ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ - --with-libnssckbi=/usr/lib/${{ matrix.host }}/nss - name: Rebuild statically run: | @@ -233,7 +246,7 @@ jobs: ${{ env.release_file_bin }} build-and-test-macos: - name: (MacOS ${{ matrix.arch }}) Build curl-impersonate and run the tests + name: (macOS ${{ matrix.arch }}) Build curl-impersonate and run the tests runs-on: ${{ matrix.os }} strategy: matrix: @@ -263,6 +276,8 @@ jobs: brew install curl # More dependencies for the tests brew install tcpdump nghttp2 + # zstd + brew install zstd - name: Check out the repo uses: actions/checkout@v2 @@ -285,7 +300,7 @@ jobs: - name: Run configure script run: | mkdir ${{ runner.temp }}/install - ./configure --prefix=${{ runner.temp }}/install + ./configure LDFLAGS="-L$(brew --prefix zstd)/lib" CPPFLAGS="-I$(brew --prefix zstd)/include" --prefix=${{ runner.temp }}/install # Cache the build of BoringSSL, which is the longest part of the build # We must cache the .zip as well, otherwise the Makefile will @@ -326,12 +341,12 @@ jobs: # For now we can only run the tests when not cross compiling, since the # tests run the curl-impersonate binary locally. - - name: Run the tests - run: | - cd tests - # sudo is needed for capturing packets - python_bin=$(which python3) - sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }} + # - name: Run the tests + # run: | + # cd tests + # # sudo is needed for capturing packets + # python_bin=$(which python3) + # sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp }}/install --capture-interface ${{ matrix.capture_interface }} # Upload pre-compiled binaries to GitHub releases page and Github actions archives. - name: Create tar release files for libcurl-impersonate @@ -360,7 +375,7 @@ jobs: # Recompile curl-impersonate statically when doing a release. - name: Reconfigure statically run: | - ./configure --prefix=${{ runner.temp }}/install --enable-static + ./configure LDFLAGS="-L$(brew --prefix zstd)/lib" CPPFLAGS="-I$(brew --prefix zstd)/include" --prefix=${{ runner.temp }}/install --enable-static - name: Rebuild statically run: | diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index b576fc3a..cc8034a1 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -5,7 +5,8 @@ name: Publish Docker image on: push: tags: - - 'v*' + - disabled + # - 'v*' jobs: push-docker-image: diff --git a/INSTALL.md b/INSTALL.md index a877b1f4..dabb172a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -8,7 +8,7 @@ There are currently three build options depending on your use case: * [Cross compiling](#cross-compiling) using an autotools-based Makefile * [Docker container build](#docker-build) -There are two versions of `curl-impersonate` for technical reasons. The **chrome** version is used to impersonate Chrome, Edge and Safari. The **firefox** version is used to impersonate Firefox. +Unlike the upstream project, there is only one version in this fork, namely the Chrome version, for impersonating all main stream browsers. ## Native build @@ -18,18 +18,14 @@ Install dependencies for building all the components: ```sh sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool -# For the Firefox version only -sudo apt install python3-pip libnss3 -pip install gyp-next -export PATH="$PATH:~/.local/bin" # Add gyp to PATH -# For the Chrome version only sudo apt install golang-go unzip +sudo apt install zstd libzstd-dev ``` Clone this repository: ```sh -git clone https://github.com/lwthiker/curl-impersonate.git +git clone https://github.com/yifeikong/curl-impersonate.git cd curl-impersonate ``` @@ -38,10 +34,7 @@ Configure and compile: ```sh mkdir build && cd build ../configure -# Build and install the Firefox version -make firefox-build -sudo make firefox-install -# Build and install the Chrome version +# Build and install make chrome-build sudo make chrome-install # You may need to update the linker's cache to find libcurl-impersonate @@ -55,14 +48,12 @@ This will install curl-impersonate, libcurl-impersonate and the wrapper scripts After installation you can run the wrapper scripts, e.g.: ```sh -curl_ff98 https://www.wikipedia.org -curl_chrome99 https://www.wikipedia.org +curl_chrome119 https://www.wikipedia.org ``` or run directly with you own flags: ```sh -curl-impersonate-ff https://www.wikipedia.org curl-impersonate-chrome https://www.wikipedia.org ``` @@ -78,13 +69,7 @@ yum install cmake3 python3 python3-pip yum install ninja-build # OR pip3 install ninja -``` - -For the Firefox version, install NSS and gyp: - -```sh -yum install nss nss-pem -pip3 install gyp-next +yum install zstd libzstd-devel ``` For the Chrome version, install Go. @@ -102,16 +87,14 @@ Install dependencies for building all the components: ```sh brew install pkg-config make cmake ninja autoconf automake libtool -# For the Firefox version only -brew install sqlite nss -pip3 install gyp-next -# For the Chrome version only +brwe install zstd brew install go ``` Clone this repository: -``` -git clone https://github.com/lwthiker/curl-impersonate.git + +```sh +git clone https://github.com/yifeikong/curl-impersonate.git cd curl-impersonate ``` @@ -120,10 +103,7 @@ Configure and compile: ```sh mkdir build && cd build ../configure -# Build and install the Firefox version -gmake firefox-build -sudo gmake firefox-install -# Build and install the Chrome version +# Build and install gmake chrome-build sudo gmake chrome-install # Optionally remove all the build files @@ -134,25 +114,6 @@ cd ../ && rm -Rf build To compile curl-impersonate statically with libcurl-impersonate, pass `--enable-static` to the `configure` script. -### A note about the Firefox version - -The Firefox version compiles a static version of nss, Firefox's TLS library. -For NSS to have a list of root certificates, curl attempts to load at runtime `libnssckbi`, one of the NSS libraries. -If you get the error: - -```sh -curl: (60) Peer's Certificate issuer is not recognized -``` - -or - -```sh -curl: (77) Problem with the SSL CA cert (path? access rights?) -``` - -, make sure that NSS is installed (see above). -If the issue persists it might be that NSS is installed in a non-standard location on your system. -Please open an issue in that case. ## Cross compiling @@ -160,7 +121,7 @@ There is some basic support for cross compiling curl-impersonate. It is currently being used to build curl-impersonate for ARM64 (aarch64) systems from x86-64 systems. Cross compiling is similar to the usual build but a bit trickier: -* You'd have to build zlib for the target architecture so that curl can link with it. +* You'd have to build zlib and zstd for the target architecture so that curl can link with it. * Some paths have to be specified manually since curl's own build system can't determine their location. An example build for aarch64 on Ubuntu x86_64: @@ -170,25 +131,21 @@ sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu ./configure --host=aarch64-linux-gnu \ --with-zlib=/path/to/compiled/zlib \ + --with-zstd=/path/to/compiled/zstd \ --with-ca-path=/etc/ssl/certs \ - --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt \ - --with-libnssckbi=/usr/lib/aarch64-linux-gnu/nss + --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt make chrome-build -make firefox-build ``` The flags mean as follows: -`--with-zlib` is the location of a compiled zlib library for the target architecture. +`--with-zlib/zstd` is the location of a compiled zlib/zstd library for the target architecture. `--with-ca-path` and `--with-ca-bundle` will be passed to curl's configure script as is. -`--with-libnssckbi` indicates the location of libnssckbi.so on the target system. This file contains the certificates needed by curl. This must be supplied if NSS is not installed in a standard location (i.e. not in `/usr/lib`). ## Docker build The Docker build is a bit more reproducible and serves as the reference implementation. It creates a Debian-based Docker image with the binaries. -### Chrome version - [`chrome/Dockerfile`](chrome/Dockerfile) is a debian-based Dockerfile that will build curl with all the necessary modifications and patches. Build it like the following: ```sh @@ -202,21 +159,3 @@ The resulting binaries and libraries are in the `/usr/local` directory, which co * `libcurl-impersonate-chrome.so`, `libcurl-impersonate.so` - libcurl compiled with impersonation support. See [libcurl-impersonate](README.md#libcurl-impersonate) for more details. You can use them inside the docker, copy them out using `docker cp` or use them in a multi-stage docker build. - -### Firefox version - -Build with: - -```sh -docker build -t curl-impersonate-ff firefox/ -``` - -The resulting binaries and libraries are in the `/usr/local` directory, which contains: - -* `curl-impersonate-ff`, `curl-impersonate` - The curl binary that can impersonate Firefox. It is compiled statically against libcurl, nss, and libnghttp2 so that it won't conflict with any existing libraries on your system. You can use it from the container or copy it out. Tested to work on Ubuntu 20.04. -* `curl_ff91esr`, `curl_ff95`, `...` - Wrapper scripts that launch `curl-impersonate` with all the needed flags. -* `libcurl-impersonate-ff.so`, `libcurl-impersonate.so` - libcurl compiled with impersonation support. See [libcurl-impersonate](README.md#libcurl-impersonate) for more details. - -If you use it outside the container, install the following dependency: - -* `sudo apt install libnss3`. Even though nss is statically compiled into `curl-impersonate`, it is still necessary to install libnss3 because curl dynamically loads `libnssckbi.so`, a file containing Mozilla's list of trusted root certificates. Alternatively, use `curl -k` to disable certificate verification. diff --git a/LICENSE b/LICENSE index 6d58f284..6c0834f7 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ MIT License Copyright (c) 2022 lwthiker +Copyright (c) 2024 yifeikong Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile.in b/Makefile.in index 6e969ceb..f3cff53e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -11,25 +11,19 @@ SUBJOBS := 4 BROTLI_VERSION := 1.0.9 # In case this is changed, update build-and-test-make.yml as well -NSS_VERSION := nss-3.92 -NSS_URL := https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_92_RTM/src/nss-3.92-with-nspr-4.35.tar.gz # In case this is changed, update build-and-test-make.yml as well BORING_SSL_COMMIT := d24a38200fef19150eef00cad35b138936c08767 NGHTTP2_VERSION := nghttp2-1.56.0 NGHTTP2_URL := https://github.com/nghttp2/nghttp2/releases/download/v1.56.0/nghttp2-1.56.0.tar.bz2 -CURL_VERSION := curl-8.1.1 +CURL_VERSION := curl-8_5_0 brotli_install_dir := $(abspath brotli-$(BROTLI_VERSION)/out/installed) brotli_static_libs := $(brotli_install_dir)/lib/libbrotlicommon-static.a $(brotli_install_dir)/lib/libbrotlidec-static.a -nss_install_dir := $(abspath $(NSS_VERSION)/dist/Release) -nss_static_libs := $(nss_install_dir)/lib/libnss_static.a boringssl_install_dir := $(abspath boringssl/build) boringssl_static_libs := $(boringssl_install_dir)/lib/libssl.a $(boringssl_install_dir)/lib/libcrypto.a nghttp2_install_dir := $(abspath $(NGHTTP2_VERSION)/installed) nghttp2_static_libs := $(nghttp2_install_dir)/lib/libnghttp2.a -# Dependencies needed to compile the Firefox version -firefox_libs := $(brotli_static_libs) $(nss_static_libs) $(nghttp2_static_libs) # Dependencies needed to compile the Chrome version chrome_libs := $(brotli_static_libs) $(boringssl_static_libs) $(nghttp2_static_libs) @@ -46,13 +40,12 @@ build = @build@ static_build = @static_build@ # Whether the user provided a specific find for zlib with_zlib = @with_zlib@ +# Whether the user provided a specific find for libzstd +with_zstd = @with_zstd@ # Path to be passed to curl's --with-ca-bundle configure option. with_ca_bundle = @with_ca_bundle@ # Path to be passed to curl's --with-ca-path configure option. with_ca_path = @with_ca_path@ -# Path to be passed to curl's --with-libnssckbi configure option (an option -# added for curl-impersonate). -with_libnssckbi = @with_libnssckbi@ CC = @CC@ CXX = @CXX@ @@ -65,55 +58,6 @@ help: ## Show this help message .PHONY: help .DEFAULT_GOAL := help -firefox-build: $(CURL_VERSION)/.firefox ## Build the Firefox version of curl-impersonate - cd $(CURL_VERSION) - # Don't pass this Makefile's MAKEFLAGS - $(MAKE) MAKEFLAGS=-j$(SUBJOBS) -.PHONY: firefox-build - -firefox-checkbuild: ## Run basic checks on the built binary -ifeq ($(host),$(build)) - cd $(CURL_VERSION) - # Make sure all needed features were compiled in - ./src/curl-impersonate-ff -V | grep -q zlib - ./src/curl-impersonate-ff -V | grep -q brotli - ./src/curl-impersonate-ff -V | grep -q nghttp2 - ./src/curl-impersonate-ff -V | grep -q NSS - $(info Build OK) -else - $(info Cross compiling, skipping checkbuild) -endif -.PHONY: firefox-checkbuild - -firefox-install: ## Install the Firefox version of curl-impersonate after build - cd $(CURL_VERSION) - $(MAKE) install-exec MAKEFLAGS= - # Wrapper scripts for the Firefox version (e.g. 'curl_ff98') - install $(srcdir)/firefox/curl_ff* @bindir@ -.PHONY: firefox-install - -firefox-install-strip: ## Like 'firefox-install', but strip binaries for smaller size - cd $(CURL_VERSION) - $(MAKE) install-exec MAKEFLAGS= - # We could have used 'install-strip' but then the docs would be installed as well. - # Instead strip manually. - $(STRIP) @bindir@/curl-impersonate-ff - # Wrapper scripts for the Firefox version (e.g. 'curl_ff98') - install $(srcdir)/firefox/curl_ff* @bindir@ -.PHONY: firefox-install-strip - -firefox-uninstall: ## Uninstall the Firefox version of curl-impersonate after 'make install' - cd $(CURL_VERSION) - $(MAKE) uninstall MAKEFLAGS= - rm -Rf @bindir@/curl_ff* -.PHONY: firefox-uninstall - -firefox-clean: ## Clean build artifacts of the Firefox version. Use after re-running './configure' - cd $(CURL_VERSION) - $(MAKE) clean MAKEFLAGS= - rm -f .firefox -.PHONY: firefox-clean - chrome-build: $(CURL_VERSION)/.chrome ## Build the Chrome version of curl-impersonate cd $(CURL_VERSION) # Don't pass this Makefile's MAKEFLAGS @@ -138,7 +82,7 @@ chrome-install: ## Install the Chrome version of curl-impersonate after build cd $(CURL_VERSION) $(MAKE) install-exec MAKEFLAGS= # Wrapper scripts for the Chrome version (e.g. 'curl_chrome99') - install $(srcdir)/chrome/curl_chrome* $(srcdir)/chrome/curl_edge* $(srcdir)/chrome/curl_safari* @bindir@ + install $(srcdir)/chrome/curl_* @bindir@ .PHONY: chrome-install chrome-install-strip: ## Like 'chrome-install', but strip binaries for smaller size @@ -165,10 +109,9 @@ chrome-clean: ## Clean build artifacts of the Chrome version. Use after re-runni clean: ## Remove all build artifacts, including dependencies rm -Rf brotli-$(BROTLI_VERSION).tar.gz brotli-$(BROTLI_VERSION) - rm -Rf $(NSS_VERSION).tar.gz $(NSS_VERSION) rm -Rf boringssl.zip boringssl rm -Rf $(NGHTTP2_VERSION).tar.bz2 $(NGHTTP2_VERSION) - rm -Rf $(CURL_VERSION).tar.xz $(CURL_VERSION) + rm -Rf $(CURL_VERSION).tar.gz $(CURL_VERSION) brotli-$(BROTLI_VERSION).tar.gz: curl -L "https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz" \ @@ -206,60 +149,12 @@ $(brotli_static_libs): brotli-$(BROTLI_VERSION).tar.gz @cmake@ --build . --config Release --target install --parallel $(SUBJOBS) -$(NSS_VERSION).tar.gz: - curl -L -o $(NSS_VERSION).tar.gz $(NSS_URL) - -$(nss_static_libs): $(NSS_VERSION).tar.gz - tar xf $(NSS_VERSION).tar.gz - -ifeq ($(host),$(build)) - # Native build, use NSS' build script. - cd $(NSS_VERSION)/nss - ./build.sh -o --disable-tests --static --python=python3 -j $(SUBJOBS) -else - # We are cross compiling. - # Cross compiling NSS is not supported by its build script and is poorly - # documented. We need to compile NSPR manually and only then compile nss. - case $(host_cpu) in \ - *64*) \ - use_64="1"; \ - nspr_configure_flags="--enable-64bit"; \ - ;; \ - *) \ - use_64="0"; \ - nspr_configure_flags=""; \ - ;; \ - esac - - # Cross-compile nspr separately - cd $(NSS_VERSION)/nspr - ./configure --prefix=$(nss_install_dir) \ - --disable-debug --enable-optimize \ - --target=$(host_alias) \ - $$nspr_configure_flags - $(MAKE) MAKEFLAGS= - $(MAKE) install MAKEFLAGS= - - # Now we can run ./build.sh with the already built nspr - cd ../nss - CC=$(CC) CXX=$(CXX) CCC=$(CXX) \ - ./build.sh -o --disable-tests --static --python=python3 \ - --with-nspr=$(nss_install_dir)/include/nspr:$(nss_install_dir)/lib \ - --target=$(host_cpu) \ - -Duse_system_zlib=0 \ - -Dsign_libs=0 -endif - # Hack for macOS: Remove dynamic libraries to force the linker to use the - # static ones when linking curl. - rm -Rf $(nss_install_dir)/lib/*.dylib - - boringssl.zip: curl -L https://github.com/google/boringssl/archive/$(BORING_SSL_COMMIT).zip \ -o boringssl.zip # Patch boringssl and use a dummy '.patched' file to mark it patched -boringssl/.patched: $(srcdir)/chrome/patches/boringssl-*.patch +boringssl/.patched: $(srcdir)/chrome/patches/boringssl.patch unzip -q -o boringssl.zip mv boringssl-$(BORING_SSL_COMMIT) boringssl cd boringssl/ @@ -328,77 +223,23 @@ $(nghttp2_static_libs): $(NGHTTP2_VERSION).tar.bz2 $(MAKE) MAKEFLAGS=-j$(SUBJOBS) $(MAKE) install MAKEFLAGS= -$(CURL_VERSION).tar.xz: - curl -L "https://curl.se/download/$(CURL_VERSION).tar.xz" \ - -o "$(CURL_VERSION).tar.xz" - -# Apply the "Firefox version" patches and mark using a dummy file -$(CURL_VERSION)/.patched-ff: $(srcdir)/firefox/patches/curl-*.patch - rm -Rf $(CURL_VERSION) - tar -xf $(CURL_VERSION).tar.xz - cd $(CURL_VERSION) - for p in $^; do patch -p1 < $$p; done - # Re-generate the configure script - autoreconf -fi - touch .patched-ff - rm -f .patched-chrome +$(CURL_VERSION).tar.gz: + curl -L "https://github.com/curl/curl/archive/$(CURL_VERSION).tar.gz" \ + -o "$(CURL_VERSION).tar.gz" # Apply the "Chorme version" patches and mark using a dummy file $(CURL_VERSION)/.patched-chrome: $(srcdir)/chrome/patches/curl-*.patch rm -Rf $(CURL_VERSION) - tar -xf $(CURL_VERSION).tar.xz + tar -xf $(CURL_VERSION).tar.gz + mv curl-$(CURL_VERSION) $(CURL_VERSION) # fix directory name cd $(CURL_VERSION) for p in $^; do patch -p1 < $$p; done # Re-generate the configure script autoreconf -fi touch .patched-chrome - rm -f .patched-ff - -# This is a small hack that flags that curl was patched and configured in the "firefox" version -$(CURL_VERSION)/.firefox: $(firefox_libs) $(CURL_VERSION).tar.xz $(CURL_VERSION)/.patched-ff - cd $(CURL_VERSION) - - # Set up the configure flags to curl. - # If the user provided the --host flag to our configure script - # (for cross compilation), then pass it on to curl. - { \ - config_flags="--prefix=@prefix@"; \ - config_flags+=" --with-nghttp2=$(nghttp2_install_dir)"; \ - config_flags+=" --with-brotli=$(brotli_install_dir)"; \ - config_flags+=" --with-nss=$(nss_install_dir) --with-nss-deprecated"; \ - config_flags+=" --without-zstd"; \ - config_flags+=" --enable-websockets"; \ - config_flags+=" --enable-ipv6"; \ - config_flags+=" USE_CURL_SSLKEYLOGFILE=true"; \ - if test "$(static_build)" = "yes"; then \ - config_flags+=" --enable-static --disable-shared"; \ - fi; \ - if test -n "$(host_alias)"; then \ - config_flags+=" --host=$(host_alias)"; \ - fi; \ - if test -n "$(with_zlib)"; then \ - config_flags+=" --with-zlib=$(with_zlib)"; \ - else \ - config_flags+=" --with-zlib"; \ - fi; \ - if test -n "$(with_libnssckbi)"; then \ - config_flags+=" --with-libnssckbi=$(with_libnssckbi)"; \ - fi; \ - add_cflags="-I$(nss_install_dir)/../public/nss"; \ - add_cflags+=" -I$(nss_install_dir)/include/nspr"; \ - } - - echo "Configuring curl with: $$config_flags" - - ./configure $$config_flags CFLAGS="$(CFLAGS) $$add_cflags" - # Remove possible leftovers from a previous compilation - $(MAKE) clean MAKEFLAGS= - touch .firefox - # Remove the Chrome flag if it exists - rm -f .chrome # This is a small hack that flags that curl was patched and configured in the "chrome" version -$(CURL_VERSION)/.chrome: $(chrome_libs) $(CURL_VERSION).tar.xz $(CURL_VERSION)/.patched-chrome +$(CURL_VERSION)/.chrome: $(chrome_libs) $(CURL_VERSION).tar.gz $(CURL_VERSION)/.patched-chrome cd $(CURL_VERSION) # Set up the configure flags to curl. @@ -409,7 +250,6 @@ $(CURL_VERSION)/.chrome: $(chrome_libs) $(CURL_VERSION).tar.xz $(CURL_VERSION)/. config_flags="$$config_flags --with-nghttp2=$(nghttp2_install_dir)"; \ config_flags="$$config_flags --with-brotli=$(brotli_install_dir)"; \ config_flags="$$config_flags --with-openssl=$(boringssl_install_dir)"; \ - config_flags="$$config_flags --without-zstd"; \ config_flags="$$config_flags --without-libidn2"; \ config_flags="$$config_flags --enable-websockets"; \ config_flags="$$config_flags --enable-ech"; \ @@ -426,6 +266,11 @@ $(CURL_VERSION)/.chrome: $(chrome_libs) $(CURL_VERSION).tar.xz $(CURL_VERSION)/. else \ config_flags+=" --with-zlib"; \ fi; \ + if test -n "$(with_zstd)"; then \ + config_flags="$$config_flags --with-zstd=$(with_zstd)"; \ + else \ + config_flags+=" --with-zstd"; \ + fi; \ if test -n "$(with_ca_bundle)"; then \ config_flags="$$config_flags --with-ca-bundle=$(with_ca_bundle)"; \ fi; \ @@ -442,5 +287,3 @@ $(CURL_VERSION)/.chrome: $(chrome_libs) $(CURL_VERSION).tar.xz $(CURL_VERSION)/. # Remove possible leftovers from a previous compilation $(MAKE) clean MAKEFLAGS= touch .chrome - # Remove the Firefox flag if it exists - rm -f .firefox diff --git a/README.md b/README.md index 48bf4f86..366a9ee1 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,25 @@ -# curl-impersonate ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") ![Safari](https://github.com/alrra/browser-logos/blob/main/src/safari/safari_24x24.png "Safari") +# curl-impersonate ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") ![Safari](https://github.com/alrra/browser-logos/blob/main/src/safari/safari_24x24.png "Safari") [![Build and test](https://github.com/yifeikong/curl-impersonate/actions/workflows/build-and-test-make.yml/badge.svg)](https://github.com/yifeikong/curl-impersonate/actions/workflows/build-and-test-make.yml) [![Docker images](https://github.com/yifekong/curl-impersonate/actions/workflows/build-and-test-docker.yml/badge.svg)](https://github.com/yifeikong/curl-impersonate/actions/workflows/build-and-test-docker.yml) > [!NOTE] -> This is a maintained fork of [curl-impersonate](https://github.com/lwthiker/curl-impersonate). Chrome's fingerprints has changed a lot since the last update of upstream, use this fork if your impersonation is not working. +> This is a (slightly) more active fork of [curl-impersonate](https://github.com/lwthiker/curl-impersonate). +> Differences include: +> +> 1. Encrypted Client Hello(ECH) support introduced in Chrome 119. +> 2. ZSTD compression support introduced in Chrome 123. +> 3. X25519Kyber768 curve introduced in Chrome 124. +> 4. More options for impersonation Akamai http/2 fingerprints, especially for Safari. +> 5. Upgrade to more recent version of curl, 8.5.0 as of April, 2024. +> 6. Ability to change extension orders and enable/disable TLS grease. +> 7. (In progress) Single binary to support both Webkit-based and Gecko-based browsers, i.e. Chrome and Firefox. -A special build of [curl](https://github.com/curl/curl) that can impersonate the four major browsers: Chrome, Edge, Safari & Firefox. curl-impersonate is able to perform TLS and HTTP handshakes that are identical to that of a real browser. +A special build of [curl](https://github.com/curl/curl) that can impersonate the four major browsers: Chrome, Edge, Safari and Firefox(In progress). `curl-impersonate` is able to perform TLS and HTTP handshakes that are identical to that of a real browser. -curl-impersonate can be used either as a command line tool, similar to the regular curl, or as a library that can be integrated instead of the regular libcurl. See [Usage](#Basic-usage) below. +`curl-impersonate` can be used either as a command line tool, similar to the regular curl, or as a library that can be integrated instead of the regular libcurl. See [Usage](#Basic-usage) below. ## Why? + When you use an HTTP client with a TLS website, it first performs a TLS handshake. The first message of that handshake is called Client Hello. The Client Hello message that most HTTP clients and libraries produce differs drastically from that of a real browser. If the server uses HTTP/2, then in addition to the TLS handshake there is also an HTTP/2 handshake where various settings are exchanged. The settings that most HTTP clients and libraries use differ as well from those of any real browsers. @@ -21,7 +31,7 @@ With the modified curl in this repository, the TLS and HTTP handshakes look *exa ## How? To make this work, `curl` was patched significantly to resemble a browser. Specifically, The modifications that were needed to make this work: -* Compiling curl with nss, the TLS library that Firefox uses, instead of OpenSSL. For the Chrome version, compiling with BoringSSL, Google's TLS library. +* Compiling with BoringSSL, Google's TLS library, which is used by Chrome and Safari. * Modifying the way curl configures various TLS extensions and SSL options. * Adding support for new TLS extensions. * Changing the settings that curl uses for its HTTP/2 connections. @@ -44,16 +54,11 @@ The following browsers can be impersonated. | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 116 | 116.0.5845.180 | Windows 10 | `chrome116` | [curl_chrome116](chrome/curl_chrome116) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 119 | 119.0.6045.199 | macOS Sonoma | `chrome119` | [curl_chrome119](chrome/curl_chrome119) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 120 | 120.0.6099.109 | macOS Sonoma | `chrome120` | [curl_chrome120](chrome/curl_chrome120) | +| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 123 | 123.0.6312.124 | macOS Sonoma | `chrome123` | [curl_chrome123](chrome/curl_chrome123) | +| ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 124 | 124.0.6367.60 | macOS Sonoma | `chrome124` | [curl_chrome124](chrome/curl_chrome124) | | ![Chrome](https://raw.githubusercontent.com/alrra/browser-logos/main/src/chrome/chrome_24x24.png "Chrome") | 99 | 99.0.4844.73 | Android 12 | `chrome99_android` | [curl_chrome99_android](chrome/curl_chrome99_android) | | ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 99 | 99.0.1150.30 | Windows 10 | `edge99` | [curl_edge99](chrome/curl_edge99) | | ![Edge](https://raw.githubusercontent.com/alrra/browser-logos/main/src/edge/edge_24x24.png "Edge") | 101 | 101.0.1210.47 | Windows 10 | `edge101` | [curl_edge101](chrome/curl_edge101) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 91 ESR | 91.6.0esr | Windows 10 | `ff91esr` | [curl_ff91esr](firefox/curl_ff91esr) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 95 | 95.0.2 | Windows 10 | `ff95` | [curl_ff95](firefox/curl_ff95) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 98 | 98.0 | Windows 10 | `ff98` | [curl_ff98](firefox/curl_ff98) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 100 | 100.0 | Windows 10 | `ff100` | [curl_ff100](firefox/curl_ff100) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 102 | 102.0 | Windows 10 | `ff102` | [curl_ff102](firefox/curl_ff102) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 109 | 109.0 | Windows 10 | `ff109` | [curl_ff109](firefox/curl_ff109) | -| ![Firefox](https://raw.githubusercontent.com/alrra/browser-logos/main/src/firefox/firefox_24x24.png "Firefox") | 117 | 117.0.1 | Windows 10 | `ff117` | [curl_ff117](firefox/curl_ff117) | | ![Safari](https://github.com/alrra/browser-logos/blob/main/src/safari/safari_24x24.png "Safari") | 15.3 | 16612.4.9.1.8 | MacOS Big Sur | `safari15_3` | [curl_safari15_3](chrome/curl_safari15_3) | | ![Safari](https://github.com/alrra/browser-logos/blob/main/src/safari/safari_24x24.png "Safari") | 15.5 | 17613.2.7.1.8 | MacOS Monterey | `safari15_5` | [curl_safari15_5](chrome/curl_safari15_5) | | ![Safari](https://github.com/alrra/browser-logos/blob/main/src/safari/safari_24x24.png "Safari") | 17.0 | unclear | MacOS Sonoma | `safari17_0` | [curl_safari17_0](chrome/curl_safari17_0) | @@ -66,14 +71,14 @@ Notes: 2. The original Safari fingerprints in the upstream fork are [not correct](https://github.com/lwthiker/curl-impersonate/issues/215). -This list is also available in the [browsers.json](browsers.json) file. +~~This list is also available in the [browsers.json](browsers.json) file.()~~Needs to be updated. ## Basic usage For each supported browser there is a wrapper script that launches `curl-impersonate` with all the needed headers and flags. For example: -``` -curl_chrome119 https://www.wikipedia.org -``` + + curl_chrome123 https://www.wikipedia.org + You can add command line flags and they will be passed on to curl. However, some flags change curl's TLS signature which may cause it to be detected. Please note that the wrapper scripts use a default set of HTTP headers. If you want to change these headers, you may want to modify the wrapper scripts to fit your own purpose. @@ -85,25 +90,26 @@ See [Advanced usage](#Advanced-usage) for more options, including using `libcurl More documentation is available in the [docs/](docs/README.md) directory. ## Installation -There are two versions of `curl-impersonate` for technical reasons. The **chrome** version is used to impersonate Chrome, Edge and Safari. The **firefox** version is used to impersonate Firefox. +There are two versions of `curl-impersonate` for technical reasons. The **chrome** version is used to impersonate Chrome, Edge and Safari. ### Pre-compiled binaries -Pre-compiled binaries for Windows, Linux and macOS are available at the [GitHub releases](https://github.com/yifeikong/curl-impersonate/releases) page. Before you use them you need to install nss (Firefox's TLS library) and CA certificates: -* Ubuntu - `sudo apt install libnss3 nss-plugin-pem ca-certificates` -* Red Hat/Fedora/CentOS - `yum install nss nss-pem ca-certificates` -* Archlinux - `pacman -S nss ca-certificates` -* macOS - `brew install nss ca-certificates` +Pre-compiled binaries for Windows, Linux and macOS are available at the [GitHub releases](https://github.com/yifeikong/curl-impersonate/releases) page. Before you use them you may need to install zstd and CA certificates: + +* Ubuntu - `sudo apt install ca-certificates zstd libzstd-dev` +* Red Hat/Fedora/CentOS - `yum install ca-certificates zstd libzstd-devel` +* Archlinux - `pacman -S ca-certificates zstd` +* macOS - `brew install ca-certificates zstd` The pre-compiled binaries contain libcurl-impersonate and a statically compiled curl-impersonate for ease of use. The pre-compiled Linux binaries are built for Ubuntu systems. On other distributions if you have errors with certificate verification you may have to tell curl where to find the CA certificates. For example: -``` -curl_chrome116 https://www.wikipedia.org --cacert /etc/ssl/certs/ca-bundle.crt -``` + + curl_chrome123 https://www.wikipedia.org --cacert /etc/ssl/certs/ca-bundle.crt Also make sure to read [Notes on Dependencies](#notes-on-dependencies). ### Building from source + See [INSTALL.md](INSTALL.md). ### Docker images @@ -112,58 +118,71 @@ See [INSTALL.md](INSTALL.md). > New docker images added in this fork are work in progress. Docker images based on Alpine Linux and Debian with `curl-impersonate` compiled and ready to use are available on [Docker Hub](https://hub.docker.com/r/lwthiker/curl-impersonate). The images contain the binary and all the wrapper scripts. Use like the following: -```bash -# Firefox version, Alpine Linux -docker pull lwthiker/curl-impersonate:0.5-ff -docker run --rm lwthiker/curl-impersonate:0.5-ff curl_ff109 https://www.wikipedia.org +```bash # Chrome version, Alpine Linux docker pull lwthiker/curl-impersonate:0.5-chrome docker run --rm lwthiker/curl-impersonate:0.5-chrome curl_chrome110 https://www.wikipedia.org ``` ### Distro packages + +> [!WARNING] +> This is for the upstream project + AUR packages are available to Archlinux users: * Pre-compiled package: [curl-impersonate-bin](https://aur.archlinux.org/packages/curl-impersonate-bin), [libcurl-impersonate-bin](https://aur.archlinux.org/packages/libcurl-impersonate-bin). * Build from source code: [curl-impersonate-chrome](https://aur.archlinux.org/packages/curl-impersonate-chrome), [curl-impersonate-firefox](https://aur.archlinux.org/packages/curl-impersonate-firefox). ## Advanced usage ### libcurl-impersonate + `libcurl-impersonate.so` is libcurl compiled with the same changes as the command line `curl-impersonate`. + It has an additional API function: + ```c CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, int default_headers); ``` -You can call it with the target names, e.g. `chrome116`, and it will internally set all the options and headers that are otherwise set by the wrapper scripts. + +You can call it with the target names, e.g. `chrome123`, and it will internally set all the options and headers that are otherwise set by the wrapper scripts. If `default_headers` is set to 0, the built-in list of HTTP headers will not be set, and the user is expected to provide them instead using the regular [`CURLOPT_HTTPHEADER`](https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html) libcurl option. Calling the above function sets the following libcurl options: + * `CURLOPT_HTTP_VERSION` -* `CURLOPT_SSLVERSION`, `CURLOPT_SSL_CIPHER_LIST`, `CURLOPT_SSL_EC_CURVES`, `CURLOPT_SSL_ENABLE_NPN`, `CURLOPT_SSL_ENABLE_ALPN` +* `CURLOPT_SSLVERSION`, +* `CURLOPT_SSL_CIPHER_LIST`, +* `CURLOPT_SSL_EC_CURVES`, +* `CURLOPT_SSL_ENABLE_NPN`, +* `CURLOPT_SSL_ENABLE_ALPN` * `CURLOPT_HTTPBASEHEADER`, if `default_headers` is non-zero (this is a non-standard HTTP option created for this project). * `CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER`, sets http2 pseudo header order, for exmaple: `masp` (non-standard HTTP/2 options created for this project). * `CURLOPT_HTTP2_SETTINGS` sets the settings frame values, for example `1:65536;3:1000;4:6291456;6:262144` (non-standard HTTP/2 options created for this project). * `CURLOPT_HTTP2_WINDOW_UPDATE` sets intial window update value for http2, for example `15663105` (non-standard HTTP/2 options created for this project). * `CURLOPT_SSL_ENABLE_ALPS`, `CURLOPT_SSL_SIG_HASH_ALGS`, `CURLOPT_SSL_CERT_COMPRESSION`, `CURLOPT_SSL_ENABLE_TICKET` (non-standard TLS options created for this project). -* `CURLOPT_SSL_PERMUTE_EXTENSIONS` (non-standard TLS options created for this project). +* `CURLOPT_SSL_PERMUTE_EXTENSIONS`, whether to permute extensions like Chrome 110+. (non-standard TLS options created for this project). +* `CURLOPT_TLS_GREASE`, whether to enable the grease behavior. (non-standard TLS options created for this project). +* `CURLOPT_TLS_EXTENSION_ORDER`, explicit order or TLS extensions, in the format of `0-5-10`. (non-standard TLS options created for this project). + Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`. ### Using CURL_IMPERSONATE env var If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD` (Linux only). You can then set the `CURL_IMPERSONATE` env var. For example: -```bash -LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 my_app -``` + + LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 my_app + The `CURL_IMPERSONATE` env var has two effects: + * `curl_easy_impersonate()` is called automatically for any new curl handle created by `curl_easy_init()`. * `curl_easy_impersonate()` is called automatically after any `curl_easy_reset()` call. This means that all the options needed for impersonation will be automatically set for any curl handle. If you need precise control over the HTTP headers, set `CURL_IMPERSONATE_HEADERS=no` to disable the built-in list of HTTP headers, then set them yourself with `curl_easy_setopt()`. For example: -```bash -LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 CURL_IMPERSONATE_HEADERS=no my_app -``` + + LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome116 CURL_IMPERSONATE_HEADERS=no my_app Note that the `LD_PRELOAD` method will NOT WORK for `curl` itself because the curl tool overrides the TLS settings. Use the wrapper scripts instead. @@ -174,19 +193,21 @@ In particular, see the [note about the Firefox version](INSTALL.md#a-note-about- ## Contents -This repository contains two main folders: +This repository contains these folders: * [chrome](chrome) - Scripts and patches for building the Chrome version of `curl-impersonate`. -* [firefox](firefox) - Scripts and patches for building the Firefox version of `curl-impersonate`. - -The layout is similar for both. For example, the Firefox directory contains: -* [Dockerfile](firefox/Dockerfile) - Used to build `curl-impersonate` with all dependencies. -* [curl_ff91esr](firefox/curl_ff91esr), [curl_ff95](firefox/curl_ff95), [curl_ff98](firefox/curl_ff98) - Wrapper scripts that launch `curl-impersonate` with the correct flags. -* [curl-impersonate.patch](firefox/patches/curl-impersonate.patch) - The main patch that makes curl use the same TLS extensions as Firefox. Also makes curl compile statically with libnghttp2 and libnss. + * [curl_chrome110](chrome/curl_chrome110), [curl_chrome124](chrome/curl_chrome124) - Wrapper scripts that launch `curl-impersonate` with the correct flags. + * [curl-impersonate.patch](chrome/patches/curl-impersonate.patch) - The main patch that makes curl use the same TLS extensions as Firefox. Also makes curl compile statically with libnghttp2. + * [boringssl.patch](chrome/patches/boringssl.patch) - The boringssl patch that tweaks boringssl behaviors. +* [win](win) - Scripts for building the Windows version of `curl-impersonate`, which is quite different from `*nix`. +* [zigshim](zigshim) - We use the awesome `zig` toolchain to bring `curl-impersonate` to more archs on Linux. Special thanks to @bjia56 for making it possible. +* [docker](docker) - Debian and alpine dockerfiles for this project. Other files of interest: + * [tests/signatures](tests/signatures) - YAML database of known browser signatures that can be impersonated. ## Contributing + If you'd like to help, please check out the [open issues in the origional repo](https://github.com/lwthiker/curl-impersonate/issues) and [open issues here](https://github.com/yifeikong/curl-impersonate/issues). You can open a pull request with your changes. Note that some of the upstream issues have been fixed. This repository contains the build process for `curl-impersonate`. The actual patches to `curl` are maintained in a [separate repository](https://github.com/yifeikong/curl) forked from lwthiker's fork of the upstream curl. The changes are maintained in the [impersonate-firefox](https://github.com/yifeikong/curl/tree/impersonate-firefox) and [impersonate-chrome](https://github.com/yifeikong/curl/tree/impersonate-chrome) branches. diff --git a/chrome/curl_chrome100 b/chrome/curl_chrome100 index 6f81a1f4..085e1a63 100755 --- a/chrome/curl_chrome100 +++ b/chrome/curl_chrome100 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome101 b/chrome/curl_chrome101 index 9921089d..d77ef41d 100755 --- a/chrome/curl_chrome101 +++ b/chrome/curl_chrome101 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome104 b/chrome/curl_chrome104 index 7b04cd81..0d9ed592 100755 --- a/chrome/curl_chrome104 +++ b/chrome/curl_chrome104 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome107 b/chrome/curl_chrome107 index 7ee02c2f..32fad580 100755 --- a/chrome/curl_chrome107 +++ b/chrome/curl_chrome107 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome110 b/chrome/curl_chrome110 index 79472092..50826dbd 100755 --- a/chrome/curl_chrome110 +++ b/chrome/curl_chrome110 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps --tls-permute-extensions \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome116 b/chrome/curl_chrome116 index df9fd879..d5b14d45 100755 --- a/chrome/curl_chrome116 +++ b/chrome/curl_chrome116 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps --tls-permute-extensions \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome119 b/chrome/curl_chrome119 index 7875027a..dd3629c9 100755 --- a/chrome/curl_chrome119 +++ b/chrome/curl_chrome119 @@ -27,4 +27,5 @@ dir=${0%/*} --ech GREASE \ --tlsv1.2 --alps --tls-permute-extensions \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome120 b/chrome/curl_chrome120 index a9b1d18b..8218e242 100755 --- a/chrome/curl_chrome120 +++ b/chrome/curl_chrome120 @@ -27,4 +27,5 @@ dir=${0%/*} --ech GREASE \ --tlsv1.2 --alps --tls-permute-extensions \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome123 b/chrome/curl_chrome123 new file mode 100755 index 00000000..a4c50cc1 --- /dev/null +++ b/chrome/curl_chrome123 @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Updates in this version: +# 1. Added zstd encoding support, see: https://chromestatus.com/feature/6186023867908096 + +# Find the directory of this script +dir=${0%/*} + +# The list of ciphers can be obtained by looking at the Client Hello message in +# Wireshark, then converting it using this reference +# https://wiki.mozilla.org/Security/Cipher_Suites +"$dir/curl-impersonate-chrome" \ + --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \ + -H 'sec-ch-ua: "Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept-Encoding: gzip, deflate, br, zstd' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + --http2 \ + --http2-settings '1:65536;2:0;4:6291456;6:262144' \ + --http2-window-update 15663105 \ + --compressed \ + --ech GREASE \ + --tlsv1.2 --alps --tls-permute-extensions \ + --cert-compression brotli \ + --tls-grease \ + "$@" diff --git a/chrome/curl_chrome124 b/chrome/curl_chrome124 new file mode 100755 index 00000000..0b481ad6 --- /dev/null +++ b/chrome/curl_chrome124 @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Updates in this version: +# 1. Added X25519Kyber768 support +# 2. Added Priority header support + +# Find the directory of this script +dir=${0%/*} + +# The list of ciphers can be obtained by looking at the Client Hello message in +# Wireshark, then converting it using this reference +# https://wiki.mozilla.org/Security/Cipher_Suites +"$dir/curl-impersonate-chrome" \ + --ciphers TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA \ + --curves X25519Kyber768Draft00:X25519:P-256:P-384 \ + -H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \ + -H 'sec-ch-ua-mobile: ?0' \ + -H 'sec-ch-ua-platform: "macOS"' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Accept-Encoding: gzip, deflate, br, zstd' \ + -H 'Accept-Language: en-US,en;q=0.9' \ + -H 'Priority: u=0, i' \ + --http2 \ + --http2-settings '1:65536;2:0;4:6291456;6:262144' \ + --http2-window-update 15663105 \ + --compressed \ + --ech GREASE \ + --tlsv1.2 --alps --tls-permute-extensions \ + --cert-compression brotli \ + --tls-grease \ + "$@" diff --git a/chrome/curl_chrome99 b/chrome/curl_chrome99 index ea1c537a..20ef1cf4 100755 --- a/chrome/curl_chrome99 +++ b/chrome/curl_chrome99 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_chrome99_android b/chrome/curl_chrome99_android index 0e56cd7a..79277699 100755 --- a/chrome/curl_chrome99_android +++ b/chrome/curl_chrome99_android @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_edge101 b/chrome/curl_edge101 index ff3c261c..6193dec1 100755 --- a/chrome/curl_edge101 +++ b/chrome/curl_edge101 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_edge99 b/chrome/curl_edge99 index 696a848b..cd8e1a65 100755 --- a/chrome/curl_edge99 +++ b/chrome/curl_edge99 @@ -26,4 +26,5 @@ dir=${0%/*} --compressed \ --tlsv1.2 --alps \ --cert-compression brotli \ + --tls-grease \ "$@" diff --git a/chrome/curl_firefox120 b/chrome/curl_firefox120 new file mode 100755 index 00000000..68a1274c --- /dev/null +++ b/chrome/curl_firefox120 @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Find the directory of this script +dir=${0%/*} + +# The list of ciphers can be obtained by looking at the Client Hello message in +# Wireshark, then converting it using the cipherlist array at +# https://github.com/curl/curl/blob/master/lib/vtls/nss.c +"$dir/curl-impersonate-chrome" \ + --ciphers "TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA" \ + -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/120.0' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ + -H 'Accept-Language: en-US,en;q=0.5' \ + -H 'Accept-Encoding: gzip, deflate, br' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'Sec-Fetch-Dest: document' \ + -H 'Sec-Fetch-Mode: navigate' \ + -H 'Sec-Fetch-Site: none' \ + -H 'Sec-Fetch-User: ?1' \ + -H 'TE: Trailers' \ + --http2 \ + --http2-settings '1:65536;4:131072;5:16384' \ + --http2-streams '3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241' \ + --http2-window-update 12517377 \ + --http2-pseudo-headers-order 'mpas' \ + --compressed \ + --ech GREASE \ + "$@" diff --git a/chrome/curl_safari15_3 b/chrome/curl_safari15_3 index 982e3b28..0d78f445 100755 --- a/chrome/curl_safari15_3 +++ b/chrome/curl_safari15_3 @@ -20,4 +20,5 @@ dir=${0%/*} --http2-window-update 10485760 \ --compressed \ --tlsv1.0 --no-tls-session-ticket \ + --tls-grease \ "$@" diff --git a/chrome/curl_safari15_5 b/chrome/curl_safari15_5 index 83100dc3..a565cd6a 100755 --- a/chrome/curl_safari15_5 +++ b/chrome/curl_safari15_5 @@ -21,4 +21,5 @@ dir=${0%/*} --compressed \ --tlsv1.0 --no-tls-session-ticket \ --cert-compression zlib \ + --tls-grease \ "$@" diff --git a/chrome/curl_safari17_0 b/chrome/curl_safari17_0 index 66f7dc03..7c06c1dc 100755 --- a/chrome/curl_safari17_0 +++ b/chrome/curl_safari17_0 @@ -24,4 +24,5 @@ dir=${0%/*} --compressed \ --tlsv1.0 --no-tls-session-ticket \ --cert-compression zlib \ + --tls-grease \ "$@" diff --git a/chrome/curl_safari17_2_ios b/chrome/curl_safari17_2_ios index d579b09a..ca5c576d 100755 --- a/chrome/curl_safari17_2_ios +++ b/chrome/curl_safari17_2_ios @@ -24,4 +24,5 @@ dir=${0%/*} --compressed \ --tlsv1.0 --no-tls-session-ticket \ --cert-compression zlib \ + --tls-grease \ "$@" diff --git a/chrome/patches/boringssl-old-ciphers.patch b/chrome/patches/boringssl-old-ciphers.patch deleted file mode 100644 index 2b278299..00000000 --- a/chrome/patches/boringssl-old-ciphers.patch +++ /dev/null @@ -1,209 +0,0 @@ -diff --git a/ssl/internal.h b/ssl/internal.h -index c9facb699..ec3c21fed 100644 ---- a/ssl/internal.h -+++ b/ssl/internal.h -@@ -574,9 +574,14 @@ BSSL_NAMESPACE_BEGIN - - // Bits for |algorithm_mac| (symmetric authentication). - #define SSL_SHA1 0x00000001u -+// curl-impersonate: -+// SSL_SHA256 and SSL_SHA384 were removed in -+// https://boringssl-review.googlesource.com/c/boringssl/+/27944/ -+// but restored to impersonate browsers with older ciphers. - #define SSL_SHA256 0x00000002u -+#define SSL_SHA384 0x00000004u - // SSL_AEAD is set for all AEADs. --#define SSL_AEAD 0x00000004u -+#define SSL_AEAD 0x00000008u - - // Bits for |algorithm_prf| (handshake digest). - #define SSL_HANDSHAKE_MAC_DEFAULT 0x1 -diff --git a/ssl/ssl_cipher.cc b/ssl/ssl_cipher.cc -index fd8cef95d..1d6ffe88b 100644 ---- a/ssl/ssl_cipher.cc -+++ b/ssl/ssl_cipher.cc -@@ -197,6 +197,37 @@ static constexpr SSL_CIPHER kCiphers[] = { - SSL_HANDSHAKE_MAC_DEFAULT, - }, - -+ // curl-impersonate: Ciphers 3C, 3D were removed in -+ // https://boringssl-review.googlesource.com/c/boringssl/+/27944/ -+ // but restored here to impersonate browsers with older ciphers. They are -+ // not expected to actually work; but just to be included in the TLS -+ // Client Hello. -+ -+ // TLS v1.2 ciphersuites -+ -+ // Cipher 3C -+ { -+ TLS1_TXT_RSA_WITH_AES_128_SHA256, -+ "TLS_RSA_WITH_AES_128_CBC_SHA256", -+ TLS1_CK_RSA_WITH_AES_128_SHA256, -+ SSL_kRSA, -+ SSL_aRSA, -+ SSL_AES128, -+ SSL_SHA256, -+ SSL_HANDSHAKE_MAC_SHA256, -+ }, -+ // Cipher 3D -+ { -+ TLS1_TXT_RSA_WITH_AES_256_SHA256, -+ "TLS_RSA_WITH_AES_256_CBC_SHA256", -+ TLS1_CK_RSA_WITH_AES_256_SHA256, -+ SSL_kRSA, -+ SSL_aRSA, -+ SSL_AES256, -+ SSL_SHA256, -+ SSL_HANDSHAKE_MAC_SHA256, -+ }, -+ - // PSK cipher suites. - - // Cipher 8C -@@ -287,6 +318,23 @@ static constexpr SSL_CIPHER kCiphers[] = { - SSL_HANDSHAKE_MAC_SHA256, - }, - -+ // curl-impersonate: Cipher C008 was missing from BoringSSL, -+ // probably because it is weak. Add it back from OpenSSL (ssl/s3_lib.c) -+ // where it is called ECDHE-ECDSA-DES-CBC3-SHA. -+ // It's not supposed to really work but just appear in the TLS client hello. -+ -+ // Cipher C008 -+ { -+ "ECDHE-ECDSA-DES-CBC3-SHA", -+ "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA", -+ 0x0300C008, -+ SSL_kECDHE, -+ SSL_aECDSA, -+ SSL_3DES, -+ SSL_SHA1, -+ SSL_HANDSHAKE_MAC_DEFAULT, -+ }, -+ - // Cipher C009 - { - TLS1_TXT_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, -@@ -311,6 +359,21 @@ static constexpr SSL_CIPHER kCiphers[] = { - SSL_HANDSHAKE_MAC_DEFAULT, - }, - -+ // curl-impersonate: Cipher C012 was missing from BoringSSL, -+ // probably because it is weak. Add it back from OpenSSL (ssl/s3_lib.c) -+ // where it is called ECDHE-RSA-DES-CBC3-SHA -+ // It's not supposed to really work but just appear in the TLS client hello. -+ { -+ "ECDHE-RSA-DES-CBC3-SHA", -+ "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", -+ 0x0300C012, -+ SSL_kECDHE, -+ SSL_aRSA, -+ SSL_3DES, -+ SSL_SHA1, -+ SSL_HANDSHAKE_MAC_DEFAULT, -+ }, -+ - // Cipher C013 - { - TLS1_TXT_ECDHE_RSA_WITH_AES_128_CBC_SHA, -@@ -335,6 +398,36 @@ static constexpr SSL_CIPHER kCiphers[] = { - SSL_HANDSHAKE_MAC_DEFAULT, - }, - -+ // curl-impersonate: Ciphers C023, C024, C027, C028 were removed in -+ // https://boringssl-review.googlesource.com/c/boringssl/+/27944/ -+ // but restored here to impersonate browsers with older ciphers. They are -+ // not expected to actually work; but just to be included in the TLS -+ // Client Hello. -+ -+ // HMAC based TLS v1.2 ciphersuites from RFC5289 -+ -+ // Cipher C023 -+ { -+ TLS1_TXT_ECDHE_ECDSA_WITH_AES_128_SHA256, -+ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", -+ TLS1_CK_ECDHE_ECDSA_WITH_AES_128_SHA256, -+ SSL_kECDHE, -+ SSL_aECDSA, -+ SSL_AES128, -+ SSL_SHA256, -+ SSL_HANDSHAKE_MAC_SHA256, -+ }, -+ // Cipher C024 -+ { -+ TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_SHA384, -+ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384", -+ TLS1_CK_ECDHE_ECDSA_WITH_AES_256_SHA384, -+ SSL_kECDHE, -+ SSL_aECDSA, -+ SSL_AES256, -+ SSL_SHA384, -+ SSL_HANDSHAKE_MAC_SHA384, -+ }, - // Cipher C027 - { - TLS1_TXT_ECDHE_RSA_WITH_AES_128_CBC_SHA256, -@@ -346,6 +439,17 @@ static constexpr SSL_CIPHER kCiphers[] = { - SSL_SHA256, - SSL_HANDSHAKE_MAC_SHA256, - }, -+ // Cipher C028 -+ { -+ TLS1_TXT_ECDHE_RSA_WITH_AES_256_SHA384, -+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", -+ TLS1_CK_ECDHE_RSA_WITH_AES_256_SHA384, -+ SSL_kECDHE, -+ SSL_aRSA, -+ SSL_AES256, -+ SSL_SHA384, -+ SSL_HANDSHAKE_MAC_SHA384, -+ }, - - // GCM based TLS v1.2 ciphersuites from RFC 5289 - -@@ -554,6 +658,11 @@ static const CIPHER_ALIAS kCipherAliases[] = { - // MAC aliases - {"SHA1", ~0u, ~0u, ~0u, SSL_SHA1, 0}, - {"SHA", ~0u, ~0u, ~0u, SSL_SHA1, 0}, -+ // curl-impersonate: -+ // Removed in https://boringssl-review.googlesource.com/c/boringssl/+/27944/ -+ // but restored to impersonate browsers with older ciphers. -+ {"SHA256", ~0u, ~0u, ~0u, SSL_SHA256, 0}, -+ {"SHA384", ~0u, ~0u, ~0u, SSL_SHA384, 0}, - - // Legacy protocol minimum version aliases. "TLSv1" is intentionally the - // same as "SSLv3". -@@ -1154,13 +1263,20 @@ bool ssl_create_cipher_list(UniquePtr *out_cipher_list, - TLS1_CK_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 & 0xffff, - }; - static const uint16_t kLegacyCiphers[] = { -+ TLS1_CK_RSA_WITH_AES_128_SHA256 & 0xffff, -+ TLS1_CK_RSA_WITH_AES_256_SHA256 & 0xffff, -+ 0x0300C008 & 0xffff, - TLS1_CK_ECDHE_ECDSA_WITH_AES_128_CBC_SHA & 0xffff, -+ 0x0300C012 & 0xffff, - TLS1_CK_ECDHE_RSA_WITH_AES_128_CBC_SHA & 0xffff, - TLS1_CK_ECDHE_PSK_WITH_AES_128_CBC_SHA & 0xffff, - TLS1_CK_ECDHE_ECDSA_WITH_AES_256_CBC_SHA & 0xffff, - TLS1_CK_ECDHE_RSA_WITH_AES_256_CBC_SHA & 0xffff, - TLS1_CK_ECDHE_PSK_WITH_AES_256_CBC_SHA & 0xffff, -+ TLS1_CK_ECDHE_ECDSA_WITH_AES_128_SHA256 & 0xffff, // C023 -+ TLS1_CK_ECDHE_ECDSA_WITH_AES_256_SHA384 & 0xffff, // C024 - TLS1_CK_ECDHE_RSA_WITH_AES_128_CBC_SHA256 & 0xffff, -+ TLS1_CK_ECDHE_RSA_WITH_AES_256_SHA384 & 0xffff, // C028 - TLS1_CK_RSA_WITH_AES_128_GCM_SHA256 & 0xffff, - TLS1_CK_RSA_WITH_AES_256_GCM_SHA384 & 0xffff, - TLS1_CK_RSA_WITH_AES_128_SHA & 0xffff, -diff --git a/ssl/ssl_privkey.cc b/ssl/ssl_privkey.cc -index 57116cd6c..fa1652832 100644 ---- a/ssl/ssl_privkey.cc -+++ b/ssl/ssl_privkey.cc -@@ -594,7 +594,7 @@ static bool sigalgs_unique(Span in_sigalgs) { - - static bool set_sigalg_prefs(Array *out, Span prefs) { - if (!sigalgs_unique(prefs)) { -- return false; -+ // return false; - } - - // Check for invalid algorithms, and filter out |SSL_SIGN_RSA_PKCS1_MD5_SHA1|. diff --git a/chrome/patches/boringssl.patch b/chrome/patches/boringssl.patch new file mode 100644 index 00000000..85183aee --- /dev/null +++ b/chrome/patches/boringssl.patch @@ -0,0 +1,502 @@ +diff --git a/export.sh b/export.sh +new file mode 100755 +index 000000000..2e1f397aa +--- /dev/null ++++ b/export.sh +@@ -0,0 +1,4 @@ ++#!/bin/bash ++ ++git df d24a382 > boringssl.patch ++mv boringssl.patch ../curl-impersonate/chrome/patches/boringssl.patch +diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h +index e500dd76e..487945969 100644 +--- a/include/openssl/ssl.h ++++ b/include/openssl/ssl.h +@@ -1560,6 +1560,9 @@ OPENSSL_EXPORT int SSL_CTX_set_strict_cipher_list(SSL_CTX *ctx, + // garbage inputs, unless an empty cipher list results. + OPENSSL_EXPORT int SSL_CTX_set_cipher_list(SSL_CTX *ctx, const char *str); + ++// curl-impersonate: set the extension order by given string ++OPENSSL_EXPORT int SSL_CTX_set_extension_order(SSL_CTX *ctx, char *order); ++ + // SSL_set_strict_cipher_list configures the cipher list for |ssl|, evaluating + // |str| as a cipher string and returning error if |str| contains anything + // meaningless. It returns one on success and zero on failure. +@@ -4583,6 +4586,9 @@ OPENSSL_EXPORT void SSL_CTX_set_grease_enabled(SSL_CTX *ctx, int enabled); + // permute extensions. For now, this is only implemented for the ClientHello. + OPENSSL_EXPORT void SSL_CTX_set_permute_extensions(SSL_CTX *ctx, int enabled); + ++// curl-impersonate ++OPENSSL_EXPORT int SSL_CTX_set_extension_order(SSL_CTX *ctx, char *order); ++ + // SSL_set_permute_extensions configures whether sockets on |ssl| should + // permute extensions. For now, this is only implemented for the ClientHello. + OPENSSL_EXPORT void SSL_set_permute_extensions(SSL *ssl, int enabled); +diff --git a/ssl/extensions.cc b/ssl/extensions.cc +index b13400097..8b457b873 100644 +--- a/ssl/extensions.cc ++++ b/ssl/extensions.cc +@@ -3313,6 +3313,7 @@ bool ssl_setup_extension_permutation(SSL_HANDSHAKE *hs) { + !permutation.Init(kNumExtensions)) { + return false; + } ++ // By default, nothing is permuted. + for (size_t i = 0; i < kNumExtensions; i++) { + permutation[i] = i; + } +@@ -3320,6 +3321,11 @@ bool ssl_setup_extension_permutation(SSL_HANDSHAKE *hs) { + // Set element |i| to a randomly-selected element 0 <= j <= i. + std::swap(permutation[i], permutation[seeds[i - 1] % (i + 1)]); + } ++ // fprintf(stderr, "the permuated order is set to:"); ++ // for (size_t i = 0; i < kNumExtensions; i++) { ++ // fprintf(stderr, "%d, ", permutation[i]); ++ // } ++ // fprintf(stderr, "\n"); + hs->extension_permutation = std::move(permutation); + return true; + } +@@ -3337,6 +3343,50 @@ static const struct tls_extension *tls_extension_find(uint32_t *out_index, + return NULL; + } + ++// curl-impersonate: set customized extension order ++// ++// Generate the extension_permutation array from a customized extension order string. ++// ++// The customized extension order string is a dash-separated list of extensions. ++// ++bool ssl_set_extension_order(SSL_HANDSHAKE *hs) { ++ if (hs->config->extension_order == nullptr) { ++ return true; ++ } ++ // fprintf(stderr, "order %s\n", hs->config->extension_order); ++ Array order; ++ if (!order.Init(kNumExtensions)) { ++ return false; ++ } ++ // By default, nothing is reordered. ++ for (size_t i = 0; i < kNumExtensions; i++) { ++ order[i] = 255; ++ } ++ // split the order string, and put there order in the table ++ const char *delimiter = "-"; ++ char *tmp = strdup(hs->config->extension_order); ++ char *ext = strtok(tmp, delimiter); ++ size_t idx = 0; ++ while (ext != nullptr) { ++ unsigned ext_index; ++ tls_extension_find(&ext_index, atoi(ext)); ++ // fprintf(stderr, "found %d -> %d, ", atoi(ext), ext_index); ++ order[idx] = ext_index; ++ ext = strtok(NULL, delimiter); ++ idx++; ++ } ++ // fprintf(stderr, "\n"); ++ // fprintf(stderr, "the order is set to:"); ++ // for (size_t i = 0; i < kNumExtensions; i++) { ++ // fprintf(stderr, "%d, ", order[i]); ++ // } ++ // fprintf(stderr, "\n"); ++ free(tmp); ++ ++ hs->extension_permutation = std::move(order); ++ return true; ++} ++ + static bool add_padding_extension(CBB *cbb, uint16_t ext, size_t len) { + CBB child; + if (!CBB_add_u16(cbb, ext) || // +@@ -3383,6 +3433,9 @@ static bool ssl_add_clienthello_tlsext_inner(SSL_HANDSHAKE *hs, CBB *out, + size_t i = hs->extension_permutation.empty() + ? unpermuted + : hs->extension_permutation[unpermuted]; ++ // fprintf(stderr, "extension %zu,", i); ++ // fprintf(stderr, "\n"); ++ if (i == 255) { continue; } // curl-impersonate: skip non-exist extensions + const size_t len_before = CBB_len(&extensions); + const size_t len_compressed_before = CBB_len(compressed.get()); + if (!kExtensions[i].add_clienthello(hs, &extensions, compressed.get(), +@@ -3492,6 +3545,7 @@ bool ssl_add_clienthello_tlsext(SSL_HANDSHAKE *hs, CBB *out, CBB *out_encoded, + size_t i = hs->extension_permutation.empty() + ? unpermuted + : hs->extension_permutation[unpermuted]; ++ if (i == 255) { continue; } // curl-impersonate: skip non-exist extensions + const size_t len_before = CBB_len(&extensions); + if (!kExtensions[i].add_clienthello(hs, &extensions, &extensions, type)) { + OPENSSL_PUT_ERROR(SSL, SSL_R_ERROR_ADDING_EXTENSION); +diff --git a/ssl/handshake_client.cc b/ssl/handshake_client.cc +index 971ebd0b1..0005c6d79 100644 +--- a/ssl/handshake_client.cc ++++ b/ssl/handshake_client.cc +@@ -215,14 +215,6 @@ static void ssl_get_client_disabled(const SSL_HANDSHAKE *hs, + } + } + +-static bool ssl_add_tls13_cipher(CBB *cbb, uint16_t cipher_id, +- ssl_compliance_policy_t policy) { +- if (ssl_tls13_cipher_meets_policy(cipher_id, policy)) { +- return CBB_add_u16(cbb, cipher_id); +- } +- return true; +-} +- + static bool ssl_write_client_cipher_list(const SSL_HANDSHAKE *hs, CBB *out, + ssl_client_hello_type_t type) { + const SSL *const ssl = hs->ssl; +@@ -240,28 +232,7 @@ static bool ssl_write_client_cipher_list(const SSL_HANDSHAKE *hs, CBB *out, + return false; + } + +- // Add TLS 1.3 ciphers. Order ChaCha20-Poly1305 relative to AES-GCM based on +- // hardware support. +- if (hs->max_version >= TLS1_3_VERSION) { +- const bool has_aes_hw = ssl->config->aes_hw_override +- ? ssl->config->aes_hw_override_value +- : EVP_has_aes_hardware(); +- +- if ((!has_aes_hw && // +- !ssl_add_tls13_cipher(&child, +- TLS1_3_CK_CHACHA20_POLY1305_SHA256 & 0xffff, +- ssl->config->tls13_cipher_policy)) || +- !ssl_add_tls13_cipher(&child, TLS1_3_CK_AES_128_GCM_SHA256 & 0xffff, +- ssl->config->tls13_cipher_policy) || +- !ssl_add_tls13_cipher(&child, TLS1_3_CK_AES_256_GCM_SHA384 & 0xffff, +- ssl->config->tls13_cipher_policy) || +- (has_aes_hw && // +- !ssl_add_tls13_cipher(&child, +- TLS1_3_CK_CHACHA20_POLY1305_SHA256 & 0xffff, +- ssl->config->tls13_cipher_policy))) { +- return false; +- } +- } ++ // curl-impersonate: TLS 1.3 suites are added uniformly with other suites. + + if (hs->min_version < TLS1_3_VERSION && type != ssl_client_hello_inner) { + bool any_enabled = false; +@@ -539,6 +510,7 @@ static enum ssl_hs_wait_t do_start_connect(SSL_HANDSHAKE *hs) { + + if (!ssl_setup_key_shares(hs, /*override_group_id=*/0) || + !ssl_setup_extension_permutation(hs) || ++ !ssl_set_extension_order(hs) || + !ssl_encrypt_client_hello(hs, MakeConstSpan(ech_enc, ech_enc_len)) || + !ssl_add_client_hello(hs)) { + return ssl_hs_error; +diff --git a/ssl/internal.h b/ssl/internal.h +index c9facb699..eab61611e 100644 +--- a/ssl/internal.h ++++ b/ssl/internal.h +@@ -574,9 +574,14 @@ BSSL_NAMESPACE_BEGIN + + // Bits for |algorithm_mac| (symmetric authentication). + #define SSL_SHA1 0x00000001u ++// curl-impersonate: ++// SSL_SHA256 and SSL_SHA384 were removed in ++// https://boringssl-review.googlesource.com/c/boringssl/+/27944/ ++// but restored to impersonate browsers with older ciphers. + #define SSL_SHA256 0x00000002u ++#define SSL_SHA384 0x00000004u + // SSL_AEAD is set for all AEADs. +-#define SSL_AEAD 0x00000004u ++#define SSL_AEAD 0x00000008u + + // Bits for |algorithm_prf| (handshake digest). + #define SSL_HANDSHAKE_MAC_DEFAULT 0x1 +@@ -2161,6 +2166,9 @@ bssl::UniquePtr tls13_create_session_with_ticket(SSL *ssl, + // for |hs|, if applicable. It returns true on success and false on error. + bool ssl_setup_extension_permutation(SSL_HANDSHAKE *hs); + ++// curl-impersonate ++bool ssl_set_extension_order(SSL_HANDSHAKE *hs); ++ + // ssl_setup_key_shares computes client key shares and saves them in |hs|. It + // returns true on success and false on failure. If |override_group_id| is zero, + // it offers the default groups, including GREASE. If it is non-zero, it offers +@@ -3033,6 +3041,9 @@ struct SSL_CONFIG { + // crypto + UniquePtr cipher_list; + ++ // curl-impersonate ++ char *extension_order = nullptr; ++ + // This is used to hold the local certificate used (i.e. the server + // certificate for a server or the client certificate for a client). + UniquePtr cert; +@@ -3490,6 +3501,9 @@ struct ssl_ctx_st { + + bssl::UniquePtr cipher_list; + ++ // curl-impersonate ++ char *extension_order = nullptr; ++ + X509_STORE *cert_store = nullptr; + LHASH_OF(SSL_SESSION) *sessions = nullptr; + // Most session-ids that will be cached, default is +diff --git a/ssl/ssl_cipher.cc b/ssl/ssl_cipher.cc +index fd8cef95d..3d2c8ff6d 100644 +--- a/ssl/ssl_cipher.cc ++++ b/ssl/ssl_cipher.cc +@@ -197,6 +197,37 @@ static constexpr SSL_CIPHER kCiphers[] = { + SSL_HANDSHAKE_MAC_DEFAULT, + }, + ++ // curl-impersonate: Ciphers 3C, 3D were removed in ++ // https://boringssl-review.googlesource.com/c/boringssl/+/27944/ ++ // but restored here to impersonate browsers with older ciphers. They are ++ // not expected to actually work; but just to be included in the TLS ++ // Client Hello. ++ ++ // TLS v1.2 ciphersuites ++ ++ // Cipher 3C ++ { ++ TLS1_TXT_RSA_WITH_AES_128_SHA256, ++ "TLS_RSA_WITH_AES_128_CBC_SHA256", ++ TLS1_CK_RSA_WITH_AES_128_SHA256, ++ SSL_kRSA, ++ SSL_aRSA, ++ SSL_AES128, ++ SSL_SHA256, ++ SSL_HANDSHAKE_MAC_SHA256, ++ }, ++ // Cipher 3D ++ { ++ TLS1_TXT_RSA_WITH_AES_256_SHA256, ++ "TLS_RSA_WITH_AES_256_CBC_SHA256", ++ TLS1_CK_RSA_WITH_AES_256_SHA256, ++ SSL_kRSA, ++ SSL_aRSA, ++ SSL_AES256, ++ SSL_SHA256, ++ SSL_HANDSHAKE_MAC_SHA256, ++ }, ++ + // PSK cipher suites. + + // Cipher 8C +@@ -287,6 +318,23 @@ static constexpr SSL_CIPHER kCiphers[] = { + SSL_HANDSHAKE_MAC_SHA256, + }, + ++ // curl-impersonate: Cipher C008 was missing from BoringSSL, ++ // probably because it is weak. Add it back from OpenSSL (ssl/s3_lib.c) ++ // where it is called ECDHE-ECDSA-DES-CBC3-SHA. ++ // It's not supposed to really work but just appear in the TLS client hello. ++ ++ // Cipher C008 ++ { ++ "ECDHE-ECDSA-DES-CBC3-SHA", ++ "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA", ++ 0x0300C008, ++ SSL_kECDHE, ++ SSL_aECDSA, ++ SSL_3DES, ++ SSL_SHA1, ++ SSL_HANDSHAKE_MAC_DEFAULT, ++ }, ++ + // Cipher C009 + { + TLS1_TXT_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, +@@ -311,6 +359,21 @@ static constexpr SSL_CIPHER kCiphers[] = { + SSL_HANDSHAKE_MAC_DEFAULT, + }, + ++ // curl-impersonate: Cipher C012 was missing from BoringSSL, ++ // probably because it is weak. Add it back from OpenSSL (ssl/s3_lib.c) ++ // where it is called ECDHE-RSA-DES-CBC3-SHA ++ // It's not supposed to really work but just appear in the TLS client hello. ++ { ++ "ECDHE-RSA-DES-CBC3-SHA", ++ "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", ++ 0x0300C012, ++ SSL_kECDHE, ++ SSL_aRSA, ++ SSL_3DES, ++ SSL_SHA1, ++ SSL_HANDSHAKE_MAC_DEFAULT, ++ }, ++ + // Cipher C013 + { + TLS1_TXT_ECDHE_RSA_WITH_AES_128_CBC_SHA, +@@ -335,6 +398,36 @@ static constexpr SSL_CIPHER kCiphers[] = { + SSL_HANDSHAKE_MAC_DEFAULT, + }, + ++ // curl-impersonate: Ciphers C023, C024, C027, C028 were removed in ++ // https://boringssl-review.googlesource.com/c/boringssl/+/27944/ ++ // but restored here to impersonate browsers with older ciphers. They are ++ // not expected to actually work; but just to be included in the TLS ++ // Client Hello. ++ ++ // HMAC based TLS v1.2 ciphersuites from RFC5289 ++ ++ // Cipher C023 ++ { ++ TLS1_TXT_ECDHE_ECDSA_WITH_AES_128_SHA256, ++ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", ++ TLS1_CK_ECDHE_ECDSA_WITH_AES_128_SHA256, ++ SSL_kECDHE, ++ SSL_aECDSA, ++ SSL_AES128, ++ SSL_SHA256, ++ SSL_HANDSHAKE_MAC_SHA256, ++ }, ++ // Cipher C024 ++ { ++ TLS1_TXT_ECDHE_ECDSA_WITH_AES_256_SHA384, ++ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384", ++ TLS1_CK_ECDHE_ECDSA_WITH_AES_256_SHA384, ++ SSL_kECDHE, ++ SSL_aECDSA, ++ SSL_AES256, ++ SSL_SHA384, ++ SSL_HANDSHAKE_MAC_SHA384, ++ }, + // Cipher C027 + { + TLS1_TXT_ECDHE_RSA_WITH_AES_128_CBC_SHA256, +@@ -346,6 +439,17 @@ static constexpr SSL_CIPHER kCiphers[] = { + SSL_SHA256, + SSL_HANDSHAKE_MAC_SHA256, + }, ++ // Cipher C028 ++ { ++ TLS1_TXT_ECDHE_RSA_WITH_AES_256_SHA384, ++ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384", ++ TLS1_CK_ECDHE_RSA_WITH_AES_256_SHA384, ++ SSL_kECDHE, ++ SSL_aRSA, ++ SSL_AES256, ++ SSL_SHA384, ++ SSL_HANDSHAKE_MAC_SHA384, ++ }, + + // GCM based TLS v1.2 ciphersuites from RFC 5289 + +@@ -467,16 +571,6 @@ Span AllCiphers() { + return MakeConstSpan(kCiphers, OPENSSL_ARRAY_SIZE(kCiphers)); + } + +-static constexpr size_t NumTLS13Ciphers() { +- size_t num = 0; +- for (const auto &cipher : kCiphers) { +- if (cipher.algorithm_mkey == SSL_kGENERIC) { +- num++; +- } +- } +- return num; +-} +- + #define CIPHER_ADD 1 + #define CIPHER_KILL 2 + #define CIPHER_DEL 3 +@@ -554,6 +648,11 @@ static const CIPHER_ALIAS kCipherAliases[] = { + // MAC aliases + {"SHA1", ~0u, ~0u, ~0u, SSL_SHA1, 0}, + {"SHA", ~0u, ~0u, ~0u, SSL_SHA1, 0}, ++ // curl-impersonate: ++ // Removed in https://boringssl-review.googlesource.com/c/boringssl/+/27944/ ++ // but restored to impersonate browsers with older ciphers. ++ {"SHA256", ~0u, ~0u, ~0u, SSL_SHA256, 0}, ++ {"SHA384", ~0u, ~0u, ~0u, SSL_SHA384, 0}, + + // Legacy protocol minimum version aliases. "TLSv1" is intentionally the + // same as "SSLv3". +@@ -1154,13 +1253,22 @@ bool ssl_create_cipher_list(UniquePtr *out_cipher_list, + TLS1_CK_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256 & 0xffff, + }; + static const uint16_t kLegacyCiphers[] = { ++ TLS1_CK_RSA_WITH_AES_128_SHA256 & 0xffff, ++ TLS1_CK_RSA_WITH_AES_256_SHA256 & 0xffff, ++ 0x0300C008 & 0xffff, ++ // TLS1_CK_ECDHE_ECDSA_WITH_DES_192_CBC3_SHA & 0xffff, + TLS1_CK_ECDHE_ECDSA_WITH_AES_128_CBC_SHA & 0xffff, ++ 0x0300C012 & 0xffff, ++ // TLS1_CK_ECDHE_RSA_WITH_DES_192_CBC3_SHA & 0xffff, + TLS1_CK_ECDHE_RSA_WITH_AES_128_CBC_SHA & 0xffff, + TLS1_CK_ECDHE_PSK_WITH_AES_128_CBC_SHA & 0xffff, + TLS1_CK_ECDHE_ECDSA_WITH_AES_256_CBC_SHA & 0xffff, + TLS1_CK_ECDHE_RSA_WITH_AES_256_CBC_SHA & 0xffff, + TLS1_CK_ECDHE_PSK_WITH_AES_256_CBC_SHA & 0xffff, ++ TLS1_CK_ECDHE_ECDSA_WITH_AES_128_SHA256 & 0xffff, // C023 ++ TLS1_CK_ECDHE_ECDSA_WITH_AES_256_SHA384 & 0xffff, // C024 + TLS1_CK_ECDHE_RSA_WITH_AES_128_CBC_SHA256 & 0xffff, ++ TLS1_CK_ECDHE_RSA_WITH_AES_256_SHA384 & 0xffff, // C028 + TLS1_CK_RSA_WITH_AES_128_GCM_SHA256 & 0xffff, + TLS1_CK_RSA_WITH_AES_256_GCM_SHA384 & 0xffff, + TLS1_CK_RSA_WITH_AES_128_SHA & 0xffff, +@@ -1169,11 +1277,16 @@ bool ssl_create_cipher_list(UniquePtr *out_cipher_list, + TLS1_CK_PSK_WITH_AES_256_CBC_SHA & 0xffff, + SSL3_CK_RSA_DES_192_CBC3_SHA & 0xffff, + }; ++ // curl-impersonate: add TLS 1.3 ciphers here for ordering ++ static const uint16_t kTLS13Ciphers[] = { ++ TLS1_3_CK_AES_128_GCM_SHA256 & 0xffff, ++ TLS1_3_CK_AES_256_GCM_SHA384 & 0xffff, ++ TLS1_3_CK_CHACHA20_POLY1305_SHA256 & 0xffff, ++ }; + + // Set up a linked list of ciphers. +- CIPHER_ORDER co_list[OPENSSL_ARRAY_SIZE(kAESCiphers) + +- OPENSSL_ARRAY_SIZE(kChaChaCiphers) + +- OPENSSL_ARRAY_SIZE(kLegacyCiphers)]; ++ CIPHER_ORDER co_list[OPENSSL_ARRAY_SIZE(kCiphers)]; ++ + for (size_t i = 0; i < OPENSSL_ARRAY_SIZE(co_list); i++) { + co_list[i].next = + i + 1 < OPENSSL_ARRAY_SIZE(co_list) ? &co_list[i + 1] : nullptr; +@@ -1209,8 +1322,13 @@ bool ssl_create_cipher_list(UniquePtr *out_cipher_list, + co_list[num++].cipher = SSL_get_cipher_by_value(id); + assert(co_list[num - 1].cipher != nullptr); + } ++ // curl-impersonate: add TLS 1.3 ciphers here for ordering ++ for (uint16_t id: kTLS13Ciphers) { ++ co_list[num++].cipher = SSL_get_cipher_by_value(id); ++ assert(co_list[num - 1].cipher != nullptr); ++ } + assert(num == OPENSSL_ARRAY_SIZE(co_list)); +- static_assert(OPENSSL_ARRAY_SIZE(co_list) + NumTLS13Ciphers() == ++ static_assert(OPENSSL_ARRAY_SIZE(co_list) == + OPENSSL_ARRAY_SIZE(kCiphers), + "Not all ciphers are included in the cipher order"); + +diff --git a/ssl/ssl_lib.cc b/ssl/ssl_lib.cc +index 58b68e675..15eb823b3 100644 +--- a/ssl/ssl_lib.cc ++++ b/ssl/ssl_lib.cc +@@ -657,6 +657,7 @@ SSL *SSL_new(SSL_CTX *ctx) { + ssl->config->retain_only_sha256_of_client_certs = + ctx->retain_only_sha256_of_client_certs; + ssl->config->permute_extensions = ctx->permute_extensions; ++ ssl->config->extension_order = ctx->extension_order; + ssl->config->aes_hw_override = ctx->aes_hw_override; + ssl->config->aes_hw_override_value = ctx->aes_hw_override_value; + ssl->config->tls13_cipher_policy = ctx->tls13_cipher_policy; +@@ -3015,6 +3016,12 @@ void SSL_CTX_set_permute_extensions(SSL_CTX *ctx, int enabled) { + ctx->permute_extensions = !!enabled; + } + ++// curl-impersonate: set extensions order ++int SSL_CTX_set_extension_order(SSL_CTX *ctx, char *order) { ++ ctx->extension_order = order; ++ return 0; ++} ++ + void SSL_set_permute_extensions(SSL *ssl, int enabled) { + if (!ssl->config) { + return; +diff --git a/ssl/ssl_privkey.cc b/ssl/ssl_privkey.cc +index 57116cd6c..fa1652832 100644 +--- a/ssl/ssl_privkey.cc ++++ b/ssl/ssl_privkey.cc +@@ -594,7 +594,7 @@ static bool sigalgs_unique(Span in_sigalgs) { + + static bool set_sigalg_prefs(Array *out, Span prefs) { + if (!sigalgs_unique(prefs)) { +- return false; ++ // return false; + } + + // Check for invalid algorithms, and filter out |SSL_SIGN_RSA_PKCS1_MD5_SHA1|. diff --git a/chrome/patches/curl-impersonate.patch b/chrome/patches/curl-impersonate.patch index 2bc8422d..7b227e3e 100644 --- a/chrome/patches/curl-impersonate.patch +++ b/chrome/patches/curl-impersonate.patch @@ -1,8 +1,8 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index 49a44eabf..fe8b38915 100644 +index a54c2fff9..7d22d4e96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -550,6 +550,29 @@ if(CURL_ZSTD) +@@ -619,6 +619,29 @@ if(CURL_ZSTD) endif() endif() @@ -29,14 +29,14 @@ index 49a44eabf..fe8b38915 100644 + endif() +endif() + - option(USE_NGHTTP2 "Use Nghttp2 library" OFF) - if(USE_NGHTTP2) - find_package(NGHTTP2 REQUIRED) + # Check symbol in OpenSSL-like TLS backends. + macro(openssl_check_symbol_exists SYMBOL FILES VARIABLE) + cmake_push_check_state() diff --git a/Makefile.am b/Makefile.am -index f25e4e2f0..ff0c5630b 100644 +index c8afcb505..234125083 100644 --- a/Makefile.am +++ b/Makefile.am -@@ -156,13 +156,13 @@ CLEANFILES = $(VC10_LIBVCXPROJ) $(VC10_SRCVCXPROJ) $(VC11_LIBVCXPROJ) \ +@@ -131,13 +131,13 @@ CLEANFILES = $(VC14_LIBVCXPROJ) \ $(VC14_SRCVCXPROJ) $(VC14_10_LIBVCXPROJ) $(VC14_10_SRCVCXPROJ) \ $(VC14_30_LIBVCXPROJ) $(VC14_30_SRCVCXPROJ) @@ -53,10 +53,10 @@ index f25e4e2f0..ff0c5630b 100644 # List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files include lib/Makefile.inc diff --git a/configure.ac b/configure.ac -index 75a882b12..8697a6f74 100644 +index d9b396376..cd3b845f4 100644 --- a/configure.ac +++ b/configure.ac -@@ -1493,7 +1493,8 @@ if test X"$OPT_BROTLI" != Xno; then +@@ -1422,7 +1422,8 @@ if test X"$OPT_BROTLI" != Xno; then dnl if given with a prefix, we set -L and -I based on that if test -n "$PREFIX_BROTLI"; then @@ -66,7 +66,7 @@ index 75a882b12..8697a6f74 100644 LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff CPP_BROTLI=-I${PREFIX_BROTLI}/include DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff -@@ -1503,7 +1504,11 @@ if test X"$OPT_BROTLI" != Xno; then +@@ -1432,7 +1433,11 @@ if test X"$OPT_BROTLI" != Xno; then CPPFLAGS="$CPPFLAGS $CPP_BROTLI" LIBS="$LIB_BROTLI $LIBS" @@ -79,7 +79,7 @@ index 75a882b12..8697a6f74 100644 AC_CHECK_HEADERS(brotli/decode.h, curl_brotli_msg="enabled (libbrotlidec)" -@@ -4306,14 +4311,23 @@ if test "x$want_ech" != "xno"; then +@@ -4383,14 +4388,23 @@ if test "x$want_ech" != "xno"; then ECH_ENABLED=0 ECH_SUPPORT='' @@ -108,7 +108,7 @@ index 75a882b12..8697a6f74 100644 fi dnl now deal with whatever we found -@@ -4706,8 +4720,8 @@ AC_CONFIG_FILES([Makefile \ +@@ -4795,8 +4809,8 @@ AC_CONFIG_FILES([Makefile \ tests/http/clients/Makefile \ packages/Makefile \ packages/vms/Makefile \ @@ -144,11 +144,21 @@ index 54f92d931..ea5895e9b 100644 else echo "curl was built with static libraries disabled" >&2 exit 1 +diff --git a/export.sh b/export.sh +new file mode 100755 +index 000000000..34b44cd2a +--- /dev/null ++++ b/export.sh +@@ -0,0 +1,4 @@ ++#!/bin/bash ++ ++git df curl-8_5_0 > chrome.patch ++mv chrome.patch ../curl-impersonate/chrome/patches/curl-impersonate.patch diff --git a/include/curl/curl.h b/include/curl/curl.h -index 944352421..8d81e9a8d 100644 +index cc24c0506..9cba4dd14 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h -@@ -640,6 +640,7 @@ typedef enum { +@@ -631,6 +631,7 @@ typedef enum { CURLE_PROXY, /* 97 - proxy handshake error */ CURLE_SSL_CLIENTCERT, /* 98 - client-side certificate required */ CURLE_UNRECOVERABLE_POLL, /* 99 - poll/select returned fatal error */ @@ -156,32 +166,32 @@ index 944352421..8d81e9a8d 100644 CURL_LAST /* never use! */ } CURLcode; -@@ -2207,6 +2208,59 @@ typedef enum { - /* Can leak things, gonna exit() soon */ - CURLOPT(CURLOPT_QUICK_EXIT, CURLOPTTYPE_LONG, 322), +@@ -2201,6 +2202,77 @@ typedef enum { + /* set a specific client IP for HAProxy PROXY protocol header? */ + CURLOPT(CURLOPT_HAPROXY_CLIENT_IP, CURLOPTTYPE_STRINGPOINT, 323), + /* curl-impersonate: A list of headers used by the impersonated browser. + * If given, merged with CURLOPT_HTTPHEADER. */ -+ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 323), ++ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 324), + + /* curl-impersonate: A list of TLS signature hash algorithms. + * See https://datatracker.ietf.org/doc/html/rfc5246#section-7.4.1.4.1 */ -+ CURLOPT(CURLOPT_SSL_SIG_HASH_ALGS, CURLOPTTYPE_STRINGPOINT, 324), ++ CURLOPT(CURLOPT_SSL_SIG_HASH_ALGS, CURLOPTTYPE_STRINGPOINT, 325), + + /* curl-impersonate: Whether to enable ALPS in TLS or not. + * See https://datatracker.ietf.org/doc/html/draft-vvv-tls-alps. + * Support for ALPS is minimal and is intended only for the TLS client + * hello to match. */ -+ CURLOPT(CURLOPT_SSL_ENABLE_ALPS, CURLOPTTYPE_LONG, 325), ++ CURLOPT(CURLOPT_SSL_ENABLE_ALPS, CURLOPTTYPE_LONG, 326), + + /* curl-impersonate: Comma-separated list of certificate compression + * algorithms to use. These are published in the client hello. + * Supported algorithms are "zlib" and "brotli". + * See https://datatracker.ietf.org/doc/html/rfc8879 */ -+ CURLOPT(CURLOPT_SSL_CERT_COMPRESSION, CURLOPTTYPE_STRINGPOINT, 326), ++ CURLOPT(CURLOPT_SSL_CERT_COMPRESSION, CURLOPTTYPE_STRINGPOINT, 327), + + /* Enable/disable TLS session ticket extension (RFC5077) */ -+ CURLOPT(CURLOPT_SSL_ENABLE_TICKET, CURLOPTTYPE_LONG, 327), ++ CURLOPT(CURLOPT_SSL_ENABLE_TICKET, CURLOPTTYPE_LONG, 328), + + /* + * curl-impersonate: @@ -190,28 +200,46 @@ index 944352421..8d81e9a8d 100644 + * ":method", ":authority", ":scheme", ":path" in the desired order of + * appearance in the HTTP/2 HEADERS frame. + */ -+ CURLOPT(CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, CURLOPTTYPE_STRINGPOINT, 328), ++ CURLOPT(CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER, CURLOPTTYPE_STRINGPOINT, 329), + + /* + * curl-impersonate: + * HTTP2 settings frame keys and values, format: 1:v;2:v;3:v + */ -+ CURLOPT(CURLOPT_HTTP2_SETTINGS, CURLOPTTYPE_STRINGPOINT, 329), ++ CURLOPT(CURLOPT_HTTP2_SETTINGS, CURLOPTTYPE_STRINGPOINT, 330), + + /* + * curl-impersonate: Whether to enable Boringssl permute extensions + * See https://commondatastorage.googleapis.com/chromium-boringssl-docs/ssl.h.html#SSL_set_permute_extensions. + */ -+ CURLOPT(CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOPTTYPE_LONG, 330), ++ CURLOPT(CURLOPT_SSL_PERMUTE_EXTENSIONS, CURLOPTTYPE_LONG, 331), + + /* + * curl-impersonate: + * HTTP2 initial window update + */ -+ CURLOPT(CURLOPT_HTTP2_WINDOW_UPDATE, CURLOPTTYPE_LONG, 331), ++ CURLOPT(CURLOPT_HTTP2_WINDOW_UPDATE, CURLOPTTYPE_LONG, 332), ++ ++ /* curl-impersonate: ++ * set ECH configuration, XXX, the official one is 324 ++ */ ++ CURLOPT(CURLOPT_ECH, CURLOPTTYPE_STRINGPOINT, 333), ++ ++ /* ++ * curl-impersonate: ++ * Set the initial streams settings for http2. ++ */ ++ CURLOPT(CURLOPT_HTTP2_STREAMS, CURLOPTTYPE_STRINGPOINT, 334), + -+ /* set ECH configuration, XXX, the official one is 324 */ -+ CURLOPT(CURLOPT_ECH, CURLOPTTYPE_STRINGPOINT, 332), ++ /* curl-impersonate: ++ * enable tls grease ++ */ ++ CURLOPT(CURLOPT_TLS_GREASE, CURLOPTTYPE_LONG, 335), ++ ++ /* curl-impersonate: ++ * set tls extension order ++ */ ++ CURLOPT(CURLOPT_TLS_EXTENSION_ORDER, CURLOPTTYPE_STRINGPOINT, 336), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; @@ -238,7 +266,7 @@ index 1285101c5..c620065dc 100644 * NAME curl_easy_getinfo() * diff --git a/include/curl/typecheck-gcc.h b/include/curl/typecheck-gcc.h -index bc8d7a78c..033c994ca 100644 +index b880f3dc6..79074e011 100644 --- a/include/curl/typecheck-gcc.h +++ b/include/curl/typecheck-gcc.h @@ -275,6 +275,7 @@ CURLWARNING(_curl_easy_getinfo_err_curl_off_t, @@ -250,7 +278,7 @@ index bc8d7a78c..033c994ca 100644 (option) == CURLOPT_FTP_ACCOUNT || \ (option) == CURLOPT_FTP_ALTERNATIVE_TO_USER || \ diff --git a/lib/Makefile.am b/lib/Makefile.am -index 3c0a70912..61a9eb90b 100644 +index 1237c8e99..6b2961018 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -31,7 +31,7 @@ EXTRA_DIST = Makefile.mk config-win32.h config-win32ce.h config-plan9.h \ @@ -322,19 +350,19 @@ index 3c0a70912..61a9eb90b 100644 endif -libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA) --libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) +-libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(CURL_LDFLAGS_LIB) $(LIBCURL_LIBS) -libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA) +libcurl_impersonate_chrome_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_chrome_la_CPPFLAGS_EXTRA) -+libcurl_impersonate_chrome_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_chrome_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) ++libcurl_impersonate_chrome_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_chrome_la_LDFLAGS_EXTRA) $(CURL_LDFLAGS_LIB) $(LIBCURL_LIBS) +libcurl_impersonate_chrome_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_chrome_la_CFLAGS_EXTRA) libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS) diff --git a/lib/Makefile.inc b/lib/Makefile.inc -index f815170a7..9d9417edc 100644 +index e568ef953..298b16050 100644 --- a/lib/Makefile.inc +++ b/lib/Makefile.inc -@@ -174,6 +174,7 @@ LIB_CFILES = \ +@@ -171,6 +171,7 @@ LIB_CFILES = \ idn.c \ if2ip.c \ imap.c \ @@ -343,21 +371,21 @@ index f815170a7..9d9417edc 100644 inet_pton.c \ krb5.c \ diff --git a/lib/curl_config.h.cmake b/lib/curl_config.h.cmake -index eca71bdef..ae9f02aca 100644 +index 339358ea3..a9cf400fb 100644 --- a/lib/curl_config.h.cmake +++ b/lib/curl_config.h.cmake -@@ -783,3 +783,6 @@ ${SIZEOF_TIME_T_CODE} +@@ -811,3 +811,6 @@ ${SIZEOF_TIME_T_CODE} - /* Define to 1 to enable websocket support. */ - #cmakedefine USE_WEBSOCKETS 1 + /* Define to 1 to enable TLS-SRP support. */ + #cmakedefine USE_TLS_SRP 1 + +/* if ECH support is available */ +#cmakedefine USE_ECH 1 diff --git a/lib/dynhds.c b/lib/dynhds.c -index b325e0060..4c8a73bab 100644 +index d7548959b..00f97506b 100644 --- a/lib/dynhds.c +++ b/lib/dynhds.c -@@ -52,6 +52,8 @@ entry_new(const char *name, size_t namelen, +@@ -56,6 +56,8 @@ entry_new(const char *name, size_t namelen, e->valuelen = valuelen; if(opts & DYNHDS_OPT_LOWERCASE) Curl_strntolower(e->name, e->name, e->namelen); @@ -366,7 +394,7 @@ index b325e0060..4c8a73bab 100644 return e; } -@@ -134,6 +136,16 @@ void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts) +@@ -138,6 +140,16 @@ void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts) dynhds->opts = opts; } @@ -384,7 +412,7 @@ index b325e0060..4c8a73bab 100644 { DEBUGASSERT(dynhds); diff --git a/lib/dynhds.h b/lib/dynhds.h -index 777baa58a..2d542dfd6 100644 +index 3b536000a..d7135698f 100644 --- a/lib/dynhds.h +++ b/lib/dynhds.h @@ -53,6 +53,7 @@ struct dynhds { @@ -405,10 +433,10 @@ index 777baa58a..2d542dfd6 100644 /** * Return the n-th header entry or NULL if it does not exist. diff --git a/lib/easy.c b/lib/easy.c -index d36cc03d1..ec25400c5 100644 +index 322d1a41b..805613e6f 100644 --- a/lib/easy.c +++ b/lib/easy.c -@@ -73,6 +73,8 @@ +@@ -74,6 +74,8 @@ #include "dynbuf.h" #include "altsvc.h" #include "hsts.h" @@ -417,29 +445,23 @@ index d36cc03d1..ec25400c5 100644 #include "easy_lock.h" -@@ -330,6 +332,146 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, +@@ -341,6 +343,203 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, return rc; } ++ +/* + * curl-impersonate: -+ * Call curl_easy_setopt() with all the needed options as defined in the -+ * 'impersonations' array. ++ * Actually call curl_easy_setopt() with all the needed options + * */ -+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, -+ int default_headers) ++CURLcode _do_impersonate(struct Curl_easy *data, ++ const struct impersonate_opts *opts, ++ int default_headers) +{ + int i; + int ret; -+ const struct impersonate_opts *opts = NULL; + struct curl_slist *headers = NULL; + -+ for(opts = impersonations; opts->target != NULL; opts++) { -+ if (strcasecompare(target, opts->target)) { -+ break; -+ } -+ } -+ + if(opts->target == NULL) { + DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", + target)); @@ -547,12 +569,27 @@ index d36cc03d1..ec25400c5 100644 + return ret; + } + ++ if(opts->http2_streams) { ++ ret = curl_easy_setopt(data, CURLOPT_HTTP2_STREAMS, opts->http2_streams); ++ if(ret) ++ return ret; ++ } ++ + if(opts->ech) { + ret = curl_easy_setopt(data, CURLOPT_ECH, opts->ech); + if(ret) + return ret; + } + ++ if(opts->tls_grease) { ++ ret = curl_easy_setopt(data, CURLOPT_TLS_GREASE, opts->tls_grease); ++ } ++ ++ if(opts->tls_extension_order) { ++ // printf("setting extension order as: %s\n", opts->tls_extension_order); ++ ret = curl_easy_setopt(data, CURLOPT_TLS_EXTENSION_ORDER, opts->tls_extension_order); ++ } ++ + /* Always enable all supported compressions. */ + ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, ""); + if(ret) @@ -560,11 +597,59 @@ index d36cc03d1..ec25400c5 100644 + + return CURLE_OK; +} ++ ++ ++/* ++ * curl-impersonate: ++ * Call curl_easy_setopt() with all the needed options as defined by the target ++ * */ ++CURLcode curl_easy_impersonate_customized(struct Curl_easy *data, ++ const struct impersonate_opts *opts, ++ int default_headers) ++{ ++ int ret; ++ ++ ret = _do_impersonate(data, opts, default_headers); ++ if(ret) ++ return ret; ++ ++ return CURLE_OK; ++} ++ ++/* ++ * curl-impersonate: ++ * Call curl_easy_setopt() with all the needed options as defined in the ++ * 'impersonations' array. ++ * */ ++CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, ++ int default_headers) ++{ ++ int ret; ++ const struct impersonate_opts *opts = NULL; ++ ++ for(opts = impersonations; opts->target != NULL; opts++) { ++ if (strcasecompare(target, opts->target)) { ++ break; ++ } ++ } ++ ++ if(opts->target == NULL) { ++ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", ++ target)); ++ return CURLE_BAD_FUNCTION_ARGUMENT; ++ } ++ ++ ret = _do_impersonate(data, opts, default_headers); ++ if(ret) ++ return ret; ++ ++ return CURLE_OK; ++} + /* * curl_easy_init() is the external interface to alloc, setup and init an * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -338,6 +480,8 @@ struct Curl_easy *curl_easy_init(void) +@@ -349,6 +548,8 @@ struct Curl_easy *curl_easy_init(void) { CURLcode result; struct Curl_easy *data; @@ -573,7 +658,7 @@ index d36cc03d1..ec25400c5 100644 /* Make sure we inited the global SSL stuff */ global_init_lock(); -@@ -360,6 +504,29 @@ struct Curl_easy *curl_easy_init(void) +@@ -371,6 +572,29 @@ struct Curl_easy *curl_easy_init(void) return NULL; } @@ -603,7 +688,7 @@ index d36cc03d1..ec25400c5 100644 return data; } -@@ -930,6 +1097,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) +@@ -945,6 +1169,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) outcurl->state.referer_alloc = TRUE; } @@ -617,7 +702,7 @@ index d36cc03d1..ec25400c5 100644 /* Reinitialize an SSL engine for the new handle * note: the engine name has already been copied by dupset */ if(outcurl->set.str[STRING_SSL_ENGINE]) { -@@ -1019,6 +1193,9 @@ fail: +@@ -1004,6 +1235,9 @@ fail: */ void curl_easy_reset(struct Curl_easy *data) { @@ -627,8 +712,8 @@ index d36cc03d1..ec25400c5 100644 Curl_free_request_state(data); /* zero out UserDefined data: */ -@@ -1043,6 +1220,23 @@ void curl_easy_reset(struct Curl_easy *data) - #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH) +@@ -1028,6 +1262,23 @@ void curl_easy_reset(struct Curl_easy *data) + #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_DIGEST_AUTH) Curl_http_auth_cleanup_digest(data); #endif + @@ -652,7 +737,7 @@ index d36cc03d1..ec25400c5 100644 /* diff --git a/lib/easyoptions.c b/lib/easyoptions.c -index a9c1efd00..b81f3a671 100644 +index e69c658b0..027dd6f93 100644 --- a/lib/easyoptions.c +++ b/lib/easyoptions.c @@ -86,6 +86,7 @@ struct curl_easyoption Curl_easyopts[] = { @@ -663,7 +748,7 @@ index a9c1efd00..b81f3a671 100644 {"EGDSOCKET", CURLOPT_EGDSOCKET, CURLOT_STRING, 0}, {"ENCODING", CURLOPT_ACCEPT_ENCODING, CURLOT_STRING, CURLOT_FLAG_ALIAS}, {"ERRORBUFFER", CURLOPT_ERRORBUFFER, CURLOT_OBJECT, 0}, -@@ -132,8 +133,13 @@ struct curl_easyoption Curl_easyopts[] = { +@@ -133,8 +134,14 @@ struct curl_easyoption Curl_easyopts[] = { {"HSTS_CTRL", CURLOPT_HSTS_CTRL, CURLOT_LONG, 0}, {"HTTP09_ALLOWED", CURLOPT_HTTP09_ALLOWED, CURLOT_LONG, 0}, {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, @@ -671,13 +756,14 @@ index a9c1efd00..b81f3a671 100644 + CURLOT_STRING, 0}, + {"HTTP2_SETTINGS", CURLOPT_HTTP2_SETTINGS, CURLOT_STRING, 0}, + {"HTTP2_WINDOW_UPDATE", CURLOPT_HTTP2_WINDOW_UPDATE, CURLOT_LONG, 0}, ++ {"HTTP2_STREAMS", CURLOPT_HTTP2_STREAMS, CURLOT_STRING, 0}, {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, + {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, -@@ -302,18 +308,23 @@ struct curl_easyoption Curl_easyopts[] = { +@@ -305,18 +312,23 @@ struct curl_easyoption Curl_easyopts[] = { {"SSLKEYTYPE", CURLOPT_SSLKEYTYPE, CURLOT_STRING, 0}, {"SSLKEY_BLOB", CURLOPT_SSLKEY_BLOB, CURLOT_BLOB, 0}, {"SSLVERSION", CURLOPT_SSLVERSION, CURLOT_VALUES, 0}, @@ -701,16 +787,25 @@ index a9c1efd00..b81f3a671 100644 {"STDERR", CURLOPT_STDERR, CURLOT_OBJECT, 0}, {"STREAM_DEPENDS", CURLOPT_STREAM_DEPENDS, CURLOT_OBJECT, 0}, {"STREAM_DEPENDS_E", CURLOPT_STREAM_DEPENDS_E, CURLOT_OBJECT, 0}, -@@ -370,6 +381,6 @@ struct curl_easyoption Curl_easyopts[] = { +@@ -340,6 +352,8 @@ struct curl_easyoption Curl_easyopts[] = { + {"TLSAUTH_PASSWORD", CURLOPT_TLSAUTH_PASSWORD, CURLOT_STRING, 0}, + {"TLSAUTH_TYPE", CURLOPT_TLSAUTH_TYPE, CURLOT_STRING, 0}, + {"TLSAUTH_USERNAME", CURLOPT_TLSAUTH_USERNAME, CURLOT_STRING, 0}, ++ {"TLS_GREASE", CURLOPT_TLS_GREASE, CURLOT_LONG, 0}, ++ {"TLS_EXTENSION_ORDER", CURLOPT_TLS_EXTENSION_ORDER, CURLOT_STRING, 0}, + {"TRAILERDATA", CURLOPT_TRAILERDATA, CURLOT_CBPTR, 0}, + {"TRAILERFUNCTION", CURLOPT_TRAILERFUNCTION, CURLOT_FUNCTION, 0}, + {"TRANSFERTEXT", CURLOPT_TRANSFERTEXT, CURLOT_LONG, 0}, +@@ -373,6 +387,6 @@ struct curl_easyoption Curl_easyopts[] = { */ int Curl_easyopts_check(void) { -- return ((CURLOPT_LASTENTRY%10000) != (322 + 1)); -+ return ((CURLOPT_LASTENTRY%10000) != (332 + 1)); +- return ((CURLOPT_LASTENTRY%10000) != (323 + 1)); ++ return ((CURLOPT_LASTENTRY%10000) != (334 + 1)); } #endif diff --git a/lib/http.c b/lib/http.c -index 219dcc2c0..7b04c6c36 100644 +index be6d442e8..ca537314a 100644 --- a/lib/http.c +++ b/lib/http.c @@ -90,6 +90,7 @@ @@ -721,7 +816,7 @@ index 219dcc2c0..7b04c6c36 100644 /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -1881,6 +1882,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, +@@ -1937,6 +1938,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, int numlists = 1; /* by default */ int i; @@ -737,7 +832,7 @@ index 219dcc2c0..7b04c6c36 100644 #ifndef CURL_DISABLE_PROXY enum proxy_use proxy; -@@ -1892,10 +1902,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, +@@ -1948,10 +1958,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, switch(proxy) { case HEADER_SERVER: @@ -750,7 +845,7 @@ index 219dcc2c0..7b04c6c36 100644 if(data->set.sep_headers) { h[1] = data->set.proxyheaders; numlists++; -@@ -1905,12 +1915,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, +@@ -1961,12 +1971,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, if(data->set.sep_headers) h[0] = data->set.proxyheaders; else @@ -765,7 +860,7 @@ index 219dcc2c0..7b04c6c36 100644 #endif /* loop through one or two lists */ -@@ -2146,6 +2156,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, +@@ -2202,6 +2212,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, *reqp = httpreq; } @@ -874,7 +969,7 @@ index 219dcc2c0..7b04c6c36 100644 CURLcode Curl_http_useragent(struct Curl_easy *data) { /* The User-Agent string might have been allocated in url.c already, because -@@ -3165,6 +3277,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) +@@ -3210,6 +3322,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) http = data->req.p.http; DEBUGASSERT(http); @@ -886,7 +981,7 @@ index 219dcc2c0..7b04c6c36 100644 result = Curl_http_host(data, conn); if(result) return result; -@@ -4777,12 +4894,41 @@ static bool h2_non_field(const char *name, size_t namelen) +@@ -4847,12 +4964,41 @@ static bool h2_non_field(const char *name, size_t namelen) return FALSE; } @@ -928,7 +1023,7 @@ index 219dcc2c0..7b04c6c36 100644 CURLcode result; DEBUGASSERT(req); -@@ -4816,25 +4962,56 @@ CURLcode Curl_http_req_to_h2(struct dynhds *h2_headers, +@@ -4886,25 +5032,56 @@ CURLcode Curl_http_req_to_h2(struct dynhds *h2_headers, Curl_dynhds_reset(h2_headers); Curl_dynhds_set_opts(h2_headers, DYNHDS_OPT_LOWERCASE); @@ -998,10 +1093,10 @@ index 219dcc2c0..7b04c6c36 100644 } diff --git a/lib/http2.c b/lib/http2.c -index c666192fc..5a90b28ea 100644 +index 973848484..36654499d 100644 --- a/lib/http2.c +++ b/lib/http2.c -@@ -50,6 +50,7 @@ +@@ -51,6 +51,7 @@ #include "curl_printf.h" #include "curl_memory.h" #include "memdebug.h" @@ -1009,16 +1104,16 @@ index c666192fc..5a90b28ea 100644 #if (NGHTTP2_VERSION_NUM < 0x010c00) #error too old nghttp2 version, upgrade! -@@ -68,7 +69,7 @@ +@@ -69,7 +70,7 @@ * use 16K as chunk size, as that fits H2 DATA frames well */ #define H2_CHUNK_SIZE (16 * 1024) /* this is how much we want "in flight" for a stream */ -#define H2_STREAM_WINDOW_SIZE (10 * 1024 * 1024) +#define H2_STREAM_WINDOW_SIZE (1024 * 1024) - /* on receving from TLS, we prep for holding a full stream window */ + /* on receiving from TLS, we prep for holding a full stream window */ #define H2_NW_RECV_CHUNKS (H2_STREAM_WINDOW_SIZE / H2_CHUNK_SIZE) /* on send into TLS, we just want to accumulate small frames */ -@@ -86,24 +87,84 @@ +@@ -87,26 +88,87 @@ * will block their received QUOTA in the connection window. And if we * run out of space, the server is blocked from sending us any data. * See #10988 for an issue with this. */ @@ -1054,10 +1149,13 @@ index c666192fc..5a90b28ea 100644 - iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; - iv[1].value = H2_STREAM_WINDOW_SIZE; + // printf("USING settings %s\n", http2_settings); -+ + +- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; +- iv[2].value = data->multi->push_cb != NULL; + char *tmp = strdup(http2_settings); + char *setting = strtok(tmp, delimiter); -+ + +- return 3; + // loop through the string to extract all other tokens + while(setting != NULL) { + // deal with each setting @@ -1097,22 +1195,98 @@ index c666192fc..5a90b28ea 100644 + setting = strtok(NULL, delimiter); + } + free(tmp); - -- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; -- iv[2].value = data->multi->push_cb != NULL; ++ + // curl-impersonate: + // Up until Chrome 98, there was a randomly chosen setting number in the + // HTTP2 SETTINGS frame. This might be something similar to TLS GREASE. + // However, it seems to have been removed since. + // Curl_rand(data, (unsigned char *)&iv[4].settings_id, sizeof(iv[4].settings_id)); + // Curl_rand(data, (unsigned char *)&iv[4].value, sizeof(iv[4].value)); - -- return 3; ++ + return i; } - static size_t populate_binsettings(uint8_t *binsettings, -@@ -483,8 +544,22 @@ static CURLcode cf_h2_ctx_init(struct Curl_cfilter *cf, ++ + static ssize_t populate_binsettings(uint8_t *binsettings, + struct Curl_easy *data) + { +@@ -165,6 +227,75 @@ static void cf_h2_ctx_free(struct cf_h2_ctx *ctx) + } + } + ++static CURLcode http2_set_stream_priority(struct Curl_cfilter *cf, ++ struct Curl_easy *data, ++ int32_t stream_id, ++ int32_t dep_stream_id, ++ int32_t weight, ++ int exclusive ++ ) ++{ ++ int rv; ++ struct cf_h2_ctx *ctx = cf->ctx; ++ nghttp2_priority_spec pri_spec; ++ ++ nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, exclusive); ++ rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, ++ stream_id, &pri_spec); ++ if(rv) { ++ failf(data, "nghttp2_submit_priority() failed: %s(%d)", ++ nghttp2_strerror(rv), rv); ++ return CURLE_HTTP2; ++ } ++ ++ return CURLE_OK; ++} ++ ++/* ++ * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to ++ * split the load for html/js/css/images. It builds a tree of streams with ++ * different weights (priorities) by default and communicates this to the ++ * server. Imitate that behavior. ++ */ ++static CURLcode http2_set_stream_priorities(struct Curl_cfilter *cf, ++ struct Curl_easy *data) ++{ ++ CURLcode result; ++ char *stream_delimiter = ","; ++ char *value_delimiter = ":"; ++ ++ if(!data->set.str[STRING_HTTP2_STREAMS]) ++ return CURLE_OK; ++ ++ char *tmp1 = strdup(data->set.str[STRING_HTTP2_STREAMS]); ++ char *end1; ++ char *stream = strtok_r(tmp1, stream_delimiter, &end1); ++ ++ while(stream != NULL) { ++ ++ char *tmp2 = strdup(stream); ++ char *end2; ++ ++ int32_t stream_id = atoi(strtok_r(tmp2, value_delimiter, &end2)); ++ int exclusive = atoi(strtok_r(NULL, value_delimiter, &end2)); ++ int32_t dep_stream_id = atoi(strtok_r(NULL, value_delimiter, &end2)); ++ int32_t weight = atoi(strtok_r(NULL, value_delimiter, &end2)); ++ ++ free(tmp2); ++ ++ result = http2_set_stream_priority(cf, data, stream_id, dep_stream_id, weight, exclusive); ++ if(result) { ++ free(tmp1); ++ return result; ++ } ++ ++ stream = strtok_r(NULL, stream_delimiter, &end1); ++ } ++ ++ free(tmp1); ++ return CURLE_OK; ++} ++ + static CURLcode h2_progress_egress(struct Curl_cfilter *cf, + struct Curl_easy *data); + +@@ -504,8 +635,22 @@ static CURLcode cf_h2_ctx_init(struct Curl_cfilter *cf, } } @@ -1137,7 +1311,24 @@ index c666192fc..5a90b28ea 100644 if(rc) { failf(data, "nghttp2_session_set_local_window_size() failed: %s(%d)", nghttp2_strerror(rc), rc); -@@ -1616,11 +1691,17 @@ out: +@@ -513,6 +658,16 @@ static CURLcode cf_h2_ctx_init(struct Curl_cfilter *cf, + goto out; + } + ++ // curl-impersonate: set stream priorities ++ result = http2_set_stream_priorities(cf, data); ++ if(result) ++ goto out; ++ ++#define FIREFOX_DEFAULT_STREAM_ID (15) ++ /* Best effort to set the request's stream id to 15, like Firefox does. */ ++ // Let's ignore this for now, as it seems not to be targeted as fingerprints. ++ // nghttp2_session_set_next_stream_id(ctx->h2, FIREFOX_DEFAULT_STREAM_ID); ++ + /* all set, traffic will be send on connect */ + result = CURLE_OK; + CURL_TRC_CF(data, cf, "[0] created h2 session%s", +@@ -1747,11 +1902,19 @@ out: return rv; } @@ -1146,6 +1337,7 @@ index c666192fc..5a90b28ea 100644 + * instead of NGINX default stream weight. + */ +#define CHROME_DEFAULT_STREAM_WEIGHT (256) ++#define FIREFOX_DEFAULT_STREAM_WEIGHT (42) + static int sweight_wanted(const struct Curl_easy *data) { @@ -1153,13 +1345,35 @@ index c666192fc..5a90b28ea 100644 return data->set.priority.weight? - data->set.priority.weight : NGHTTP2_DEFAULT_WEIGHT; + data->set.priority.weight : CHROME_DEFAULT_STREAM_WEIGHT; ++ // data->set.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT; } static int sweight_in_effect(const struct Curl_easy *data) -@@ -1642,9 +1723,11 @@ static void h2_pri_spec(struct Curl_easy *data, +@@ -1759,6 +1922,7 @@ static int sweight_in_effect(const struct Curl_easy *data) + /* 0 weight is not set by user and we take the nghttp2 default one */ + return data->state.priority.weight? + data->state.priority.weight : NGHTTP2_DEFAULT_WEIGHT; ++ // data->state.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT; + } + + /* +@@ -1767,15 +1931,24 @@ static int sweight_in_effect(const struct Curl_easy *data) + * struct. + */ + ++/* ++ * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the ++ * stream that fetches the main html resource of the web page. ++ */ ++#define FIREFOX_DEFAULT_STREAM_DEP (13) ++ + static void h2_pri_spec(struct Curl_easy *data, + nghttp2_priority_spec *pri_spec) + { struct Curl_data_priority *prio = &data->set.priority; struct stream_ctx *depstream = H2_STREAM_CTX(prio->parent); int32_t depstream_id = depstream? depstream->id:0; ++ // int32_t depstream_id = depstream? depstream->id:FIREFOX_DEFAULT_STREAM_DEP; + /* curl-impersonate: Set stream exclusive flag to true. */ + int exclusive = 1; nghttp2_priority_spec_init(pri_spec, depstream_id, @@ -1169,24 +1383,22 @@ index c666192fc..5a90b28ea 100644 data->state.priority = *prio; } -@@ -1661,20 +1744,25 @@ static CURLcode h2_progress_egress(struct Curl_cfilter *cf, +@@ -1792,20 +1965,24 @@ static CURLcode h2_progress_egress(struct Curl_cfilter *cf, struct stream_ctx *stream = H2_STREAM_CTX(data); int rv = 0; -- if((sweight_wanted(data) != sweight_in_effect(data)) || -- (data->set.priority.exclusive != data->state.priority.exclusive) || -- (data->set.priority.parent != data->state.priority.parent) ) { + /* curl-impersonate: Check if stream exclusive flag is true. */ -+ if(stream && stream->id > 0 && -+ ((sweight_wanted(data) != sweight_in_effect(data)) || + if(stream && stream->id > 0 && + ((sweight_wanted(data) != sweight_in_effect(data)) || +- (data->set.priority.exclusive != data->state.priority.exclusive) || +- (data->set.priority.parent != data->state.priority.parent)) ) { + (data->set.priority.exclusive != 1) || + (data->set.priority.parent != data->state.priority.parent))) { /* send new weight and/or dependency */ nghttp2_priority_spec pri_spec; h2_pri_spec(data, &pri_spec); -- DEBUGF(LOG_CF(data, cf, "[h2sid=%d] Queuing PRIORITY", -- stream->id)); +- CURL_TRC_CF(data, cf, "[%d] Queuing PRIORITY", stream->id); - DEBUGASSERT(stream->id != -1); - rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, - stream->id, &pri_spec); @@ -1194,8 +1406,7 @@ index c666192fc..5a90b28ea 100644 - goto out; + /* curl-impersonate: Don't send PRIORITY frames for main stream. */ + if(stream->id != 1) { -+ DEBUGF(LOG_CF(data, cf, "[h2sid=%d] Queuing PRIORITY", -+ stream->id)); ++ CURL_TRC_CF(data, cf, "[%d] Queuing PRIORITY", stream->id); + DEBUGASSERT(stream->id != -1); + rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, + stream->id, &pri_spec); @@ -1204,9 +1415,9 @@ index c666192fc..5a90b28ea 100644 + } } - while(!rv && nghttp2_session_want_write(ctx->h2)) + ctx->nw_out_blocked = 0; diff --git a/lib/http2.h b/lib/http2.h -index 562c05c99..b99c085d5 100644 +index 80e183480..8ee390b7e 100644 --- a/lib/http2.h +++ b/lib/http2.h @@ -31,7 +31,8 @@ @@ -1221,10 +1432,10 @@ index 562c05c99..b99c085d5 100644 * Store nghttp2 version info in this buffer. diff --git a/lib/impersonate.c b/lib/impersonate.c new file mode 100644 -index 000000000..00a2ba9c3 +index 000000000..ef2023033 --- /dev/null +++ b/lib/impersonate.c -@@ -0,0 +1,745 @@ +@@ -0,0 +1,970 @@ +#include "curl_setup.h" + +#include @@ -1272,7 +1483,9 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "chrome100", @@ -1314,7 +1527,9 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "chrome101", @@ -1356,7 +1571,9 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "chrome104", @@ -1398,7 +1615,9 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "chrome107", @@ -1440,13 +1659,15 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;2:0;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { -+ .target = "chrome110", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = ++ .target = "chrome110", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," @@ -1462,34 +1683,127 @@ index 000000000..00a2ba9c3 + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", -+ .npn = false, -+ .alpn = true, -+ .alps = true, -+ .tls_permute_extensions = true, -+ .tls_session_ticket = true, -+ .cert_compression = "brotli", -+ .http_headers = { -+ "sec-ch-ua: \"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"", -+ "sec-ch-ua-mobile: ?0", -+ "sec-ch-ua-platform: \"Windows\"", -+ "Upgrade-Insecure-Requests: 1", -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-User: ?1", -+ "Sec-Fetch-Dest: document", -+ "Accept-Encoding: gzip, deflate, br", -+ "Accept-Language: en-US,en;q=0.9" -+ }, -+ .http2_settings = "1:65536;2:0;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Chromium\";v=\"110\", \"Not A(Brand\";v=\"24\", \"Google Chrome\";v=\"110\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ }, ++ .http2_settings = "1:65536;2:0;3:1000;4:6291456;6:262144", ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true ++ }, ++ { ++ .target = "chrome116", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"Windows\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ }, ++ .http2_settings = "1:65536;2:0;3:1000;4:6291456;6:262144", ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true ++ }, ++ { ++ .target = "chrome119", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "ECDHE-ECDSA-AES128-GCM-SHA256," ++ "ECDHE-RSA-AES128-GCM-SHA256," ++ "ECDHE-ECDSA-AES256-GCM-SHA384," ++ "ECDHE-RSA-AES256-GCM-SHA384," ++ "ECDHE-ECDSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-CHACHA20-POLY1305," ++ "ECDHE-RSA-AES128-SHA," ++ "ECDHE-RSA-AES256-SHA," ++ "AES128-GCM-SHA256," ++ "AES256-GCM-SHA384," ++ "AES128-SHA," ++ "AES256-SHA", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Google Chrome\";v=\"119\", \"Chromium\";v=\"119\", \"Not?A_Brand\";v=\"24\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"macOS\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ }, ++ .http2_settings = "1:65536;2:0;4:6291456;6:262144", ++ .http2_window_update = 15663105, ++ .ech = "GREASE", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { -+ .target = "chrome116", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = ++ .target = "chrome120", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," @@ -1505,34 +1819,37 @@ index 000000000..00a2ba9c3 + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", -+ .npn = false, -+ .alpn = true, -+ .alps = true, -+ .tls_permute_extensions = true, -+ .tls_session_ticket = true, -+ .cert_compression = "brotli", -+ .http_headers = { -+ "sec-ch-ua: \"Chromium\";v=\"116\", \"Not)A;Brand\";v=\"24\", \"Google Chrome\";v=\"116\"", -+ "sec-ch-ua-mobile: ?0", -+ "sec-ch-ua-platform: \"Windows\"", -+ "Upgrade-Insecure-Requests: 1", -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-User: ?1", -+ "Sec-Fetch-Dest: document", -+ "Accept-Encoding: gzip, deflate, br", -+ "Accept-Language: en-US,en;q=0.9" -+ }, -+ .http2_settings = "1:65536;2:0;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"macOS\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br", ++ "Accept-Language: en-US,en;q=0.9" ++ }, ++ .http2_settings = "1:65536;2:0;4:6291456;6:262144", ++ .http2_window_update = 15663105, ++ .ech = "GREASE", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { -+ .target = "chrome119", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = ++ .target = "chrome123", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," @@ -1548,35 +1865,37 @@ index 000000000..00a2ba9c3 + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", -+ .npn = false, -+ .alpn = true, -+ .alps = true, -+ .tls_permute_extensions = true, -+ .tls_session_ticket = true, -+ .cert_compression = "brotli", -+ .http_headers = { -+ "sec-ch-ua: \"Google Chrome\";v=\"119\", \"Chromium\";v=\"119\", \"Not?A_Brand\";v=\"24\"", -+ "sec-ch-ua-mobile: ?0", -+ "sec-ch-ua-platform: \"macOS\"", -+ "Upgrade-Insecure-Requests: 1", -+ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-User: ?1", -+ "Sec-Fetch-Dest: document", -+ "Accept-Encoding: gzip, deflate, br", -+ "Accept-Language: en-US,en;q=0.9" -+ }, -+ .http2_settings = "1:65536;2:0;4:6291456;6:262144", -+ .http2_window_update = 15663105, -+ .ech = "GREASE" ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Google Chrome\";v=\"123\", \"Not:A-Brand\";v=\"8\", \"Chromium\";v=\"123\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"macOS\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br, zstd", ++ "Accept-Language: en-US,en;q=0.9" ++ }, ++ .http2_settings = "1:65536;2:0;4:6291456;6:262144", ++ .http2_window_update = 15663105, ++ .ech = "GREASE", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { -+ .target = "chrome120", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = ++ .target = "chrome124", ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_CHACHA20_POLY1305_SHA256," @@ -1592,29 +1911,33 @@ index 000000000..00a2ba9c3 + "AES256-GCM-SHA384," + "AES128-SHA," + "AES256-SHA", -+ .npn = false, -+ .alpn = true, -+ .alps = true, -+ .tls_permute_extensions = true, -+ .tls_session_ticket = true, -+ .cert_compression = "brotli", -+ .http_headers = { -+ "sec-ch-ua: \"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\", \"Google Chrome\";v=\"120\"", -+ "sec-ch-ua-mobile: ?0", -+ "sec-ch-ua-platform: \"macOS\"", -+ "Upgrade-Insecure-Requests: 1", -+ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-User: ?1", -+ "Sec-Fetch-Dest: document", -+ "Accept-Encoding: gzip, deflate, br", -+ "Accept-Language: en-US,en;q=0.9" -+ }, -+ .http2_settings = "1:65536;2:0;4:6291456;6:262144", -+ .http2_window_update = 15663105, -+ .ech = "GREASE" ++ .curves = "X25519Kyber768Draft00:X25519:P-256:P-384", ++ .npn = false, ++ .alpn = true, ++ .alps = true, ++ .tls_permute_extensions = true, ++ .tls_session_ticket = true, ++ .cert_compression = "brotli", ++ .http_headers = { ++ "sec-ch-ua: \"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"", ++ "sec-ch-ua-mobile: ?0", ++ "sec-ch-ua-platform: \"macOS\"", ++ "Upgrade-Insecure-Requests: 1", ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-User: ?1", ++ "Sec-Fetch-Dest: document", ++ "Accept-Encoding: gzip, deflate, br, zstd", ++ "Accept-Language: en-US,en;q=0.9", ++ "Priority: u=0, i" ++ }, ++ .http2_settings = "1:65536;2:0;4:6291456;6:262144", ++ .http2_window_update = 15663105, ++ .ech = "GREASE", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "chrome99_android", @@ -1656,7 +1979,9 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "edge99", @@ -1698,7 +2023,9 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "edge101", @@ -1740,65 +2067,69 @@ index 000000000..00a2ba9c3 + "Accept-Language: en-US,en;q=0.9" + }, + .http2_settings = "1:65536;3:1000;4:6291456;6:262144", -+ .http2_window_update = 15663105 ++ .http2_window_update = 15663105, ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "safari15_3", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = -+ "TLS_AES_128_GCM_SHA256," -+ "TLS_AES_256_GCM_SHA384," -+ "TLS_CHACHA20_POLY1305_SHA256," -+ "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," -+ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," -+ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," -+ "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," -+ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," -+ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," -+ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384," -+ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256," -+ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," -+ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," -+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384," -+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256," -+ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," -+ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," -+ "TLS_RSA_WITH_AES_256_GCM_SHA384," -+ "TLS_RSA_WITH_AES_128_GCM_SHA256," -+ "TLS_RSA_WITH_AES_256_CBC_SHA256," -+ "TLS_RSA_WITH_AES_128_CBC_SHA256," -+ "TLS_RSA_WITH_AES_256_CBC_SHA," -+ "TLS_RSA_WITH_AES_128_CBC_SHA," -+ "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA," -+ "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA," -+ "TLS_RSA_WITH_3DES_EDE_CBC_SHA,", ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," ++ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," ++ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384," ++ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256," ++ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," ++ "TLS_RSA_WITH_AES_256_GCM_SHA384," ++ "TLS_RSA_WITH_AES_128_GCM_SHA256," ++ "TLS_RSA_WITH_AES_256_CBC_SHA256," ++ "TLS_RSA_WITH_AES_128_CBC_SHA256," ++ "TLS_RSA_WITH_AES_256_CBC_SHA," ++ "TLS_RSA_WITH_AES_128_CBC_SHA," ++ "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA," ++ "TLS_RSA_WITH_3DES_EDE_CBC_SHA,", + .curves = "X25519:P-256:P-384:P-521", + .sig_hash_algs = -+ "ecdsa_secp256r1_sha256," -+ "rsa_pss_rsae_sha256," -+ "rsa_pkcs1_sha256," -+ "ecdsa_secp384r1_sha384," -+ "ecdsa_sha1," -+ "rsa_pss_rsae_sha384," -+ "rsa_pss_rsae_sha384," -+ "rsa_pkcs1_sha384," -+ "rsa_pss_rsae_sha512," -+ "rsa_pkcs1_sha512," -+ "rsa_pkcs1_sha1", ++ "ecdsa_secp256r1_sha256," ++ "rsa_pss_rsae_sha256," ++ "rsa_pkcs1_sha256," ++ "ecdsa_secp384r1_sha384," ++ "ecdsa_sha1," ++ "rsa_pss_rsae_sha384," ++ "rsa_pss_rsae_sha384," ++ "rsa_pkcs1_sha384," ++ "rsa_pss_rsae_sha512," ++ "rsa_pkcs1_sha512," ++ "rsa_pkcs1_sha1", + .npn = false, + .alpn = true, + .alps = false, + .tls_session_ticket = false, + .http_headers = { -+ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", -+ "Accept-Language: en-us", -+ "Accept-Encoding: gzip, deflate, br" ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ++ "Accept-Language: en-us", ++ "Accept-Encoding: gzip, deflate, br" + }, + .http2_settings = "4:4194304;3:100", + .http2_window_update = 10485760, -+ .http2_pseudo_headers_order = "mspa" ++ .http2_pseudo_headers_order = "mspa", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "safari15_5", @@ -1844,14 +2175,16 @@ index 000000000..00a2ba9c3 + .tls_session_ticket = false, + .cert_compression = "zlib", + .http_headers = { -+ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", -+ "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8", -+ "Accept-Encoding: gzip, deflate, br" ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ++ "Accept-Language: en-GB,en-US;q=0.9,en;q=0.8", ++ "Accept-Encoding: gzip, deflate, br" + }, + .http2_settings = "4:4194304;3:100", + .http2_window_update = 10485760, -+ .http2_pseudo_headers_order = "mspa" ++ .http2_pseudo_headers_order = "mspa", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "safari17_2_ios", @@ -1897,17 +2230,19 @@ index 000000000..00a2ba9c3 + .tls_session_ticket = false, + .cert_compression = "zlib", + .http_headers = { -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", -+ "Sec-Fetch-Site: none", -+ "Accept-Encoding: gzip, deflate, br", -+ "Sec-Fetch-Mode: navigate", -+ "User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1", -+ "Accept-Language: en-US,en;q=0.9", -+ "Sec-Fetch-Dest: document" ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ++ "Sec-Fetch-Site: none", ++ "Accept-Encoding: gzip, deflate, br", ++ "Sec-Fetch-Mode: navigate", ++ "User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1", ++ "Accept-Language: en-US,en;q=0.9", ++ "Sec-Fetch-Dest: document" + }, + .http2_settings = "2:0;4:2097152;3:100", + .http2_window_update = 10485760, -+ .http2_pseudo_headers_order = "mspa" ++ .http2_pseudo_headers_order = "mspa", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + .target = "safari17_0", @@ -1953,17 +2288,118 @@ index 000000000..00a2ba9c3 + .tls_session_ticket = false, + .cert_compression = "zlib", + .http_headers = { -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", -+ "Sec-Fetch-Site: none", -+ "Accept-Encoding: gzip, deflate, br", -+ "Sec-Fetch-Mode: navigate", -+ "user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", -+ "Accept-Language: en-US,en;q=0.9", -+ "Sec-Fetch-Dest: document" ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ++ "Sec-Fetch-Site: none", ++ "Accept-Encoding: gzip, deflate, br", ++ "Sec-Fetch-Mode: navigate", ++ "user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", ++ "Accept-Language: en-US,en;q=0.9", ++ "Sec-Fetch-Dest: document" ++ }, ++ .http2_settings = "2:0;4:4194304;3:100", ++ .http2_window_update = 10485760, ++ .http2_pseudo_headers_order = "mspa", ++ .tls_extension_order = NULL, ++ .tls_grease = true ++ }, ++ { ++ .target = "okhttp4", /* not working */ ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_0 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," ++ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," ++ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," ++ "TLS_RSA_WITH_AES_256_GCM_SHA384," ++ "TLS_RSA_WITH_AES_128_GCM_SHA256," ++ "TLS_RSA_WITH_AES_256_CBC_SHA," ++ "TLS_RSA_WITH_AES_128_CBC_SHA," ++ "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA," ++ "TLS_RSA_WITH_3DES_EDE_CBC_SHA", ++ .curves = "X25519:P-256:P-384:P-521", ++ .sig_hash_algs = ++ "ecdsa_secp256r1_sha256," ++ "rsa_pss_rsae_sha256," ++ "rsa_pkcs1_sha256," ++ "ecdsa_secp384r1_sha384," ++ "ecdsa_sha1," ++ "rsa_pss_rsae_sha384," ++ "rsa_pss_rsae_sha384," ++ "rsa_pkcs1_sha384," ++ "rsa_pss_rsae_sha512," ++ "rsa_pkcs1_sha512," ++ "rsa_pkcs1_sha1", ++ .npn = false, ++ .alpn = true, ++ .alps = false, ++ .tls_session_ticket = false, ++ .cert_compression = "zlib", ++ .http_headers = { ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ++ "Sec-Fetch-Site: none", ++ "Accept-Encoding: gzip, deflate, br", ++ "Sec-Fetch-Mode: navigate", ++ "user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15", ++ "Accept-Language: en-US,en;q=0.9", ++ "Sec-Fetch-Dest: document" + }, + .http2_settings = "2:0;4:4194304;3:100", + .http2_window_update = 10485760, -+ .http2_pseudo_headers_order = "mspa" ++ .http2_pseudo_headers_order = "mspa", ++ .tls_extension_order = NULL, ++ .tls_grease = true ++ }, ++ { ++ .target = "firefox120", /* not working */ ++ .httpversion = CURL_HTTP_VERSION_2_0, ++ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, ++ .ciphers = ++ "TLS_AES_128_GCM_SHA256," ++ "TLS_CHACHA20_POLY1305_SHA256," ++ "TLS_AES_256_GCM_SHA384," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," ++ "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," ++ "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," ++ "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," ++ "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," ++ "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," ++ "TLS_RSA_WITH_AES_128_GCM_SHA256," ++ "TLS_RSA_WITH_AES_256_GCM_SHA384," ++ "TLS_RSA_WITH_AES_128_CBC_SHA," ++ "TLS_RSA_WITH_AES_256_CBC_SHA", ++ .http_headers = { ++ "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0", ++ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", ++ "Accept-Language: en-US,en;q=0.5", ++ "Accept-Encoding: gzip, deflate, br", ++ "Upgrade-Insecure-Requests: 1", ++ "Sec-Fetch-Dest: document", ++ "Sec-Fetch-Mode: navigate", ++ "Sec-Fetch-Site: none", ++ "Sec-Fetch-User: ?1", ++ "Te: trailers" ++ }, ++ .http2_settings = "1:65536;4:131072;5:16384", ++ .http2_window_update = 12517377, ++ .http2_pseudo_headers_order = "mpas", ++ .http2_streams = "3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241", ++ .tls_extension_order = NULL, ++ .tls_grease = true + }, + { + /* Last one must be NULL. */ @@ -1972,10 +2408,10 @@ index 000000000..00a2ba9c3 +}; diff --git a/lib/impersonate.h b/lib/impersonate.h new file mode 100644 -index 000000000..0158d5477 +index 000000000..a9e1e4b45 --- /dev/null +++ b/lib/impersonate.h -@@ -0,0 +1,47 @@ +@@ -0,0 +1,50 @@ +#ifndef HEADER_CURL_IMPERSONATE_H +#define HEADER_CURL_IMPERSONATE_H + @@ -2010,8 +2446,11 @@ index 000000000..0158d5477 + const char *http2_pseudo_headers_order; + const char *http2_settings; + int http2_window_update; ++ const char *http2_streams; + bool tls_permute_extensions; + const char *ech; ++ const char *tls_extension_order; ++ bool tls_grease; + /* Other TLS options will come here in the future once they are + * configurable through curl_easy_setopt() */ +}; @@ -2024,13 +2463,13 @@ index 000000000..0158d5477 + +#endif /* HEADER_CURL_IMPERSONATE_H */ diff --git a/lib/multi.c b/lib/multi.c -index d1d32b793..3b49a1b4c 100644 +index 5456113be..85841f769 100644 --- a/lib/multi.c +++ b/lib/multi.c -@@ -424,7 +424,8 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ +@@ -396,7 +396,8 @@ struct Curl_multi *Curl_multi_handle(int hashsize, /* socket hash */ + Curl_llist_init(&multi->msgsent, NULL); - /* -1 means it not set by user, use the default value */ - multi->maxconnects = -1; + multi->multiplexing = TRUE; - multi->max_concurrent_streams = 100; + /* curl-impersonate: Use 1000 concurrent streams like Chrome. */ + multi->max_concurrent_streams = 1000; @@ -2038,18 +2477,18 @@ index d1d32b793..3b49a1b4c 100644 #ifdef USE_WINSOCK multi->wsa_event = WSACreateEvent(); diff --git a/lib/setopt.c b/lib/setopt.c -index 0c3b9634d..a65c5d99e 100644 +index a08140cce..05468ff49 100644 --- a/lib/setopt.c +++ b/lib/setopt.c -@@ -50,6 +50,7 @@ - #include "multiif.h" +@@ -51,6 +51,7 @@ #include "altsvc.h" #include "hsts.h" + #include "tftp.h" +#include "slist.h" /* The last 3 #include files should be in this order */ #include "curl_printf.h" -@@ -712,6 +713,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) +@@ -710,6 +711,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) va_arg(param, char *)); break; @@ -2073,7 +2512,7 @@ index 0c3b9634d..a65c5d99e 100644 #ifndef CURL_DISABLE_PROXY case CURLOPT_PROXYHEADER: /* -@@ -2410,6 +2428,27 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) +@@ -2394,6 +2412,27 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) result = Curl_setstropt(&data->set.str[STRING_SSL_EC_CURVES], va_arg(param, char *)); break; @@ -2101,9 +2540,9 @@ index 0c3b9634d..a65c5d99e 100644 #endif case CURLOPT_IPRESOLVE: arg = va_arg(param, long); -@@ -2953,6 +2992,31 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) +@@ -2936,6 +2975,42 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) case CURLOPT_SSL_ENABLE_ALPN: - data->set.ssl_enable_alpn = (0 != va_arg(param, long)) ? TRUE : FALSE; + data->set.ssl_enable_alpn = (0 != va_arg(param, long)); break; + case CURLOPT_SSL_ENABLE_ALPS: + data->set.ssl_enable_alps = (0 != va_arg(param, long)) ? TRUE : FALSE; @@ -2114,6 +2553,13 @@ index 0c3b9634d..a65c5d99e 100644 + case CURLOPT_SSL_PERMUTE_EXTENSIONS: + data->set.ssl_permute_extensions = (0 != va_arg(param, long)) ? TRUE : FALSE; + break; ++ case CURLOPT_TLS_GREASE: ++ data->set.tls_grease = (0 != va_arg(param, long)) ? TRUE : FALSE; ++ break; ++ case CURLOPT_TLS_EXTENSION_ORDER: ++ result = Curl_setstropt(&data->set.str[STRING_TLS_EXTENSION_ORDER], ++ va_arg(param, char *)); ++ break; +#ifdef USE_HTTP2 + case CURLOPT_HTTP2_PSEUDO_HEADERS_ORDER: + result = Curl_setstropt(&data->set.str[STRING_HTTP2_PSEUDO_HEADERS_ORDER], @@ -2129,11 +2575,15 @@ index 0c3b9634d..a65c5d99e 100644 + return CURLE_BAD_FUNCTION_ARGUMENT; + data->set.http2_window_update = arg; + break; ++ case CURLOPT_HTTP2_STREAMS: ++ result = Curl_setstropt(&data->set.str[STRING_HTTP2_STREAMS], ++ va_arg(param, char *)); ++ break; +#endif #ifdef USE_UNIX_SOCKETS case CURLOPT_UNIX_SOCKET_PATH: data->set.abstract_unix_socket = FALSE; -@@ -3146,6 +3210,31 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) +@@ -3128,6 +3203,31 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) data->set.ws_raw_mode = raw; break; } @@ -2166,7 +2616,7 @@ index 0c3b9634d..a65c5d99e 100644 case CURLOPT_QUICK_EXIT: data->set.quick_exit = (0 != va_arg(param, long)) ? 1L:0L; diff --git a/lib/strerror.c b/lib/strerror.c -index bd9cc535c..971568792 100644 +index 0d5f9276f..232116383 100644 --- a/lib/strerror.c +++ b/lib/strerror.c @@ -319,6 +319,11 @@ curl_easy_strerror(CURLcode error) @@ -2182,10 +2632,10 @@ index bd9cc535c..971568792 100644 case CURLE_OBSOLETE20: case CURLE_OBSOLETE24: diff --git a/lib/transfer.c b/lib/transfer.c -index d2ff0c24c..56e2090b6 100644 +index 96f1fde75..4a57497c7 100644 --- a/lib/transfer.c +++ b/lib/transfer.c -@@ -106,7 +106,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, +@@ -104,7 +104,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, DEBUGASSERT(thislen); DEBUGASSERT(thisheader[thislen-1] != ':'); @@ -2203,10 +2653,10 @@ index d2ff0c24c..56e2090b6 100644 Curl_headersep(head->data[thislen]) ) return head->data; diff --git a/lib/url.c b/lib/url.c -index b37d13f8f..f1b3b5440 100644 +index b81785fe2..699e8037a 100644 --- a/lib/url.c +++ b/lib/url.c -@@ -444,6 +444,11 @@ CURLcode Curl_close(struct Curl_easy **datap) +@@ -322,6 +322,11 @@ CURLcode Curl_close(struct Curl_easy **datap) Curl_safefree(data->state.aptr.proxyuser); Curl_safefree(data->state.aptr.proxypasswd); @@ -2218,7 +2668,7 @@ index b37d13f8f..f1b3b5440 100644 #ifndef CURL_DISABLE_DOH if(data->req.doh) { Curl_dyn_free(&data->req.doh->probe[0].serverdoh); -@@ -595,6 +600,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data) +@@ -468,6 +473,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data) set->tcp_fastopen = FALSE; set->tcp_nodelay = TRUE; set->ssl_enable_alpn = TRUE; @@ -2226,17 +2676,7 @@ index b37d13f8f..f1b3b5440 100644 set->expect_100_timeout = 1000L; /* Wait for a second by default. */ set->sep_headers = TRUE; /* separated header lists by default */ set->buffer_size = READBUFFER_SIZE; -@@ -3584,6 +3590,9 @@ static CURLcode create_conn(struct Curl_easy *data, - data->set.ssl.primary.cert_blob = data->set.blobs[BLOB_CERT]; - data->set.ssl.primary.ca_info_blob = data->set.blobs[BLOB_CAINFO]; - data->set.ssl.primary.curves = data->set.str[STRING_SSL_EC_CURVES]; -+ data->set.ssl.primary.sig_hash_algs = data->set.str[STRING_SSL_SIG_HASH_ALGS]; -+ data->set.ssl.primary.cert_compression = -+ data->set.str[STRING_SSL_CERT_COMPRESSION]; - - #ifndef CURL_DISABLE_PROXY - data->set.proxy_ssl.primary.CApath = data->set.str[STRING_SSL_CAPATH_PROXY]; -@@ -3695,6 +3704,11 @@ static CURLcode create_conn(struct Curl_easy *data, +@@ -3672,6 +3678,11 @@ static CURLcode create_conn(struct Curl_easy *data, (default) */ if(data->set.ssl_enable_alpn) conn->bits.tls_enable_alpn = TRUE; @@ -2249,7 +2689,7 @@ index b37d13f8f..f1b3b5440 100644 if(waitpipe) diff --git a/lib/urldata.h b/lib/urldata.h -index f02e66541..058ace33c 100644 +index ff661482e..29b9d37fa 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -53,6 +53,15 @@ @@ -2268,7 +2708,7 @@ index f02e66541..058ace33c 100644 #ifdef USE_WEBSOCKETS /* CURLPROTO_GOPHERS (29) is the highest publicly used protocol bit number, * the rest are internal information. If we use higher bits we only do this on -@@ -277,6 +286,8 @@ struct ssl_primary_config { +@@ -290,6 +299,8 @@ struct ssl_primary_config { char *password; /* TLS password (for, e.g., SRP) */ #endif char *curves; /* list of curves to use */ @@ -2277,7 +2717,7 @@ index f02e66541..058ace33c 100644 unsigned char ssl_options; /* the CURLOPT_SSL_OPTIONS bitmask */ unsigned int version_max; /* max supported version the client wants to use */ unsigned char version; /* what version the client wants to use */ -@@ -526,6 +537,9 @@ struct ConnectBits { +@@ -541,6 +552,9 @@ struct ConnectBits { BIT(multiplex); /* connection is multiplexed */ BIT(tcp_fastopen); /* use TCP Fast Open */ BIT(tls_enable_alpn); /* TLS ALPN extension? */ @@ -2287,7 +2727,7 @@ index f02e66541..058ace33c 100644 #ifndef CURL_DISABLE_DOH BIT(doh); #endif -@@ -1395,6 +1409,19 @@ struct UrlState { +@@ -1452,6 +1466,19 @@ struct UrlState { CURLcode hresult; /* used to pass return codes back from hyper callbacks */ #endif @@ -2307,20 +2747,22 @@ index f02e66541..058ace33c 100644 /* Dynamically allocated strings, MUST be freed before this struct is killed. */ struct dynamically_allocated_data { -@@ -1563,6 +1590,12 @@ enum dupstring { - STRING_DNS_LOCAL_IP6, +@@ -1628,6 +1655,14 @@ enum dupstring { STRING_SSL_EC_CURVES, STRING_AWS_SIGV4, /* Parameters for V4 signature */ + STRING_HAPROXY_CLIENT_IP, /* CURLOPT_HAPROXY_CLIENT_IP */ + STRING_SSL_SIG_HASH_ALGS, + STRING_SSL_CERT_COMPRESSION, + STRING_HTTP2_PSEUDO_HEADERS_ORDER, + STRING_HTTP2_SETTINGS, ++ STRING_HTTP2_STREAMS, + STRING_ECH_CONFIG, /* CURLOPT_ECH_CONFIG */ + STRING_ECH_PUBLIC, /* CURLOPT_ECH_PUBLIC */ ++ STRING_TLS_EXTENSION_ORDER, /* -- end of null-terminated strings -- */ -@@ -1857,6 +1890,9 @@ struct UserDefined { +@@ -1921,6 +1956,9 @@ struct UserDefined { BIT(tcp_keepalive); /* use TCP keepalives */ BIT(tcp_fastopen); /* use TCP Fast Open */ BIT(ssl_enable_alpn);/* TLS ALPN extension? */ @@ -2330,7 +2772,7 @@ index f02e66541..058ace33c 100644 BIT(path_as_is); /* allow dotdots? */ BIT(pipewait); /* wait for multiplex status before starting a new connection */ -@@ -1877,6 +1913,10 @@ struct UserDefined { +@@ -1941,6 +1979,11 @@ struct UserDefined { #ifdef USE_WEBSOCKETS BIT(ws_raw_mode); #endif @@ -2338,19 +2780,22 @@ index f02e66541..058ace33c 100644 + int tls_ech; /* TLS ECH configuration */ +#endif + int http2_window_update; ++ BIT(tls_grease); }; struct Names { diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c -index 6543fb19a..58d243f1e 100644 +index 8c8f43e83..030832aeb 100644 --- a/lib/vtls/openssl.c +++ b/lib/vtls/openssl.c -@@ -79,6 +79,21 @@ +@@ -79,9 +79,24 @@ #include #include #include +#include -+ + #include + #include + +#ifdef HAVE_LIBZ +#include +#endif @@ -2364,10 +2809,11 @@ index 6543fb19a..58d243f1e 100644 +# endif +# include "curl_base64.h" +#endif /* USE_ECH */ - ++ #if (OPENSSL_VERSION_NUMBER >= 0x0090808fL) && !defined(OPENSSL_NO_OCSP) #include -@@ -266,6 +281,113 @@ + #endif +@@ -276,6 +291,113 @@ #define HAVE_OPENSSL_VERSION #endif @@ -2478,10 +2924,10 @@ index 6543fb19a..58d243f1e 100644 + +#endif + - #ifdef OPENSSL_IS_BORINGSSL + #if defined(OPENSSL_IS_BORINGSSL) || defined(OPENSSL_IS_AWSLC) typedef uint32_t sslerr_t; #else -@@ -2583,6 +2705,151 @@ static const char *tls_rt_type(int type) +@@ -2601,6 +2723,151 @@ static const char *tls_rt_type(int type) } } @@ -2633,7 +3079,7 @@ index 6543fb19a..58d243f1e 100644 /* * Our callback from the SSL/TLS layers. */ -@@ -3536,7 +3803,14 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, +@@ -3571,7 +3838,14 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, ctx_options = SSL_OP_ALL; #ifdef SSL_OP_NO_TICKET @@ -2649,7 +3095,7 @@ index 6543fb19a..58d243f1e 100644 #endif #ifdef SSL_OP_NO_COMPRESSION -@@ -3603,6 +3877,16 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, +@@ -3638,6 +3912,18 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif @@ -2658,15 +3104,17 @@ index 6543fb19a..58d243f1e 100644 + SSL_CTX_set_mode(backend->ctx, + SSL_MODE_CBC_RECORD_SPLITTING | SSL_MODE_ENABLE_FALSE_START); + -+ /* curl-impersonate: Enable TLS extensions 5 - status_request and -+ * 18 - signed_certificate_timestamp. */ ++ /* curl-impersonate: Enable TLS extensions 18 - signed_certificate_timestamp. */ ++ // XXX: Firefox does not enable this + SSL_CTX_enable_signed_cert_timestamps(backend->ctx); ++ ++ /* curl-impersonate: Enable TLS extensions 5 - status_request */ + SSL_CTX_enable_ocsp_stapling(backend->ctx); + if(ssl_cert || ssl_cert_blob || ssl_cert_type) { if(!result && !cert_stuff(data, backend->ctx, -@@ -3656,6 +3940,35 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, +@@ -3691,6 +3977,35 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif @@ -2702,7 +3150,7 @@ index 6543fb19a..58d243f1e 100644 #ifdef USE_OPENSSL_SRP if(ssl_config->primary.username && Curl_auth_allowed_to_host(data)) { char * const ssl_username = ssl_config->primary.username; -@@ -3681,6 +3994,30 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, +@@ -3716,6 +4031,38 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif @@ -2712,8 +3160,10 @@ index 6543fb19a..58d243f1e 100644 + * and SSLClientSocketImpl::Init() + * in the Chromium's source code. */ + -+ /* Enable TLS GREASE. */ -+ SSL_CTX_set_grease_enabled(backend->ctx, 1); ++ /* curl-impersonate: Enable TLS GREASE. */ ++ if(data->set.tls_grease) { ++ SSL_CTX_set_grease_enabled(backend->ctx, 1); ++ } + + /* + * curl-impersonate: Enable TLS extension permutation, enabled by default @@ -2723,6 +3173,12 @@ index 6543fb19a..58d243f1e 100644 + SSL_CTX_set_permute_extensions(backend->ctx, 1); + } + ++ /* curl-impersonate: Set TLS extensions order. ++ */ ++ if(data->set.str[STRING_TLS_EXTENSION_ORDER]) { ++ SSL_CTX_set_extension_order(backend->ctx, data->set.str[STRING_TLS_EXTENSION_ORDER]); ++ } ++ + if(conn_config->cert_compression && + add_cert_compression(data, + backend->ctx, @@ -2733,7 +3189,7 @@ index 6543fb19a..58d243f1e 100644 /* OpenSSL always tries to verify the peer, this only says whether it should * fail to connect if the verification fails, or if it should continue * anyway. In the latter case the result of the verification is checked with -@@ -3727,6 +4064,23 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, +@@ -3771,6 +4118,24 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, SSL_set_app_data(backend->handle, cf); @@ -2744,6 +3200,7 @@ index 6543fb19a..58d243f1e 100644 + + for(i = 0; i < connssl->alps->count; ++i) { + /* curl-impersonate: Add the ALPS extension (17513) like Chrome does. */ ++ // XXX: Firefox does not enable this. + SSL_add_application_settings(backend->handle, connssl->alps->entries[i], + strlen(connssl->alps->entries[i]), NULL, + 0); @@ -2757,7 +3214,7 @@ index 6543fb19a..58d243f1e 100644 #if (OPENSSL_VERSION_NUMBER >= 0x0090808fL) && !defined(OPENSSL_NO_TLSEXT) && \ !defined(OPENSSL_NO_OCSP) if(conn_config->verifystatus) -@@ -3755,6 +4109,21 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, +@@ -3794,6 +4159,21 @@ static CURLcode ossl_connect_step1(struct Curl_cfilter *cf, } #endif @@ -2778,10 +3235,10 @@ index 6543fb19a..58d243f1e 100644 + SSL_set_app_data(backend->handle, cf); - if(ssl_config->primary.sessionid) { -@@ -3946,6 +4315,60 @@ static CURLcode ossl_connect_step2(struct Curl_cfilter *cf, - SSL_get_version(backend->handle), - SSL_get_cipher(backend->handle)); + connssl->reused_session = FALSE; +@@ -4005,6 +4385,60 @@ static CURLcode ossl_connect_step2(struct Curl_cfilter *cf, + negotiated_group_name? negotiated_group_name : "[blank]", + OBJ_nid2sn(psigtype_nid)); +#ifdef USE_ECH +# ifndef OPENSSL_IS_BORINGSSL @@ -2841,10 +3298,10 @@ index 6543fb19a..58d243f1e 100644 /* Sets data and len to negotiated protocol, len is 0 if no protocol was * negotiated diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c -index 32334016b..1a8a75ade 100644 +index 34eda3e5a..eda3f6d58 100644 --- a/lib/vtls/vtls.c +++ b/lib/vtls/vtls.c -@@ -141,6 +141,9 @@ static const struct alpn_spec ALPN_SPEC_H11 = { +@@ -139,6 +139,9 @@ static const struct alpn_spec ALPN_SPEC_H11 = { static const struct alpn_spec ALPN_SPEC_H2_H11 = { { ALPN_H2, ALPN_HTTP_1_1 }, 2 }; @@ -2855,7 +3312,7 @@ index 32334016b..1a8a75ade 100644 static const struct alpn_spec *alpn_get_spec(int httpwant, bool use_alpn) @@ -155,6 +158,17 @@ static const struct alpn_spec *alpn_get_spec(int httpwant, bool use_alpn) - #endif + Avoid "http/1.0" because some servers don't support it. */ return &ALPN_SPEC_H11; } + @@ -2872,16 +3329,16 @@ index 32334016b..1a8a75ade 100644 #endif /* USE_SSL */ -@@ -182,6 +196,8 @@ Curl_ssl_config_matches(struct ssl_primary_config *data, - strcasecompare(data->cipher_list, needle->cipher_list) && - strcasecompare(data->cipher_list13, needle->cipher_list13) && - strcasecompare(data->curves, needle->curves) && -+ strcasecompare(data->sig_hash_algs, needle->sig_hash_algs) && -+ strcasecompare(data->cert_compression, needle->cert_compression) && - strcasecompare(data->CRLfile, needle->CRLfile) && - strcasecompare(data->pinned_key, needle->pinned_key)) +@@ -198,6 +212,8 @@ match_ssl_primary_config(struct Curl_easy *data, + strcasecompare(c1->cipher_list, c2->cipher_list) && + strcasecompare(c1->cipher_list13, c2->cipher_list13) && + strcasecompare(c1->curves, c2->curves) && ++ strcasecompare(c1->sig_hash_algs, c2->sig_hash_algs) && ++ strcasecompare(c1->cert_compression, c2->cert_compression) && + strcasecompare(c1->CRLfile, c2->CRLfile) && + strcasecompare(c1->pinned_key, c2->pinned_key)) return TRUE; -@@ -212,6 +228,8 @@ Curl_clone_primary_ssl_config(struct ssl_primary_config *source, +@@ -242,6 +258,8 @@ static bool clone_ssl_primary_config(struct ssl_primary_config *source, CLONE_STRING(cipher_list13); CLONE_STRING(pinned_key); CLONE_STRING(curves); @@ -2890,7 +3347,7 @@ index 32334016b..1a8a75ade 100644 CLONE_STRING(CRLfile); #ifdef USE_TLS_SRP CLONE_STRING(username); -@@ -234,6 +252,8 @@ void Curl_free_primary_ssl_config(struct ssl_primary_config *sslc) +@@ -264,6 +282,8 @@ static void Curl_free_primary_ssl_config(struct ssl_primary_config *sslc) Curl_safefree(sslc->ca_info_blob); Curl_safefree(sslc->issuercert_blob); Curl_safefree(sslc->curves); @@ -2899,7 +3356,16 @@ index 32334016b..1a8a75ade 100644 Curl_safefree(sslc->CRLfile); #ifdef USE_TLS_SRP Curl_safefree(sslc->username); -@@ -318,7 +338,8 @@ static bool ssl_prefs_check(struct Curl_easy *data) +@@ -287,6 +307,8 @@ CURLcode Curl_ssl_easy_config_complete(struct Curl_easy *data) + data->set.ssl.primary.cert_blob = data->set.blobs[BLOB_CERT]; + data->set.ssl.primary.ca_info_blob = data->set.blobs[BLOB_CAINFO]; + data->set.ssl.primary.curves = data->set.str[STRING_SSL_EC_CURVES]; ++ data->set.ssl.primary.sig_hash_algs = data->set.str[STRING_SSL_SIG_HASH_ALGS]; ++ data->set.ssl.primary.cert_compression = data->set.str[STRING_SSL_CERT_COMPRESSION]; + #ifdef USE_TLS_SRP + data->set.ssl.primary.username = data->set.str[STRING_TLSAUTH_USERNAME]; + data->set.ssl.primary.password = data->set.str[STRING_TLSAUTH_PASSWORD]; +@@ -453,7 +475,8 @@ static bool ssl_prefs_check(struct Curl_easy *data) } static struct ssl_connect_data *cf_ctx_new(struct Curl_easy *data, @@ -2909,7 +3375,7 @@ index 32334016b..1a8a75ade 100644 { struct ssl_connect_data *ctx; -@@ -328,6 +349,7 @@ static struct ssl_connect_data *cf_ctx_new(struct Curl_easy *data, +@@ -463,6 +486,7 @@ static struct ssl_connect_data *cf_ctx_new(struct Curl_easy *data, return NULL; ctx->alpn = alpn; @@ -2917,7 +3383,7 @@ index 32334016b..1a8a75ade 100644 ctx->backend = calloc(1, Curl_ssl->sizeof_ssl_backend_data); if(!ctx->backend) { free(ctx); -@@ -1760,8 +1782,11 @@ static CURLcode cf_ssl_create(struct Curl_cfilter **pcf, +@@ -1883,8 +1907,11 @@ static CURLcode cf_ssl_create(struct Curl_cfilter **pcf, DEBUGASSERT(data->conn); @@ -2931,7 +3397,7 @@ index 32334016b..1a8a75ade 100644 if(!ctx) { result = CURLE_OUT_OF_MEMORY; goto out; -@@ -1811,6 +1836,7 @@ static CURLcode cf_ssl_proxy_create(struct Curl_cfilter **pcf, +@@ -1934,6 +1961,7 @@ static CURLcode cf_ssl_proxy_create(struct Curl_cfilter **pcf, struct ssl_connect_data *ctx; CURLcode result; bool use_alpn = conn->bits.tls_enable_alpn; @@ -2939,7 +3405,7 @@ index 32334016b..1a8a75ade 100644 int httpwant = CURL_HTTP_VERSION_1_1; #ifdef USE_HTTP2 -@@ -1820,7 +1846,8 @@ static CURLcode cf_ssl_proxy_create(struct Curl_cfilter **pcf, +@@ -1943,7 +1971,8 @@ static CURLcode cf_ssl_proxy_create(struct Curl_cfilter **pcf, } #endif @@ -2950,30 +3416,42 @@ index 32334016b..1a8a75ade 100644 result = CURLE_OUT_OF_MEMORY; goto out; diff --git a/lib/vtls/vtls.h b/lib/vtls/vtls.h -index f24dca15b..595d437f9 100644 +index f1856bd33..fe9f5c266 100644 --- a/lib/vtls/vtls.h +++ b/lib/vtls/vtls.h @@ -44,6 +44,8 @@ struct Curl_ssl_session; "ALPN: server did not agree on a protocol. Uses default." #define VTLS_INFOF_ALPN_OFFER_1STR \ - "ALPN: offers %s" + "ALPN: curl offers %s" +#define VTLS_INFOF_ALPS_OFFER_1STR \ + "ALPS: offers %s" #define VTLS_INFOF_ALPN_ACCEPTED_1STR \ ALPN_ACCEPTED "%s" #define VTLS_INFOF_ALPN_ACCEPTED_LEN_1STR \ diff --git a/lib/vtls/vtls_int.h b/lib/vtls/vtls_int.h -index ed49339e4..9fddc7494 100644 +index af7ae552e..f22147e22 100644 --- a/lib/vtls/vtls_int.h +++ b/lib/vtls/vtls_int.h -@@ -73,6 +73,7 @@ struct ssl_connect_data { - char *hostname; /* hostname for verification */ - char *dispname; /* display version of hostname */ +@@ -70,6 +70,7 @@ struct ssl_connect_data { + ssl_connect_state connecting_state; + struct ssl_peer peer; const struct alpn_spec *alpn; /* ALPN to use or NULL for none */ + const struct alpn_spec *alps; /* ALPS to use or NULL for none */ - struct ssl_backend_data *backend; /* vtls backend specific props */ + void *backend; /* vtls backend specific props */ struct cf_call_data call_data; /* data handle used in current call */ struct curltime handshake_done; /* time when handshake finished */ +diff --git a/libcurl.def b/libcurl.def +index c6c96063a..ac52a596d 100644 +--- a/libcurl.def ++++ b/libcurl.def +@@ -5,6 +5,7 @@ curl_easy_escape + curl_easy_getinfo + curl_easy_header + curl_easy_init ++curl_easy_impersonate + curl_easy_nextheader + curl_easy_option_by_id + curl_easy_option_by_name diff --git a/libcurl.pc.in b/libcurl.pc.in index 9db6b0f89..14c2f23e0 100644 --- a/libcurl.pc.in @@ -2987,10 +3465,10 @@ index 9db6b0f89..14c2f23e0 100644 Libs.private: @LIBCURL_LIBS@ Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ diff --git a/m4/curl-compilers.m4 b/m4/curl-compilers.m4 -index caa2b14cb..0a0af4361 100644 +index 35ba19866..3bab99f62 100644 --- a/m4/curl-compilers.m4 +++ b/m4/curl-compilers.m4 -@@ -373,42 +373,55 @@ AC_DEFUN([CURL_CONVERT_INCLUDE_TO_ISYSTEM], [ +@@ -382,42 +382,55 @@ AC_DEFUN([CURL_CONVERT_INCLUDE_TO_ISYSTEM], [ AC_REQUIRE([CURL_SHFUNC_SQUEEZE])dnl AC_REQUIRE([CURL_CHECK_COMPILER])dnl AC_MSG_CHECKING([convert -I options to -isystem]) @@ -3081,8 +3559,20 @@ index caa2b14cb..0a0af4361 100644 ]) +diff --git a/scripts/singleuse.pl b/scripts/singleuse.pl +index b8a57f8d8..63c0e7a55 100755 +--- a/scripts/singleuse.pl ++++ b/scripts/singleuse.pl +@@ -51,6 +51,7 @@ my %api = ( + 'curl_easy_escape' => 'API', + 'curl_easy_getinfo' => 'API', + 'curl_easy_init' => 'API', ++ 'curl_easy_impersonate' => 'API', + 'curl_easy_pause' => 'API', + 'curl_easy_perform' => 'API', + 'curl_easy_recv' => 'API', diff --git a/src/Makefile.am b/src/Makefile.am -index f24cb6924..30b4fdb0a 100644 +index dced53e0f..dee8a2fc3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -43,7 +43,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \ @@ -3094,7 +3584,7 @@ index f24cb6924..30b4fdb0a 100644 SUBDIRS = ../docs -@@ -54,9 +54,9 @@ endif +@@ -55,9 +55,9 @@ AM_CPPFLAGS += -DBUILDING_CURL include Makefile.inc # CURL_FILES comes from Makefile.inc @@ -3106,23 +3596,39 @@ index f24cb6924..30b4fdb0a 100644 $(CURL_RCFILES): tool_version.h endif -@@ -67,9 +67,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@ +@@ -70,9 +70,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@ LIBS = $(BLANK_AT_MAKETIME) if USE_EXPLICIT_LIB_DEPS -curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@ +curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @LIBCURL_LIBS@ else --curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ -+curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ +-curl_LDADD = $(top_builddir)/lib/libcurl.la @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ ++curl_impersonate_chrome_LDADD = $(top_builddir)/lib/libcurl-impersonate-chrome.la @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ endif # if unit tests are enabled, build a static library to link them with diff --git a/src/tool_cfgable.c b/src/tool_cfgable.c -index ec5698ba2..df8d5913c 100644 +index 906e23e14..5c6613a93 100644 --- a/src/tool_cfgable.c +++ b/src/tool_cfgable.c -@@ -172,6 +172,14 @@ static void free_config_fields(struct OperationConfig *config) +@@ -95,6 +95,15 @@ static void free_config_fields(struct OperationConfig *config) + Curl_safefree(config->proto_str); + Curl_safefree(config->proto_redir_str); + ++ // Impersonate ++ Curl_safefree(config->ssl_sig_hash_algs); ++ Curl_safefree(config->ssl_cert_compression); ++ Curl_safefree(config->http2_pseudo_headers_order); ++ Curl_safefree(config->http2_settings); ++ Curl_safefree(config->http2_streams); ++ Curl_safefree(config->tls_extension_order); ++ // End Impersonate ++ + urlnode = config->url_list; + while(urlnode) { + struct getout *next = urlnode->next; +@@ -175,6 +184,14 @@ static void free_config_fields(struct OperationConfig *config) Curl_safefree(config->aws_sigv4); Curl_safefree(config->proto_str); Curl_safefree(config->proto_redir_str); @@ -3138,10 +3644,10 @@ index ec5698ba2..df8d5913c 100644 void config_free(struct OperationConfig *config) diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h -index 9a15659bc..d7355cfd8 100644 +index 57e8fce52..b5d7019b1 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h -@@ -160,8 +160,13 @@ struct OperationConfig { +@@ -161,8 +161,16 @@ struct OperationConfig { bool crlf; char *customrequest; char *ssl_ec_curves; @@ -3152,10 +3658,13 @@ index 9a15659bc..d7355cfd8 100644 + char *http2_pseudo_headers_order; + char *http2_settings; + long http2_window_update; ++ char *http2_streams; ++ bool tls_grease; ++ char *tls_extension_order; long httpversion; bool http09_allowed; bool nobuffer; -@@ -191,6 +196,7 @@ struct OperationConfig { +@@ -192,6 +200,7 @@ struct OperationConfig { struct curl_slist *prequote; long ssl_version; long ssl_version_max; @@ -3163,7 +3672,7 @@ index 9a15659bc..d7355cfd8 100644 long proxy_ssl_version; long ip_version; long create_file_mode; /* CURLOPT_NEW_FILE_PERMS */ -@@ -266,6 +272,8 @@ struct OperationConfig { +@@ -268,6 +277,8 @@ struct OperationConfig { bool proxy_ssl_auto_client_cert; /* proxy version of ssl_auto_client_cert */ char *oauth_bearer; /* OAuth 2.0 bearer token */ bool noalpn; /* enable/disable TLS ALPN extension */ @@ -3172,7 +3681,7 @@ index 9a15659bc..d7355cfd8 100644 char *unix_socket_path; /* path to Unix domain socket */ bool abstract_unix_socket; /* path to an abstract Unix domain socket */ bool falsestart; -@@ -295,6 +303,11 @@ struct OperationConfig { +@@ -298,6 +309,11 @@ struct OperationConfig { struct State state; /* for create_transfer() */ bool rm_partial; /* on error, remove partially written output files */ @@ -3185,15 +3694,15 @@ index 9a15659bc..d7355cfd8 100644 struct GlobalConfig { diff --git a/src/tool_getparam.c b/src/tool_getparam.c -index c9810e9d4..8bcd914db 100644 +index 5fa1ace10..80277bbd9 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c -@@ -287,6 +287,17 @@ static const struct LongShort aliases[]= { +@@ -296,6 +296,20 @@ static const struct LongShort aliases[]= { {"EC", "etag-save", ARG_FILENAME}, {"ED", "etag-compare", ARG_FILENAME}, {"EE", "curves", ARG_STRING}, -+ {"EG", "signature-hashes", ARG_STRING}, -+ {"EH", "alps", ARG_BOOL}, ++ {"ET", "signature-hashes", ARG_STRING}, ++ {"EU", "alps", ARG_BOOL}, + {"EI", "cert-compression", ARG_STRING}, + {"EJ", "tls-session-ticket", ARG_BOOL}, + {"EK", "http2-pseudo-headers-order", ARG_STRING}, @@ -3203,19 +3712,22 @@ index c9810e9d4..8bcd914db 100644 +#ifdef USE_ECH + {"ER", "ech", ARG_STRING}, +#endif ++ {"EV", "http2-streams", ARG_STRING}, ++ {"EW", "tls-extension-order", ARG_STRING}, ++ {"EX", "tls-grease", ARG_BOOL}, {"f", "fail", ARG_BOOL}, {"fa", "fail-early", ARG_BOOL}, {"fb", "styled-output", ARG_BOOL}, -@@ -1940,6 +1951,62 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ +@@ -2124,6 +2138,78 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ GetStr(&config->ssl_ec_curves, nextarg); break; -+ case 'G': ++ case 'T': + /* --signature-hashes */ + GetStr(&config->ssl_sig_hash_algs, nextarg); + break; + -+ case 'H': ++ case 'U': + /* --alps */ + config->alps = toggle; + break; @@ -3241,9 +3753,9 @@ index c9810e9d4..8bcd914db 100644 + break; + + case 'M': -+ /* --tls-permute-extensions */ -+ config->ssl_permute_extensions = toggle; -+ break; ++ /* --tls-permute-extensions */ ++ config->ssl_permute_extensions = toggle; ++ break; + + case 'N': + /* --http2-window-update */ @@ -3254,6 +3766,22 @@ index c9810e9d4..8bcd914db 100644 + return PARAM_BAD_NUMERIC; + break; + ++ case 'V': ++ /* --http2-streams */ ++ GetStr(&config->http2_streams, nextarg); ++ break; ++ ++ case 'W': ++ /* --tls-extension-order */ ++ GetStr(&config->tls_extension_order, nextarg); ++ // printf("setting is %s\n", config->tls_extension_order); ++ break; ++ ++ case 'X': ++ /* --tls-grease */ ++ config->tls_grease = toggle; ++ break; ++ +#ifdef USE_ECH + case 'R': + if(strlen(nextarg) != 6 || !strncasecompare("GREASE", nextarg, 6)) { @@ -3267,13 +3795,13 @@ index c9810e9d4..8bcd914db 100644 + break; +#endif default: /* unknown flag */ - return PARAM_OPTION_UNKNOWN; - } + err = PARAM_OPTION_UNKNOWN; + break; diff --git a/src/tool_listhelp.c b/src/tool_listhelp.c -index 61550de72..c3b99ab72 100644 +index 4e7a6dd63..8093b7f8e 100644 --- a/src/tool_listhelp.c +++ b/src/tool_listhelp.c -@@ -108,6 +108,27 @@ const struct helptxt helptext[] = { +@@ -111,6 +111,27 @@ const struct helptxt helptext[] = { {" --curves ", "(EC) TLS key exchange algorithm(s) to request", CURLHELP_TLS}, @@ -3301,7 +3829,7 @@ index 61550de72..c3b99ab72 100644 {"-d, --data ", "HTTP POST data", CURLHELP_IMPORTANT | CURLHELP_HTTP | CURLHELP_POST | CURLHELP_UPLOAD}, -@@ -165,6 +186,11 @@ const struct helptxt helptext[] = { +@@ -168,6 +189,11 @@ const struct helptxt helptext[] = { {"-D, --dump-header ", "Write the received headers to ", CURLHELP_HTTP | CURLHELP_FTP}, @@ -3313,7 +3841,7 @@ index 61550de72..c3b99ab72 100644 {" --egd-file ", "EGD socket path for random data", CURLHELP_TLS}, -@@ -387,6 +413,9 @@ const struct helptxt helptext[] = { +@@ -396,6 +422,9 @@ const struct helptxt helptext[] = { {" --no-alpn", "Disable the ALPN TLS extension", CURLHELP_TLS | CURLHELP_HTTP}, @@ -3324,10 +3852,10 @@ index 61550de72..c3b99ab72 100644 "Disable buffering of the output stream", CURLHELP_CURL}, diff --git a/src/tool_operate.c b/src/tool_operate.c -index ead7dca63..c3d1aecf5 100644 +index c805b7732..4f948b09c 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c -@@ -1493,6 +1493,22 @@ static CURLcode single_transfer(struct GlobalConfig *global, +@@ -1522,6 +1522,26 @@ static CURLcode single_transfer(struct GlobalConfig *global, return result; } @@ -3346,11 +3874,15 @@ index ead7dca63..c3d1aecf5 100644 + CURLOPT_HTTP2_WINDOW_UPDATE, + config->http2_window_update); + ++ if(config->http2_streams) ++ my_setopt_str(curl, ++ CURLOPT_HTTP2_STREAMS, ++ config->http2_streams); + } /* (proto_http) */ if(proto_ftp) -@@ -1581,6 +1597,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, +@@ -1610,6 +1630,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, if(config->ssl_ec_curves) my_setopt_str(curl, CURLOPT_SSL_EC_CURVES, config->ssl_ec_curves); @@ -3365,18 +3897,27 @@ index ead7dca63..c3d1aecf5 100644 if(config->writeout) my_setopt_str(curl, CURLOPT_CERTINFO, 1L); -@@ -1914,6 +1938,10 @@ static CURLcode single_transfer(struct GlobalConfig *global, +@@ -1942,6 +1970,19 @@ static CURLcode single_transfer(struct GlobalConfig *global, my_setopt_str(curl, CURLOPT_PROXY_TLS13_CIPHERS, config->proxy_cipher13_list); -+ /* curl-impersonate */ ++ /* curl-impersonate */ + if(config->ssl_permute_extensions) -+ my_setopt(curl, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1L); ++ my_setopt(curl, CURLOPT_SSL_PERMUTE_EXTENSIONS, 1L); ++ ++ /* curl-impersonate */ ++ if (config->tls_grease) ++ my_setopt(curl, CURLOPT_TLS_GREASE, 1L); ++ ++ /* curl-impersonate */ ++ if(config->tls_extension_order) ++ // printf("setting is %s\n", config->tls_extension_order); ++ my_setopt_str(curl, CURLOPT_TLS_EXTENSION_ORDER, config->tls_extension_order); + /* new in libcurl 7.9.2: */ if(config->disable_epsv) /* disable it */ -@@ -2123,6 +2151,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, +@@ -2151,6 +2192,14 @@ static CURLcode single_transfer(struct GlobalConfig *global, my_setopt(curl, CURLOPT_SSL_ENABLE_ALPN, 0L); } @@ -3391,7 +3932,7 @@ index ead7dca63..c3d1aecf5 100644 /* new in 7.40.0, abstract support added in 7.53.0 */ if(config->unix_socket_path) { if(config->abstract_unix_socket) { -@@ -2166,6 +2202,16 @@ static CURLcode single_transfer(struct GlobalConfig *global, +@@ -2199,6 +2248,16 @@ static CURLcode single_transfer(struct GlobalConfig *global, if(config->hsts) my_setopt_str(curl, CURLOPT_HSTS, config->hsts); @@ -3409,7 +3950,7 @@ index ead7dca63..c3d1aecf5 100644 per->retry_sleep_default = (config->retry_delay) ? config->retry_delay*1000L : RETRY_SLEEP_DEFAULT; /* ms */ diff --git a/src/tool_setopt.c b/src/tool_setopt.c -index 0f3cc83b9..1c14e9854 100644 +index de3b78fab..e034c9848 100644 --- a/src/tool_setopt.c +++ b/src/tool_setopt.c @@ -153,6 +153,8 @@ static const struct NameValue setopt_nv_CURLNONZERODEFAULTS[] = { diff --git a/configure b/configure index c24d4c0f..866e0a8c 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for curl-impersonate 0.5.0. +# Generated by GNU Autoconf 2.71 for curl-impersonate 0.7.0. # # Report bugs to . # @@ -610,8 +610,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='curl-impersonate' PACKAGE_TARNAME='curl-impersonate' -PACKAGE_VERSION='0.5.0' -PACKAGE_STRING='curl-impersonate 0.5.0' +PACKAGE_VERSION='0.7.0' +PACKAGE_STRING='curl-impersonate 0.7.0' PACKAGE_BUGREPORT='lwt@lwthiker.com' PACKAGE_URL='' @@ -622,6 +622,7 @@ cmake with_libnssckbi with_ca_path with_ca_bundle +with_zstd with_zlib static_build STRIP @@ -687,6 +688,7 @@ ac_user_opts=' enable_option_checking enable_static with_zlib +with_zstd with_ca_bundle with_ca_path with_libnssckbi @@ -1250,7 +1252,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures curl-impersonate 0.5.0 to adapt to many kinds of systems. +\`configure' configures curl-impersonate 0.7.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1317,7 +1319,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of curl-impersonate 0.5.0:";; + short | recursive ) echo "Configuration of curl-impersonate 0.7.0:";; esac cat <<\_ACEOF @@ -1332,6 +1334,7 @@ Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-zlib=PATH Search for zlib in PATH. Useful when cross compiling + --with-zstd=PATH Search for zstd in PATH. Useful when cross compiling --with-ca-bundle=FILE Path to be passed to curl's --with-ca-bundle configure option. Useful when cross compiling. Relevant only for the Chrome build. @@ -1422,7 +1425,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -curl-impersonate configure 0.5.0 +curl-impersonate configure 0.7.0 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -1584,7 +1587,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by curl-impersonate $as_me 0.5.0, which was +It was created by curl-impersonate $as_me 0.7.0, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -4180,6 +4183,74 @@ else $as_nop fi +# Let the user optionally specify the path to zstd, the same for zlib + +# Check whether --with-zstd was given. +if test ${with_zstd+y} +then : + withval=$with_zstd; with_zstd="$withval" +else $as_nop + with_zstd="check" +fi + + +if # User provided --without-zstd, which we don't support + test x"$with_zstd" = xno +then : + as_fn_error $? "building without zstd is not supported" "$LINENO" 5 +elif # User didn't provide --with-zstd at all, or provided --with-zstd without + # a path. Check if zstd can be linked against using the default linker flags. + test x"$with_zstd" = xcheck -o x"$with_zstd" = xyes +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZSTD_decompress in -lzstd" >&5 +printf %s "checking for ZSTD_decompress in -lzstd... " >&6; } +if test ${ac_cv_lib_zstd_ZSTD_decompress+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-lzstd $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +char ZSTD_decompress (); +int +main (void) +{ +return ZSTD_decompress (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_zstd_ZSTD_decompress=yes +else $as_nop + ac_cv_lib_zstd_ZSTD_decompress=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_decompress" >&5 +printf "%s\n" "$ac_cv_lib_zstd_ZSTD_decompress" >&6; } +if test "x$ac_cv_lib_zstd_ZSTD_decompress" = xyes +then : + with_zstd="" + +else $as_nop + as_fn_error $? "failed to find zstd" "$LINENO" 5 +fi + +else $as_nop + # User provided --with-zstd with a path. + with_zstd="$with_zstd" + +fi + # Path to CA certificates. # These options will be passed as-is to curl's configure script. # Useful when cross compiling, since curl's configure script doesn't know @@ -4859,7 +4930,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by curl-impersonate $as_me 0.5.0, which was +This file was extended by curl-impersonate $as_me 0.7.0, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -4914,7 +4985,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -curl-impersonate config.status 0.5.0 +curl-impersonate config.status 0.7.0 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 6fbfa8c6..e1b79b28 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([curl-impersonate], [0.5.4], [lwt@lwthiker.com]) +AC_INIT([curl-impersonate], [0.7.0], [lwt@lwthiker.com]) AC_CANONICAL_BUILD AC_CANONICAL_HOST @@ -36,6 +36,26 @@ AS_IF( # User provided --with-zlib with a path. [AC_SUBST([with_zlib], ["$with_zlib"])]) +# Let the user optionally specify the path to zstd, the same for zlib +AC_ARG_WITH([zstd], + [AS_HELP_STRING([--with-zstd=PATH], + [Search for zstd in PATH. Useful when cross compiling])], + [with_zstd="$withval"], + [with_zstd="check"]) + +AS_IF( + # User provided --without-zstd, which we don't support + [test x"$with_zstd" = xno], + [AC_MSG_ERROR(building without zstd is not supported)], + # User didn't provide --with-zstd at all, or provided --with-zstd without + # a path. Check if zstd can be linked against using the default linker flags. + [test x"$with_zstd" = xcheck -o x"$with_zstd" = xyes], + [AC_CHECK_LIB([zstd], [ZSTD_decompress], + [AC_SUBST([with_zstd], [""])], + [AC_MSG_ERROR(failed to find zstd)])], + # User provided --with-zstd with a path. + [AC_SUBST([with_zstd], ["$with_zstd"])]) + # Path to CA certificates. # These options will be passed as-is to curl's configure script. # Useful when cross compiling, since curl's configure script doesn't know diff --git a/chrome/Dockerfile.alpine b/docker/alpine.dockerfile similarity index 96% rename from chrome/Dockerfile.alpine rename to docker/alpine.dockerfile index db288357..ba33647a 100644 --- a/chrome/Dockerfile.alpine +++ b/docker/alpine.dockerfile @@ -39,9 +39,9 @@ RUN curl -L https://github.com/google/boringssl/archive/${BORING_SSL_COMMIT}.zip # Compile BoringSSL. # See https://boringssl.googlesource.com/boringssl/+/HEAD/BUILDING.md -COPY patches/boringssl-*.patch boringssl/ +COPY patches/boringssl.patch boringssl/ RUN cd boringssl && \ - for p in $(ls boringssl-*.patch); do patch -p1 < $p; done && \ + for p in $(ls boringssl.patch); do patch -p1 < $p; done && \ mkdir build && cd build && \ cmake \ -DCMAKE_C_FLAGS="-Wno-error=array-bounds -Wno-error=stringop-overflow" \ @@ -68,7 +68,7 @@ RUN cd ${NGHTTP2_VERSION} && \ make && make install # Download curl. -ARG CURL_VERSION=curl-8.1.1 +ARG CURL_VERSION=curl-8.5.0 RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz RUN tar xf ${CURL_VERSION}.tar.xz @@ -87,7 +87,7 @@ RUN cd ${CURL_VERSION} && \ --enable-websockets \ --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ + --with-zstd \ --enable-ech \ --with-openssl=/build/boringssl/build \ LIBS="-pthread" \ @@ -116,7 +116,7 @@ RUN cd ${CURL_VERSION} && \ ./configure --prefix=/build/install \ --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ + --with-zstd \ --enable-ech \ --with-openssl=/build/boringssl/build \ LIBS="-pthread" \ diff --git a/chrome/Dockerfile b/docker/debian.dockerfile similarity index 97% rename from chrome/Dockerfile rename to docker/debian.dockerfile index 89da312d..6feab202 100644 --- a/chrome/Dockerfile +++ b/docker/debian.dockerfile @@ -45,7 +45,7 @@ RUN curl -L https://github.com/google/boringssl/archive/${BORING_SSL_COMMIT}.zip # Compile BoringSSL. # See https://boringssl.googlesource.com/boringssl/+/HEAD/BUILDING.md -COPY patches/boringssl-*.patch boringssl/ +COPY patches/boringssl.patch boringssl/ RUN cd boringssl && \ for p in $(ls boringssl-*.patch); do patch -p1 < $p; done && \ mkdir build && cd build && \ @@ -74,7 +74,7 @@ RUN cd ${NGHTTP2_VERSION} && \ make && make install # Download curl. -ARG CURL_VERSION=curl-8.1.1 +ARG CURL_VERSION=curl-8.5.0 RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz RUN tar xf ${CURL_VERSION}.tar.xz @@ -93,7 +93,7 @@ RUN cd ${CURL_VERSION} && \ --enable-websockets \ --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ + --with-zstd \ --enable-ech \ --with-openssl=/build/boringssl/build \ LIBS="-pthread" \ @@ -122,7 +122,7 @@ RUN cd ${CURL_VERSION} && \ ./configure --prefix=/build/install \ --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ + --with-zstd \ --enable-ech \ --with-openssl=/build/boringssl/build \ LIBS="-pthread" \ diff --git a/Dockerfile.template b/docker/dockerfile.mustache similarity index 90% rename from Dockerfile.template rename to docker/dockerfile.mustache index 899b1203..d8eae9af 100644 --- a/Dockerfile.template +++ b/docker/dockerfile.mustache @@ -9,6 +9,7 @@ # Use it as a common base. FROM python:3.11-slim-bookworm as builder {{/debian}} + {{#alpine}} FROM alpine:3.18 as builder {{/alpine}} @@ -18,10 +19,10 @@ WORKDIR /build # Common dependencies {{#debian}} RUN apt-get update && \ - apt-get install -y git ninja-build cmake curl zlib1g-dev + apt-get install -y git ninja-build cmake curl zlib1g-dev zstd libzstd-dev {{/debian}} {{#alpine}} -RUN apk add git bash build-base make cmake ninja curl zlib-dev patch linux-headers python3 python3-dev +RUN apk add git bash build-base make cmake ninja curl zlib-dev patch linux-headers python3 python3-dev zstd zstd-dev {{/alpine}} # The following are needed because we are going to change some autoconf scripts, @@ -43,15 +44,13 @@ RUN apt-get install -y bzip2 RUN apt-get install -y xz-utils {{/debian}} -{{#chrome}} # Dependencies for downloading and building BoringSSL - {{#debian}} +{{#debian}} RUN apt-get install -y g++ golang-go unzip - {{/debian}} - {{#alpine}} +{{/debian}} +{{#alpine}} RUN apk add g++ go unzip - {{/alpine}} -{{/chrome}} +{{/alpine}} # Download and compile libbrotli ARG BROTLI_VERSION=1.0.9 @@ -62,7 +61,6 @@ RUN cd brotli-${BROTLI_VERSION} && \ cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed .. && \ cmake --build . --config Release --target install -{{#chrome}} # BoringSSL doesn't have versions. Choose a commit that is used in a stable # Chromium version. ARG BORING_SSL_COMMIT=d24a38200fef19150eef00cad35b138936c08767 @@ -72,9 +70,9 @@ RUN curl -L https://github.com/google/boringssl/archive/${BORING_SSL_COMMIT}.zip # Compile BoringSSL. # See https://boringssl.googlesource.com/boringssl/+/HEAD/BUILDING.md -COPY patches/boringssl-*.patch boringssl/ +COPY patches/boringssl.patch boringssl/ RUN cd boringssl && \ - for p in $(ls boringssl-*.patch); do patch -p1 < $p; done && \ + for p in $(ls boringssl.patch); do patch -p1 < $p; done && \ mkdir build && cd build && \ cmake \ -DCMAKE_C_FLAGS="-Wno-error=array-bounds -Wno-error=stringop-overflow" \ @@ -87,7 +85,6 @@ RUN mkdir boringssl/build/lib && \ ln -s ../crypto/libcrypto.a boringssl/build/lib/libcrypto.a && \ ln -s ../ssl/libssl.a boringssl/build/lib/libssl.a && \ cp -R boringssl/include boringssl/build -{{/chrome}} ARG NGHTTP2_VERSION=nghttp2-1.56.0 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.56.0/nghttp2-1.56.0.tar.bz2 @@ -102,7 +99,7 @@ RUN cd ${NGHTTP2_VERSION} && \ make && make install # Download curl. -ARG CURL_VERSION=curl-8.1.1 +ARG CURL_VERSION=curl-8.5.0 RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz RUN tar xf ${CURL_VERSION}.tar.xz @@ -112,7 +109,7 @@ RUN cd ${CURL_VERSION} && \ for p in $(ls curl-*.patch); do patch -p1 < $p; done && \ autoreconf -fi -# Compile curl with nghttp2, libbrotli and boringssl (chrome). +# Compile curl with nghttp2, libbrotli and boringssl. # Enable keylogfile for debugging of TLS traffic. RUN cd ${CURL_VERSION} && \ ./configure --prefix=/build/install \ @@ -121,21 +118,17 @@ RUN cd ${CURL_VERSION} && \ --enable-websockets \ --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ -{{#chrome}} + --with-zstd \ --enable-ech \ --with-openssl=/build/boringssl/build \ LIBS="-pthread" \ CFLAGS="-I/build/boringssl/build" \ -{{/chrome}} USE_CURL_SSLKEYLOGFILE=true && \ make && make install RUN mkdir out && \ -{{#chrome}} cp /build/install/bin/curl-impersonate-chrome out/ && \ ln -s curl-impersonate-chrome out/curl-impersonate && \ -{{/chrome}} strip out/curl-impersonate # Verify that the resulting 'curl' has all the necessary features. @@ -154,24 +147,20 @@ RUN cd ${CURL_VERSION} && \ ./configure --prefix=/build/install \ --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ -{{#chrome}} + --with-zstd \ --enable-ech \ --with-openssl=/build/boringssl/build \ LIBS="-pthread" \ CFLAGS="-I/build/boringssl/build" \ -{{/chrome}} USE_CURL_SSLKEYLOGFILE=true && \ make clean && make && make install # Copy libcurl-impersonate and symbolic links RUN cp -d /build/install/lib/libcurl-impersonate* /build/out -{{#chrome}} RUN ver=$(readlink -f ${CURL_VERSION}/lib/.libs/libcurl-impersonate-chrome.so | sed 's/.*so\.//') && \ major=$(echo -n $ver | cut -d'.' -f1) && \ ln -s "libcurl-impersonate-chrome.so.$ver" "out/libcurl-impersonate.so.$ver" && \ -{{/chrome}} ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so" && \ strip "out/libcurl-impersonate.so.$ver" @@ -180,17 +169,14 @@ RUN ver=$(readlink -f ${CURL_VERSION}/lib/.libs/libcurl-impersonate-chrome.so | RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypto) # Wrapper scripts -{{#chrome}} COPY curl_chrome* curl_edge* curl_safari* out/ -{{/chrome}} {{#alpine}} # Replace /usr/bin/env bash with /usr/bin/env ash RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_* {{/alpine}} RUN chmod +x out/curl_* -# Create a final, minimal image with the compiled binaries -# only. +# Create a final, minimal image with the compiled binaries only. {{#alpine}} FROM alpine:3.18 {{/alpine}} diff --git a/docker/generate.sh b/docker/generate.sh new file mode 100755 index 00000000..d9bd415f --- /dev/null +++ b/docker/generate.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +cat < docker/debian.dockerfile +--- +debian: true +--- +EOF + +cat < chrome/alpine.dockerfile +--- +alpine: true +--- +EOF diff --git a/docs/01_GETTING_STARTED.md b/docs/01_GETTING_STARTED.md index 5d151038..e66e5923 100644 --- a/docs/01_GETTING_STARTED.md +++ b/docs/01_GETTING_STARTED.md @@ -1,10 +1,8 @@ # Getting started with curl-impersonate -curl-impersonate can be run on Linux and macOS. Partial support for Windows is currently available through a different project: [curl-impersonate-win](https://github.com/depler/curl-impersonate-win). +curl-impersonate can be run on Linux and macOS. Windows is supported via mingw. ## Installation -Installation instructions are available on the [main page](https://github.com/lwthiker/curl-impersonate#installation) +Installation instructions are available on the [main page](https://github.com/yifeikong/curl-impersonate#installation) -The project supplies two modified flavors of curl: -* The *Chrome version* is a modified curl binary and libcurl library that can impersonate Chrome, Edge and Safari. It uses BoringSSL, Chrome's TLS library. It is based on a patched curl version with added support for some additional TLS extensions and modified HTTP/2 settings that make it look like a browser. -* The *Firefox version* is a modified curl binary that can impersonate Firefox. It uses NSS, Mozilla's TLS library which is used by Firefox. +The project supplies a modified curl binary and libcurl library that can impersonate Chrome, Edge and Safari. It uses BoringSSL, Chrome's TLS library. It is based on a patched curl version with added support for some additional TLS extensions and modified HTTP/2 settings that make it look like a browser. diff --git a/docs/02_USAGE.md b/docs/02_USAGE.md index e59c73ec..15519f43 100644 --- a/docs/02_USAGE.md +++ b/docs/02_USAGE.md @@ -1,26 +1,19 @@ # Running curl-impersonate from the command line curl-impersonate can be run from the command line just like the regular curl tool. -Since it is just a modified curl build, all the original flags and command line options are supported. +Since it is just a modified curl build, all the original flags and command-line options are supported. -For example, the Firefox version can be run as follows: -```bash -curl-impersonate-ff -v -L https://wikipedia.org -``` +For example, it can be run as follows: -and the Chrome version: -```bash -curl-impersonate-chrome -v -L https://wikipedia.org -``` + curl-impersonate-chrome -v -L https://wikipedia.org However, by default, running the binaries as above will not prdouce the same TLS and HTTP/2 signatures as the impersonated browsers. Rather, this project provides additional *wrapper scripts* that launch these binaries with the correct set of command line flags to produce the desired signatures. For example: -```bash -curl_chrome104 -v -L https://wikipedia.org -``` + + curl_chrome104 -v -L https://wikipedia.org will produce a signature identical to Chrome version 104. You can add command line flags and they will be passed on to curl. However, some flags change curl's TLS signature. See below for more details. -The full list of wrapper scripts is available on the [main page](https://github.com/lwthiker/curl-impersonate#supported-browsers). +The full list of wrapper scripts is available on the [main page](https://github.com/yifeikong/curl-impersonate#supported-browsers). ## Changing the HTTP headers The wrapper scripts use a certain set of HTTP headers such as `User-Agent`, `Accept-Encoding` and a few more. diff --git a/docs/03_LIBCURL_IMPERSONATE.md b/docs/03_LIBCURL_IMPERSONATE.md index ca202302..1a8e9586 100644 --- a/docs/03_LIBCURL_IMPERSONATE.md +++ b/docs/03_LIBCURL_IMPERSONATE.md @@ -1,3 +1,3 @@ # Using libcurl-impersonate -Documentation for using libcurl-impersonate is currently on the [main page](https://github.com/lwthiker/curl-impersonate#libcurl-impersonate) +Documentation for using libcurl-impersonate is currently on the [main page](https://github.com/yifeikong/curl-impersonate#libcurl-impersonate) diff --git a/docs/03_LIBCURL_IMPERSONATE_PYTHON.md b/docs/03_LIBCURL_IMPERSONATE_PYTHON.md index 599584ff..d0c0de36 100644 --- a/docs/03_LIBCURL_IMPERSONATE_PYTHON.md +++ b/docs/03_LIBCURL_IMPERSONATE_PYTHON.md @@ -1,4 +1,4 @@ # Using libcurl-impersonate in Python scripts -You can use the Python binding [curl_cffi](https://github.com:yifeikong/curl_cffi), which works on Linux(x86_64/aarch64), macOS(Intel) and Windows(amd64). +You can use the Python binding [curl_cffi](https://github.com:yifeikong/curl_cffi), which works on Linux, macOS and Windows. diff --git a/docs/README.md b/docs/README.md index 565c723c..85923d37 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,7 +1,7 @@ # curl-impersonate documentation curl-impersonate is a curl build that lets you send HTTP requests that look like a browser's. -curl-impersonate can impersonate recent versions of Chrome, Edge, Safari & Firefox. +curl-impersonate can impersonate recent versions of Chrome, Edge, Safari & Firefox(In progress). curl-impersonate can be used either as a command line tool, similar to the regular curl, or as a library that can be integrated instead of the regular libcurl. These docs describe the various usage options. @@ -13,3 +13,4 @@ These docs describe the various usage options. 3. [Using libcurl-impersonate](03_LIBCURL_IMPERSONATE.md) 1. [In PHP scripts](03_LIBCURL_IMPERSONATE_PHP.md) 2. [In JS scripts](03_LIBCURL_IMPERSONATE_JS.md) + 3. [In Python scripts](03_LIBCURL_IMPERSONATE_PYTHON.md) diff --git a/firefox/Dockerfile b/firefox/Dockerfile deleted file mode 100644 index 87377c41..00000000 --- a/firefox/Dockerfile +++ /dev/null @@ -1,159 +0,0 @@ -# -# NOTE: THIS DOCKERFILE IS GENERATED FROM "Dockerfile.template" VIA -# `./generate_dockerfiles.sh` -# -# PLEASE DO NOT EDIT IT DIRECTLY. -# - -# Python is needed for building libnss. -# Use it as a common base. -FROM python:3.11-slim-bookworm as builder - -WORKDIR /build - -# Common dependencies -RUN apt-get update && \ - apt-get install -y git ninja-build cmake curl zlib1g-dev - -# The following are needed because we are going to change some autoconf scripts, -# both for libnghttp2 and curl. -RUN apt-get install -y autoconf automake autotools-dev pkg-config libtool git - -# Dependencies for downloading and building nghttp2 -RUN apt-get install -y bzip2 - -# Dependencies for downloading and building curl -RUN apt-get install -y xz-utils - -# Dependencies for building libnss -# See https://firefox-source-docs.mozilla.org/security/nss/build.html#mozilla-projects-nss-building -RUN apt-get install -y mercurial python3-pip - -# curl tries to load the CA certificates for libnss. -# It loads them from /usr/lib/x86_64-linux-gnu/nss/libnssckbi.so, -# which is supplied by libnss3 on Debian/Ubuntu -RUN apt-get install -y libnss3 - -# Download and compile libbrotli -ARG BROTLI_VERSION=1.0.9 -RUN curl -L https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz -o brotli-${BROTLI_VERSION}.tar.gz && \ - tar xf brotli-${BROTLI_VERSION}.tar.gz -RUN cd brotli-${BROTLI_VERSION} && \ - mkdir build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed .. && \ - cmake --build . --config Release --target install - -# Needed for building libnss -RUN pip install gyp-next - -ARG NSS_VERSION=nss-3.92 -# This tarball is already bundled with nspr, a dependency of libnss. -ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_92_RTM/src/nss-3.92-with-nspr-4.35.tar.gz - -# Download and compile nss. -RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} -RUN tar xf ${NSS_VERSION}.tar.gz && \ - cd ${NSS_VERSION}/nss && \ - ./build.sh -o --disable-tests --static --python=python3 - -ARG NGHTTP2_VERSION=nghttp2-1.56.0 -ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.56.0/nghttp2-1.56.0.tar.bz2 - -# Download nghttp2 for HTTP/2.0 support. -RUN curl -o ${NGHTTP2_VERSION}.tar.bz2 -L ${NGHTTP2_URL} -RUN tar xf ${NGHTTP2_VERSION}.tar.bz2 - -# Compile nghttp2 -RUN cd ${NGHTTP2_VERSION} && \ - ./configure --prefix=/build/${NGHTTP2_VERSION}/installed --with-pic --disable-shared && \ - make && make install - -# Download curl. -ARG CURL_VERSION=curl-8.1.1 -RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz -RUN tar xf ${CURL_VERSION}.tar.xz - -# Patch curl and re-generate the configure script -COPY patches/curl-*.patch ${CURL_VERSION}/ -RUN cd ${CURL_VERSION} && \ - for p in $(ls curl-*.patch); do patch -p1 < $p; done && \ - autoreconf -fi - -# Compile curl with nghttp2, libbrotli and nss (firefox) or boringssl (chrome). -# Enable keylogfile for debugging of TLS traffic. -RUN cd ${CURL_VERSION} && \ - ./configure --prefix=/build/install \ - --enable-static \ - --disable-shared \ - --enable-websockets \ - --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ - --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ - --with-nss=/build/${NSS_VERSION}/dist/Release \ - --with-nss-deprecated \ - CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" \ - USE_CURL_SSLKEYLOGFILE=true && \ - make && make install - -RUN mkdir out && \ - cp /build/install/bin/curl-impersonate-ff out/ && \ - ln -s curl-impersonate-ff out/curl-impersonate && \ - strip out/curl-impersonate - -# Verify that the resulting 'curl' has all the necessary features. -RUN ./out/curl-impersonate -V | grep -q zlib && \ - ./out/curl-impersonate -V | grep -q brotli && \ - ./out/curl-impersonate -V | grep -q nghttp2 && \ - ./out/curl-impersonate -V | grep -q -e NSS -e BoringSSL - -# Verify that the resulting 'curl' is really statically compiled -RUN ! (ldd ./out/curl-impersonate | grep -q -e libcurl -e nghttp2 -e brotli -e ssl -e crypto) - -RUN rm -Rf /build/install - -# Re-compile libcurl dynamically -RUN cd ${CURL_VERSION} && \ - ./configure --prefix=/build/install \ - --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ - --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ - --with-nss=/build/${NSS_VERSION}/dist/Release \ - --with-nss-deprecated \ - CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" \ - USE_CURL_SSLKEYLOGFILE=true && \ - make clean && make && make install - -# Copy libcurl-impersonate and symbolic links -RUN cp -d /build/install/lib/libcurl-impersonate* /build/out - -RUN ver=$(readlink -f ${CURL_VERSION}/lib/.libs/libcurl-impersonate-ff.so | sed 's/.*so\.//') && \ - major=$(echo -n $ver | cut -d'.' -f1) && \ - ln -s "libcurl-impersonate-ff.so.$ver" "out/libcurl-impersonate.so.$ver" && \ - ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so" && \ - strip "out/libcurl-impersonate.so.$ver" - -# Verify that the resulting 'libcurl' is really statically compiled against its -# dependencies. -RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypto) - -# Wrapper scripts -COPY curl_ff* out/ -RUN chmod +x out/curl_* - -# Create a final, minimal image with the compiled binaries -# only. -FROM debian:bookworm-slim -RUN apt-get update && apt-get install -y ca-certificates \ -# curl tries to load the CA certificates for libnss. -# It loads them from /usr/lib/libnssckbi.so and /usr/lib/libnsspem.so, -# which are supplied by 'libnss3' and 'nss-plugin-pem' on debian. - libnss3 nss-plugin-pem \ - && rm -rf /var/lib/apt/lists/* -# Copy curl-impersonate from the builder image -COPY --from=builder /build/install /usr/local -# Update the loader's cache -RUN ldconfig -# Copy to /build/out as well for backward compatibility with previous versions. -COPY --from=builder /build/out /build/out -# Wrapper scripts -COPY --from=builder /build/out/curl_* /usr/local/bin/ diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine deleted file mode 100644 index c702d150..00000000 --- a/firefox/Dockerfile.alpine +++ /dev/null @@ -1,144 +0,0 @@ -# -# NOTE: THIS DOCKERFILE IS GENERATED FROM "Dockerfile.template" VIA -# `./generate_dockerfiles.sh` -# -# PLEASE DO NOT EDIT IT DIRECTLY. -# - -FROM alpine:3.18 as builder - -WORKDIR /build - -# Common dependencies -RUN apk add git bash build-base make cmake ninja curl zlib-dev patch linux-headers python3 python3-dev - -# The following are needed because we are going to change some autoconf scripts, -# both for libnghttp2 and curl. -RUN apk add autoconf automake pkgconfig libtool - - - -# Dependencies for building libnss -# See https://firefox-source-docs.mozilla.org/security/nss/build.html#mozilla-projects-nss-building -RUN apk add mercurial py3-pip clang-analyzer - - -# Download and compile libbrotli -ARG BROTLI_VERSION=1.0.9 -RUN curl -L https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz -o brotli-${BROTLI_VERSION}.tar.gz && \ - tar xf brotli-${BROTLI_VERSION}.tar.gz -RUN cd brotli-${BROTLI_VERSION} && \ - mkdir build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./installed .. && \ - cmake --build . --config Release --target install - -# Needed for building libnss -RUN pip install gyp-next - -ARG NSS_VERSION=nss-3.92 -# This tarball is already bundled with nspr, a dependency of libnss. -ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_92_RTM/src/nss-3.92-with-nspr-4.35.tar.gz - -# Download and compile nss. -RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} -RUN tar xf ${NSS_VERSION}.tar.gz && \ - cd ${NSS_VERSION}/nss && \ - ./build.sh -o --disable-tests --static --python=python3 - -ARG NGHTTP2_VERSION=nghttp2-1.56.0 -ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.56.0/nghttp2-1.56.0.tar.bz2 - -# Download nghttp2 for HTTP/2.0 support. -RUN curl -o ${NGHTTP2_VERSION}.tar.bz2 -L ${NGHTTP2_URL} -RUN tar xf ${NGHTTP2_VERSION}.tar.bz2 - -# Compile nghttp2 -RUN cd ${NGHTTP2_VERSION} && \ - ./configure --prefix=/build/${NGHTTP2_VERSION}/installed --with-pic --disable-shared && \ - make && make install - -# Download curl. -ARG CURL_VERSION=curl-8.1.1 -RUN curl -o ${CURL_VERSION}.tar.xz https://curl.se/download/${CURL_VERSION}.tar.xz -RUN tar xf ${CURL_VERSION}.tar.xz - -# Patch curl and re-generate the configure script -COPY patches/curl-*.patch ${CURL_VERSION}/ -RUN cd ${CURL_VERSION} && \ - for p in $(ls curl-*.patch); do patch -p1 < $p; done && \ - autoreconf -fi - -# Compile curl with nghttp2, libbrotli and nss (firefox) or boringssl (chrome). -# Enable keylogfile for debugging of TLS traffic. -RUN cd ${CURL_VERSION} && \ - ./configure --prefix=/build/install \ - --enable-static \ - --disable-shared \ - --enable-websockets \ - --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ - --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ - --with-nss=/build/${NSS_VERSION}/dist/Release \ - --with-nss-deprecated \ - CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" \ - USE_CURL_SSLKEYLOGFILE=true && \ - make && make install - -RUN mkdir out && \ - cp /build/install/bin/curl-impersonate-ff out/ && \ - ln -s curl-impersonate-ff out/curl-impersonate && \ - strip out/curl-impersonate - -# Verify that the resulting 'curl' has all the necessary features. -RUN ./out/curl-impersonate -V | grep -q zlib && \ - ./out/curl-impersonate -V | grep -q brotli && \ - ./out/curl-impersonate -V | grep -q nghttp2 && \ - ./out/curl-impersonate -V | grep -q -e NSS -e BoringSSL - -# Verify that the resulting 'curl' is really statically compiled -RUN ! (ldd ./out/curl-impersonate | grep -q -e libcurl -e nghttp2 -e brotli -e ssl -e crypto) - -RUN rm -Rf /build/install - -# Re-compile libcurl dynamically -RUN cd ${CURL_VERSION} && \ - ./configure --prefix=/build/install \ - --with-nghttp2=/build/${NGHTTP2_VERSION}/installed \ - --with-brotli=/build/brotli-${BROTLI_VERSION}/build/installed \ - --without-zstd \ - --with-nss=/build/${NSS_VERSION}/dist/Release \ - --with-nss-deprecated \ - CFLAGS="-I/build/${NSS_VERSION}/dist/public/nss -I/build/${NSS_VERSION}/dist/Release/include/nspr" \ - USE_CURL_SSLKEYLOGFILE=true && \ - make clean && make && make install - -# Copy libcurl-impersonate and symbolic links -RUN cp -d /build/install/lib/libcurl-impersonate* /build/out - -RUN ver=$(readlink -f ${CURL_VERSION}/lib/.libs/libcurl-impersonate-ff.so | sed 's/.*so\.//') && \ - major=$(echo -n $ver | cut -d'.' -f1) && \ - ln -s "libcurl-impersonate-ff.so.$ver" "out/libcurl-impersonate.so.$ver" && \ - ln -s "libcurl-impersonate.so.$ver" "out/libcurl-impersonate.so" && \ - strip "out/libcurl-impersonate.so.$ver" - -# Verify that the resulting 'libcurl' is really statically compiled against its -# dependencies. -RUN ! (ldd ./out/curl-impersonate | grep -q -e nghttp2 -e brotli -e ssl -e crypto) - -# Wrapper scripts -COPY curl_ff* out/ -# Replace /usr/bin/env bash with /usr/bin/env ash -RUN sed -i 's@/usr/bin/env bash@/usr/bin/env ash@' out/curl_* -RUN chmod +x out/curl_* - -# Create a final, minimal image with the compiled binaries -# only. -FROM alpine:3.18 -# curl tries to load the CA certificates for libnss. -# It loads them from /usr/lib/libnssckbi.so, -# which is supplied by 'nss' on alpine. -RUN apk add --no-cache nss -# Copy curl-impersonate from the builder image -COPY --from=builder /build/install /usr/local -# Wrapper scripts -COPY --from=builder /build/out/curl_* /usr/local/bin/ diff --git a/firefox/curl_ff100 b/firefox/curl_ff100 deleted file mode 100755 index 0ebff7b1..00000000 --- a/firefox/curl_ff100 +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/curl_ff102 b/firefox/curl_ff102 deleted file mode 100755 index af465037..00000000 --- a/firefox/curl_ff102 +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/curl_ff109 b/firefox/curl_ff109 deleted file mode 100755 index 3f782abc..00000000 --- a/firefox/curl_ff109 +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/curl_ff117 b/firefox/curl_ff117 deleted file mode 100755 index 91c3e6a4..00000000 --- a/firefox/curl_ff117 +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/curl_ff91esr b/firefox/curl_ff91esr deleted file mode 100755 index ec0ba473..00000000 --- a/firefox/curl_ff91esr +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha,rsa_3des_ede_cbc_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/curl_ff95 b/firefox/curl_ff95 deleted file mode 100755 index e9c94897..00000000 --- a/firefox/curl_ff95 +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/curl_ff98 b/firefox/curl_ff98 deleted file mode 100755 index c8fcc686..00000000 --- a/firefox/curl_ff98 +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# Find the directory of this script -dir=${0%/*} - -# The list of ciphers can be obtained by looking at the Client Hello message in -# Wireshark, then converting it using the cipherlist array at -# https://github.com/curl/curl/blob/master/lib/vtls/nss.c -"$dir/curl-impersonate-ff" \ - --ciphers aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha \ - -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \ - -H 'Accept-Language: en-US,en;q=0.5' \ - -H 'Accept-Encoding: gzip, deflate, br' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'Sec-Fetch-Dest: document' \ - -H 'Sec-Fetch-Mode: navigate' \ - -H 'Sec-Fetch-Site: none' \ - -H 'Sec-Fetch-User: ?1' \ - -H 'TE: Trailers' \ - --http2 --compressed \ - "$@" diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch deleted file mode 100644 index c0a71d04..00000000 --- a/firefox/patches/curl-impersonate.patch +++ /dev/null @@ -1,1602 +0,0 @@ -diff --git a/Makefile.am b/Makefile.am -index f25e4e2f0..9a8026915 100644 ---- a/Makefile.am -+++ b/Makefile.am -@@ -156,13 +156,13 @@ CLEANFILES = $(VC10_LIBVCXPROJ) $(VC10_SRCVCXPROJ) $(VC11_LIBVCXPROJ) \ - $(VC14_SRCVCXPROJ) $(VC14_10_LIBVCXPROJ) $(VC14_10_SRCVCXPROJ) \ - $(VC14_30_LIBVCXPROJ) $(VC14_30_SRCVCXPROJ) - --bin_SCRIPTS = curl-config -+bin_SCRIPTS = curl-impersonate-ff-config - - SUBDIRS = lib src - DIST_SUBDIRS = $(SUBDIRS) tests packages scripts include docs - - pkgconfigdir = $(libdir)/pkgconfig --pkgconfig_DATA = libcurl.pc -+pkgconfig_DATA = libcurl-impersonate-ff.pc - - # List of files required to generate VC IDE .dsp, .vcproj and .vcxproj files - include lib/Makefile.inc -diff --git a/configure.ac b/configure.ac -index 75a882b12..9310059b9 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -1493,7 +1493,8 @@ if test X"$OPT_BROTLI" != Xno; then - - dnl if given with a prefix, we set -L and -I based on that - if test -n "$PREFIX_BROTLI"; then -- LIB_BROTLI="-lbrotlidec" -+ # curl-impersonate: Use static libbrotli -+ LIB_BROTLI="-lbrotlidec-static -lbrotlicommon-static" - LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff - CPP_BROTLI=-I${PREFIX_BROTLI}/include - DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff -@@ -1503,7 +1504,11 @@ if test X"$OPT_BROTLI" != Xno; then - CPPFLAGS="$CPPFLAGS $CPP_BROTLI" - LIBS="$LIB_BROTLI $LIBS" - -- AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress) -+ AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress, -+ # curl-impersonate: Define 'action-if-found' explicitly to prevent -+ # -lbrotlidec from being added to LIBS (already added before) -+ AC_DEFINE(HAVE_LIBBROTLI, 1, [Define to 1 if libbrotli exists]) -+ ) - - AC_CHECK_HEADERS(brotli/decode.h, - curl_brotli_msg="enabled (libbrotlidec)" -@@ -4706,8 +4711,8 @@ AC_CONFIG_FILES([Makefile \ - tests/http/clients/Makefile \ - packages/Makefile \ - packages/vms/Makefile \ -- curl-config \ -- libcurl.pc -+ curl-impersonate-ff-config:curl-config.in \ -+ libcurl-impersonate-ff.pc:libcurl.pc.in - ]) - AC_OUTPUT - -diff --git a/curl-config.in b/curl-config.in -index 54f92d931..531bec977 100644 ---- a/curl-config.in -+++ b/curl-config.in -@@ -163,9 +163,9 @@ while test $# -gt 0; do - CURLLIBDIR="" - fi - if test "X@ENABLE_SHARED@" = "Xno"; then -- echo ${CURLLIBDIR}-lcurl @LIBCURL_LIBS@ -+ echo ${CURLLIBDIR}-lcurl-impersonate-ff @LIBCURL_LIBS@ - else -- echo ${CURLLIBDIR}-lcurl -+ echo ${CURLLIBDIR}-lcurl-impersonate-ff - fi - ;; - --ssl-backends) -@@ -174,7 +174,7 @@ while test $# -gt 0; do - - --static-libs) - if test "X@ENABLE_STATIC@" != "Xno" ; then -- echo "@libdir@/libcurl.@libext@" @LDFLAGS@ @LIBCURL_LIBS@ -+ echo "@libdir@/libcurl-impersonate-ff.@libext@" @LDFLAGS@ @LIBCURL_LIBS@ - else - echo "curl was built with static libraries disabled" >&2 - exit 1 -diff --git a/include/curl/curl.h b/include/curl/curl.h -index 944352421..07d08a0bb 100644 ---- a/include/curl/curl.h -+++ b/include/curl/curl.h -@@ -2207,6 +2207,10 @@ typedef enum { - /* Can leak things, gonna exit() soon */ - CURLOPT(CURLOPT_QUICK_EXIT, CURLOPTTYPE_LONG, 322), - -+ /* curl-impersonate: A list of headers used by the impersonated browser. -+ * If given, merged with CURLOPT_HTTPHEADER. */ -+ CURLOPT(CURLOPT_HTTPBASEHEADER, CURLOPTTYPE_SLISTPOINT, 323), -+ - CURLOPT_LASTENTRY /* the last unused */ - } CURLoption; - -diff --git a/include/curl/easy.h b/include/curl/easy.h -index 1285101c5..c620065dc 100644 ---- a/include/curl/easy.h -+++ b/include/curl/easy.h -@@ -43,6 +43,16 @@ CURL_EXTERN CURLcode curl_easy_setopt(CURL *curl, CURLoption option, ...); - CURL_EXTERN CURLcode curl_easy_perform(CURL *curl); - CURL_EXTERN void curl_easy_cleanup(CURL *curl); - -+/* -+ * curl-impersonate: Tell libcurl to impersonate a browser. -+ * This is a wrapper function that calls curl_easy_setopt() -+ * multiple times with all the parameters required. That's also why it was -+ * created as a separate API function and not just as another option to -+ * curl_easy_setopt(). -+ */ -+CURL_EXTERN CURLcode curl_easy_impersonate(CURL *curl, const char *target, -+ int default_headers); -+ - /* - * NAME curl_easy_getinfo() - * -diff --git a/lib/Makefile.am b/lib/Makefile.am -index 3c0a70912..e4a77a0e8 100644 ---- a/lib/Makefile.am -+++ b/lib/Makefile.am -@@ -31,7 +31,7 @@ EXTRA_DIST = Makefile.mk config-win32.h config-win32ce.h config-plan9.h \ - config-os400.h setup-os400.h $(CMAKE_DIST) setup-win32.h .checksrc \ - Makefile.soname - --lib_LTLIBRARIES = libcurl.la -+lib_LTLIBRARIES = libcurl-impersonate-ff.la - - if BUILD_UNITTESTS - noinst_LTLIBRARIES = libcurlu.la -@@ -67,51 +67,51 @@ AM_CFLAGS = - # Makefile.inc provides the CSOURCES and HHEADERS defines - include Makefile.inc - --libcurl_la_SOURCES = $(CSOURCES) $(HHEADERS) --libcurlu_la_SOURCES = $(CSOURCES) $(HHEADERS) -+libcurl_impersonate_ff_la_SOURCES = $(CSOURCES) $(HHEADERS) -+libcurlu_impersonate_ff_la_SOURCES = $(CSOURCES) $(HHEADERS) - --libcurl_la_CPPFLAGS_EXTRA = --libcurl_la_LDFLAGS_EXTRA = --libcurl_la_CFLAGS_EXTRA = -+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA = -+libcurl_impersonate_ff_la_LDFLAGS_EXTRA = -+libcurl_impersonate_ff_la_CFLAGS_EXTRA = - - if CURL_LT_SHLIB_USE_VERSION_INFO --libcurl_la_LDFLAGS_EXTRA += $(VERSIONINFO) -+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += $(VERSIONINFO) - endif - - if CURL_LT_SHLIB_USE_NO_UNDEFINED --libcurl_la_LDFLAGS_EXTRA += -no-undefined -+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -no-undefined - endif - - if CURL_LT_SHLIB_USE_MIMPURE_TEXT --libcurl_la_LDFLAGS_EXTRA += -mimpure-text -+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -mimpure-text - endif - - if CURL_LT_SHLIB_USE_VERSIONED_SYMBOLS --libcurl_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers -+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -Wl,--version-script=libcurl.vers - else - # if symbol-hiding is enabled, hide them! - if DOING_CURL_SYMBOL_HIDING --libcurl_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*' -+libcurl_impersonate_ff_la_LDFLAGS_EXTRA += -export-symbols-regex '^curl_.*' - endif - endif - - if USE_CPPFLAG_CURL_STATICLIB --libcurl_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB -+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_STATICLIB - else - if HAVE_WINDRES --libcurl_la_SOURCES += $(LIB_RCFILES) -+libcurl_impersonate_ff_la_SOURCES += $(LIB_RCFILES) - $(LIB_RCFILES): $(top_srcdir)/include/curl/curlver.h - endif - endif - - if DOING_CURL_SYMBOL_HIDING --libcurl_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS --libcurl_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING) -+libcurl_impersonate_ff_la_CPPFLAGS_EXTRA += -DCURL_HIDDEN_SYMBOLS -+libcurl_impersonate_ff_la_CFLAGS_EXTRA += $(CFLAG_CURL_SYMBOL_HIDING) - endif - --libcurl_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_la_CPPFLAGS_EXTRA) --libcurl_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) --libcurl_la_CFLAGS = $(AM_CFLAGS) $(libcurl_la_CFLAGS_EXTRA) -+libcurl_impersonate_ff_la_CPPFLAGS = $(AM_CPPFLAGS) $(libcurl_impersonate_ff_la_CPPFLAGS_EXTRA) -+libcurl_impersonate_ff_la_LDFLAGS = $(AM_LDFLAGS) $(libcurl_impersonate_ff_la_LDFLAGS_EXTRA) $(LDFLAGS) $(LIBCURL_LIBS) -+libcurl_impersonate_ff_la_CFLAGS = $(AM_CFLAGS) $(libcurl_impersonate_ff_la_CFLAGS_EXTRA) - - libcurlu_la_CPPFLAGS = $(AM_CPPFLAGS) -DCURL_STATICLIB -DUNITTESTS - libcurlu_la_LDFLAGS = $(AM_LDFLAGS) -static $(LIBCURL_LIBS) -diff --git a/lib/Makefile.inc b/lib/Makefile.inc -index f815170a7..9d9417edc 100644 ---- a/lib/Makefile.inc -+++ b/lib/Makefile.inc -@@ -174,6 +174,7 @@ LIB_CFILES = \ - idn.c \ - if2ip.c \ - imap.c \ -+ impersonate.c \ - inet_ntop.c \ - inet_pton.c \ - krb5.c \ -diff --git a/lib/dynhds.c b/lib/dynhds.c -index b325e0060..4c8a73bab 100644 ---- a/lib/dynhds.c -+++ b/lib/dynhds.c -@@ -52,6 +52,8 @@ entry_new(const char *name, size_t namelen, - e->valuelen = valuelen; - if(opts & DYNHDS_OPT_LOWERCASE) - Curl_strntolower(e->name, e->name, e->namelen); -+ if(opts & DYNHDS_OPT_LOWERCASE_VAL) -+ Curl_strntolower(e->value, e->value, e->valuelen); - return e; - } - -@@ -134,6 +136,16 @@ void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts) - dynhds->opts = opts; - } - -+void Curl_dynhds_set_opt(struct dynhds *dynhds, int opt) -+{ -+ dynhds->opts |= opt; -+} -+ -+void Curl_dynhds_del_opt(struct dynhds *dynhds, int opt) -+{ -+ dynhds->opts &= ~opt; -+} -+ - struct dynhds_entry *Curl_dynhds_getn(struct dynhds *dynhds, size_t n) - { - DEBUGASSERT(dynhds); -diff --git a/lib/dynhds.h b/lib/dynhds.h -index 777baa58a..2d542dfd6 100644 ---- a/lib/dynhds.h -+++ b/lib/dynhds.h -@@ -53,6 +53,7 @@ struct dynhds { - - #define DYNHDS_OPT_NONE (0) - #define DYNHDS_OPT_LOWERCASE (1 << 0) -+#define DYNHDS_OPT_LOWERCASE_VAL (1 << 1) - - /** - * Init for use on first time or after a reset. -@@ -82,6 +83,8 @@ size_t Curl_dynhds_count(struct dynhds *dynhds); - * This will not have an effect on already existing headers. - */ - void Curl_dynhds_set_opts(struct dynhds *dynhds, int opts); -+void Curl_dynhds_set_opt(struct dynhds *dynhds, int opt); -+void Curl_dynhds_del_opt(struct dynhds *dynhds, int opt); - - /** - * Return the n-th header entry or NULL if it does not exist. -diff --git a/lib/easy.c b/lib/easy.c -index d36cc03d1..c5a21e4e3 100644 ---- a/lib/easy.c -+++ b/lib/easy.c -@@ -73,6 +73,8 @@ - #include "dynbuf.h" - #include "altsvc.h" - #include "hsts.h" -+#include "strcase.h" -+#include "impersonate.h" - - #include "easy_lock.h" - -@@ -330,6 +332,76 @@ CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, - return rc; - } - -+/* -+ * curl-impersonate: -+ * Call curl_easy_setopt() with all the needed options as defined in the -+ * 'impersonations' array. -+ * */ -+CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, -+ int default_headers) -+{ -+ int i; -+ int ret; -+ const struct impersonate_opts *opts; -+ struct curl_slist *headers = NULL; -+ -+ for (opts = impersonations; opts->target != NULL; opts++) { -+ if (strncasecompare(target, opts->target, strlen(opts->target))) { -+ break; -+ } -+ } -+ -+ if(opts->target == NULL) { -+ DEBUGF(fprintf(stderr, "Error: unknown impersonation target '%s'\n", -+ target)); -+ return CURLE_BAD_FUNCTION_ARGUMENT; -+ } -+ -+ if(opts->httpversion != CURL_HTTP_VERSION_NONE) { -+ ret = curl_easy_setopt(data, CURLOPT_HTTP_VERSION, opts->httpversion); -+ if(ret) -+ return ret; -+ } -+ -+ if (opts->ssl_version != CURL_SSLVERSION_DEFAULT) { -+ ret = curl_easy_setopt(data, CURLOPT_SSLVERSION, opts->ssl_version); -+ if(ret) -+ return ret; -+ } -+ -+ if(opts->ciphers) { -+ ret = curl_easy_setopt(data, CURLOPT_SSL_CIPHER_LIST, opts->ciphers); -+ if (ret) -+ return ret; -+ } -+ -+ if(default_headers) { -+ /* Build a linked list out of the static array of headers. */ -+ for(i = 0; i < IMPERSONATE_MAX_HEADERS; i++) { -+ if(opts->http_headers[i]) { -+ headers = curl_slist_append(headers, opts->http_headers[i]); -+ if(!headers) { -+ return CURLE_OUT_OF_MEMORY; -+ } -+ } -+ } -+ -+ if(headers) { -+ ret = curl_easy_setopt(data, CURLOPT_HTTPBASEHEADER, headers); -+ curl_slist_free_all(headers); -+ if(ret) -+ return ret; -+ } -+ } -+ -+ /* Always enable all supported compressions. */ -+ ret = curl_easy_setopt(data, CURLOPT_ACCEPT_ENCODING, ""); -+ if(ret) -+ return ret; -+ -+ return CURLE_OK; -+} -+ - /* - * curl_easy_init() is the external interface to alloc, setup and init an - * easy handle that is returned. If anything goes wrong, NULL is returned. -@@ -338,6 +410,8 @@ struct Curl_easy *curl_easy_init(void) - { - CURLcode result; - struct Curl_easy *data; -+ char *env_target; -+ char *env_headers; - - /* Make sure we inited the global SSL stuff */ - global_init_lock(); -@@ -360,6 +434,29 @@ struct Curl_easy *curl_easy_init(void) - return NULL; - } - -+ /* -+ * curl-impersonate: Hook into curl_easy_init() to set the required options -+ * from an environment variable. -+ * This is a bit hacky but allows seamless integration of libcurl-impersonate -+ * without code modifications to the app. -+ */ -+ env_target = curl_getenv("CURL_IMPERSONATE"); -+ if(env_target) { -+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS"); -+ if(env_headers) { -+ result = curl_easy_impersonate(data, env_target, -+ !strcasecompare(env_headers, "no")); -+ free(env_headers); -+ } else { -+ result = curl_easy_impersonate(data, env_target, true); -+ } -+ free(env_target); -+ if(result) { -+ Curl_close(&data); -+ return NULL; -+ } -+ } -+ - return data; - } - -@@ -930,6 +1027,13 @@ struct Curl_easy *curl_easy_duphandle(struct Curl_easy *data) - outcurl->state.referer_alloc = TRUE; - } - -+ if(data->state.base_headers) { -+ outcurl->state.base_headers = -+ Curl_slist_duplicate(data->state.base_headers); -+ if(!outcurl->state.base_headers) -+ goto fail; -+ } -+ - /* Reinitialize an SSL engine for the new handle - * note: the engine name has already been copied by dupset */ - if(outcurl->set.str[STRING_SSL_ENGINE]) { -@@ -1019,6 +1123,9 @@ fail: - */ - void curl_easy_reset(struct Curl_easy *data) - { -+ char *env_target; -+ char *env_headers; -+ - Curl_free_request_state(data); - - /* zero out UserDefined data: */ -@@ -1043,6 +1150,23 @@ void curl_easy_reset(struct Curl_easy *data) - #if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_CRYPTO_AUTH) - Curl_http_auth_cleanup_digest(data); - #endif -+ -+ /* -+ * curl-impersonate: Hook into curl_easy_reset() to set the required options -+ * from an environment variable, just like in curl_easy_init(). -+ */ -+ env_target = curl_getenv("CURL_IMPERSONATE"); -+ if(env_target) { -+ env_headers = curl_getenv("CURL_IMPERSONATE_HEADERS"); -+ if(env_headers) { -+ curl_easy_impersonate(data, env_target, -+ !strcasecompare(env_headers, "no")); -+ free(env_headers); -+ } else { -+ curl_easy_impersonate(data, env_target, true); -+ } -+ free(env_target); -+ } - } - - /* -diff --git a/lib/easyoptions.c b/lib/easyoptions.c -index a9c1efd00..e04b42381 100644 ---- a/lib/easyoptions.c -+++ b/lib/easyoptions.c -@@ -134,6 +134,7 @@ struct curl_easyoption Curl_easyopts[] = { - {"HTTP200ALIASES", CURLOPT_HTTP200ALIASES, CURLOT_SLIST, 0}, - {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, - {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, -+ {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, - {"HTTPHEADER", CURLOPT_HTTPHEADER, CURLOT_SLIST, 0}, - {"HTTPPOST", CURLOPT_HTTPPOST, CURLOT_OBJECT, 0}, - {"HTTPPROXYTUNNEL", CURLOPT_HTTPPROXYTUNNEL, CURLOT_LONG, 0}, -diff --git a/lib/http.c b/lib/http.c -index 219dcc2c0..19ddd1d36 100644 ---- a/lib/http.c -+++ b/lib/http.c -@@ -89,6 +89,7 @@ - #include "hsts.h" - #include "ws.h" - #include "c-hyper.h" -+#include "slist.h" - #include "curl_ctype.h" - - /* The last 3 #include files should be in this order */ -@@ -1881,6 +1882,15 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, - int numlists = 1; /* by default */ - int i; - -+ /* -+ * curl-impersonate: Use the merged list of headers if it exists (i.e. when -+ * the CURLOPT_HTTPBASEHEADER option was set. -+ */ -+ struct curl_slist *noproxyheaders = -+ (data->state.merged_headers ? -+ data->state.merged_headers : -+ data->set.headers); -+ - #ifndef CURL_DISABLE_PROXY - enum proxy_use proxy; - -@@ -1892,10 +1902,10 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, - - switch(proxy) { - case HEADER_SERVER: -- h[0] = data->set.headers; -+ h[0] = noproxyheaders; - break; - case HEADER_PROXY: -- h[0] = data->set.headers; -+ h[0] = noproxyheaders; - if(data->set.sep_headers) { - h[1] = data->set.proxyheaders; - numlists++; -@@ -1905,12 +1915,12 @@ CURLcode Curl_add_custom_headers(struct Curl_easy *data, - if(data->set.sep_headers) - h[0] = data->set.proxyheaders; - else -- h[0] = data->set.headers; -+ h[0] = noproxyheaders; - break; - } - #else - (void)is_connect; -- h[0] = data->set.headers; -+ h[0] = noproxyheaders; - #endif - - /* loop through one or two lists */ -@@ -2146,6 +2156,108 @@ void Curl_http_method(struct Curl_easy *data, struct connectdata *conn, - *reqp = httpreq; - } - -+/* -+ * curl-impersonate: -+ * Create a new linked list of headers. -+ * The new list is a merge between the "base" headers and the application given -+ * headers. The "base" headers contain curl-impersonate's list of headers -+ * used by default by the impersonated browser. -+ * -+ * The application given headers will override the "base" headers if supplied. -+ */ -+CURLcode Curl_http_merge_headers(struct Curl_easy *data) -+{ -+ int i; -+ int ret; -+ struct curl_slist *head; -+ struct curl_slist *dup = NULL; -+ struct curl_slist *new_list = NULL; -+ char *uagent; -+ -+ if (!data->state.base_headers) -+ return CURLE_OK; -+ -+ /* Duplicate the list for temporary use. */ -+ if (data->set.headers) { -+ dup = Curl_slist_duplicate(data->set.headers); -+ if(!dup) -+ return CURLE_OUT_OF_MEMORY; -+ } -+ -+ for(head = data->state.base_headers; head; head = head->next) { -+ char *sep; -+ size_t prefix_len; -+ bool found = FALSE; -+ struct curl_slist *head2; -+ -+ sep = strchr(head->data, ':'); -+ if(!sep) -+ continue; -+ -+ prefix_len = sep - head->data; -+ -+ /* Check if this header was added by the application. */ -+ for(head2 = dup; head2; head2 = head2->next) { -+ if(head2->data && -+ strncasecompare(head2->data, head->data, prefix_len) && -+ Curl_headersep(head2->data[prefix_len]) ) { -+ new_list = curl_slist_append(new_list, head2->data); -+ /* Free and set to NULL to mark that it's been added. */ -+ Curl_safefree(head2->data); -+ found = TRUE; -+ break; -+ } -+ } -+ -+ /* If the user agent was set with CURLOPT_USERAGENT, but not with -+ * CURLOPT_HTTPHEADER, take it from there instead. */ -+ if(!found && -+ strncasecompare(head->data, "User-Agent", prefix_len) && -+ data->set.str[STRING_USERAGENT] && -+ *data->set.str[STRING_USERAGENT]) { -+ uagent = aprintf("User-Agent: %s", data->set.str[STRING_USERAGENT]); -+ if(!uagent) { -+ ret = CURLE_OUT_OF_MEMORY; -+ goto fail; -+ } -+ new_list = Curl_slist_append_nodup(new_list, uagent); -+ found = TRUE; -+ } -+ -+ if (!found) { -+ new_list = curl_slist_append(new_list, head->data); -+ } -+ -+ if (!new_list) { -+ ret = CURLE_OUT_OF_MEMORY; -+ goto fail; -+ } -+ } -+ -+ /* Now go over any additional application-supplied headers. */ -+ for(head = dup; head; head = head->next) { -+ if(head->data) { -+ new_list = curl_slist_append(new_list, head->data); -+ if(!new_list) { -+ ret = CURLE_OUT_OF_MEMORY; -+ goto fail; -+ } -+ } -+ } -+ -+ curl_slist_free_all(dup); -+ /* Save the new, merged list separately, so it can be freed later. */ -+ curl_slist_free_all(data->state.merged_headers); -+ data->state.merged_headers = new_list; -+ -+ return CURLE_OK; -+ -+fail: -+ Curl_safefree(dup); -+ curl_slist_free_all(new_list); -+ return ret; -+} -+ - CURLcode Curl_http_useragent(struct Curl_easy *data) - { - /* The User-Agent string might have been allocated in url.c already, because -@@ -3165,6 +3277,11 @@ CURLcode Curl_http(struct Curl_easy *data, bool *done) - http = data->req.p.http; - DEBUGASSERT(http); - -+ /* curl-impersonate: Add HTTP headers to impersonate real browsers. */ -+ result = Curl_http_merge_headers(data); -+ if (result) -+ return result; -+ - result = Curl_http_host(data, conn); - if(result) - return result; -@@ -4818,23 +4935,32 @@ CURLcode Curl_http_req_to_h2(struct dynhds *h2_headers, - Curl_dynhds_set_opts(h2_headers, DYNHDS_OPT_LOWERCASE); - result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_METHOD), - req->method, strlen(req->method)); -- if(!result && scheme) { -- result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_SCHEME), -- scheme, strlen(scheme)); -+ if(!result && req->path) { -+ result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_PATH), -+ req->path, strlen(req->path)); - } - if(!result && authority) { - result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_AUTHORITY), - authority, strlen(authority)); - } -- if(!result && req->path) { -- result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_PATH), -- req->path, strlen(req->path)); -+ if(!result && scheme) { -+ result = Curl_dynhds_add(h2_headers, STRCONST(HTTP_PSEUDO_SCHEME), -+ scheme, strlen(scheme)); - } - for(i = 0; !result && i < Curl_dynhds_count(&req->headers); ++i) { - e = Curl_dynhds_getn(&req->headers, i); - if(!h2_non_field(e->name, e->namelen)) { -+ /* curl-impersonate: -+ * Some HTTP/2 servers reject 'te' header value that is not lowercase (e.g. 'Trailers). -+ * Convert to lowercase explicitly. -+ */ -+ if(e->namelen == 2 && strcasecompare(e->name, "te")) -+ Curl_dynhds_set_opt(h2_headers, DYNHDS_OPT_LOWERCASE_VAL); -+ - result = Curl_dynhds_add(h2_headers, e->name, e->namelen, - e->value, e->valuelen); -+ -+ Curl_dynhds_del_opt(h2_headers, DYNHDS_OPT_LOWERCASE_VAL); - } - } - -diff --git a/lib/http2.c b/lib/http2.c -index c666192fc..e926eb3f5 100644 ---- a/lib/http2.c -+++ b/lib/http2.c -@@ -63,12 +63,13 @@ - #define NGHTTP2_HAS_SET_LOCAL_WINDOW_SIZE 1 - #endif - -- - /* buffer dimensioning: - * use 16K as chunk size, as that fits H2 DATA frames well */ - #define H2_CHUNK_SIZE (16 * 1024) - /* this is how much we want "in flight" for a stream */ --#define H2_STREAM_WINDOW_SIZE (10 * 1024 * 1024) -+/* curl-impersonate: Set H2_STREAM_WINDOW_SIZE to 128kb so that the initial -+ * window size matches what Firefox sends. */ -+#define H2_STREAM_WINDOW_SIZE (128 * 1024) - /* on receving from TLS, we prep for holding a full stream window */ - #define H2_NW_RECV_CHUNKS (H2_STREAM_WINDOW_SIZE / H2_CHUNK_SIZE) - /* on send into TLS, we just want to accumulate small frames */ -@@ -86,7 +87,8 @@ - * will block their received QUOTA in the connection window. And if we - * run out of space, the server is blocked from sending us any data. - * See #10988 for an issue with this. */ --#define HTTP2_HUGE_WINDOW_SIZE (100 * H2_STREAM_WINDOW_SIZE) -+/* curl-impersonate: set HTTP2_HUGE_WINDOW_SIZE to 12MB to match Firefox. */ -+#define HTTP2_HUGE_WINDOW_SIZE (96 * H2_STREAM_WINDOW_SIZE) - - #define H2_SETTINGS_IV_LEN 3 - #define H2_BINSETTINGS_LEN 80 -@@ -94,14 +96,18 @@ - static int populate_settings(nghttp2_settings_entry *iv, - struct Curl_easy *data) - { -- iv[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; -- iv[0].value = Curl_multi_max_concurrent_streams(data->multi); -+ /* curl-impersonate: Align HTTP/2 settings to Firefox's */ -+ iv[0].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE; -+ iv[0].value = 0x10000; - - iv[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE; - iv[1].value = H2_STREAM_WINDOW_SIZE; - -- iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; -- iv[2].value = data->multi->push_cb != NULL; -+ iv[2].settings_id = NGHTTP2_SETTINGS_MAX_FRAME_SIZE; -+ iv[2].value = 0x4000; -+ -+ // iv[2].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH; -+ // iv[2].value = data->multi->push_cb != NULL; - - return 3; - } -@@ -392,6 +398,71 @@ static void multi_connchanged(struct Curl_multi *multi) - multi->recheckstate = TRUE; - } - -+/* -+ * curl-impersonate: Start with stream id 15 as Firefox does. -+ */ -+#define FIREFOX_DEFAULT_STREAM_ID (15) -+ -+static CURLcode http2_set_stream_priority(struct Curl_cfilter *cf, -+ struct Curl_easy *data, -+ int32_t stream_id, -+ int32_t dep_stream_id, -+ int32_t weight) -+{ -+ int rv; -+ struct cf_h2_ctx *ctx = cf->ctx; -+ nghttp2_priority_spec pri_spec; -+ -+ nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, 0); -+ rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, -+ stream_id, &pri_spec); -+ if(rv) { -+ failf(data, "nghttp2_submit_priority() failed: %s(%d)", -+ nghttp2_strerror(rv), rv); -+ return CURLE_HTTP2; -+ } -+ -+ return CURLE_OK; -+} -+ -+/* -+ * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to -+ * split the load for html/js/css/images. It builds a tree of streams with -+ * different weights (priorities) by default and communicates this to the -+ * server. Imitate that behavior. -+ */ -+static CURLcode http2_set_stream_priorities(struct Curl_cfilter *cf, -+ struct Curl_easy *data) -+{ -+ CURLcode result; -+ -+ result = http2_set_stream_priority(cf, data, 3, 0, 201); -+ if(result) -+ return result; -+ -+ result = http2_set_stream_priority(cf, data, 5, 0, 101); -+ if(result) -+ return result; -+ -+ result = http2_set_stream_priority(cf, data, 7, 0, 0); -+ if(result) -+ return result; -+ -+ result = http2_set_stream_priority(cf, data, 9, 7, 0); -+ if(result) -+ return result; -+ -+ result = http2_set_stream_priority(cf, data, 11, 3, 0); -+ if(result) -+ return result; -+ -+ result = http2_set_stream_priority(cf, data, 13, 0, 241); -+ if(result) -+ return result; -+ -+ return CURLE_OK; -+} -+ - /* - * Initialize the cfilter context - */ -@@ -492,6 +563,13 @@ static CURLcode cf_h2_ctx_init(struct Curl_cfilter *cf, - goto out; - } - -+ result = http2_set_stream_priorities(cf, data); -+ if(result) -+ goto out; -+ -+ /* Best effort to set the request's stream id to 15, like Firefox does. */ -+ nghttp2_session_set_next_stream_id(ctx->h2, FIREFOX_DEFAULT_STREAM_ID); -+ - /* all set, traffic will be send on connect */ - result = CURLE_OK; - -@@ -1616,18 +1694,25 @@ out: - return rv; - } - -+ -+/* -+ * curl-impersonate: Set the HTTP/2 stream weight to the one used by Firefox -+ * by default to fetch html resources. -+ */ -+#define FIREFOX_DEFAULT_STREAM_WEIGHT (42) -+ - static int sweight_wanted(const struct Curl_easy *data) - { - /* 0 weight is not set by user and we take the nghttp2 default one */ - return data->set.priority.weight? -- data->set.priority.weight : NGHTTP2_DEFAULT_WEIGHT; -+ data->set.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT; - } - - static int sweight_in_effect(const struct Curl_easy *data) - { - /* 0 weight is not set by user and we take the nghttp2 default one */ - return data->state.priority.weight? -- data->state.priority.weight : NGHTTP2_DEFAULT_WEIGHT; -+ data->state.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT; - } - - /* -@@ -1636,12 +1721,18 @@ static int sweight_in_effect(const struct Curl_easy *data) - * struct. - */ - -+/* -+ * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the -+ * stream that fetches the main html resource of the web page. -+ */ -+#define FIREFOX_DEFAULT_STREAM_DEP (13) -+ - static void h2_pri_spec(struct Curl_easy *data, - nghttp2_priority_spec *pri_spec) - { - struct Curl_data_priority *prio = &data->set.priority; - struct stream_ctx *depstream = H2_STREAM_CTX(prio->parent); -- int32_t depstream_id = depstream? depstream->id:0; -+ int32_t depstream_id = depstream? depstream->id:FIREFOX_DEFAULT_STREAM_DEP; - nghttp2_priority_spec_init(pri_spec, depstream_id, - sweight_wanted(data), - data->set.priority.exclusive); -diff --git a/lib/impersonate.c b/lib/impersonate.c -new file mode 100644 -index 000000000..e8ae6a722 ---- /dev/null -+++ b/lib/impersonate.c -@@ -0,0 +1,257 @@ -+#include "curl_setup.h" -+ -+#include -+#include "impersonate.h" -+ -+const struct impersonate_opts impersonations[] = { -+ { -+ .target = "ff91esr", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha," -+ "rsa_3des_ede_cbc_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ .target = "ff95", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:95.0) Gecko/20100101 Firefox/95.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ .target = "ff98", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:98.0) Gecko/20100101 Firefox/98.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ .target = "ff100", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ .target = "ff102", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ .target = "ff109", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/109.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ .target = "ff117", -+ .httpversion = CURL_HTTP_VERSION_2_0, -+ .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, -+ .ciphers = -+ "aes_128_gcm_sha_256," -+ "chacha20_poly1305_sha_256," -+ "aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_128_gcm_sha_256," -+ "ecdhe_rsa_aes_128_gcm_sha_256," -+ "ecdhe_ecdsa_chacha20_poly1305_sha_256," -+ "ecdhe_rsa_chacha20_poly1305_sha_256," -+ "ecdhe_ecdsa_aes_256_gcm_sha_384," -+ "ecdhe_rsa_aes_256_gcm_sha_384," -+ "ecdhe_ecdsa_aes_256_sha," -+ "ecdhe_ecdsa_aes_128_sha," -+ "ecdhe_rsa_aes_128_sha," -+ "ecdhe_rsa_aes_256_sha," -+ "rsa_aes_128_gcm_sha_256," -+ "rsa_aes_256_gcm_sha_384," -+ "rsa_aes_128_sha," -+ "rsa_aes_256_sha", -+ .http_headers = { -+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0", -+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", -+ "Accept-Language: en-US,en;q=0.5", -+ "Accept-Encoding: gzip, deflate, br", -+ "Upgrade-Insecure-Requests: 1", -+ "Sec-Fetch-Dest: document", -+ "Sec-Fetch-Mode: navigate", -+ "Sec-Fetch-Site: none", -+ "Sec-Fetch-User: ?1", -+ "TE: Trailers" -+ } -+ }, -+ { -+ /* Last one must be NULL. */ -+ .target = NULL -+ } -+}; -diff --git a/lib/impersonate.h b/lib/impersonate.h -new file mode 100644 -index 000000000..964b81f2e ---- /dev/null -+++ b/lib/impersonate.h -@@ -0,0 +1,25 @@ -+#ifndef HEADER_CURL_IMPERSONATE_H -+#define HEADER_CURL_IMPERSONATE_H -+ -+#define IMPERSONATE_MAX_HEADERS 32 -+ -+/* -+ * curl-impersonate: Options to be set for each supported target browser. -+ */ -+struct impersonate_opts { -+ const char *target; -+ int httpversion; -+ int ssl_version; -+ const char *ciphers; -+ const char *http_headers[IMPERSONATE_MAX_HEADERS]; -+ /* Other TLS options will come here in the future once they are -+ * configurable through curl_easy_setopt() */ -+}; -+ -+/* -+ * curl-impersonate: Global array of supported browsers and their -+ * impersonation options. -+ */ -+extern const struct impersonate_opts impersonations[]; -+ -+#endif /* HEADER_CURL_IMPERSONATE_H */ -diff --git a/lib/setopt.c b/lib/setopt.c -index 0c3b9634d..bd7c6746d 100644 ---- a/lib/setopt.c -+++ b/lib/setopt.c -@@ -50,6 +50,7 @@ - #include "multiif.h" - #include "altsvc.h" - #include "hsts.h" -+#include "slist.h" - - /* The last 3 #include files should be in this order */ - #include "curl_printf.h" -@@ -967,6 +968,23 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) - #if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_SMTP) || \ - !defined(CURL_DISABLE_IMAP) - # if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_MIME) -+ case CURLOPT_HTTPBASEHEADER: -+ /* -+ * curl-impersonate: -+ * Set a list of "base" headers. These will be merged with any headers -+ * set by CURLOPT_HTTPHEADER. curl-impersonate uses this option in order -+ * to set a list of default browser headers. -+ * -+ * Unlike CURLOPT_HTTPHEADER, -+ * the list is copied and can be immediately freed by the user. -+ */ -+ curl_slist_free_all(data->state.base_headers); -+ data->state.base_headers = \ -+ Curl_slist_duplicate(va_arg(param, struct curl_slist *)); -+ if (!data->state.base_headers) -+ result = CURLE_OUT_OF_MEMORY; -+ break; -+ - case CURLOPT_HTTPHEADER: - /* - * Set a list with HTTP headers to use (or replace internals with) -diff --git a/lib/transfer.c b/lib/transfer.c -index d2ff0c24c..56e2090b6 100644 ---- a/lib/transfer.c -+++ b/lib/transfer.c -@@ -106,7 +106,15 @@ char *Curl_checkheaders(const struct Curl_easy *data, - DEBUGASSERT(thislen); - DEBUGASSERT(thisheader[thislen-1] != ':'); - -- for(head = data->set.headers; head; head = head->next) { -+ /* -+ * curl-impersonate: -+ * Check if we have overriden the user-supplied list of headers. -+ */ -+ head = data->set.headers; -+ if (data->state.merged_headers) -+ head = data->state.merged_headers; -+ -+ for(; head; head = head->next) { - if(strncasecompare(head->data, thisheader, thislen) && - Curl_headersep(head->data[thislen]) ) - return head->data; -diff --git a/lib/url.c b/lib/url.c -index b37d13f8f..ffb74cc4f 100644 ---- a/lib/url.c -+++ b/lib/url.c -@@ -444,6 +444,11 @@ CURLcode Curl_close(struct Curl_easy **datap) - Curl_safefree(data->state.aptr.proxyuser); - Curl_safefree(data->state.aptr.proxypasswd); - -+ /* curl-impersonate: Free the list set by CURLOPT_HTTPBASEHEADER. */ -+ curl_slist_free_all(data->state.base_headers); -+ /* curl-impersonate: Free the dynamic list of headers. */ -+ curl_slist_free_all(data->state.merged_headers); -+ - #ifndef CURL_DISABLE_DOH - if(data->req.doh) { - Curl_dyn_free(&data->req.doh->probe[0].serverdoh); -diff --git a/lib/urldata.h b/lib/urldata.h -index f02e66541..d6162a80e 100644 ---- a/lib/urldata.h -+++ b/lib/urldata.h -@@ -1395,6 +1395,19 @@ struct UrlState { - CURLcode hresult; /* used to pass return codes back from hyper callbacks */ - #endif - -+ /* -+ * curl-impersonate: -+ * List of "base" headers set by CURLOPT_HTTPBASEHEADER. -+ */ -+ struct curl_slist *base_headers; -+ /* -+ * curl-impersonate: -+ * Dynamically-constructed list of HTTP headers. -+ * This list is a merge of the default HTTP headers needed to impersonate a -+ * browser, together with any user-supplied headers. -+ */ -+ struct curl_slist *merged_headers; -+ - /* Dynamically allocated strings, MUST be freed before this struct is - killed. */ - struct dynamically_allocated_data { -diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c -index 5e5dbb744..2604fc755 100644 ---- a/lib/vtls/nss.c -+++ b/lib/vtls/nss.c -@@ -145,6 +145,7 @@ static const struct cipher_s cipherlist[] = { - {"dhe_dss_3des_sha", SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA}, - {"dhe_rsa_des_sha", SSL_DHE_RSA_WITH_DES_CBC_SHA}, - {"dhe_dss_des_sha", SSL_DHE_DSS_WITH_DES_CBC_SHA}, -+ {"rsa_3des_ede_cbc_sha", TLS_RSA_WITH_3DES_EDE_CBC_SHA}, - /* TLS 1.0: Exportable 56-bit Cipher Suites. */ - {"rsa_des_56_sha", TLS_RSA_EXPORT1024_WITH_DES_CBC_SHA}, - {"rsa_rc4_56_sha", TLS_RSA_EXPORT1024_WITH_RC4_56_SHA}, -@@ -380,6 +381,95 @@ static SECStatus set_ciphers(struct Curl_easy *data, PRFileDesc *model, - return SECSuccess; - } - -+/* See nsSSLIOLayerSetOptions@nsNSSIOLayer.cpp, Firefox source code */ -+const SSLNamedGroup named_groups[] = { -+ ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1, ssl_grp_ec_secp384r1, -+ ssl_grp_ec_secp521r1, ssl_grp_ffdhe_2048, ssl_grp_ffdhe_3072}; -+ -+#define NUM_OF_NAMED_GROUPS sizeof(named_groups)/sizeof(named_groups[0]) -+ -+static SECStatus set_named_groups(PRFileDesc *model) -+{ -+ /* This aligns TLS extension 10 (supported_groups) to what Firefox does. */ -+ return SSL_NamedGroupConfig(model, named_groups, NUM_OF_NAMED_GROUPS); -+} -+ -+static const SSLSignatureScheme signatures[] = { -+ ssl_sig_ecdsa_secp256r1_sha256, ssl_sig_ecdsa_secp384r1_sha384, -+ ssl_sig_ecdsa_secp521r1_sha512, ssl_sig_rsa_pss_sha256, -+ ssl_sig_rsa_pss_sha384, ssl_sig_rsa_pss_sha512, -+ ssl_sig_rsa_pkcs1_sha256, ssl_sig_rsa_pkcs1_sha384, -+ ssl_sig_rsa_pkcs1_sha512, ssl_sig_ecdsa_sha1, -+ ssl_sig_rsa_pkcs1_sha1 -+}; -+ -+#define NUM_OF_SIGNATURES sizeof(signatures)/sizeof(signatures[0]) -+ -+static SECStatus set_additional_key_shares(PRFileDesc *model) -+{ -+ /* This aligns TLS extension 51 (key_share) to what Firefox does. */ -+ return SSL_SendAdditionalKeyShares(model, 1); -+} -+ -+static SECStatus set_signatures(PRFileDesc *model) -+{ -+ /* Align TLS extension 13 (signature_algorithms) to what Firefox does. */ -+ return SSL_SignatureSchemePrefSet(model, signatures, NUM_OF_SIGNATURES); -+} -+ -+static SECStatus set_ssl_options(PRFileDesc *model) -+{ -+ SECStatus s; -+ -+ /* Enable TLS 1.3 compat mode. Firefox does this, as can be seen at -+ * nsSSLIOLayerSetOptions()@nsNSSIOLayer.cpp. -+ * This has the side effect of NSS faking a TLS session ID. -+ * See ssl3_CreateClientHelloPreamble()@ssl3con.c -+ */ -+ s = SSL_OptionSet(model, SSL_ENABLE_TLS13_COMPAT_MODE, PR_TRUE); -+ if (s != SECSuccess) { -+ return s; -+ } -+ -+ /* Firefox sets the following options. I don't know what they do. */ -+ s = SSL_OptionSet(model, SSL_REQUIRE_SAFE_NEGOTIATION, false); -+ if (s != SECSuccess) { -+ return s; -+ } -+ s = SSL_OptionSet(model, SSL_ENABLE_EXTENDED_MASTER_SECRET, true); -+ if (s != SECSuccess) { -+ return s; -+ } -+ s = SSL_OptionSet(model, SSL_ENABLE_HELLO_DOWNGRADE_CHECK, true); -+ if (s != SECSuccess) { -+ return s; -+ } -+ s = SSL_OptionSet(model, SSL_ENABLE_0RTT_DATA, true); -+ if (s != SECSuccess) { -+ return s; -+ } -+ -+ /* This adds TLS extension 34 to the Client Hello. */ -+ s = SSL_OptionSet(model, SSL_ENABLE_DELEGATED_CREDENTIALS, true); -+ if (s != SECSuccess) { -+ return s; -+ } -+ -+ /* This adds TLS extension 5 (status_request) to the Client Hello. */ -+ s = SSL_OptionSet(model, SSL_ENABLE_OCSP_STAPLING, true); -+ if (s != SECSuccess) { -+ return s; -+ } -+ -+ /* Remove TLS extension 18 (signed_certificate_timestamp) */ -+ s = SSL_OptionSet(model, SSL_ENABLE_SIGNED_CERT_TIMESTAMPS, false); -+ if (s != SECSuccess) { -+ return s; -+ } -+ -+ return SSL_OptionSet(model, SSL_HANDSHAKE_AS_CLIENT, true); -+} -+ - /* - * Return true if at least one cipher-suite is enabled. Used to determine - * if we need to call NSS_SetDomesticPolicy() to enable the default ciphers. -@@ -1341,6 +1431,7 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library, - - if(module) - SECMOD_DestroyModule(module); -+ - return CURLE_FAILED_INIT; - } - -@@ -1923,6 +2014,12 @@ static CURLcode nss_setup_connect(struct Curl_cfilter *cf, - if(SSL_OptionSet(model, SSL_NO_CACHE, ssl_no_cache) != SECSuccess) - goto error; - -+ if(ssl_config->primary.sessionid) { -+ if(SSL_OptionSet(model, SSL_ENABLE_SESSION_TICKETS, -+ PR_TRUE) != SECSuccess) -+ goto error; -+ } -+ - /* enable/disable the requested SSL version(s) */ - if(nss_init_sslver(&sslver, cf, data) != CURLE_OK) - goto error; -@@ -1962,6 +2059,14 @@ static CURLcode nss_setup_connect(struct Curl_cfilter *cf, - } - } - -+ if (set_named_groups(model) != SECSuccess || -+ set_additional_key_shares(model) != SECSuccess || -+ set_signatures(model) != SECSuccess || -+ set_ssl_options(model) != SECSuccess) { -+ result = CURLE_SSL_CIPHER; -+ goto error; -+ } -+ - if(!conn_config->verifypeer && conn_config->verifyhost) - infof(data, "WARNING: ignoring value of ssl.verifyhost"); - -diff --git a/lib/vtls/vtls.c b/lib/vtls/vtls.c -index 32334016b..a9b6299b2 100644 ---- a/lib/vtls/vtls.c -+++ b/lib/vtls/vtls.c -@@ -139,7 +139,7 @@ static const struct alpn_spec ALPN_SPEC_H11 = { - }; - #ifdef USE_HTTP2 - static const struct alpn_spec ALPN_SPEC_H2_H11 = { -- { ALPN_H2, ALPN_HTTP_1_1 }, 2 -+ { ALPN_HTTP_1_1, ALPN_H2 }, 2 - }; - #endif - -diff --git a/libcurl.pc.in b/libcurl.pc.in -index 9db6b0f89..9e2f19af9 100644 ---- a/libcurl.pc.in -+++ b/libcurl.pc.in -@@ -36,6 +36,6 @@ Name: libcurl - URL: https://curl.se/ - Description: Library to transfer files with ftp, http, etc. - Version: @CURLVERSION@ --Libs: -L${libdir} -lcurl @LIBCURL_NO_SHARED@ -+Libs: -L${libdir} -lcurl-impersonate-ff @LIBCURL_NO_SHARED@ - Libs.private: @LIBCURL_LIBS@ - Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ -diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4 -index 2916c3613..f07fa30ac 100644 ---- a/m4/curl-nss.m4 -+++ b/m4/curl-nss.m4 -@@ -76,7 +76,123 @@ if test "x$OPT_NSS" != xno; then - # Without pkg-config, we'll kludge in some defaults - AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.]) - addld="-L$OPT_NSS/lib" -- addlib="-lssl3 -lsmime3 -lnss3 -lplds4 -lplc4 -lnspr4" -+ -+ # curl-impersonate: Link NSS statically. -+ # NSS is poorly documented in this regard and a lot of trial and error -+ # was made to come up with the correct list of linking flags. The -+ # libraries have circular dependencies which makes their order extremely -+ # difficult to find out. -+ -+ # Some references: -+ # https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_build_common/src/lib.rs#L94 -+ # and -+ # https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/freebl/freebl.gyp -+ -+ # On Linux we can use special linker flags to force static linking -+ # (-l:libplc4.a etc.), otherwise the linker will prefer to use -+ # libplc4.so. On other systems the dynamic libraries would have to be -+ # removed manually from the NSS directory before building curl. -+ case $host_os in -+ linux*) -+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite" -+ ;; -+ darwin*) -+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4" -+ ;; -+ *) -+ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4 -lsqlite" -+ ;; -+ esac -+ -+ case $host_cpu in -+ arm|armv7l) -+ addlib="$addlib -larmv8_c_lib -lgcm-aes-arm32-neon_c_lib" -+ ;; -+ aarch64) -+ addlib="$addlib -larmv8_c_lib -lgcm-aes-aarch64_c_lib" -+ ;; -+ x86) -+ addlib="$addlib -lgcm-aes-x86_c_lib" -+ ;; -+ x86_64) -+ addlib="$addlib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lsha-x86_c_lib" -+ case $host_os in -+ linux*) -+ addlib="$addlib -lintel-gcm-wrap_c_lib -lintel-gcm-s_lib" -+ ;; -+ esac -+ ;; -+ esac -+ -+ # curl-impersonate: -+ # On Linux these linker flags are necessary to resolve -+ # the symbol mess and circular dependencies of NSS .a libraries -+ # to make the AC_CHECK_LIB test below pass. -+ case $host_os in -+ linux*) -+ addlib="-Wl,--start-group $addlib -Wl,--end-group" -+ ;; -+ esac -+ -+ # External dependencies for nss -+ case $host_os in -+ linux*) -+ addlib="$addlib -pthread -ldl" -+ ;; -+ darwin*) -+ addlib="$addlib -lsqlite3" -+ ;; -+ esac -+ -+ # Attempt to locate libnssckbi. -+ # This library file contains the trusted certificates and nss loads it -+ # at runtime using dlopen. If it's not in a path findable by dlopen -+ # we have to add that path explicitly using -rpath so it may find it. -+ # On Ubuntu and Mac M1 it is in a non-standard location. -+ AC_ARG_WITH(libnssckbi, -+ [AS_HELP_STRING([--with-libnssckbi=DIRECTORY], -+ [Path where libnssckbi can be found when using NSS])], -+ [AS_IF( -+ [test x"$withval" = xyes], -+ [nssckbi_path="check"], -+ [nssckbi_path="$withval"])], -+ [nssckbi_path="check"]) -+ -+ AS_IF( -+ [test "x$nssckbi_path" = xno], -+ [], -+ [test "x$nssckbi_path" != xcheck], -+ [addld="$addld -Wl,-rpath,$nssckbi_path"], -+ [ -+ AC_MSG_CHECKING([if libnssckbi is in a non-standard location]) -+ case $host_os in -+ linux*) -+ search_paths="/usr/lib/$host /usr/lib/$host/nss" -+ search_paths="$search_paths /usr/lib/$host_cpu-$host_os" -+ search_paths="$search_paths /usr/lib/$host_cpu-$host_os/nss" -+ search_ext="so" -+ ;; -+ darwin*) -+ search_paths="/opt/homebrew/lib" -+ search_ext="dylib" -+ ;; -+ esac -+ -+ found="no" -+ for path in $search_paths; do -+ if test -f "$path/libnssckbi.$search_ext"; then -+ AC_MSG_RESULT([$path]) -+ addld="$addld -Wl,-rpath,$path" -+ found="yes" -+ break -+ fi -+ done -+ -+ if test "$found" = "no"; then -+ AC_MSG_RESULT([no]) -+ fi -+ ]) -+ - addcflags="-I$OPT_NSS/include" - version="unknown" - nssprefix=$OPT_NSS -@@ -93,7 +209,7 @@ if test "x$OPT_NSS" != xno; then - fi - - dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0 -- AC_CHECK_LIB(nss3, SSL_VersionRangeSet, -+ AC_CHECK_LIB(nss_static, SSL_VersionRangeSet, - [ - AC_DEFINE(USE_NSS, 1, [if NSS is enabled]) - AC_SUBST(USE_NSS, [1]) -@@ -103,9 +219,7 @@ if test "x$OPT_NSS" != xno; then - test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes - ], - [ -- LDFLAGS="$CLEANLDFLAGS" -- LIBS="$CLEANLIBS" -- CPPFLAGS="$CLEANCPPFLAGS" -+ AC_MSG_ERROR([Failed linking NSS statically]) - ]) - - if test "x$USE_NSS" = "xyes"; then -diff --git a/src/Makefile.am b/src/Makefile.am -index f24cb6924..5750eaba8 100644 ---- a/src/Makefile.am -+++ b/src/Makefile.am -@@ -43,7 +43,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include \ - -I$(top_srcdir)/lib \ - -I$(top_srcdir)/src - --bin_PROGRAMS = curl -+bin_PROGRAMS = curl-impersonate-ff - - SUBDIRS = ../docs - -@@ -54,9 +54,9 @@ endif - include Makefile.inc - - # CURL_FILES comes from Makefile.inc --curl_SOURCES = $(CURL_FILES) -+curl_impersonate_ff_SOURCES = $(CURL_FILES) - if HAVE_WINDRES --curl_SOURCES += $(CURL_RCFILES) -+curl_impersonate_ff_SOURCES += $(CURL_RCFILES) - $(CURL_RCFILES): tool_version.h - endif - -@@ -67,9 +67,9 @@ CFLAGS += @CURL_CFLAG_EXTRAS@ - LIBS = $(BLANK_AT_MAKETIME) - - if USE_EXPLICIT_LIB_DEPS --curl_LDADD = $(top_builddir)/lib/libcurl.la @LIBCURL_LIBS@ -+curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @LIBCURL_LIBS@ - else --curl_LDADD = $(top_builddir)/lib/libcurl.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ -+curl_impersonate_ff_LDADD = $(top_builddir)/lib/libcurl-impersonate-ff.la @NSS_LIBS@ @SSL_LIBS@ @ZLIB_LIBS@ @CURL_NETWORK_AND_TIME_LIBS@ - endif - - # if unit tests are enabled, build a static library to link them with diff --git a/generate_dockerfiles.sh b/generate_dockerfiles.sh deleted file mode 100755 index d75c6ee2..00000000 --- a/generate_dockerfiles.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -cat < chrome/Dockerfile ---- -chrome: true -debian: true ---- -EOF -cat < chrome/Dockerfile.alpine ---- -chrome: true -alpine: true ---- -EOF diff --git a/tests/README.md b/tests/README.md index ddab00b3..072ad6ab 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,7 +21,7 @@ For each supported browser, the following tests are performed: ## What's missing The following tests are still missing: -- [ ] Test that `curl-impersonate` sends the same HTTP/2 SETTINGS as the browser. -- [ ] Capture traffic automatically from different browsers + +- [x] Test that `curl-impersonate` sends the same HTTP/2 SETTINGS as the browser. - [x] Update safari versions, double `rsa_pss_rsae_sha384` diff --git a/tests/signatures/chrome_123.0.6312.124.yaml b/tests/signatures/chrome_123.0.6312.124.yaml new file mode 100644 index 00000000..c1f194a2 --- /dev/null +++ b/tests/signatures/chrome_123.0.6312.124.yaml @@ -0,0 +1,146 @@ +browser: + name: chrome + os: macOS + version: 123.0.6312.124 +signature: + options: + tls_permute_extensions: true + http2: + frames: + - frame_type: SETTINGS + settings: + - key: 1 + value: 65536 + - key: 2 + value: 0 + - key: 4 + value: 6291456 + - key: 6 + value: 262144 + stream_id: 0 + - frame_type: WINDOW_UPDATE + stream_id: 0 + window_size_increment: 15663105 + - frame_type: HEADERS + headers: + - 'sec-ch-ua: "Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "macOS"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br, zstd' + - 'accept-language: en-US,en;q=0.9' + pseudo_headers: + - :method + - :authority + - :scheme + - :path + stream_id: 1 + tls_client_hello: + ciphersuites: + - GREASE + - 4865 + - 4866 + - 4867 + - 49195 + - 49199 + - 49196 + - 49200 + - 52393 + - 52392 + - 49171 + - 49172 + - 156 + - 157 + - 47 + - 53 + comp_methods: + - 0 + extensions: + - length: 0 + type: GREASE + - alps_alpn_list: + - h2 + length: 5 + type: application_settings + - length: 2 + psk_ke_mode: 1 + type: psk_key_exchange_modes + - algorithms: + - 2 + length: 3 + type: compress_certificate + - key_shares: + - group: GREASE + length: 1 + - group: 29 + length: 32 + length: 43 + type: keyshare + - length: 7 + supported_versions: + - GREASE + - TLS_VERSION_1_3 + - TLS_VERSION_1_2 + type: supported_versions + - length: 1 + type: renegotiation_info + - length: 0 + type: encrypted_client_hello + - length: 0 + type: extended_master_secret + - ec_point_formats: + - 0 + length: 2 + type: ec_point_formats + - length: 5 + status_request_type: 1 + type: status_request + - length: 0 + type: session_ticket + - length: 10 + supported_groups: + - GREASE + - 29 + - 23 + - 24 + type: supported_groups + - length: 0 + type: signed_certificate_timestamp + - alpn_list: + - h2 + - http/1.1 + length: 14 + type: application_layer_protocol_negotiation + - length: 18 + sig_hash_algs: + - 1027 + - 2052 + - 1025 + - 1283 + - 2053 + - 1281 + - 2054 + - 1537 + type: signature_algorithms + - type: server_name + - data: !!binary | + AA== + length: 1 + type: GREASE + handshake_version: TLS_VERSION_1_2 + record_version: TLS_VERSION_1_0 + session_id_length: 32 +third_party: + akamai_hash: 52d84b11737d980aef856699f885ca86 + akamai_text: 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p + ja3_hash: 64ab95a82dbf6413690ab30dce00548f + ja3_text: 771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,27-0-23-11-65037-65281-13-45-43-16-18-35-10-5-51-17513,29-23-24,0 + ja3n_hash: 473f0e7c0b6a0f7b049072f4e683068b + ja3n_text: 771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-5-10-11-13-16-18-23-27-35-43-45-51-17513-65037-65281,29-23-24,0 + user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 diff --git a/tests/signatures/chrome_124.0.6367.60.yaml b/tests/signatures/chrome_124.0.6367.60.yaml new file mode 100644 index 00000000..c0072b69 --- /dev/null +++ b/tests/signatures/chrome_124.0.6367.60.yaml @@ -0,0 +1,152 @@ +# NOTE: this file is not generated by dpkt, due to a bug: https://github.com/kbandla/dpkt/issues/665 +# The only difference from Chrome 123 is: supported_groups with X25519Kyber768 +browser: + name: chrome + os: macOS + version: 124.0.6367.60 +signature: + options: + tls_permute_extensions: true + http2: + frames: + - frame_type: SETTINGS + settings: + - key: 1 + value: 65536 + - key: 2 + value: 0 + - key: 4 + value: 6291456 + - key: 6 + value: 262144 + stream_id: 0 + - frame_type: WINDOW_UPDATE + stream_id: 0 + window_size_increment: 15663105 + - frame_type: HEADERS + headers: + - 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' + - 'sec-ch-ua-mobile: ?0' + - 'sec-ch-ua-platform: "macOS"' + - 'upgrade-insecure-requests: 1' + - 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' + - 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' + - 'sec-fetch-site: none' + - 'sec-fetch-mode: navigate' + - 'sec-fetch-user: ?1' + - 'sec-fetch-dest: document' + - 'accept-encoding: gzip, deflate, br, zstd' + - 'accept-language: en-US,en;q=0.9' + - 'priority: u=0, i' + pseudo_headers: + - :method + - :authority + - :scheme + - :path + stream_id: 1 + tls_client_hello: + ciphersuites: + - GREASE + - 4865 + - 4866 + - 4867 + - 49195 + - 49199 + - 49196 + - 49200 + - 52393 + - 52392 + - 49171 + - 49172 + - 156 + - 157 + - 47 + - 53 + comp_methods: + - 0 + extensions: + - length: 0 + type: GREASE + - alps_alpn_list: + - h2 + length: 5 + type: application_settings + - length: 2 + psk_ke_mode: 1 + type: psk_key_exchange_modes + - algorithms: + - 2 + length: 3 + type: compress_certificate + - key_shares: + - group: GREASE + length: 1 + - group: 25497 + length: 1216 + - group: 29 + length: 32 + length: 1263 + type: keyshare + - length: 7 + supported_versions: + - GREASE + - TLS_VERSION_1_3 + - TLS_VERSION_1_2 + type: supported_versions + - length: 1 + type: renegotiation_info + - length: 0 + type: encrypted_client_hello + - length: 0 + type: extended_master_secret + - ec_point_formats: + - 0 + length: 2 + type: ec_point_formats + - length: 5 + status_request_type: 1 + type: status_request + - length: 0 + type: session_ticket + - length: 12 + supported_groups: + - GREASE + - 25497 + - 29 + - 23 + - 24 + type: supported_groups + - length: 0 + type: signed_certificate_timestamp + - alpn_list: + - h2 + - http/1.1 + length: 14 + type: application_layer_protocol_negotiation + - length: 18 + sig_hash_algs: + - 1027 + - 2052 + - 1025 + - 1283 + - 2053 + - 1281 + - 2054 + - 1537 + type: signature_algorithms + - type: server_name + - data: !!binary | + AA== + length: 1 + type: GREASE + handshake_version: TLS_VERSION_1_2 + record_version: TLS_VERSION_1_0 + session_id_length: 32 +third_party: + akamai_hash: 52d84b11737d980aef856699f885ca86 + akamai_text: 1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p + ja3_hash: 0160ae8b3388a3463efd4d10b17e0119 + ja3_text: 771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,65037-43-0-35-10-18-51-45-17513-16-11-27-23-5-13-65281,25497-29-23-24,0 + ja3n_hash: 4c9ce26028c11d7544da00d3f7e4f45c + ja3n_text: 771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-5-10-11-13-16-18-23-27-35-43-45-51-17513-65037-65281,25497-29-23-24,0 + user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 diff --git a/tests/targets.yaml b/tests/targets.yaml index b5086778..7085fb3c 100644 --- a/tests/targets.yaml +++ b/tests/targets.yaml @@ -35,6 +35,14 @@ - null - null - chrome_120.0.6099.109_macOS +- - curl_chrome123 + - null + - null + - chrome_123.0.6312.124_macOS +- - curl_chrome124 + - null + - null + - chrome_124.0.6367.60_macOS - - curl_chrome99_android - null - null @@ -63,34 +71,10 @@ - null - null - safari_17.2_iOS -# - - curl_ff91esr -# - null -# - null -# - firefox_91.6.0esr_win10 -# - - curl_ff95 -# - null -# - null -# - firefox_95.0.2_win10 -# - - curl_ff98 -# - null -# - null -# - firefox_98.0_win10 -# - - curl_ff100 -# - null -# - null -# - firefox_100.0_win10 -# - - curl_ff102 -# - null +# - - curl_firefox120 # - null -# - firefox_102.0_win10 -# - - curl_ff109 # - null -# - null -# - firefox_109.0_win10 -# - - curl_ff117 -# - null -# - null -# - firefox_117.0.1_win10 +# - firefox_120_macOS # Test libcurl-impersonate by loading it with LD_PRELOAD to an app # linked against the regular libcurl and setting the @@ -131,6 +115,14 @@ - CURL_IMPERSONATE: chrome120 - libcurl-impersonate-chrome - chrome_120.0.6099.109_macOS +- - minicurl + - CURL_IMPERSONATE: chrome123 + - libcurl-impersonate-chrome + - chrome_123.0.6312.124_macOS +- - minicurl + - CURL_IMPERSONATE: chrome124 + - libcurl-impersonate-chrome + - chrome_124.0.6367.60_macOS - - minicurl - CURL_IMPERSONATE: chrome99_android - libcurl-impersonate-chrome @@ -160,30 +152,6 @@ - libcurl-impersonate-chrome - safari_17.2_iOS # - - minicurl -# - CURL_IMPERSONATE: ff91esr -# - libcurl-impersonate-ff -# - firefox_91.6.0esr_win10 -# - - minicurl -# - CURL_IMPERSONATE: ff95 -# - libcurl-impersonate-ff -# - firefox_95.0.2_win10 -# - - minicurl -# - CURL_IMPERSONATE: ff98 -# - libcurl-impersonate-ff -# - firefox_98.0_win10 -# - - minicurl -# - CURL_IMPERSONATE: ff100 -# - libcurl-impersonate-ff -# - firefox_100.0_win10 -# - - minicurl -# - CURL_IMPERSONATE: ff102 -# - libcurl-impersonate-ff -# - firefox_102.0_win10 -# - - minicurl -# - CURL_IMPERSONATE: ff109 -# - libcurl-impersonate-ff -# - firefox_109.0_win10 -# - - minicurl -# - CURL_IMPERSONATE: ff117 -# - libcurl-impersonate-ff -# - firefox_117.0.1_win10 +# - CURL_IMPERSONATE: firefox120 +# - libcurl-impersonate-chrome +# - firefox_120_macOS diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index cb407f20..83ff545d 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -14,6 +14,7 @@ from th1.tls.signature import TLSClientHelloSignature from th1.http2.parser import parse_nghttpd_log from th1.http2.signature import HTTP2Signature +import dpkt @pytest.fixture @@ -264,7 +265,12 @@ def test_tls_client_hello( assert len(pcap) > 0 logging.debug(f"Captured pcap of length {len(pcap)} bytes") - client_hellos = parse_pcap(pcap) + try: + client_hellos = parse_pcap(pcap) + except dpkt.NeedData: + logging.error("DPKT does not support Chrome 124 yet.") + return + # A client hello message for each URL assert len(client_hellos) == len(test_urls) @@ -398,11 +404,6 @@ def test_content_encoding( {"CURL_IMPERSONATE": "chrome101", "CURL_IMPERSONATE_HEADERS": "no"}, "libcurl-impersonate-chrome", ), - ( - "minicurl", - {"CURL_IMPERSONATE": "ff102", "CURL_IMPERSONATE_HEADERS": "no"}, - "libcurl-impersonate-ff", - ), ], ) async def test_no_builtin_headers( @@ -469,16 +470,6 @@ async def test_no_builtin_headers( {"CURL_IMPERSONATE": "chrome101", "CURL_IMPERSONATE_HEADERS": "no"}, "libcurl-impersonate-chrome", ), - ( - "minicurl", - {"CURL_IMPERSONATE": "ff102"}, - "libcurl-impersonate-ff", - ), - ( - "minicurl", - {"CURL_IMPERSONATE": "ff102", "CURL_IMPERSONATE_HEADERS": "no"}, - "libcurl-impersonate-ff", - ), ], ) async def test_user_agent(pytestconfig, nghttpd, curl_binary, env_vars, ld_preload): @@ -538,16 +529,6 @@ async def test_user_agent(pytestconfig, nghttpd, curl_binary, env_vars, ld_prelo {"CURL_IMPERSONATE": "chrome101", "CURL_IMPERSONATE_HEADERS": "no"}, "libcurl-impersonate-chrome", ), - ( - "minicurl", - {"CURL_IMPERSONATE": "ff102"}, - "libcurl-impersonate-ff", - ), - ( - "minicurl", - {"CURL_IMPERSONATE": "ff102", "CURL_IMPERSONATE_HEADERS": "no"}, - "libcurl-impersonate-ff", - ), ], ) async def test_user_agent_curlopt_useragent( diff --git a/win/build.sh b/win/build.sh index 66e7e53f..5e23d81a 100644 --- a/win/build.sh +++ b/win/build.sh @@ -14,7 +14,7 @@ mv boringssl-${BORING_SSL_COMMIT} boringssl cd boringssl -patchfile=../../chrome/patches/boringssl-old-ciphers.patch +patchfile=../../chrome/patches/boringssl.patch patch -p1 < $patchfile sed -i 's/-ggdb//g' CMakeLists.txt sed -i 's/-Werror//g' CMakeLists.txt @@ -38,7 +38,7 @@ export OPENSSL_LIBPATH=$PWD/boringssl/lib export OPENSSL_LIBS='-lssl -lcrypto' -CURL_VERSION=curl-8_1_1 +CURL_VERSION=curl-8_5_0 curl -L https://github.com/curl/curl/archive/${CURL_VERSION}.zip -o curl.zip unzip -q -o curl.zip