diff --git a/include/curl/curl.h b/include/curl/curl.h index 2199a34fcd83..29339db77055 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -2255,6 +2255,12 @@ typedef enum { /* set ECH configuration, XXX, the official one is 324 */ CURLOPT(CURLOPT_ECH, CURLOPTTYPE_STRINGPOINT, 333), + /* + * curl-impersonate: + * Set the initial streams settings for http2. + */ + CURLOPT(CURLOPT_HTTP2_STREAMS, CURLOPTTYPE_STRINGPOINT, 334), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/lib/easy.c b/lib/easy.c index d11892540b4e..95a6dff18a0c 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -463,6 +463,12 @@ CURLcode _do_impersonate(struct Curl_easy *data, return ret; } + if(opts->http2_streams) { + ret = curl_easy_setopt(data, CURLOPT_HTTP2_STREAMS, opts->http2_streams); + if(ret) + return ret; + } + if(opts->ech) { ret = curl_easy_setopt(data, CURLOPT_ECH, opts->ech); if(ret) @@ -488,7 +494,7 @@ CURLcode curl_easy_impersonate_customized(struct Curl_easy *data, { int ret; - ret = _do_impersonate(data, opts, default_headers) + ret = _do_impersonate(data, opts, default_headers); if(ret) return ret; @@ -518,7 +524,7 @@ CURLcode curl_easy_impersonate(struct Curl_easy *data, const char *target, return CURLE_BAD_FUNCTION_ARGUMENT; } - ret = _do_impersonate(data, opts, default_headers) + ret = _do_impersonate(data, opts, default_headers); if(ret) return ret; diff --git a/lib/easyoptions.c b/lib/easyoptions.c index 585a6638b1a1..aa848d66b266 100644 --- a/lib/easyoptions.c +++ b/lib/easyoptions.c @@ -138,6 +138,7 @@ struct curl_easyoption Curl_easyopts[] = { CURLOT_STRING, 0}, {"HTTP2_SETTINGS", CURLOPT_HTTP2_SETTINGS, CURLOT_STRING, 0}, {"HTTP2_WINDOW_UPDATE", CURLOPT_HTTP2_WINDOW_UPDATE, CURLOT_LONG, 0}, + {"HTTP2_STREAMS", CURLOPT_HTTP2_STREAMS, CURLOT_STRING, 0}, {"HTTPAUTH", CURLOPT_HTTPAUTH, CURLOT_VALUES, 0}, {"HTTPGET", CURLOPT_HTTPGET, CURLOT_LONG, 0}, {"HTTPBASEHEADER", CURLOPT_HTTPBASEHEADER, CURLOT_SLIST, 0}, @@ -384,6 +385,6 @@ struct curl_easyoption Curl_easyopts[] = { */ int Curl_easyopts_check(void) { - return ((CURLOPT_LASTENTRY%10000) != (333 + 1)); + return ((CURLOPT_LASTENTRY%10000) != (334 + 1)); } #endif diff --git a/lib/http2.c b/lib/http2.c index a4bda9b9704c..36654499de10 100644 --- a/lib/http2.c +++ b/lib/http2.c @@ -168,6 +168,7 @@ static int populate_settings(nghttp2_settings_entry *iv, return i; } + static ssize_t populate_binsettings(uint8_t *binsettings, struct Curl_easy *data) { @@ -226,6 +227,75 @@ static void cf_h2_ctx_free(struct cf_h2_ctx *ctx) } } +static CURLcode http2_set_stream_priority(struct Curl_cfilter *cf, + struct Curl_easy *data, + int32_t stream_id, + int32_t dep_stream_id, + int32_t weight, + int exclusive + ) +{ + int rv; + struct cf_h2_ctx *ctx = cf->ctx; + nghttp2_priority_spec pri_spec; + + nghttp2_priority_spec_init(&pri_spec, dep_stream_id, weight, exclusive); + rv = nghttp2_submit_priority(ctx->h2, NGHTTP2_FLAG_NONE, + stream_id, &pri_spec); + if(rv) { + failf(data, "nghttp2_submit_priority() failed: %s(%d)", + nghttp2_strerror(rv), rv); + return CURLE_HTTP2; + } + + return CURLE_OK; +} + +/* + * curl-impersonate: Firefox uses an elaborate scheme of http/2 streams to + * split the load for html/js/css/images. It builds a tree of streams with + * different weights (priorities) by default and communicates this to the + * server. Imitate that behavior. + */ +static CURLcode http2_set_stream_priorities(struct Curl_cfilter *cf, + struct Curl_easy *data) +{ + CURLcode result; + char *stream_delimiter = ","; + char *value_delimiter = ":"; + + if(!data->set.str[STRING_HTTP2_STREAMS]) + return CURLE_OK; + + char *tmp1 = strdup(data->set.str[STRING_HTTP2_STREAMS]); + char *end1; + char *stream = strtok_r(tmp1, stream_delimiter, &end1); + + while(stream != NULL) { + + char *tmp2 = strdup(stream); + char *end2; + + int32_t stream_id = atoi(strtok_r(tmp2, value_delimiter, &end2)); + int exclusive = atoi(strtok_r(NULL, value_delimiter, &end2)); + int32_t dep_stream_id = atoi(strtok_r(NULL, value_delimiter, &end2)); + int32_t weight = atoi(strtok_r(NULL, value_delimiter, &end2)); + + free(tmp2); + + result = http2_set_stream_priority(cf, data, stream_id, dep_stream_id, weight, exclusive); + if(result) { + free(tmp1); + return result; + } + + stream = strtok_r(NULL, stream_delimiter, &end1); + } + + free(tmp1); + return CURLE_OK; +} + static CURLcode h2_progress_egress(struct Curl_cfilter *cf, struct Curl_easy *data); @@ -588,6 +658,16 @@ static CURLcode cf_h2_ctx_init(struct Curl_cfilter *cf, goto out; } + // curl-impersonate: set stream priorities + result = http2_set_stream_priorities(cf, data); + if(result) + goto out; + +#define FIREFOX_DEFAULT_STREAM_ID (15) + /* Best effort to set the request's stream id to 15, like Firefox does. */ + // Let's ignore this for now, as it seems not to be targeted as fingerprints. + // nghttp2_session_set_next_stream_id(ctx->h2, FIREFOX_DEFAULT_STREAM_ID); + /* all set, traffic will be send on connect */ result = CURLE_OK; CURL_TRC_CF(data, cf, "[0] created h2 session%s", @@ -1827,12 +1907,14 @@ static ssize_t http2_handle_stream_close(struct Curl_cfilter *cf, * instead of NGINX default stream weight. */ #define CHROME_DEFAULT_STREAM_WEIGHT (256) +#define FIREFOX_DEFAULT_STREAM_WEIGHT (42) static int sweight_wanted(const struct Curl_easy *data) { /* 0 weight is not set by user and we take the nghttp2 default one */ return data->set.priority.weight? data->set.priority.weight : CHROME_DEFAULT_STREAM_WEIGHT; + // data->set.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT; } static int sweight_in_effect(const struct Curl_easy *data) @@ -1840,6 +1922,7 @@ static int sweight_in_effect(const struct Curl_easy *data) /* 0 weight is not set by user and we take the nghttp2 default one */ return data->state.priority.weight? data->state.priority.weight : NGHTTP2_DEFAULT_WEIGHT; + // data->state.priority.weight : FIREFOX_DEFAULT_STREAM_WEIGHT; } /* @@ -1848,12 +1931,19 @@ static int sweight_in_effect(const struct Curl_easy *data) * struct. */ +/* + * curl-impersonate: By default Firefox uses stream 13 as the "parent" of the + * stream that fetches the main html resource of the web page. + */ +#define FIREFOX_DEFAULT_STREAM_DEP (13) + static void h2_pri_spec(struct Curl_easy *data, nghttp2_priority_spec *pri_spec) { struct Curl_data_priority *prio = &data->set.priority; struct stream_ctx *depstream = H2_STREAM_CTX(prio->parent); int32_t depstream_id = depstream? depstream->id:0; + // int32_t depstream_id = depstream? depstream->id:FIREFOX_DEFAULT_STREAM_DEP; /* curl-impersonate: Set stream exclusive flag to true. */ int exclusive = 1; nghttp2_priority_spec_init(pri_spec, depstream_id, diff --git a/lib/impersonate.c b/lib/impersonate.c index 00a2ba9c397e..1b6eeb9e9de4 100644 --- a/lib/impersonate.c +++ b/lib/impersonate.c @@ -738,6 +738,45 @@ const struct impersonate_opts impersonations[] = { .http2_window_update = 10485760, .http2_pseudo_headers_order = "mspa" }, + { + .target = "firefox120", + .httpversion = CURL_HTTP_VERSION_2_0, + .ssl_version = CURL_SSLVERSION_TLSv1_2 | CURL_SSLVERSION_MAX_DEFAULT, + .ciphers = + "TLS_AES_128_GCM_SHA256," + "TLS_CHACHA20_POLY1305_SHA256," + "TLS_AES_256_GCM_SHA384," + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256," + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384," + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384," + "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA," + "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA," + "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA," + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA," + "TLS_RSA_WITH_AES_128_GCM_SHA256," + "TLS_RSA_WITH_AES_256_GCM_SHA384," + "TLS_RSA_WITH_AES_128_CBC_SHA," + "TLS_RSA_WITH_AES_256_CBC_SHA", + .http_headers = { + "User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.5", + "Accept-Encoding: gzip, deflate, br", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1", + "Te: trailers" + }, + .http2_settings = "1:65536;4:131072;5:16384", + .http2_window_update = 12517377, + .http2_pseudo_headers_order = "mpas", + .http2_streams = "3:0:0:201,5:0:0:101,7:0:0:1,9:0:7:1,11:0:3:1,13:0:0:241" + }, { /* Last one must be NULL. */ .target = NULL diff --git a/lib/impersonate.h b/lib/impersonate.h index 0158d5477a9c..e0e92fc3c736 100644 --- a/lib/impersonate.h +++ b/lib/impersonate.h @@ -32,6 +32,7 @@ struct impersonate_opts { const char *http2_pseudo_headers_order; const char *http2_settings; int http2_window_update; + const char *http2_streams; bool tls_permute_extensions; const char *ech; /* Other TLS options will come here in the future once they are diff --git a/lib/setopt.c b/lib/setopt.c index fe468ca87785..f64a89f983ba 100644 --- a/lib/setopt.c +++ b/lib/setopt.c @@ -2999,6 +2999,10 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param) return CURLE_BAD_FUNCTION_ARGUMENT; data->set.http2_window_update = arg; break; + case CURLOPT_HTTP2_STREAMS: + result = Curl_setstropt(&data->set.str[STRING_HTTP2_STREAMS], + va_arg(param, char *)); + break; #endif #ifdef USE_UNIX_SOCKETS case CURLOPT_UNIX_SOCKET_PATH: diff --git a/lib/urldata.h b/lib/urldata.h index b78408094b23..dc95784b0013 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1659,6 +1659,7 @@ enum dupstring { STRING_SSL_CERT_COMPRESSION, STRING_HTTP2_PSEUDO_HEADERS_ORDER, STRING_HTTP2_SETTINGS, + STRING_HTTP2_STREAMS, STRING_ECH_CONFIG, /* CURLOPT_ECH_CONFIG */ STRING_ECH_PUBLIC, /* CURLOPT_ECH_PUBLIC */ diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 5174bfaa7786..ea51bff00749 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -168,6 +168,7 @@ struct OperationConfig { char *http2_pseudo_headers_order; char *http2_settings; long http2_window_update; + char *http2_streams; long httpversion; bool http09_allowed; bool nobuffer; diff --git a/src/tool_getparam.c b/src/tool_getparam.c index a5df5601a357..199f5746b69c 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -307,6 +307,7 @@ static const struct LongShort aliases[]= { #ifdef USE_ECH {"ER", "ech", ARG_STRING}, #endif + {"EV", "http2-streams", ARG_STRING}, {"f", "fail", ARG_BOOL}, {"fa", "fail-early", ARG_BOOL}, {"fb", "styled-output", ARG_BOOL}, @@ -2179,6 +2180,11 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ return PARAM_BAD_NUMERIC; break; + case 'V': + /* --http2-streams */ + GetStr(&config->http2_streams, nextarg); + break; + #ifdef USE_ECH case 'R': if(strlen(nextarg) != 6 || !strncasecompare("GREASE", nextarg, 6)) { diff --git a/src/tool_operate.c b/src/tool_operate.c index 81ba306709ff..01743da1c3d5 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -1537,6 +1537,10 @@ static CURLcode single_transfer(struct GlobalConfig *global, CURLOPT_HTTP2_WINDOW_UPDATE, config->http2_window_update); + if(config->http2_streams) + my_setopt_str(curl, + CURLOPT_HTTP2_STREAMS, + config->http2_streams); } /* (proto_http) */