diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74fbae5ee..fd14e28b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,8 +72,10 @@ jobs: - "feat.: rustls-tls" - "feat.: rustls-tls-manual-roots" - "feat.: rustls-tls-native-roots" + - "feat.: rustls-tls-no-provider" - "feat.: native-tls" - "feat.: default-tls and rustls-tls" + - "feat.: rustls-tls and rustls-tls-no-provider" - "feat.: cookies" - "feat.: blocking" - "feat.: blocking only" @@ -131,8 +133,12 @@ jobs: features: "--no-default-features --features rustls-tls-manual-roots" - name: "feat.: rustls-tls-native-roots" features: "--no-default-features --features rustls-tls-native-roots" + - name: "feat.: rustls-tls-no-provider" + features: "--no-default-features --features rustls-tls-no-provider" - name: "feat.: native-tls" features: "--features native-tls" + - name: "feat.: rustls-tls and rustls-tls-no-provider" + features: "--features rustls-tls,rustls-tls-no-provider" - name: "feat.: default-tls and rustls-tls" features: "--features rustls-tls" - name: "feat.: cookies" @@ -212,8 +218,11 @@ jobs: with: toolchain: 'stable' - #- name: Check - # run: RUSTFLAGS="--cfg reqwest_unstable" cargo check --features http3 + - name: Check + run: cargo test --features http3 + env: + RUSTFLAGS: --cfg reqwest_unstable + RUSTDOCFLAGS: --cfg reqwest_unstable docs: name: Docs @@ -263,7 +272,7 @@ jobs: run: | cargo clean cargo update -Z minimal-versions - cargo update -p proc-macro2 --precise 1.0.60 + cargo update -p proc-macro2 --precise 1.0.62 cargo check cargo check --all-features @@ -291,6 +300,7 @@ jobs: cargo update cargo update -p log --precise 0.4.18 cargo update -p tokio --precise 1.29.1 + cargo update -p url --precise 2.5.0 - uses: Swatinem/rust-cache@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dea63e5a..1e6ecd9d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## v0.12.7 + +- Revert adding `impl Service>` for `Client`. + +## v0.12.6 + +- Add support for `danger_accept_invalid_hostnames` for `rustls`. +- Add `impl Service>` for `Client` and `&'_ Client`. +- Add support for `!Sync` bodies in `Body::wrap_stream()`. +- Enable happy eyeballs when `hickory-dns` is used. +- Fix `Proxy` so that `HTTP(S)_PROXY` values take precendence over `ALL_PROXY`. +- Fix `blocking::RequestBuilder::header()` from unsetting `sensitive` on passed header values. + +## v0.12.5 + +- Add `blocking::ClientBuilder::dns_resolver()` method to change DNS resolver in blocking client. +- Add `http3` feature back, still requiring `reqwest_unstable`. +- Add `rustls-tls-no-provider` Cargo feature to use rustls without a crypto provider. +- Fix `Accept-Encoding` header combinations. +- Fix http3 resolving IPv6 addresses. +- Internal: upgrade to rustls 0.23. + ## v0.12.4 - Add `zstd` support, enabled with `zstd` Cargo feature. diff --git a/Cargo.toml b/Cargo.toml index e0c4cab97..044757de8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "reqwest" -version = "0.12.4" +version = "0.12.7" description = "higher level HTTP client library" keywords = ["http", "request", "client"] categories = ["web-programming::http-client", "wasm"] @@ -33,7 +33,7 @@ default = ["default-tls", "charset", "http2", "macos-system-configuration"] # functionality for it. default-tls = ["dep:hyper-tls", "dep:native-tls-crate", "__tls", "dep:tokio-native-tls"] -http2 = ["h2", "hyper/http2", "hyper-util/http2"] +http2 = ["h2", "hyper/http2", "hyper-util/http2", "hyper-rustls?/http2"] # Enables native-tls specific functionality not available by default. native-tls = ["default-tls"] @@ -41,11 +41,14 @@ native-tls-alpn = ["native-tls", "native-tls-crate?/alpn", "hyper-tls?/alpn"] native-tls-vendored = ["native-tls", "native-tls-crate?/vendored"] rustls-tls = ["rustls-tls-webpki-roots"] -rustls-tls-manual-roots = ["__rustls"] -rustls-tls-webpki-roots = ["dep:webpki-roots", "__rustls"] -rustls-tls-native-roots = ["dep:rustls-native-certs", "__rustls"] +rustls-tls-no-provider = ["rustls-tls-manual-roots-no-provider"] -blocking = ["futures-channel/sink", "futures-util/io", "futures-util/sink", "tokio/rt-multi-thread", "tokio/sync"] +rustls-tls-manual-roots = ["__rustls", "__rustls-ring"] +rustls-tls-webpki-roots = ["dep:webpki-roots", "hyper-rustls?/webpki-tokio", "__rustls", "__rustls-ring"] +rustls-tls-native-roots = ["dep:rustls-native-certs", "hyper-rustls?/native-tokio", "__rustls", "__rustls-ring"] +rustls-tls-manual-roots-no-provider = ["__rustls"] + +blocking = ["dep:futures-channel", "futures-channel?/sink", "futures-util/io", "futures-util/sink", "tokio/sync"] charset = ["dep:encoding_rs"] @@ -75,18 +78,19 @@ socks = ["dep:tokio-socks"] macos-system-configuration = ["dep:system-configuration"] # Experimental HTTP/3 client. -# Disabled while waiting for quinn to upgrade. -#http3 = ["rustls-tls-manual-roots", "dep:h3", "dep:h3-quinn", "dep:quinn", "dep:futures-channel"] +http3 = ["rustls-tls-manual-roots", "dep:h3", "dep:h3-quinn", "dep:quinn", "dep:slab", "dep:futures-channel"] + # Internal (PRIVATE!) features used to aid testing. -# Don't rely on these whatsoever. They may disappear at anytime. +# Don't rely on these whatsoever. They may disappear at any time. # Enables common types used for TLS. Useless on its own. __tls = ["dep:rustls-pemfile", "tokio/io-util"] # Enables common rustls code. # Equivalent to rustls-tls-manual-roots but shorter :) -__rustls = ["dep:hyper-rustls", "dep:tokio-rustls", "dep:rustls", "__tls", "dep:rustls-pemfile", "rustls-pki-types"] +__rustls = ["dep:hyper-rustls", "dep:tokio-rustls", "dep:rustls", "__tls", "dep:rustls-pemfile", "dep:rustls-pki-types"] +__rustls-ring = ["hyper-rustls?/ring", "tokio-rustls?/ring", "rustls?/ring", "quinn?/ring"] # When enabled, disable using the cached SYS_PROXIES. __internal_proxy_sys_no_cache = [] @@ -94,14 +98,14 @@ __internal_proxy_sys_no_cache = [] [dependencies] base64 = "0.22" http = "1" -url = "2.2" +url = "2.4" bytes = "1.0" serde = "1.0" serde_urlencoded = "0.7.1" tower-service = "0.3" -futures-core = { version = "0.3.0", default-features = false } -futures-util = { version = "0.3.0", default-features = false } -sync_wrapper = "0.1.2" +futures-core = { version = "0.3.28", default-features = false } +futures-util = { version = "0.3.28", default-features = false } +sync_wrapper = { version = "1.0", features = ["futures"] } # Optional deps... @@ -114,15 +118,15 @@ mime_guess = { version = "2.0", default-features = false, optional = true } encoding_rs = { version = "0.8", optional = true } http-body = "1" http-body-util = "0.1" -hyper = { version = "1", features = ["http1", "client"] } +hyper = { version = "1.1", features = ["http1", "client"] } hyper-util = { version = "0.1.3", features = ["http1", "client", "client-legacy", "tokio"] } h2 = { version = "0.4", optional = true } -once_cell = "1" -log = "0.4" +once_cell = "1.18" +log = "0.4.17" mime = "0.3.16" -percent-encoding = "2.1" +percent-encoding = "2.3" tokio = { version = "1.0", default-features = false, features = ["net", "time"] } -pin-project-lite = "0.2.0" +pin-project-lite = "0.2.11" ipnet = "2.3" # Optional deps... @@ -134,20 +138,20 @@ native-tls-crate = { version = "0.2.10", optional = true, package = "native-tls" tokio-native-tls = { version = "0.3.0", optional = true } # rustls-tls -hyper-rustls = { version = "0.26.0", default-features = false, optional = true } -rustls = { version = "0.22.2", optional = true } +hyper-rustls = { version = "0.27.0", default-features = false, optional = true, features = ["http1", "tls12"] } +rustls = { version = "0.23.4", optional = true, default-features = false, features = ["std", "tls12"] } rustls-pki-types = { version = "1.1.0", features = ["alloc"] ,optional = true } -tokio-rustls = { version = "0.25", optional = true } +tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["tls12"] } webpki-roots = { version = "0.26.0", optional = true } rustls-native-certs = { version = "0.7", optional = true } ## cookies -cookie_crate = { version = "0.17.0", package = "cookie", optional = true } -cookie_store = { version = "0.20.0", optional = true } +cookie_crate = { version = "0.18.0", package = "cookie", optional = true } +cookie_store = { version = "0.21.0", optional = true } ## compression async-compression = { version = "0.4.0", default-features = false, features = ["tokio"], optional = true } -tokio-util = { version = "0.7.1", default-features = false, features = ["codec", "io"], optional = true } +tokio-util = { version = "0.7.9", default-features = false, features = ["codec", "io"], optional = true } ## socks tokio-socks = { version = "0.5.1", optional = true } @@ -156,41 +160,43 @@ tokio-socks = { version = "0.5.1", optional = true } hickory-resolver = { version = "0.24", optional = true, features = ["tokio-runtime"] } # HTTP/3 experimental support -h3 = { version = "0.0.4", optional = true } -h3-quinn = { version = "0.0.5", optional = true } -quinn = { version = "0.10", default-features = false, features = ["tls-rustls", "ring", "runtime-tokio"], optional = true } +h3 = { version = "0.0.6", optional = true } +h3-quinn = { version = "0.0.7", optional = true } +quinn = { version = "0.11.1", default-features = false, features = ["rustls", "runtime-tokio"], optional = true } +slab = { version = "0.4.9", optional = true } # just to get minimal versions working with quinn futures-channel = { version = "0.3", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] env_logger = "0.10" hyper = { version = "1.1.0", default-features = false, features = ["http1", "http2", "client", "server"] } -hyper-util = { version = "0.1", features = ["http1", "http2", "client", "client-legacy", "server-auto", "tokio"] } +hyper-util = { version = "0.1.3", features = ["http1", "http2", "client", "client-legacy", "server-auto", "tokio"] } serde = { version = "1.0", features = ["derive"] } -libflate = "1.0" -brotli_crate = { package = "brotli", version = "3.3.0" } +libflate = "2.1" +brotli_crate = { package = "brotli", version = "6.0.0" } zstd_crate = { package = "zstd", version = "0.13" } doc-comment = "0.3" tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread"] } -futures-util = { version = "0.3.0", default-features = false, features = ["std", "alloc"] } +futures-util = { version = "0.3.28", default-features = false, features = ["std", "alloc"] } +rustls = { version = "0.23", default-features = false, features = ["ring"] } [target.'cfg(windows)'.dependencies] -winreg = "0.52.0" +windows-registry = "0.2" [target.'cfg(target_os = "macos")'.dependencies] -system-configuration = { version = "0.5.1", optional = true } +system-configuration = { version = "0.6.0", optional = true } # wasm [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.45" serde_json = "1.0" -wasm-bindgen = "0.2.68" +wasm-bindgen = "0.2.89" wasm-bindgen-futures = "0.4.18" wasm-streams = { version = "0.4", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies.web-sys] -version = "0.3.25" +version = "0.3.28" features = [ "AbortController", "AbortSignal", @@ -210,9 +216,12 @@ features = [ ] [target.'cfg(target_arch = "wasm32")'.dev-dependencies] -wasm-bindgen = { version = "0.2.68", features = ["serde-serialize"] } +wasm-bindgen = { version = "0.2.89", features = ["serde-serialize"] } wasm-bindgen-test = "0.3" +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(reqwest_unstable)'] } + [[example]] name = "blocking" path = "examples/blocking.rs" diff --git a/src/async_impl/body.rs b/src/async_impl/body.rs index e44fc4faf..d545a0d19 100644 --- a/src/async_impl/body.rs +++ b/src/async_impl/body.rs @@ -48,6 +48,7 @@ pin_project! { } /// Converts any `impl Body` into a `impl Stream` of just its DATA frames. +#[cfg(any(feature = "stream", feature = "multipart",))] pub(crate) struct DataStream(pub(crate) B); impl Body { @@ -88,7 +89,7 @@ impl Body { #[cfg_attr(docsrs, doc(cfg(feature = "stream")))] pub fn wrap_stream(stream: S) -> Body where - S: futures_core::stream::TryStream + Send + Sync + 'static, + S: futures_core::stream::TryStream + Send + 'static, S::Error: Into>, Bytes: From, { @@ -98,7 +99,7 @@ impl Body { #[cfg(any(feature = "stream", feature = "multipart", feature = "blocking"))] pub(crate) fn stream(stream: S) -> Body where - S: futures_core::stream::TryStream + Send + Sync + 'static, + S: futures_core::stream::TryStream + Send + 'static, S::Error: Into>, Bytes: From, { @@ -106,11 +107,11 @@ impl Body { use http_body::Frame; use http_body_util::StreamBody; - let body = http_body_util::BodyExt::boxed(StreamBody::new( + let body = http_body_util::BodyExt::boxed(StreamBody::new(sync_wrapper::SyncStream::new( stream .map_ok(|d| Frame::data(Bytes::from(d))) .map_err(Into::into), - )); + ))); Body { inner: Inner::Streaming(body), } @@ -394,11 +395,25 @@ where pub(crate) type ResponseBody = http_body_util::combinators::BoxBody>; -pub(crate) fn response( - body: hyper::body::Incoming, +pub(crate) fn boxed(body: B) -> ResponseBody +where + B: hyper::body::Body + Send + Sync + 'static, + B::Error: Into>, +{ + use http_body_util::BodyExt; + + body.map_err(box_err).boxed() +} + +pub(crate) fn response( + body: B, deadline: Option>>, read_timeout: Option, -) -> ResponseBody { +) -> ResponseBody +where + B: hyper::body::Body + Send + Sync + 'static, + B::Error: Into>, +{ use http_body_util::BodyExt; match (deadline, read_timeout) { @@ -421,6 +436,7 @@ where // ===== impl DataStream ===== +#[cfg(any(feature = "stream", feature = "multipart",))] impl futures_core::Stream for DataStream where B: HttpBody + Unpin, diff --git a/src/async_impl/client.rs b/src/async_impl/client.rs index 22519f535..0a7281c42 100644 --- a/src/async_impl/client.rs +++ b/src/async_impl/client.rs @@ -91,7 +91,7 @@ struct Config { // NOTE: When adding a new field, update `fmt::Debug for ClientBuilder` accepts: Accepts, headers: HeaderMap, - #[cfg(feature = "native-tls")] + #[cfg(feature = "__tls")] hostname_verification: bool, #[cfg(feature = "__tls")] certs_verification: bool, @@ -188,7 +188,7 @@ impl ClientBuilder { error: None, accepts: Accepts::default(), headers, - #[cfg(feature = "native-tls")] + #[cfg(feature = "__tls")] hostname_verification: true, #[cfg(feature = "__tls")] certs_verification: true, @@ -388,10 +388,7 @@ impl ClientBuilder { } } - #[cfg(feature = "native-tls")] - { - tls.danger_accept_invalid_hostnames(!config.hostname_verification); - } + tls.danger_accept_invalid_hostnames(!config.hostname_verification); tls.danger_accept_invalid_certs(!config.certs_verification); @@ -500,7 +497,7 @@ impl ClientBuilder { } #[cfg(feature = "__rustls")] TlsBackend::Rustls => { - use crate::tls::NoVerifier; + use crate::tls::{IgnoreHostname, NoVerifier}; // Set root certificates. let mut root_cert_store = rustls::RootCertStore::empty(); @@ -565,10 +562,38 @@ impl ClientBuilder { return Err(crate::error::builder("empty supported tls versions")); } + // Allow user to have installed a runtime default. + // If not, we use ring. + let provider = rustls::crypto::CryptoProvider::get_default() + .map(|arc| arc.clone()) + .unwrap_or_else(|| { + #[cfg(not(feature = "__rustls-ring"))] + panic!("No provider set"); + + #[cfg(feature = "__rustls-ring")] + Arc::new(rustls::crypto::ring::default_provider()) + }); + // Build TLS config - let config_builder = - rustls::ClientConfig::builder_with_protocol_versions(&versions) - .with_root_certificates(root_cert_store); + let signature_algorithms = provider.signature_verification_algorithms; + let config_builder = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&versions) + .map_err(|_| crate::error::builder("invalid TLS versions"))?; + + let config_builder = if !config.certs_verification { + config_builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(NoVerifier)) + } else if !config.hostname_verification { + config_builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(IgnoreHostname::new( + root_cert_store, + signature_algorithms, + ))) + } else { + config_builder.with_root_certificates(root_cert_store) + }; // Finalize TLS config let mut tls = if let Some(id) = config.identity { @@ -577,12 +602,6 @@ impl ClientBuilder { config_builder.with_no_client_auth() }; - // Certificate verifier - if !config.certs_verification { - tls.dangerous() - .set_certificate_verifier(Arc::new(NoVerifier)); - } - tls.enable_sni = config.tls_sni; // ALPN protocol @@ -1305,6 +1324,8 @@ impl ClientBuilder { /// # Example /// /// ``` + /// # #[cfg(all(feature = "__rustls", not(feature = "__rustls-ring")))] + /// # let _ = rustls::crypto::ring::default_provider().install_default(); /// use std::net::IpAddr; /// let local_addr = IpAddr::from([12, 4, 1, 8]); /// let client = reqwest::Client::builder() @@ -1324,6 +1345,8 @@ impl ClientBuilder { /// # Example /// /// ``` + /// # #[cfg(all(feature = "__rustls", not(feature = "__rustls-ring")))] + /// # let _ = rustls::crypto::ring::default_provider().install_default(); /// let interface = "lo"; /// let client = reqwest::Client::builder() /// .interface(interface) @@ -1459,9 +1482,17 @@ impl ClientBuilder { /// /// # Optional /// - /// This requires the optional `native-tls` feature to be enabled. - #[cfg(feature = "native-tls")] - #[cfg_attr(docsrs, doc(cfg(feature = "native-tls")))] + /// This requires the optional `default-tls`, `native-tls`, or `rustls-tls(-...)` + /// feature to be enabled. + #[cfg(feature = "__tls")] + #[cfg_attr( + docsrs, + doc(cfg(any( + feature = "default-tls", + feature = "native-tls", + feature = "rustls-tls" + ))) + )] pub fn danger_accept_invalid_hostnames( mut self, accept_invalid_hostname: bool, @@ -1689,14 +1720,7 @@ impl ClientBuilder { self } - /// Enables the [hickory-dns](hickory_resolver) async resolver instead of a default threadpool - /// using `getaddrinfo`. - /// - /// If the `hickory-dns` feature is turned on, the default option is enabled. - /// - /// # Optional - /// - /// This requires the optional `hickory-dns` feature to be enabled + #[doc(hidden)] #[cfg(feature = "hickory-dns")] #[cfg_attr(docsrs, doc(cfg(feature = "hickory-dns")))] #[deprecated(note = "use `hickory_dns` instead")] @@ -1713,6 +1737,11 @@ impl ClientBuilder { /// # Optional /// /// This requires the optional `hickory-dns` feature to be enabled + /// + /// # Warning + /// + /// The hickory resolver does not work exactly the same, or on all the platforms + /// that the default resolver does #[cfg(feature = "hickory-dns")] #[cfg_attr(docsrs, doc(cfg(feature = "hickory-dns")))] pub fn hickory_dns(mut self, enable: bool) -> ClientBuilder { @@ -1720,22 +1749,10 @@ impl ClientBuilder { self } - /// Disables the hickory-dns async resolver. - /// - /// This method exists even if the optional `hickory-dns` feature is not enabled. - /// This can be used to ensure a `Client` doesn't use the hickory-dns async resolver - /// even if another dependency were to enable the optional `hickory-dns` feature. + #[doc(hidden)] #[deprecated(note = "use `no_hickory_dns` instead")] pub fn no_trust_dns(self) -> ClientBuilder { - #[cfg(feature = "hickory-dns")] - { - self.hickory_dns(false) - } - - #[cfg(not(feature = "hickory-dns"))] - { - self - } + self.no_hickory_dns() } /// Disables the hickory-dns async resolver. @@ -1798,7 +1815,7 @@ impl ClientBuilder { /// The default is false. #[cfg(feature = "http3")] #[cfg_attr(docsrs, doc(cfg(all(reqwest_unstable, feature = "http3",))))] - pub fn set_tls_enable_early_data(mut self, enabled: bool) -> ClientBuilder { + pub fn tls_early_data(mut self, enabled: bool) -> ClientBuilder { self.config.tls_enable_early_data = enabled; self } @@ -1810,7 +1827,7 @@ impl ClientBuilder { /// [`TransportConfig`]: https://docs.rs/quinn/latest/quinn/struct.TransportConfig.html #[cfg(feature = "http3")] #[cfg_attr(docsrs, doc(cfg(all(reqwest_unstable, feature = "http3",))))] - pub fn set_quic_max_idle_timeout(mut self, value: Duration) -> ClientBuilder { + pub fn http3_max_idle_timeout(mut self, value: Duration) -> ClientBuilder { self.config.quic_max_idle_timeout = Some(value); self } @@ -1821,10 +1838,14 @@ impl ClientBuilder { /// Please see docs in [`TransportConfig`] in [`quinn`]. /// /// [`TransportConfig`]: https://docs.rs/quinn/latest/quinn/struct.TransportConfig.html + /// + /// # Panics + /// + /// Panics if the value is over 2^62. #[cfg(feature = "http3")] #[cfg_attr(docsrs, doc(cfg(all(reqwest_unstable, feature = "http3",))))] - pub fn set_quic_stream_receive_window(mut self, value: VarInt) -> ClientBuilder { - self.config.quic_stream_receive_window = Some(value); + pub fn http3_stream_receive_window(mut self, value: u64) -> ClientBuilder { + self.config.quic_stream_receive_window = Some(value.try_into().unwrap()); self } @@ -1834,10 +1855,14 @@ impl ClientBuilder { /// Please see docs in [`TransportConfig`] in [`quinn`]. /// /// [`TransportConfig`]: https://docs.rs/quinn/latest/quinn/struct.TransportConfig.html + /// + /// # Panics + /// + /// Panics if the value is over 2^62. #[cfg(feature = "http3")] #[cfg_attr(docsrs, doc(cfg(all(reqwest_unstable, feature = "http3",))))] - pub fn set_quic_receive_window(mut self, value: VarInt) -> ClientBuilder { - self.config.quic_receive_window = Some(value); + pub fn http3_conn_receive_window(mut self, value: u64) -> ClientBuilder { + self.config.quic_receive_window = Some(value.try_into().unwrap()); self } @@ -1848,7 +1873,7 @@ impl ClientBuilder { /// [`TransportConfig`]: https://docs.rs/quinn/latest/quinn/struct.TransportConfig.html #[cfg(feature = "http3")] #[cfg_attr(docsrs, doc(cfg(all(reqwest_unstable, feature = "http3",))))] - pub fn set_quic_send_window(mut self, value: u64) -> ClientBuilder { + pub fn http3_send_window(mut self, value: u64) -> ClientBuilder { self.config.quic_send_window = Some(value); self } @@ -2218,7 +2243,7 @@ impl Config { f.field("tcp_nodelay", &true); } - #[cfg(feature = "native-tls")] + #[cfg(feature = "__tls")] { if !self.hostname_verification { f.field("danger_accept_invalid_hostnames", &true); @@ -2379,7 +2404,7 @@ impl PendingRequest { self.project().headers } - #[cfg(feature = "http2")] + #[cfg(any(feature = "http2", feature = "http3"))] fn retry_error(mut self: Pin<&mut Self>, err: &(dyn std::error::Error + 'static)) -> bool { use log::trace; @@ -2439,7 +2464,7 @@ impl PendingRequest { } } -#[cfg(feature = "http2")] +#[cfg(any(feature = "http2", feature = "http3"))] fn is_retryable_error(err: &(dyn std::error::Error + 'static)) -> bool { // pop the legacy::Error let err = if let Some(err) = err.source() { @@ -2457,6 +2482,7 @@ fn is_retryable_error(err: &(dyn std::error::Error + 'static)) -> bool { } } + #[cfg(feature = "http2")] if let Some(cause) = err.source() { if let Some(err) = cause.downcast_ref::() { // They sent us a graceful shutdown, try with a new connection! @@ -2533,7 +2559,7 @@ impl Future for PendingRequest { crate::error::request(e).with_url(self.url.clone()) )); } - Poll::Ready(Ok(res)) => res, + Poll::Ready(Ok(res)) => res.map(super::body::boxed), Poll::Pending => return Poll::Pending, }, #[cfg(feature = "http3")] @@ -2747,6 +2773,8 @@ fn add_cookie_header(headers: &mut HeaderMap, cookie_store: &dyn cookie::CookieS #[cfg(test)] mod tests { + #![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] + #[tokio::test] async fn execute_request_rejects_invalid_urls() { let url_str = "hxxps://www.rust-lang.org/"; diff --git a/src/async_impl/decoder.rs b/src/async_impl/decoder.rs index 4e05428ae..d742e6d35 100644 --- a/src/async_impl/decoder.rs +++ b/src/async_impl/decoder.rs @@ -1,4 +1,10 @@ use std::fmt; +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate" +))] use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; @@ -15,9 +21,16 @@ use async_compression::tokio::bufread::ZstdDecoder; #[cfg(feature = "deflate")] use async_compression::tokio::bufread::ZlibDecoder; -use bytes::Bytes; +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate", + feature = "blocking", +))] use futures_core::Stream; -use futures_util::stream::Peekable; + +use bytes::Bytes; use http::HeaderMap; use hyper::body::Body as HttpBody; use hyper::body::Frame; @@ -38,7 +51,6 @@ use tokio_util::codec::{BytesCodec, FramedRead}; use tokio_util::io::StreamReader; use super::body::ResponseBody; -use crate::error; #[derive(Clone, Copy, Debug)] pub(super) struct Accepts { @@ -74,7 +86,13 @@ pub(crate) struct Decoder { inner: Inner, } -type PeekableIoStream = Peekable; +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate" +))] +type PeekableIoStream = futures_util::stream::Peekable; #[cfg(any( feature = "gzip", @@ -114,11 +132,30 @@ enum Inner { Pending(Pin>), } +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate" +))] /// A future attempt to poll the response body for EOF so we know whether to use gzip or not. struct Pending(PeekableIoStream, DecoderType); +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate", + feature = "blocking", +))] pub(crate) struct IoStream(B); +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate" +))] enum DecoderType { #[cfg(feature = "gzip")] Gzip, @@ -376,11 +413,24 @@ impl HttpBody for Decoder { } } +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate", + feature = "blocking", +))] fn empty() -> ResponseBody { use http_body_util::{combinators::BoxBody, BodyExt, Empty}; BoxBody::new(Empty::new().map_err(|never| match never {})) } +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate" +))] impl Future for Pending { type Output = Result; @@ -429,6 +479,13 @@ impl Future for Pending { } } +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate", + feature = "blocking", +))] impl Stream for IoStream where B: HttpBody + Unpin, @@ -447,7 +504,7 @@ where continue; } } - Some(Err(err)) => Poll::Ready(Some(Err(error::into_io(err.into())))), + Some(Err(err)) => Poll::Ready(Some(Err(crate::error::into_io(err.into())))), None => Poll::Ready(None), }; } @@ -484,9 +541,9 @@ impl Accepts { (true, true, true, false) => Some("gzip, br, zstd"), (true, true, false, false) => Some("gzip, br"), (true, false, true, true) => Some("gzip, zstd, deflate"), - (true, false, false, true) => Some("gzip, zstd, deflate"), + (true, false, false, true) => Some("gzip, deflate"), (false, true, true, true) => Some("br, zstd, deflate"), - (false, true, false, true) => Some("br, zstd, deflate"), + (false, true, false, true) => Some("br, deflate"), (true, false, true, false) => Some("gzip, zstd"), (true, false, false, false) => Some("gzip"), (false, true, true, false) => Some("br, zstd"), @@ -561,3 +618,57 @@ impl Default for Accepts { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accepts_as_str() { + fn format_accept_encoding(accepts: &Accepts) -> String { + let mut encodings = vec![]; + if accepts.is_gzip() { + encodings.push("gzip"); + } + if accepts.is_brotli() { + encodings.push("br"); + } + if accepts.is_zstd() { + encodings.push("zstd"); + } + if accepts.is_deflate() { + encodings.push("deflate"); + } + encodings.join(", ") + } + + let state = [true, false]; + let mut permutations = Vec::new(); + + #[allow(unused_variables)] + for gzip in state { + for brotli in state { + for zstd in state { + for deflate in state { + permutations.push(Accepts { + #[cfg(feature = "gzip")] + gzip, + #[cfg(feature = "brotli")] + brotli, + #[cfg(feature = "zstd")] + zstd, + #[cfg(feature = "deflate")] + deflate, + }); + } + } + } + } + + for accepts in permutations { + let expected = format_accept_encoding(&accepts); + let got = accepts.as_str().unwrap_or(""); + assert_eq!(got, expected.as_str()); + } + } +} diff --git a/src/async_impl/h3_client/connect.rs b/src/async_impl/h3_client/connect.rs index ec732f66a..2cce1cf47 100644 --- a/src/async_impl/h3_client/connect.rs +++ b/src/async_impl/h3_client/connect.rs @@ -6,6 +6,7 @@ use h3::client::SendRequest; use h3_quinn::{Connection, OpenStreams}; use http::Uri; use hyper_util::client::legacy::connect::dns::Name; +use quinn::crypto::rustls::QuicClientConfig; use quinn::{ClientConfig, Endpoint, TransportConfig}; use std::net::{IpAddr, SocketAddr}; use std::str::FromStr; @@ -29,7 +30,8 @@ impl H3Connector { local_addr: Option, transport_config: TransportConfig, ) -> Result { - let mut config = ClientConfig::new(Arc::new(tls)); + let quic_client_config = Arc::new(QuicClientConfig::try_from(tls)?); + let mut config = ClientConfig::new(quic_client_config); // FIXME: Replace this when there is a setter. config.transport_config(Arc::new(transport_config)); @@ -45,7 +47,11 @@ impl H3Connector { } pub async fn connect(&mut self, dest: Uri) -> Result { - let host = dest.host().ok_or("destination must have a host")?; + let host = dest + .host() + .ok_or("destination must have a host")? + .trim_start_matches('[') + .trim_end_matches(']'); let port = dest.port_u16().unwrap_or(443); let addrs = if let Some(addr) = IpAddr::from_str(host).ok() { diff --git a/src/async_impl/h3_client/mod.rs b/src/async_impl/h3_client/mod.rs index ce877cdfe..aa1bc4422 100644 --- a/src/async_impl/h3_client/mod.rs +++ b/src/async_impl/h3_client/mod.rs @@ -4,13 +4,13 @@ pub(crate) mod connect; pub(crate) mod dns; mod pool; +use crate::async_impl::body::ResponseBody; use crate::async_impl::h3_client::pool::{Key, Pool, PoolClient}; use crate::error::{BoxError, Error, Kind}; use crate::{error, Body}; use connect::H3Connector; use futures_util::future; use http::{Request, Response}; -use hyper::Body as HyperBody; use log::trace; use std::future::Future; use std::pin::Pin; @@ -49,7 +49,7 @@ impl H3Client { mut self, key: Key, req: Request, - ) -> Result, Error> { + ) -> Result, Error> { let mut pooled = match self.get_pooled_client(key).await { Ok(client) => client, Err(e) => return Err(error::request(e)), @@ -76,11 +76,11 @@ impl H3Client { } pub(crate) struct H3ResponseFuture { - inner: Pin, Error>> + Send>>, + inner: Pin, Error>> + Send>>, } impl Future for H3ResponseFuture { - type Output = Result, Error>; + type Output = Result, Error>; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { self.inner.as_mut().poll(cx) diff --git a/src/async_impl/h3_client/pool.rs b/src/async_impl/h3_client/pool.rs index d6442c81a..100a0935d 100644 --- a/src/async_impl/h3_client/pool.rs +++ b/src/async_impl/h3_client/pool.rs @@ -5,6 +5,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::time::Instant; +use crate::async_impl::body::ResponseBody; use crate::error::{BoxError, Error, Kind}; use crate::Body; use bytes::Buf; @@ -13,7 +14,6 @@ use h3::client::SendRequest; use h3_quinn::{Connection, OpenStreams}; use http::uri::{Authority, Scheme}; use http::{Request, Response, Uri}; -use hyper::body as HyperBody; use log::trace; pub(super) type Key = (Scheme, Authority); @@ -125,9 +125,20 @@ impl PoolClient { pub async fn send_request( &mut self, req: Request, - ) -> Result, BoxError> { + ) -> Result, BoxError> { + use http_body_util::{BodyExt, Full}; + use hyper::body::Body as _; + let (head, req_body) = req.into_parts(); - let req = Request::from_parts(head, ()); + let mut req = Request::from_parts(head, ()); + + if let Some(n) = req_body.size_hint().exact() { + if n > 0 { + req.headers_mut() + .insert(http::header::CONTENT_LENGTH, n.into()); + } + } + let mut stream = self.inner.send_request(req).await?; match req_body.as_bytes() { @@ -146,7 +157,11 @@ impl PoolClient { resp_body.extend(chunk.chunk()) } - Ok(resp.map(|_| HyperBody::from(resp_body))) + let resp_body = Full::new(resp_body.into()) + .map_err(|never| match never {}) + .boxed(); + + Ok(resp.map(|_| resp_body)) } } diff --git a/src/async_impl/multipart.rs b/src/async_impl/multipart.rs index 75198ca0a..525876dde 100644 --- a/src/async_impl/multipart.rs +++ b/src/async_impl/multipart.rs @@ -3,9 +3,16 @@ use std::borrow::Cow; use std::fmt; use std::pin::Pin; +#[cfg(feature = "stream")] +use std::io; +#[cfg(feature = "stream")] +use std::path::Path; + use bytes::Bytes; use mime_guess::Mime; use percent_encoding::{self, AsciiSet, NON_ALPHANUMERIC}; +#[cfg(feature = "stream")] +use tokio::fs::File; use futures_core::Stream; use futures_util::{future, stream, StreamExt}; @@ -82,6 +89,33 @@ impl Form { self.part(name, Part::text(value)) } + /// Adds a file field. + /// + /// The path will be used to try to guess the filename and mime. + /// + /// # Examples + /// + /// ```no_run + /// # async fn run() -> std::io::Result<()> { + /// let form = reqwest::multipart::Form::new() + /// .file("key", "/path/to/file").await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// Errors when the file cannot be opened. + #[cfg(feature = "stream")] + #[cfg_attr(docsrs, doc(cfg(feature = "stream")))] + pub async fn file(self, name: T, path: U) -> io::Result
+ where + T: Into>, + U: AsRef, + { + Ok(self.part(name, Part::file(path).await?)) + } + /// Adds a customized Part. pub fn part(self, name: T, part: Part) -> Form where @@ -218,6 +252,30 @@ impl Part { Part::new(value.into(), Some(length)) } + /// Makes a file parameter. + /// + /// # Errors + /// + /// Errors when the file cannot be opened. + #[cfg(feature = "stream")] + #[cfg_attr(docsrs, doc(cfg(feature = "stream")))] + pub async fn file>(path: T) -> io::Result { + let path = path.as_ref(); + let file_name = path + .file_name() + .map(|filename| filename.to_string_lossy().into_owned()); + let ext = path.extension().and_then(|ext| ext.to_str()).unwrap_or(""); + let mime = mime_guess::from_ext(ext).first_or_octet_stream(); + let file = File::open(path).await?; + let field = Part::stream(file).mime(mime); + + Ok(if let Some(file_name) = file_name { + field.file_name(file_name) + } else { + field + }) + } + fn new(value: Body, body_length: Option) -> Part { Part { meta: PartMetadata::new(), diff --git a/src/async_impl/request.rs b/src/async_impl/request.rs index 665710430..76b40a788 100644 --- a/src/async_impl/request.rs +++ b/src/async_impl/request.rs @@ -309,6 +309,9 @@ impl RequestBuilder { /// # Ok(()) /// # } /// ``` + /// + /// In additional the request's body, the Content-Type and Content-Length fields are + /// appropriately set. #[cfg(feature = "multipart")] #[cfg_attr(docsrs, doc(cfg(feature = "multipart")))] pub fn multipart(self, mut multipart: multipart::Form) -> RequestBuilder { @@ -649,6 +652,8 @@ impl TryFrom for HttpRequest { #[cfg(test)] mod tests { + #![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] + use super::{Client, HttpRequest, Request, RequestBuilder, Version}; use crate::Method; use serde::Serialize; diff --git a/src/async_impl/response.rs b/src/async_impl/response.rs index 4088bd647..17be37030 100644 --- a/src/async_impl/response.rs +++ b/src/async_impl/response.rs @@ -35,7 +35,7 @@ pub struct Response { impl Response { pub(super) fn new( - res: hyper::Response, + res: hyper::Response, url: Url, accepts: Accepts, total_timeout: Option>>, @@ -432,7 +432,7 @@ impl Response { impl fmt::Debug for Response { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("Response") - .field("url", self.url()) + .field("url", &self.url().as_str()) .field("status", &self.status()) .field("headers", self.headers()) .finish() diff --git a/src/blocking/body.rs b/src/blocking/body.rs index dd44c6fa2..4782b368c 100644 --- a/src/blocking/body.rs +++ b/src/blocking/body.rs @@ -296,7 +296,7 @@ async fn send_future(sender: Sender) -> Result<(), crate::Error> { // // We need to know whether there is any data to send before // we check the transmission channel (with poll_ready below) - // because somestimes the receiver disappears as soon as is + // because sometimes the receiver disappears as soon as is // considers the data is completely transmitted, which may // be true. // diff --git a/src/blocking/client.rs b/src/blocking/client.rs index 7b020e733..d4b973ee6 100644 --- a/src/blocking/client.rs +++ b/src/blocking/client.rs @@ -16,6 +16,7 @@ use tokio::sync::{mpsc, oneshot}; use super::request::{Request, RequestBuilder}; use super::response::Response; use super::wait; +use crate::dns::Resolve; #[cfg(feature = "__tls")] use crate::tls; #[cfg(feature = "__tls")] @@ -669,9 +670,17 @@ impl ClientBuilder { /// /// # Optional /// - /// This requires the optional `native-tls` feature to be enabled. - #[cfg(feature = "native-tls")] - #[cfg_attr(docsrs, doc(cfg(feature = "native-tls")))] + /// This requires the optional `default-tls`, `native-tls`, or `rustls-tls(-...)` + /// feature to be enabled. + #[cfg(feature = "__tls")] + #[cfg_attr( + docsrs, + doc(cfg(any( + feature = "default-tls", + feature = "native-tls", + feature = "rustls-tls" + ))) + )] pub fn danger_accept_invalid_hostnames(self, accept_invalid_hostname: bool) -> ClientBuilder { self.with_inner(|inner| inner.danger_accept_invalid_hostnames(accept_invalid_hostname)) } @@ -920,6 +929,15 @@ impl ClientBuilder { self.with_inner(|inner| inner.resolve_to_addrs(domain, addrs)) } + /// Override the DNS resolver implementation. + /// + /// Pass an `Arc` wrapping a trait object implementing `Resolve`. + /// Overrides for specific names passed to `resolve` and `resolve_to_addrs` will + /// still be applied on top of this resolver. + pub fn dns_resolver(self, resolver: Arc) -> ClientBuilder { + self.with_inner(|inner| inner.dns_resolver(resolver)) + } + // private fn with_inner(mut self, func: F) -> ClientBuilder diff --git a/src/blocking/multipart.rs b/src/blocking/multipart.rs index 8f7c7bc84..4f18a2aae 100644 --- a/src/blocking/multipart.rs +++ b/src/blocking/multipart.rs @@ -104,7 +104,7 @@ impl Form { /// /// ```no_run /// # fn run() -> std::io::Result<()> { - /// let files = reqwest::blocking::multipart::Form::new() + /// let form = reqwest::blocking::multipart::Form::new() /// .file("key", "/path/to/file")?; /// # Ok(()) /// # } diff --git a/src/blocking/request.rs b/src/blocking/request.rs index 94f89b46a..8ac112444 100644 --- a/src/blocking/request.rs +++ b/src/blocking/request.rs @@ -167,6 +167,14 @@ impl RequestBuilder { } } + /// Assemble a builder starting from an existing `Client` and a `Request`. + pub fn from_parts(client: Client, request: Request) -> RequestBuilder { + RequestBuilder { + client, + request: crate::Result::Ok(request), + } + } + /// Add a `Header` to this Request. /// /// ```rust @@ -203,7 +211,12 @@ impl RequestBuilder { match >::try_from(key) { Ok(key) => match >::try_from(value) { Ok(mut value) => { - value.set_sensitive(sensitive); + // We want to potentially make an unsensitive header + // to be sensitive, not the reverse. So, don't turn off + // a previously sensitive header. + if sensitive { + value.set_sensitive(true); + } req.headers_mut().append(key, value); } Err(e) => error = Some(crate::error::builder(e.into())), @@ -550,6 +563,15 @@ impl RequestBuilder { self.request } + /// Build a `Request`, which can be inspected, modified and executed with + /// `Client::execute()`. + /// + /// This is similar to [`RequestBuilder::build()`], but also returns the + /// embedded `Client`. + pub fn build_split(self) -> (Client, crate::Result) { + (self.client, self.request) + } + /// Constructs the Request and sends it the target URL, returning a Response. /// /// # Errors diff --git a/src/dns/hickory.rs b/src/dns/hickory.rs index 042707006..a94160b2d 100644 --- a/src/dns/hickory.rs +++ b/src/dns/hickory.rs @@ -1,9 +1,12 @@ //! DNS resolution via the [hickory-resolver](https://github.com/hickory-dns/hickory-dns) crate -use hickory_resolver::{lookup_ip::LookupIpIntoIter, system_conf, TokioAsyncResolver}; +use hickory_resolver::{ + config::LookupIpStrategy, error::ResolveError, lookup_ip::LookupIpIntoIter, system_conf, + TokioAsyncResolver, +}; use once_cell::sync::OnceCell; -use std::io; +use std::fmt; use std::net::SocketAddr; use std::sync::Arc; @@ -22,6 +25,9 @@ struct SocketAddrs { iter: LookupIpIntoIter, } +#[derive(Debug)] +struct HickoryDnsSystemConfError(ResolveError); + impl Resolve for HickoryDnsResolver { fn resolve(&self, name: Name) -> Resolving { let resolver = self.clone(); @@ -46,13 +52,23 @@ impl Iterator for SocketAddrs { } /// Create a new resolver with the default configuration, -/// which reads from `/etc/resolve.conf`. -fn new_resolver() -> io::Result { - let (config, opts) = system_conf::read_system_conf().map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("error reading DNS system conf: {e}"), - ) - })?; +/// which reads from `/etc/resolve.conf`. The options are +/// overridden to look up for both IPv4 and IPv6 addresses +/// to work with "happy eyeballs" algorithm. +fn new_resolver() -> Result { + let (config, mut opts) = system_conf::read_system_conf().map_err(HickoryDnsSystemConfError)?; + opts.ip_strategy = LookupIpStrategy::Ipv4AndIpv6; Ok(TokioAsyncResolver::tokio(config, opts)) } + +impl fmt::Display for HickoryDnsSystemConfError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("error reading DNS system conf for hickory-dns") + } +} + +impl std::error::Error for HickoryDnsSystemConfError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} diff --git a/src/error.rs b/src/error.rs index 12fca8b9c..ca7413fd6 100644 --- a/src/error.rs +++ b/src/error.rs @@ -172,7 +172,7 @@ impl fmt::Debug for Error { builder.field("kind", &self.inner.kind); if let Some(ref url) = self.inner.url { - builder.field("url", url); + builder.field("url", &url.as_str()); } if let Some(ref source) = self.inner.source { builder.field("source", source); @@ -287,6 +287,13 @@ pub(crate) fn upgrade>(e: E) -> Error { // io::Error helpers +#[cfg(any( + feature = "gzip", + feature = "zstd", + feature = "brotli", + feature = "deflate", + feature = "blocking", +))] pub(crate) fn into_io(e: BoxError) -> io::Error { io::Error::new(io::ErrorKind::Other, e) } diff --git a/src/lib.rs b/src/lib.rs index d62cb8210..cf3d39d0f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,8 +139,11 @@ //! //! System proxies look in environment variables to set HTTP or HTTPS proxies. //! -//! `HTTP_PROXY` or `http_proxy` provide http proxies for http connections while +//! `HTTP_PROXY` or `http_proxy` provide HTTP proxies for HTTP connections while //! `HTTPS_PROXY` or `https_proxy` provide HTTPS proxies for HTTPS connections. +//! `ALL_PROXY` or `all_proxy` provide proxies for both HTTP and HTTPS connections. +//! If both the all proxy and HTTP or HTTPS proxy variables are set the more specific +//! HTTP or HTTPS proxies take precedence. //! //! These can be overwritten by adding a [`Proxy`] to `ClientBuilder` //! i.e. `let proxy = reqwest::Proxy::http("https://secure.example")?;` diff --git a/src/proxy.rs b/src/proxy.rs index 17670cf4a..5be207a8a 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -29,10 +29,6 @@ use system_configuration::{ sys::schema_definitions::kSCPropNetProxiesHTTPSPort, sys::schema_definitions::kSCPropNetProxiesHTTPSProxy, }; -#[cfg(target_os = "windows")] -use winreg::enums::HKEY_CURRENT_USER; -#[cfg(target_os = "windows")] -use winreg::RegKey; /// Configuration of a proxy that a `Client` should pass requests to. /// @@ -897,6 +893,13 @@ fn insert_proxy(proxies: &mut SystemProxyMap, scheme: impl Into, addr: S fn get_from_environment() -> SystemProxyMap { let mut proxies = HashMap::new(); + if !(insert_from_env(&mut proxies, "http", "ALL_PROXY") + && insert_from_env(&mut proxies, "https", "ALL_PROXY")) + { + insert_from_env(&mut proxies, "http", "all_proxy"); + insert_from_env(&mut proxies, "https", "all_proxy"); + } + if is_cgi() { if log::log_enabled!(log::Level::Warn) && env::var_os("HTTP_PROXY").is_some() { log::warn!("HTTP_PROXY environment variable ignored in CGI"); @@ -909,13 +912,6 @@ fn get_from_environment() -> SystemProxyMap { insert_from_env(&mut proxies, "https", "https_proxy"); } - if !(insert_from_env(&mut proxies, "http", "ALL_PROXY") - && insert_from_env(&mut proxies, "https", "ALL_PROXY")) - { - insert_from_env(&mut proxies, "http", "all_proxy"); - insert_from_env(&mut proxies, "https", "all_proxy"); - } - proxies } @@ -937,12 +933,11 @@ fn is_cgi() -> bool { #[cfg(target_os = "windows")] fn get_from_platform_impl() -> Result, Box> { - let hkcu = RegKey::predef(HKEY_CURRENT_USER); - let internet_setting: RegKey = - hkcu.open_subkey("Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings")?; + let internet_setting = windows_registry::CURRENT_USER + .open("Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings")?; // ensure the proxy is enable, if the value doesn't exist, an error will returned. - let proxy_enable: u32 = internet_setting.get_value("ProxyEnable")?; - let proxy_server: String = internet_setting.get_value("ProxyServer")?; + let proxy_enable = internet_setting.get_u32("ProxyEnable")?; + let proxy_server = internet_setting.get_string("ProxyServer")?; Ok((proxy_enable == 1).then_some(proxy_server)) } @@ -1300,7 +1295,10 @@ mod tests { assert_eq!(p.host(), "127.0.0.1"); assert_eq!(all_proxies.len(), 2); - assert!(all_proxies.values().all(|p| p.host() == "127.0.0.2")); + // Set by ALL_PROXY + assert_eq!(all_proxies["https"].host(), "127.0.0.2"); + // Overwritten by the more specific HTTP_PROXY + assert_eq!(all_proxies["http"].host(), "127.0.0.1"); } #[cfg(any(target_os = "windows", target_os = "macos"))] @@ -1984,11 +1982,6 @@ mod test { url::ParseError::InvalidIpv6Address, ); } - - #[test] - fn invalid_domain_character() { - check_parse_error("http://abc 123/", url::ParseError::InvalidDomainCharacter); - } } } } diff --git a/src/tls.rs b/src/tls.rs index 8f979b15b..83f3feee8 100644 --- a/src/tls.rs +++ b/src/tls.rs @@ -47,7 +47,9 @@ #[cfg(feature = "__rustls")] use rustls::{ client::danger::HandshakeSignatureValid, client::danger::ServerCertVerified, - client::danger::ServerCertVerifier, DigitallySignedStruct, Error as TLSError, SignatureScheme, + client::danger::ServerCertVerifier, crypto::WebPkiSupportedAlgorithms, + server::ParsedCertificate, DigitallySignedStruct, Error as TLSError, RootCertStore, + SignatureScheme, }; #[cfg(feature = "__rustls")] use rustls_pki_types::{ServerName, UnixTime}; @@ -571,6 +573,71 @@ impl ServerCertVerifier for NoVerifier { } } +#[cfg(feature = "__rustls")] +#[derive(Debug)] +pub(crate) struct IgnoreHostname { + roots: RootCertStore, + signature_algorithms: WebPkiSupportedAlgorithms, +} + +#[cfg(feature = "__rustls")] +impl IgnoreHostname { + pub(crate) fn new( + roots: RootCertStore, + signature_algorithms: WebPkiSupportedAlgorithms, + ) -> Self { + Self { + roots, + signature_algorithms, + } + } +} + +#[cfg(feature = "__rustls")] +impl ServerCertVerifier for IgnoreHostname { + fn verify_server_cert( + &self, + end_entity: &rustls_pki_types::CertificateDer<'_>, + intermediates: &[rustls_pki_types::CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + now: UnixTime, + ) -> Result { + let cert = ParsedCertificate::try_from(end_entity)?; + + rustls::client::verify_server_cert_signed_by_trust_anchor( + &cert, + &self.roots, + intermediates, + now, + self.signature_algorithms.all, + )?; + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls_pki_types::CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.signature_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls_pki_types::CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.signature_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.signature_algorithms.supported_schemes() + } +} + /// Hyper extension carrying extra TLS layer information. /// Made available to clients on responses when `tls_info` is set. #[derive(Clone)] diff --git a/src/wasm/body.rs b/src/wasm/body.rs index 751cf586e..241aa8173 100644 --- a/src/wasm/body.rs +++ b/src/wasm/body.rs @@ -182,6 +182,10 @@ impl fmt::Debug for Body { } } +// Can use new methods in web-sys when requiring v0.2.93. +// > `init.method(m)` to `init.set_method(m)` +// For now, ignore their deprecation. +#[allow(deprecated)] #[cfg(test)] mod tests { use crate::Body; diff --git a/src/wasm/client.rs b/src/wasm/client.rs index 9f22a896b..e33ed26d1 100644 --- a/src/wasm/client.rs +++ b/src/wasm/client.rs @@ -182,6 +182,10 @@ impl fmt::Debug for ClientBuilder { } } +// Can use new methods in web-sys when requiring v0.2.93. +// > `init.method(m)` to `init.set_method(m)` +// For now, ignore their deprecation. +#[allow(deprecated)] async fn fetch(req: Request) -> crate::Result { // Build the js Request let mut init = web_sys::RequestInit::new(); diff --git a/src/wasm/request.rs b/src/wasm/request.rs index c373e5c8f..e6f51ebc1 100644 --- a/src/wasm/request.rs +++ b/src/wasm/request.rs @@ -115,6 +115,14 @@ impl RequestBuilder { RequestBuilder { client, request } } + /// Assemble a builder starting from an existing `Client` and a `Request`. + pub fn from_parts(client: crate::Client, request: crate::Request) -> crate::RequestBuilder { + crate::RequestBuilder { + client, + request: crate::Result::Ok(request), + } + } + /// Modify the query string of the URL. /// /// Modifies the URL of this request, adding the parameters provided. @@ -349,6 +357,15 @@ impl RequestBuilder { self.request } + /// Build a `Request`, which can be inspected, modified and executed with + /// `Client::execute()`. + /// + /// This is similar to [`RequestBuilder::build()`], but also returns the + /// embedded `Client`. + pub fn build_split(self) -> (Client, crate::Result) { + (self.client, self.request) + } + /// Constructs the Request and sends it to the target URL, returning a /// future Response. /// diff --git a/tests/badssl.rs b/tests/badssl.rs index 9b001d070..28d284324 100644 --- a/tests/badssl.rs +++ b/tests/badssl.rs @@ -1,4 +1,5 @@ #![cfg(not(target_arch = "wasm32"))] +#![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] #[cfg(all(feature = "__tls", not(feature = "rustls-tls-manual-roots")))] #[tokio::test] @@ -74,7 +75,7 @@ async fn test_badssl_no_built_in_roots() { assert!(result.is_err()); } -#[cfg(feature = "native-tls")] +#[cfg(any(feature = "native-tls", feature = "rustls-tls"))] #[tokio::test] async fn test_badssl_wrong_host() { let text = reqwest::Client::builder() diff --git a/tests/client.rs b/tests/client.rs index ea1eaa8ca..18aaf4e99 100644 --- a/tests/client.rs +++ b/tests/client.rs @@ -1,4 +1,5 @@ #![cfg(not(target_arch = "wasm32"))] +#![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] mod support; use support::server; @@ -60,7 +61,7 @@ async fn auto_headers() { } #[tokio::test] -async fn donot_set_conent_length_0_if_have_no_body() { +async fn donot_set_content_length_0_if_have_no_body() { let server = server::http(move |req| async move { let headers = req.headers(); assert_eq!(headers.get(CONTENT_LENGTH), None); @@ -70,7 +71,7 @@ async fn donot_set_conent_length_0_if_have_no_body() { http::Response::default() }); - let url = format!("http://{}/conent-length", server.addr()); + let url = format!("http://{}/content-length", server.addr()); let res = reqwest::Client::builder() .no_proxy() .build() @@ -83,6 +84,35 @@ async fn donot_set_conent_length_0_if_have_no_body() { assert_eq!(res.status(), reqwest::StatusCode::OK); } +#[cfg(feature = "http3")] +#[tokio::test] +async fn http3_request_full() { + use http_body_util::BodyExt; + + let server = server::http3(move |req| async move { + assert_eq!(req.headers()[CONTENT_LENGTH], "5"); + let reqb = req.collect().await.unwrap().to_bytes(); + assert_eq!(reqb, "hello"); + http::Response::default() + }); + + let url = format!("https://{}/content-length", server.addr()); + let res = reqwest::Client::builder() + .http3_prior_knowledge() + .danger_accept_invalid_certs(true) + .build() + .expect("client builder") + .post(url) + .version(http::Version::HTTP_3) + .body("hello") + .send() + .await + .expect("request"); + + assert_eq!(res.version(), http::Version::HTTP_3); + assert_eq!(res.status(), reqwest::StatusCode::OK); +} + #[tokio::test] async fn user_agent() { let server = server::http(move |req| async move { @@ -216,6 +246,7 @@ async fn overridden_dns_resolution_with_gai() { server.addr().port() ); let client = reqwest::Client::builder() + .no_proxy() .resolve(overridden_domain, server.addr()) .build() .expect("client builder"); @@ -240,6 +271,7 @@ async fn overridden_dns_resolution_with_gai_multiple() { // the server runs on IPv4 localhost, so provide both IPv4 and IPv6 and let the happy eyeballs // algorithm decide which address to use. let client = reqwest::Client::builder() + .no_proxy() .resolve_to_addrs( overridden_domain, &[ @@ -272,6 +304,7 @@ async fn overridden_dns_resolution_with_hickory_dns() { server.addr().port() ); let client = reqwest::Client::builder() + .no_proxy() .resolve(overridden_domain, server.addr()) .hickory_dns(true) .build() @@ -298,6 +331,7 @@ async fn overridden_dns_resolution_with_hickory_dns_multiple() { // the server runs on IPv4 localhost, so provide both IPv4 and IPv6 and let the happy eyeballs // algorithm decide which address to use. let client = reqwest::Client::builder() + .no_proxy() .resolve_to_addrs( overridden_domain, &[ @@ -383,6 +417,7 @@ async fn http2_upgrade() { } #[cfg(feature = "default-tls")] +#[cfg_attr(feature = "http3", ignore = "enabling http3 seems to break this, why?")] #[tokio::test] async fn test_allowed_methods() { let resp = reqwest::Client::builder() @@ -466,7 +501,7 @@ async fn test_tls_info() { assert!(tls_info.is_none()); } -// NOTE: using the default "curernt_thread" runtime here would cause the test to +// NOTE: using the default "current_thread" runtime here would cause the test to // fail, because the only thread would block until `panic_rx` receives a // notification while the client needs to be driven to get the graceful shutdown // done. diff --git a/tests/multipart.rs b/tests/multipart.rs index 8b5149e1d..bac6da314 100644 --- a/tests/multipart.rs +++ b/tests/multipart.rs @@ -175,3 +175,58 @@ fn blocking_file_part() { assert_eq!(res.url().as_str(), &url); assert_eq!(res.status(), reqwest::StatusCode::OK); } + +#[cfg(feature = "stream")] +#[tokio::test] +async fn async_impl_file_part() { + let _ = env_logger::try_init(); + + let form = reqwest::multipart::Form::new() + .file("foo", "Cargo.lock") + .await + .unwrap(); + + let fcontents = std::fs::read_to_string("Cargo.lock").unwrap(); + + let expected_body = format!( + "\ + --{0}\r\n\ + Content-Disposition: form-data; name=\"foo\"; filename=\"Cargo.lock\"\r\n\ + Content-Type: application/octet-stream\r\n\r\n\ + {1}\r\n\ + --{0}--\r\n\ + ", + form.boundary(), + fcontents + ); + + let ct = format!("multipart/form-data; boundary={}", form.boundary()); + + let server = server::http(move |req| { + let ct = ct.clone(); + let expected_body = expected_body.clone(); + async move { + assert_eq!(req.method(), "POST"); + assert_eq!(req.headers()["content-type"], ct); + assert_eq!(req.headers()["transfer-encoding"], "chunked"); + + let full = req.collect().await.unwrap().to_bytes(); + + assert_eq!(full, expected_body.as_bytes()); + + http::Response::default() + } + }); + + let url = format!("http://{}/multipart/3", server.addr()); + + let res = reqwest::Client::new() + .post(&url) + .multipart(form) + .send() + .await + .unwrap(); + + assert_eq!(res.url().as_str(), &url); + assert_eq!(res.status(), reqwest::StatusCode::OK); +} diff --git a/tests/proxy.rs b/tests/proxy.rs index 231de25d8..9231a3267 100644 --- a/tests/proxy.rs +++ b/tests/proxy.rs @@ -1,4 +1,5 @@ #![cfg(not(target_arch = "wasm32"))] +#![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] mod support; use support::server; diff --git a/tests/redirect.rs b/tests/redirect.rs index c98c799ef..c496d90d3 100644 --- a/tests/redirect.rs +++ b/tests/redirect.rs @@ -1,4 +1,5 @@ #![cfg(not(target_arch = "wasm32"))] +#![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] mod support; use http_body_util::BodyExt; use reqwest::Body; diff --git a/tests/support/server.cert b/tests/support/server.cert new file mode 100644 index 000000000..e573f2a52 Binary files /dev/null and b/tests/support/server.cert differ diff --git a/tests/support/server.key b/tests/support/server.key new file mode 100644 index 000000000..757035e24 Binary files /dev/null and b/tests/support/server.key differ diff --git a/tests/support/server.rs b/tests/support/server.rs index f9c45b4d2..43742b60e 100644 --- a/tests/support/server.rs +++ b/tests/support/server.rs @@ -52,6 +52,7 @@ where F2: FnOnce(&mut Builder) -> Bu + Send + 'static, { // Spawn new runtime in thread to prevent reactor execution context conflict + let test_name = thread::current().name().unwrap_or("").to_string(); thread::spawn(move || { let rt = runtime::Builder::new_current_thread() .enable_all() @@ -68,7 +69,7 @@ where let (panic_tx, panic_rx) = std_mpsc::channel(); let tname = format!( "test({})-support-server", - thread::current().name().unwrap_or("") + test_name, ); thread::Builder::new() .name(tname) @@ -110,3 +111,109 @@ where .join() .unwrap() } + +#[cfg(feature = "http3")] +pub fn http3(func: F1) -> Server +where + F1: Fn(http::Request>) -> Fut + + Clone + + Send + + 'static, + Fut: Future> + Send + 'static, +{ + use bytes::Buf; + use http_body_util::BodyExt; + use quinn::crypto::rustls::QuicServerConfig; + use std::sync::Arc; + + // Spawn new runtime in thread to prevent reactor execution context conflict + let test_name = thread::current().name().unwrap_or("").to_string(); + thread::spawn(move || { + let rt = runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("new rt"); + + let cert = std::fs::read("tests/support/server.cert").unwrap().into(); + let key = std::fs::read("tests/support/server.key").unwrap().try_into().unwrap(); + + let mut tls_config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![cert], key) + .unwrap(); + tls_config.max_early_data_size = u32::MAX; + tls_config.alpn_protocols = vec![b"h3".into()]; + + let server_config = quinn::ServerConfig::with_crypto(Arc::new(QuicServerConfig::try_from(tls_config).unwrap())); + let endpoint = rt.block_on(async move { + quinn::Endpoint::server(server_config, "[::1]:0".parse().unwrap()).unwrap() + }); + let addr = endpoint.local_addr().unwrap(); + + let (shutdown_tx, mut shutdown_rx) = oneshot::channel(); + let (panic_tx, panic_rx) = std_mpsc::channel(); + let tname = format!( + "test({})-support-server", + test_name, + ); + thread::Builder::new() + .name(tname) + .spawn(move || { + rt.block_on(async move { + + loop { + tokio::select! { + _ = &mut shutdown_rx => { + break; + } + Some(accepted) = endpoint.accept() => { + let conn = accepted.await.expect("accepted"); + let mut h3_conn = h3::server::Connection::new(h3_quinn::Connection::new(conn)).await.unwrap(); + let func = func.clone(); + tokio::spawn(async move { + while let Ok(Some((req, stream))) = h3_conn.accept().await { + let func = func.clone(); + tokio::spawn(async move { + let (mut tx, rx) = stream.split(); + let body = futures_util::stream::unfold(rx, |mut rx| async move { + match rx.recv_data().await { + Ok(Some(mut buf)) => { + Some((Ok(hyper::body::Frame::data(buf.copy_to_bytes(buf.remaining()))), rx)) + }, + Ok(None) => None, + Err(err) => { + Some((Err(err), rx)) + } + } + }); + let body = BodyExt::boxed(http_body_util::StreamBody::new(body)); + let resp = func(req.map(move |()| body)).await; + let (parts, mut body) = resp.into_parts(); + let resp = http::Response::from_parts(parts, ()); + tx.send_response(resp).await.unwrap(); + + while let Some(Ok(frame)) = body.frame().await { + if let Ok(data) = frame.into_data() { + tx.send_data(data).await.unwrap(); + } + } + tx.finish().await.unwrap(); + }); + } + }); + } + } + } + let _ = panic_tx.send(()); + }); + }) + .expect("thread spawn"); + Server { + addr, + panic_rx, + shutdown_tx: Some(shutdown_tx), + } + }) + .join() + .unwrap() +} diff --git a/tests/timeouts.rs b/tests/timeouts.rs index c18fecdbe..c3649ea9f 100644 --- a/tests/timeouts.rs +++ b/tests/timeouts.rs @@ -1,4 +1,5 @@ #![cfg(not(target_arch = "wasm32"))] +#![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] mod support; use support::server; diff --git a/tests/upgrade.rs b/tests/upgrade.rs index 5ea72acc2..7a67c0457 100644 --- a/tests/upgrade.rs +++ b/tests/upgrade.rs @@ -1,4 +1,5 @@ #![cfg(not(target_arch = "wasm32"))] +#![cfg(not(feature = "rustls-tls-manual-roots-no-provider"))] mod support; use support::server; use tokio::io::{AsyncReadExt, AsyncWriteExt};