diff --git a/changelog.d/+retry-on-http-incomplete-message.fixed.md b/changelog.d/+retry-on-http-incomplete-message.fixed.md new file mode 100644 index 00000000000..592904039de --- /dev/null +++ b/changelog.d/+retry-on-http-incomplete-message.fixed.md @@ -0,0 +1 @@ +Retry http request (intproxy) on hyper IncompleteMessage error. diff --git a/mirrord/intproxy/src/proxies/incoming.rs b/mirrord/intproxy/src/proxies/incoming.rs index 6d1e223ef03..966d1175acb 100644 --- a/mirrord/intproxy/src/proxies/incoming.rs +++ b/mirrord/intproxy/src/proxies/incoming.rs @@ -620,7 +620,8 @@ impl IncomingProxy { } // Retry on known errors. Err(error @ InterceptorError::Reset) - | Err(error @ InterceptorError::ConnectionClosedTooSoon(..)) => { + | Err(error @ InterceptorError::ConnectionClosedTooSoon(..)) + | Err(error @ InterceptorError::IncompleteMessage(..)) => { tracing::warn!(%error, ?request, "Failed to read first frames of streaming HTTP response"); let interceptor = self diff --git a/mirrord/intproxy/src/proxies/incoming/interceptor.rs b/mirrord/intproxy/src/proxies/incoming/interceptor.rs index 5bf5301a6c5..2d6486d709f 100644 --- a/mirrord/intproxy/src/proxies/incoming/interceptor.rs +++ b/mirrord/intproxy/src/proxies/incoming/interceptor.rs @@ -71,6 +71,10 @@ pub enum InterceptorError { /// The layer closed connection too soon to send a request. #[error("connection closed too soon")] ConnectionClosedTooSoon(HttpRequestFallback), + + #[error("incomplete message")] + IncompleteMessage(HttpRequestFallback), + /// Received a request with an unsupported HTTP version. #[error("{0:?} is not supported")] UnsupportedHttpVersion(Version), @@ -282,6 +286,19 @@ impl HttpConnection { None, )) } + Err(InterceptorError::Hyper(e)) if e.is_incomplete_message() => { + tracing::warn!( + "Sending request to local application failed with: {e:?}. \ + Connection closed before the message could complete!" + ); + tracing::trace!( + ?request, + "Retrying the request, see \ + [https://github.com/hyperium/hyper/issues/2136] for more info." + ); + + Err(InterceptorError::IncompleteMessage(request)) + } Err(fail) => { tracing::warn!(?fail, "Request to local application failed!"); @@ -385,11 +402,13 @@ impl HttpConnection { Ok(response) => return Ok(response), Err(error @ InterceptorError::Reset) - | Err(error @ InterceptorError::ConnectionClosedTooSoon(_)) => { + | Err(error @ InterceptorError::ConnectionClosedTooSoon(_)) + | Err(error @ InterceptorError::IncompleteMessage(_)) => { tracing::warn!( ?request, %error, - "Either the connection closed or we got a reset, retrying!" + "Either the connection closed, the message is incomplete, \ + or we got a reset, retrying!" ); let Some(backoff) = backoffs.next() else {