Retry http request (intproxy) on hyper IncompleteMessage error. (meta…

…lbear-co#2934) * Retry http request (intproxy) on hyper IncompleteMessage error. * also applies for streamed * changelog * apply great extra dot suggestion * another neat suggestion involving dots --------- Co-authored-by: Aviram Hassan <[email protected]>
Razz4780 · Nov 27, 2024 · a73fb72 · a73fb72
1 parent 13ac0a0
commit a73fb72
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 3 deletions.
diff --git a/changelog.d/+retry-on-http-incomplete-message.fixed.md b/changelog.d/+retry-on-http-incomplete-message.fixed.md
@@ -0,0 +1 @@
+Retry http request (intproxy) on hyper IncompleteMessage error.
diff --git a/mirrord/intproxy/src/proxies/incoming.rs b/mirrord/intproxy/src/proxies/incoming.rs
@@ -620,7 +620,8 @@ impl IncomingProxy {
             }
             // Retry on known errors.
             Err(error @ InterceptorError::Reset)
-            | Err(error @ InterceptorError::ConnectionClosedTooSoon(..)) => {
+            | Err(error @ InterceptorError::ConnectionClosedTooSoon(..))
+            | Err(error @ InterceptorError::IncompleteMessage(..)) => {
                 tracing::warn!(%error, ?request, "Failed to read first frames of streaming HTTP response");
 
                 let interceptor = self

diff --git a/mirrord/intproxy/src/proxies/incoming/interceptor.rs b/mirrord/intproxy/src/proxies/incoming/interceptor.rs
@@ -71,6 +71,10 @@ pub enum InterceptorError {
     /// The layer closed connection too soon to send a request.
     #[error("connection closed too soon")]
     ConnectionClosedTooSoon(HttpRequestFallback),
+
+    #[error("incomplete message")]
+    IncompleteMessage(HttpRequestFallback),
+
     /// Received a request with an unsupported HTTP version.
     #[error("{0:?} is not supported")]
     UnsupportedHttpVersion(Version),
@@ -282,6 +286,19 @@ impl HttpConnection {
                     None,
                 ))
             }
+            Err(InterceptorError::Hyper(e)) if e.is_incomplete_message() => {
+                tracing::warn!(
+                    "Sending request to local application failed with: {e:?}. \
+                        Connection closed before the message could complete!"
+                );
+                tracing::trace!(
+                    ?request,
+                    "Retrying the request, see \
+                    [https://github.com/hyperium/hyper/issues/2136] for more info."
+                );
+
+                Err(InterceptorError::IncompleteMessage(request))
+            }
 
             Err(fail) => {
                 tracing::warn!(?fail, "Request to local application failed!");
@@ -385,11 +402,13 @@ impl HttpConnection {
                 Ok(response) => return Ok(response),
 
                 Err(error @ InterceptorError::Reset)
-                | Err(error @ InterceptorError::ConnectionClosedTooSoon(_)) => {
+                | Err(error @ InterceptorError::ConnectionClosedTooSoon(_))
+                | Err(error @ InterceptorError::IncompleteMessage(_)) => {
                     tracing::warn!(
                         ?request,
                         %error,
-                        "Either the connection closed or we got a reset, retrying!"
+                        "Either the connection closed, the message is incomplete, \
+                        or we got a reset, retrying!"
                     );
 
                     let Some(backoff) = backoffs.next() else {