From a73fb72c0c434ca7f57e1805f6b7cd7e4630e7c4 Mon Sep 17 00:00:00 2001
From: meowjesty <43983236+meowjesty@users.noreply.github.com>
Date: Wed, 27 Nov 2024 09:54:45 -0300
Subject: [PATCH] Retry http request (intproxy) on hyper IncompleteMessage
 error. (#2934)

* Retry http request (intproxy) on hyper IncompleteMessage error.

* also applies for streamed

* changelog

* apply great extra dot suggestion

* another neat suggestion involving dots

---------

Co-authored-by: Aviram Hassan <aviram@metalbear.co>
---
 ...+retry-on-http-incomplete-message.fixed.md |  1 +
 mirrord/intproxy/src/proxies/incoming.rs      |  3 ++-
 .../src/proxies/incoming/interceptor.rs       | 23 +++++++++++++++++--
 3 files changed, 24 insertions(+), 3 deletions(-)
 create mode 100644 changelog.d/+retry-on-http-incomplete-message.fixed.md

diff --git a/changelog.d/+retry-on-http-incomplete-message.fixed.md b/changelog.d/+retry-on-http-incomplete-message.fixed.md
new file mode 100644
index 00000000000..592904039de
--- /dev/null
+++ b/changelog.d/+retry-on-http-incomplete-message.fixed.md
@@ -0,0 +1 @@
+Retry http request (intproxy) on hyper IncompleteMessage error.
diff --git a/mirrord/intproxy/src/proxies/incoming.rs b/mirrord/intproxy/src/proxies/incoming.rs
index 6d1e223ef03..966d1175acb 100644
--- a/mirrord/intproxy/src/proxies/incoming.rs
+++ b/mirrord/intproxy/src/proxies/incoming.rs
@@ -620,7 +620,8 @@ impl IncomingProxy {
             }
             // Retry on known errors.
             Err(error @ InterceptorError::Reset)
-            | Err(error @ InterceptorError::ConnectionClosedTooSoon(..)) => {
+            | Err(error @ InterceptorError::ConnectionClosedTooSoon(..))
+            | Err(error @ InterceptorError::IncompleteMessage(..)) => {
                 tracing::warn!(%error, ?request, "Failed to read first frames of streaming HTTP response");
 
                 let interceptor = self
diff --git a/mirrord/intproxy/src/proxies/incoming/interceptor.rs b/mirrord/intproxy/src/proxies/incoming/interceptor.rs
index 5bf5301a6c5..2d6486d709f 100644
--- a/mirrord/intproxy/src/proxies/incoming/interceptor.rs
+++ b/mirrord/intproxy/src/proxies/incoming/interceptor.rs
@@ -71,6 +71,10 @@ pub enum InterceptorError {
     /// The layer closed connection too soon to send a request.
     #[error("connection closed too soon")]
     ConnectionClosedTooSoon(HttpRequestFallback),
+
+    #[error("incomplete message")]
+    IncompleteMessage(HttpRequestFallback),
+
     /// Received a request with an unsupported HTTP version.
     #[error("{0:?} is not supported")]
     UnsupportedHttpVersion(Version),
@@ -282,6 +286,19 @@ impl HttpConnection {
                     None,
                 ))
             }
+            Err(InterceptorError::Hyper(e)) if e.is_incomplete_message() => {
+                tracing::warn!(
+                    "Sending request to local application failed with: {e:?}. \
+                        Connection closed before the message could complete!"
+                );
+                tracing::trace!(
+                    ?request,
+                    "Retrying the request, see \
+                    [https://github.com/hyperium/hyper/issues/2136] for more info."
+                );
+
+                Err(InterceptorError::IncompleteMessage(request))
+            }
 
             Err(fail) => {
                 tracing::warn!(?fail, "Request to local application failed!");
@@ -385,11 +402,13 @@ impl HttpConnection {
                 Ok(response) => return Ok(response),
 
                 Err(error @ InterceptorError::Reset)
-                | Err(error @ InterceptorError::ConnectionClosedTooSoon(_)) => {
+                | Err(error @ InterceptorError::ConnectionClosedTooSoon(_))
+                | Err(error @ InterceptorError::IncompleteMessage(_)) => {
                     tracing::warn!(
                         ?request,
                         %error,
-                        "Either the connection closed or we got a reset, retrying!"
+                        "Either the connection closed, the message is incomplete, \
+                        or we got a reset, retrying!"
                     );
 
                     let Some(backoff) = backoffs.next() else {