From 74b88b4f20aba484bd691248370e6f44d02fee63 Mon Sep 17 00:00:00 2001 From: vthiery Date: Wed, 19 Aug 2020 00:06:20 +0200 Subject: [PATCH 1/7] Handle request cancellation properly When a request is being cancelled by the caller via the context, even though the underlying HTTP client should not perform any more request and directly return, the backoff mechanism still kicks in and leaves the caller waiting for multiple backoff periods for nothing. --- httpclient/client.go | 10 ++++++++ httpclient/client_test.go | 37 +++++++++++++++++++++++++++++ hystrix/hystrix_client.go | 8 +++++++ hystrix/hystrix_client_test.go | 43 ++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+) diff --git a/httpclient/client.go b/httpclient/client.go index 5f03c5b..d8f93fb 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -153,6 +153,16 @@ func (c *Client) Do(request *http.Request) (*http.Response, error) { } if err != nil { + // If the request context has already been cancelled, don't retry + ctx := request.Context() + select { + case <-ctx.Done(): + multiErr.Push(err.Error()) + c.reportRequestEnd(request, response) + return nil, multiErr.HasError() + default: + } + multiErr.Push(err.Error()) c.reportError(request, err) backoffTime := c.retrier.NextInterval(i) diff --git a/httpclient/client_test.go b/httpclient/client_test.go index dbca35d..f213d35 100644 --- a/httpclient/client_test.go +++ b/httpclient/client_test.go @@ -2,6 +2,7 @@ package httpclient import ( "bytes" + "context" "io/ioutil" "net/http" "net/http/httptest" @@ -417,6 +418,42 @@ func TestHTTPClientGetReturnsErrorOnFailure(t *testing.T) { assert.Nil(t, response) } +func TestHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { + count := 0 + noOfRetries := 3 + // Set a huge backoffInterval that we won't have to wait anyway + backoffInterval := 1 * time.Hour + maximumJitterInterval := 1 * time.Millisecond + + client := NewClient( + WithHTTPTimeout(10*time.Millisecond), + WithRetryCount(noOfRetries), + WithRetrier(heimdall.NewRetrier(heimdall.NewConstantBackoff(backoffInterval, maximumJitterInterval))), + ) + + ctx, cancel := context.WithCancel(context.Background()) + + dummyHandler := func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{ "response": "something went wrong" }`)) + count++ + // Cancel the context after the first call + cancel() + } + + server := httptest.NewServer(http.HandlerFunc(dummyHandler)) + defer server.Close() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + require.NoError(t, err) + + response, err := client.Do(req.WithContext(ctx)) + require.Error(t, err, "should have failed to make request") + require.Nil(t, response) + + assert.Equal(t, 1, count) +} + func TestPluginMethodsCalled(t *testing.T) { client := NewClient(WithHTTPTimeout(10 * time.Millisecond)) mockPlugin := &MockPlugin{} diff --git a/hystrix/hystrix_client.go b/hystrix/hystrix_client.go index 01c0ca3..2d16e0b 100644 --- a/hystrix/hystrix_client.go +++ b/hystrix/hystrix_client.go @@ -205,6 +205,14 @@ func (hhc *Client) Do(request *http.Request) (*http.Response, error) { }, hhc.fallbackFunc) if err != nil { + // If the request context has already been cancelled, don't retry + ctx := request.Context() + select { + case <-ctx.Done(): + return nil, err + default: + } + backoffTime := hhc.retrier.NextInterval(i) time.Sleep(backoffTime) continue diff --git a/hystrix/hystrix_client_test.go b/hystrix/hystrix_client_test.go index 65a7bbd..b09e8fb 100644 --- a/hystrix/hystrix_client_test.go +++ b/hystrix/hystrix_client_test.go @@ -2,6 +2,7 @@ package hystrix import ( "bytes" + "context" "io/ioutil" "net/http" "net/http/httptest" @@ -343,6 +344,48 @@ func BenchmarkHystrixHTTPClientRetriesGetOnFailure(b *testing.B) { } } +func TestHystrixHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { + count := 0 + noOfRetries := 3 + // Set a huge backoffInterval that we won't have to wait anyway + backoffInterval := 1 * time.Hour + maximumJitterInterval := 1 * time.Millisecond + + client := NewClient( + WithHTTPTimeout(10*time.Millisecond), + WithCommandName("some_command_name"), + WithHystrixTimeout(10*time.Millisecond), + WithMaxConcurrentRequests(100), + WithErrorPercentThreshold(10), + WithSleepWindow(100), + WithRequestVolumeThreshold(10), + WithRetryCount(noOfRetries), + WithRetrier(heimdall.NewRetrier(heimdall.NewConstantBackoff(backoffInterval, maximumJitterInterval))), + ) + + ctx, cancel := context.WithCancel(context.Background()) + + dummyHandler := func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{ "response": "something went wrong" }`)) + count++ + // Cancel the context after the first call + cancel() + } + + server := httptest.NewServer(http.HandlerFunc(dummyHandler)) + defer server.Close() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + require.NoError(t, err) + + response, err := client.Do(req.WithContext(ctx)) + require.Error(t, err, "should have failed to make request") + require.Nil(t, response) + + assert.Equal(t, 1, count) +} + func TestHystrixHTTPClientRetriesPostOnFailure(t *testing.T) { count := 0 backoffInterval := 1 * time.Millisecond From 224576576c05f8b40b6e121fc2ea405bc7b59955 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Wed, 9 Sep 2020 15:09:01 +0200 Subject: [PATCH 2/7] feat: use context.WithTimeout instead of sleep Signed-off-by: Yoan Blanc --- httpclient/client.go | 50 ++++++++++++++++++++++++---------- hystrix/hystrix_client.go | 23 ++++++++++------ hystrix/hystrix_client_test.go | 11 ++++++-- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/httpclient/client.go b/httpclient/client.go index d8f93fb..ffb6979 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -2,6 +2,7 @@ package httpclient import ( "bytes" + "context" "io" "io/ioutil" "net/http" @@ -138,6 +139,7 @@ func (c *Client) Do(request *http.Request) (*http.Response, error) { multiErr := &valkyrie.MultiError{} var response *http.Response +outter: for i := 0; i <= c.retryCount; i++ { if response != nil { response.Body.Close() @@ -153,28 +155,48 @@ func (c *Client) Do(request *http.Request) (*http.Response, error) { } if err != nil { - // If the request context has already been cancelled, don't retry - ctx := request.Context() - select { - case <-ctx.Done(): - multiErr.Push(err.Error()) - c.reportRequestEnd(request, response) - return nil, multiErr.HasError() - default: - } - multiErr.Push(err.Error()) c.reportError(request, err) + backoffTime := c.retrier.NextInterval(i) - time.Sleep(backoffTime) - continue + ctx, cancel := context.WithTimeout(context.Background(), backoffTime) + + select { + case <-ctx.Done(): + cancel() + + continue + + case <-request.Context().Done(): + cancel() + + multiErr.Push(request.Context().Err().Error()) + c.reportError(request, err) + + // If the request context has already been cancelled, don't retry + break outter + } } + c.reportRequestEnd(request, response) if response.StatusCode >= http.StatusInternalServerError { backoffTime := c.retrier.NextInterval(i) - time.Sleep(backoffTime) - continue + ctx, cancel := context.WithTimeout(context.Background(), backoffTime) + + select { + case <-ctx.Done(): + cancel() + continue + case <-request.Context().Done(): + cancel() + + multiErr.Push(request.Context().Err().Error()) + c.reportError(request, err) + + // If the request context has already been cancelled, don't retry + break outter + } } multiErr = &valkyrie.MultiError{} // Clear errors if any iteration succeeds diff --git a/hystrix/hystrix_client.go b/hystrix/hystrix_client.go index 2d16e0b..fdba698 100644 --- a/hystrix/hystrix_client.go +++ b/hystrix/hystrix_client.go @@ -2,6 +2,7 @@ package hystrix import ( "bytes" + "context" "io" "io/ioutil" "net/http" @@ -181,6 +182,7 @@ func (hhc *Client) Do(request *http.Request) (*http.Response, error) { request.Body = ioutil.NopCloser(bodyReader) // prevents closing the body between retries } +outter: for i := 0; i <= hhc.retryCount; i++ { if response != nil { response.Body.Close() @@ -205,17 +207,22 @@ func (hhc *Client) Do(request *http.Request) (*http.Response, error) { }, hhc.fallbackFunc) if err != nil { - // If the request context has already been cancelled, don't retry - ctx := request.Context() + backoffTime := hhc.retrier.NextInterval(i) + ctx, cancel := context.WithTimeout(context.Background(), backoffTime) + select { + case <-request.Context().Done(): + cancel() + + // If the request context has been cancelled, don't retry + err = request.Context().Err() + break outter + case <-ctx.Done(): - return nil, err - default: - } + cancel() - backoffTime := hhc.retrier.NextInterval(i) - time.Sleep(backoffTime) - continue + continue + } } break diff --git a/hystrix/hystrix_client_test.go b/hystrix/hystrix_client_test.go index b09e8fb..42effc9 100644 --- a/hystrix/hystrix_client_test.go +++ b/hystrix/hystrix_client_test.go @@ -308,6 +308,7 @@ func TestHystrixHTTPClientRetriesGetOnFailure5xx(t *testing.T) { response, err := client.Get(server.URL, http.Header{}) require.NoError(t, err) + defer response.Body.Close() assert.Equal(t, 4, count) @@ -345,6 +346,8 @@ func BenchmarkHystrixHTTPClientRetriesGetOnFailure(b *testing.B) { } func TestHystrixHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { + t.Skip("Skip: concurrency issues... to be fixed in hystrix") + count := 0 noOfRetries := 3 // Set a huge backoffInterval that we won't have to wait anyway @@ -366,11 +369,13 @@ func TestHystrixHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) dummyHandler := func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(`{ "response": "something went wrong" }`)) - count++ // Cancel the context after the first call cancel() + + w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"response": "something went wrong"}`)) + count++ } server := httptest.NewServer(http.HandlerFunc(dummyHandler)) From bd0d2c890f1744c4770fabb5d9ac0c841e5bd914 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Tue, 15 Sep 2020 16:55:23 +0200 Subject: [PATCH 3/7] fix: undo changes make to hystrix Signed-off-by: Yoan Blanc --- hystrix/hystrix_client.go | 19 ++------------ hystrix/hystrix_client_test.go | 48 ---------------------------------- 2 files changed, 2 insertions(+), 65 deletions(-) diff --git a/hystrix/hystrix_client.go b/hystrix/hystrix_client.go index fdba698..01c0ca3 100644 --- a/hystrix/hystrix_client.go +++ b/hystrix/hystrix_client.go @@ -2,7 +2,6 @@ package hystrix import ( "bytes" - "context" "io" "io/ioutil" "net/http" @@ -182,7 +181,6 @@ func (hhc *Client) Do(request *http.Request) (*http.Response, error) { request.Body = ioutil.NopCloser(bodyReader) // prevents closing the body between retries } -outter: for i := 0; i <= hhc.retryCount; i++ { if response != nil { response.Body.Close() @@ -208,21 +206,8 @@ outter: if err != nil { backoffTime := hhc.retrier.NextInterval(i) - ctx, cancel := context.WithTimeout(context.Background(), backoffTime) - - select { - case <-request.Context().Done(): - cancel() - - // If the request context has been cancelled, don't retry - err = request.Context().Err() - break outter - - case <-ctx.Done(): - cancel() - - continue - } + time.Sleep(backoffTime) + continue } break diff --git a/hystrix/hystrix_client_test.go b/hystrix/hystrix_client_test.go index 42effc9..65a7bbd 100644 --- a/hystrix/hystrix_client_test.go +++ b/hystrix/hystrix_client_test.go @@ -2,7 +2,6 @@ package hystrix import ( "bytes" - "context" "io/ioutil" "net/http" "net/http/httptest" @@ -308,7 +307,6 @@ func TestHystrixHTTPClientRetriesGetOnFailure5xx(t *testing.T) { response, err := client.Get(server.URL, http.Header{}) require.NoError(t, err) - defer response.Body.Close() assert.Equal(t, 4, count) @@ -345,52 +343,6 @@ func BenchmarkHystrixHTTPClientRetriesGetOnFailure(b *testing.B) { } } -func TestHystrixHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { - t.Skip("Skip: concurrency issues... to be fixed in hystrix") - - count := 0 - noOfRetries := 3 - // Set a huge backoffInterval that we won't have to wait anyway - backoffInterval := 1 * time.Hour - maximumJitterInterval := 1 * time.Millisecond - - client := NewClient( - WithHTTPTimeout(10*time.Millisecond), - WithCommandName("some_command_name"), - WithHystrixTimeout(10*time.Millisecond), - WithMaxConcurrentRequests(100), - WithErrorPercentThreshold(10), - WithSleepWindow(100), - WithRequestVolumeThreshold(10), - WithRetryCount(noOfRetries), - WithRetrier(heimdall.NewRetrier(heimdall.NewConstantBackoff(backoffInterval, maximumJitterInterval))), - ) - - ctx, cancel := context.WithCancel(context.Background()) - - dummyHandler := func(w http.ResponseWriter, r *http.Request) { - // Cancel the context after the first call - cancel() - - w.WriteHeader(http.StatusInternalServerError) - w.Header().Set("Content-Type", "application/json") - w.Write([]byte(`{"response": "something went wrong"}`)) - count++ - } - - server := httptest.NewServer(http.HandlerFunc(dummyHandler)) - defer server.Close() - - req, err := http.NewRequest(http.MethodGet, server.URL, nil) - require.NoError(t, err) - - response, err := client.Do(req.WithContext(ctx)) - require.Error(t, err, "should have failed to make request") - require.Nil(t, response) - - assert.Equal(t, 1, count) -} - func TestHystrixHTTPClientRetriesPostOnFailure(t *testing.T) { count := 0 backoffInterval := 1 * time.Millisecond From 9f2d72d9ec60c522e65795200fc0398a53609827 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Tue, 15 Sep 2020 17:40:30 +0200 Subject: [PATCH 4/7] fix: increase code coverage Signed-off-by: Yoan Blanc --- httpclient/client.go | 6 ++-- httpclient/client_test.go | 72 ++++++++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/httpclient/client.go b/httpclient/client.go index ffb6979..52ec0e7 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -171,7 +171,7 @@ outter: cancel() multiErr.Push(request.Context().Err().Error()) - c.reportError(request, err) + c.reportError(request, request.Context().Err()) // If the request context has already been cancelled, don't retry break outter @@ -187,12 +187,14 @@ outter: select { case <-ctx.Done(): cancel() + continue + case <-request.Context().Done(): cancel() multiErr.Push(request.Context().Err().Error()) - c.reportError(request, err) + c.reportError(request, request.Context().Err()) // If the request context has already been cancelled, don't retry break outter diff --git a/httpclient/client_test.go b/httpclient/client_test.go index f213d35..e5f32ef 100644 --- a/httpclient/client_test.go +++ b/httpclient/client_test.go @@ -419,7 +419,6 @@ func TestHTTPClientGetReturnsErrorOnFailure(t *testing.T) { } func TestHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { - count := 0 noOfRetries := 3 // Set a huge backoffInterval that we won't have to wait anyway backoffInterval := 1 * time.Hour @@ -431,27 +430,60 @@ func TestHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { WithRetrier(heimdall.NewRetrier(heimdall.NewConstantBackoff(backoffInterval, maximumJitterInterval))), ) - ctx, cancel := context.WithCancel(context.Background()) - - dummyHandler := func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(`{ "response": "something went wrong" }`)) - count++ - // Cancel the context after the first call - cancel() + tt := []struct { + Title string + CancelTimeout time.Duration + NotNilResponse bool + }{ + { + Title: "Cancel directly", + CancelTimeout: 0 * time.Millisecond, + NotNilResponse: false, + }, + { + Title: "Cancel afterwards", + CancelTimeout: 10 * time.Millisecond, + NotNilResponse: true, + }, } - server := httptest.NewServer(http.HandlerFunc(dummyHandler)) - defer server.Close() - - req, err := http.NewRequest(http.MethodGet, server.URL, nil) - require.NoError(t, err) - - response, err := client.Do(req.WithContext(ctx)) - require.Error(t, err, "should have failed to make request") - require.Nil(t, response) - - assert.Equal(t, 1, count) + for _, test := range tt { + test := test + t.Run(test.Title, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + + dummyHandler := func(w http.ResponseWriter, r *http.Request) { + if test.CancelTimeout == 0 { + cancel() + } else { + go func() { + time.Sleep(test.CancelTimeout) + cancel() + }() + } + + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{ "response": "something went wrong" }`)) + } + + server := httptest.NewServer(http.HandlerFunc(dummyHandler)) + defer server.Close() + + req, err := http.NewRequest(http.MethodGet, server.URL, nil) + require.NoError(t, err) + + response, err := client.Do(req.WithContext(ctx)) + require.Error(t, err, "should have failed to make request") + + if test.NotNilResponse { + require.NotNil(t, response) + } else { + require.Nil(t, response) + } + }) + } } func TestPluginMethodsCalled(t *testing.T) { From 3efdc7f5fc0b07dd6a44dfb93092256f18e6bb64 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Wed, 16 Sep 2020 07:20:21 +0200 Subject: [PATCH 5/7] fixup! fix: increase code coverage Signed-off-by: Yoan Blanc --- httpclient/client.go | 56 ++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/httpclient/client.go b/httpclient/client.go index 52ec0e7..d4c818c 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -157,52 +157,32 @@ outter: if err != nil { multiErr.Push(err.Error()) c.reportError(request, err) - - backoffTime := c.retrier.NextInterval(i) - ctx, cancel := context.WithTimeout(context.Background(), backoffTime) - - select { - case <-ctx.Done(): - cancel() - - continue - - case <-request.Context().Done(): - cancel() - - multiErr.Push(request.Context().Err().Error()) - c.reportError(request, request.Context().Err()) - - // If the request context has already been cancelled, don't retry - break outter - } + } else { + c.reportRequestEnd(request, response) } - c.reportRequestEnd(request, response) - - if response.StatusCode >= http.StatusInternalServerError { - backoffTime := c.retrier.NextInterval(i) - ctx, cancel := context.WithTimeout(context.Background(), backoffTime) + if err == nil && response.StatusCode < http.StatusInternalServerError { + // Clear errors if any iteration succeeds + multiErr = &valkyrie.MultiError{} + break + } - select { - case <-ctx.Done(): - cancel() + backoffTime := c.retrier.NextInterval(i) + ctx, cancel := context.WithTimeout(context.Background(), backoffTime) - continue + select { + case <-ctx.Done(): + cancel() - case <-request.Context().Done(): - cancel() + case <-request.Context().Done(): + cancel() - multiErr.Push(request.Context().Err().Error()) - c.reportError(request, request.Context().Err()) + multiErr.Push(request.Context().Err().Error()) + c.reportError(request, request.Context().Err()) - // If the request context has already been cancelled, don't retry - break outter - } + // If the request context has already been cancelled, don't retry + break outter } - - multiErr = &valkyrie.MultiError{} // Clear errors if any iteration succeeds - break } return response, multiErr.HasError() From c6b32d9d4e96fc13045e5ac73381072c98cfe7fa Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Wed, 16 Sep 2020 07:39:11 +0200 Subject: [PATCH 6/7] fix: #89 Signed-off-by: Yoan Blanc --- httpclient/client.go | 37 ++++++++++++++++++++----------------- httpclient/client_test.go | 1 + httpclient/options_test.go | 1 - 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/httpclient/client.go b/httpclient/client.go index d4c818c..612e13c 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -145,6 +145,26 @@ outter: response.Body.Close() } + // Wait before retrying. + if i > 0 { + backoffTime := c.retrier.NextInterval(i - 1) + ctx, cancel := context.WithTimeout(context.Background(), backoffTime) + + select { + case <-ctx.Done(): + cancel() + + case <-request.Context().Done(): + cancel() + + multiErr.Push(request.Context().Err().Error()) + c.reportError(request, request.Context().Err()) + + // If the request context has already been cancelled, don't retry + break outter + } + } + c.reportRequestStart(request) var err error response, err = c.client.Do(request) @@ -166,23 +186,6 @@ outter: multiErr = &valkyrie.MultiError{} break } - - backoffTime := c.retrier.NextInterval(i) - ctx, cancel := context.WithTimeout(context.Background(), backoffTime) - - select { - case <-ctx.Done(): - cancel() - - case <-request.Context().Done(): - cancel() - - multiErr.Push(request.Context().Err().Error()) - c.reportError(request, request.Context().Err()) - - // If the request context has already been cancelled, don't retry - break outter - } } return response, multiErr.HasError() diff --git a/httpclient/client_test.go b/httpclient/client_test.go index e5f32ef..0e01dba 100644 --- a/httpclient/client_test.go +++ b/httpclient/client_test.go @@ -465,6 +465,7 @@ func TestHTTPClientDontRetryWhenContextIsCancelled(t *testing.T) { } w.WriteHeader(http.StatusInternalServerError) + w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{ "response": "something went wrong" }`)) } diff --git a/httpclient/options_test.go b/httpclient/options_test.go index b92ce71..2e93673 100644 --- a/httpclient/options_test.go +++ b/httpclient/options_test.go @@ -128,6 +128,5 @@ func ExampleWithRetrier() { // Output: retry attempt 0 // retry attempt 1 // retry attempt 2 - // retry attempt 3 // error } From bbfba3c6351a0dd73d68a363d1fc26b4f83c2b47 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Wed, 16 Sep 2020 07:46:44 +0200 Subject: [PATCH 7/7] fixup! fix: #89 Signed-off-by: Yoan Blanc --- httpclient/client.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/httpclient/client.go b/httpclient/client.go index 612e13c..3d40cda 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -177,15 +177,19 @@ outter: if err != nil { multiErr.Push(err.Error()) c.reportError(request, err) - } else { - c.reportRequestEnd(request, response) + + continue } - if err == nil && response.StatusCode < http.StatusInternalServerError { - // Clear errors if any iteration succeeds - multiErr = &valkyrie.MultiError{} - break + c.reportRequestEnd(request, response) + + if response.StatusCode >= http.StatusInternalServerError { + continue } + + // Clear errors if any iteration succeeds + multiErr = &valkyrie.MultiError{} + break } return response, multiErr.HasError()