Clone request in retry strategy (#342)

Add request cloning in between retries.

src/Okta.Sdk.IntegrationTests/DefaultRetryStrategyScenarios.cs

@@ -0,0 +1,60 @@
+// <copyright file="DefaultRetryStrategyScenarios.cs" company="Okta, Inc">
+// Copyright (c) 2014 - present Okta, Inc. All rights reserved.
+// Licensed under the Apache 2.0 license. See the LICENSE file in the project root for full license information.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using Okta.Sdk.Configuration;
+using Xunit;
+
+namespace Okta.Sdk.IntegrationTests
+{
+    public class DefaultRetryStrategyScenarios
+    {
+        [Fact]
+        public void RetryShouldNotThrowInvalidOperationException()
+        {
+            List<string> guidList = new List<string>();
+
+            for (int i = 0; i < 150; i++)
+            {
+                guidList.Add(Guid.NewGuid().ToString());
+            }
+
+            var oktaClient = new OktaClient(
+                null,
+                new HttpClient(),
+                null,
+                new DefaultRetryStrategy(
+                    OktaClientConfiguration.DefaultMaxRetries,
+                    OktaClientConfiguration.DefaultRequestTimeout));
+
+            // Forcing several parallel requests to trigger rate limiting
+            guidList
+                .AsParallel()
+                .ForAll(t =>
+                {
+                    var log = oktaClient.Logs
+                        .GetLogs(
+                            q: $"eventType eq \"user.lifecycle.create\" and target.id eq \"{t.Trim()}\"",
+                            since: DateTime.Now.Add(TimeSpan.FromDays(-180d)).ToString("yyyy-MM-dd"))
+                        .Select(ev => ev.Actor.AlternateId)
+                        .FirstOrDefault();
+                    try
+                    {
+                        Assert.True(log.Result == null, "not found");
+                    }
+                    catch (AggregateException e)
+                    {
+                        foreach (var ex in e.Flatten().InnerExceptions)
+                        {
+                            Assert.False(ex.GetType() == typeof(InvalidOperationException) && ex.Message.Contains("The request message was already sent"), "The request shouldn't be reused.");
+                        }
+                    }
+                });
+        }
+    }
+}

src/Okta.Sdk.UnitTests/DefaultRetryStrategyShould.cs

@@ -34,7 +34,7 @@ public async Task RetryOperationUntilMaxRetriesIsReached()
             request.RequestUri = new Uri("https://foo.dev");
 
             var operation = Substitute.For<Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>>>();
-            operation(request, default(CancellationToken)).Returns(response);
+            operation(request, default(CancellationToken)).ReturnsForAnyArgs(response);
 
             operation(request, default(CancellationToken)).Result.StatusCode.Should().Be(429);
             operation.ClearReceivedCalls();
@@ -65,7 +65,7 @@ public async Task RetryOnlyWith429Responses()
             request.RequestUri = new Uri("https://foo.dev");
 
             var operation = Substitute.For<Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>>>();
-            operation(request, default(CancellationToken)).Returns(x => response, x => successResponse);
+            operation(request, default(CancellationToken)).ReturnsForAnyArgs(x => response, x => successResponse);
 
             var retryStrategy = new DefaultRetryStrategy(5, 0);
 
@@ -93,10 +93,11 @@ public async Task AddOktaHeadersInRetry()
             var numberOfExecutions = 0;
             var operation = Substitute.For<Func<HttpRequestMessage, CancellationToken, Task<HttpResponseMessage>>>();
 
-            operation(request, default(CancellationToken)).Returns(
+            operation(request, default(CancellationToken)).ReturnsForAnyArgs(
                 x =>
                 {
-                    requestHeadersDictionary.Add(numberOfExecutions, request.Headers.ToList());
+                    var receivedRequest = (HttpRequestMessage)x.Args().GetValue(0);
+                    requestHeadersDictionary.Add(numberOfExecutions, receivedRequest.Headers.ToList());
                     numberOfExecutions++;
 
                     return response;
@@ -105,6 +106,7 @@ public async Task AddOktaHeadersInRetry()
             var retryStrategy = new DefaultRetryStrategy(1, 0);
 
             var retryResponse = await retryStrategy.WaitAndRetryAsync(request, default(CancellationToken), operation);
+
             numberOfExecutions.Should().Be(2);
             retryResponse.StatusCode.Should().Be((HttpStatusCode)429);
 

src/Okta.Sdk/DefaultRetryStrategy.cs

@@ -6,6 +6,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
@@ -79,7 +80,7 @@ public async Task<HttpResponseMessage> WaitAndRetryAsync(HttpRequestMessage requ
                     {
                         await Task.Delay(delayTimeSpan, cancellationToken).ConfigureAwait(false);
                         response.Headers.TryGetValues("X-Okta-Request-Id", out var requestId);
-                        AddRetryOktaHeaders(request, requestId.FirstOrDefault(), numberOfRetries);
+                        request = await AddRetryOktaHeadersAsync(request, requestId.FirstOrDefault(), numberOfRetries).ConfigureAwait(false);
                     }
                     else
                     {
@@ -102,19 +103,23 @@ public async Task<HttpResponseMessage> WaitAndRetryAsync(HttpRequestMessage requ
         public bool IsRetryable(HttpResponseMessage response)
             => response != null && _retryableStatusCodes.Contains(response.StatusCode);
 
-        private void AddRetryOktaHeaders(HttpRequestMessage request, string requestId, int numberOfRetries)
+        private async Task<HttpRequestMessage> AddRetryOktaHeadersAsync(HttpRequestMessage request, string requestId, int numberOfRetries)
         {
-            if (!request.Headers.Contains("X-Okta-Retry-For"))
+            var clonedRequest = await CloneHttpRequestMessageAsync(request).ConfigureAwait(false);
+
+            if (!clonedRequest.Headers.Contains("X-Okta-Retry-For"))
             {
-                request.Headers.Add("X-Okta-Retry-For", requestId);
+                clonedRequest.Headers.Add("X-Okta-Retry-For", requestId);
             }
 
-            if (request.Headers.Contains("X-Okta-Retry-Count"))
+            if (clonedRequest.Headers.Contains("X-Okta-Retry-Count"))
             {
-                request.Headers.Remove("X-Okta-Retry-Count");
+                clonedRequest.Headers.Remove("X-Okta-Retry-Count");
             }
 
-            request.Headers.Add("X-Okta-Retry-Count", numberOfRetries.ToString());
+            clonedRequest.Headers.Add("X-Okta-Retry-Count", numberOfRetries.ToString());
+
+            return clonedRequest;
         }
 
         private TimeSpan CalculateDelay(HttpResponseMessage response)
@@ -146,5 +151,42 @@ private TimeSpan CalculateDelay(HttpResponseMessage response)
 
             return backoffSeconds;
         }
+
+        private static async Task<HttpRequestMessage> CloneHttpRequestMessageAsync(HttpRequestMessage request)
+        {
+            HttpRequestMessage clonedRequest = new HttpRequestMessage(request.Method, request.RequestUri);
+
+            if (request.Content != null)
+            {
+                // Copy the request's content (via a MemoryStream) into the cloned object
+                var memoryStream = new MemoryStream();
+                await request.Content.CopyToAsync(memoryStream).ConfigureAwait(false);
+                memoryStream.Position = 0;
+                clonedRequest.Content = new StreamContent(memoryStream);
+
+                // Copy the content headers
+                if (request.Content.Headers != null)
+                {
+                    foreach (var header in request.Content.Headers)
+                    {
+                        clonedRequest.Content.Headers.Add(header.Key, header.Value);
+                    }
+                }
+            }
+
+            clonedRequest.Version = request.Version;
+
+            foreach (KeyValuePair<string, object> property in request.Properties)
+            {
+                clonedRequest.Properties.Add(property);
+            }
+
+            foreach (KeyValuePair<string, IEnumerable<string>> header in request.Headers)
+            {
+                clonedRequest.Headers.TryAddWithoutValidation(header.Key, header.Value);
+            }
+
+            return clonedRequest;
+        }
     }
 }