Skip to content

Commit

Permalink
SNOW-1739483 fixes after CR
Browse files Browse the repository at this point in the history
  • Loading branch information
@sfc-gh-dstempniak
sfc-gh-dstempniak committed Oct 22, 2024
1 parent 797bfcd commit 85bd602
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 18 deletions.
30 changes: 16 additions & 14 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -525,12 +525,12 @@ private void HandleFileHeaderErr(Exception ex, SFFileMetadata fileMetadata)

switch (ex)
{
case AmazonS3Exception err:
if (err.ErrorCode == EXPIRED_TOKEN || err.ErrorCode == HttpStatusCode.BadRequest.ToString())
case AmazonS3Exception exAws:
if (exAws.ErrorCode == EXPIRED_TOKEN || exAws.ErrorCode == HttpStatusCode.BadRequest.ToString())
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
}
else if (err.ErrorCode == NO_SUCH_KEY)
else if (exAws.ErrorCode == NO_SUCH_KEY)
{
fileMetadata.resultStatus = ResultStatus.NOT_FOUND_FILE.ToString();
}
Expand All @@ -541,7 +541,7 @@ private void HandleFileHeaderErr(Exception ex, SFFileMetadata fileMetadata)

break;
default:
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();;
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
break;
}
}
Expand All @@ -557,20 +557,21 @@ private void HandleUploadFileErr(Exception ex, SFFileMetadata fileMetadata)

switch (ex)
{
case AmazonS3Exception err:
if (err.ErrorCode == EXPIRED_TOKEN)
case AmazonS3Exception exAws:
if (exAws.ErrorCode == EXPIRED_TOKEN)
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
}
else
{
fileMetadata.lastError = err;
fileMetadata.lastError = exAws;
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
}
break;

default:
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();;
case Exception exOther:
fileMetadata.lastError = exOther;
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
break;
}
}
Expand All @@ -586,20 +587,21 @@ private void HandleDownloadFileErr(Exception ex, SFFileMetadata fileMetadata)

switch (ex)
{
case AmazonS3Exception err:
if (err.ErrorCode == EXPIRED_TOKEN)
case AmazonS3Exception exAws:
if (exAws.ErrorCode == EXPIRED_TOKEN)
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
}
else
{
fileMetadata.lastError = err;
fileMetadata.lastError = exAws;
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
}
break;

default:
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();;
case Exception exOther:
fileMetadata.lastError = exOther;
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
break;
}
}
Expand Down
22 changes: 18 additions & 4 deletions Snowflake.Data/Core/HttpUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,9 +378,9 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
UriUpdater updater = new UriUpdater(requestMessage.RequestUri, includeRetryReason);
int retryCount = 0;

long startTimeInMilliseconds = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
while (true)
{

try
{
childCts = null;
Expand All @@ -401,13 +401,12 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
lastException = e;
if (cancellationToken.IsCancellationRequested)
{
logger.Debug("SF rest request timeout or explicit cancel called.");
logger.Info("SF rest request timeout or explicit cancel called.");
cancellationToken.ThrowIfCancellationRequested();
}
else if (childCts != null && childCts.Token.IsCancellationRequested)
{
logger.Warn("Http request timeout. Retry the request");
totalRetryTime += (int)httpTimeout.TotalSeconds;
}
else
{
Expand All @@ -426,6 +425,8 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
}
}

totalRetryTime = (int)((DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - startTimeInMilliseconds) / 1000);

if (childCts != null)
{
childCts.Dispose();
Expand Down Expand Up @@ -464,6 +465,19 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
logger.Info("Response returned was null.");
}

if (restTimeout.TotalSeconds > 0 && totalRetryTime > restTimeout.TotalSeconds)
{
logger.Debug($"stop retry as connection_timeout {restTimeout.TotalSeconds} sec. reached");
if (response != null)
{
return response;

Check warning on line 473 in Snowflake.Data/Core/HttpUtil.cs

View check run for this annotation

Codecov / codecov/patch

Snowflake.Data/Core/HttpUtil.cs#L472-L473 

Added lines #L472 - L473 were not covered by tests
}
var errorMessage = $"http request failed and connection_timeout {restTimeout.TotalSeconds} sec. reached.\n";
errorMessage += $"Last exception encountered: {lastException}";
logger.Error(errorMessage);
throw new OperationCanceledException(errorMessage);
}

retryCount++;
if ((maxRetryCount > 0) && (retryCount > maxRetryCount))
{
Expand All @@ -486,7 +500,6 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
logger.Debug($"Sleep {backOffInSec} seconds and then retry the request, retryCount: {retryCount}");

await Task.Delay(TimeSpan.FromSeconds(backOffInSec), cancellationToken).ConfigureAwait(false);
totalRetryTime += backOffInSec;

var jitter = GetJitter(backOffInSec);

Expand All @@ -504,6 +517,7 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
backOffInSec *= 2;
}

totalRetryTime = (int)((DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - startTimeInMilliseconds) / 1000);
if ((restTimeout.TotalSeconds > 0) && (totalRetryTime + backOffInSec > restTimeout.TotalSeconds))
{
// No need to wait more than necessary if it can be avoided.
Expand Down

0 comments on commit 85bd602

Please sign in to comment.