Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SNOW-948463 Removed error information for PUT command for GCP #801

Merged
merged 1 commit into from
Oct 27, 2023
Merged

SNOW-948463 Removed error information for PUT command for GCP #801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 50 additions & 3 deletions Snowflake.Data.Tests/IntegrationTests/SFPutGetTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,47 @@ public void TestPutFileRelativePathAsteriskWildcard()
}
}

[Test]
// presigned url is enabled on CI so we need to disable the test
// it should be enabled when downscoped credential is the default option
[IgnoreOnEnvIs("snowflake_cloud_env", new [] { "GCP" })]
public void TestPutFileWithoutOverwriteFlagSkipsSecondUpload()
{
// Set the PUT query variables
t_inputFilePath = $"{Guid.NewGuid()}.csv";
t_internalStagePath = $"@{t_schemaName}.{t_stageName}";

PrepareFileData(t_inputFilePath);

using (var conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();
PutFile(conn, expectedStatus: ResultStatus.UPLOADED);
VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
PutFile(conn, expectedStatus: ResultStatus.SKIPPED);
}
}

[Test]
public void TestPutFileWithOverwriteFlagRunsSecondUpload()
{
var overwriteAttribute = "OVERWRITE=TRUE";

// Set the PUT query variables
t_inputFilePath = $"{Guid.NewGuid()}.csv";
t_internalStagePath = $"@{t_schemaName}.{t_stageName}";

PrepareFileData(t_inputFilePath);

using (var conn = new SnowflakeDbConnection(ConnectionString))
{
conn.Open();
PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
VerifyFilesAreUploaded(conn, new List<string> { t_inputFilePath }, t_internalStagePath);
PutFile(conn, overwriteAttribute, expectedStatus: ResultStatus.UPLOADED);
}
}

[Test]
public void TestPutDirectoryAsteriskWildcard()
{
Expand Down Expand Up @@ -418,21 +459,26 @@ private static bool IsCompressedByTheDriver()
}

// PUT - upload file from local directory to the stage
private void PutFile(SnowflakeDbConnection conn)
void PutFile(
SnowflakeDbConnection conn,
String additionalAttribute = "",
ResultStatus expectedStatus = ResultStatus.UPLOADED)
{
using (var command = conn.CreateCommand())
{
// Prepare PUT query
string putQuery =
$"PUT file://{t_inputFilePath} {t_internalStagePath} AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}";
$"PUT file://{t_inputFilePath} {t_internalStagePath}" +
$" AUTO_COMPRESS={(t_autoCompress ? "TRUE" : "FALSE")}" +
$" {additionalAttribute}";

// Upload file
command.CommandText = putQuery;
var reader = command.ExecuteReader();
Assert.IsTrue(reader.Read());

// Check file status
Assert.AreEqual(ResultStatus.UPLOADED.ToString(),
Assert.AreEqual(expectedStatus.ToString(),
reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ResultStatus));
// Check source and destination compression type
if (t_autoCompress)
Expand All @@ -449,6 +495,7 @@ private void PutFile(SnowflakeDbConnection conn)
Assert.AreEqual(SFFileCompressionTypes.NONE.Name,
reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.DestinationCompressionType));
}
Assert.IsNull(reader.GetString((int)SFResultSet.PutGetResponseRowTypeInfo.ErrorDetails));
}
}

Expand Down
4 changes: 2 additions & 2 deletions Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ public void TestUploadFile(HttpStatusCode httpStatusCode, ResultStatus expectedR
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>()))
mockBlobClient.Setup(client => client.Upload(It.IsAny<Stream>(), It.IsAny<bool>(), It.IsAny<CancellationToken>()))
.Returns(() => MockAzureClient.createMockResponseForBlobContentInfo(key));

return mockBlobClient.Object;
Expand Down Expand Up @@ -251,7 +251,7 @@ public async Task TestUploadFileAsync(HttpStatusCode httpStatusCode, ResultStatu
.Returns<string>((blobName) =>
{
var mockBlobClient = new Mock<BlobClient>();
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<CancellationToken>()))
mockBlobClient.Setup(client => client.UploadAsync(It.IsAny<Stream>(), It.IsAny<bool>(),It.IsAny<CancellationToken>()))
.Returns(async () => await Task.Run(() => MockAzureClient.createMockResponseForBlobContentInfo(key)).ConfigureAwait(false));

return mockBlobClient.Object;
Expand Down
8 changes: 7 additions & 1 deletion Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,12 +201,18 @@ private void AssertForGetFileHeaderTests(ResultStatus expectedResultStatus, File
{
Assert.AreEqual(MockGCSClient.ContentLength, fileHeader.contentLength);
Assert.AreEqual(MockGCSClient.SFCDigest, fileHeader.digest);
Assert.IsNull(_fileMetadata.lastError);
}
else if (expectedResultStatus == ResultStatus.NOT_FOUND_FILE)
{
Assert.IsNull(fileHeader);
Assert.IsNull(_fileMetadata.lastError);
}
else
{
Assert.IsNull(fileHeader);
Assert.IsNotNull(_fileMetadata.lastError);
}

Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);
}

Expand Down
11 changes: 5 additions & 6 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFGCSClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,9 +444,7 @@ private void HandleDownloadResponse(HttpWebResponse response, SFFileMetadata fil
private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFileMetadata fileMetadata)
{
Logger.Error("Failed to get file header for presigned url: " + ex.Message);

fileMetadata.lastError = ex;


HttpWebResponse response = (HttpWebResponse)ex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized ||
response.StatusCode == HttpStatusCode.Forbidden ||
Expand All @@ -457,6 +455,7 @@ private SFFileMetadata HandleFileHeaderErrForPresignedUrls(WebException ex, SFFi
else
{
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
fileMetadata.lastError = ex;
}

return fileMetadata;
Expand All @@ -472,19 +471,18 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi
{
Logger.Error("Failed to get file header for non-presigned url: " + ex.Message);

// If file doesn't exist, GET request fails
fileMetadata.lastError = ex;
sfc-gh-knozderko marked this conversation as resolved.
Show resolved Hide resolved

HttpWebResponse response = (HttpWebResponse)ex.Response;
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
fileMetadata.resultStatus = ResultStatus.RENEW_TOKEN.ToString();
fileMetadata.lastError = ex;
}
else if (response.StatusCode == HttpStatusCode.Forbidden ||
response.StatusCode == HttpStatusCode.InternalServerError ||
response.StatusCode == HttpStatusCode.ServiceUnavailable)
{
fileMetadata.resultStatus = ResultStatus.NEED_RETRY.ToString();
fileMetadata.lastError = ex;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
Expand All @@ -493,6 +491,7 @@ private SFFileMetadata HandleFileHeaderErrForGeneratedUrls(WebException ex, SFFi
else
{
fileMetadata.resultStatus = ResultStatus.ERROR.ToString();
fileMetadata.lastError = ex;
}
return fileMetadata;
}
Expand Down
4 changes: 2 additions & 2 deletions Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ public void UploadFile(SFFileMetadata fileMetadata, Stream fileBytesStream, SFEn
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
blobClient.Upload(fileBytesStream);
blobClient.Upload(fileBytesStream, overwrite: true);
blobClient.SetMetadata(metadata);
}
catch (RequestFailedException ex)
Expand Down Expand Up @@ -221,7 +221,7 @@ public async Task UploadFileAsync(SFFileMetadata fileMetadata, Stream fileBytesS
{
// Issue the POST/PUT request
fileBytesStream.Position = 0;
await blobClient.UploadAsync(fileBytesStream, cancellationToken).ConfigureAwait(false);
await blobClient.UploadAsync(fileBytesStream, true, cancellationToken).ConfigureAwait(false);
blobClient.SetMetadata(metadata);
}
catch (RequestFailedException ex)
Expand Down