From e2272f107f84b504371a58ac5a9c10ef4c76f52e Mon Sep 17 00:00:00 2001 From: Krzysztof Nozderko Date: Mon, 2 Dec 2024 15:45:12 +0100 Subject: [PATCH] SNOW-1569293 Read encryption headers in a case insensitive way --- .../UnitTests/SFAzureClientTest.cs | 35 ++++++++++- .../UnitTests/SFGCSClientTest.cs | 63 +++++++++++++++---- .../UnitTests/SFS3ClientTest.cs | 55 +++++++++++----- .../FileTransfer/StorageClient/SFS3Client.cs | 23 +++++-- .../StorageClient/SFSnowflakeAzureClient.cs | 27 ++++++-- 5 files changed, 164 insertions(+), 39 deletions(-) diff --git a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs index 08b85a9b5..745f5eaeb 100644 --- a/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs @@ -21,7 +21,7 @@ namespace Snowflake.Data.Tests.UnitTests using Azure; using Azure.Storage.Blobs.Models; - [TestFixture] + [TestFixture, NonParallelizable] class SFAzureClientTest : SFBaseTest { // Mock data for file metadata @@ -377,5 +377,38 @@ public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultSta // Assert Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus); } + + [Test] + public void TestEncryptionMetadataReadingIsCaseInsensitive() + { + // arrange + var metadata = new Dictionary + { + { + "ENCRYPTIONDATA", + @"{ + ""ContentEncryptionIV"": ""initVector"", + ""WrappedContentKey"": { + ""EncryptedKey"": ""key"" + } + }" + }, + { "MATDESC", "description" }, + { "SFCDIGEST", "something"} + }; + var blobProperties = BlobsModelFactory.BlobProperties(metadata: metadata, contentLength: 10); + var mockBlobServiceClient = new Mock(); + _client = new SFSnowflakeAzureClient(_fileMetadata.stageInfo, mockBlobServiceClient.Object); + + // act + var fileHeader = _client.HandleFileHeaderResponse(ref _fileMetadata, blobProperties); + + // assert + Assert.AreEqual(ResultStatus.UPLOADED.ToString(), _fileMetadata.resultStatus); + Assert.AreEqual("something", fileHeader.digest); + Assert.AreEqual("initVector", fileHeader.encryptionMetadata.iv); + Assert.AreEqual("key", fileHeader.encryptionMetadata.key); + Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc); + } } } diff --git a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs index 9d336125e..ed1257894 100644 --- a/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs @@ -3,21 +3,22 @@ */ using System; +using NUnit.Framework; +using Snowflake.Data.Core; +using Snowflake.Data.Core.FileTransfer.StorageClient; +using Snowflake.Data.Core.FileTransfer; +using System.Collections.Generic; +using System.Net; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading.Tasks; +using System.Threading; +using Snowflake.Data.Tests.Mock; +using Moq; namespace Snowflake.Data.Tests.UnitTests { - using NUnit.Framework; - using Snowflake.Data.Core; - using Snowflake.Data.Core.FileTransfer.StorageClient; - using Snowflake.Data.Core.FileTransfer; - using System.Collections.Generic; - using System.Net; - using System.IO; - using System.Threading.Tasks; - using System.Threading; - using Snowflake.Data.Tests.Mock; - using Moq; - [TestFixture, NonParallelizable] class SFGCSClientTest : SFBaseTest { @@ -371,6 +372,44 @@ public void TestUseUriWithRegionsWhenNeeded(string region, string endPoint, bool Assert.AreEqual(expectedRequestUri, uri); } + [Test] + [TestCase("some-header-name", "SOME-HEADER-NAME")] + [TestCase("SOME-HEADER-NAME", "some-header-name")] + public void TestGcsHeadersAreCaseInsensitiveForHttpResponseMessage(string headerNameToAdd, string headerNameToGet) + { + // arrange + const string HeaderValue = "someValue"; + var responseMessage = new HttpResponseMessage( HttpStatusCode.OK ) {Content = new StringContent( "Response content" ) }; + responseMessage.Headers.Add(headerNameToAdd, HeaderValue); + + // act + var header = responseMessage.Headers.GetValues(headerNameToGet); + + // assert + Assert.NotNull(header); + Assert.AreEqual(1, header.Count()); + Assert.AreEqual(HeaderValue, header.First()); + } + + [Test] + [TestCase("some-header-name", "SOME-HEADER-NAME")] + [TestCase("SOME-HEADER-NAME", "some-header-name")] + public void TestGcsHeadersAreCaseInsensitiveForWebHeaderCollection(string headerNameToAdd, string headerNameToGet) + { + // arrange + const string HeaderValue = "someValue"; + var headers = new WebHeaderCollection(); + headers.Add(headerNameToAdd, HeaderValue); + + // act + var header = headers.GetValues(headerNameToGet); + + // assert + Assert.NotNull(header); + Assert.AreEqual(1, header.Count()); + Assert.AreEqual(HeaderValue, header.First()); + } + private void AssertForDownloadFileTests(ResultStatus expectedResultStatus) { if (expectedResultStatus == ResultStatus.DOWNLOADED) diff --git a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs index 5432b0121..0f2de32a6 100644 --- a/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs +++ b/Snowflake.Data.Tests/UnitTests/SFS3ClientTest.cs @@ -1,27 +1,25 @@ /* - * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. + * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved. */ using System; -using Amazon.S3.Encryption; +using NUnit.Framework; +using Snowflake.Data.Core; +using Snowflake.Data.Core.FileTransfer.StorageClient; +using Snowflake.Data.Core.FileTransfer; +using System.Collections.Generic; +using Amazon.S3; +using Snowflake.Data.Tests.Mock; +using System.Threading.Tasks; +using Amazon; +using System.Threading; +using System.IO; +using Moq; +using Amazon.S3.Model; namespace Snowflake.Data.Tests.UnitTests { - using NUnit.Framework; - using Snowflake.Data.Core; - using Snowflake.Data.Core.FileTransfer.StorageClient; - using Snowflake.Data.Core.FileTransfer; - using System.Collections.Generic; - using Amazon.S3; - using Snowflake.Data.Tests.Mock; - using System.Threading.Tasks; - using Amazon; - using System.Threading; - using System.IO; - using Moq; - using Amazon.S3.Model; - - [TestFixture] + [TestFixture, NonParallelizable] class SFS3ClientTest : SFBaseTest { // Mock data for file metadata @@ -320,6 +318,29 @@ public async Task TestDownloadFileAsync(string awsStatusCode, ResultStatus expec AssertForDownloadFileTests(expectedResultStatus); } + [Test] + public void TestEncryptionMetadataReadingIsCaseInsensitive() + { + // arrange + var mockAmazonS3Client = new Mock(AwsKeyId, AwsSecretKey, AwsToken, _clientConfig); + _client = new SFS3Client(_fileMetadata.stageInfo, MaxRetry, Parallel, _proxyCredentials, mockAmazonS3Client.Object); + var response = new GetObjectResponse(); + response.Metadata.Add(SFS3Client.AMZ_IV.ToUpper(), "initVector"); + response.Metadata.Add(SFS3Client.AMZ_KEY.ToUpper(), "key"); + response.Metadata.Add(SFS3Client.AMZ_MATDESC.ToUpper(), "description"); + response.Metadata.Add(SFS3Client.SFC_DIGEST.ToUpper(), "something"); + + // act + var fileHeader = _client.HandleFileHeaderResponse(ref _fileMetadata, response); + + // assert + Assert.AreEqual(ResultStatus.UPLOADED.ToString(), _fileMetadata.resultStatus); + Assert.AreEqual("something", fileHeader.digest); + Assert.AreEqual("initVector", fileHeader.encryptionMetadata.iv); + Assert.AreEqual("key", fileHeader.encryptionMetadata.key); + Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc); + } + private void AssertForDownloadFileTests(ResultStatus expectedResultStatus) { if (expectedResultStatus == ResultStatus.DOWNLOADED) diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs index ea0eb3fd0..524dc23c1 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFS3Client.cs @@ -9,6 +9,7 @@ using Snowflake.Data.Log; using System; using System.IO; +using System.Linq; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -266,26 +267,38 @@ private GetObjectRequest GetFileHeaderRequest(ref AmazonS3Client client, SFFileM /// The S3 file metadata. /// The Amazon S3 response. /// The file header of the S3 file. - private FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, GetObjectResponse response) + internal FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, GetObjectResponse response) { // Update the result status of the file metadata fileMetadata.resultStatus = ResultStatus.UPLOADED.ToString(); SFEncryptionMetadata encryptionMetadata = new SFEncryptionMetadata { - iv = response.Metadata[AMZ_IV], - key = response.Metadata[AMZ_KEY], - matDesc = response.Metadata[AMZ_MATDESC] + iv = GetMetadataCaseInsensitive(response.Metadata, AMZ_IV), + key = GetMetadataCaseInsensitive(response.Metadata, AMZ_KEY), + matDesc = GetMetadataCaseInsensitive(response.Metadata, AMZ_MATDESC) }; return new FileHeader { - digest = response.Metadata[SFC_DIGEST], + digest = GetMetadataCaseInsensitive(response.Metadata, SFC_DIGEST), contentLength = response.ContentLength, encryptionMetadata = encryptionMetadata }; } + private string GetMetadataCaseInsensitive(MetadataCollection metadataCollection, string metadataKey) + { + var value = metadataCollection[metadataKey]; + if (value != null) + return value; + if (string.IsNullOrEmpty(metadataKey)) + return null; + var keysCaseInsensitive = metadataCollection.Keys + .Where(key => $"x-amz-meta-{metadataKey}".Equals(key, StringComparison.OrdinalIgnoreCase)); + return keysCaseInsensitive.Any() ? metadataCollection[keysCaseInsensitive.First()] : null; + } + /// /// Set the client configuration common to both client with and without client-side /// encryption. diff --git a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs index 98c2694cb..d13dc01b9 100644 --- a/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs +++ b/Snowflake.Data/Core/FileTransfer/StorageClient/SFSnowflakeAzureClient.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Azure; using Azure.Storage.Blobs.Models; using Newtonsoft.Json; @@ -154,30 +155,48 @@ public async Task GetFileHeaderAsync(SFFileMetadata fileMetadata, Ca /// The S3 file metadata. /// The Amazon S3 response. /// The file header of the S3 file. - private FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, BlobProperties response) + internal FileHeader HandleFileHeaderResponse(ref SFFileMetadata fileMetadata, BlobProperties response) { fileMetadata.resultStatus = ResultStatus.UPLOADED.ToString(); SFEncryptionMetadata encryptionMetadata = null; - if (response.Metadata.TryGetValue("encryptiondata", out var encryptionDataStr)) + if (TryGetMetadataValueCaseInsensitive(response, "encryptiondata", out var encryptionDataStr)) { dynamic encryptionData = JsonConvert.DeserializeObject(encryptionDataStr); encryptionMetadata = new SFEncryptionMetadata { iv = encryptionData["ContentEncryptionIV"], key = encryptionData.WrappedContentKey["EncryptedKey"], - matDesc = response.Metadata["matdesc"] + matDesc = GetMetadataValueCaseInsensitive(response, "matdesc") }; } return new FileHeader { - digest = response.Metadata["sfcdigest"], + digest = GetMetadataValueCaseInsensitive(response, "sfcdigest"), contentLength = response.ContentLength, encryptionMetadata = encryptionMetadata }; } + private bool TryGetMetadataValueCaseInsensitive(BlobProperties properties, string metadataKey, out string metadataValue) + { + if (properties.Metadata.TryGetValue(metadataKey, out metadataValue)) + return true; + if (string.IsNullOrEmpty(metadataKey)) + return false; + var keysCaseInsensitive = properties.Metadata.Keys + .Where(key => metadataKey.Equals(key, StringComparison.OrdinalIgnoreCase)); + return keysCaseInsensitive.Any() ? properties.Metadata.TryGetValue(keysCaseInsensitive.First(), out metadataValue) : false; + } + + private string GetMetadataValueCaseInsensitive(BlobProperties properties, string metadataKey) + { + if (TryGetMetadataValueCaseInsensitive(properties, metadataKey, out var metadataValue)) + return metadataValue; + throw new KeyNotFoundException($"The given key '{metadataKey}' was not present in the dictionary."); + } + /// /// Upload the file to the Azure location. ///