Merge branch 'master' into SNOW-1757860_bc_fips_complient_encryption

.github/workflows/main.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['net6.0', 'net7.0', 'net8.0']
+        dotnet: ['net6.0', 'net7.0', 'net8.0', 'net9.0']
         cloud_env: ['AZURE', 'GCP', 'AWS']
         target_framework: ['netstandard2.0', 'netstandard2.1']
     steps:
@@ -43,6 +43,7 @@ jobs:
           dotnet-version: |
             6.0.x
             8.0.x
+            9.0.x
           dotnet-quality: 'ga'
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -109,6 +110,7 @@ jobs:
           dotnet-version: |
             6.0.x
             8.0.x
+            9.0.x
           dotnet-quality: 'ga'
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -163,7 +165,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['net6.0', 'net7.0', 'net8.0']
+        dotnet: ['net6.0', 'net7.0', 'net8.0', 'net9.0']
         cloud_env: ['AZURE', 'GCP', 'AWS']
         target_framework: ['netstandard2.0', 'netstandard2.1']
     steps:
@@ -174,6 +176,7 @@ jobs:
           dotnet-version: |
             6.0.x
             8.0.x
+            9.0.x
           dotnet-quality: 'ga'
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -226,7 +229,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dotnet: ['net6.0', 'net7.0', 'net8.0']
+        dotnet: ['net6.0', 'net7.0', 'net8.0', 'net9.0']
         cloud_env: ['AZURE', 'GCP', 'AWS']
         target_framework: ['netstandard2.0', 'netstandard2.1']
     steps:
@@ -237,6 +240,7 @@ jobs:
           dotnet-version: |
             6.0.x
             8.0.x
+            9.0.x
           dotnet-quality: 'ga'
       - name: Setup Python
         uses: actions/setup-python@v5

Jenkinsfile

@@ -28,7 +28,7 @@ timestamps {
       string(name: 'branch', value: 'main'),
       string(name: 'client_git_commit', value: scmInfo.GIT_COMMIT),
       string(name: 'client_git_branch', value: scmInfo.GIT_BRANCH),
-      string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'dotnet-ubuntu204-net8'),
+      string(name: 'TARGET_DOCKER_TEST_IMAGE', value: 'dotnet-ubuntu204-net9'),
       string(name: 'parent_job', value: env.JOB_NAME),
       string(name: 'parent_build_number', value: env.BUILD_NUMBER)
     ]

README.md

@@ -4,7 +4,7 @@
 [![NuGet](https://img.shields.io/nuget/v/Snowflake.Data.svg)](https://www.nuget.org/packages/Snowflake.Data/)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
-The Snowflake .NET connector supports the the following .NET framework and libraries versions:
+The Snowflake .NET connector supports the following .NET framework and libraries versions:
 
 - .NET Framework 4.6.2
 - .NET Framework 4.7.1
@@ -14,6 +14,7 @@ The Snowflake .NET connector supports the the following .NET framework and libra
 - .NET 6.0
 - .NET 7.0
 - .NET 8.0
+- .NET 9.0
 
 Disclaimer: While the connector targets netstandard2.0 and may work with versions in its [support matrix](https://learn.microsoft.com/en-us/dotnet/standard/net-standard?tabs=net-standard-2-0#select-net-standard-version), only the versions listed above are supported and tested by the connector
 

Snowflake.Data.Tests/Snowflake.Data.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFrameworks>net6.0;net7.0;net8.0;net462;net471;net472;net48;net481</TargetFrameworks>
-    <TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">net6.0;net7.0;net8.0;</TargetFrameworks>
+    <TargetFrameworks>net6.0;net7.0;net8.0;net9.0;net462;net471;net472;net48;net481</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
     <Title>Snowflake.Data.Tests</Title>
     <Description>Snowflake Connector for .NET</Description>
     <Company>Snowflake Computing, Inc</Company>

Snowflake.Data.Tests/UnitTests/PutGetStageInfoTest.cs

@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2024 Snowflake Computing Inc. All rights reserved.
+ */
+
+using System.Collections.Generic;
+using NUnit.Framework;
+using Snowflake.Data.Core;
+using Snowflake.Data.Core.FileTransfer;
+
+namespace Snowflake.Data.Tests.UnitTests
+{
+    [TestFixture]
+    public class PutGetStageInfoTest
+    {
+        [Test]
+        [TestCaseSource(nameof(TestCases))]
+        public void TestGcsRegionalUrl(string region, bool useRegionalUrl, string endPoint, string expectedGcsEndpoint)
+        {
+            // arrange
+            var stageInfo = CreateGcsStageInfo(region, useRegionalUrl, endPoint);
+
+            // act
+            var gcsCustomEndpoint = stageInfo.GcsCustomEndpoint();
+
+            // assert
+            Assert.AreEqual(expectedGcsEndpoint, gcsCustomEndpoint);
+        }
+
+        internal static IEnumerable<object[]> TestCases()
+        {
+            yield return new object[] { "US-CENTRAL1", false, null, null };
+            yield return new object[] { "US-CENTRAL1", false, "", null };
+            yield return new object[] { "US-CENTRAL1", false, "null", null };
+            yield return new object[] { "US-CENTRAL1", false, "    ", null };
+            yield return new object[] { "US-CENTRAL1", false, "example.com", "example.com" };
+            yield return new object[] { "ME-CENTRAL2", false, null, "storage.me-central2.rep.googleapis.com" };
+            yield return new object[] { "ME-CENTRAL2", true, null, "storage.me-central2.rep.googleapis.com" };
+            yield return new object[] { "ME-CENTRAL2", true, "", "storage.me-central2.rep.googleapis.com" };
+            yield return new object[] { "ME-CENTRAL2", true, "  ", "storage.me-central2.rep.googleapis.com" };
+            yield return new object[] { "ME-CENTRAL2", true, "example.com", "example.com" };
+            yield return new object[] { "US-CENTRAL1", true, null, "storage.us-central1.rep.googleapis.com" };
+            yield return new object[] { "US-CENTRAL1", true, "", "storage.us-central1.rep.googleapis.com" };
+            yield return new object[] { "US-CENTRAL1", true, " ", "storage.us-central1.rep.googleapis.com" };
+            yield return new object[] { "US-CENTRAL1", true, "null", "storage.us-central1.rep.googleapis.com" };
+            yield return new object[] { "US-CENTRAL1", true, "example.com", "example.com" };
+        }
+
+        private PutGetStageInfo CreateGcsStageInfo(string region, bool useRegionalUrl, string endPoint) =>
+            new PutGetStageInfo
+            {
+                locationType = SFRemoteStorageUtil.GCS_FS,
+                location = "some location",
+                path = "some path",
+                region = region,
+                storageAccount = "some storage account",
+                isClientSideEncrypted = true,
+                stageCredentials = new Dictionary<string, string>(),
+                presignedUrl = "some pre-signed url",
+                endPoint = endPoint,
+                useRegionalUrl = useRegionalUrl
+            };
+    }
+}

Snowflake.Data.Tests/UnitTests/SFAzureClientTest.cs

@@ -21,7 +21,7 @@ namespace Snowflake.Data.Tests.UnitTests
     using Azure;
     using Azure.Storage.Blobs.Models;
 
-    [TestFixture]
+    [TestFixture, NonParallelizable]
     class SFAzureClientTest : SFBaseTest
     {
         // Mock data for file metadata
@@ -377,5 +377,38 @@ public async Task TestDownloadFileAsync(HttpStatusCode httpStatusCode, ResultSta
             // Assert
             Assert.AreEqual(expectedResultStatus.ToString(), _fileMetadata.resultStatus);
         }
+
+        [Test]
+        public void TestEncryptionMetadataReadingIsCaseInsensitive()
+        {
+            // arrange
+            var metadata = new Dictionary<string, string>
+            {
+                {
+                    "ENCRYPTIONDATA",
+                    @"{
+                        ""ContentEncryptionIV"": ""initVector"",
+                        ""WrappedContentKey"": {
+                            ""EncryptedKey"": ""key""
+                        }
+                    }"
+                },
+                { "MATDESC", "description" },
+                { "SFCDIGEST", "something"}
+            };
+            var blobProperties = BlobsModelFactory.BlobProperties(metadata: metadata, contentLength: 10);
+            var mockBlobServiceClient = new Mock<BlobServiceClient>();
+            _client = new SFSnowflakeAzureClient(_fileMetadata.stageInfo, mockBlobServiceClient.Object);
+
+            // act
+            var fileHeader = _client.HandleFileHeaderResponse(ref _fileMetadata, blobProperties);
+
+            // assert
+            Assert.AreEqual(ResultStatus.UPLOADED.ToString(), _fileMetadata.resultStatus);
+            Assert.AreEqual("something", fileHeader.digest);
+            Assert.AreEqual("initVector", fileHeader.encryptionMetadata.iv);
+            Assert.AreEqual("key", fileHeader.encryptionMetadata.key);
+            Assert.AreEqual("description", fileHeader.encryptionMetadata.matDesc);
+        }
     }
 }

Snowflake.Data.Tests/UnitTests/SFEnvironmentTest.cs

@@ -22,6 +22,8 @@ public void TestRuntimeExtraction()
             expectedVersion = "7.0";
 #elif NET8_0
             expectedVersion = "8.0";
+#elif NET9_0
+            expectedVersion = "9.0";
 #endif
 
             // Act

Snowflake.Data.Tests/UnitTests/SFGCSClientTest.cs

@@ -1,24 +1,25 @@
 /*
- * Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
+ * Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
  */
 
 using System;
+using NUnit.Framework;
+using Snowflake.Data.Core;
+using Snowflake.Data.Core.FileTransfer.StorageClient;
+using Snowflake.Data.Core.FileTransfer;
+using System.Collections.Generic;
+using System.Net;
+using System.IO;
+using System.Linq;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Threading;
+using Snowflake.Data.Tests.Mock;
+using Moq;
 
 namespace Snowflake.Data.Tests.UnitTests
 {
-    using NUnit.Framework;
-    using Snowflake.Data.Core;
-    using Snowflake.Data.Core.FileTransfer.StorageClient;
-    using Snowflake.Data.Core.FileTransfer;
-    using System.Collections.Generic;
-    using System.Net;
-    using System.IO;
-    using System.Threading.Tasks;
-    using System.Threading;
-    using Snowflake.Data.Tests.Mock;
-    using Moq;
-
-    [TestFixture]
+    [TestFixture, NonParallelizable]
     class SFGCSClientTest : SFBaseTest
     {
         // Mock data for file metadata
@@ -340,6 +341,75 @@ public async Task TestDownloadFileAsync(HttpStatusCode? httpStatusCode, ResultSt
             AssertForDownloadFileTests(expectedResultStatus);
         }
 
+        [Test]
+        [TestCase("us-central1", null, null, "https://storage.googleapis.com/mock-customer-stage/mock-id/tables/mock-key/")]
+        [TestCase("us-central1", "example.com", null, "https://example.com/mock-customer-stage/mock-id/tables/mock-key/")]
+        [TestCase("us-central1", "https://example.com", null, "https://example.com/mock-customer-stage/mock-id/tables/mock-key/")]
+        [TestCase("us-central1", null, true, "https://storage.us-central1.rep.googleapis.com/mock-customer-stage/mock-id/tables/mock-key/")]
+        [TestCase("me-central2", null, null, "https://storage.me-central2.rep.googleapis.com/mock-customer-stage/mock-id/tables/mock-key/")]
+        public void TestUseUriWithRegionsWhenNeeded(string region, string endPoint, bool useRegionalUrl, string expectedRequestUri)
+        {
+            var fileMetadata = new SFFileMetadata()
+            {
+                stageInfo = new PutGetStageInfo()
+                {
+                    endPoint = endPoint,
+                    location = Location,
+                    locationType = SFRemoteStorageUtil.GCS_FS,
+                    path = LocationPath,
+                    presignedUrl = null,
+                    region = region,
+                    stageCredentials = _stageCredentials,
+                    storageAccount = null,
+                    useRegionalUrl = useRegionalUrl
+                }
+            };
+
+            // act
+            var uri = _client.FormBaseRequest(fileMetadata, "PUT").RequestUri.ToString();
+
+            // assert
+            Assert.AreEqual(expectedRequestUri, uri);
+        }
+
+        [Test]
+        [TestCase("some-header-name", "SOME-HEADER-NAME")]
+        [TestCase("SOME-HEADER-NAME", "some-header-name")]
+        public void TestGcsHeadersAreCaseInsensitiveForHttpResponseMessage(string headerNameToAdd, string headerNameToGet)
+        {
+            // arrange
+            const string HeaderValue = "someValue";
+            var responseMessage = new HttpResponseMessage( HttpStatusCode.OK ) {Content =  new StringContent( "Response content" ) };
+            responseMessage.Headers.Add(headerNameToAdd, HeaderValue);
+
+            // act
+            var header = responseMessage.Headers.GetValues(headerNameToGet);
+
+            // assert
+            Assert.NotNull(header);
+            Assert.AreEqual(1, header.Count());
+            Assert.AreEqual(HeaderValue, header.First());
+        }
+
+        [Test]
+        [TestCase("some-header-name", "SOME-HEADER-NAME")]
+        [TestCase("SOME-HEADER-NAME", "some-header-name")]
+        public void TestGcsHeadersAreCaseInsensitiveForWebHeaderCollection(string headerNameToAdd, string headerNameToGet)
+        {
+            // arrange
+            const string HeaderValue = "someValue";
+            var headers = new WebHeaderCollection();
+            headers.Add(headerNameToAdd, HeaderValue);
+
+            // act
+            var header = headers.GetValues(headerNameToGet);
+
+            // assert
+            Assert.NotNull(header);
+            Assert.AreEqual(1, header.Count());
+            Assert.AreEqual(HeaderValue, header.First());
+        }
+
         private void AssertForDownloadFileTests(ResultStatus expectedResultStatus)
         {
             if (expectedResultStatus == ResultStatus.DOWNLOADED)