From d6f89ea6f298e6b7459fa8246a22d8efe1aeb045 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 13:41:09 +0200 Subject: [PATCH 01/85] Add baseline benchmarks for `Microsoft.Azure.Cosmos.Encryption.Custom` --- .../src/AssemblyInfo.cs | 3 +- .../EncryptionBenchmark.cs | 102 ++++++++++++++++++ ...Encryption.Custom.Performance.Tests.csproj | 26 +++++ .../Program.cs | 21 ++++ .../Readme.md | 19 ++++ .../TestDoc.cs | 62 +++++++++++ Microsoft.Azure.Cosmos.sln | 31 +++++- 7 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs index c9e211085f..aca9899cc7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs @@ -5,4 +5,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.Tests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] -[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] \ No newline at end of file +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs new file mode 100644 index 0000000000..2e92274e6c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -0,0 +1,102 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using System.IO; + using BenchmarkDotNet.Attributes; + using Microsoft.Data.Encryption.Cryptography; + using Moq; + + [RPlotExporter] + public partial class EncryptionBenchmark + { + private static readonly byte[] DekData = Enumerable.Repeat((byte)0, 32).ToArray(); + private static readonly DataEncryptionKeyProperties DekProperties = new( + "id", + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + DekData, + new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); + private static readonly Mock StoreProvider = new(); + + private TestDoc? testDoc; + private CosmosEncryptor? encryptor; + private Custom.DataEncryptionKey? dek; + + private EncryptionOptions? encryptionOptions; + private byte[]? encryptedData; + private byte[]? plaintext; + + [Params(1, 10, 100)] + public int DocumentSizeInKb { get; set; } + + [GlobalSetup] + public async Task Setup() + { + StoreProvider + .Setup(x => x.UnwrapKey(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(DekData); + + this.dek = this.CreateMdeDek(); + + Mock keyProvider = new(); + keyProvider + .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(() => this.dek); + + this.encryptor = new(keyProvider.Object); + this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); + + this.encryptionOptions = CreateEncryptionOptions(); + + this.plaintext = EncryptionProcessor.BaseSerializer.ToStream(this.testDoc).ToArray(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext), + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + using MemoryStream memoryStream = new MemoryStream(); + + encryptedStream.CopyTo(memoryStream); + this.encryptedData = memoryStream.ToArray(); + } + + [Benchmark] + public async Task Encrypt() + { + await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext!), + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } + + [Benchmark] + public async Task Decrypt() + { + await EncryptionProcessor.DecryptAsync( + new MemoryStream(this.encryptedData!), + this.encryptor, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } + + private Custom.DataEncryptionKey CreateMdeDek() + { + return new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue); + } + + private static EncryptionOptions CreateEncryptionOptions() + { + EncryptionOptions options = new() + { + DataEncryptionKeyId = "dekId", + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt + }; + + return options; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj new file mode 100644 index 0000000000..3599027dc2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -0,0 +1,26 @@ + + + + Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests + Exe + net6 + enable + enable + + + + + + + + + + + + + true + true + ..\..\..\testkey.snk + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs new file mode 100644 index 0000000000..5304787e46 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs @@ -0,0 +1,21 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using BenchmarkDotNet.Configs; + using BenchmarkDotNet.Diagnosers; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Running; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + internal class Program + { + public static void Main(string[] args) + { + ManualConfig dontRequireSlnToRunBenchmarks = ManualConfig + .Create(DefaultConfig.Instance) + .AddJob(Job.MediumRun.WithToolchain(InProcessEmitToolchain.Instance)) + .AddDiagnoser(MemoryDiagnoser.Default); + + BenchmarkRunner.Run(dontRequireSlnToRunBenchmarks, args); + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md new file mode 100644 index 0000000000..967eed7430 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -0,0 +1,19 @@ +``` ini + +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores +.NET SDK=9.0.100-preview.7.24407.12 + [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 + +Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 +LaunchCount=2 WarmupCount=10 + +``` +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **47.90 μs** | **1.284 μs** | **1.842 μs** | **4.5776** | **1.1597** | **-** | **56.22 KB** | +| Decrypt | 1 | 59.67 μs | 1.041 μs | 1.558 μs | 5.2490 | 1.3428 | - | 64.79 KB | +| **Encrypt** | **10** | **154.57 μs** | **2.728 μs** | **3.998 μs** | **20.7520** | **4.1504** | **-** | **255.95 KB** | +| Decrypt | 10 | 220.03 μs | 6.124 μs | 8.585 μs | 29.0527 | 5.8594 | - | 357.41 KB | +| **Encrypt** | **100** | **2,761.51 μs** | **213.677 μs** | **319.822 μs** | **218.7500** | **173.8281** | **142.5781** | **2459.89 KB** | +| Decrypt | 100 | 2,445.99 μs | 136.839 μs | 200.577 μs | 347.6563 | 300.7813 | 253.9063 | 3406.33 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs new file mode 100644 index 0000000000..824045c489 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs @@ -0,0 +1,62 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using System.Text; + using Newtonsoft.Json; + + public partial class EncryptionBenchmark + { + internal class TestDoc + { + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; + + [JsonProperty("id")] + public string Id { get; set; } + + public string NonSensitive { get; set; } + + public string SensitiveStr { get; set; } + + public int SensitiveInt { get; set; } + + public Dictionary SensitiveDict { get; set; } + + public TestDoc() + { + } + + public static TestDoc Create(int approximateSize = -1) + { + return new TestDoc() + { + Id = Guid.NewGuid().ToString(), + NonSensitive = Guid.NewGuid().ToString(), + SensitiveStr = Guid.NewGuid().ToString(), + SensitiveInt = new Random().Next(), + SensitiveDict = GenerateBigDictionary(approximateSize), + }; + } + + private static Dictionary GenerateBigDictionary(int approximateSize) + { + const int stringSize = 100; + int items = Math.Max(1, approximateSize / stringSize); + + return Enumerable.Range(1, items).ToDictionary(x => x.ToString(), y => GenerateRandomString(stringSize)); + } + + private static string GenerateRandomString(int size) + { + Random rnd = new Random(); + const string characters = "abcdefghijklmnopqrstuvwxyz0123456789"; + + StringBuilder sb = new(); + for (int i = 0; i < size; i++) + { + sb.Append(characters[rnd.Next(0, characters.Length)]); + } + return sb.ToString(); + } + } + + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index d412905195..6fa5e4c3f2 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35209.166 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -34,6 +34,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Encr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FaultInjection", "Microsoft.Azure.Cosmos\FaultInjection\src\FaultInjection.csproj", "{021DDC27-02EF-42C4-9A9E-AA600833C2EE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests", "Microsoft.Azure.Cosmos.Encryption.Custom\tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj", "{CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Cover|Any CPU = Cover|Any CPU @@ -164,6 +166,30 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|x64.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|Any CPU.Build.0 = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|x64.ActiveCfg = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -175,6 +201,7 @@ Global {D7C78D76-A740-4129-BAAE-894640F95D74} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} {F87719DB-BB52-4B12-9D9C-F6AE30BAB3D7} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} {B5B3631D-AC2F-4257-855D-D6FE12F20B60} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C6A1D820-CB03-4DE6-87D1-46EF476F0040} From 8928b880d43b58aea3fe3bc78ff6ba0dc40198a2 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 14:05:57 +0200 Subject: [PATCH 02/85] Cleanup --- .../EncryptionBenchmark.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 2e92274e6c..7a1bb21549 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -18,7 +18,6 @@ public partial class EncryptionBenchmark private TestDoc? testDoc; private CosmosEncryptor? encryptor; - private Custom.DataEncryptionKey? dek; private EncryptionOptions? encryptionOptions; private byte[]? encryptedData; @@ -34,12 +33,10 @@ public async Task Setup() .Setup(x => x.UnwrapKey(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(DekData); - this.dek = this.CreateMdeDek(); - Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => this.dek); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); @@ -55,8 +52,7 @@ public async Task Setup() new CosmosDiagnosticsContext(), CancellationToken.None); - using MemoryStream memoryStream = new MemoryStream(); - + using MemoryStream memoryStream = new MemoryStream(); encryptedStream.CopyTo(memoryStream); this.encryptedData = memoryStream.ToArray(); } @@ -82,11 +78,6 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private Custom.DataEncryptionKey CreateMdeDek() - { - return new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue); - } - private static EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() From 9dc7d5416803b524a86374e21c14aa9bf75abe02 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 15:02:31 +0200 Subject: [PATCH 03/85] Use set of static test data for benchmarks --- .../EncryptionBenchmark.cs | 17 ++++++++++++----- ...s.Encryption.Custom.Performance.Tests.csproj | 12 ++++++++++++ .../Readme.md | 12 ++++++------ .../sampledata/testdoc-100kb.json | 1 + .../sampledata/testdoc-10kb.json | 1 + .../sampledata/testdoc-1kb.json | 1 + 6 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 7a1bb21549..d8e4c7d8a2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -16,7 +16,6 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); - private TestDoc? testDoc; private CosmosEncryptor? encryptor; private EncryptionOptions? encryptionOptions; @@ -39,11 +38,8 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); - this.encryptionOptions = CreateEncryptionOptions(); - - this.plaintext = EncryptionProcessor.BaseSerializer.ToStream(this.testDoc).ToArray(); + this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( new MemoryStream(this.plaintext), @@ -89,5 +85,16 @@ private static EncryptionOptions CreateEncryptionOptions() return options; } + + private byte[] LoadTestDoc() + { + string name = $"Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.sampledata.testdoc-{this.DocumentSizeInKb}kb.json"; + using Stream resourceStream = typeof(EncryptionBenchmark).Assembly.GetManifestResourceStream(name)!; + + byte[] buffer = new byte[resourceStream!.Length]; + resourceStream.Read(buffer, 0, buffer.Length); + + return buffer; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 3599027dc2..e32a42c36c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -8,6 +8,18 @@ enable + + + + + + + + + + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 967eed7430..bb4836273b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **47.90 μs** | **1.284 μs** | **1.842 μs** | **4.5776** | **1.1597** | **-** | **56.22 KB** | -| Decrypt | 1 | 59.67 μs | 1.041 μs | 1.558 μs | 5.2490 | 1.3428 | - | 64.79 KB | -| **Encrypt** | **10** | **154.57 μs** | **2.728 μs** | **3.998 μs** | **20.7520** | **4.1504** | **-** | **255.95 KB** | -| Decrypt | 10 | 220.03 μs | 6.124 μs | 8.585 μs | 29.0527 | 5.8594 | - | 357.41 KB | -| **Encrypt** | **100** | **2,761.51 μs** | **213.677 μs** | **319.822 μs** | **218.7500** | **173.8281** | **142.5781** | **2459.89 KB** | -| Decrypt | 100 | 2,445.99 μs | 136.839 μs | 200.577 μs | 347.6563 | 300.7813 | 253.9063 | 3406.33 KB | +| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | +| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | +| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | +| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | +| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | +| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json new file mode 100644 index 0000000000..21d40c5f05 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json @@ -0,0 +1 @@ +{"id":"e010f76d-17ce-4165-8225-35ce05333ee3","NonSensitive":"bbdb1cda-3444-4cf1-9a0f-063ce785476e","SensitiveStr":"b4f1b63c-96f3-4739-a5c1-ef1ca06f458e","SensitiveInt":770489940,"SensitiveDict":{"1":"qwt2ya60f3t7vst0j3cs9fqdiv658do98f5dfmh55yq5kjk3zu7p9ogbr8ph8c7h0iuttixja7b30vq52z9wqujjkf43utdsirsg","2":"vjvr3z8ns12xrhyxfnh5aptwqb3ejtkoql1od8jdxurgfxk2xwavu88de98b5he7v08d77guujejjrui08czkn4k7arrr1ijaqy6","3":"7kg7uasrf2i10b3itl13lxpzlt2tefzh7tk5r2eyxeu98b39l0zm7p2rk6ivvwyeti6q8wlkwebemw5i6nrnq1uog5j0kzlvnem9","4":"qasrvpil3s88t8f3fmr9i8jie3tqji5pxo0iaui21v97ay7o9myfs00w56euwd3pz3y5311rul9g42wuro9rb5eal42k6zo5m897","5":"sgu94rueqoqrpiljdekiiezjp4s1ds6xsd37yvcm7ulqoeq9h1xasaqgv4u067bbzi2uhd2ixbqh2ol1v3y6ndwhznxcs1l80ynz","6":"8z9uw1q9lobl3vqzj9n9lolcjtgdmyhg6riiabrrn1cj7fozwgmtetvv5xfnh8mg5avdib5csgl6hrrpvs8xgrqfv9atz1rqfrfn","7":"y7mo0hhc12sr7unppcwkfkfkavu47ym1c4ogwx0em6t3ept6ksfhxgn80645cy4q0njaisjc76o0l74ek1wyu83n55zkycuc909d","8":"ml3gxghqanig46gb0t5x0eowc8x6lwkosh054jcypjkj0sdykaync2o4853k1cm1ttod7a5htzyw9tnl3ch6iypopdh3271a5b7i","9":"jd3pcvu99al4qt476yjfauw31la95wjm07vxv4g5b9y32i3holfe38921chp54h3qx21bb7h42ho51caslx4lvsq4a4uv0r6chhi","10":"8mfl983nto693ihpwy9y3mo3c6lba8f4epr2anq7r26g5loauf5jt204wpmid3vcw7wq4vxfss5pgdco9ipgp5pcpivlrn0f0e22","11":"3zf5k550nhtqah3lmwj0qn6z940np7jpu3wmra6i22sf6wce1tftz7u9uur5vjts9qvixj41f19f0afi39inw2icke3pz1zj33lf","12":"xiog07lz2g0fvctld0l8ltfmamd1jgd7mmxh3z1xmshkrhkwhc0joa3g2zj7mwn2s1dbeo9tfpcwa32cqutxp34924i5pxy80tjv","13":"fq7tmr6pofoppj4xloubu493zlldoyqreciapdsohlyp49731bp6kw7nk3hvsd5m90nyqvddgmkt7ku61yulpdzy7xn1tqq3bnba","14":"5p03agq9dpmtmtrdeojlkq1llb9oxxrbg07yhc4dt34po7nf0hf89epk38pfxzgl6l8ru3g9tavh346dnfmlagfgumaqvevy526x","15":"edi9zs677mgmnhzjvjmc7ormtzl4oa1yt97ei7n441to010w3hbrk0uj4rzulb7f39w8a20cbqcxrhzrhsta4bz1bnnv8rgyo4x5","16":"1z986f9aidg55rii7volmthfr48beb29l5ttabi684pgwlxnveyhq82v0zuv7i2mcx6mo8r17qh6wuqqasw4cyu7kmgumly2ckzd","17":"rlz20xqzaqdec6ec4hqaswz55f1ckop30vpgpm6g18ub3ksakdjg8azt2rgpnwg6ruyp28x2sniqt3zctasrc69p7txmt97s340k","18":"v80p03aa9x9sqz5qcl1iljzy3um0kxxwa1z1nfna60f6ma48c023y7c0833th99bgta4qjrwir0d1c6pud8gspz63yrn8av8fdct","19":"ijrq4hmjsv4binbc03anhfbnl418rfvcfgzyce08g5hhudpwtj7ii05ri64kcife7uh9jdl1gz9bzyzjpr2st0qbnjomihd1yg7w","20":"j3ipcp1gyzqizwbtumcv430jek7siwi978lgxij5yiru5q98xo3q5oiqx87v7d05bctwchk7oj8xfyj7194jsbz5z6kr6bvpi4hk","21":"5lwdli6md1iacxxbqp9zc12tp26w0qw4b5dg87aif54mb66111nip0jyherf5obz91n6v8bdnchiqfulwsdsu4kwfeud4z0uvaog","22":"zl6ztc3ncnc61kwd0crhk7fpwignjggdivhje1yrvgtv31ksk63ttxe9001fa03jxzmwogh2ombrgkvdokihndkcjhhxhrn2ruon","23":"p4uemv64y8497cnexs05ih3i3of1k9112cedrrvz1j7zzv7tg3i4a5avbcdvw23vjs8ivnec7zdjxzihnpm7d6k6m5fqt09fbnrx","24":"gtgky4vi7x6oskpnqlv8eymkru8j19fy9rubxspr6zkjpm1zin34c1hacu21cp4xfq5hsclhmf443avr8g9hn51cr3r9h55et3se","25":"iqp21mfuklwrix35ixwuwxqxwmt4jhywfb5h3t11q4e5p1p1zqptjbipvrbhilc5icbvltfcv7vfcnsncjqfz8yut4h9gaymj06e","26":"4x7gkhll44182spczbl4bmtyhzs9mmefwxrfx6v72owz16ug59nrgch8o8wle0x8fm0090uyck8bkgxxl8fssnrftaypf5uwx84f","27":"cdwh6okkrd4f3pgyvessf2i9hl6bq4vhzvwucr2eyu0l53vq86pdwr6qxoe1vq8iwtzbfu2nwe5eg23zeyas10c2p9z5o9pjpl4p","28":"k20tzkucfijvtux1087365s69lrsg6axm2f7v03g4riygpzxj4e54rxfvvcw49xbyrq9oaofrroxw5owpbnihv3d35b72xolycpd","29":"fjy7dhkw5w06fm7vzlpjg29vg03605p2hivrrndrs4su432heklth18ns080z9ek8ihr6bssxc4mn5lzrmim4077qo7i1njqxyjx","30":"ld3s0rpja4naxd72v9u7op8rfmojlrv13z15tg1vmquwo4dvm2z1qpr6qn0hm4npva740kj582ltfzl66qnx35oec4fkul7kdftw","31":"fxro95cche8o7lprcf1sbl9u0biasa2nj9xqgvzaogep46rx6mbxtjs99lnrzgqson4nxz8x2rpegb2l9g0k0kepz4s9lbkr2sn3","32":"8cqqhyjskso2lh5xnjzx5qpei5149hfb9951wqwojdnbn86gaq1i2rpof4k2rzqfib5qqisfahg3awi1ug6owmezzwh3jsb9k7di","33":"mt8alt08khh3yitrp47npz6bcnbu6oqr9aiwyocww87k0wwtyqtwz2hyan7vkmnyz8ldnbsxinxbx9kep8ajofladz5ni84l5xrw","34":"zvm7or3haiy5bv03njeo03vs0f5gk6c4nnloq5vmoaq3v89zgm94350n7o3t6p7cv64i3igklsylnd9beiqp9dfwzgphffsi6pnm","35":"f0wzkqyacsnlft7g8ytcz3v6itbvrssn08pq0jkiufoyarteuemej4qgfmad88h1pq1daph09ovmm5dgzotrti0lcj8zf64sgb3p","36":"cubf6nafgxt71f6lr6d1b887i4r6pwscllaslo7pfznjtz1ijwf1e3hl4idg1d27k1s0k4hy5k1snztzkqn535735e66ecbhxq8l","37":"exyo0bmxxrd7rbiq7wd4wrta6rlc9x4oyxa09ctq259acpauqa413cpmz28usjg2hijcvarug64vccyvg8h6o6fznrcex4mmu95z","38":"3xlf2u73l8yt2r2v1qpavh2e81yyejgahelmeje1ckowmtlf1jqxiddkpj120cw54yax77jdk724qyfluvjqdkbinjd5zn019x53","39":"nqle32172bknqrze6zwy4t4jaft6z14kmnufopk1dxygrrw248joqpd2mzcgb3wt82g7mn8kmfvy8qzdfz0pjf4eych9jyjeiau4","40":"aq32vdqky0tkvfii1vyxsh862czpdwi8g12j0vs8ssdqrqbyc535h356uuipp0sljwkn92qjqcj3g1miz6tnymm952uiy3yz01wa","41":"dmwfuq186j6dnedbp6rsj0ge3g07xg66g12udj3unku6va3h40c6bpmueq5cx00q0ysal22q0pjzia5pujokm83xs209eorz7tk4","42":"s4rv0mwelpvtg1o53vr9jlwkqgnb9cb0omfor8qxc77vgjlfluedwzbnzd3cgbgbbkvtyjfs1y2gkjafwq18mejsfvhu3kxt9kj9","43":"khxgqvw2q6wxpcokidcg88gpm610ig4cs6dephbeutyzg020jzabmnuitizz7unv3c8d0hbwe535y8flmargdvfs3r9az2r5myg9","44":"roc8qqxdmavb744qd6pe3d2cv2dti8og2wcr1yhxkyynsbld0kscrcc82cgsuay7x5xnu4qwni1774dymzxjvxwhtw99d61bngdo","45":"58l4eccws8nldagemuu0hcp4b1tftohobh07agj4yxk7a2czjycnu41ohxtn1mjl2aeht7u99cwo4qqp7wvxzlcmhrj84amu1l1u","46":"nash8eow6tb49k1m1nhzodmnvhmlz5af70n77yefa0g9xmhlh840g877ts6kyahedhjvugf3joui44ntpia9gzxt00z6jipfantt","47":"d7nhmhwgj7vghf6qggkxw2zev1xs2f95l7h990lxoyc6ktdvfe4gsqx0iscqa0t2mtclhjps7uhz1an4gnpe6a0ba0u527dgfawi","48":"vb6tbv7cazdydqvi8qcnj1fadfbqcic9s8a8p674oo5jjbe3wc05rlovz6covhac3qm3jxonbycv31wlktx9cspetuqoe93k6hcn","49":"vnrsuz93okofehv60xlv0f4wht3whoz4asenw87yakja2luioht825b9rg2ry64vvsyuzyt2bhoeqnz1wabyhytaqujbgjhr3sog","50":"fmckh5p1ku854hz1rah2poma0rw6i7y2ylyboskao82pd19h3vvrsd19xlw4rsgiubzfa8q6j6nskiznxomilsn82lsl394prxrx","51":"6lxfcvmismjli2x3tbte38gq0ee81zpen4lthbmgh5htvccyz8x82tjogmb1xcivmon4udo4a9uwr535ltxhobojjtej9yjv543a","52":"1s21v23ajpgy5ff51tie14oz0zvhrrvu19dmpfo6ycb6m4p478ihs1onut69ab6959bn1mmonmb5vqw71stp70vcsc5fo037xgbp","53":"wfwla1k144s3s2xxecdb7cs2ndm3t5w6y4zhem3mwv89qypwzi7z0tkcacx6jj73721jr8sqx6uplhwojooqv20162pw39k8y6i2","54":"cek6rjkudrcdbwkti0dd37a40d4t44gxz2kf096vr6ff3nyyh8h6mrh1fcfplkrck7u7c0pzasr4f17tx0zwrs4w9ca0hizxfumr","55":"oqt2q6qp31qrcl21qqfxq0j9v05885c3f7u0y2voyg2kotoyuhjzaj6xlj5kuyj7aclvyne8jirgwlghplpkb38tacyv1yqxhyda","56":"erc8ma2ql6nc8lxs9cezywd515dayhxujopnt8oj385qp9shxwca8zqqin88jabsiql1ceejufknu1gmzgp6h1cv4r0ukj7lw1m5","57":"gwwulnmocf1kcz35xho022tsy9ec1j3wa1sa1m9rlnwt6z7rq79pe1zehwk4v2f6p6gizlbyeaskbgufqps6lelubczgbv7haczy","58":"nawt8ac0cli65jfc44itfm6xw8rl9y3tvc4dg7rir67imzn0reli2lzl7rt0wqb2hx6bsj0fi6fdmisiume4kp66e3tymbkv5qvb","59":"dy80gxpm9sooohykpx8cejnvrn1bz5vqrpzwp3zqmuvj5lepavnx68j12v8ly4hygnoe31aqp16mu5r5tmvksum2cpl0afc0wyw2","60":"h1s7tg8733ikglzojmq4lla16g5mfixje4vxq0ajp7sik6wuw1ctpzm01xwa584is8bsrrmv5s2pvy3xnyrslnh2jtd09uqsp2xs","61":"qwn8ksa9vwucdfgl71dicvrsv4sgh4olbfvnigd62jolal1lu22ttjebweg3nabs0u3uzhbv5bdwmxmtmu5dcjmiymb197n5wurq","62":"xbg0byt7xmtw6gjcbsyo9lvzg7amfl46reuzpzpokrt5vo1gfle9q42m6vw4vaoqu1b0h9fuupwnebkl6ish7a901nnwh0b890ly","63":"lj0xzdhvtnii10ikqxwd7vtd1cz41z1cx9fcpbv494zltl88dj0g7mzhlbygt44bwqj4xoxt22e8txvrh3ocp18rodwznbgyt63g","64":"lvvnygyywhqqcspmbrn1eas0uqfta08pbjhqhmnxywwr10kyqb5tv654k9h6jdhvwehbdimqndb7e65roppp88vah2u9zqf1eqwe","65":"iwzxqnfa0we8s333i0qo1csyptswuzx8zstm93bi8u4trl2mme5rsxdogfsmoez7fpqr8zbka3pnet1a0z8eqiltfb55ag2wrdyc","66":"2bfivp8qdt3s58wtykddfys4j3ypossy3arlvjwbp2sj9gd4wctbjd8xmr8um3z2xihw9l5c6w0jlj373ks9wdjtb1wh8a7yhjb6","67":"nufqjuqrnzeyx4s16jv4kwavw4micmfevqvmi9uffqt4ft960pgibsscpmg5k410jmnv6eehflc4jqow87asxll4f4wi8h6x9lar","68":"kfb02ufcimu9k7x9h5u7t2t08kn65l7huhe4n25ukuzz50ik11xn3qk159hzfccfde86cc0j1gnv7w4mxokynv5o1augohcykogp","69":"2pq2flugd0gfs7mlwwkeikvwoegy3a1i2rqqmhae3z5g8plwonu25q9xm0bgnumfq8o2w1hkz39nd9zi0wqlx72fj8i8ykr3wfa0","70":"khgghtv7w7uw00l73xes0hf0auabq192tjpr0vjhkwrthexbgnoektv4yklvitb306wk0rs6hqr56sy6rc8c5wonfa7x5e0dksle","71":"67w6l85ln872nsleeet2vht06cq9b94zfoq44ilr48nqzef1c8j54xwtygwebwmwcn2yi4cxpazm7v0lyjs7ldvuc6kuvl7kb2ks","72":"we7ylh7k9hy1vx0v72ljolms34vetgzqotsuwv163viy2k5jf12tx1jb9lt5qhzyp48pa2gtolyck4b5k4v7vb5xf0ohbtk9a5r4","73":"ndg6ufbv635xi7ow63jqlgr4rqqdvh6h082rkz8ui911q8nlfxh1vwc1zat3hy642hepuqdfxzfe96bcvd2aano4c7izytayd17f","74":"71ploor208vumkdgt2xshtljz961tjep6248xrc5efpm5krijwqtisrxj9ms50n8uemiau26fhp3rdyopgzjneat11y02cnobrzh","75":"z9ebfmy95tikhengt79kztvht0n156dev010qzvw201vs9lg6f5vqrn3sbv7upl6l750a8co385xpfblwdlzpndigh0tl2005tvv","76":"t6rozbasnvi30kk9za0jieyy5ff60jdv80w7a3nn8m0mr3y8vc1hqu8rs1s302iwq5qn3am5sdmny0tg4uhrsc47chhfbk4s2xft","77":"kqpowkkv2pdq4c7y4kpqz3parbgv9v4pk2wesfzxr6ssnlhpdpdrh1zayke7gswtu7lgmd17tum34vazwgy6p8c2evwhzwtxdkix","78":"hw570z97ayivd746etnt9aatnne9g9cnfbtrn7qlfvwns6pfx5lhx811aslmjxy3z7ht3rivut96qb1rx0n0qvminwrbj8wgc4wt","79":"b33snb68uejym9bvckbto56vnyqvgss6axbfi619x948hn7nxvurltj4zf7iqub59585p5sasw3kno5t58ot2rm3qewgpfivr37g","80":"h20v5q35etwdsw4z05j8abjx5qny6uk79h241u0mmg2s1xdtyvtbpcd6i9w3b4napggmwhzla921i7m5ki05s7zftaeoo2vqba4g","81":"d9qiueg2lv78brwmnci1lloon61j307fhcsrnh4s1artjok4q8iat05foh0pczziuabx1c0g7sjzmh152n5ojbwgh1iz1z1xbf60","82":"ie2y7y8i2xshl1u72l9jex8ql2n56i5i27cogue4lcmlgjo06roxu7dbg8b2zw07bpptms7pq0ktkz09vyxdru6gdluk4225luga","83":"2mx1x5jc2eyvx9o9de4izikzz5cliihjpntdd1wmemzkhkufzieri9fvgjmuur9ky6vszw8j5z1pvq5vuqi2yjm1a4u7tb1l8efp","84":"5w2ufrqbi9pahyby6w1xqqsgtea53yyg4tdrnmo56fcb275gzrt3s8f9sctfrovjcg4exdodf29ksvyqnoh5je289jl92vcuc2qv","85":"7jd2nqzrong3f1xei3g0ytl3qap9j53ov3nn7hkvmyvrb2wnbrifmqrw3him28l4rfs2kr43r7kf9cwg7xacecgdhx5aw5to6c1w","86":"91y3qy89mitteonfotvif1vsa9lul5d0l7n4h5bm2rcu4lf7mnnt3tctt08y50412g88scm2n5rch377qg1rksjjus0atbbxefph","87":"lywssvp6ojt7pdrrqp0cqpph6d1qc0l4oopstqvgq3n520b9gq1ovlk15m298lizfont3m7g08o7w1zxq17uu1q97i9fo1bfhcfg","88":"w8ritbk5qlnv93f5kjaoojrlgyo0vkfluomx4qgqipznczn5lfy8bjqavclww6861d1on3ymn23x7j03kp0pgjrtxk5mii5iifft","89":"9q1rbjidwgzwtm46wd1ngnxjaq0jeolqnmb68hmotj3u3oui4fzmfmsj89c31tgdfsys9uf18p4nq9yvpyaynlzurn5zsbakua5m","90":"5aa97rpo3qs2nwsdzieiqzwtz0cheiqnwapivhq6t6slqwaki5e8utqc2lp91a0l3og7rgq48xvbrrynj1von1aa7l6cmfxu9lju","91":"if2mnsvvb5h2w0t1r4zmgx1ngph1n72drqp8evn7g40lo45zxsgszdvf50t8kk6rzd3ayge3i143rcmx43v7etbrx7lhsr5xvbug","92":"ev81wrtb95922kbs1ffgolvkmklz9na3ka9nz91chasajeuj38c23l02kpelh3bsuk9b53siffw2hw5d83n5cuoeugyrr6a6pu9a","93":"t5m0u61zhkmqp26ongg41g85wpir0o3868lk5gmh1uk9ijp6lfjf7v1k747ivtpelgayprg2ah6ud0zxq7421728apgezzbf7war","94":"bsxlwh2eikokss3yc8t9rpmyf37bt1ojvbsj0d1op3qavu73pekx3k3k0x3xl2prh35f25tk7j5vxoo66f1oq7cyulwp640xxvd0","95":"ur09m5a9sliiz6f8zjhbzl4r97xma384mzd0x2n54pn90s5er7higdxgohqkmi2964bw95gl3n7xijli27j8bn4rntuz9xufjs9g","96":"qdn4wd4qyh6s61gkhk3kas52dynw0hvggo202hsfw7f29k3t0xxse1f52cs8drd7dhxa28szrw42v5eg6f6sw9eiinu6wu9n0rau","97":"g7jnpbm5k3pfbxgft6wzh9onf7temjqamstaf8nuwmmk4zm5os5yru1511oavg2grch5n8p4mzdmxs62qmbeijabazn3ud8tsgfw","98":"yfmvs91kw3brgcm2tido1kmzvh8jx1s28yggdb0k8pv7ehxjsf2ymfphtlzvnknd8b00ano24n94luqd6tkuzlhbseld0wfwdg6z","99":"50bvqawehrwh7tp0brwrkb7bjql7rx1y53r96pv4dg23k2dfcaxytx1uhhcl7cfhucys7epmshumhtr3f5rx8twuf8blgqnsavzv","100":"n4vgzqcbik4a57o99qeydl2i8l47h1fleqalqzbvz1atnzs1kr4taud7uim88fjwnyljz9vajufdomjn83o3xrk7t697ys8rz64l","101":"rshrxc3kemqvxosz9pxi0finflmlubuyfovl02bdj04a4ur960542wovv86ay39y6g9i22yem77uefnl4le3jpsyo0yicpony6ga","102":"y1cdt4miizwzyztn5hrzdkc3qyqe955wbe87l2modj7955vdir34ikk12euhyf5iwkalojo5kq1l0wz2kvpd6vnibsr0fpsym00x","103":"p1a143tn8st9brom7f0o5iuvzn85mn57fgk4f7r7f94mdsxia232zk69cxprm874cqyijqn6vtwbuhtk17zs1ajy2pb8c6pgfl0v","104":"mtksiir2wnkvu0sytgkxn8fk5maso8z44g1s1qwsns7v9s2y5hnaq7k5uen9tvf24ear0j9hg3lnv9xirbn5s9d4cgsy5b0fgsrk","105":"26pk3x1pgyrh876mrm9ur68mwm2g8pupacp75bmzw6s4td6tkkbxp18q74hp5385mb2tls59yjb1cea5xeqpvoyful54peaffn9k","106":"wdc33hhknm0s6xa2ms3c933315fu9tlnbftvqsmlnb7cgj5sozj88cafkggxzcv4a68uw1c2y3h8ju4j2g1tl5y96b78nl5n2dzp","107":"dq76qb4ddn89kbqobm3jdnze3u55yyv3b3ob08hvl9wzq0fter0szmsbi35mhc9b7uy8qqikrn75vmt1m7s4hmfsz7q81s8s1j2f","108":"ku66tgz9onoezgl93yzlnvc2ogsncj6mf5ppm2lpsft8qnnp3hdelpqdbm780mnoyhvgd3a37i0pkfspcqmgr6a4zdqqhc0kn5tz","109":"981llvf8f6sy848kxqvj93ynyewu0t5yqsfzj34avl401dp1cb8ynld2s7l2twa7v811aiwfmjwj1l7xs6bhv3posj03ygk34m7c","110":"pkiaie7k5fj1qbt6p8cxdfloi5n6l8z1x2i9zr1jrz4smorokglakqggt2zbc0wtig1e76vuq3l09rt3g93nobfes5odek9nzv0n","111":"lwdw7f1shu67x92hqo54tmgt7m5jvni7nopejdajnznqdmx2ano6rsiq9q1ojf6z2xkg167x68tqjh15xx0hnb26ai87vri90oap","112":"fuclp0s7g0jlsxligsjfqmvnik7f3dx7r2g1mue6d9vz82djb1520a2smurfziw1xxgeg6pfp3rm9gt3x98kmfm9jfu6k0ep1mot","113":"5oa4993yc7wrvk4bqxs0evalp7iviqtjx8lpdy5wksqi4iw4elytm3dglazy5rz57mykho6n11jovhhsus2sjskaa4y427emej2n","114":"95iupp7zqdyyj1d961pf4kyks478456ubu2elf6wei7ttky3ll4aokxwhn2xq6dusu4kntesdvdth73w5fvf6urvvra6wtc9wxzy","115":"aefa23tkcavwvcrjyir21kcpzg00fvs2mn5nin95gf3zp71k2zlbzp0y11bqfsm6bgc8dya160dg48oiccm2goi1ieb2tduwn0kx","116":"vqpqd9adk380670hj75oga83mz2dcd8682gbemlk4ds1gxwfmeym3ioz7qnyrmp1spf3w6ieshl6vy90aehuw5t1qu2iz2chz6md","117":"yu6we6kux1bh6rgl025lg307uewyhxcl62hvf2di7dgofvgm2h36iloz29tk5vgalpxgivgutqcfpvi868hl8o7azwydx1n8s1ui","118":"gg39a0q2apf08ay0kc7ol72r65d6d0fz8yd4xojqvybdm5jn4cfklwgpq0mzf8yve7kupwa7ibfvyx2u29gaf7u30345mwgx9t4n","119":"vc1wo5p00i0yonwqyy22o025zuwa20xa0z4sdnf7rw3ados9agi28a4nwanyymrofdq150hdxdqvih1ejdyp3hnu87nbp7rp4vs3","120":"wfc40963gjei71n0out369i47nb8txlde4ltkgpp80dj1rdczyjqbxzylg3up27wukyu5kz75hmd9vp77l9wgb7tu9qac9otfr9y","121":"vvthm368h5pj1lry2g6fplllw138p4ttfkhw6jspwvganr29yya2cunhzdgcm5y3w5br78nji97flhdjze17rrcdpi4srkbrewve","122":"6t066ihtfxpby0myjh5lydg29xwei2flyu104frvdng2h0bj7dtztwjuy9iefnmksd2ah1hqh7w9fnohibrtytluw4qgbuhoy737","123":"20vxvfteyc9sj621383iu0mdnxk6wuztxacy9fpwu3fxs8h6p279zoerz5avg4mw00znjpb8zopg242yotiidkbf68ugonqn8v1v","124":"vypofg2ckee2wg1781esdljnby494qd3ppwm31g8vewl2nivswnjdo2p06ztwefbii0cn3avrzfkmlyj2h5o35jzl5hfvl81crnl","125":"254ljzbhdykiqi9lpk75q33my0mbbkfd1l3ampk9ftdmhlj9tbgdn89sohxwavk6ivl5wq2oxevgxq141854zihvqxn5vrxs6lus","126":"zbvlq8z3t55glru9z9msmqo2r3velahmzvkuwlry08js98xuf4ed1bbbt18l1s10mugbxhzyxxcv1qdrk7n52witpco392skvmnj","127":"ira6h7258jnob5mpulirv9tc4qxetqlspx86uq3k2f77g6zid0e1wnmzjn1q5ddavil1hgcloqvmuk71eudlb4e7syd939uxkt77","128":"lyvzv39pxfadodp6jqohpt43oppxrjxiusiwkms4lsbc03y5p1jkg9qk4em9j1g1vp17w4gwaqt13xfqlsbstjl1zdtfi7h92o1b","129":"ob6lxyotqm5l592nu6syzsdbyi0ulcxaxpa7mf4p1fwxoc9oo4duur26xcfvckljj3jjwfzmrn7z9ts2r09o7x7av5ty1adovayk","130":"7qqf40tfz035w3rne9yh71pxais7vxq3k30j75v67ibcxdy4eri3y8ui63knejx5jh7631s9y79yqvp6atmmchayolm5zxaoxl8t","131":"qdwx94xdcs8tjae3gl2ckknwgb6fcquj353tf37zxeosjw2wzhzlllgrbfso7udfvxw409a78f1l9phu3segx6kuyybzuj5lmwzh","132":"eipoy5qe89eumkzj2us7f91it29rpbdzbog676cgk6681wxy64qnzkwl7abwh9flom04ocrqvmte7il6sdxz9t46lanffbp0wz98","133":"auh4qxynzq00kvt82hfyrdf2c3i9tqu1r0e6wqob17039zah0hc8q21u9g9klo4wwi2vxeen4jx1m50vo08j56pndmuphc49mhcr","134":"p2ioa4o4m7y0dv27q87co1a8ziknj8zwta7u7r3kztb3r6ajtcu79l8f81fg43orh4q5ygm4jee93h9bdmjg03b7686ghov87rj3","135":"ryunmeyyfsnczf5lu5hs5hvle33o59oknko7yunfcyif55nsp6mn591ilgro2ka27hmgu33n6fnextreyomlz3cwz3egl30gbhg2","136":"gnlj5uhkg8ihh4e8n1y2xj5ia00w4cyvtcsimc9c7j57it0pql5o2f10yiqxtbt8uvb32r3a22n9uib4m79tczfmlgvlzj1fvorz","137":"i0jgb8b4e183if1el77bt6830qr1nu6hgpfis2c0oc55ktn1i76gsf8bu9ef5n7xvrqwgbr0klp4tq2x8v493f6rddj2mpl99ffg","138":"llaop10i5w2l32rb6x1bg5s6nyk0604idvhbvd9kxpyoubwvx897vsw127wdqnbesnrz96xx2q9xo8zips73om4lg9t4d2o8r4nj","139":"ipsivxvo4za2bdlamkoi1wqyksc9xfhicm4xkfyqv5b0ts1lrr678pbof1sfgodneuno4psl2rllxucoi5def8cgpvoo29hvrpfe","140":"bzqlwpqexie0lvmnpurps5byzt8nl7t1ktvx9piivnfsb4pggaxtzz6qiu4znbwm8d2ruw26b2xvuswjvveo8gg2y7ca29xlhnd5","141":"dysj4859se2zciqa2tru6coy3shpadzhlzdwzdmb9neuflj606z47n0vg6fsie0iwy6ecdz2kon1p05pg5ymf1roc9wrvktyxyhp","142":"14dgb7adghlrfh8j5mqe65d61ezpdnnah1in87eq1itqk9uunbw64xjvtnmcf5k6b7jaewkxl2u40pnjdstcouf2sjvjfrb9n5x8","143":"uyeyp1wrv550kwhzcjgjvse9b10hprmxepe41qdvnd1ijyqz1kz69bbrin84ppwk77r8qgbkwte51bbjj8ohvrgqksm0ziceah17","144":"rxp6nwflgt46peyn17ly1n6istpdi5l8gmxnfmet408ssyumkp7vfmgeymni6560qtq7c5qdtdvsems2llhx5uaijjhp59ip73s8","145":"lscf1gesmh0n7hu33e0qqjwse50ff4t305ia9n74yjjrexmcyndhd61a6acv8xzq2ula47bnjns81gn8uvf2p1upqkg0z03n8qj5","146":"tq3s0jkfqz3102szln1k53oknok6s1o8w4mv3t8lify1kjmiviu4wl7szk1i0t92a07fz4zix8x7am0jiahzcjndiuton0mgk6jr","147":"zdf00kis6um4ul985uf7uiw1qd0whtztt0wdzzwxla6fv0vhej4xg5i9c6sdva99tag40syufjkjbnk6cd9i643iko0q8m98pb3j","148":"3eqw8mdsc57tf0r9jjhb9oec91ek2wc8shqe37650wksrtpcf0zpzpvsjrrus7zknnv1jdo1poj7vwuii6zv0t28qug693ls16ud","149":"xxgn57ac4bbpfzjzhie9ai5d5sxvel3f1k2b4b9p58h2xlxtxm7nuduutlhq2bjchb86wdjwjlkkrmwufi3spmiem6u8duhk06nj","150":"5s05qp7b8g3n32ngw4qje26zn5j0g46fgkthzhv6wbgv98lc79y072emympkrnr8960swcjmdo9htq7ocqq9gmbdci0ss7p0bfk2","151":"hh0stgfz8j7mhefdlksgcbxboy64tbhfn3aa3a9f42h93gynpaobe4zwsfzj77kkg7ipon7t41qrq65rg94mv6j2wtvywi37ts2a","152":"0o6q83bcvj96kzb9g6p52w14ecp36wv99slpwbvw9s0k5x1gi4o4ws17n7gjhcz1p5m1juxcrkvthmj2tnj240xsrxiwk86ir4m2","153":"wx8y710o74ei8q05i2xfx6sv6x1jq6svtife2hlu8atv6g039xv2szmlog6sew9o75pzh6mnxlibhkzwxbqlnvb3vzkikna3gtra","154":"rsu4o5fg5aqb4so3lxhqkz7g2z9jelcxa1v0wixg9afzczfdlsdr4cgeyws4n9jgcb63sw7ki4y0jmb1r199pbmyke0ie7f1mvg0","155":"jk1d0uvbwrxnq233omqsj3yegrpn5slrj0ayecjew23xl615yrghpnw8p1z19rh1cg4jskiohpt85civid9qwsofn1dvpt2z6yfw","156":"mmo7wp1sqxp2vg6qjqk8sump5ho1v8yi484r9jvhdp5qrhviiga99cedt6ghmoc43uot3f9lcudiou1nbk3p54rp9hv4693v64x0","157":"11p5n80zir9tveeqqoxpcs6lk97lwq8pu07tsuk6ztxh6dzbhbidwddlvm6ohik96m3hulmeny1oaz2xcoltysw4ruw7evc3jz0u","158":"norsygqksro5t9soo7o4r6z9aa5q473ohxx6t2twiok01s0ujigk6m0mmctuctp2kzm1il235nmuzyts39j8whqo1ejnknr8nklw","159":"ei4a1t02cxtiwp47372eukapcuaey8axg3fio41rd8dpv0xlpqlnk30ettrpnkbvwsx7q8oixp02lwuiv6kiqk0k31r8ztof7ql3","160":"x5um8vgsffdpv4sk7vinwiqrd8dau2wmo5ln3dfwioet3seq6bmkshwql44x7lz83ov6cjh9bp5e05utglxtg4ww7att12x4ipd0","161":"8iin5zqyxi8njhudrfo5np24jhuywla2ahozjtp4dfesbxjqlsm7689kicibhtq0re0m6w6u1ouetyiaylstnycw6ps1u04omg98","162":"mujm15w253de7yzbh6f1eejyz4w41vjqcu49zum5e9e0y772itvygc0ht3pgll75a0w5qqd2vicvl3uziouy3uh55h2jf6nlv0hh","163":"qq5l9zlz95jzghjp7a9qjf0yvj4e7z0xefl49bgqnlpol1i868t2s0zo6ww2ztptrn9g00puhc06n8pwk8e6k0s4m30bychnmaph","164":"s6xzfglc56drmw1yhcswffcy9ed3k8c20seuvhonjteha2uuw0x21fzxvnn6c2wbwp8hljle1exenj1f0x2h3xodpq90w0kuhsno","165":"c28rfhxgc5jogi72abnfe68jksoj3cfyrubx8y1geldeakv5vs4sgr1ajk1hprwqwxovkdfstxwymntg63uwmo6bbqfuxzwosr7u","166":"bbvr6ogo1n4qbov9c258giqkykki1cbvbgcadq4jbneqlklft39zi9pxo8pbtwpxpgso5vctmj7s5boh2gy5jh276fyi4cme4pyb","167":"tewrnprftjspuwiv0f04u48ltnvrhuwxd282lz4257dojeu9htgnbjq1v5yap9lvo5sno4gxxuehexvukpmks0rtt4hiu2vbbi0h","168":"xg3gdx1tyww9iki7l3i2pd83zk1y4kxxx4qpxr1yaw5c2uen3nep1fzjbgvp42n4d5gepvtdhtkasf2qu92lyidq759pjawzblpk","169":"yxi6s7ho8coddoh31dp364nnmw0oe0kvd4ggvi1jiseptpw0vyt27jrai4z0a26gb61qj69psy3g3290678uou8e6kdzdxcwh713","170":"u5qy4hia9ww2e02m6tpviw0y5ax31u1to42g5qeti5t7jaaopo9x3izvloqjyo2d87im7ios6p098uab6ok3j3eikmfs6lggfhp8","171":"n4dpqer9o7uoqggp2lf9w2ghgcs53s1p83glyn3pd1sc0lj02aj8ljisq6ax5es0pd61uasj0dltf6qgygf9338gp59ipi4i2kby","172":"l37u7vgl3z0im64afsyimnzxojir31wrxp8nxqwfywiw7m3ncikolekmvmly5h718ik89m4oon2b5h1xva8rl5j8hcff8lslvqs6","173":"7hyqq9kopaiy2dg0aozztfx8v0a93w66e95cnzo77tsb4acl8yz8p7dyxgwoy3ijdqp4xtjs34v50amid2tw7cfxbsog7z2r96w0","174":"9mbjoifdaws7z5le50b4j45ompjal8nb631pgykp6gyp1ilu6hk456laa1pe38x1f31hlpmc9nw21s65u9q00yqee74oi30l3qvr","175":"3tw7r5pblv4uzw8wlcxqn4jydyhyw7r5wiurtyjyx9qu7ssf8dozj20mntyhplak35fuzi5nn00le1fzi4mgjg797at2fzwsa7e7","176":"1bknz0fu1um6ef033sc8tvy11f6w7vj1915rk114uhrdslvkrohnxs0cpjzmc7s7cxmxplx9rjnnxkquv7l1p51jayoofn8p8qo5","177":"wblmdn0ifyz2ys7fc3k2cqmsk2exowqb0mthd815fniiztx3gt346hopvoxp3g6x32chv09z1mo9ltdim6v8zwh25fdwy1c2fom3","178":"p0t3b5tzyjz3cdcpv3z1n2r5g3isr3qhmqa6hvj0xc4z4r9aag1kp8908vn5cvmig8ywc3qrxxs69pmkym8xqbm48p3yu7dwzcz7","179":"a5qqa34wuhxc3ruu7n8b4l5qm2wngk3anwucs5gg4bqkv0p8d49up9f6tn0aesao8sbk8l3ga6na4b3vqgrgi1gtqiwz3es6mu69","180":"ogqlfc9tfmilnxlmw393ql8xosgogk8x1w21oooc8s2whfot0phrjcfjn144xflkxmg1rfu01ynbzfbqk3urmeqrlca3swvw2y8n","181":"p6ftcmd57t2oi7gtbibqz3n1kbjvgmu5ns0s4yjcye2kqvvzrwdrrpl7vruvz3rnjeelg3adqg6ug03ak63pq5akspkdo35yeo0t","182":"9y1hz17xd0keubs2k0f0wfoz4cp8kl41obfw3mpg9qq9lqg7dpps3lj9gntwxpf3m0nldfnpoiacsi2y6uptdjw5hxd7mpe5tp4d","183":"6hkatfwu1iwxp8uj1udxaf1nxq9jsqsa0ruoyepd2nvaty4ktu8f0di54idlbnc0zmt33c2gdcukli99ses1a4e8ulj260toyazk","184":"12ey5y7e0xa1sv6mi2oug64xnf6ahhh2y9vs0svb6wr0pzls6gf9z4i0c8lz1ba6lif895zqzt89tugrzxnibxtzvdqq30ny0evm","185":"oozfionyl4jwntkwvvcqqvhe0e8b0alk8c9dnzgemzyj306yjgrh35r0jp15su3lw08s49i2adr0qr1ci7kpokxn7t2xit23y9bj","186":"6b40y9eygp22qjxtpr42tqmdt0576jvuys72yndp34praz4avb9ahhkodgce6mffls9ixsawdx4u1375yorbtlgieia71ffggjeq","187":"73y08uczc9n1thgd7trfo322tvcbb6e4o6kqwib1d1d50xkcegxedsgtrzp22pjulb8144rrreie521fw7ngcn677lb10z4o90mp","188":"gzh0von2y1ktdw6g51rsky9k4rls6wckmnj1152bljzrxmqeqvxxj2gm1dep1zhmd2ca0vea12wyyv1p7005px5bvtwn5nx5thf4","189":"bm0ctfspl5yqm2g92iuwzdwpj0i49r8nlbjyzly59ni7ifg1xwzuwzbgadvhx1p4sue09hikdaggw0b2984kqely37q3tmgurv7m","190":"taox16teqf3fta3xl4s46wdjypd9g3l6yn3rm5ftxlr6tyuh7ncjyvkovxa8vwau5w1rd2z3d88kcst85q1m408xkkr9mw7jnp5x","191":"r4n48ua4bayusx9r17k9st1puvaxer4mcbnmlnw4gjutrh4tn7cgvy6vak6vwnqeo1sux9z3yma8lci5vnnfmjoyhdl9bipof63j","192":"p4p4nq4m9ustmfaf7x7j95uqdv0vgbxwvk0oclqa1p331envxhe3gr0habj3gd20f1ozbsnn4gakkjh8ovgqiq401w9xxtlv8qa7","193":"aszgfxbuk59w273zpe5kjgvc32yc416r1y89i4ak46j4mwnrnz2pyd7mbqcw2pb4h01gxdgpo30j9aykv8rxpneltkk7yjr2r9di","194":"o2pdhzcsv1o29ktad3mbfwna3dig5nk4j0r0vpvyh373875y7rchd4mjpk4mbj6rrraot7h1rsf02jp1qxbylc67cj93upa8dczi","195":"1l1qvchrse3nm8u5a0ds7pkht00y44gct3x1evjh3f1y55pxcs7dywb86iepi4yabhbkvfz5mahrgrg5idk1f3xbidcnc98khw6g","196":"copf2pxwpkv2voaiovk4b4p1ur5kt1pok6kl88declzvm1f7m50gyp560pg3wijp6ufwt5ywkwq4qlq6z09v0qbusdeqytyhivqz","197":"6j6kpqvccc1699j3pdc8empzdbkssfkcsusinpnb7gqg7mhgviul9plmufwi1rotwvezlbd0jgtl8pn1fd0xz7kb9hxiu0wwyx98","198":"vvcfq29cal0l0hvlvspm00cge4vyajaakq8iyt4xspdugh7whcv5tmx71hm1ixkz6isykvugkdb2491sz2j6eu2p646uvfjw4wbq","199":"0avefsocl4t22ufq59kvpj164drpeimmirqwx69hipwsigz0yz00mtjrl6rxsb40b842ewfcqofjmx7gwbond4tqr18yu6ouui90","200":"n0totrxt01rmwfux09juaeandw4awgjuyhzwmm32z31soae21ghngdq7wgas8k4ltm6nxykbm5s833ika73x9xs68n69olso8xeo","201":"zwahegre5ceimwekcetvlpuyy12vjtnvdj8grrm432jk5361gxp7cdiaspxmqo0kkpm74uj48m3crbrsfw3jqu3uhk6ahpzlcib9","202":"t1omoddaixp78pyk8rxdrvdl0umrc50y6nob06c5jyvtjv3zjd6qyuf522fqzfm0p7tygolkbn1zfcgnwjbi5lkp4xsmwl4ha768","203":"nr2watu2zm6snip8azqpqndlcr1985e2sun6rn5t8l4w6lkh2ojkguy3opuowv4sb4lx746jdogjxw1d8v4j5s82mc8n8ed50ymu","204":"1npkepf55p91uetzwmastwx4u1mxb6wilbjjgy7brjfuqb14xdypsit3kfpa9rqq5p4ydyp3osg3bzuegrr2caakww8lgls3s97s","205":"gdnwraa3cc4nu0mledhaa5tldua1fjyes7xb42u0uom7m11funo10pc54szeuy8i9a7rhi7ju5d8xx3yuzwvpq0uo603owmfb6iy","206":"iwcvx6ika5cemtmxuqsakgr0grq40bsr1lrp3b2hzgqlceemyv3zuwcgre5t5b7pis7wurgzxfbjpbcg1eoipf97se378jgm19cy","207":"8uaj61vb6yov0g0r9heq1r5db3wije2fhzqwtjwntw5lrdl7k8y0diwgixe4b01r1ximpjdkwagjzo9stq01x2twofo22qpdllv6","208":"ia27xs3vklx7mqzmc55pq5nake8qwztzfpnq6nqmb6lkbovmlu25oybz94fflpt5ggebsxfhu0qdr5pfpyl9dn4kp1598ymyvc4x","209":"3zstth9y18cfg1golz59q04mujpfbiwmkgqfgp2l9yzwgie3r6ctl2xupee55isl7w8hzhsb1op90ocu4gb73vuhniurczma57kh","210":"tnsk31k8wdtkktb70vt90sdsih8ma2yifta99zndc7ocbz4zk8u37braj17q2gec0kak590fowz2dhqctr374unvl6q81nwob818","211":"6jnlv2jwpr6ntsoy2f6xrslwf0bngcy9sjkfagkut66191y861vv2ixvl3mjx6ec9wcdz0hoqrg2dlizdu1yygjhhtwa8fpxxunb","212":"taresodaf6olvfcd50q6bychop2r98ts8nl8zygdxxa3okuyt0q1arzj66n59relkamnfet4oechuck6j8wj4y8ec50xv0i2ldei","213":"3n7knb90u1k97qa0jdj6mixxatdoqkpxakag4q5qv7zt6ympcorch1zpdatgqgjflsff0u3p805ikp5p3qvh9h9dokpq8wv9ekm0","214":"bkqr4cznpu6vfuleckdmud1bgveu938s89rrvuyvyrciz657lpbsiodeuxcevzhnyttxbqwnitzi0wlvb51dq2apqdoz4y1hm0vv","215":"mpo9lf5alb0mj5jq2tx14uh7kudqryl10deszheu40omerhhoauts0hf84ak2q90q2rxhnoi2o73ytojf8tbivpnv7akuqv4ci7n","216":"io6udvetempw6y0ufn6wbu62qf46wqmzdzeu7zgnfxtmw2eol0yn2jgi3a3i7bx4hqdymaw41l1w3oqd2jk3lyfr7twb6pi2uwah","217":"lmsd7f1zw1usugeo3cylv5yu4cnzm71cchrxnamnt9cnixw5x6l6cwiue6oqzgf0vuntcyaz4d6y14ivt0w96cish7dobgefk7i2","218":"09i4zvl56r63vuoswh5pwttziumkvggol5q3nd57g2zkuv7nfvxkgy0fgjoe0yjm95enw7vxd8z1h5mr7qwgw5lyittopdegdnwc","219":"22kapnn8cztge0mcwywgw918wuns53gwdznhs40cdseq46f8vj5pzbtnv8nzji6ng0x98yowwdzrrn0gxqi6h5pfdl9hwl3j4koq","220":"jg8rdmqbtzrtviejcu8xpfcfzp4ozxe468sw2o626x6vrbrxeawmcfouw5epu73uz94o7wh5q3h7marty8pis89t2qkpmab6fhp3","221":"n5fbgqd0qx7r2yrkzrm5liwf4v7zcksko2hawjfooqd8zh94x0xdgm3m76akqyc5yuahpkpp9gdcfg64dmkiw1sb0ryz1e2xdhbl","222":"kd5rnytjdtuwhrd0ykn89747cis850ko6gs0yuhrfvbms53zzmpvdjzwpi2cq485hzf8xwwktxj8llw1nf093eo62dibllvmpqc8","223":"81eb44k3s9ys1jsrf0g6cm5doptjk8wle7n0akdyqqvy0wlukjrcjb7hn53x660blvouyyhcq977vdjrumfltnf7pnsmlh1hf46a","224":"6fv21cijhs7nzbr1lqq5e7n4h27lmbm642lkkkegwuw68fkm2dhvv6i0oh3vysnie9f7yyqd4yu0pzw011mfqslxmcf2gw5f9qwl","225":"yiijqry8hwytdmus68ovqdqpae5s0xoti1flga81y5bh62togivwuls3u54dxyxh58o8hnjsdug016kvx2jrrzqirjz43mjryp31","226":"u23rt8wc8ngl4qizth3dutel0fok7yzwcxd8ilee6t63xi2wwuuoy1hykffajjsiziispv8lbdd68dfject0dq51junm57u6k0kc","227":"193ywst9pjjiq3hprgh1cu5kg7rrd27m4jrkck690x9q2al5v42qrispfkelpk4dqb06dkplmt2o5u2djlwjw4llme8mc8f7ncx2","228":"hiov41beoaxc173t70bunakplv03osl6p06fbdhju15howp9mce1vo7ormeqsga2ris2r7h8weqgvoywwu1pjo0sk3w2r0ztswzt","229":"6035xk6cebnjdzhai3wyprk7bm38sfnt8jjplzhd5gzt389mybeofq01cjrt15wrewo3gx0rnhm1jujc2xfb5r6r9p2iot4ywiu8","230":"gqc05ebl1devdzh8g3rkdpkta2ox1bkv1h52rco89zr63aajxle3nfrxjocdf5ht8zk7p1qdfv95svh6zof8yh68tbs2pe4why7a","231":"ujtk5d9zbimmde9njin5jenstdvaaxiuxa65if599vluf9q6h0qhig7pwc4pmawwyg2rr9ybnbp1l56nskk7wwvd1vq3j3bcp5uy","232":"4bgbf8h5b1mfqx6mz8vjs1ni8am2tzjibn0j6eaisdkmemy741s0gcmffi2qcydpmthgtwhf4tlvi24abg6rtxjn2i249in0slq4","233":"i4ur9rgl4b7kcfeny68dekqslavyj9lz0ij5ll653keel8a5u0g8hmbrdsob385oksquapt2b04ly2hem5sbzyrcfelhjujriqgu","234":"d8p494ktrrnpvbmnoef44qh31wihfr10s8ohg6ubueqftt82eq3s1591h76xd1odo0o2u1cky8c538fznw3u9dpfrdc0ae9om40m","235":"2gpyxo41echmtzl6h78smeqjgo60rholodqs6yrpaq5n30o1nfpmfasd7bffrbhqkg0jskajd6g8vou3nizy04lccme2d6kjslea","236":"wbqoimou2qw5h4kvrq3e4gdbpbwemtry1va3fcmnd6prem8bjh8ki7x2vllx2tja8ykx92k1ivdijsu1yk562eif9a0bnfhwkqvh","237":"mhed39q7p3tqhvqmvgd8q83x2rrfngg2yt2xhyrt0v1uwqjnm9b64zh622tm3vr06sgoiwwpks6bk9qot5lp4ywhpaxkop3xn57k","238":"w3rrm6gersrxmucc3irmkm0k7jh9oi57xdyyvz3qir5xwktq2grbuvyzgti06q4w9tolclan8wppxrjp7qc6hvzivjbymzv2o30g","239":"mrq0igvclfgn6qvkmlhmt27pj1z9whjq3kpcqoz3rtx64rf89hx5nvyq8kvg5fghzlvysvpjg9t2q8umpht54tbrdq717roayol3","240":"giyo3qlbprv4wawjcekwfqhg9nt797on2c7k8zr87rn58mmru4gweovkg9a97b1dn2muiqjysd8ixaovo5g1p04vqealv40we7ln","241":"zokxszxcq2ur0p1ahunfr4zrp6jazk2grtx6g8r3rhrkfrb9p14kq1ebftldyer0gfc4a5nngli98o68mvzvzmxxwsvcx11lt8qx","242":"6op97cd95c53x9fubn3si501cdg85u5v648p2szlnn741ztykgtbx19gp554k19z2s4pbl2w05a6izsfegtu9vkyi8lp8o05l4s2","243":"xc6ste9w0r2ruyyi6qffmaxk1rc5vx2ia4hqqh6m1yjc4psldtjenfi1g3cnq3xf899gqp6s7i77p6hpv9lvcut97ewl2437l04l","244":"dcsxx79kj5h6rndk5xoz4bh99xjua6sohhlu3v2dzht1qb4zbc5hcj13y0sy6qlpzax69u6ikigk8xur12stdturlniv1srqqrre","245":"n465xaamzkblvoslmnyeptie144s09lqtd92g6j5ze31ujigu9x67zbfhl98masihmezh2dw1oxs0udu4eqnjeuuchvv2u3jm5m6","246":"hscwsotlvb1tw0mcckee1srv0xjfhxmxt8fngsh5muv818bq7zmpy6v0n4wbyqjnj3hprrmxydnrezu5fyvmnc0rhvk2wkr82gss","247":"ddgkdwc5ctn4rge4dv8lwg0hf13o6zxuws4ket4kfq2c2bml8z038kg43kns25gznch9cs4w7m9wms3ge4cstlzu22qnywqi8vfr","248":"ddlqfrcq17z803smwed7seah64qgftvlamw0svpxa47le16xezcus1i2mdixywmsj00y6u8gvagp18pmu6yiv6526v36ea96e6ji","249":"07w0tr2387j6mwf6uzhd93k7gmb8at1eqpydgv6wqnyimlzmb4a7okh2kgyjsx1cv9632w26va8gmeyc6brctxbutvl7xjb8pxnu","250":"ot5nnmawmyv5hrz7nvcaue2ceigumgs3b16pzg5n3tjbgdcgpckqfaxmphg3jmuo6e9znvlg725h5wdkzrur1oipfv14nzlznjyn","251":"fugi6rsba8j43an1nqm5kpewa9yraxfvc7he8a8hv4f3c8ervlx0tyufvkj6upqrbndidmllzpyt8j7okpmpiblvyrve8zyh9jn7","252":"oro5cwpnl39jbbdi637vg82v1o3relixb1fb3riu9xp6ts9nlnq0wab7fee2gpy57po5iheu2z70muu7mhajrdqgt27fihc5eq2m","253":"nt915dq1mrghzcsfbxkdy8yf36grwioatp8ymdyzcql6d0ycaf470ljv6g8hs20sngjrnhbjwd58eye8bfsapo7xf61v5xt3ncjd","254":"x4zpunhsly1oxl7ukzfihopp4y52f8xi74g4lc9z6k038moz4bqdjptemtuso49nbmhiawr00gg945us4lcl0jo359xlqvsguz0i","255":"gqzbdy2v3v0ik8ox9xdzuizkp7c1h1mtww76jrg76pzz0soj5og8zp9dj8928lsca6h0lbgtgfqqvdt97v4w3hwdzzf1f0w0xgy6","256":"ad0b02xcu60txwnycdhaprv7urjpkugsj274vp7doxf8n1x5bplxeuq1deotrptl2tqcabvzi7ghphu0ok0jpg3coxchu4iofr6h","257":"onoqwscn0gp43q036bikgjppf0ze8ehk95p3fwllltwqcqp2shfvf4d12ksykxmsif2n6w3m8n34hlkpuzhw6eoq17zui4su9dj0","258":"sgyi79w1hsirfcir1480a0lkbek3akhx6tm2qbiweym7bmz7m47rzyymvl460nnaytmhjnj2cal0jz7y8bdkxmq2htob776crp20","259":"tu6wi2ad81jdjpjjwmvmw57imidpajnt9kayk0bh540og822mtucvq6t93ienc3kl6p5fu03iq1x6943yxgcehfxdbkymm6r0o04","260":"sl0jxxelpwib759254lsi79g7oae1s4nr0if7uu1yow98ukbw1feidcxfol5pomym1xitjlq6xonnsjbp4lw5m8e6oj6957brb1z","261":"w5s7x93m38pukdcctr00ghztim4341wc6pr5a5qqy675b2ft3qc7o6dlo4kfnu3iqji7tkzs2mtz0swb2fbi9wbmd1ytrkhyxt02","262":"hyved1bwr1iwmkavp3yfx8c1e5oq33j8glxawkzach9nvoz6gyht1g231kh5el8yj9q9ls58433c1v2cxl18gb9k35mri0ehjmc7","263":"74804nxt7ryr1zfnjahoid70knwbbrjtbd5yyiuigo6jhbsg324s1t3xj1vhw2hdi4775hbh43uvx6ld183dzm3dqi9ty8ixozvq","264":"v0ddn48n4algmzvw6rov3nijtnbnvhk7hvlndac838w9xh4mb3v7zavl34273dkksapm4c97vzj7r8rs1akywvgcn9skj4wd5kad","265":"ommt727rc86pniqjyvrcd5384ptimc7wpjw5b7beoxfs964hjtzi5sn0m3nclx9iwj5lamjf3wo3l44ka1vxp7tfo6dygzp06g81","266":"zt1b5vqubh87910bqln6r62mupvlqpvzhy0ox0wt44ru901zkdb0rk1comc2cicws7fc3ip5t4ayvx4xo8ao2a2350jjj4p296ac","267":"la0qxee68yjtlg2z2cq19rbo5s0ih5izknsv96yc6v93o6x0o0h1tvqmlwdkndvt8jct9rc4w078ca0cyncgdtmm85eghj8tp0a8","268":"olygxlhap90cjmq3v3xq2ztvo37z3omsqkdicqi0j9x1bfmob4vjpczh060dm6w21adrzrdu26ido49lpp3htdjpgneyx7mf5v1i","269":"lx9y6munabb3bxzubm0u4za9f5cdwwzmxatl0zdx2go13m91fmd2uwbqb79x8tarz9rez6a11q8f5m1d9irlbr9b5p86jl4gguf3","270":"4hgpu6ql3hnd56qrojuwz19apvo315w0g83d4x71v0bg1fmnga4mx67xhmm0adagdyb09gqgza16fpu97ihc580g52rlie98rt2b","271":"uxhtovmqb73il2xug10plxadcdboxx5iz261dkp2jz8mh0umbr5qj090ggnczoj6n364wg91pdr9wwf9uid1fohx40okmopm8h8v","272":"a9g64lifplg5ew424qgc237y39xumwrmnxg8fn3kip3ifgas1maicryywyqm1l9m1hfx5a8lqp1hug7h5a1u7ic0313zyzdb1m7s","273":"3al95nyd2wnntgwmnnq4507vhpsz5j4jelbqg8ycohx5knwg8mp0rnnvacqwr0tq6pa3t4onq4l8vb6puwcuymc7djhqf3ia69ps","274":"9ji7l0lryah9ej3q2qaodk9dyj5fqj5ojjd1clhcssxg3kr8t76ounwckhgv22em1l82m18p6dmxg4zwh4rjiaz4g9n09lzywsl3","275":"qn7o4athcmquuyxqys7g3g9fxkw9nrm0c0fsal5cd8vtcrv51nr27uiv8zya9dseub33yuyd5m7x2vljtgvb8nxjb008a0t0jxeq","276":"fajei46378uoxmonsusr1ku54makmftzvk7ay8dmwf954wvm36kqz6c4m76q1qs7ivyqkhig2i8lt8aqiuvsrcr884rsfoaumdsm","277":"agysqhzj55s3q2368hqbu1685oaws0d62xxz1aymq96sa1dqm1e84o16jd5nah1yaq8urb969n8ngc2hlt433ai4sgvl0ctylg85","278":"nc1nfr1ugld1hmej55lobh6fqj6oxkhvywgi795g3vj5yaqfk19khbg2b1sdt6dk51qfkc4phrffdsegleyr5k6b9ryk2kht10t6","279":"uf3sxah3fgjnlhyt6mt5zsmzqeplxrzf6fuvl5o52h6a0u8i33cxr8fnl8yfsrwmu9pvqugqddiy6l7yws95atluaxlvnhjxt65a","280":"yk4hhulczm9wnaforud1pck3xgwjfoz5za7ul165kavnxx11sz1bebjxi04h0uxtm1xz69xdssm8t5l7g6vf7s7gkqr7pwgapceo","281":"spnajuhu4kzzefdaql25hy8xuvi81jvl0fslmnqy73o6pbdvzxdzegfmdx4og5clnizw4wjcmim82dpuv9j6kbpyix8470y5cics","282":"9xalr9bbyx4l7dj25uns6ejmxwk4eekl5bbm2a1zr3uhl4bl1bdb3p5y84gng6pori7agtacrsksbydav9ettjvolvhxi2tj3fun","283":"pp098lligamo3l1o00xyrzsdb7f9aorw82ucoy4gmwfzbj2lf8hwqt48cppnidf90r9d44esyky4qxd4xzw52mqkuyk3ktn4hemk","284":"fy8z894cxdaaegsxqkrlwowsbecagv0dc9p5qc76lx793j8u7j1kq1mxosd13scn69j9jxbvb423tsmcznuv7x0gblh73jlt30kj","285":"fsggcf2z0a9g84ntgttte9nq4m1c6urfk1wv4hvh7tq84wthcylp0ooboovorl52rnru02chgrvzfgndn4cb4k1l8cvv9meqltlr","286":"266yxddkwyp4kbb1stqeauo1tot8u1tbxz1j8pe37aliw3895hwrwt2ij5ozq0t4r6mdahtbsa20vpo1kvggapk0303rzbvufmnj","287":"botzitjf3i4hv0x2vkgmo0ogeulrjswl8l6ytypcyotmjhq1pe2ct1ol993w4t3mv4khy6zu7h6y7xflgd3dwki49zuqi1wx0hbp","288":"wku24ugphzwye1czch3pjwtm1ilvuiqlpkrub9hxactxsyofd4me60pvro585hz41s17hp5jsn4muvlky5pp37tjmro10htsnlop","289":"fnpv4mz05a8ollbkx15orh68eeezxrtem2doio6gu61tk1knuzjf4yrc6s7qn49yzxwjttr2ldkwvqfysg19d4y7pp40u85xgu36","290":"douy6rp6ahad9tdukcb9nt1wqfdmy4nxbmo1h1erss3bypgew4fqo87oxoibjckld6ueg0t6ppvbjjgmdy10gicb9n5emlbgmrag","291":"el46w0yojmnjtnxbwkh5rorzo7mkbuw8lg6emzyuki5jtibbjw7ynq13lzjo3d6nz42ik82aiicu2zicef9es31zvd6fqo98vvtr","292":"ko8j1arfpenkhuji2khk0as1u61zyid6kgxx3hyy6irzgizocua81dsomwz5er838tpeamuhajgyrlz0elbz43cahmys071on4ak","293":"a6biniojx4iehr75ujz055z5wrz8wj71sto35r514xsvby6p7hp2xls4r1i4eiq9grei3qrzrix7hm8pxf4tn8zvkomjisihe52g","294":"ha3q43hodotlx24tg7qwk6zp2wef25qrpnl3qc0g6gbnfh537gdmlvt4z2pubbvsv66lycrg1b6vruhd3gu2tplgwfc2av5325lm","295":"y4if54atbult4op25a32dzjufb4gmxuv8yl8jl58iy76nc99o67lzg1i8qi5sburqyiw2qr3124hevf93f8mmrbcmkd0m4w19ku0","296":"awk0gw46j5b34vjuag1w6nrrxuw11el90jabn9hy0wmrhshhz6h52stsxzkxl2rux115ny6j7rrx3hilioq5v7w1d9inaep894js","297":"y3mfc92usxiww6dz94cyxwagtgapxgrvxgd8wgkeu8wql8aujc5q93yfv0wfm2s0zgll8ui1h989djy9ho33qbw3targrhciv25u","298":"ghk62vmgvuktv9izo5d0kd6uqnuqp62jslqdf6uu5wtg2ubd0p8goz40qnpyrxnp3y7mn5nw781797x247yizqnyoth441nm8kkh","299":"5qka0xnwhmneyeiexmy17upf2qtng3x826zggpyb8dszgizqm5jxdxuj9myfikjkrii3ate13f3l7i23ih5jhvhaneordytdih6t","300":"mmkbxr8a14dggftmck9acko7gwvztsircdghwx5fg3c4smpv61pqr1xy5i73jcj54veumjq0t9dwqot4i3kwbdvsr4ccutnus3i5","301":"nkephg57di0viv5jfqr46f3j8s1zhwvfw1j58r9sun8c07780xaq7o0d3narf5poyyu1f29rlahgxmmv3z1lvh9rfg9tsnuk1oc8","302":"8r7zlckmcsuj5ex4zeh5i9ngp4d26gk199niawg7iucvgv0w0wbtx4c5w719i39c6xqt1wx0qm9zhoifoog377416lh23f8bbjfl","303":"j9y39v9dkbxhyqztlxhb2wf91o699xnecnlgfbqt1r0pj4byzdjaa847938ufsciyop46usbxuj9p1n4eh2bvzojpsw2f9st5nhm","304":"nxvt1406oufyd9mkzdz3uurhrdnsqs2oca7s56y0y7gtjx28rhrgnhugm42ywygjo7llxyf7okcbaq2enudvbbajb8f1pplqup86","305":"jxu3xidnabkl9uwemu0ylgx9ofywf5nle09kwee351851uhdjwpyfj3je0h6y95ompxc2rqmkj31k6ijb6w7ovxo35p0s167ichn","306":"ggylyk9o65bag9di0ah9k7ojihhnnzciu9jiobkuq09psxdq5un14xxlqby3kz1o5xhvtkylun7tu64tj0hxxeuqzhzmheco5jim","307":"84cm15fwt79egxyz6xde26xkkoxq43l80gpe6ujii9wbk12wgis5t9lsromlc24mkl89xv4bmm4a760h6llu185g9gsfgrvizdq5","308":"5ow0sj4i42w9hd1mehl0sfcdlw4b5pauuf88tevnfn40ejr04yljmrba2l8ylcvvhdp3xvx9z024alwrtnjssl793eeiomydhly2","309":"a09yqsohounsgtgotguhmb7b4jot2k6y4x36u0fgviiwt5skn3jerywqckzmsnbsv7ppp0avj186sldple5o7cim6ipqyaffhnr4","310":"jd4h6k5skcow05dkwtu5c8cvb8tttkgocrjhntlnw0i36nah08u0lptfrycy45n03aolsk7juu27r5es6c6c8jaj29uqe0rbdh42","311":"7smqsgux61oklg46rc4tkxrfht7r5b4dha08bxs0y2yl08vj90y9nt7iruhdlnh910h9r8a9fsg7kpx917v0op8ofy50m63e9lq7","312":"ipdhzzz2ar4g5eqj1fpkrmdxcr4gh7hr07q8p7furq09jgqpp07odfarq53urjg72113wm6fm4u99g2nlmld6hu4himyb28ftoi2","313":"neqwuvevi30txnlvtga2in1dx0z47xkwabsc6mmp6cjk4iicy83o7648lj6qdjia9nau0ziz14hm1n4xarty7zw7u5mwfo0nezgk","314":"86fc49yx3206f79cfm3tcf6gwzj4qo3fd2offemkmoajxmzkfflli7b8n9hcaragv5p63vqx0fenhqhtlto1ja7xb090munhttn2","315":"w0jhwj11zo7bz8riuu8pldewgpuu572pirllykvoc7eb5171ivrnlk03h9h6a5qk6byy1t5zuth9q44yic0kjrsistvkx3emkfpj","316":"2ct2kf2xn1rgrbmj77erhqk1k4926etcjk2439bxr55bdlyj3r59g6jah4gnvk8e4cb7wwe1kefrscgocfyskyeuphwpw2eyiach","317":"5ko62r2megjctr65ekf9avenfnijl6d7rrb2ktqlw6tshye33u6m5ewuynjrnvkqwqr5iuufzhk9eaefawm9l08feasxxr6wgjgv","318":"j0lfkupk01zhnls0cj0au5zsnbvsx8cjpvh2qhoqytdkq9gc5bo5nt5ooe88n3ykj98rpl5g0si4t9jvujzmu3057ly3xsc9c72g","319":"gaseufrixyspsybukvtao1frsf7ykc5htlzcwp66vs0otsshsqv3c4awco92du6mpn0j7874ng8zxx5hr9ws9lsopszeqsd9z2g9","320":"lgubizy2kyf3g1x0e17u80ytwy3ipajfisd6b4wszu3vlzcsoqpq1ujwoiswmxbo5mnuozfeh5ov17m6hcuv44neodjkzfdwhbsk","321":"ewkowtovxd534to4firdhcq9f2jne5k3a5zft6rm40w77r45ylth1alooa625oags8ledoe01j26ukv5z4nx2xw4o3aotbrw03gn","322":"00q651pf87xfvtrwfiopx0nbj3z2ume22v6dj5xsysjyb9yi6zpkvgohsyn2q4uq4umj2kiu9csfbb0522n03py3gzsmf03o4g78","323":"htss69ku78ovuek1q2bmm6kgax5e3340q0cg0ixjmkru90wi5qqo2wfvd8obmeg0lzp8pvm3ej55bmznj2711jkvqyb3am1518qc","324":"6djvxvip06br4q2kfudl2fh4aq5wfgyi53p9zgz5v7ltkzgi94eytsv7ljxoe69cc5g3pfdt37dv00nwwgdo92wcfukngt8u1kte","325":"0udb9hobwjn0pks13gwz1xrc75nv2v429g8r710bc03ftwfo4nprb9bx0xrrrl4m6afppzj8wyjzfaf7353zh7i1r1y7fw1twaom","326":"x4fu6h4s67jkt0ut27wpl5j91h8v5adupzw5kyd0vnjkkkhzwz8u7udrtnscbhakc9tk7gvdlteq0tzn3o19gb23poagmkgl6kyb","327":"ckypkqgnc5rljrntoh7rlbr8tsoyl3klschd7wbrgvgwfvlhy3tpoefwmjrnkf8ca365kh5jubdsvesvs41pb9ko8sb97d432kxl","328":"bmgd3sudjg7nkvcqbwfp38n7koc90w8s8a16wsd8mq9bk1baig41fz322hwow7yj68usg5ab5d7ds8ghb1jwiogmi0bdtkuhu4iq","329":"4c0qrtl3ninj3wrea6truor16t7jrqzikiv67uza3t877ppkp7q5t4h9k040cv7x5mpr27tx51h2ib0oflgr3q43meepiklfgtzl","330":"1bgyqeghb9o3zgioyd5ypknz7tgk85dc5ir1fshcnyxlhpyd5lt7jomdn7v7s6xno0lj64zazhh51b7h5ubgzs0aj68nqs9jk6ib","331":"e0uokc0jfk77tzvp0cy549mw5f9sd5qwa43e7w042vz9cjn07lluo7awgvjnyi82z50yun5pv9ntv6o4whfwkuuc83zg4ox044ps","332":"5m8v9upuq34n58o73qgh8sfxt5sabptv1jxiowg0nvw8lrzx6kk6haggmyokodx0ivnthrn4mcq7dwkpgrgoxprgmu7se2hcbelc","333":"908resz5htm6m523dxgabprf7dcqwyrbu9t8cd6j4j4nh7cj7035sge3oiw6bjvo9onqgyq5m8ha15ltqr5wswwfmdr6kezlqt9s","334":"niho65zqro82aoszbf5yny6yhfyit4s54wd2pk3p1g4vc8x403za2x5kmqsj0i86gvqwqsnoh36iqoqxyd2hbly7fw4j6chvcj2j","335":"psuo9zsus44q98idts3djlfjfpbfjsnoxtfshq5ltubroyhrc9gvf06ij3jjmfu68ugr7o64smyhg4zvqjd5weaj8kuylyvzcvgp","336":"6yq4va0ywj6j51itaw6eviyd944ug8cmboui2gdzmyzyzgm6p7mtuqb4t74d3tc1r8slc4vtfu94zei9yjvdobv5rh7nbvfazpk9","337":"7biir77e7yjqj2wk9rq0nesew9gbymotgw6x43r2artyd00vonhg0ouqqjl70n1y4yc3sjsmaveruae8o553cb7wfjnmgj2p5nlx","338":"hb6engteflht93o1x6ld6minxwjw7al0l9kvfaz6oxh31vemd5p3hjag12s1imz8oz21zbhz052f7n51ntb4iz4ndlahjcnltjl6","339":"lumepbqo9xfyqerwyv47u754vsc4krvpyg7y5kiihi26wy9x3zlhvn9e4dqvg619oo4x4lyb84lxa71ydb8o12xbq9nglitod4zw","340":"vdzqdm4b74k9mnsi9s7rs2r9g19bi8syabrgxuskyev239882x70tmtz41v6ewf1bg8blui943kkmlc7mzroukt2i5llw5jbovt5","341":"0bjxxztrakvnh9n8z1c6ma7ghynhto4ojzrzpy4h3hn4asfbe7yrzq7svuzoghvw8z4f9kv4cy01epk0c8hltiwyqirelt8wxbz7","342":"w73vi9j5v7pep4cvecjpi3e41gqgzxrpt7l2h3q9qazliqra2cuwy7iejy97jei99bhthljyeklw5vtvx6ohj0f0z41n2q7eipq9","343":"vs4rmiqakaxdnkzq5ffny7kz2eypko7w2ybqj7sg731d3y7w40w3i7c9vv2nekdsnb9rfx84osvohai6jmgvze54qhbeidbze3ae","344":"gkxvudif8im7f18ajn6z6nru8ic3rv9d4wghue9rsfvsn2euirticlhfmp7z6gxrbqpqx9cbw1qsuympzvi4d8kukwmyao5rsll7","345":"90furtlq83xnr87fqc57853hum0n9jbicmsrtwtf6ro07grlgwz9inlnflh9tgep93divvmxiwcrcxthiybyry18qkb80qs08pma","346":"ggymo37wltus38pfovqtosv9uudz37of6keg6sosoh96ahndn122iffpax4beqiesh3qyzpuv7a05uay6udopttby8g4701mkxig","347":"24a8f83l8v3u8ua52lti9y7t3idz7o89t3c0sjjybogip99raq99dpn7uttuhq9drvvuc6le7ug3rer5yanvsdo1ppvvpk141ld2","348":"yg72a89cpnhu6zvz6krniuc00apkd3bile9w3znraop2fykoci8yrz6i2yz40wwwm52a4vuebf78f818ycw4z8w65dwwiu1wxuuw","349":"f0q6we7pkh52ouxhzl7ady6xua4wzsj2ouv3c9igwy97kunhwdn9bxbycpwaam3lfds30n5fwkkfiwojp1yko3q5lep47x7u7y33","350":"4j2i4g8tpwcxp5vwysjr568qmjxdf83zkdvtairfc9h82qh3k024shosckjycfj8gis12oiyeco5axhmwi9sns6q4i65arp08g7u","351":"tn5bsav8brp78rkfbuf40oel7gc4cbb26kg1y4a4suf0j3f0niqbt0wbmgmwsvrdugiaa148m780l859edwmlxh1pyju8715gcv0","352":"4650kxhmis3f7bxit0kdhcwmav50ppjqrw2vlok9i46j98do1460i5474z5jhsudxh3u0i79brs42kau1jpr8a62g15owv1cjazy","353":"mj16maj1z84uw9vak3vl8u6u47j9jwz6j1kqopm5pxoffrzh8fsuhrgqt8pguqsar49aua0kdf07yg1vidkjjrlgye7z2ul7mat7","354":"64h3e8824o3ajdgyo3nfa2i8yhbgdnpyaig64249w1aa6d65qn21ssww3c3pozs7lt7afm3bqm9gbod8frr17wxf0yy0153i2yrv","355":"fheo3nup91flrfo1wlt5f4qssabh40schhswik2lgl8jiprsluqsq4w5vqyh3mh59gzhsva350dg857foqc28h6ot54ufo418xux","356":"upublsqr58ph43ykpfogfalnvhssiib1a2g1v4uf5kmy1t24uok8uf2nwm4zh4gn9kmtk0o6f1fmme7u025pzjr0ytsj1wu0x8vx","357":"udonfzun3extjswqqnmk15ygv4h1sdk9rgyo73aq2bp0bf53u01ah85eztqtx9o9qbekxx7sqz0v049dya6bo8nipu9z5huuy239","358":"6c3aqje1ungh1cn1u943ezcnxcs7absl4imj3czgxo2urbiuhrbnlccskabqcw5t0ssw6mar38obs7abpcxkeflk4z6s9o5zyfwa","359":"41cvdg3blkcoihuo9xim85mkihwbr8lmasssnauq1j6bhel84mnilpobrmj2nfmzffxugry7xk718j3v0xwnc1uwwdlt029be2g9","360":"3orgk0wpkukkgiabte3nhraceszpewklqkbwk4gxvqvz2iicqsherae7aspa0u6jztot0jioc3vhe9e8640kildm0anq8hk6g4tm","361":"rcs47al62pxn09u8k4nyjcijppzbkxswdx3exxegykvwyzm3z58fwqppxnh8z1welwb8m63m1kxq9tnd7blv0j9q3qiyq1et7hxx","362":"574btostqyc93pe3spalqr8i2fmwigf89rk73euwdmtqmdaap987c1df6geuolh2bnwf6ascdb4yjux9v8y7hainuv7c849mub1m","363":"7z5y2hc5olj2wcyhjsbzfuvj9ws0jegf4i7xp63i0nbjamphpyctlsi3x49e7j7g2qbnv45urq4y6sby19e0jevuh2vxd1l5bapz","364":"grw1dc9exe3xjh0bzqtazlkijo7t2q9nbg025u0p9spwnvwf6qbngmjnqioakw7lc1ksoiapciekpviwvgj3815t8egamnvf1htb","365":"qa2jakmsangnl5buglagkx8hdl4vi4n5m6qn9w1gyw4vt3ishdov8wlzcu2gbzsleozqa959o9du54kh6wv95pusqe8tggxa5yxw","366":"vexwze5vxbz5ule4nsyqbbrxdgcye7fv2s0w8ib0bg0wqq2r2r099al3otyeyzzk4u9ynmuszgithbje8je780qctdbzvgeeg5q9","367":"mvu0pwvn8mz8qp3valdmuqh3q7lx6vcv7fwlhicrdamt5c76xjc1xvw9qbff8sypg44aicore7gxuv4q0f6f3tih14e9wfgrujzc","368":"7wbkgw13ikdxrac5gg95rrfiybbk9jyikqh5uam0dpfirr2l5lo0xpd2pipyovj21h11xb7fldik1z1hvpubesa79ho7kz7vd2mr","369":"xz514qvhxqpisouq0of0z6cc37tvib4lhq6xfqfiul7xkfgk8c6yfyne1a6u9pxyr1mmnp42jpr90nsy0fv7htd2l6xyhb4prk8j","370":"928hdgvmpf7f3n964l4ex44vn5a5wlyrhuus6mb03biazftl9fslktd3mughfubbt8ak2zlexiktpqmzs6gszp1jn8al7ns4sq2b","371":"1nai1mmqbp6rmwa0dztin200k4ag601fxpofdpm5kelcwnkfc2qliuxx9fxrixepwk84nagepc9u7o7oor5huyvcn5quor9yh8cb","372":"u71dzywnlfhuvllz9xbshfwl925nk5nok9bhkeyfcladsll92slhu1te67ewrsdagfen6nagk74a503bmkpwwoq6mh0e3poqy6jy","373":"uk7wpfrj92q54q6kzro6ony4fog1j5fn50zbv0brr2vly0pinbbv2bshu5ekpgqks9xe6w5rupzjvfl3yrobo2evcduuehhcs6jw","374":"uweei4v3e7xpmdobxn5a6put2ih6zm6xlyxerupyktwl8rltoyasscpp48p4dteaz67hdpxp40k1u8vxqty6im55o4kamjyyju84","375":"5tiybs5izql4p9n57waj94y57ha1rs39f5koa3ompthxg9eluq2dgjie86xkevyqvxb25ooyhz44muabnqj9mdz9naumxgyn8qiw","376":"hpkxklug93n2afa4wa63d2a6gdtma9c67tdlvrtysqjp69ys5y6rl0wjmxl7p8s67y93pknijaaqx3y7fmookfsu32nmg8xw0xp5","377":"g8mvnk8h607l9jmw2ycme53ryso83ifgxmuoub0ojnb9a1p294cecrjfmd5jtz615p7d6vw27aot7xkohipp9a5vpykchw54j25c","378":"aiit1xn71lyvh578a6gda10uc0slk0p9v5bwqq0olbnnhxbqpiilwafq25srwv98sh9h3pzv9qiqgfvfg1rvesx5rrbt05bx8faz","379":"t0nw3ez8x42tf3jj2fdvywemj8lmhy1ifd5jj6fn9zpots0wioq8xvxae58y9xk4yf6ryiencls839wn1zlhbu029ungyzzrsr8o","380":"hh2d02oeojdiworiizi37m0fqli1p6xquzkss8jzmh0t8zkyf09tzm3uo4jxkjv790ybcwtne8ichjxvx9viwt6s1scdjg24qo23","381":"knerewlittk0je5mms30h22gcgoh2sy40rkqdhlekmh4urs4rm2z99jm9hocvsvclstkmjqs40cxj5dyc0jz8pgdyz4y1g0wjng4","382":"75e113l0j2l0x5rmm8u4u1ruuk83c1h3l4hh645snnp742ub665xy1grbm5m24qt7l743i5mjrbqnzv48wc2obl662vtvtl5c6h8","383":"i98m7q9v08tgi9606459sibb7qscp2ezxvxyg1p8vflopxerjrgqc7d085mnwi6pgxz89xv1wnbkk623kwz3z5iipoee1mwi2x03","384":"mvhs2arp6oznij66n0q5mu5675bolyzmfkzxybjusa9dbgqplrzpv2y3jdd9hzgol2xmfi9h4v6ra62c5cejbpcc7pa17kw07f8o","385":"mevxw2bghrlsnepi2fiqdk8xqi67hj52xvqmq0vce09e23mz13u4gsnabll71gqut4l8a05r9j03augxfz89a6rag6rgnb5d5ebb","386":"ak1rz2jyt03ds0i4a5xl447isccm97sq97ou2b42i8sw48kis6mfbi0qnxz39685vpum6d87ilsfyyjjm7gz1mgcgescwsc78707","387":"pxv6oslksusezhr0yy93jdlgnoe1v9azfa8k6wpi21gatcwo35nuw5m1dwggyuik1srvwj5jt0m2671y5ktbum13apg46okrncem","388":"93kcuwzfvlcefbrrmkxitva4oqx70bcsk1kamthyaxi4ix8crn99gv8hzzb9mzko3l9x8kgivb7syfhdk2p6kk9dah25vx2c4ft2","389":"b723kcyls7kcnqt8zykbofgchsjupvkgng7urh1dsa5csgwrnzm6pxdsvujemf7xugk9g0td4p2y1nz3sxmfi3jcufa57n2ivjel","390":"rv5uyrdv271ne2a6m1efytfpbc17b516wgujtbwvexpt2tcj25tkhmmtco2moz63wk4649ytuf4wmpnyrrl50riq8kqe328jofix","391":"t3ik1zh2sh4v3968wpy8w19o6bcoo6ldfvxsfg8iexicuiaoesbrqgvnsylz083kt4si5k2mu6dv23knpmy7axib9jh8wbaccshx","392":"e92dcc0irnmgnj0ioemsl9yxbp1rzo1jex9clyoys5f6xfh39kvljjqp0ofw4xplx1ske5kftsqve7cziodcsrfvyb4e99i21tkr","393":"fpox5lpifj361nh7goht9895ylqu6fy9l1tlsafjanttewgdx76sm9ul7nsli7cd9s2vtmf4k47mrqk4x1j4ebwfuy2mn6o3cyp0","394":"yuwqna2p58jx8s55ejlv90p4mjw22dhw2waw53z3q3v502j7rf63begcicm1x0od9rzvrwm7rrw7c682dmqutn2evru39u1c1z24","395":"mn2xlqohma3ui82gffymqbewptpw2onwy4h3ylr2tizyh813zq3lxv29fvd3bm2v1z57xkxiknh0sfkdbdw9qyp413jmgzmpbu7j","396":"qcx44m5r7ld7bx1v0rafw9s25ccvn0z92mt67zxh3wa5sar97do7wuhjt156f4evqy4d7pyw9qsrun9ocpjj6cz3nyqpa9vbmbgb","397":"2y19u5o6yhmy29qa2tb00q9yhzef1vc5nk64t3j7zszg2ajjtjqh6ikb7cupd7l2ys740by8wzt5g9x9u98hkmco0kk3x9rwbs9u","398":"ai2a3pcl6iwjyh2560zvyl5jntx5l4ynq6r2f733ory7ag2ov4cxdl6xn2jon6ibdiyhagagi7j8heqjunus6dvvowwc4jl08ch6","399":"idl6ibqfoydgbobeb4g6fa1dzr7j59yaatytd0auneej1xuss5s3n8ytj4rdtj5uknbfffs6ek12n8qr81rc60pd6swnezhiy1l0","400":"6fe08pe0qrvu149taltnzrgc3hubo8ewhqtn2mioib1rv8475ha7kdn7ew4zjshnqwxmichguqzd3xgawlt53mnyxt5sp6s3diod","401":"womx7pdqod21p1z0m00zg1mn0mlsk7toc0poask6cugh53mvmg6hifreyq1g9zuj5jp2pq8iqf1md6iumlk8p30z0rhro6oze109","402":"97dswr6wcpywdxb6t3st72efi1fev0rntjse7wefu3wtttgh75ljh96wonjw3jpeo1wsmtc5g23apa1db7e3kvgauzpgyhjyohyy","403":"s8p0kgm17d3n5vidbqar6cfzhp3r5omsyoev4o4c095xcq38rbrsi99j4bxzj3ha2vrt5ppvj84haaoui963g0lh4069oqr7bpjj","404":"av7zxfko8b6sgod8l20f4mlbbel0fon8it3kcb29yhoovt7bhdiwo6ocv2gyz85wnb09dr1d19vtfufqb5n664xsjigi204td8la","405":"s6mc55lx3bt77s5qd9yfp4cqt7zqvwk371d6rhomsbqijsh8wi4m7c78v93m3m3od6l1ms3u0102wu32w1i38xhwvdrrwt69l8fq","406":"6degdj1hl6ihhdmjuvkvrdnakv3qfnexm2dog32jyy2x7zim3wm4yowk4ahn2g8izly0l9p3c26hpvtitpre1qz0nzma8165ruv1","407":"bqrua5fwc5izezwtjog7da8ol9w667zukxoy1b964h7dfseayjgnefwktikx0ax9q4q0h3qqzxri5kus2y0p2x6w1s4vekbiqfcg","408":"nldfhruadwxz4d6yvwzq58g82c19pq4beb8gsox407rx4jehmw5evbocg0apox1qrzsijm56zwqhwxrigexryz2tm872ssdecede","409":"s31tivw3h43bi75ba0oiy7whhhagzkdfvdzvtiooyrbo5ii6mc9e5oz95oz6igmu57pj2mityjj349x0uu8pb7oi2ajt5ckpcxbl","410":"17wc0pwd2j9ca6oqpc8dsw5mega4sibkcotv6bn8o4ockob9q1vyfirx3iisfhvrou35yidr9l2wsyia3o27n6utfg79eel2dpq4","411":"yafxdx08077vtemttwuskzwrdhvxpfcpxvsv8q4110zl1srzmc0kqb7bvdoplpsuno4sihs707z23pwss45onqd22nrury1l5rev","412":"wutclbo8hphq1al36tf35bdrxs6awaru4colkbd64s4nxyj2ownjary44a9z6hlf8u88h704iiqsk92oxe7yyfvpyawf98phafdo","413":"7jtb3lituqgyyqsyo9tmxfwd8viqj87yqmvpqm9pikbg8r99a75xvwox26zfd0m2kx8g4kybxjq14ycp1nzxy17m6brnf79ms2lg","414":"cl4dssn6dzx2ykjjlm0ybv42gv3s3v6f09rpvt7l557l7dtd64um4xmuzm11qcmobbp7v6n62ehrvslsngzsv6vlwvmiheozglue","415":"89v8s0bi0pds992kn8anezr24scskvhqbtr3ru0mdlujzklflpss7mk1dwtqrwomivxnjyr8602jzdhyzs8bcbffjzo29vxpap18","416":"i3bpc878piadmpk3859bhci0c6q1cuta931zxq3fcxs5jwok6cthokjkqhoq2l8gcs0ix60ix6qmmecgk8m54y1b2smvi2f5h4qc","417":"bhaadegfnhfvt0cvkxy570dpyp9uehxpc1idartnfzab16ctrlap8vceodvifueyrj0inaaji45wsh3cp0k57vrdfbwckveupzz6","418":"5hyk2xdr4jwrq1m8gc7u7qeh7novty16wenv36rwtkpg2jidcz2dnmiro0jc1zbkie3p27u9zo4lc4fentnadjh42il3r47sserc","419":"d09ds0bbonp9e614mxz14wmqvunh5wbg935yvow4ku2xqw02jnabkny09fn1j9bi5i8ozj7kykx0z5c9r180rctu7cf74gh3fnkm","420":"uyjezu2rwo1239e1yddlb1lvac8cn378pfg6t2di7dpeabs91m21742gnkbu0pfqv1jtbjar94m71r3s2zzele7i1v4wfuwabdhv","421":"wj5s32xrqhoq0sng5abhjcmgp1ymmgpgwejg88jl2w2p1wng1ser1lyyxt5torf9g0nmgy8755bltkunjyzin6x1zg6w1thyn733","422":"es9vqlb2z8d6c8c29s0bxvfdmiya14a6p15ar0qpecmn11di1jbr7oqse1ag5saxkoqxbdbofwzx9zkken5kyk8ivetcn22sxwpt","423":"gzgfavessknzcpxr9eieer0ndaahdaqmi900s005dymqcfryf5ih4nkr8sk3hrmvdtytk3kpxmvmj9h9xr0vyr6btmq9s0ojgi9a","424":"5lz9l0sn2t2uep30hjk2utxf9xycyzjpo0xi9kfz2k1efswr1ahh883dumv8q6k09eeagslyq1gfkxwtw1ubybuc76o1wddz2cim","425":"mws2hut7tibotflvwiz68qa3fk7aeyaloyoy6srg67ntd1feuehr5wiu7swsxegssgp7vanzyjqajjvnxl0ugqp4wug9ktuj5bsw","426":"vsmtgf5fip4bsvmkoe66u1qeydnvdfqw8vjviixc9cg8cbl0d522o86s0cp1hdomubj3bxy7ntwvcou56f990gr91t282avuifve","427":"niorezcolbxa8drw3y6p64j9iuzg8e8kzkk6t7ywfditqamoailgshh6bbor39r1vy2oiyw594q04od4jsxe802kh7tyqwathnml","428":"80p3h96htd9tuanfrpmca0m40aa0d61s8ubx9c32khnf8qlk8imhtlslqhbm9xctq3iyihm749zcxfvhxcbuchtsgfb8fsjlyule","429":"bplqzpn3rvd3cob652mw7yp0rpbeb74dx5461ap9elht4ruwlzbbbsk9ll7vu6hd7l3x296gjxhr6g4dxa5nzffhf45idzbgil4o","430":"trgclw6vb9xnea1owh72k9xfz2vdb48qzjyi5ryd663uth7qa2u0ajyub4m9gh5i0mvj4qzv3gq7itqx6ohw4fupnv2dqqefekha","431":"aitz66h50dj52bumpqiy0u0w0x28vyc22ck8ae2gawe1hha0u7hknv1htn5cpnu2y5ylrfxnmgjjue05gki1uwjgdxl8hd9fl9g3","432":"p6kvb1pddhxd3h8b02fws3s1sej19vowj06key26om95jloui5yt38njzghdhnn81ykcym0xawozcr0vx4s6rtx00uxp2p7e1337","433":"2bl4x9p6u740q5aph8u5nv4qiz3ipwfnau6e1lwrr57ndq6b0twccxw9nj8eqr4f5bbkv3kfncv9n3accchqe5vdco2ezb0qxco0","434":"y6ruq7wgyly8tvo3rzub37ywsl3ob2f9wjgmlpdqk9p7s1ypojr8jaox4xjyw1ooqkygkcfwd0lf12mgwzlqcd7lp3ieths59xnc","435":"sfxzveu7tjxuioe2jg15p5l129z22itqbdustlbkw10se807urxjup9ywkvp2y12hpepoz5v55yw95ns97x9thqad61ktlfw0oh4","436":"6olg39m448a6n5k649hw4ebchh6i5dqgqtsff7a9aedt2iqnhon2e0w2rle97frnab0rffijoh97uq57faygy5i8fvjbuq2edjog","437":"wugdz52uqk3opki764jannermfy6q6ouza4evvhkw4e7lvsfus8nnkg4lism26rwy5o0f8m8491sfkyo83szw7zbdsweg0f5fmwa","438":"bx9h3s2uwdsljvzpvxptqbp71ob38oyjhtjnlc1vocf8tthtb13v2xcksfldzsgcddog34qxxmu91wfkwjk0q54sord8hxw1sfxg","439":"ihch5v5q2d12e0t33b8t9u5q6tqbpyq77zxyfz406bgklkcqq9b28w83p3loq62nt8ieesdghaghjfozxuve530jbrn2l24foitk","440":"62tky9o09dvhhw1jfrpdyburspevrj0q227qh3yrk9rlrsutp35qdkza891mk21c358bg7ycon2lewjsgy276nkqj20n8imcab3x","441":"mai54klkjzewlxp4w0yoo8zpebrkwcy3rakpzvtn6lits1zf3mvnvyu7tmgqloztn8oypo0qpaighuyz7q0bfni4ofhj1xrm1nyu","442":"c4lnzl41n2ctoegft4yl3bpbxtm7nk8vwrxj4fi3tcn77uhqre1r3qlze6ygx3m994x2z2qwez6jreqrh6b1li4jfba235lp5dvr","443":"aenhju3o8oc8fcry0qun7iokth1uko3fekz8v7h8gh6t64lpz8gyzchcvvlmqtrjfpmnmzpetny15vr9d8nsunc0qypuea6wr4dp","444":"3oplrkm2wmqzchcl66gp5rnmjsiqgjzh62i044syw9j05jjrzw9zzp00k73ggzw781btsc1jvgct2jcy8eb4l0ivhmoe2jzepx33","445":"hqtr4gmi6j00gb2luu7kq9x2j9ck73iw39w0sqfzmvalupc4mhiz8om1a57rojn12sz2niuoqbacbvxzqufxobrki8f4y6u3opc2","446":"qd0xvcjvxxvr60mu0ezsbrf9u9998ich242zlwt2621qanqarll6cn3v8i9hh8uxdcw9g4e0kgpdzx3ul058uas75j7ho45w3he2","447":"c6cgf79x0qss5hegby8io1jvipobga6rmp2bt731qnkbpd41jduhfzik181txf49n8r57tws9r25zvr64nps7l3i8nimdcu4qauc","448":"rn93xyppbrkb1lcp7ztk97imstt00qluapk9jn2pjbrrwvrfadna6hl6klpodmgggdj1iq7eq06lt155oksbsr2ytfoefg7zehq7","449":"hir70v1xmq5thhg2sapog9a3tvgn1orc1n618zb4eatqkl8xum0t4jmlhtnejlgi8dtcw7kfwpm14oe9cbsmaia0cljxsltzmm28","450":"54h0wd2grbbmi4433yz524ft6c8qo2yu6w2trwlej80omimk38l6yofwbmlkin6cmz7pdzrzrbca92m3qcvlzimftqqe7hj1bgwz","451":"akixve5q5rjjar7mcpem8ng8oshgkzaeaxsg3vgwdugpdh3b3g82vaf2y1t0gg8oy126mfp6pho8k7ic67e55rieshfl57i3qhvs","452":"aibjtqxl7m2baib3wp5bmclfmaoydvbndnqyon48e79ppm39tdcbgtwrga5m7kg34kqwe5h3v6vkhdj3um8jpkw0vbll1i2vq19l","453":"sb9k3qp989ej1q0pc306pwpy6vz7yypkv5b5uj4e9nomdojw0e1tgtb3dpvzywr1m0ej8x5fmegjlqqlcuynq0650fjtlsbfkpoc","454":"trf4hzqsykypes7ipxzl6xrlwre9xn2xesb0juzbu9pnh34264vayonbuql21ortyqeomqjh5z6hzze2eu831asbuut7op3p29k5","455":"973epdsmbtbk3ebdd6vse4znvnsd4u3jp3ti96umnhp5y52e5tt533kvypt4zbii6fqay54kmuhn5q0n66yf6o7zwybs94i2350f","456":"tkm0wc4l73e7v15i84bkxg2t1tjzlahspf91dq0dz42tzb4ymfjozefzb6tz19tash8hog49s6l8rk1vsn2wjjt37rd4kp67dkkg","457":"xibmh6bgqlryyorzoto1ta1v1vtmoog968ykbheqj2h9orgng8ngs9uydi5ms9n7ejzzvzobpei8ek05s2vvzhjxc0vbya3yko5g","458":"6genn2eo919lgv69cm2vcfey11yudsxp57abdycl29o2xb0crfuoz3q49r6zodyn958cjkcghhuyx2qt82q2pjiyygt0ocqy1ob3","459":"2h4ddskqpv4gb96oi0l92y3kkeg2mtgjxw6uoqixd1p4xx3cmo56hph80c0pt4fsqtysntmb1trbu30gz1yq84e1ekresmfnan4k","460":"8tc8wjy68u0pzpth57n69hfbso3bj7u2z1k51cx04kupthuqgdlccu0ppq9rgndlwxyb356ocb3liap3ng4hbz738dsozx6vbtqv","461":"gryhntub2e5o8mt8os6u35y46phkke2up53x9cqug1qzqwco04a1y1v7mjdu1imo8gaebp9f9yul6kv56h4n18gwbz16yss16pfw","462":"q9umil81mkp9xkdhxnulwvalt28b460hbe1rbzimmylq491kskgc02o9k1knn404alnod4hpjzagbrvvsz1ix2kq7ch1ar9n9de9","463":"yye7egsfnczqbb79r223ug8gh40cwwrv1oopeqglp042be261yrc2rgkud4erhtarz2usxzjb0ha15u0h52j00aygf7x5mrjpvvf","464":"86tsvwlujqwz44d351t6qxbv0uxd3x7r3jvaqr67qk6rym5r1zq4lr7jtdz34ehe3khrzszzm20cdfpnm2vyo48miq5951z3kw5r","465":"bfk9xz6ici053ckvlt1w03srkm8gi8k9i0p0jhqz2g280xa4ltlb2tnbfgngsaf0u9jm92wqwyduxbmdrg0vphlbmtxdiw57sdll","466":"m2pjllsibwwsc94rjp73fwrywoq0r338nyowztc4943nakarohrwrx47epzsxwsmrnunsr171xtsss5yx0gp98aurf2p4t9op1gc","467":"1k2zs51jb23o9i9crg0wm46y8603vsyrp1vmyah6dpgvuwk2rtnewmv1teq473mm2nggag65bb3i86sno7ep3uzf3d925ymirq6j","468":"0sahijqerkbqszjkkt394kyewltma8sf3y0rd46mct9h99zzh8tgiyxc98cg1y6n637qrhinaul0ncl6epg1x8ruf45xwwdmmtq4","469":"utwixxmp049np2twi77srciscmd0ir5yrlm4n6v9lnbvq0ei1cdixpide31gidaidglx5gvd50gwnyypt3tfneb7wenqurxmiamr","470":"ylatmsnb6zskkemsqwupm9hhmpxwqtxjn89mobjimjs5z97okm0tvm9j220q390jb2o9o8oo2w3ay1qbvu6qplue0tayokdmcsqf","471":"uexfrh9yu2m6umfhx17n21q2ns15fphlmxjn2z8tg4xul5t6pskao2kwezmhpbkayvtky31hll4ya8ckakkyh0q7j9c5od8ha0kc","472":"koye9ucw4rade2mvoxzt6u6h7kc26y2dzbfotd9u8f263n7mpfafm9h5qsnosgj5z9rodrdy9ha23d1vx7p03vpdp35q6kxg9gf1","473":"gkv1qcvo81skxjwz8t6wpn4un7ebjcaq9zof3b17iimtrp2dtv7ujxvxu65stvis5x1cvioc39y958y42tbp95lvjiu3sl516i4g","474":"9a7g78t79us4matvg93drvqw82mx11dp6tzitlgx5nxac8kvvmv1ekdrfbkrkew5r7kddelmb95zl55o6s2pwgbcsm0hgb73kx0s","475":"mghwmiymsi2r90cjb2nneq4tmxrnamv1w30ccl1s3bt1hs6uxolgvgfe6p1zmuvlnjtwhbi9hx2mrg3joxz32qww86c488d7ybap","476":"fx4g6eze4at1l492o3e2qn6c88bynd4g2lnpeigrehapufsrilso8tx5wvb0fg2d2sp1g4hksj59tv5u6fanmk6ys5u559z8pb3l","477":"rt6o4ns2cu29vvx4yci635xzht4pashu3pjhw081nkvlu40kb13aextm31t8zhg270loopdmtpcls2eu3qksilyo34prx9fweyhz","478":"61b7pqgyv93g977l6icver0ffy9y126gwqpc7rfbm9z7zbdku2ppp1jlt2sylmc3p45b3wlosgmq4mxm5ugg4xr6e8zl1q93qxwj","479":"64yybusbxxba58ewlr69g4vn4opmxplw3d15eawj4bdr5z7k31vtmny3lbaohwunpom6nivpdn9nnuooxvvhgb4vmk0pot81j44u","480":"600qxie3w145qt4pve93irbnpz8zgdlqovhqhwa06uby2yh1pcic4bx0zcctzhnl8tl87l76f95ni08lcq8pg2h9cdmuw7r0yvaw","481":"fej8lfarvbkg4u6xyxyzjkmuvpg0ae4o6md5jpkkmkfp2vpvvoyi6p5svkjata8g1hz3vgc3v2x1uw61oqquubxv7mzst8u6iw9n","482":"vlik553sae88ag75xk62nv9zbf9hmru7jigopin0y21ovoe7t6irckh06uzhb0fbphikiou9pcs935uosttxnjviag9orq7pqij3","483":"uzaztoovytaakc500769jedzh1hf9ron1iqqxmq73jll86mumnwa8c9nb13eumalf4n37k6ya9jftlmekx15fzoec8sn2l92yf8r","484":"s1cq2zg0pk55kq6ls8wfjq7c5o6u87gj60a3fbgtobog6o2c8ne97z3tkkpznsu2kcg2hb8ko77fszgw5notk5j3ymbnse8o62us","485":"np4v4u5i6ghgl83vvh7rnq2ud3azvgtjjqgw5w4a85t6kuzm1x1p4modg4xucixa31cikwtvd0uhkkor2j9cleuqtc3ggyt4lpcx","486":"oc20v7fow0i7t8d9iohxbk6pc9fyf4nin9q1l44xzepf6thqn5p8n8qy5o26zbzqdmdk9k7je871hf06gcqwu7hs6k3ivxai08d2","487":"tqwxacukfxjw9vea89m5tmf47tfy0cpiwz2jjch8ajt5kq50gyyk9udwblpjgnjnyy1vv5rlczc7ysw4lln1l95m05c6cu5mktkm","488":"813js17sms7mlrcxfb9mltwxa9h59uhsgdczb5ds4amdx5gyjlwxhamdxdmkjhwdnvalhjvts5bolgi4k6u0r25ydprtl94bpg9f","489":"dtevbmvx2f75tl1yge10ugnkbwkxgostrme5o9856uwgqo5nxhkujr7jjdhdw76040i5lrbpfqvk54y16semxoufee3djp7jbiwh","490":"9h9en0dni2vxg65s4n1wfuhytbxulnj3zjhc2566cft5td0gykkncr4popci1t8s8swu08msx6xzyhyin06mnsqkqwbz99rm1sju","491":"4i5sbnfkm0b0j4i1pamlndg4dowlmvfeunfddpeurpzinx4mweiiklxhyng803cb151ru9piq0shwrqidcimdt4rnbp6oo0oy7kh","492":"e2l3ws3qvag8skkw9hh1cr1idg5wkffffmfcrk78ce3yv0xlfabuxlszwfviuvyyj7o22t933vxvt07ddnmf5366rr4zmnmwlcoi","493":"w65leyn32wyxjck76ewui1dzku5xovec3r306sgzzazwluxx2346rou4832ja2k6z50zid76yl2haq135q442bewt93hwrdreeht","494":"xl3mfoq7rb4wc4xa8jt01u1atl7nt6d26fjkjop3pb0g1fc9bv28l3idj77315hbx5fb20645aeeo70rnsyti9p4fey3oascpkzg","495":"awfp9om6qdzf2g07k433eb9jvswo1weq56nc0lmzea3es46bccqmab7h3lcvo3aad183s8g7ggcyenpcsfqp7t7l469kq0rvon7t","496":"qf7yonw7boeeqci12cm8j8oh7nx0nz36178plw0n33xeogezc8f0qmrcpkalcto0y2ik620o14bep4twygbcfpgsmc1r0eu6eja6","497":"43zt0dkrpiakrs3gufel8ga22jaucff7h37yrwhtz2z03widgr24oesdhb1ljgr7vy79tfhi0hch88nmjs7t4csc00y0ibe289q2","498":"e0b9n8yxe7tnb9jhhvbfi1znbrb10xp349oj8p40580buut13fe4xsyxrqxu9phpajglvjcn3w1z9435swdf0gwl3gn9aqg2oovr","499":"ufymtqpdfwemfupskqufgclvimibq0hjqgf1uiv528xldskr83ncnpza0zfxzj3pcq7yg1qyqoxst1n4rztf4jo0s06l6j591twb","500":"j9ks7rrhmu288fvsgd5gkddhdch02yrd2o6mxmt9ydhce1ek92ym1njtwb3z5b0sybwob3nucueshoo7di615zxiyv2m5fdmkbos","501":"fe5c41z8t89ozvl2tjk2i8whcfkvls7ncu8msi4ccsevtkdebp2un9xncl3l1zjfv78gwgysgegf6oy3tuv56ilztkg93igndq2a","502":"8mdipv0sd055najbc8o7hippk8lspsey7y935419jyoy9zqxyz8iuxbxeqt1qv0pnfkj4f3zplaefl3svw9yay0kofldmgr7d6y0","503":"tgmr1o2gkeq19l22i1dg8gvn8w6em6stvlghz2rfh49tylq20ikftbqo1so7hug83ppapf2qbfl63m1hk8x9bx7pywwka9t2d4qc","504":"dk6argmtkr5ye2c0qtq3y938ptzv09vur5i3zenlio44txxvlporo141zzc2w4p79ri17gb0da707pc5lbkgw4psdayklaqv602k","505":"8zb2ev4tyo876uljjyafa6sy0i92geiydhsbk1xuzuz6yqvwuw6m2ie04mvofyqj7m8ydnvw5s38tndrd8fo3nyp449ob84u1mos","506":"jppoeghrzgck0qf12k36vbpg9di1fbzri87j9hezt9w1827lruecee5rillthxat62p26zhqz3dzcg0kr8w1uceuh7bl7kb4xo3i","507":"14w2n2jo8r9w84zxopmadzqx3x77jf8a1crdllvn1iyiyce131m2pqopv9ueymrekz7qc7dsp3rf22nuijaphlesu50x0ly4kjiy","508":"bndtxnload9a8a2l6krv8g8dv5zou88arsi3a3pd2ujhvjbujzcifp55i0cr8i61zzdxanbx7p8nzay94lofatvnm1csku2511xm","509":"q7qhm1qcemomt00kfrgy104r9exp8xphdp9g2jdk0t5t20606ovlcxfrgioslbpgnrku7colv9eir1cxvb7kdkd5pqy72ejvx4xx","510":"5rhhrvac3dp1hk4xnqz7yn7y6uk6lp5q2fttswp66izquqmqu9cou1egcttp0piczeel43uu5sak6lf7v87ju89uzijbrv88ko2z","511":"golevrg7x618lnrd95eh7b104j7b07dzpjxr7ce0m6twprdcelhn5zehe0atvi1k0vwfus67ifrboairhjf74il2kbj3fmvsmp5n","512":"muufm657k90zo60sxpdnsos7bvxdn0jqguuq414jv3ohdng7ztn53fd010uqxeiwraercpy5ed4s746m9pp2m5x7l926ugxvqg2o","513":"dohhv0xi3pp4v6zcm6z1ik13bxjosd6tvpqo53mszz4ylzvce26umn4p4de85gr13ry1b49nof51v0qy847f628zclrvamzfzwx6","514":"zon72ug0k30iq317vn0lo2wqeslv367pq3rrmav682gzv7u3734aiq3oq8r8vlnf1mf84tk5322hfnpmsg3pq387blc20r8x2xd4","515":"r3x8kibf7zhi3o5edo0ck664ddzl8xjmthoqbqcd4rhpzgp58vdjk7gg3l0hw31rc3psdm2sqjthuf19gg4e8xgfjbg3h566wlnb","516":"gjhn5lloej02p0jf0a18t505j7xxr0s08tyyke5pr75yyxcldwacdvumczzvqw2m7q620pgxjtwicuo43c4rzprklq4gziqvvrpp","517":"anjwzd51pjg9iwkjs5zrnrpr8msfnn75y4c1b4n6bq52qj58sgjq71i24jnhwaxvvnbd9icvw70832sjra4sd3fs6is9isokqhc5","518":"h9ncqmk3pnaelifyjqlzq3m6g7cvpq0dydhpowj5i3vkoktu6kl0cj1w6wq0nqttaqv6yqysb43ffyfmjkub7fmjd59h3izb2kbn","519":"6ec656xgdsrn8e4eaxlpc6fjx2eebb6st47c7mvqq49w3sukjsn6dynw7cvwi3v44u3099zjdf6yswwt1iaowyymawzywe81ou01","520":"0ydbii1gt52iv7txy667s48c3jh3sqni8bax8g2aux4dl2nefogjms73v9p0v71lgl17p4k9lv523oci67pdr17km3accodrwgr9","521":"rtz9dclarg4x1r6evb1kha3c0klujekv71sjibblj1dzko73e3eue5zexbnhozyuw9k4r6s3fk9ojpdrli5gkszcqu59qkv53xx3","522":"2zyvdbycp53ha5dqq648zcznhornutfa0vkq110mw06bm3p9fwbdiiqtudxqr5cij17meoxjnj9a91wf4j1coo61dmet3bfjhgev","523":"zbwoy607nxpr4da64f3v41u4jcvhnpfhd0u19h82ml3l5fk35f4srxwdp4we34bui1m0ejgxjqc6ib0nuu4g87qpha3gjx3goqh3","524":"0hja42ox9idw2brll15w7324jxyegu9mkndsa7ix874ob2nthg6nksrfrchnval7twpv39tmmurzstva3lscdo66d3c4qayots8i","525":"tultofg7zc5xdj19yc1o5bdnyckkqj2pnxpx8g7qrx9bbcj7cy6x191yu43c3jc28nx6bl4twin5n0at05nvismsw6ifjoom1aar","526":"m8q6o1rilae4xashshef6w6y0r1fj8ctn9rto63cuu8dfezmy9b0m9y6b6azxywgt24y3jnpskdu04cnanx5jcig7x3g31435dr9","527":"cko8zmpgi5ac5pqy6yvb4k34xsnkg55l4r6l1y0cnk8n01mzd1qz3tmeuc6zfc55vqsu6g00ian0mzvfr4psmb4qu61tbacnlu05","528":"x9c1068f62r4uj7fyjiq42e8bc52wc4mixj3bjhja2v078f1mzqvihjtpip3fco1ovmyfe654vqffd6k3aui2iqsnx146zmz7fkk","529":"2v8al31dgezvhvk4so1ws9wjrykljw41imasy3g30jeof6cxw4nuhuly8v0v3qkdzjqirhum4d78wqpijl9xo1wxy6v5kgux4od7","530":"1xx03mbz9rv6tp7uschw9oa5b2vdnrzzz9utf29jsvk5awkd9mi498pqur4k91l6ewhky0r2at4iw80l2v28do6171tlk8oq50ed","531":"5vl1kydxvcl48wmdn8v790kxtt4gwuxs6emuvhojfodvfnodgb3tvqjj32prz4fo6804umzbx3u3okrt9lqctj3lbmuo9ggc6ag1","532":"stii8pmklxzec7vrnx4wmzdyvmwst58z2ft9vdnshx7hd1v0275hr6eermtvusicav662chlgtvxdc8w83uyzu0je555r5jzuip9","533":"h06r7jdvo6fwnlh9sj0m8r8sp19usttfu2z5albizrh9v96fnv2mnh1ishq9x1frtth41nmlt3vgs96vsvhu5p9bdatz0gmasilk","534":"ttmvht215eqfn84ye2jz849rub0zclfgqdxdlioarriddpboiechng2ltln0nsjwdpvuy3550guhaldbwdagqc0k2j89r9ms5nz3","535":"jomunzugrs3yo0182ajvukmhq20g1tjilcsmlao91o3ryrs1ilfmgi927ayedicpodmwn0m36hsjlczjeg45qizrq0gi7azznmds","536":"bmopm5purphawkwzbfhgzob82mg8j48lnt863vq3s4wcdl0ep7v5zrma7oi7dxvveqy4jucc9rs3ba6j1swf706khnp53na8ye2g","537":"919ck84tjbq2na4hzuqva7ko2zybh3j7sz4s0btipoqnt8ovu6cn9ksd78b72bfla7ai86ifonm322nduujzyh5qk54la8qgdtr6","538":"twdhor5s84p7s4pmfx9i9kieaumk7fixmvhfb7l9qy11bf9bo9dwqzb5wxdsjm5kurxl411m5qxovt83lg8jqlngjb7orvdo2zv9","539":"73mol29jn7vvx2059rnel458qo4beyfj2onwl3w20zi1q874gj15lka5a3wqn0ldk6gr3iuj7w9bs3e45uyysplmq6yjdtzuumu2","540":"vrfa4sgx47mblogidi7ai8wwjwrct6w5bfaogdy05rvhk3x11dla4sxjei80uytvqn1c8lrea1mcwx32r38o150arn8anhqh2urn","541":"3arlzgyrlj2rx1a8uzhgr1jqfr45upqo2wwgpt9an2yltsu65efyqpjhryya9n7h1tlcugppja25uhw0ymf75yronlf3nowybr3a","542":"k86g9g4fsjotesaynhexny4r8kfmjw21q13ac4phg942bxs0tjhlcynp5hmmkq0es36o5fg4crrvof0pth8b9ioyiqxbsdx5388d","543":"r96sokibjy8v7ucs0zq0rjayqo0hxt2bs44jpztbqz69ys99agj3qesnypncmzae70c6w7mz3nv9hts1nabxrl7hndpqcv5qjlsh","544":"rb94hi4fw7vvfoufa8yniomtli878vs56muluzzpuqrk2xtt1iqjx5a1me5b6sk2onx25ttt611sr1q65izzuuvbqaaok1ag4j7n","545":"0cqkfiq3t342h8x6mmc0e64qxvdxz92iod0n05omjtekgm6fhk4ldg2wwh1bjyuglsci29bmuizyemuz8df2vdqpn2uljm7gmo28","546":"3v99vb69shqdffe4u2fjqny1lna6yphijlx2et37bmpw3zz25z327rj45mvl5aelbe2f42w5izbs2o1tadw6usqs96jeydve8vdm","547":"7fq6j6ay2kideoz7vqrp6x1hss5prb84txxxeubf0wswlmzkk1iac3maun2n8shfqz8vdzi3u3zbw58w88q2rk0awwhkz4hd6h9o","548":"o1yghmo44orog7yq75oe6om3ebqpw88p2t8amm5mcgtjs9eqj2a2mkd0puasoql2j1uwlqu6klh9v7tsg96r0h0mukb3gqdqur15","549":"329ru3bk83s3hlzgdinup6trqqbbvjsout7v5p2t4ziyhcob4d9fs0x11o09ox040xcbchj4egn3fsuk46tt75atd8nes5p9kika","550":"eo7dri6scqdo0z2agr19m8b358pss71nhoavt57a1o27oko4eqlcc80zqiy7o2q3mt049l0olmex4dto6lxis17khgs8npi8mvuu","551":"arm486cp0s286ekwc0kbpgcug7mfx36sr9f7twq8fy290qpwc3vfaq94x9f2na95pki5zzc907dk7c6skwrlhlc9wp8hjchkvpxo","552":"0z34hfgfjmaf655ptal5cdwmo76ybjrpg90zcvo4pe3qt6vpcegvredqulebxha08z70qec0q2gl6kpquaezj9b06in07iwh8zkc","553":"udll8yf85k9qgakhddvi8u5mjfdj8jo00d4nyxcrgiej9fjyhnhid5fjy0ny7q0t6jrsta92z3maxg2hk8l74jueujobexinpu5o","554":"qac2iqc8vkhthpt1q1r6anst42vtvrtrwmx5lhr7god2jqv8qyz5xcj8qy5dlq4ctr40rrk54hrzmhj4qoei28d2nuteu924jzem","555":"dlke37py65740x2iv774zvxwev4w4ynnz3ycf3ljgpzuvvbzl3zjfbd2j1th6sy5axq792vwqm37o268bd2qij90tzf1p5l2gze3","556":"u2it1zm5al63kq53td9jm5m8pk572ycmi6gv5uj2g1tt0ynla6au4omescro4zfa8ex5buwz8uw9dhofhp09xjo7c4snxay3x128","557":"4rtaaxk9cefpg5jqdahmc2ki9sbkfvnx0qv6yldjg7dwb70ar8u0xeohv5whe25tdnc68odzkqbfi43e93ulett72clmqnbdy4n5","558":"0hicz2mn0qo9may5qjfxc7xi96fbyk38glxb6c0b6ryll9uw62h5edr274o4rq2a8sg6wnk2ssd4oh6xu5429flphjhis3fxi8py","559":"307ucsn6tdy5jg783pedaf6e7j51vrixan824kpx0pgcozuq4hi5k69y466rrb75h1wcl54m8u5cdqxgicbe201i7cy59hpfp1ha","560":"z0s2w84pmr8g51t92h8dcl2vubmd4a0ntt9cwq8jqgz651fd15tne1hf84su3jyvsaxccj95b132v1nqiq6lfu93pgh3qq01288z","561":"8z1gq8wc9oxf4eokursuvttm8cowzvzpwo5xfxol7pguhj8qnhnjzw3z4cukhpo0rhjwldj783dbbo0jbl4c7vbcecwhpnwizx1a","562":"nsawtoguaadabxksyoqry2ux9nywtbahagvqr5uxum1l12ph0fb1dcu5mqq5ukmalmclsgfdsxlkjztnk6z6knxq5gf6scyxjm7i","563":"tfq9b65abptub6gg5jyu43b8wkjp7ca5c580802wuw6x348wvlvp73m5yd8uyqhun0unbhlr332a4bx0qnye5163dq86sirbokfh","564":"ryembkx50dwpzotuxwxexct0rgzhe28s87r2c6ujotdjs3s7119pd6y10n1vfdi631gs74zkt7xw7hksfy396xs0ot2tdrgfqauv","565":"1r2esqcz3gjmonky5nfe6owbilyn3zgbnaiu9q2q89mbr3t0rtc45gv9jtaph4t8dzqu8g1xegz4i8h0u59w5bekbt9qa6bm7mys","566":"3q430c99syavh9gm4lk7qr9j81p5vyfljy75sthkf4dv2g2suxo6ix4g9as6tebclvq5nsv94sf48jryd04owm8t965laafzh4ik","567":"30lrvku7w4c50mvv93ij5s1sp0fckcjfbg1782v3yvjp3fdpym16q005buoh4zqzh70gtif71v5od6q3oo9vibbmy7m6ak6d98lh","568":"es8okisw5v1xukmxdmj7g3pd5j6ozdk8cgsd4wqkawiymc9v6gogmiviaspugbg6z4fu82thsud7tvron0b26lna5y2dbh9sjrfi","569":"cbyhhacg6i70zkjbvg1xl2imp7jtz58d8ogm7k7k4qgqecjguvz259wsgq87dw6aa02d1mi7g5qspanewymuwzk5yh8y7luye23g","570":"bljbw99mzq7nvt8cj84sb1m89rlelstz8h2m0dbtu6iq3yckbkslysewpr4wgpxa8ybzo272loz028mnyazilzd1hjdt2kz9cg3k","571":"i5i6u18f1rm0ht7nzo522d2nbvpnhb9qxb1nvng9og0b01ddiu9hnpb1gfm63tfuvxcp8a05h1m3k3cp4uk2os8vmreowwffjjir","572":"5ik8152m4htuqfjkhxyksmhrtllyz721k39cwnc0uwa7acuu6vwat2wfv39l4smuo3o3plsqcyyhd8hyvszzfro3a6wtauk1debv","573":"ad5wfsx3z3t9yfwzidekb52icrfju5eizkn5moxnroma47zhat9c3b2t9ze9qu5nkoaymxbv9xsgmdwak13k1c4slfk3upl9y0f3","574":"7m80y8rkmc0ezizdyqdh83kixriuk5duawkuifjvarm9xmdmxucl3tu91quaf5ascnj5zt5sgajfwoj5j7bhtxnqsuyrdnyg42yb","575":"g457cbb5nvgclogzvb7zewov5w0o9sx3ceojw7arsj2hcspt1j4kulfp6q46o0tnb9f97yuerik428ndgm5b08dmac10qty2p8qm","576":"vxq9e7uj5g1lqazi0e2wqnmidv031cduz5thacgutvb63ylatz92xktkn6v7t3041cqvi80wpr90dtli3jb3bwzhj3148w0mizki","577":"wdk8axflbh9q6y7hvbw5k2iau4yplz9yqlnofv4p613ah577wa83sccb87kjyydwihvawkcb4t20n2cvhmd11uu5kun0fqa1hzri","578":"nhhgxx10mhl7zbonsshj0gnvcka8ameotlpk97wpjtkehoajookg12c6lov8h838k0ethenn9fgxigy9u3nm5a0vzgl54cnp8ft7","579":"pb8ziyl3s3n2s4160tem2yw7vziwi8u1dt9u9nph5u4aihg9bkge48lh79es4taq9sdzymzq8zphso8u149rpmmdqjleykqtk311","580":"s67h6b2226kffticji2a42wbj0zd57c8gromyg8rid0kc2jqdxlq3ri4uscxsrbi5xmr26h4h8q6xurfqckwd7a2seb42tcj81x2","581":"vuyo5088uh4czhi2y4hnztcspu7rmf4vqpqjrpadoi7pfcguhf32afmmv0pfdo2orgkkkqmiahvip321p1xzxedcq031k2fijngw","582":"9tbm3wi1pb1pbjr9ze31f9l076u7ktrhkofvnt2zfvz4k2sov2ocwzgqrwcd0964fjp3t7rpc29jhez1h8jab9e1l45neol7unzt","583":"kfaxi60xf8v8tjxvjqbm0wvxquub7hppq9z0dwmofed7al1vhckum53aa44hfvj1lz9ufqupcjk6a05tntfnmx5ni8bvid4a7ek2","584":"3j99cevtenbtrrjwhp9jcwn7hky1nfkufoa846dkvyr19r7rgxlo82jp3f32xzmza62egirnvnxv65vtb1nw3k1fivhmmuo683zn","585":"lqkdt05ni89exc9kpvlfsyh556cqlc3om0znx30yyt4b7lr4pyua0couw3uyd2mriwy9esombjhx0p7tplguv56n2k00d2usx9uy","586":"t5ojv54lol8evfz8bmzdunfjiz136hdqj2mg1x6oofu6i2zfm6aneijknd6cixhlmrt3hmasw96s8fa5lupky8lp0ckoc5mecb2d","587":"1ucva79elx45xmskwwfe3gg9bycmdtrb6mvuujzgu6ay6rten5ydvrrohuc8lj35o7tqysrsli7olmi7yg1comd3zidurs7r4qhp","588":"jmvu46j18k1vkx5kak90mhyk7pc2yhtw4vi61vcbz6vhaj053tj5ytu4jg3ltobh515ws6pjgglcshxcyz1ht6apynwo3sehs2tm","589":"df3r1j2e4bru8erj5pijyk9kx7kzvphw8mbkajbuejao8otpc52p5h139rje67wo6qlbyxw8mfd7sua645g1f3c2n6jfyejxgrp4","590":"t7yyzvs0byrtocvj4j9v19ughdnbtm2mh1frsaucwelxld1lexh50l2nrqnayaru4wcppg7g0sn186fwxz9ja40ftijwjct57psr","591":"cjqxdkn0vys64gbp8w0ru2j88m3mr05cxneru8fiqd2cukfusmpft498jn53f9xw6jvcwzjbn9mryvvvlw208vc80048pxj02uz4","592":"d15xzrnedsztz7bdocxsbuc80ffz2qxuoiaol59s4crruis9gvzdffgrldrerqkhppoadb7123kles099ccrdulhokwfvyvdysfz","593":"24fkkx16grl8y71uof0m09064hb577uto6ai3ubnxh4aq6ah8a1rr5s3v9w3fpnfu16aifsi3y7n3re7hrta9m5pb98xvbnobmv3","594":"81g2m74f2wkvmi55wvkfnzjj69c3kukqu6nhmvtigkhtzvumwz6jurdcogdzplizx4ym1isidoykrc6mlj9d2xc4i6gyocajtdow","595":"zkc14oey5zks7xjm16ha2pvvf0ufodgh3ar6n15a3brwz8g61nhivzis1e6kgh9jkjykyjm08ob1reurn4xfmy8ftq7wwlnr8wcr","596":"25uxl49ikla5ne1k1g1ge7zvr4xrev8s716702f72s9vid3m6uyhejsoch8cjm5fg6lom32u8yuq3iyu6u6oq8yngz0xeyj4x242","597":"orcpctxa7vep9dib6chsih4cj45tdt1u0u24mn8kwnd72zxno5c8auttvkpdahewm8c744h3cszd5mc8mljpuxo8rkr1l7wht9f2","598":"3dypzbkaw8g6wczjdd2cqi2tp22kd05etoa7wqpnk6nhj66h9wdp368mi51kyqk9af7xmr0whmaasailajuz5kdjmgaut0cic9dx","599":"yn77zo7fbys1h7ubyyyut5fmtucoiramymp9dwhd29xnilq2azk1c8rfalehsedinaydl9mc1r711pzu8bq4nlds7xsk7ogl0of7","600":"b2lr8j5jwu4z6xex7pgu89vrkrn0b3gn2nvp5t334d4ywte5cxztxrloxdz2kbmyl6mjyjn6jx5wjd9dfbsmi258gklv2if00rjj","601":"640c9b026sksik8sqq56teo5u5b01pv2vt1e9ovg610milc9yya7wo7vi5siqzhyarwf6h4cstev417nf7b4ybb6izo0dserp0k0","602":"9a33guewt05rsq0y2jbcm045f7hfjsd8clykf4wgsf3gamsw22v8dhieg4397ezo3efqziqxvckvi4ejsueao3m4ppc3tlhbwvn6","603":"9se6ym6l57l8x3scl6za8frz3bkwdcp4be4llb5t7u1ab3pnqoggiqor3uwg9g66zo5el1stwhp0f1cd9455zsrcugczzni46jcm","604":"obtwm5kgicg1snerg4ltgqsazv1w147bqey6b08v6b7bc7e0o58tioiaculhoje7efz5qsg7d78y42c30t07rbvo6qan2jfqii2g","605":"hru7be7oc9txcctlpn6h8as315othdv45yviegxpjctfv2c3j91n74bkkfdhab3af1pyf47o9l1hj9rmm4pf19gydv5731ru0ryf","606":"cijw3grcb45k2s2lr4e5a6kslqtb3ayjx3x71p5kfgfmmy4jxny0gvioa8eabsq8e9kk99zqpmeep0svsaxfc317as5nd5lxx0fx","607":"5bme54hwonmslq7nmw3uyj4mvvcx11o486jf3bu6s10i0h3iw7fj4yec0kvv7ndebwqo23d9kz95l5p2gv63z7n1j9vhpk1on0tv","608":"nq4477hrlriuuk82d9r6csb75y91h4yjrfcs99mrkpge43az9f6fzuc2j9shm152fwoo5tn38unu8kmmlox2kav9f38jtfvdunrf","609":"11lfjf85mfu324mzi37kk7fj4obhecoa34fdiki0ygrldni2soojg33rqdwfg7ob0iar6d98llm0tkii9ze18mg7bsgle9ewtqlr","610":"jknv7sxlug03sxfzsx94d55xdvu21kax9vgouyq7jcm7sxmtcdcxskyrm3qryztwc85vdsxjukdfx4ow6plt6v9t2zswgvg0a6nq","611":"uzwcpav2830vejqr3p7pg6w918oxeqo9db7utfkh3daq9xs4x9e0trwiy6m3oucsevtnxgbwu4uwg6kdi2hon57weixmnd50yzhw","612":"n1obb1aqus0gnc9dorkezpuhots0m6tjo8jlr1dxbpnjhqhfk0n2u5ym9akvopm9z9cpxe66icpuhyqp3hwtvtr8i8bitlv6pz6e","613":"n4jz8lleek623cy42aapej5v1vxgxt2t7pdqkmeoiqb8gpuuib53sf9mpfwhjvbzu06kb4z49ioprijka4k86okxpbvpu1mlkfzp","614":"y72drws0dgvefbvxn3lvlmpgdsg0qew08hyracbhiy7edbphf730h633rp9l61tcxowz0r8w1nd9yrpnfoappm7e6sgl4zdcj6ho","615":"0moz3ma80euu79idw0h7si22quu4mulo2snb0wokkphxnbjpmhp4ce77cjxbpo4j0jo5pcou6j8l76qol3gy1i7bz314b1ly5rh7","616":"y9az99n9wdyvyqs15yjl88x0d7m76oqjngbvgzt9h8lau0yk4tbd0ivh1zppy9knxcf8ngnojdgscxvizduuzr1jkl4jtbdjbi38","617":"gjzm1kc55e2scbw3oq73j8a5gts2lhutyzeuty7905lume42xs874ubx0pbabsq6yels1dn6xr0pyn8cisfloxcfeyx6z1pfhzy9","618":"f49dpkx5pux12s82zp2xmxv2jbykt3mx1nc63r98mvxn6pg6pvzvvylim0wpartj76qlj1o1dy9di2zue01sw55i5ru5nop6lj1c","619":"rg81dbxaa6gk38qd52bjcflfkkl54x9q73ghi9v9jtk9siir3oep6jcbthkvhzm61p0ixvnlgegpo6c3oe25ei0p0qrizqix947k","620":"tf6a6swl04aqmz5tfqv39hkw6kji0qn438hfxlyefwufu13gw1a5za3daiuzw1772t0sw6nb3f6qoffyk6u9oozo3ds9393cvno7","621":"2hqr1914t51hzd87ehd93v476lyf7cl45h7x3rs1izrtyj93ipchy7o8uwwcsswet64rsldqyvlczr0ntmvcnht7u9dscya0dnrh","622":"qd03bl06dkloyrixl8deu2fk2d45w0l8d8z66o7fei83n43r3md50obpekhq994zzrd43hq1tnnyqm5ylmep4eexwnsjuhowvzou","623":"vc3fg7a1188z6v3ww5x8dyx21kcfvk9373bs5mtfkmiputxhc7lrn00j3ovf09mzgwpyrhfzh8667328efw8t0d5e82scag1yvtb","624":"2omdag7g3oz2b7lzul10ia74shxmhgoxej21dmb4tttj48cew816ps0djmeffw0fm25b6x2rhghculj45648ikv983dbgiqxm0j3","625":"qob3e11qgd6d4gtvs6fma3oahlcnf3hsdoezdzw9k81ownc7xfq7495ejshblcuostdlkp84xwcorh8y5kayacraypyxnb6kmv5k","626":"wpt95dkbn3ihli6w9bwki3nb6z9jmu5pvao4tdbqzrnt1jmau3y8nxgcae91k2k8h9hi4mfgh6qq7lmebyqx3gv23of0plazai5r","627":"y40ykeb3a2qqkuy419xzpdkseun1mcv9sbj8mkj0aje3xthuhk0rnsa6vbv0wf3nss5zwnv5q66leu7he13vjsf0nz8sk3zwwbky","628":"74akmwshsnhk5qy6h7hzxo4dhksaq6cgsdr9xgsw878g1qrfuftitt46ez7c0260y1v538v0ti1227lqapq2fhzs2bgkwk3hahd5","629":"mmlvr992tngfmyz55nxllr2dz2kjvcae45qd4pbu76hbw64yv6buxphll839u9zzgkbz2i3arn19wjjdvznwobqjz02j0m0calii","630":"e6gv2839oq6xav0msw4uqo31hx2y38ktcnvi9y65jp84mryzua9bmyf0r6p1m3g78wto2h577ckiondzmzog4sk1bzk2jhl4tmec","631":"yikes3lg3skninontl1asq7qgzq7q4x517nkbk8yz3ivwe91naom0s2r2cnu339yjl388l8p12kuexwo123g20lowqv3uvpbtad5","632":"q64iua3ni2xh24kix7qt1c9x755duvaqdzgulxfraxt7dr2a3fyaggr36wj2rygk7j1iv42f7xxw885cx9sgm6fb3lcef0y5blyx","633":"65a05vz2tdczaxm1mxmyodfgvozlso4ds6nel8ix2465k9bindmsck66l8xwnxwwcf9kf8k7oja13rsp02wl1xv9y503cqgmyzqh","634":"7w2by8wnbdv7e4oeru03apyw9xgi0qhfvh8m415e2gpwu5t8ysw46mc8yts8vkde891pmd4wkibb2zzr1d6ghafy9yid6aq0t17a","635":"8juoz63kpyep5dy9krqp3bce9eyclym3of38bjcu7p8yyzgrdlehh2bkau9ccwuyha1frxk6dbihyrza54p426psvz9c9prnmh71","636":"hom0k597kdb599u4hvpbpjguisbkgvtd5poephl3dxjrp4pdk68pegtuk2bpnncapojok4m0r9indnlvh0jn1kde8ngm5l8xyzbe","637":"6u3pu369d82lm5vm2m0788ebsieqtxrji1upm2x2i2rrsq0n8p2wut7ua9uativxo8om8qwkd61xp05fv7tpuilyu7jirgj23mjh","638":"ghpfhklvj2venexyzzfehfl7uv1boafhgtfu9qjhe72xrnwlmc7cucj5nzhqez32buibtriolg0n7ig26veg49lzvwa2i6436see","639":"b5hacyw6yyqq9kci64qhr9j6kh1jpk7gonesrfi1nraibmno58atn2ouy54rpeeevsomxe3vnh5ilqjvtad3pjneyqmfgr3smicp","640":"7uwku55qp2ch4adijaeda4a4t1zvvyx4vmn0x8fet6k5ah54fs1cdxwdwuvwwvfhd342osydstmpuf3lnul1mwqeqbrjsevhpxa5","641":"6xvuqyvbqxhm33vbdp5a2ck4a5z7ta8ge0x1f6k0uvovcj1r27z4fp7v1qy4aypet6uao9d4ahufwst0retuzkoehjzfm9igw114","642":"evvd4s5sk8ercz2bzudfh4lcnbtrc2cx0the6bush35gvool5hii5rlyaxxr3mdg3nq34th7hec6xajwst3m1aquouqcyk306z6y","643":"7plg5jlapotfjbfzrep2oj8w3yu0c6q65lf3up2y2j04lrsquhv9fts35cf6pjc6ulqf6ctfiyokyzwm16tzvj6jlxkqoths4k45","644":"j88rounkmji19e8hq4q6frtajld2j2yjsajdqso9uc8o7r9upztls134fwttki6zy1we9lkdkp6c8nw5p91ic4k743n2flj82yrp","645":"sdlr0p0bhkw8dripr3w0a9u1ir6s9ls2vczvwqdlmxacejvwaf1ii0p9v0o1l6zyt5o5i9wqvy801uqdcrup21wvbkaf9ce8en37","646":"6tw5q9teo4lf8b0y2uxejg3lbcjrhg4xmsqwqx4muv6m7p85z9k6qgj9kdcj1903gh8czf4josaqh5uajfb0f5lil7hcj53v8fzg","647":"xs7c4nnq4ki6u49vs6exsw32467fo72dpkj4cailfp2ij1rkxx14jxjhpxloyh7blszfr0qarlfynbgvxse2pvpv486x1ewl172y","648":"u5h3bobfj5yul6p3nyahufrabsipnzgi6sposv74j6czdo6qogypa792099zq0xeiylpv4zj9evhlu05xvp2r7k8rp037pwfksuq","649":"zifnrv81pvk7axpskzvm25yllxi4f00r4931cog489eodozy9fn4vqo272d552cdrkikq8qw69x5ffc9u8bs7546k4a3w1ad1sa3","650":"vv35ojlm795ur3x9xibcvd46v8elrmk86lcre63nz4w5kgt6wr58bxfcg0vqbxiansb2fgpuetdzekeffp93m27rpbqnkfzuwdky","651":"20ksmslgcfimd36c2rmgodax3xvltbqwf2l4284ph5tlpp1hztp6wkr38202cjwgoqwwpdv0lyxfaztxyp8p2gc4wkdbels19rpy","652":"xj7vv4mctslmtjg9369kjagqxjbezqzhr0uy85h64vil2r5m6n3ymicxicpig2qv414nwnmrz59j5pod98900ltsqhihmeznbpe3","653":"63jw4nb319f0zwlr7jpgf32pyr8qt73ce69xixeo2xs5dercsiywsz2xjwhlqqz93hw8styh6tmoz3gb8lp773xz2zpggrv21xw2","654":"cli61pnr4b3czjz3l8r2xgz7rtwqs1ksu2ypnp8q5198mtl7tye85l6sogutopikha00rdkukf6c7v11v82d6j2exqmt6leudqgc","655":"21xivq5zhzru2cah5trxljy23belj9vs76kqfht2l0025x9tsp7pjb29re09m20x9e2zh8o0gaw4iri0qpv52d0g94yc5a66rgca","656":"07zk08rryj6hwpqutzc4ot7rw3ic3zf46fs4f34lftoufle4tnhga3wkidlibf32yfuoads24o1fxagvf6f5jn9abiztyrf4lnyn","657":"itulckexla6ywtj4gg38yt2gvmiqexsvbvv7cj9l3dyt9fer8pd80kd1kqib1cq5x3t3za4p1syn8mc9hd4j1id0e6ou68hs8p1v","658":"cz6ancnclhgwfn4n02vyqgcb7h93dwvgwooiutl0lw7lg2bb8qzefbylx58wtv34whiiw70hbepxsuv2f808aei84xw2cb8ogekp","659":"lx3woxtpd1bftxcawhwt0f4nndtahpuyjxnlsq5oe9m8lb8c3c8i0g2l0grp3k9i93596px6gzhcyis3pw696w4udu7tsdn7swg7","660":"cn8656350lg8p2xeu7v74rttbqqbd5myb0w9btncb4l2z3hgc3javb7kki03ew06k3vi3dmgaa2pedvx6vg06x5km3z7bjyacefk","661":"nl9uxgp4l5gtw17r2bnyvgl30log0qkn0p73553cpezk86ypw5ke5s3mflxe59d4j8huume32svp3gvd899brc1o2hxz5u2hjir3","662":"q2wtwlcamak3xq5xctql2p54mz6msoaq47bctywvpxlylplw20rv2x551vi2o1or32y8tuf0cj4utu8729asy6lowdu4nm7lera0","663":"kxcnenwdhbhi4jiazki1e539r1d74s5kbd902djd70byofbpmwkufhusg5s0u7kfgd20ifrr8c1yl01f40980gbmysj1v1klwcwl","664":"7v5fpif9affhqrtw5ezx2cxifi01tq11k0at57x3cl7stwwjgepwfnutugevghkhx5j03gb1aa5r0ogbi60a5ynbxhgpdxcfrjmn","665":"fqphuoucyz5ggt5v4fzyd554bbau3qhmm143gdn5zh2c6vncfp0x8id966bqrkbnfz3c2elyysh9wiw078oidvtd5cmjhj7n8wu5","666":"klo5e1qznb14db23nz83f556g5xgohvmudtpm7v8vgoshlkzd7th8vv00yegg922wcgm5y7kqwtjz3jpfp7qv4w3wdqx45ceum7h","667":"h6maqxvw2w6srpf2y6u27555dux3una43c110dxj3nvy91z8k9gz8ks7tvscbep5wo05trsft9gmr5stiyqp4p3t81yohjrrfw82","668":"7tdppws7p835g7y3abh5qfwnbuoctptv0zvx1ym5o1q5mg0b223hn1jiffr10lo9f5z39m1o5sby03ha48uprk0uatp6bi79oagt","669":"hk0lvq7nfdrhqaogmt3vvtvof8vdey0hi0lg6weratm1gdri6ezfsbcdzzj3wppd95fzhrd1pwprwrkew5i6jx4mhzdiqh5oq8yf","670":"x9pyguqp2kse7c6o4oc8kmxd26c3ygz9it13ysg4r1tt5b0kua070j8ctkjtoq70dgcfde8f9hxlgubdtg1zrdc1x5axzlsmnjfb","671":"ik4q3ws2qb6yyqjgkbw62scd0js4ubkbmmtd0z21pe0mz6p9o8bp4ulj7sb08i21hw8l2j374e0dhy4soifayxa8o46p3cwyp450","672":"vhf5nee9lxynz433mhtuegtfalgixz5qa4s8q2n79c0psuelvdhky2f2vghaiyj5amvfy94rhm369n0y6ptomsewz0ay30lgazju","673":"k8963gtpph5gl1cvgxqh0m5hix60x7rsd53ltiomk7mzegqejnyzc79a7tdyrp8exxly5aormccuihws5qbla28m5qiqgxo47eky","674":"40nwz3xaisoomo2n0snmg5i4mu98ci1xbi5rg839f84n3ukxooxnxal17814wxgjy7xaj0xfp8m3rb067hbpuyx9ho8cuu5qyirb","675":"818v1ev6b7veke3ixdp0zxfpl9kvimxxstzrewrbq35601350djois7x1tonus3vu6469u05ml6pmf70g45giuu3g234fl0q2o39","676":"7bszkivkyjga6r7qjcz7tbaxhjnh7fxo2yoqml806dgucm91ff9l0s6i0k7vddv6ht6vvppbmlqum2fzguewt2vvxqazqpafpiis","677":"6jfhllspd3a3zri7t1zm2r8iyrjoz8ju8hv3g12laxh9g3pzih2n86rc4ih6qz0mpda8lvxxal9erzzofkt0szdrz4fq34fdme0d","678":"nq9gqve316v0ezpyxuktyqk6sl2zpa2b37up17x4eox6amko7hhkqb1zj5bey30jj1vs9hw0yvnubit8n7ojr08y8u4ljv5uu2yi","679":"tet10cj353rrtdk7zy1qqad53tx71lzbhzrpuda4zcxihfob6j73z98ymqlrk8u3tum42wucy1n5m0axdznjqa8qryyfvejqmnks","680":"pgu56dupxx6tmc5gwlnyhgonsf1nehsc9crsl9ixbr5s8h3dr0odg0kt5f3aowwgctbjwb2j3fqp34afbp0qlqexgdwwwnkurof6","681":"tpc5q1lxxuejd13ev95atb7iiboyit4cu4sbu6yl4noeesir5vesxmjyahqm4hqrycjx3yqsu0tixtvu8ydpkkkqbqh9eao5lp5a","682":"6al56yyqb3gr7iccd1y7xjuhx3muamp4lp22gw2i7huqevv9c8u3bvaccjtfu2spfva6d6c6lm3e8bcjfi823bsrkyug68u4ce0k","683":"m9fdev4r8r4may8rnwbjl9ucrn1qgecwab14vftegp10o1d3el2p00rxpbem5khjd3lw8r5xkubo2wuyiaksny7qcw2ue8yr1253","684":"2ijtcl21r8e3zhh5gcbee2mc24pid6xi7w2lo4909yk9q26cyoyqn2rn97wi6b4juxd4x3j75ciop5dhoe3x9txp8bx92nbv8iur","685":"orr1kr274gmwuec1p3okjajj2fudvb88xf1ysmfzlgxjzys97v3847i0up4xas9g37y6j4c89wqik1h0ihsr7cf9zdgwr8xamtj0","686":"lftqz0lsyukuayu5fzxhnzgmeqlq6a94zirox4edg3omihbfdpaia2tdud5qiziil9j5t9gpak7phbrindwdpiyt0clxle6y808a","687":"i6xjgr4lr7ijdcdl37li9nseeqxy46x9hmridlnqw3dkaeclvv1daio3qscoj3jci457z3oqd0v3zqes4aydk1bggehzr3u5rsts","688":"wkyfi3vn3aoiuvp5pk1t2xin9ut30bsslg0zw4mk0wih7f7ive2wol4fcud6wm0pv1u7xzbe04elqiw578chzvy3jlsohqoiquho","689":"8o3lomk82f1qjwx106qlfp8maybwb4php9jhp5mnoh6vg8psleznzx82qzxj19lgt967pjvnv6rw96uywwizaw5f82z5dw9d4h84","690":"zf3838f0gnld4uy2h51w4cp0nertf3nb7xc3d2n866i1u2l2h6sm0ew0vy8rjnu6e8jfeyvms6hq2ru8fuvurplt1db4oy6fz02v","691":"fljg19bdlazcookvfky5vptg7m31m1mksr05ugofs6zj9sly6khjs795a656dtc3d121hqoe2r19o2bvpjx983y1qrwwqnssxgkx","692":"b9k493mjc33goq04a134ows84kg82b04ktr65j9rtm9cy7f0m60db9zw1to2v7gie8cvjx8wg0ya7pewv97zhvtrsa06yqm581aj","693":"ml2xwkua0ggftwp9shimv83qetqf5ai2osw78939vri8pxl8in322gjv5xh7v5i0kf6mtipwejk0bhdfdytib9fw6e65fqnq9vbg","694":"qlsk6z32ivocfjsl6zjr83zj3ubsmerywke8bo8s07s821dp26oxh6q0emuupqcmncixe9z4d7va7hpcfeel88bx65451n5b80qw","695":"8rt6hwlvaetuqb1lnm4ddzabs12kmb4j183drm5uijk9upwermuq1o0c3qm5xir99ffml04gmf0dlu034yr0g7hxuko3w110pc70","696":"qr39k52ai3hcyaqe2ydr7ve6hftyawbmzb0b51ettzm7bav6ltjfose73hy24fckuzk32w8ca97maxnf9v6ssc7bdlrfuvw2hxt2","697":"ajcml4mu4r02iohpkjwjegolnxd65m2u48bfprdnsaxq19wtihdb5g97bmxsy6pzl00zirsmo8t8vx2714z1o05j21qmiwa3u5vn","698":"einwoqnzmq8voly97fv27b86dghel57cthejezfiprkt0ygjaxvpw0nbnym59uev8f2bm45pcr7mwc21w3xbd1jfqiconybjk39a","699":"iix7okk51bi74ecp98t0ta5fel0eb9fs4029khyrmwn3ewjncp9u66hcz2dwli22k9aypy5ny96vuys8gf0fli7pz6cul6wew69h","700":"2p3qhjgm05i0tsv5kpqaqmonbp7ab2wu4qidufq7ki6ecz3601sm54thrfoeprcosxa13ui9zd1n2guzdyirxystcg1a91t2gxvo","701":"td4w5ho3gj0uouqsh1v7ezb47eruowr6mhlgce0xakd3tltaj1fq7tkj8b1nvd4d78l2dnkrtmyxchneum4upze0kru37zsgvmth","702":"572fer667u0tkpk1ns7imiiowlugcwrfigjagfyzxvi0v4jqzwslva0qrsx2f2xzl5839s3mq9dmwpt1jht22bij6z3nzwj62p2d","703":"o071og6om1jkgrekzaby07hf0blw9qbp1685ehotp3y1kbw6lnyo6ilzxd01qntys77uuswvzdv3uf79w1p8w0bsqjv7lh07vuw7","704":"9bc96oqo2sxlbyngrj6ve47innipgh9ztjyhccno4iw2utujb3nxrjgte1ik4s43v8316v0jzq8iixd2wgb6dqvg9x04xwxlbm9l","705":"lqkn0w7sx5ku8zyxhyc0naozd7a3dkv1pac4cgeymqtelkh8yunof9pz5qwrsj7e9ttgg2misyns3n5mmx5xadrytrbose590d53","706":"82n2romn7925dx3d7dxjr1vvr544h40x1ve2qoz2wckc2u8fffsqhkwd21fof5y5awbqm5vva2stki8d61d5sjbve6ukqrdyon1h","707":"9jfr8ohhjmw2rdvpeozluoijobwr43sldxgntky0iikeya7uxndnrvqzedzqxbi8ut3ixh6gdq77r0sx61rhdd8r5bga5lo18tyz","708":"fvqnct1c5vfavj3f5orz81hs3e87ets1wucb1zf0gj27oogfw4p296xyjiigeo6v8mntjs7gsomkyf4y5adwb6p4s1top4cbz7u3","709":"9id7jlupso3ideg138vptrlyvszsnw6nrft7ow3mg97ijlg92c5u5bngw5pjy0veuece42nwry2rfz2zjwb0r7rdjoze0dk6egi5","710":"sl9tdzhll9unib9e5bweausz4owtv7heh12n07ni4y4wr5cjgvdybiwe5k2ikciefyzx8iip7icfbg3loxorgxpaga0ztkhbgulp","711":"5qahh9z8pp9n4s76h5x01zx9nul4isvn1sk2r52sjbjrvpawuy8fozqilee8xwxzmcptf6877yh0cjqsd9eh9znthxo59eg1f0pq","712":"8fhnw28uvhmtectlln81kt43w2j8b5lrql8i2koue6ix5drb7w03w94tuc9xkyy1jr6ktmnq0y3p2rzrvtjblsgb6b6ewk33k815","713":"fo0gn72wj7vmib3g2g8cmf28ou9r536k9cw5f04ikappxzt04fge6o5rf6foqw14d2x5vk830m3bofnx653lrc2xcbqit6479mz5","714":"wktdgtye3bx9lj8rfgpqraifjcl4xnlsgqa65p40k3pvwew678sg2rwoxjgdis6eobo3xou4oy63pnzznpcsjfdwmyozj79o0qa9","715":"2zprst4iyv7t417rfjmy2zhwbx4mh3zcn6ynjsxt0wnd6tmmo1rw56zyljx3zqh56sfao7dqs2knndf84jwi4orb8y56p95ccaau","716":"wwjvgvx4ry0f8ohvmuev27joh3xtq3ryvzjsaixrl014v7hj2b8jasp6dqwmlrq25o8rcan94bqm8t4av3yo9famz1j645g2zk71","717":"1n1199dfhbqw9ofqpsatrk5uerq4jao6r6hy9x3i83qi7wujrnycwqv56azr7y6xkziurtr72agc7k1m48ob9204xt26g9cww2i2","718":"6ak9d7yvw41rwryu623y8t5b80l31fvrtzco6oxzo408r5ol94i9zjregnljnqxcly0wer1vbr5fv0jdpbehtmxrqk0vyqxiyvbx","719":"e8z27sqmi66ezeod64qeqv69si233t7z2vu28dv9j2ux8xycegk00qqfmcmxciwhjl2dlovcxb47fv233vhqzoilg931nc6yzljb","720":"4c9xgtdpdmo0mkvfmaqabyobfhz9ie6pduqqifqojvl3f8uzoz8fs0to5i118wrv8zq0nab5w7dkicatn448oqwz8hu3mdagu4yj","721":"2fu84tyue0ekcdv1az2meh0fky4ibgfte9rii44ev22h4g2c6jjp6mekd7gylaqfrm44qh2jgfphwdhebw1bzwuuv7wzl1apbyrd","722":"xz4ojxvsw5s1zccrn4p5zt7phdhc4gvuq30a8gd78ngzkvdrj3ojwe3e7uhw9tkr32r0r7yz2p406phi75j615012oc6h2g7v07u","723":"y2y452898jdrgjv3s9z6a996qmnlkmj24r16uztbcb734h2us9hmlbploeiffxk3xcw79glhwfzrz5963kz091ipfe198d3sq7zp","724":"eldwjpbiikifq5zdjhg3d301fjow1mxhouyr8qg47iq60kwr7j26bycde33d1z5jdu0gf7oyk9ts8gdos9ef3lcin1puiewmfrg0","725":"3fr20gvd1swspit0hx2jx2xbjchoc6xe81ef5x23f7htcu1y4f0get9zh6lgfwotk4m6vov71wb3xdvo0fetslrq40ikqprdtz7l","726":"jshdc40sg1x7bkpho4myaff9sazynfxwqzvtrmkrogenarwtqo8w0goanienqfar6f9nmp8bq3pcxozzv450tqnjmlu2v2lr5324","727":"en4q96v5wckt0refklt8bcinwe9ht5z5c27ltz6hgt9mre5izqhynvtemtp9y3cfq9zkm4m2wx47uwhaixn3v1949xpq1i2clvg0","728":"nq0ztc2nwmj755gricyhl7eyrl3onxne1592awbxqgdhtd0a3v8oirbu9bz4q4zqeuso2adpgji0l2ty9mahngygr5gn2zv9qc14","729":"jc2l4r8kzhqwq0dy8qjkz7yyat9ij3k8cz8y8azacqx9ww9aybb4ykik1o5744qg855zkgih9beairhmtik9dz2x8m9wntzdzinj","730":"suduprkk0bnlzvrzxydhkg51u9ljhynasg1xth0xuhzjty5z42r905lrre4ly3ao6ax9jm1c1txsyjlnmi4arq3h1z1rr3o3wu12","731":"02pr75ptcc2lgdkdrs8hfpsswoqgkrxnhnpobhkm9y6008vabos06xj8meuzqcqyixt43c5oixbe248whs1qxzlcrl4w78xemzs7","732":"p0ixqsz1inoiv4og6nducv38fuya8paasfl15fxcejrpfo1py5sh3n8xearjoqbyo1j9ex28da9s43byd0kw75mnuu1yylr66zfe","733":"y7yo2hi6j19b4bsbgf9340bpu1l442dmuhcgtzhplvh35gwmi5jaihudt0ml1ms5ezympyofe01zqcn0eyppd8hb335iidu1m6r8","734":"q1brgdqikxhe1z0bu389v0qn931mvfs6r0i1s7yh47pyd7vhou2wqyt9r0n0hpaqhsxt1sdvxztrq1ota2ofhlc4g6ymd6q9njmn","735":"bl9u9qjfqepqweum5hwljs1284midemm2h3rvmu40ezi4n1fhw599imc0flvaq8jwc0y5ju33fcuda6dmu0x1kwuo18cm85fhisb","736":"5zzpy6kc0k1u4uwv6mtb363k0m12ylegyabfw7jhzwadjazew1wjkay24mz1p77x41eazizb2y0xafmd0opb3402mplem22ej6zy","737":"ew079okp1lmhkaxo8mytv46xetf56kw8rdyd7023pzls8cv9nilbl0d70e07jm2pxqfha5balgdccsgtj9d0gei7zc30e4oah1lt","738":"3gw1eb90h29j163y940m0aiz0dax0xwksq92h4fyyk5svb08lk4qiqask2wl5qrllldb80ilx9ta4lcg1tib8rowl8xpc0ajtg89","739":"wrdup3e7f1vz07pjt8thq4mmnr6iaeakjy1xwiz6m3lzjy3zq1pc776507pdjk0ddwzfg748am673bwumjsttahkyzvdbt4abfr9","740":"av4a447vht66s7ahti2kgvhqx77csvoqs349kkxoezrc0ejme1jiv2vpwmkl1yxhusvjtl63ya1iz5xk739fp97kid7rztmy6ddp","741":"4qs5zh5pjbjeuykqs3nosvuavofvu8j1y93m0g934k12q7kx10rodhko6lt0y9busxvkvgkpyntea1cc4xk6yugp2v2u3l09uptf","742":"tpoglelmdkvx9o3jel9jd2o64ony4s99t88r97ye1zmheeum9myya7emcwd4p61p5m6c9ravc3cs78ix8wfoz7jyexukj8qzjjrz","743":"h4qxrma6rpqyudg2ei2dqcu8pq10mx2kch37p15dnysyyc78yakapoyal8rp9ud3yn5jn73t5e10yfyuviz0p7l3yy0d6srum87c","744":"6cfg7g1vrhjqdysvizdkm75bb8afczmvthzv2ktkwqjcyyarnegbbm3h98adfefr3yas43387hfsulp8uf8jai0kx4hdiyvao22r","745":"wbj3w5qxo42au5o9egptmdemdgj5m49sd68teqqqyid6zm56sw5m9vwmw8dg3hmyskkiuo0nqozvh330mr43hfi9vyucm6bzykr1","746":"zglr0ekjdbryhnqe4ivprl78lurpbpzoa4ae7k5qgjcyn6d7lfag6o40xpezm71n25fx64bhyr09k62a81ifnwmd02k3x35epwcb","747":"nup0sp9xu411xl5ry5mevjudmwhuj4uua16gpgtekme9ap0u6dqcgn7lffgk9yy97lw02b9nruwaomcd93nwfvb9ws6isbnnywm2","748":"9mbetr8kjstsi6m15bhomkto7ha8f32bg2mc4ijv0a6cp3cls1b393hq8mnfxagqz2rytwax0mgz94dv7k86s3x1o13i2efm0n5e","749":"94801zyzm2tax4g6d3qj7pdta5etz7dikwoka2sapc9jph1ga5i8fsn2m2kkd0n7b86mzqgn7a4vmryppnv08j0i58gw6dp28q29","750":"45a0sx0bi7y1l806bhd3hpkuzoyfzo4vgv3dro1wu450gg8mdjhzdbbuzm58064mska5t3t332eum69behgqqzc6afirf45eebos","751":"8xevp087zxh6zwl46dps3ejv2r881w94hfrib7b1z6camn69hmced5r4sb7dk048g4r25934s9e6mofybq0qtzhsvzhf8d04z6vb","752":"dg1tbh9ltyzaew3yl10s0l0z2sh7nmhiq4z7scsdtsx8221ni3x9bczjseci6rn1jyf2itmoljev4gcbfzt51q0exvzxw6o0fuvq","753":"w9ya3a8lsu3xw69f1ewqqm0nffaag82wzqylaeqgtqp0ft8qjk5hv83ebytasfm7pd1wj42xf2eoh826led3ldjoh1281c7ypslp","754":"euypzvwppcfzokxfw7exi795gb4gqxmir6ev5xzt12icogf0f54p68afto9blgzs6lsjt85sd1c6tkhravy8b9p2pcu23mmilf7g","755":"39pwecodjxkjji2lj9ihq7hh2vgkwbesafgl7nb0nx1ha4ijvvtvsu5mf7wd98mlcziod11kx6qmfxpbhtja94z81wj6orcabz3y","756":"i1l0zx3qi4d9qbibfy9rskd5njuimy2zt0m4mleqbgsa0pjq1ev7qvfti435flao0kbqo9ssn72dhh7iou97xbx161jbmsm2u4r6","757":"7m9w0c4kseqibij1dkiq4jursw1p76ytizl4ywysv8t2a4o1aoc0ezmjbhx8uqn3yu7bwa7ocdu7bvc53xy2m7nh6qy345rv4pdt","758":"incxzifqswgq6x6408e4n32wh6mzpkuqhvrcvc977z23ia6ivj7yb369q8djhi7z5ow7hrvsw5xi48n8uhsue9e76ql168ywzdh3","759":"jt2x9905hkwi0enx8s91yxndec64kiqbelnrcdt1bapdfrfa0no9vzos8d2flv3y5xkz94qr9xv9tsg8vphm9ppg93yvny1rtp91","760":"we1zx8sgaiu8jq50mqgzc00dybamgva798z0bddzhprs2qv7skfz3vkrkf90avzd6vyyff4pm5rgpryaqrhkhu6s1mrpkmslmtx4","761":"nh3p7a9lybgbky7ds9v4xyo4xt1wijyt1sypce877u0fi0wgsezmc31wyzazs5pwruezip3dycoshxo5aiisuc18eoofw4zp6v4p","762":"2376lsobhftzlfob5cdc39f5udq9j1wkf2q5prohacd8648z5rujp3yr2k4xwuc4rpnt1ax1jy96kg3lz78fvdgtm4qjt3g20q7g","763":"jfctaw3vj9g7ej367c4dcfqp5ktm96x9ig2pbedidlkb3o1g4b80x85thqg5kb551j5jywd4c6rsqsktpkhmox6ud1e39b9hefnx","764":"gkc9bokz7w1hzxhkwi4wnyo3owj16odgohhs8eplyefs5ncin47b26y3z7davtosb3mw31a7fk93mrk0b30o9z437ljnmmhbgvn8","765":"xko6ga4sonz5sbd3sf5m2i61jbzauug1fl17ecouudhlf9rr27usty3urqr05500fzad06r1p3iypflemuu0qw6ax4lalcu5ei8z","766":"mg5oas0p9vg2dahdsp10l4oqbs42fhfl6ndhpizyqzttj700cgcjki9ete1ve7rt2nqk4lyrjoaq85uh8bzohbhax0mbdwh0uhm4","767":"hqfi4niis4qutppqxy320qo9037p1uolb4elbeqaqbcsvwbn18np3twnoz4flqeksxz0r673ysim0eit4qneq6vfqsw5ljd4h7xg","768":"7fa75s3vwe7rxr13jf6g4y1eyv27nakwyj9y6xg1joy0342deyoxpp3zl4cg53t6cjpbrxvy5tiey7xrguw1kr4hpxc2gdyf1x37","769":"zk5gl8gqmhos8iiswebvhsrfv9g2vt4omzi1dz92ub72pa3ojg1qa1rvsrzjncq5mmbncwpoas9stvoyjbclgryaqzgsehm0b49e","770":"lml4ugfslh5s3bl3ahsosepz6bwrs7ycxwk5di7ypfbbpy0zfe7q0ppky5luk41y1c1jxs1vwd95e1enjxu5iblyjslbkwy181go","771":"poevtlm4z6y2aavfqvmwdd3sxbhta7ip0qpfpvsra7yx5mtvyqosa348mj6fio8j5d6lo7tea2eokr4e3wfsn3ck8kyb0s0737x1","772":"uyho3h73nubsbaiu031vjuea3xv7yknovd1tbowu8pryruobhm5iylmlf4oyflgx1lvpukvhtcetjfmd7177abycj2x6swkq1ujx","773":"7zdojcimgnisxoshzrvebmqmihgip3vpswnwsey8aux8ujw271k2kcrxqbp0tsifknlr8klh60ok8h4u9efaebwgu5c3ceqbwce9","774":"hbcwjf9rdhwdmtqiainnlb4op8iu77peibhpi793kes3s6pxni9dfso3cb6e3pvm5pflmowi7eqlvqx7dm8yjtsz8uao6f76uo3p","775":"abhlgqz5770kx1yd8p1dk2ger2gfy3u24dute4q9ry65ppf38mkj5n8zevzjnx7bjmnu55p7n2vxtt0ywwkeatepnfkc6fxujvp2","776":"1ng3waserhmss5v826svy8awakvykihyxgspo8d3zhfeyyxymnhf53meme60qf0cnr5xibzs0hkn3armb0aqx3b8opvntff3n9h3","777":"ey9w9r8ycmp1jhd0l8b1dfelufqgtc325cokgmfoedo2z3cmf9odoemgf368uf0na2v1oawo0x0egyst6nnxvzaslkybriwko90d","778":"ghjgtmz1ypiwme8w301qeq1kodp2icbc90cu374p13s0sartndzyadnckud2k3x337kjyqri2lk49n1rvobc65914pfs0xt1ntw4","779":"ler97kh5q3u9hrrbb743xvv8pci76ttk2yfbg7noy6ldwplts6dgkghnirkzj2o0cc1lsftxr9a376f5ydlpsf74aijekihibnvh","780":"y8l7a3n3mn1kwgc4nlp84o2o626wnhpydjjnxxyf2eoumkiou69m2k66hxyen345togb9nrh59osjb8hvrv4zcdsab1dpsjf0n4m","781":"dlaevh8dpgaefh7wbu2l7xj1fpbq6jxg3d3tihlnaxc5jr5p8emvob3ngzac8zt3cq4wp9s5nrni1y2fjyab59zkwbpsruvqaoaa","782":"tgso0ui2pgyvkhvtzkghop3sjrt3bnstpi178h4iuoouirpbmg9fxrx1r6u2ni29z3eu77okurqjxdduotq95r6etjhto5tmnvtl","783":"ccgtcq2fy76v41gzj1fftcp60o8fgcc1wudjbyifu213ksytowc64tclq480mswc93q58zd3extgvl8qru9jbyo8jr4aji8ktdx6","784":"lbdqjwfao49mrbw59vbgv10tjzru5m743qun57wi1dnlft7bj2xkdhbm43wggl1d3dw25ek8iozkpotmbmykto84ihn1vev90ahz","785":"rccn64260sye80kjk9dxdbll6prcps8o7n0ejmyphcqkmd0jb0x4ua238izltsmfn73omjvkujo6f95pl53qo79loka2f2uaxbci","786":"nmm795kc7fj2kmbcgqs7hxdl6g6949d62wb6jbp3jed7dn4auaptqzaymdkcd7jbd6eg9mscwy045ckvg0qww6uogjpvqert51g1","787":"03lt3d61x9aaxvq6l8n3wecj713zr7ezm30eo57qjan72xwpcxu476if17ihxrmlp2xv6irrmxs69thlicpbq0vgvy4px9227tg5","788":"x509dfujlibj4xdrtwwo4hdekj2ddyg4dp3wyrztrpr13omns4fn0fhu2in0fkohpv70b4vmybb51kjn2ybx0dn1j3o3e7w62a0a","789":"n3pm74ncsxket2keqnoeu4vz5zssq9ej8beb3tzhopaaf7ihgnfj5imjtj2bha02dlnlg3i6l7y74o9udjq7p6cadp63sky13gna","790":"a2p4czm7qfgwl2zcv8i4szn2rux136810oruqafkhhr0p32hqbaijewonob16fq097xv7kfky4x2cc13x11mficug42pgaoadvi2","791":"lnm0wovofik3hs9mg2gxkyrqnomy657jw0pb7l7wsx6yl0pjaalylxxktv6d18poo7dmzpvj1zknra7ju2a5iqp4dyk4v0xj8pqc","792":"9qf3wt4tiauxdueepuoc68z3x0lgpr25mve2aj4358uma05khyv6s3w0peflueuybbkabyx9hl6qnox1tvjjbfe6s852805bkvyd","793":"atoy7iz2szkphvoolbwgxytb0eu18e0pd0w46cm04sxtah5ja4dqpbuf28z8wrjhui8gurf18rn0dh5omhezw053q5s429nzpuha","794":"pqw70og1l1oljrm4cdmivvlatkqh2c8bwzt3fx0u5lwn6iyaz46mmwywpdno0fzqeyn7yfwa6dfqtgs8mm0idax87skhi9984b4d","795":"9gy97xr2aj9qwdvfb62ewb7jgfav399k5guy49j6sl1a5mpvri6bre9bhjq4k51hul855v3lonm37bgcsancvju0xg81di8waki5","796":"riarkcly63kafa2bu5zabjxu6n7fwkk0o39rcgqjedmmqlsplsqn80ca508sabpiud5fpstmqpcigouglh3v756ovty31kr2o7rr","797":"n7rplshqivkpo5oyjjvva5kkqxk61dvw0ncj069s6q5utdxv7lry18oks4y4kze7hjxjlmll4dzei47wrl5f8bst96herbal7dt4","798":"i34ep2hghdmizd79uxkt4qkqbnkrb2vgcqggt3b0rv0vwr3trosgv84aeo8dv377zauib86yo2to80ebwo6lcxfoc7asy3qpv64d","799":"gt7hr648jkpcs4j5a9fl9lefh8m6xmuau0xfurg82xilk737eagcky8h30bq311vlctngoybdx1xynqxjbukobmhrtdwmskaina7","800":"r9t4y5f5k8lshp4qimrvl97yvlyfokr6b97mu6ydt4bnpqy31ypefsnoa7zzrh23wybxf99l4dszdbb63svgzctg5a3uu152z1io","801":"pzd8bi78huo5ur0qkomufrxjfch7ojp084lh46uou5z4u9jxa7qvcj8foogf4qsws5xomfx8aq1tggclb8gigoe54bd2huz3osxc","802":"n6d8krmmjwbmn9mtk1lzo8rj6ai76osar3azqgqftvgc87bapgrrom2smfy35o0z85d4bibgiie7awk1wg0u9aovr7pceif3n5mi","803":"esay0d6lx0wocdbrnyjduc4mmkd10n8yyouxpdoq017yw4zc5gvs682bu7mjxz5pug31ulre033843bxnjucrgl2kw9tbzv2kr6i","804":"b62hfsecwvwusec7j3jbgutfg76wfk93fy7fuli992c8d51qqfjpl6me9tvcx6vvisja2434fatjjar7g4aukdjwkls5p4e6m38j","805":"qhlds0yu9ahqypys0exg4nrrm9mu249b3y42rzfly180vazqurubn609ddfnskstuntyaho6yqgmf0bg6z5yntzidrljiwastdso","806":"9442ctiof0v5mfrcw7vz34vhfggguzr2q9xbsg33iwvkp00y9n6zm5nce5n076a3w1t5zssqtcj2eiif13j43x1y1lkzvr5dkcv1","807":"rnpg23r7givgbh84h3b79v9hby8vu77i1atdse7naes5wwbxshvnl0clqygyweqiz9ttapwfv8i7822t3d78mpgdqyx4w92k7hq8","808":"xw1wcit5vkiss4rfl4l9srhc7nfueargrkkq4feabnr6u0qfaei01xpzxbpjvl509mojqwrdlz5i756tja4zdhe1y1e1fp6dz8ev","809":"p6agafvtivwpokwmdxthtqjpuyxnuxn3qbisui6rjttvygzvs4adjhve90ldmrpuuz26l9hzoylv8szu3lvuxt5o0k1uzvki7lha","810":"nxkaxyppo2dwtbxk3ffpx64kh24dmykypssvamn3bm2rujieoiccmpkbljr6omagfkft7s58r6wvg4wcywsdt1s6yksk2egdb6dt","811":"oomuiwy53twkytk4k4pgnyfn705d4yw3oil2d3dwdupo5hosjk99wt06imme258vu3488en755cahdhoz1s40dk11jriv6cdukd5","812":"lnj1sp3b92yb1tkwjh02so2au1xh9s9dnt8wlvx90joec5pd05rkp8drizaf4q4o0ttvrmsdvo7a4k6pyxgkqkliu6fuakpwh9qd","813":"13piog5a9ccdwys95hai0i45zpi4hba4jayluq3nberlgx4efhcup6t5xuam4y4dum4968t06djbq7ac790wr1ru4u6u1yoq9wn8","814":"o929wyn62ph7kuy31swozryb8aquvhihjh87mxqr9gpwnfjrc5ctkddm4vj2lh4ilik4sot5xaz9byzgyfbynlek3pqasgci7gma","815":"46787p8gh0j6rx0j8giaorybayi9ch1mocgsfrssi5hf6dhh7e7ibfsdc5h9ljuaugklnh6okz363rcvl8v7n8sz5etlzfklzz19","816":"2iuahayddlwvytjglu7gvgeclck09hozfigxrtgd7nt9gkpfjrtnj3d3e82a69yn7ccucgkg4xva8s5rvt855pfqcbl653yw0zze","817":"58us0l1y0ffklubr3lfg1yl2v9hfri4pvbqw0d3f7n163duuxljf3ys0rwmvvzbrzfmal3gmc8u5trfob7sy5jqhjlcqxaywq276","818":"17b0o7gx0c62paf33bkcyv23rtwfnbvhr4oghc0mx5s477wb21nhjuxelgfp5ocj7q9t5dsrpjxb2y7a117q3yqan6jzwmlg6fel","819":"uvv1mg1giuuybg6erauat1pp06mw9p4aodf4oyl27lbefgh8qow63e8k49bipod38c3zpb1v895oz333nike3arc22xsxpgo8zmp","820":"zwfj5c71gyxjp7heomjpmbjyqev1ykhfo1x5apdl2s6l835fjc93g507siu4movjdnuwy72rptzuawvjui800ea0drhnssb0428n","821":"svultd74e9mmrxkmd6kgapgo1ffpfc667lrmv3lwyhxy1y1pnlfkvqwoxe1gtqjmoz0s9eqc6j3gbx6729ygiawlyd5s7ix8j6r4","822":"akof52i2c3mg6eafhgmpp4sl3ty43gvkyilxkyynui8wwe074fvckbnpku2z7y2w2jq2h4bw5g7sm7n4ydy8eqxk7x3s6ti3aw1h","823":"eqou1m7nnx48er20xgtnvi7icz58f7i795gc2ia58ejx06sl6gwci2h38gtxx1z4srm4myiklakeg903921hag8n2qij3cjblx7n","824":"wh0gur5jlaoa615e6uz1mxoqjmhlymdhr56w6c5iaoiq985wv4k76kmuoq5pr44bg50affkepd9fuzx37atgj4tqrsfv6mx6ujc6","825":"coeniqnia55guj815rvw700cymx5wws3hsc5rvsg1giq5yydaf3yfkynemlycz7u8dnjhugjrgpc3ws2jg5zy87srre6xju1mer3","826":"vkskl21xwyhw0ssnpknclyn50gcwusl06049quikckn0c9001knol2d5vylrzgxc4iuy9tdj3rq7vez8v0o2x78rhtn1q4ylmo5s","827":"0dzvd005xdqumgbezs9x4as2acqjlt3pvro6sx922iwj25ti7bt6e2q87a5840t2kt31mrz6go341aym9x3y4uqv1rxqat6c1l3z","828":"ob84s8u2u1irfg529caqsi3bqd43nbbwpmae4ha2776telgd3nq4r1qkvvot63sxm4147ym7fu4q6hsuj59aror7t3k6ck72285f","829":"6znboph7a6xfbwdugjygsun679grjt8qs3zn8xrkgosp5g3e96fi47mx9fkftkgx7nc2pvlkjiyu6zp7e1psf9xz5ko59uwxovmm","830":"d114pwxo3zezexm39kdamriiqni42y8zi24uyrgedh6a3nhsva9zbfrb4bzi9vynzm5o22szb3ewgyxfrw0f61apcvlmmx3131di","831":"kiajxo9xsw5hjun3gn4s9ct3e57hvyjrnydlg2x685h0w2wdjp10e92b4xdj355kug6ezjhtxk606dx78k8rf8vpurp9x0cmtxkc","832":"88vvvksw4vl382dwj79zo8dmaotbcdd5i9ygb3e311hmhhkn33om6jnlvmk0q3bfuui1sjk1rf1mqzlklg3od0gn3rlwco8w9ez3","833":"m07y7ij6vvveq5f2ucvlq17990musn8uql07pnkvnkg5v6cpag5dcr95s7l1h1qp91rwjk22m9iatt1lrnsv919i779kbvg03odp","834":"qq4louzyny16ii2vblctw9rkny8mwufwzfg64eb7t0h7a89gfnwxryumq26f9t13xnmpjbkryoba7nhz1fkv2wsk9yj5rxx53n6p","835":"himlzxzr8g6r36c622qdl0x4i54y78sue216vchcw59i270zuaxqfgu1s88izbue3scrrku0kso7kplxf94llgpcgoxcr9m9csn9","836":"n4dsxsunnoevgzevtohu9uf1duabp1bnjwcsmb74e3c08o6lx7jsjg4eibh0mg89zc926uhif1h1znghh0tx5midilgrufvbyo1f","837":"lwplxk6ywe0gip9dy8mdqe8e3gxy8o1uwdzd4ts0k42kwuukle4q9wt1464u80fuaumjmbp06v7usomk16btuzszjzk98iwmlwn5","838":"g1ir5vappwoaczutzzm99t8uc6xzvsd5bjlnc7hnp4xknie1iro1ogtp0n2fhcsq64yr0bd1vdh2alkua0bhb91m3qxi0mdoentw","839":"kunx9jraioxuplpixcwtbuu1kw82kqt68w0s6ic41ujm9p22fxulqx4uguzo7ewecl6wk15ct57nc3wxy2hhfx7oihs383qm39d5","840":"snbbd2kugpjz5bl8r22769k45fqc9yizh2tjbdp3ra0oknn3x2kn8yc9a9b1qxp6t5tylc2msvbhtpgb4glg7pphfqxfwpimnhdv","841":"4ytirj87se6igc9zyn7klfl6k9gsiscke5c2pthus2dilaw6um20qfmbqi7oygydvr4oxrv65goe8gvcv2f04aotx20axmd4aapj","842":"6v5f0lwu247lhfybq6ms12clb98ti1zs4n8f0mz30hes7tubnna14ei9v7ri5bxb06k2u2bgzchwvl0w8ndshp883xfxwans38uy","843":"m9nijsaak8t8m96hnrfxau2851e0l6iqwiavydu6c1uylalqvuz09c315kn3gxde3y2rfiuunrmx0tv11amt5afsqg847hjmv0yw","844":"kwcogkstmcplgwblacm7j7thldipxq89dyis1r81lgxtuipgwpyy6ts85n6x5x8oc1kyxopjmq43u9ekxzhrw4vianbabfd4j14e","845":"ry6j9fwgq63ldpcmj1akbvq55bx9jxjjz3x4weggctrzksn1icc9klo10lzoqmgnzinkrep30ncm32fcf6pr2agf79wmav8j4hqw","846":"pthsvp36sklij1pmifyhetwwu6by6xykrg8dptznmesywy75b11cm52tbzf1c1cikks3m5d1kodzyrn6y5e0pi5q5au47pde02dp","847":"06ahzgfkdv5fz82l9spy87lkwicuc4h7c0m8yi5vqt001ur16v79w2b54u2cxu8dpro1uvk1xq8xzhh82eiwcq7me4sildsks3jn","848":"m4cibzqdvauyrblmb9mkg9pfto4n0s7iobh2vfw6l9anvvtath6gg3sqbw8wbkjfolqrr689ls2voisnpiqstspg7d7rr7znnb3m","849":"gjwr5si0tsyk49wp6km5y725h439tu11c2p7xffk70mutdbv8q45c2v9eok7zs4tvzfhflco84rv9fdykgq8txm1pmb442thtanl","850":"cvx36okhq6xiydnuvgh95rfxa6l4j83lpczbpnefipn7b4vi43y9jlw722gnsgt0unan7a07szylcgvlabvf39rtocgm9w2teesu","851":"me3p7rdd70kgus5r9alkkhgp5dlv3g222yyrt93uyiucgo0j214z8th7wl6qksxr1v2arx5ikpvosi3cn349mknkfr8i7gnglvui","852":"6xnzz7ztk3sgh27r6emq0l5a7gdftzjveez3feu4e566i3alqfl2bdc8c0385k6rms9r7e0tnn1h43drsekh0ttig7syws4xoqcy","853":"3ejael1wi815e31egsmwhgyxg3r2wgjk5j3oadd5s226oby6txctp5tlzpe9ke38t7qyodkpu0ml9fet323cbd4a92zpt6bdzkuz","854":"hvq4svzlqd1fi4fpy4pfbsrhrte9couwt7v37l83np2ppz414zs21401zondm6kqkkvvdsadb1ns1dkvtil7rspxc1jej2jmcq9f","855":"khxxpcrkul9dnxl4xxqq20oii2nq6pmo9tqv3i7eq9ystfgx5962gw010ml36kl7x1w7am4cup2nq79ahqbizhwo1pnbwut8retb","856":"i8vgb3yqieortrf29q9r17rngerjjlyi118x7bilghh728qtmq6cxlobh06j930a6pgjog51kq6we64b56enojj9djhef9fm0flo","857":"ynhxebwj7kcckdj9k6st9obcldn20wtdyx7pvjphjvtqj38w8q7j3b6gd4brzy6tjlr6v98npyzmbamxrguh3mpm4n3r0f0rocmy","858":"6y4iycjj0bo26hesytgd1b1fv91n2y1jeabyurp82fszqpfot564e9wdsg00q9wesk90131fueyre3lxdgf9i2xn5h0ba98t4d1z","859":"dxk17hqwd9ji9owy8whpepyw0sn3f37h1p76g6vk2xzsu2kzj2ndwv5lqr7xb3epbph0jf40tu2jnxe2rzriebzh608dcoueqcd5","860":"sm5579b4y4ud04rmfkanux9sye42pzlk2xq5jfsc9rqjhh8kqgahmaf618trnoqt5dgb8hqxju9d5ttp05xjfy2i4s08uvhpx0gc","861":"v36lao80kluattrof08zsyr65clz6t0dzusxrqvk4oyjj8wmm7ftv432ekjk49itaaoqh0m9plb7isf5rkcrpdebprua774aqvno","862":"uugd25s4aklui7bfg3o49groa0makc25kvahy84h3jfocsw95wnpmjlsoh57r9rudeyilrz8bwv3jsir910xzg6wbk45bv65nnrk","863":"hgn06oi1clcqyps7bnqmr7jrchohzifbro6lazm0loh1qje56po2vt0ub5nmqvoewtx7bu2xs2nyyaatn67tegyq0hb9739njxs1","864":"2jqxcfxw346gyu0uihs799vl4v45joimyvnl1z9gzmslxtz0luhy6hppiydshme837f4kr2sp3yreco3xb987grskvj36z0efnky","865":"whi07uwtr34z934n5r6xnkl6bln4rrdfnv3xoabu8g9x66u81noydr4r0r1rakuwpu59rn6axkro1umb7rs4c1n14w2di7umqhuu","866":"imy4w7pvtkrqe5r3kjggk0t6faunbk4k440nef70ihz8qb8qm28c4banqztnk1n9ce4xvhrnhevix8ekqhyat8b4ryazk7f18ztv","867":"ons4nonjdptwy1deuaj2na5yte9o4zjrh8f0k06uhb5gj0da1i10rm2kdzc0sw42x8itol82vo96smx9sbwx8br07pm8ot5zvtz1","868":"c393ku3gjyo77z0vmuqstftybu3dtrb0fhf1mo7h34dmmpe1n0gjll8czqwpo9o41j4dtgjfnd6bmai5jrgakupwwiqa6b3zw66q","869":"n568l81mzziz7szzznq0xwrpkizwlmuvh7upkr0kb9u7q4b2q1b3ha39mbxkpn47igo8idp0ij0iml1n5bxy0cpmkm8eaut30udl","870":"ogtpoatowrhmy0123j4f5ks9c3f9ofwcl9slxg1uvdwrq216xwmwhlwzmuxjsu3ygs0yer5d0tbw0nkat9047xa0ziqvddxhebhw","871":"ko76qgjb4dvpa12zmaku8859zmm4fdgq8owtmtuc73v2qmqm2u5nd6purwzki9atvd0t1xr6my67h8pnjlczeur99rzwc8rw7jzy","872":"7coyxgd7ip4mk0qt5vfrkpx3lzt9l7gsf00p10zufe4pflgp5qyzruiyek1955j0tiuwstqyb35m44rraqu2wbfmvfzkf7wrma2y","873":"4u45z67gv5g1aypwn3rkrrftx8soq651o4wo7751jcycp65nfc8136ycp0nm44w3lc2rn8ek4u1so7wtthwq21agxbitv238je4m","874":"cpxli7myffi6fnhelh01ie7mxvbe0an3coxtgne40p7tb8jjfpxx1g0s1fwi66yjfx13buhi1pgupu93hn15njmtllxia7ovqwej","875":"09crkdydost7feju6p9spdbaa7viri4xc3aqqu2ewmdhv6o9nhu0m2ubhrissdmnh8rcvick9w6hsj90fg2bdi6qdm1rkxq88yor","876":"kiuqrufiftkoxjn7vm67c6hsrp5rblv58cfhvc6554a4vgped5kvz4r7hh717oy6r7o2d3lnmc0yxq451qsot8b25pukqn4yspb0","877":"vswxed30ybe7zdqs902zd2t1dzzscgbdtzjp40hmdhjav1zrkm12h3n25tfr2yok3mpscc3czur83plwmvevrbygaj1ymypv2hln","878":"xk81x5393cpshr4ecwcqycm8hkox76u8q8hs30l6wc40rc3i4bw6pgvfota77ab9c96ol3pab21v8khgnv1myzhj6tmu3l4871yx","879":"y15236vavpu8hz5wd13yfb503h3rqi2p7ycbcvlo4y3obrfx1iwl3lgpk0ihag4n0murx6nsvqzpbuhfhqxcouu686crhles6tdy","880":"rqcy5yuwnch7tcjgxchefx77rsbuelq48jlrvwenmp5xxgx1cb749757bwj7y0cfhpq6hw87b6eytonysujwc6xda6xsa6hxevd2","881":"j4t9pjpps4w16cr49xla9asklu2hb7l749xiwqkrkfyv8ri8r7msz7o5r1ikgrj0ss2k4i96s876cwjq5xnld2mgwbi6cuctlxd6","882":"19goj0bo8nhnj96k3k3tuo3h7f2z9528u1ksub7g1v2tw1rgwvaly8ks1eoqoc297s6gezsxsdl7fgpmppojz7elpf9s9d5eor2t","883":"rq5o6kk7skv4f7kpdqy9uzet7deep3ww1v13tzz2f4i67bq2ox90rwevmyw3wo9gfwypd6lrz27vs5is9mu851zr3lre789xpqgq","884":"ejqudxfcg0ajg6ww1bi88ggrod3iaf0iz9mxfgny73wltv00t5vvea75r2oj2ngbkablj3tcu89iunxrt2pix8buu21pdo755rxi","885":"o2o3d89whis4vchfa8hy3s8iaoefqxu5hm6lw8i2dl8o6m2dk8elr0rgtaa7m4lwb70fza4fe5k13k5qzvtbzd5s2f0zefqca6p8","886":"1u5twzo6innl8xa0j9ydphy722lll4r3fo22cewmlr40t18c7hvsfgweqa3a31gx5gk6g22ohh1l5ly5nzq1bg3t965luvutiyby","887":"4jw7cjn8yzrjs6iha96m0grk9g62pprjsxn8bcccl6cw5njh8p3tn91iuu8smp99qpccv052wao6wpgd94ezf56uki37bx5y95r9","888":"hyxoclybv9ku4lr54xlmblccsow98428hdxduzbwkm5zxd6231aaltupuzt86g6aokehc5ehdk2imqroa3164otcbs39bmvm4mzw","889":"iggd0tc62pamn9zda4nvo91z3y53mfdr1j2i8wa70gzo6i86a4747lwb69q9oxsp3jphkbyf0mo0txcj9lwi6ekio9709s7kxhjq","890":"ond4cocj1fogu9vggviqs4b1dq6i6fm0qkwjknej5z1vobkck6woufgxei9ikgu3wjgpvw32zm1izkgju5aiguof0jr67l6w2bk0","891":"zbzhjykvjapn0ffs5eipunwljwpkgaqqgq1g01hiaokq0d7i4jv4b6heo0l5v3fynddlkcbdb0kpb95rkti3dr5rd0sk56wuejrb","892":"dg1lfws13pbh9pr2dxk7y1e19twtk2wapohvf1ng2htykouvi1xs4nud2l19nf75d8i79sar0v53ir4kma6diwtnulf43z1q7z4r","893":"78frf5felszq1psvzztahg1cu1nq47uqhdgq6azoh4r89m97ylsh7hzxbrngf0yq4s3ozylioopdtt69oqwgsq1os4vxo75a27vo","894":"yyq9gjcfo6hdaxfhz875flrbqag43yetrntta35c76mw9wf4zn99wnit47qcbvu39h6ofqwf8q98ooyrqa3a0ehgw728ux68950r","895":"b760qrebtyhbnlib77xw8b0ic8q5j4r78t4fytlhoycm4j64becaxvvrx93ofp9bik9vl369sd3j2sv9t2w3aawcoa9rcyhetcei","896":"q00r0ttfhpn229b3k1zwdhefpgwghkoftjlt895z7vfwegdme8ja63bfqcyyxiuioxvi2d38k7duob3thk3guo6v97c0j1i03od7","897":"70g9jpcg5mtu3hsyp5gcxv9ilurdsduwcxmnlv1ro371uujzprk9104qwryuocceuwn1lig90e0azxfdldzi95wi2ks3llp78syj","898":"mu2rfxw20ucev4o4vb0vw3txrabzkr6d9768pz418aggb8gv6j87ztcpncr9rkn5qz1ib81mjkfxac6y3nj9eyap2x0b8gqhazdj","899":"7fkr5rdktjbi2n6xzlg9pplz3gt3lhjw9ivu3txz7p5fm9p7vwkatjbt912kag9zyiab8lng825k8gntvqkh2dsy7vjhssq4fqhg","900":"5zxju06dovr2qijtosykn73h3kk74191hojx71lj2cyfy812pnwjsgoy74148i8n7gp9oyhiezismb2k8rs8qs840maoke4yf8yf","901":"n33p5l101r9e67pzbo06t76c34s53nma1qezbujbq5moum0ek1tarrm9zr6ydbbpkyen4w4ypjciurbic6h82qxedt6nklz4t70v","902":"dlngm279ftcrbrppzjekyiioqhn7qlwhtf28ot766h43jp89o5538r8exaacnxd10xvh8v9vpg93bbl6gnw8ac5nm4aueokeiccg","903":"7nzghcsqra0b9peetec1ruta0bmopm3157bniydmayrs1e2ejp2ks1svdnxza9tqujr4wz85683q9iq1i6vmqftje43tk036v42s","904":"l177h7hv19i2rsemlur2ld2t14er4zx6qbcwjbv0io8q7qnbk37cosw1044h7jz54g23bbx8h8a6ecglcxlfn0rmddzdyw7smrx6","905":"ckjfhe16jpg5tw6ty13f1qso74vresx9yvqilhj5o6r0sb689h2a4qa72b7vqepycbkql3bs23lbugmkr5zstxikln7y6xingj9i","906":"su580jawbbtdova0qeis3k00r427yuuwuso9k35yhxyhhcsrmqxdhoeu86ic6jwfx1d7e5ln0l6194edg0vze8pxegw9ylsngcpx","907":"x54i314lpyn9ysssttivm5lzriodluke5bjflvcrb9ec8q8ikfd5xlmoae2bfnzme1l1vitwhbwjmzv8p9j0iwrb337fwxqbkez3","908":"ql21hpxjdkchh17c0tls3kfn9d4mq3f4oajtb5u7c0uf2qu4y6o2clvni4zf6l1ctgwynhwceldoxoouxd78940sd3dsn6gx82a8","909":"ijz14ql7jh4exjffl4eeel4ujrbecdhi8ainh110v2fn7uhtdzcz5q9azx2w8iein4chrx6c46g2bi5hp476cyx5k32v2yiq4vk6","910":"o6rnx8nkd5bwpntpcprfry26jtxjtuvry41xnyn38nc7wvgfsxwf4sbbqfaobubv3wedz8252kpnn0aq3t21ljn6z6tv3c7rl15z","911":"x8bn2mfu876wssjwigm5ff39fo1cah2uw0rpafj2qh44jc2t1ru79fmteevayaefobnrni1vk0jpit61ooq2c1lknsjf21yhlxmr","912":"oxbqfo1qk0bcye3wls83ghcjwyswiridfnqzipvyl1h9pxutu9i70m4pduvpne61f96uak69ynymb8o92u39efs5978upl406jjp","913":"oqu2nd56p8o0iqnzqlq8u0ejom9esymdpknhbb2r34hi6g1y2686km782n1qabqhkl36p6f4xabtyhts22b4ql1i9vq2hi0foge8","914":"inhjqcuoewbmo580rrr9oyyj8wgh4f8kwelszrv48jyjelzzxz5iqfcw63j5mbr11edr5jb7wtdvtch0t55cmxq1gmr8syypmj45","915":"5ogeqdj02r32tjjiko8e6933lvu4cp01vafghdz3sjo94pa5i7glmroo2my02v49exy069hkkt7pp7e1xp3gzw83fulx6mi09je6","916":"udk662j3l8nvr0k7mzn9cgbrtais8s1ak4xpfu9i4lx88cqxus5b4z5pwsztk4is061cqu8k94p5xv9ux2w98db37zrlsfjvz4lc","917":"qmlovpejm21blsyse54nthlsaif5e4s7lsysydy3ag30i1wtikeat0yjrvdr29am3c6in6uueahd905x3cl72fkddhhh8yfweivb","918":"fl7h0pgq6sox17asr9kn4b5fgx3vlko6pmbd3xiaut69j5tcbuizmbjqe8pl1aza3u3wtnvq2rnrksnzh2wh26q8dzu3hx8q7uow","919":"hwblf64j4id3rxpvv6guzupv8zaz9s8twn7vs6g9rghyn4cilrwk10mvglkt3d7fiau22oc9sj3tmr7nk85vwsd27dgrnx3jh23d","920":"2z2pvqpyo7lkb3xe3p7o0jp6co77h3647iw2yxtrizx5uug3b0ma1nf792pk634sjvb2tgkrp4ks6jyyzgglv5rlyt1i1zfu5088","921":"41j552zlvc27eeogp5yxv5lhbsq34lmx18zsejrz66qn6u5b3ihucj6mhh3jqgctt5z8xcal6yjpgutv3c0l96l1d81lwm6r1mle","922":"z6ijtmfa82gihziwo4eolzdqv7bfgabnujctmzk50a2bnbbvqte5vmjv9m16343lu6289oq27wl530740x30yzaxismdlvf22gxj","923":"jj4e6cda2c1y0z7mg4369sgv1lzhsb028ac38kj8cuekd4frng7kx34paqx8djnl37a2nnf0hny5n6g6lx4wy2sbvcs2q7i1oii7","924":"vzxf4wyhdhh6r72mwupknbc2p62yjor2iumwk6th2hzfn4avyw4w0xjh5pkgltbfvz7ayxi4ukstn7emgqhylhydz0ej6yco9qls","925":"v6li3eb1ys216a9rywkipbzt0u4c07ifsnz94jyzixkthvz24ih4c6tdsgp07epnft8kspz8v4bwlz4pb57rmturbkt4967dd5md","926":"gsvkrsv5rxem2611z7zfxvxo0bqojjcgk133vrv9m2knat67kzs5g6guo0jepxmtatdjazdu6kb954wu3filgxoxa9f3mzz1e8fc","927":"csauue74tlsmv80uu2wcbl1adsmqlde7mguktj9jyqv1s9v70ddmlf8p2vsme2okf6sjach2tb1s8hxzzweq36e2gm7wq8e6s8tb","928":"p4v1pelvojb7z1jfbsbur4x3z8fpsn5mli6m0ozmtkazt7b7s306z0uehu67fr3fugkggiydgsm86s7lkvm9dkdpgvyswo9u24u7","929":"07xoj0u96ba2jz15q2qvkuap7ksxjp2u51fwwsygvrnhiuegq73s7akbv0ziha35qjkrk268kn1okfwqqj6hwlbkc6qyf0s9y140","930":"7v0wi48w6m23qdrpu7jp9a2um4i0pcb6yn96m319iyx4bbhs6as11cvdrezy5mm8o3nndzkjffpg2gtc34uk416s6okuuj5493zd","931":"r7lpzg5lxo5jmk2fm7qgth1q11qg2wh5m4ao5c0ckv76h9vkdfevnlfqvdvksbhof71fj65vp0pmjuti68ppmu7bdqkcj44dx6gh","932":"rp8y89heyuebn40g7avh2q7rzdm6y55vl33z8c4sl4x8sm2h4qfo7kkhdyo19ddoahb32r7v4focibj3qynys98ukahj7p67lfks","933":"zvcicttq9yn2n8n4ht7gpug4mvwgicpg3w1q02hluzmwjc2zflsj3170hroaveg4dcidevsi80jc7cgwt1wz9ojsmw7j2v3zhn39","934":"yh4q8o5dn0gtj2jtqihzj6pfngrv5chth1zulxwqzg7bgz9y6jhlviw936nicegjiugea0op7at3mqlevizs5xzrqc2peysk2tet","935":"aprfep21knj1g8sr39eo5g17gw9skitze16gomoz5z7ioqu0912lhkoll5w6g4o7ilxmsaawduq2vrp8tzadf6i8jdxg7xms0nf7","936":"3wpuwet9f2toouy3d5hyqiaggj4yrje7qw2bbhfwsmc7zqu2u0kpl3m0zdh3u7rfwdmxp4poer1mp42rqjpnbddhlqyr3l1g1gp0","937":"275jycdzfpwi8lwonfgkvt2tujcp7zy6nb49vh7tpriw1hy6v36uuwp1fyxj3fwqlblmt9yzq0gtjojrrht1o6n0izkz9itpbz31","938":"b002li8hz6vcvxhkcpztfvzx8jcech060rv12248zcelijw85rzg1l0osesptwa7xiud2dn26w9wkafxep68mwn8hvfuna5ltivl","939":"1lzdqlfu9vi4ljzsiub0d25jib0la7l4odx905btb1aiool7qn7763ehpzjp9knvdsqzh9ui7n5i0hgotarhb4x2q2ciy6aqmutj","940":"vm2kiq0t2v30mvxznicbmm4soa15j3n2dc9bdcha8nsxf102w460jlprjto4k1e8cmiofmfvoym2h8xfoqf61c4vnfoz9wnxrza7","941":"nwpb1rzf8n4hg78yj5xaz1ahvl7fggs2iq9jh8iz2akkuhawbopybbit3o7gzqau72ulffm9y8dlcdods8fr8ja1v8vf3mzpmowt","942":"j5hwv7oprjqy8hp71l0f7y7eci8jdlitnozh0ihpz8emogilltd8h7qq5g4va2rpd65w4xm9vwcrpafw8erk5w8j46ibwle0umw3","943":"p7c4q2gmpz5qqs095oojezv0gxa0ge5yq70vmrjlzcrtme8e0jasjcdrkl9y1ffhlsqqvmllarw0yq01016b4qgeirls8iplhldl","944":"8vsi8lyjfaylzntpb68sez1403myrb6gvhswxb81i8pyjwx4988pra0gamv8q7133ba0anp9amtw8x76ulfsn8glsv3drzo8w7b2","945":"2mfzcvszl4fokd3q1dm7029kispx41bwaba88xdbdvrzm14qgj3zkpz8vrsl1u5lcjnc416c0rl5mhjsav1infxpu2y30xsfm9td","946":"q72icomj7mavtb32sko7f9651ounxo3dfbu5wl84p0lyeimct2pnf411uiv71n8py2mdjttoj70f3qkxzlaf4ksy4m2a6u1rrz9v","947":"upiknl09vauzl2sy9ppsichcewv4phgd5hn28lcuxsjgv1bt6w9huqu6r5hyn0hnfu8xrgycxy3f8597jjqugusbd7ht1jo3i6ej","948":"wrfvqx7vw4chewy0g7j9bs9lmkq5f1xlpfk2ve4z4onu52lgd32ymlo4bkt8t1pued2zpweuo0maoxplj73m7ezhk82shzg3cvbv","949":"sl4fcuuzu224axaqdyb3ld57f6ljjtpgrhxz0cy49tdf2lxnnfn5jsaj9rlt0vx011rvfarmi6ak00353i5c6am3v8w1d94vojtu","950":"d82ag8xaxriuu5sdgn3o7od1e4qabpssai6yu2u8ucfu3sao3owkm05j0lfwo0fcz2peidt53n949glwe83l8y470oufhyc4hv9n","951":"2k4gnpb1fr7dimp8i502mb9lkwwacq2e6mzdlj6k7wudjyr37sktfiegos5461wu64h7pl1ofr0qcp3himmph6m8qq4wb8k6anp2","952":"2g3jmo4tx6scr0h3vr2kgch1ehwfk9ptz0lvt1fkgl61dsobkajv8qhmtuhuoe2kheg4n6ldqafgphvoq2ss11et5iu8rczu9kqz","953":"1vncmql1q5lx0yo87wda1sgz0c7h2032uuy62ohmb92waqrx2ntc7x0ymkv6y6iws0bsrvq8vuzb35k4xbsp95w4l9ucnlrr7fpp","954":"azbotrm015ip8jsuffcwr2avrije935d3u2qex6dpdqy096w0p3k0e7x4t1oqjexbhac35rgnp79p7ctkcgvuavhjh8ck060i01p","955":"e2jkrd4phyc3tpnkwokjk61xf8gm4u5zhzqecwj6oci7c7dfbhbovr4ssiw2uunjx97mpzx1va67dcss6du7142oibdjb83ct15a","956":"mqev7ssxivca34cmt9tqhai0o614b6v5cnxsbe04gttl6vijxykd0pb2sde2b0y3m4jciery0fkljvtiwgx2vituxcm8pd024zq4","957":"xttkp1m9k1l64prygl22fk1ox16io69xu2fdpm8kzk1t1gecf17pis0t5ahpwfs1r7d49b4o4fsvej2eoz9ncvej8db5zsg2jx0i","958":"3ech5lhgns301tjp9aas35zmn7a45q0ly8hk3rqgcqaq983zc6dhrlc5zuer6auqfkp6krj94t9ois8b3xqop2sm8dz5cu9vgf89","959":"uv2o6pqtivwzy3y0dd5z8kap61b34xkbihsnjwmsku9zfo0ro2gbivh2mc0l38tr0iyjucn5xtzte1lhr7rxva94jexbux7l0k4h","960":"y3n4b5b9twfg8uqzo4l6fa1hmg40w92yedqke6ujl91gwu0xuq447od6c5j060mzqs17s1d2cz9rergdrp6xppgmiu8aj0kvsj8x","961":"uf8z8onb6y1yzz0s3u9db29v712zo4ui01hywiy84518b51gezzzvnjmexau0r82oul85fnujocq412rs6h7fuockah5w9crr6sv","962":"ie5qfm7u7kxmvkueza3utv7djb60pud4slxoa8kpgmnsufdahjx0wnz69byzf6kn17tilhclyt1bfa0lkqn1tor8dvgywcl6b0di","963":"olo3l5fi31xlnbkinewwgajfs4mjxhj34mxqg2t6js6hjlkqkc01riuu1hxid4d3z678fk9liudrh766h4q2ymep84w234mgxl5e","964":"8ikxr2zl1zc0ywnn3zvpe332hxryitdio0m723l2m6tozrbe8lfz5e3pn7lxztudfwtwmjb29dyg8vl1jfzfykpdcn05f45t878n","965":"97laj3w2kakt9fwdvz1vtsqixm4pflyugdhf52v0x3ovwoytyj9buzauaq368413ohegyfhnqii4ydx9yjljrqjex2xpfbb9g9be","966":"z2irnvi0vhb0gz7yz57lmjs3jun4an9laiuj7ybynefwgm81zmq3ao0qzqm7498scz0o7fkj0q8770a3x5gae8i8rqcy07j39ezw","967":"z90t6i9ontiudkjvoj7yap5ur8c3ub01lowgajdm6swdw4dun2vz6z8g6113ieqv6m4z2n51mpdd9qzcgmfccar5nxe74dvylnkv","968":"271r7419vorchfbbik0fnc2xbs0ttx73178hgbg72sc6yqtgln6dpi24qc9tync768780pr2uoedtek0fdxwki82ravh8mppli9k","969":"5vkjq6bfu76i8i9z2yh9007v0eotjqk8h3p07dc19rnkn5qrg6g9ki3ocq9n5bgt91415a44ha23y5qflxn0t86qor1bpubrkovu","970":"wq3hvlwovjulnzmk9ai4gb0qvxq7u54c401fiiacjtj0znpbywxoeejrdch4jo71s9qetbjxtec8abkfqfdm2c4t3nac5tou4uto","971":"5afwhw35f8hkw2xx61op8g0ol2v5jgrn4uwputrgi1y7j4rkuqbppo3hlxgjco8hk1sllrmth54p2nnvfr9o1btj7vxbf5s5z1uz","972":"1psfzeka0945i080m2ribwzu31a4867wu8cgeilabzx8bi0isodjj5i4syi6hdik808u7rvrt3izk2oynhx8s6416ajdtzptzbau","973":"1n7zm8qcb1fstphyc6c9zzok3ic9t8batcp3z3k2qfw5wj2xjmwsj0ihxg1z0y4fk5mewhk09z9x0olcnrv9cmqi813addm6j4l4","974":"5x97linfzhjci7hiu3jzbounj63um2v2rnbc3exbdock3qktfku2a2lu6gutdyz14k26xjmzuoh7b2bu3cg8d52eixus0c5gcmml","975":"hxv7l2b5fezs22lh73glnwo3s319mcyizc0kpipt50l7yu3kmgubr60mrd7zrszbuu875ugm1nv1ia5gs8f1fb7hislv6enu0mms","976":"x4szkh45mwdx0gawtx5jahsyt3punczvedx9tepasygbj66rvqwievl3l7xzbofhyb11t36e15ylny92algz6gd9g6fmrff2b4xv","977":"r6nbo26ftk1devjjat82zcurmyw1cyezut4x5hs68rmcb2atcje3xtmo50d9eki1qy2ckujkmmr02fsi36wxgufmnlquy1nks75u","978":"36zp0t8lxr8tqc482z49zffx7cyvzhja1jcjcuwytf5kulvgr1b0rwjnop412udorwwkqcqhsq9y6j7lrg2jis3x6ndszsvaeye8","979":"nce0461i8vfchan6vmbardg6lcau456hg158adph1194dahnglhmo05zg1ue58ih349qxn3himpkhdsqgo4sxspz2y1gbaiy9sgb","980":"elseuq2fnqvd788wurf08y7sec5meq3bbrjbc7584f2xaqyb1fcar5xwm546x5xejs3pge809zmsnmu3xin3os5dnbrgrna6wb8h","981":"1qk5cbfs4qpzfy6kdj9ss4bq4mq2446a8aav5eybwm9ejew9ibo5ne37bgf8o3tqlv18owad13zkpcv5p88614z83vzyc73xsa1m","982":"4oxuzrr3aftaf0xhhux8gkkr7awhoycb5m837grrb8wszjv11961vbpunwf20b9y97w9up8ifsuk0yxxa9ksgw479m15oda2w2kh","983":"3p342xl8ivnuq62k9rkvpcwom6wcokj8qmtnncrflqfig5x14m7exnixxf5ryv6dv927vsolqleq3blp95azgj656qizrvlpxjgb","984":"5dvr5j9z1o1cvct49uspjzwdlnet2ydp7gauj0ejp3du6z6dq7ljx4rrkvoaeq8810y8g0npur2uakr39tu0o8kvf36xlwzix224","985":"72jxwbmjtp1vfpnok82ayra3ccb6lgqqqvwujn1t2bxhp3i1gzgz8cn1j9n1bp19khyqquj5kl9vdbsranlhiyxgio2a766akxcl","986":"eroaw4hee0jilvb1r1syu4sfodddubmvhdopqf7zn13u9g4v4tles0imrkt7fhcvrfzlo3whulubcvzccm1z4h2h7w8mh7s58n9i","987":"2i78q6hbwtq5agbyl4pxvcr7vgx56yeqerugpltylr9j90uyfy42pufacn4dd85ki3i8sgd6s8t6uff3ptzs61o1kcnlghqcb9x8","988":"gpurhv5mrw0b7we9f2gan12e3evlybka3b2tg02euts86ywq3m883dal84dnggyuqcn9pidfj81aett2g18io8wlh7uxpydsbcyl","989":"o7t43sjw81jid1we8hfjf9v9pc755777wygsh0fy2aacnkxuguipr8sgpt1y67z284wm6m3oej3gc6t53xqwrn36c3i7yipc3tv7","990":"xuob6idhp7mmpkmc382bzn97nd4xu7fz5x82cp1ua267c3xylhbm5pk4wrof0wvv65lfphqmmvkvjvcywwrvarar56lm8inemsrq","991":"2yzr8wfqcnodh6awm8zfqmk4hj8l4v880amt3xuvoep2uf8bdyv9ns59b6k6zrxykv8mse4akodyy5ujnlhqiemgzmppege6rtpr","992":"8bd3u9clr4jr6se539r7ryv8quiejigat9qq0nsa96ejg616zbj25t8fixmv4kv3oowrfw4sgggqroddp5rmlw6s1akzwkyub6xk","993":"pea3ivw39ouvguusnpwcm65b68b92wwvu63w915ojfivgmbtnel49itcgo0dizg9xujvebod8llehkztowi2uu6ntpsvfe0qzh8h","994":"19wk0fkmtneg0cjojqduu8uk3l4e55y4hmpnd4r29o4urhe09rl60v90gcb25h2wrbhihtdvl7vle12igl7i4kpgh64gwczpgnji","995":"s9xttp24pkv7kg0e8tu9kbzwt2k5dylc2or9d3bwnh1nivftaudkpjp28aw46bjy31g7hnv0fblyiqmcos20aoa398s5fbk12t8j","996":"vtzqwkl7ehvn7x8k2wmjgfahjjc2yfiq52wo47ysp802fa8vmr75x7cfnbok0ev8ir0qxttwl4oi7tir49d659o4dvly1wlqvddg","997":"y55bwzpod8snrp683az73wwwt6ecvnra3uqugca4hml7ojdaf32rl7si8b04evxfurerlrnkvheuxuc5d5lhzs7484vwiof8qu8u","998":"90nzvj7u7yvn9xqeiz894mdc1lica5umaf833ia00t74scb6vo3wo1hxz6g6we7hprrix6m9oxmqm2sniyjy58zqykw2zq5u3aqi","999":"vckchrmp6rpyv6gxb4uh6gupkowxutwheafxyadnm6a8takmy6aiswcvzpnwqn1w3pwqhpj4u5ru1cwdgmgs48vmmvy9ry82fvu3","1000":"316byj1gz6u0jxl44sa6w4lxbd7sxps0fl6tf3ndtymv3ukejwejjc205xhjs7wi0kydf33rmfa27vem8s8wjxeeowh82pffqelo","1001":"1vwdfr8bvyd04hgcrn9l85a9s9affknnteb9cdgassxqu2pafxwc6ey9nlgnr4p29z6xoyxob34uzd2umgsbz8ushoverk74tdbj","1002":"2qqoluknn1sekkehllo548e193c7sm67tedpmuj9ckn8t133ysml70kv2f00io61o8wl03bcgy13ohmag1oxw3wzz8oil8irpssd","1003":"2wh70jarqyittq4vdibof4io8uq34sx4nqmfda7mpqvhgdyy2zr9u5r4wtvyi9xwv6uzqqdi6vu1gcn91q8xdu40gmnbik25t618","1004":"af6qnao0exkkg2i8uvq25z25d2wi40apz6yrrganzwvwf27lc5i0dmgabpch2vcx7jq85ol9gto11rdpow7n8auwr7neadnm91y8","1005":"mb9c1vql8ix3fbzd5iztkwta3r43q9j8g9l3n6ldbyartll1107c6yy54p30u4l4rfvdrzc6e7ank7m56vla16d02is5il80pelt","1006":"1vp2vms4h8g52ofwkdrpsrfoikikt2tz3gqk5rg54qswoi8o6its3x448sb7dop3pz4fxdpey4gfxwv0x8bk1fxuyz82glp679di","1007":"kax8ccci6m4318tqd4ail568zuesfejyz5riat3i4wazuyk86utebp2oeif4is2wcx7k7u8nfr9los82f34fp89ob02fgsae67fx","1008":"30rtqi91s7c2cfcywdgcwms73r6ujp1hp84nxoghscniye411e6ctmjw7bku9tssgyvsanuore1kr557jt4q648jr9z39wmy2oj4","1009":"d8nypgeq1i699vb3e9clt1elyuols3cbnbqiy4uc88tdvdyre424qhphysxn8sroy08utdkaoo3tto0rbv00wiphb7lav9lytxa9","1010":"q21maxhgzkoju0ltmlvngci5myt758ayyk0n6hpflkd3p6cbfhvc1gwaypbiby29l6emxwzhl3fextqfpkaygi4ke5xy06nsgprn","1011":"57n4v8bx4xyxb7tulsq5p5cw7j90vu4856fs3cbp640czpx6t1whb9mlj72do85zl7hk1a46ymcwa1wsu90vb8fxnuki9g2tkty6","1012":"42m4mbk8zdn4o2j8fa9w289srb2dvoqd4yfgi3rckqf9mqjtv7837y0yxq7xz724e9m9a2lx2hnp7i3u0q0lf6ayxpv907h4xrk0","1013":"zfjrgz2q90kef7u2is3um75lecidx0nmon113vu6pntbq91rra2xhskacs1mymcwo5bdie8pmtn3cxjlpbiljfv3iooqw0obb1ay","1014":"fwm46ndy0sj3ox2jnyljw9j88s0ghn0kaovetyhhlgnc0ye00knv8gv6dmpc2j77bcbo2goftdik0f0z94zoki7bigs3zqf6qa2n","1015":"fjnjn550gxegqz13lafp5vcdibzh33zx137nhdbyamu4jop2z207vzfltkwwd4oumwjvo3higbr4soyxsf0lo861embklviwopq0","1016":"mloj5g4hetidzvwb0spn20ijzfcn6muqk8cpspirn2eo03faamegv4nnikly93byot8o8454wh9ilwqpiw6elapp3mi0z9g2q4gp","1017":"kdpkoai3yqhhdo9zazh1ek6jl6wcq1eq0c2pfp0r786cajohiod73kxei81lhn6488as37hw8chr9m1l49bhq3v6rmaufys0dhpa","1018":"d0zj8ny7ig7c1hqta30842yz9z94aa8lsqg32ycb0fg1su29l73o9q4cloe03urozdkgkupf7p3c4yqk9iydqucfuv3ypw79wnqv","1019":"meeh7n1ie7a8hrdeuxzjhjpal1vz8nr4v269dtxt445jigfxt1qvxvy9q723b8fdhvlpqq6vwla7fxyaqcepa3nrtdo3q0fgoe3l","1020":"z2agdvx4wkpv93c929fwj2q7tsc2fw973ukcvms77vjc9ecyzzl5xc7d66amq0brw8wytpd4q9w5l77wattoa9p4hcd08hurrt53","1021":"txirvrxbc3l8huui1sk3xvwe3xbwsfqiuiivvuxqm1epf0zl1beusqhjb12cvrxba1d2zcorqw5geaeemmlpdopvsh0u78b2tfv6","1022":"eyqodex4qlozrt5bh4tl4c616hp6e05h8n4bzlrspleacfpa5k93fubgi42ki33po4gf2lfdwqr7fnr4c655faz9infp15l4pqj6","1023":"2iojlz4qcfe1ur8lq0a7mrt9l0d0bf5fubf5102e8ofo8k422vdf4omhsb0elxgif8jor8hwqdekole5ikpfftvoii6uu05472fc","1024":"kcmfk6uz12upomjoq5qmu0wz630mha2z00g3yf6mb3uxu51qh8kr90r8e607jo97y6f8psg688c0fy1jrpdxq0o95jti2bbsheki"}} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json new file mode 100644 index 0000000000..f16c3e3e5c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json @@ -0,0 +1 @@ +{"id":"add9df95-3525-412f-b782-a0dbd5d0bada","NonSensitive":"dfb280c7-62a9-4d0b-b0d4-14f311fe0ec8","SensitiveStr":"ecb4222c-d335-4508-a051-654955d9465e","SensitiveInt":1692274068,"SensitiveDict":{"1":"9qze47js1zpiabsm6rvw8opz5uy0g71bvna2nv17yv3d61iegnsyq84oh3alpmvfep8jo2hhjavj8naad5zflzolsc18iweub7v8","2":"5tdm5pa1jl2ez2jvbgm9egtkincfgz5b3xzhxsq6gvwa7d65eyay6zymriq9r2jaosid4fxwilj7fxho58c8q48zhzr4vxuzv26p","3":"f0xmwqestj0d1h1es0g1rg9c8h9owc4sdhk5ul1ycnrmj6ic6owkl7qcgjs4ijgpc5iyujxiy7rd9l7ommu16vfd9nv2xuesgi5e","4":"847h5ttzmrm3dg7ey6bsid5cszjj73x0sno0ewdl30g09e5ebq8y1x91eoy04ua5kurxg7rqpjawbdtbbojgddkadnzyhkef00ed","5":"0xrlp48ceyvtfx9299c7595ae52kokox9qv5ecbjxedu5527kkqk7r0f9jpuw0xxxt109grqlem1xxsdapd1f1z44rvw9v4ggc10","6":"nuun71pabmusdv53vef4vygbypaewtpvfakbzqeep5fayihxjh13nzlhap0cdeyq0a8fi55j499gaecfux5bv2r3en4704lkt9xb","7":"d21yrhbguvwro2jzmon21trvbarj2bc4pwa5cvxs5gurhefvx5x5ufxy2ld4lv2jzgqiyhw6r6jl7620v0zizk954czfczng73c3","8":"lbvelwr350ujsnht9c3f7ckyadbq9mosbe9obfnxu3438yl0glgrg4cosvvl9gag8zkj4tscnx8pg0sssju6jfcbv9r71xpmgof9","9":"lixhbvlktx3s4202rssta5dvgtjaxzuhbrmfrllox39cb5nfau4cje64ag4kdiu6336bq34ci06c9xrt1sear4q7htvd2b17wzh3","10":"mz7umqc743raldw7tink26gqrncj3zai6zqzzy7azjuxmp136cge2riy8crc8bruwqemov2sha4aoa05zg1ahlfei50wfcj248ky","11":"tnrt4pwho6wosdtt7zz72vyoznxykda40atnyr5va1vgxc090t00qho0yy7iu3rgru7osm50ne9qth62v4afhhr31mv4banlbsui","12":"mpaae9i713xwtj0hatveyh3rlxm97emodpckgfffa72s7ufzt2tobh42dcr8wmhv6neojw8a2fyp6s62npb0k5gar43iwrlca0x9","13":"fdneqqcny6ubyirv96rbz1x5zknqh22afqee66vxwvy7dwc5u0hgjauzzfadd563bvmptedj181c6i1qaec6y9fr12xpgx8lqjec","14":"af5o4bdkpgno2oct8iot7fovousiwd19lsct2h03tbntppvw4qirhyjihnazuggtef5vqn46pl3jrviwrddve73co6gbr4qzv1jp","15":"zwe741if01gtebnuseeejzioe7wzi3h3y5hy5tmavagwqtdiic9i75yo0c64ezballqoh313vx7ga1k9xbixr0lgbuhfu8ksrj4p","16":"eahflfvs0x5mkvxia338mh3mm3x59jknsam3gnbas10vutod1dw0f41blupybowil5y2kn3oslsgmoeikeenwx0uap0dv0co3178","17":"rdwythux8nqx42bhbbzsy7bvjcgevelktbt5aw8djdh1vfjyghzjlyq88618txcz3qjogm6j4lfxkdrpv7yrwowsblj6ma5bj5d4","18":"39q2ndc6wpxm5769z7gsknrrkhiu3bvrxngs1qdmo59sv1joso1tq3z5ozzqs3obyuoff2g4b5aztum9eesr2i7oc6m31cbfll5f","19":"k9hzf4kdk8szu0qk0mpbcr6dll8jf1r5xlstooqzs88eejzhr8brhael2l0zgeiv6whpu85e25x642l6ps2e2n8jb0afzwr3b7wx","20":"qp7m3l10r4chn05fwomvd8udambaad20gy445xsiovuv8ewhp4z4rltul3syepqhepqad3katho1kufufeybbzjt5zt85gad17zy","21":"v97uza82ru2jq2urqft4kridnnsruk85n4lndw8otyjn8n7ct9lrewomf5depga2hrptu6yqifuym5oye4gkp0qtrgbhg1k22loa","22":"1v30p1o5a0wtuwp2mftybbzphlg3xpw2jpb60zepv49j5tupvha9o2hletqxmv7vl4l8vyyq1bc7rl0ifjrrbff91e6fcdje7wet","23":"xif7ligwdstp3a3v4egf1cgioyc0xdps9pir8i4tgngzs3xqembjmzq4xrz4odzruddse00hhmxrfuicx55ig87420pouttiiod6","24":"lovq7ainqta1ozmiyqzz01nolwny3hr6f5asr5te8l3eog4uwfqb7adrt4wfssdofci1oshae5cmrnkomvim9r46bs8wkjopoqni","25":"b11srpyex3nt1sn9p30joh4in5owklvgqtdmf9oq2se898k88zegzthkqq7l7qi6acyp6h44pdoqk5bj88l5agt796fjvsxm8169","26":"w27wt6o22x41jg38ha2iqu82oiruw37w1b1u5macmxqi0t3kdh41vfu7bgf13blr4pbd4svsralppddft7k3jdpz7dfzxa49ldmt","27":"j9f9unyawqa8ci0be4nqsbnwi5a776c204k15ons23mhys7bc7ykm7ttecsyxtrixkuntxlk26qpgao9u7s51r857aqntcxt5ktq","28":"1gpsx460e7spiuc30enkygbltmtglur3a64st2ee5d7rdf5bx9xrw0hkfqc3rq293mvob6b52b7e3lk6to7bffiylgstippypzog","29":"3c4alamoof07jbl0gw3an2e4sx0jkxl1di0s32ie1jr5yomt3etdj09ck95r0h58p8vrhqxoqetdw7apxvlttbucx8tkezq8cr9x","30":"j2pcy170lm6luc1nu01ugrlcxsggpkit3sirrp3nf4qvf470lkgv2hp6c2vqsbkmto7s06twff9glywum2y3ivl0n1k333it9du4","31":"cgcli39dzt0hj0ursaxakfkpka9irepzvbsdx07ogoyl9shwhixabk1kahlvkcpidc8k687fwj9g08fnyux72ai54q9zs1w62c23","32":"5byc3br3q6l724ituqdsk7qvvjny9c6yp77n6a45zydlfgi9fbbv17ho0xthkjz5is0bvzqkcqwdf24gd3h6jmm2pubp5rirex9p","33":"q9oo1ccklal9drk1lr4dk85alx4re21xl01p6o5ofisukfscryqklxiok34jtq53owqigimbii718qvofps7wl0ecln7cbf6jlbt","34":"5ckzpge6a1cg8z5cicknjs1x55n8b2lcy17gu8uro97tyovxxi5a130ubjz19j3zwi3l72wv3sen8lq2k87acium45kx1cu2qyof","35":"tz7qg132w5qi569inb54hn70wwy7sbufreopspbtdm01elq1ozl9qa90pq0wbg9aid03yyifr4yk711pjx7z10lotp06lupypwry","36":"73dybhdkumsgmoe95rruec80v1y3qo415zxrwq1ekt6afgch0hofkmbfrkeqqexduv0rgekc1hbtap55nollt1f5sg9clla2xbgv","37":"1aqcjgkm3dxsiqa87kjwdfynlwqoe9mdc29felg1a20pmo784tfr29uv25xoa6icelyxb3forx2dw2aegt816ped3w6z7kl8tupn","38":"t7zubumt7woqcnv920fw8v959m1vy3l8yllp9ywelm7rgfb1h6s554h754bsxpdfjglqq22cn0teuf57plfbkcrtatcn8g2f0sy8","39":"qp4y1bpe55xzscelojvqe6omjus8bevbut3haht9ufl900z5w6ddfudq9c6o264f0f91iamaye01w5u7z749fgoaeaft0s3giv40","40":"hk99cvejeyshfiw5jlmecn9a7d3viwrhlcx58w6xm1v79c1lpu39pz7kzs0ubo04tcgu2rtls6dlcv2xou7zd9zbrlygra3nzu7d","41":"jkh5uwir6epqwj0qnt8jpbj85im6iibokeaqdzpp8585o8seuoje3ioxtn5eizfx408i7p02cogrllqxanw2cz8y39q9011v94c3","42":"dtfp5kw38hfh6te0y1rrgoe8qk94pqp7cnqi5vin7a73254zcui1cjapk8t8dtr5gsklakhouqm4cp1bz25f5u80zs4n5bp8h1d1","43":"0leckg1prka3sbvbhya0dk3rpwqcak8rwejwh6pkbfmc68nh1iwgfnydyrxhz2p7ygszbel2h6atla1uu6gqyze6q1eduuhw80zh","44":"gtiktm0lqhgzz88gvgov6sns4zkmibp1q57gk3vupq6o0t0mx0urmuouoo24xhk06sjelc18eejdb3pwushfw123if9h22ecpoc9","45":"kzptbjo5rkh7p0jyhrcy296hay7t8n6zzgaiorsidh1oomfpb9e5goohvhpxw6ynaqpu6cyb6nhjexgxgxvb6gam3bnui0ii0355","46":"k3q8fkon89ppant3swis0tt9dcvmk3inyzurrdc5rhg13vn66eip0vuhq2to5bfb2j78wthudjcz6px4sdg5lxfh2myjx595biob","47":"vj32urvjr93i45nk4t9m00lkhrbjrrtwqfogn8xc277iu8bbbp7rv4enwbjtg4kg1n8svxy5jejng5tqzjw0z1nnhje1nrxrdu94","48":"0fcgvl3dlrwbjkk44r055j6vmkleqzg8cbe2ne4qxq2plrue93nh3vjd3zh08h5uyw9s95du5b6kvnk8gufzmb5q8s3bjemfy700","49":"c75714w3b5flq5he0uso6wdgybxv3b2t2kwyniajlkae1bvvhsyg1pzcvzi3draxkrwfkjdbzosw506wpwywisqe0tstt8g8ta2s","50":"sg3psireybv4etr63qpur5m4aezzeopo9w9bigh6iqiwndpx2dtij4rjgj2dithrt5cg33qfoj9f7xq16c0jl62jvwvjoarrt8xt","51":"3g9efb5t9b0ty5dwg1wc7xudwvkfngx9a845z2pgx4c5rbdvgovd2k9l77svabimpfbpznqeryii7rdnu0vkeeb1or52jm0lxcij","52":"nvy08acdgmhtiy0iczl1da8o8mrvomjd1tgf4y5eboxfvrrmcf0q60vk6f8tcvxwyoukucq1w4l2ju31yu6nfwog52rjeoizqpbs","53":"4ubre02694gvfb6dqzxut4t1scpuojcswtrn0xue0jw14hxhoh15xa91zk7yhrw4uhzlj4xsrf7m18pg3cf8x7kdx9negsqa2yd9","54":"1ia31ra8p4yumhakor8w4ck02c230s4hacyynpjhhxzrus2fqz9ke16lubyxv9i6kceayfqjblu4diaplh6lja82gs4q4q6w51hh","55":"x48ku1h3h5o4omqtw1kbbi658th9bqrb51u9ud62egu6im7kk8dt8wz1oymgf04pyk8c4qj92346ahrzptovtapmr4efby47nwkc","56":"5uotabm9n5byj9gy3rwsq2hsg2taab7by2698etbj08t9zb9u51yf4i17qiio1ng0t612seq8zcz60w7r9eyeejvucg9vgg2avsy","57":"4dfn56jtup5d3mx7johhhvbz9tefgwlouslw5gfce8xxuqsbkr2x65ykntp19j0fllnxkunk64sqj80k16kznehsx2rli79elba3","58":"dnj3du5zta5vcqsz7plwe0082wk7rn2me04xunjfkmihkemgam6gsirhfadkyvjopwvvxkhe84ianyqkf9mceur97q3mwv2dbzxa","59":"kgoj6765r8xzs0106h8cwz5vu7f1l0ddvofm0gtchldwauqu2yid1hywkeru19yxk5vnjsuv5nfjl78jzqodz92b9geptc1prw12","60":"tcaue8mbhkjsfmftnhy0qg22ccy0w1fmy8d1i4xpzm9s4pwb6ivv5yw2jh8jwj7smk3mtmh9kur1t5vsojr6odhgkr1l91d5cu5f","61":"cechvcezrij2ea41rabsmiz1avp8dvflr3givg6dnqfjs0cr7esi7pb7mft3yc5g08y73795bbsk3jjhr21iqkzh6fkkbwdh2rr0","62":"celnfzj1uipd25kjh1jl74q7npkp6gntvkzkjpn5ipax4q43m062ddf4gg8pvzs6i9g2mbqphpptygltdkc5mj305dh97tdv74tf","63":"8kxk1r9zrqsv3zb3v3bkf7zxclxfydkmvtzeuh51zt0w3c1aop3zxq15vo6w5idrd154ca2umlbdpfq5roj07hozpx2u1qn72g57","64":"ogqtkhmr1df4k29yvs57ayiqght7rodqxui67wtpsmfmva3yw9u8mfloe4xt72pxl1b620jwfresmn1f3bjuayutbcdia0cxs3xd","65":"27phu3a62d1cj9wwqyz524vyo7bxoc1qad1naob2xi0uo3dw5blx8nim5gthmh7tgrng9f3rdmi5fbwoglofum8glbhzmdvsj2th","66":"hynij3cgqcggezuszytkt7cnv0qjq97lhwbirbuv1l9vqemjmrqovkbu74w1060nx359pqwwfwgo3iyy2hl3cay7s1q8darqng8o","67":"kdnnac968jogx8scgufafvwvrj0iyk3evw17zswprn8toukdonl8mh69cx1jbgn3mqm1r3v1ydeebfqg5oni5tubuljbqp13jb35","68":"q7bqn04nirvo0ls2z3tz1v2i6k01ke81w434o708nc6ajd26qi3qhd6r4dogmsw4su1fg0de0qpvw1yck8enw05q5m6zd5gipi2n","69":"rtn7298ijs0k4kt6fwhbfs8njplfx42ugyxnbj4aa5gz9wxrla2jhr6wc2idxqbwx7t4fhtn8ljon30ayarhb46u1505zcpd0s1t","70":"9zntysfwmzq1rvsi2t3sr2zh5udpy1ar3d6bqs3j0d2cgmnt26pdudw66j9rxf1vi1ztm3lb0p4gd71tmcvbdukkzo71vdg4xvif","71":"yklqa8rzjpv4o0246hw41botq7dls78itt2o0f5ihywvdurdj8icp6ag3w21uc9umx1gyicxxb18dkc68fwwp04yfuhzjt20teu1","72":"d4av0chqi949y2dg8nscntr6dgpj2j41ypt9pjctkqjmki23w15mr1inwkup397kx2phq97hjawv0l9sxrbkx6yc3id35l4tze3k","73":"vkne4012phhsdcsn1xegy5eziyxi6qtuse0z49bn7p9dvg37qn91x6okpmbklkwqur7swhmg9b8xomfv9komk8mbkpkz2dbx3wp5","74":"72rfcv52r3bvtaymza4kiv0thyvsxun7ha4yf6f066szwf4ysic410uzf03vp40h21g9vvdgdsgmqmdde0uy5ypnlsl8fhdgrp2j","75":"b1qkpvkqyf5ylylzzoef8ps90sad8zjkbdqb2m3iqjkzn1mxq7627wxwop56q8qvd1cwyh0ltj76aemqipi5hqqggnsd80afursg","76":"d35v3cunt8s3puqys02wft6uqxty8gxo77olhqvvamm8u8bs999r9gwpnqoak7gkiqlgyk0td0uv9yasuxweovng860j5d2fmclh","77":"febksm0249z1z1ebqly3bfwprefs3uxwy95geqhdlncbkudyzoy0pg7iske1w2v6fdlp5il7u9pt06zsgn9te16pc14yi3ezafse","78":"6hiv0nk4emrswr778udshxtjar9y0im36gvm2sontq9qzr718afv1xlypu4libs5nqz5d5k6h7ixdefkoi5jvuou1i7noxzz0cna","79":"eid2wkk3jp4m8j2g2aufm3tnhdesisj5mroqonuxmhovuj9cc6215h1bgxmla2ezf8khis0fpkma9oleyqh56zyppp2a4of2hwye","80":"u09hgdntqkycqckwhe0ftxgxfoqvqmla0ssvarygkfmedjdjbfuyc39ol6g56x4lbsklbm0a0g8bjqbql396xd5cb233vo0edkb2","81":"yob5q3d14pe8k4ix7dsd9x4dw0riiy84jk0694mdics2dgvhpen9x947okfh51l8adibi24q42n49mba7yczlumuadqspqhudp3n","82":"twhpcolc0nvgfkwlmyaedbj49gqzadvtoczodupznap3wgmdbkaaf98326awb6pmb6msoz6q668m6lzqn85hin1i9u743f8g8isu","83":"92v8gj43iy18bpps98ohky9we9yazlp4k4bk1z9a2rh3r7e0oa4igvth88el7bx0sur4tf8dqza4ufy3b2cnazejyj3ndqcobgrs","84":"3uwqalut4diz6xjpryaaw9aetvc1g0ajzxnypchk619zxj6kgrapjhefcsx6qoqr2elhxu418h3yf5sygzi4yh2n0xqzwy7nc9se","85":"tn2kfq5r48e27oh5g3gcnmqhwswip11a1obt5df83kuphgb157ae7qagahemhfngq5rozlwtvkwc5loh96pfrqbv70th63skbdqt","86":"s3p0q74e5f2fjwryhw6vdtm3vke54q58juq7c6bwu5m5siippv9t52nq0y5i31u1wkvmeywop7omvvatestblya5mn0z0lig05vp","87":"zdk0fm2p6xvfwmh1l6mv59om77x0bcpsocfe2hvyh4c6t8w3r44qjamiau20ccbm0jvvhk44hgf568fr9tvkabedefdne33oov9b","88":"p3idf0iajvfvck7az0ddjogayn89hblmaqo8b95rqdxvx5rbsnm4txoeno3lvtb4ckzpyen3k3cs8o6jzxreh1qb3706sr9ly5g0","89":"3fmuct4aupjbuclx22phlz7b33ew82whxntcirgcn5r8sgmc708ibjn80gtih3yfu5l7vyxbb5htk3svps5jrfmakqvurp2vnq8p","90":"25516v2ifoxqvivlpd1x6zrlzfojxyg957g8svamggl6p132sbvjirapnyf2cyekqyotmypattyk34n61072ov6sd94lhr33emg1","91":"xqf0i8yokyw98i3p1ef1ucyv9g96v6vhwx4ztkso6irzcll54t20kho4wrhpt2xont9d0l42kl5m7xuapu1s6aq0mqmq3bl8ubo2","92":"fohdadlq4fyzxxq0vp33uetzxnlxi5bdl3ihfk9grugjykt3netq4hbr0uxw1zfenjfq8b5r8egas7b3q9bdyih1o6bddebp3gys","93":"g9tobhvstwkcat294bbqtzwq0kpb2xuvxgyzow1014oe8b57o63v3l20u36aug1hdy2tqgk5uy94d7i9yv95r5wtulsqzvp97uzg","94":"snqo2jyc5dye8ejyutpw06di5tovsv25chak49ws4vzeudzmf1ux0vwp85lmr0moj5j0190ohjfdpba9exsoiiwjlicd05ak5icf","95":"9egy7ndw8sr49hl9rt8sh0b8qlapngix6pf6ysjcwrs37o0l8n3zlqxfnjzepnaxjgf6n1qu4morw8tmpavd0j1nck58lk8i1hdy","96":"g7vz9wc75rupaeqdev0gcd31nmc7iv0nf2rljrexslbis6dbm7p2tdoq2fufb7aic8mh30917mrzbsst461y0ozeqxmv03f4xz4m","97":"biiprt4jdw10dyhlxdmayxkksz30lwlt3kzw8a5nd6tp5d1v8ljdlfo3gkd8n07oxcfi2mkd58frjiwomnlju2lhgx2r8nsbntpi","98":"n5zd3mpm7bv2y8klzlq3lb5ma03j1hrwsjif51y80f60r2tk8pbk79mycz77frrljt5ddh2nbe093h7fj2ad86y35h1qvltk8lc0","99":"3krxtcd90i41pb66820rodjdopeparzaouoja8g6229gn1m18wco63gq1nxtiv6xt1bv6rs81ogui7y5zukn7eosr9tevdxwvtip","100":"6iorwaxm06cd6l6bstchrg3brbtac7ofaods2rstpya29t0a2nvnjjlnuvarwx2cj76fawfh5ukc3b4ksu5ifnkjspxcb8k68c4t","101":"l5t2x74kpznm9fy8ix1zrbthnudoafu67t9op6r975gws6f022xq1utd69p9vd8k9bh29d09e45vxogtlmff68h72po4ljeanki3","102":"1oxb7e6s0ecx3lpsntxmi5qis6l10a6japco2r82bzi3d6g5p83wcfeog057m496tlz42bd5c73z7opji12kr42a6c0mb6tdaki6"}} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json new file mode 100644 index 0000000000..0a6c195ae8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json @@ -0,0 +1 @@ +{"id":"f5b1a667-0e39-4177-986c-d05d38e1eac8","NonSensitive":"80382481-0cd4-410f-a5dc-44fad3a935ee","SensitiveStr":"8a1506f9-7b15-4f8e-96dd-50033e540260","SensitiveInt":1737683325,"SensitiveDict":{"1":"7oockvrrhrreznj9lgq38e82g96608slltnab3hco7w5alehsmf1yp4r68kojs159ycuqq9syfqvsw3gjmprt0yhbtzoityuro5b","2":"ofoz73i909m7pqhg93o2a05ozoq9x1n1k3817t2vwgym1y3nbzf3d2njgej3iqutlo4xcdyri4fyjfirflpslwg2zfvv159vujex","3":"6371pofp39fuye4mz99vdkc19wo4l1ze7ocgbb12rnwozqgg9g1zqt6fnayhlgjmdjdab4xwgycpjkr8jyn4bp2kr18lo625bznp","4":"hweoq8ktlyi81z1h6j702cted8oxrya1dsh0rm5c5by1b69vmw5czcxqx6aqlmyf20yiurbzgqylns57si31qmo76qyvx2vggynb","5":"tuqwaitw9u230otpniu6txocfi1l3k6mm62f4tkak84wuf5c979qhd2wksx84zieutzblqnpv2n8ub7uivc92kpz7pfk9huulrhd","6":"l8q06w89xgyt23pk0pizu012tjz45ptfpozh8enn3z1ipg00di1n1oot14pjqluh5hlys9s7hkkkzhwv29ob5vvmj2saqptdi4iv","7":"edjikhq40zza6rl4cnn0y92madl59ek7qdiyeszvbkbzcjyqzbjw1dsuni1qspumqj56iza25j839czjt0p0yyl6n7pg7ekfj548","8":"2nqoipmh21552ymx1wvfef4t629tmb33cjom6orbjaa1sb4b7f5xagonllk0q25vqz0murnaseha01s99bktvjcltl0d1892qd8e","9":"gag5o0xxrdhwq7x6m9r0eyp9jg8f6oarcoizpr4j7hlj3ry8n7e5s7yseh8g7n5dweuqcoulbo0gb71hng5c6p73qfddhdfadz1i","10":"wxb3ykip4tehje4uikysdru15n9jvyh5fgcsl1m802lfx6c892vxt1ryyfrlknfp97a0antnfs3vpjref21j0lystuno14t7izg6"}} \ No newline at end of file From 38a198a22128c17980f424672633a4e9934edd03 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 11:11:13 +0200 Subject: [PATCH 04/85] Add non-allocating APIs to encryptors --- .../src/AeadAes256CbcHmac256Algorithm.cs | 19 ++++++++ .../src/CosmosEncryptor.cs | 30 +++++++++++++ .../src/DataEncryptionKey.cs | 22 ++++++++++ .../src/Encryptor.cs | 44 +++++++++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 21 ++++++--- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 30 +++++++++++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 30 +++++++++++++ 8 files changed, 192 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index df416c3efd..73049fd9ca 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -137,6 +137,20 @@ public override byte[] EncryptData(byte[] plainText) return this.EncryptData(plainText, hasAuthenticationTag: true); } + /// + /// Encryption Algorithm + /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + /// + /// Plaintext data to be encrypted + /// Returns the ciphertext corresponding to the plaintext. + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + /// /// Encryption Algorithm /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits @@ -418,5 +432,10 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length); return authenticationTag; } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 462bd56a1f..20dda27ee5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,21 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + /// public override async Task EncryptAsync( byte[] plainText, @@ -67,5 +82,20 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index bcee51d0e1..c505199dd6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -29,6 +29,17 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract byte[] EncryptData(byte[] plainText); + /// + /// Encrypts the plainText with a data encryption key. + /// + /// Plain text value to be encrypted. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Encrypted value. + public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -36,6 +47,17 @@ public abstract class DataEncryptionKey /// Plain text. public abstract byte[] DecryptData(byte[] cipherText); + /// + /// Decrypts the cipherText with a data encryption key. + /// + /// Ciphertext value to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Plain text. + public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 1e7f272ce5..73eeaa0890 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -27,6 +27,28 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Encrypts the plainText using the key and algorithm provided. + /// + /// Plain text. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Cipher text. + public abstract Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -40,5 +62,27 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Decrypts the cipherText using the key and algorithm provided. + /// + /// Ciphertext to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Plain text. + public abstract Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 68d863114e..0377c4257f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -12,6 +12,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey { + private const byte Version = 1; + private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm; private readonly byte[] unwrapKey; @@ -65,7 +67,8 @@ public MdeEncryptionAlgorithm( dekProperties.WrappedDataEncryptionKey); this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( protectedDataEncryptionKey, - encryptionType); + encryptionType, + Version); } else { @@ -80,11 +83,9 @@ public MdeEncryptionAlgorithm( this.RawKey = rawKey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( plaintextDataEncryptionKey, - encryptionType); - + encryptionType, + Version); } - - } /// @@ -125,5 +126,15 @@ public override byte[] DecryptData(byte[] cipherText) { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText); } + + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Encrypt(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ed4cca0596..ef53408b4a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index fd83ef528d..525b6b8d7f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1763,6 +1763,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -1776,6 +1796,16 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 85b7bf3f36..5526551480 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2248,6 +2248,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2261,6 +2281,16 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } From e0bb8bf39a41454e7ee8b89426c2a433bf4b6073 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 14:24:46 +0200 Subject: [PATCH 05/85] WIP --- .../src/AeadAes256CbcHmac256Algorithm.cs | 26 +++++++++++ .../src/CosmosEncryptor.cs | 10 +++++ .../src/DataEncryptionKey.cs | 14 ++++++ .../src/EncryptionProcessor.cs | 21 ++++++--- .../src/Encryptor.cs | 28 ++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 13 +++++- .../EmulatorTests/LegacyEncryptionTests.cs | 10 +++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 20 +++++++++ .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 +++++++++++++++++++ 9 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index 73049fd9ca..a51e09726b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -26,11 +26,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey /// private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8; + /// + /// Authentication tag size in bytes + /// + private const int AuthenticationTagSizeInBytes = KeySizeInBytes; + /// /// Block size in bytes. AES uses 16 byte blocks. /// private const int BlockSizeInBytes = 16; + /// + /// Size of Initialization Vector in bytes. + /// + private const int IvSizeInBytes = 16; + /// /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) /// @@ -437,5 +447,21 @@ public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cip { throw new NotImplementedException(); } + + public override int GetEncryptByteCount(int plainTextLength) + { + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + throw new NotImplementedException(); + } + + private static int GetCipherTextLength(int inputSize) + { + return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 20dda27ee5..1a0f093176 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -97,5 +97,15 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index c505199dd6..65890ec941 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -40,6 +40,13 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Size of input when encrypted. + public abstract int GetEncryptByteCount(int plainTextLength); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -58,6 +65,13 @@ public abstract class DataEncryptionKey /// Plain text. public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Upper bound size of the input when decrypted. + public abstract int GetDecryptByteCount(int cipherTextLength); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index af10de5e11..bb5beda440 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -99,19 +99,28 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - cipherText = await encryptor.EncryptAsync( + int cipherTextLength = await encryptor.GetEncryptBytesCount( + plainText.Length, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm); + + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + cipherTextWithTypeMarker[0] = (byte)typeMarker; + + int encryptedBytesCount = await encryptor.EncryptAsync( plainText, + plainTextOffset: 0, + plainTextLength: plainText.Length, + cipherTextWithTypeMarker, + outputOffset: 1, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - if (cipherText == null) + if (encryptedBytesCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - - byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1]; - cipherTextWithTypeMarker[0] = (byte)typeMarker; - Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length); + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 73eeaa0890..9362c6f2a0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -49,6 +49,20 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Size of input when encrypted. + public abstract Task GetEncryptBytesCount( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -84,5 +98,19 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Upper bound size of the input when decrypted. + public abstract Task GetDecryptBytesCount( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 0377c4257f..ba606e9412 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -104,7 +104,8 @@ public MdeEncryptionAlgorithm( this.RawKey = rawkey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( dataEncryptionKey, - encryptionType); + encryptionType, + Version); } /// @@ -136,5 +137,15 @@ public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cip { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); } + + public override int GetEncryptByteCount(int plainTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetEncryptByteCount(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetDecryptByteCount(cipherTextLength); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 525b6b8d7f..7b8d85e878 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1806,6 +1806,16 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 5526551480..69924c0a7a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2291,6 +2291,26 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs new file mode 100644 index 0000000000..329e06e7c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -0,0 +1,43 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System.Linq; + using System.Text; + + [TestClass] + public class AeadAes256CbcHmac256AlgorithmTests + { + private static readonly byte[] RootKey = new byte[32]; + + private static AeadAes256CbcHmac256EncryptionKey key; + private static AeadAes256CbcHmac256Algorithm algorithm; + + [ClassInitialize] + public static void ClassInitialize(TestContext testContext) + { + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); + AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); + } + + [TestMethod] + public void EncryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; + + int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); + byte[] cipherTextBytes = new byte[cipherTextLength]; + + int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); + Assert.Equals(encrypted, cipherTextLength); + + byte[] decrypted = algorithm.DecryptData(cipherTextBytes); + + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); + } + } +} From 9c3c276e389db6bd2971d73bd4ad722e7ab88737 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 15:38:12 +0200 Subject: [PATCH 06/85] Revert solution update --- Microsoft.Azure.Cosmos.sln | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index 6fa5e4c3f2..b1d77052bf 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.12.35209.166 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject From f33538c684911a9f8d472c800477f0f54d036f9e Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 10:14:19 +0200 Subject: [PATCH 07/85] Add implementation, fix tests --- .../src/CosmosEncryptor.cs | 28 ++++++++++++++++--- .../src/EncryptionProcessor.cs | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 18 +++++++++--- .../Readme.md | 14 +++++----- .../MdeEncryptionProcessorTests.cs | 15 +++++++++- .../TestCommon.cs | 17 ++++++++++- 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 1a0f093176..34c616b227 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -98,14 +98,34 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetEncryptByteCount(plainTextLength); } - public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetDecryptByteCount(cipherTextLength); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index bb5beda440..3d93d0be8a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -120,7 +120,7 @@ public static async Task EncryptAsync( { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 7b8d85e878..d7f8c5a22b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1807,14 +1807,24 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); } - public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index bb4836273b..44aae54dd0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-preview.7.24407.12 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | -| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | -| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | -| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | -| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | +| **Encrypt** | **1** | **81.01 μs** | **2.595 μs** | **3.804 μs** | **4.7607** | **1.5869** | **-** | **58.87 KB** | +| Decrypt | 1 | 68.96 μs | 1.627 μs | 2.333 μs | 5.0049 | 1.2207 | - | 61.57 KB | +| **Encrypt** | **10** | **190.05 μs** | **5.716 μs** | **8.556 μs** | **16.1133** | **3.9063** | **-** | **197.66 KB** | +| Decrypt | 10 | 215.60 μs | 4.424 μs | 6.484 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **2,261.05 μs** | **212.952 μs** | **298.529 μs** | **148.4375** | **78.1250** | **74.2188** | **1785.47 KB** | +| Decrypt | 100 | 3,139.31 μs | 316.105 μs | 473.131 μs | 224.6094 | 175.7813 | 130.8594 | 3066.66 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 1396e7e06e..562cc0f0a7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -24,7 +24,7 @@ public class MdeEncryptionProcessorTests private const string dekId = "dekId"; [ClassInitialize] - public static void ClassInitilize(TestContext testContext) + public static void ClassInitialize(TestContext testContext) { _ = testContext; MdeEncryptionProcessorTests.encryptionOptions = new EncryptionOptions() @@ -38,9 +38,22 @@ public static void ClassInitilize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] plainText, int plainTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index 63968723e6..a0ecdab14e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,11 +27,26 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } + internal static int EncryptData(byte[] plainText, int inputOffset, byte[] output, int outputOffset) + { + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, plainText.Length); + + return cipherText.Length; + } + internal static byte[] DecryptData(byte[] cipherText) { return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - + + internal static int DecryptData(byte[] cipherText, int inputOffset, byte[] output, int outputOffset) + { + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset).ToArray()); + Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); + return plainText.Length; + } + internal static Stream ToStream(T input) { string s = JsonConvert.SerializeObject(input); From 6923fd2fdb29a831e6eadd01e26463c2e312d254 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 18:17:17 +0200 Subject: [PATCH 08/85] Switch to randomized encryption for benchmarks --- .../EncryptionBenchmark.cs | 2 +- ...smos.Encryption.Custom.Performance.Tests.csproj | 6 ------ .../Readme.md | 14 +++++++------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index d8e4c7d8a2..3655043648 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -35,7 +35,7 @@ public async Task Setup() Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.encryptionOptions = CreateEncryptionOptions(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index e32a42c36c..4deb4d0edf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -8,12 +8,6 @@ enable - - - - - - diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index bb4836273b..2807f8c809 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-preview.7.24407.12 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | -| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | -| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | -| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | -| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | +| **Encrypt** | **1** | **61.45 μs** | **1.676 μs** | **2.457 μs** | **4.9438** | **1.2207** | **-** | **61.25 KB** | +| Decrypt | 1 | 77.89 μs | 1.959 μs | 2.933 μs | 5.7373 | 1.4648 | - | 71.22 KB | +| **Encrypt** | **10** | **171.64 μs** | **3.341 μs** | **4.791 μs** | **21.2402** | **3.6621** | **-** | **260.97 KB** | +| Decrypt | 10 | 255.57 μs | 7.833 μs | 11.724 μs | 29.2969 | 4.3945 | - | 363.84 KB | +| **Encrypt** | **100** | **2,601.33 μs** | **215.481 μs** | **322.522 μs** | **199.2188** | **125.0000** | **123.0469** | **2464.88 KB** | +| Decrypt | 100 | 3,156.06 μs | 321.419 μs | 481.084 μs | 355.4688 | 300.7813 | 261.7188 | 3413.05 KB | From 03ef682ba696e192985b8ddd0632176882bedcd5 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 18:05:07 +0200 Subject: [PATCH 09/85] Some more array pooling --- Directory.Build.props | 1 + .../src/ArrayPoolManager.cs | 47 ++++++ .../src/CosmosEncryptor.cs | 8 +- .../src/EncryptionProcessor.cs | 147 +++++++++++++----- .../src/Encryptor.cs | 4 +- .../src/Mirrored/UnixDateTimeConverter.cs | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 4 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 4 +- .../EncryptionBenchmark.cs | 2 +- .../Readme.md | 16 +- .../MdeEncryptionProcessorTests.cs | 12 +- .../TestCommon.cs | 10 +- Microsoft.Azure.Cosmos.lutconfig | 6 + 13 files changed, 196 insertions(+), 67 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs create mode 100644 Microsoft.Azure.Cosmos.lutconfig diff --git a/Directory.Build.props b/Directory.Build.props index 04fc0bc918..7b9401e44a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,6 +12,7 @@ 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) $(DefineConstants);PREVIEW;ENCRYPTIONPREVIEW + NU1903 diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs new file mode 100644 index 0000000000..b09a919a5f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -0,0 +1,47 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Buffers; + using System.Collections.Generic; + + internal class ArrayPoolManager : IDisposable + { + private List rentedBuffers = new List(); + private bool disposedValue; + + public byte[] Rent(int minimumLength) + { + byte[] buffer = ArrayPool.Shared.Rent(minimumLength); + this.rentedBuffers.Add(buffer); + return buffer; + } + + protected virtual void Dispose(bool disposing) + { + if (!this.disposedValue) + { + if (disposing) + { + foreach (byte[] buffer in this.rentedBuffers) + { + ArrayPool.Shared.Return(buffer); + } + } + + this.rentedBuffers = null; + this.disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 34c616b227..0288ee79db 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,7 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + /// public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( @@ -83,6 +84,7 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + /// public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( @@ -98,7 +100,8 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + /// + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -113,7 +116,8 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + /// + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3d93d0be8a..e7afe78c2a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; + using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -25,6 +26,9 @@ internal static class EncryptionProcessor // UTF-8 encoding. private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); + private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); private static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings() { @@ -80,6 +84,8 @@ public static async Task EncryptAsync( byte[] cipherText = null; TypeMarker typeMarker; + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + switch (encryptionOptions.EncryptionAlgorithm) { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: @@ -97,20 +103,26 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); + (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } - int cipherTextLength = await encryptor.GetEncryptBytesCount( + int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( plainText.Length, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); + cipherTextWithTypeMarker[0] = (byte)typeMarker; int encryptedBytesCount = await encryptor.EncryptAsync( plainText, plainTextOffset: 0, - plainTextLength: plainText.Length, + plainTextLength, cipherTextWithTypeMarker, outputOffset: 1, encryptionOptions.DataEncryptionKeyId, @@ -121,16 +133,16 @@ public static async Task EncryptAsync( throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - itemJObj[propertyName] = cipherTextWithTypeMarker; + itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); pathsEncrypted.Add(pathToEncrypt); } encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); break; case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: @@ -166,11 +178,11 @@ public static async Task EncryptAsync( } encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 2, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: cipherText, - encryptionOptions.PathsToEncrypt); + encryptionFormatVersion: 2, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: cipherText, + encryptionOptions.PathsToEncrypt); break; default: @@ -278,6 +290,8 @@ private static async Task MdeEncAlgoDecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + JObject plainTextJObj = new JObject(); foreach (string path in encryptionProperties.EncryptedPaths) { @@ -288,25 +302,31 @@ private static async Task MdeEncAlgoDecryptObjectAsync( } byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) { continue; } - byte[] cipherText = new byte[cipherTextWithTypeMarker.Length - 1]; - Buffer.BlockCopy(cipherTextWithTypeMarker, 1, cipherText, 0, cipherTextWithTypeMarker.Length - 1); + int plainTextLength = await encryptor.GetDecryptBytesCountAsync( + cipherTextWithTypeMarker.Length - 1, + encryptionProperties.DataEncryptionKeyId, + encryptionProperties.EncryptionAlgorithm); + + byte[] plainText = arrayPoolManager.Rent(plainTextLength); - byte[] plainText = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + int decryptedCount = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( encryptionProperties, - cipherText, + cipherTextWithTypeMarker, + cipherTextOffset: 1, + cipherTextWithTypeMarker.Length - 1, + plainText, encryptor, diagnosticsContext, cancellationToken); EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText, + plainText.AsSpan(0, decryptedCount), plainTextJObj, propertyName); } @@ -340,9 +360,12 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static async Task MdeEncAlgoDecryptPropertyAsync( + private static async Task MdeEncAlgoDecryptPropertyAsync( EncryptionProperties encryptionProperties, byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] buffer, Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) @@ -352,18 +375,22 @@ private static async Task MdeEncAlgoDecryptPropertyAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - byte[] plainText = await encryptor.DecryptAsync( + int decryptedCount = await encryptor.DecryptAsync( cipherText, + cipherTextOffset, + cipherTextLength, + buffer, + outputOffset: 0, encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - if (plainText == null) + if (decryptedCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); } - return plainText; + return decryptedCount; } private static async Task LegacyEncAlgoDecryptContentAsync( @@ -483,49 +510,71 @@ private static JObject RetrieveEncryptionProperties( return encryptionPropertiesJObj; } - private static (TypeMarker, byte[]) Serialize(JToken propertyValue) + private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) { + byte[] buffer; + int length; switch (propertyValue.Type) { case JTokenType.Undefined: Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null); + return (default, null, -1); case JTokenType.Null: Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null); + return (TypeMarker.Null, null, -1); case JTokenType.Boolean: - return (TypeMarker.Boolean, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlBoolSerializer); + return (TypeMarker.Boolean, buffer, length); case JTokenType.Float: - return (TypeMarker.Double, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlDoubleSerializer); + return (TypeMarker.Double, buffer, length); case JTokenType.Integer: - return (TypeMarker.Long, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlLongSerializer); + return (TypeMarker.Long, buffer, length); case JTokenType.String: - return (TypeMarker.String, SqlVarCharSerializer.Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeString(propertyValue.ToObject()); + return (TypeMarker.String, buffer, length); case JTokenType.Array: - return (TypeMarker.Array, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Array, buffer, length); case JTokenType.Object: - return (TypeMarker.Object, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Object, buffer, length); default: throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); } + + (byte[] bytes, int length) SerializeFixed(IFixedSizeSerializer serializer) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int bytes = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, bytes); + } + + (byte[] bytes, int length) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int bytes = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, bytes); + } } private static void DeserializeAndAddProperty( TypeMarker typeMarker, - byte[] serializedBytes, + ReadOnlySpan serializedBytes, JObject jObject, string key) { switch (typeMarker) { case TypeMarker.Boolean: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlBoolSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Double: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlDoubleSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Long: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlLongSerializer.Deserialize(serializedBytes)); break; case TypeMarker.String: jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); @@ -587,5 +636,27 @@ await EncryptionProcessor.DecryptAsync( // and corresponding decrypted properties are added back in the documents. return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); } + + internal static int GetOriginalBase64Length(string base64string) + { + if (string.IsNullOrEmpty(base64string)) + { + return 0; + } + + int paddingCount = 0; + int characterCount = base64string.Length; + if (base64string[characterCount - 1] == '=') + { + paddingCount++; + } + + if (base64string[characterCount - 2] == '=') + { + paddingCount++; + } + + return (3 * (characterCount / 4)) - paddingCount; + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 9362c6f2a0..469d069313 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -57,7 +57,7 @@ public abstract Task EncryptAsync( /// Identifier for the encryption algorithm. /// Token for cancellation. /// Size of input when encrypted. - public abstract Task GetEncryptBytesCount( + public abstract Task GetEncryptBytesCountAsync( int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, @@ -107,7 +107,7 @@ public abstract Task DecryptAsync( /// Identifier for the encryption algorithm. /// Token for cancellation. /// Upper bound size of the input when decrypted. - public abstract Task GetDecryptBytesCount( + public abstract Task GetDecryptBytesCountAsync( int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs index 9fffa1e3cb..30c23a3a92 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs @@ -59,7 +59,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist try { - totalSeconds = Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); + totalSeconds = System.Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); } catch { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index d7f8c5a22b..9150a739cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1807,7 +1807,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -1817,7 +1817,7 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 69924c0a7a..9f63b3545b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2292,7 +2292,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -2302,7 +2302,7 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index d8e4c7d8a2..3655043648 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -35,7 +35,7 @@ public async Task Setup() Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.encryptionOptions = CreateEncryptionOptions(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 44aae54dd0..74a360b0bf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **81.01 μs** | **2.595 μs** | **3.804 μs** | **4.7607** | **1.5869** | **-** | **58.87 KB** | -| Decrypt | 1 | 68.96 μs | 1.627 μs | 2.333 μs | 5.0049 | 1.2207 | - | 61.57 KB | -| **Encrypt** | **10** | **190.05 μs** | **5.716 μs** | **8.556 μs** | **16.1133** | **3.9063** | **-** | **197.66 KB** | -| Decrypt | 10 | 215.60 μs | 4.424 μs | 6.484 μs | 24.6582 | 4.8828 | - | 303.41 KB | -| **Encrypt** | **100** | **2,261.05 μs** | **212.952 μs** | **298.529 μs** | **148.4375** | **78.1250** | **74.2188** | **1785.47 KB** | -| Decrypt | 100 | 3,139.31 μs | 316.105 μs | 473.131 μs | 224.6094 | 175.7813 | 130.8594 | 3066.66 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |-----------:|----------:|----------:|-----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **108.1 μs** | **6.13 μs** | **8.18 μs** | **105.1 μs** | **4.3945** | **1.4648** | **-** | **56.71 KB** | +| Decrypt | 1 | 131.0 μs | 7.17 μs | 10.51 μs | 127.6 μs | 5.3711 | 1.8311 | - | 66.1 KB | +| **Encrypt** | **10** | **277.8 μs** | **13.18 μs** | **18.91 μs** | **282.8 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | +| Decrypt | 10 | 486.4 μs | 57.69 μs | 86.34 μs | 438.9 μs | 23.4375 | 5.8594 | - | 287.62 KB | +| **Encrypt** | **100** | **3,187.7 μs** | **309.58 μs** | **443.99 μs** | **3,045.0 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.54 KB** | +| Decrypt | 100 | 4,828.9 μs | 313.12 μs | 468.67 μs | 4,833.8 μs | 218.7500 | 167.9688 | 125.0000 | 2845.11 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 562cc0f0a7..462bc4122c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -38,22 +38,22 @@ public static void ClassInitialize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] plainText, int plainTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + .ReturnsAsync((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index a0ecdab14e..cf043ac178 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,10 +27,10 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } - internal static int EncryptData(byte[] plainText, int inputOffset, byte[] output, int outputOffset) + internal static int EncryptData(byte[] plainText, int inputOffset, int inputLength, byte[] output, int outputOffset) { - byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset).ToArray()); - Buffer.BlockCopy(cipherText, 0, output, outputOffset, plainText.Length); + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, cipherText.Length); return cipherText.Length; } @@ -40,9 +40,9 @@ internal static byte[] DecryptData(byte[] cipherText) return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - internal static int DecryptData(byte[] cipherText, int inputOffset, byte[] output, int outputOffset) + internal static int DecryptData(byte[] cipherText, int inputOffset, int inputLength, byte[] output, int outputOffset) { - byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset).ToArray()); + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset, inputLength).ToArray()); Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); return plainText.Length; } diff --git a/Microsoft.Azure.Cosmos.lutconfig b/Microsoft.Azure.Cosmos.lutconfig new file mode 100644 index 0000000000..596a860301 --- /dev/null +++ b/Microsoft.Azure.Cosmos.lutconfig @@ -0,0 +1,6 @@ + + + true + true + 180000 + \ No newline at end of file From 1044a89aa79e98ba08d4a40dd8fa5c39f47d76e7 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 19 Sep 2024 16:24:54 +0200 Subject: [PATCH 10/85] Streaming deserialization --- .../src/ArrayPoolManager.cs | 16 +- .../src/EncryptionProcessor.cs | 32 +++- .../src/MemoryTextReader.cs | 161 ++++++++++++++++++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- .../Readme.md | 16 +- .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 ----- .../TestCommon.cs | 36 ++-- 8 files changed, 229 insertions(+), 79 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index b09a919a5f..8cafd07c5d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -8,14 +8,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Buffers; using System.Collections.Generic; - internal class ArrayPoolManager : IDisposable + internal class ArrayPoolManager : IDisposable { - private List rentedBuffers = new List(); + private List rentedBuffers = new List(); private bool disposedValue; - public byte[] Rent(int minimumLength) + public T[] Rent(int minimumLength) { - byte[] buffer = ArrayPool.Shared.Rent(minimumLength); + T[] buffer = ArrayPool.Shared.Rent(minimumLength); this.rentedBuffers.Add(buffer); return buffer; } @@ -26,9 +26,9 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - foreach (byte[] buffer in this.rentedBuffers) + foreach (T[] buffer in this.rentedBuffers) { - ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(buffer, clearArray: true); } } @@ -44,4 +44,8 @@ public void Dispose() GC.SuppressFinalize(this); } } + + internal class ArrayPoolManager : ArrayPoolManager + { + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e7afe78c2a..53478d915d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -544,18 +544,18 @@ private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedByt throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); } - (byte[] bytes, int length) SerializeFixed(IFixedSizeSerializer serializer) + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) { byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int bytes = serializer.Serialize(propertyValue.ToObject(), buffer); - return (buffer, bytes); + int length = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, length); } - (byte[] bytes, int length) SerializeString(string value) + (byte[], int) SerializeString(string value) { byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int bytes = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, bytes); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); } } @@ -580,15 +580,31 @@ private static void DeserializeAndAddProperty( jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Array: - jObject.Add(key, JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings)); + DeserializeAndAddProperty(serializedBytes); break; case TypeMarker.Object: - jObject.Add(key, JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings)); + DeserializeAndAddProperty(serializedBytes); break; default: Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); break; } + + void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + where T : JToken + { + using ArrayPoolManager manager = new ArrayPoolManager(); + + char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); + + JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); + + using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); + using JsonTextReader reader = new JsonTextReader(memoryTextReader); + + jObject.Add(key, serializer.Deserialize(reader)); + } } private enum TypeMarker : byte diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs new file mode 100644 index 0000000000..b8996ca802 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs @@ -0,0 +1,161 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + + /// + /// Adjusted implementation of .Net StringReader reading from a Memory instead of a string. + /// + internal class MemoryTextReader : TextReader + { + private Memory chars; + private int length; + private int pos; + private bool closed; + + public MemoryTextReader(Memory chars) + { + this.chars = chars; + this.length = chars.Length; + } + + public override void Close() + { + this.Dispose(true); + } + + protected override void Dispose(bool disposing) + { + this.chars = null; + this.pos = 0; + this.length = 0; + this.closed = true; + base.Dispose(disposing); + } + + [Pure] + public override int Peek() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + if (this.pos == this.length) + { + return -1; + } + + return this.chars.Span[this.pos]; + } + + public override int Read() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + if (this.pos == this.length) + { + return -1; + } + + return this.chars.Span[this.pos++]; + } + + public override int Read(char[] buffer, int index, int count) + { + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if (index < 0) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + if (buffer.Length - index < count) + { + throw new ArgumentOutOfRangeException(); + } + + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + int n = this.length - this.pos; + if (n > 0) + { + if (n > count) + { + n = count; + } + + this.chars.Span.Slice(this.pos, n).CopyTo(buffer.AsSpan(index, n)); + this.pos += n; + } + + return n; + } + + public override string ReadToEnd() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + this.pos = this.length; + return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); + } + + public override string ReadLine() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + int i = this.pos; + while (i < this.length) + { + char ch = this.chars.Span[i]; + if (ch == '\r' || ch == '\n') + { + string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + this.pos = i + 1; + if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') + { + this.pos++; + } + + return result; + } + + i++; + } + + if (i > this.pos) + { + string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + this.pos = i; + return result; + } + + return null; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ef53408b4a..d940744205 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 9f63b3545b..94bcd21c46 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(32, unwrapcount); + Assert.AreEqual(64, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 74a360b0bf..5d231e7843 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |-----------:|----------:|----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **108.1 μs** | **6.13 μs** | **8.18 μs** | **105.1 μs** | **4.3945** | **1.4648** | **-** | **56.71 KB** | -| Decrypt | 1 | 131.0 μs | 7.17 μs | 10.51 μs | 127.6 μs | 5.3711 | 1.8311 | - | 66.1 KB | -| **Encrypt** | **10** | **277.8 μs** | **13.18 μs** | **18.91 μs** | **282.8 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | -| Decrypt | 10 | 486.4 μs | 57.69 μs | 86.34 μs | 438.9 μs | 23.4375 | 5.8594 | - | 287.62 KB | -| **Encrypt** | **100** | **3,187.7 μs** | **309.58 μs** | **443.99 μs** | **3,045.0 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.54 KB** | -| Decrypt | 100 | 4,828.9 μs | 313.12 μs | 468.67 μs | 4,833.8 μs | 218.7500 | 167.9688 | 125.0000 | 2845.11 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |-----------:|----------:|----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **134.2 μs** | **24.65 μs** | **36.89 μs** | **4.5166** | **1.4648** | **-** | **56.71 KB** | +| Decrypt | 1 | 122.4 μs | 2.57 μs | 3.77 μs | 5.1270 | 1.5869 | - | 64.01 KB | +| **Encrypt** | **10** | **309.7 μs** | **40.78 μs** | **61.04 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | +| Decrypt | 10 | 435.5 μs | 28.40 μs | 40.73 μs | 21.4844 | 5.3711 | - | 265.22 KB | +| **Encrypt** | **100** | **3,666.4 μs** | **285.40 μs** | **418.34 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.51 KB** | +| Decrypt | 100 | 4,928.2 μs | 441.88 μs | 661.39 μs | 195.3125 | 121.0938 | 101.5625 | 2617.38 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs deleted file mode 100644 index 329e06e7c5..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Tests -{ - using Microsoft.Azure.Cosmos.Encryption.Custom; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using System.Linq; - using System.Text; - - [TestClass] - public class AeadAes256CbcHmac256AlgorithmTests - { - private static readonly byte[] RootKey = new byte[32]; - - private static AeadAes256CbcHmac256EncryptionKey key; - private static AeadAes256CbcHmac256Algorithm algorithm; - - [ClassInitialize] - public static void ClassInitialize(TestContext testContext) - { - AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); - AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); - } - - [TestMethod] - public void EncryptUsingBufferDecryptsSuccessfully() - { - byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; - - int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); - byte[] cipherTextBytes = new byte[cipherTextLength]; - - int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); - Assert.Equals(encrypted, cipherTextLength); - - byte[] decrypted = algorithm.DecryptData(cipherTextBytes); - - Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); - } - } -} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index cf043ac178..81d8683e57 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -70,7 +70,7 @@ private static JObject ParseStream(Stream stream) internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt" }; + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveArr", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } @@ -83,17 +83,12 @@ internal class TestDoc public int SensitiveInt { get; set; } - public TestDoc() - { - } + public string[] SensitiveArr { get; set; } - public TestDoc(TestDoc other) + public Dictionary SensitiveDict { get; set; } + + public TestDoc() { - this.Id = other.Id; - this.PK = other.PK; - this.NonSensitive = other.NonSensitive; - this.SensitiveStr = other.SensitiveStr; - this.SensitiveInt = other.SensitiveInt; } public override bool Equals(object obj) @@ -103,7 +98,9 @@ public override bool Equals(object obj) && this.PK == doc.PK && this.NonSensitive == doc.NonSensitive && this.SensitiveInt == doc.SensitiveInt - && this.SensitiveStr == this.SensitiveStr; + && this.SensitiveStr == doc.SensitiveStr + && this.SensitiveArr?.Equals(doc.SensitiveArr) == true + && this.SensitiveDict?.Equals(doc.SensitiveDict) == true; } public override int GetHashCode() @@ -114,6 +111,8 @@ public override int GetHashCode() hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveStr); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveInt); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveArr); + hashCode = (hashCode * -1521134295) + EqualityComparer>.Default.GetHashCode(this.SensitiveDict); return hashCode; } @@ -125,7 +124,20 @@ public static TestDoc Create(string partitionKey = null) PK = partitionKey ?? Guid.NewGuid().ToString(), NonSensitive = Guid.NewGuid().ToString(), SensitiveStr = Guid.NewGuid().ToString(), - SensitiveInt = new Random().Next() + SensitiveInt = new Random().Next(), + SensitiveArr = new string[] + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }, + SensitiveDict = new Dictionary + { + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + } }; } From 05bfc508daa7f673c3da34a0887ccbb88f77b8eb Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Fri, 20 Sep 2024 09:12:17 +0200 Subject: [PATCH 11/85] Cleanup --- .../src/Mirrored/UnixDateTimeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs index 30c23a3a92..9fffa1e3cb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs @@ -59,7 +59,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist try { - totalSeconds = System.Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); + totalSeconds = Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); } catch { From 5629f74a2a12074ad1fcf8968eb175e35a5a54da Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Fri, 20 Sep 2024 10:06:31 +0200 Subject: [PATCH 12/85] Update MDE and rerun benchmarks --- .../Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../Readme.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ed4cca0596..ef53408b4a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2807f8c809..2d388b505d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **61.45 μs** | **1.676 μs** | **2.457 μs** | **4.9438** | **1.2207** | **-** | **61.25 KB** | -| Decrypt | 1 | 77.89 μs | 1.959 μs | 2.933 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **171.64 μs** | **3.341 μs** | **4.791 μs** | **21.2402** | **3.6621** | **-** | **260.97 KB** | -| Decrypt | 10 | 255.57 μs | 7.833 μs | 11.724 μs | 29.2969 | 4.3945 | - | 363.84 KB | -| **Encrypt** | **100** | **2,601.33 μs** | **215.481 μs** | **322.522 μs** | **199.2188** | **125.0000** | **123.0469** | **2464.88 KB** | -| Decrypt | 100 | 3,156.06 μs | 321.419 μs | 481.084 μs | 355.4688 | 300.7813 | 261.7188 | 3413.05 KB | +| **Encrypt** | **1** | **44.46 μs** | **0.661 μs** | **0.969 μs** | **4.2114** | **1.0376** | **-** | **51.96 KB** | +| Decrypt | 1 | 56.00 μs | 1.062 μs | 1.589 μs | 5.0049 | 1.2817 | - | 61.57 KB | +| **Encrypt** | **10** | **131.08 μs** | **0.893 μs** | **1.309 μs** | **16.3574** | **3.1738** | **-** | **200.9 KB** | +| Decrypt | 10 | 174.88 μs | 3.443 μs | 4.938 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **2,052.43 μs** | **230.487 μs** | **344.982 μs** | **160.1563** | **107.4219** | **83.9844** | **1891.44 KB** | +| Decrypt | 100 | 2,791.54 μs | 284.376 μs | 425.641 μs | 234.3750 | 166.0156 | 140.6250 | 3066.91 KB | From 495d2c4d317c18f03c94f31fb6792d1f947fb8d0 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 11:11:13 +0200 Subject: [PATCH 13/85] Add non-allocating APIs to encryptors --- .../src/AeadAes256CbcHmac256Algorithm.cs | 45 ++++++++++++ .../src/CosmosEncryptor.cs | 64 +++++++++++++++++ .../src/DataEncryptionKey.cs | 36 ++++++++++ .../src/EncryptionProcessor.cs | 19 +++-- .../src/Encryptor.cs | 72 +++++++++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 34 +++++++-- .../EmulatorTests/LegacyEncryptionTests.cs | 50 +++++++++++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 52 +++++++++++++- .../Readme.md | 12 ++-- .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 +++++++++++ .../MdeEncryptionProcessorTests.cs | 19 ++++- .../TestCommon.cs | 58 ++++++++++----- 12 files changed, 465 insertions(+), 39 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index df416c3efd..a51e09726b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -26,11 +26,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey /// private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8; + /// + /// Authentication tag size in bytes + /// + private const int AuthenticationTagSizeInBytes = KeySizeInBytes; + /// /// Block size in bytes. AES uses 16 byte blocks. /// private const int BlockSizeInBytes = 16; + /// + /// Size of Initialization Vector in bytes. + /// + private const int IvSizeInBytes = 16; + /// /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) /// @@ -137,6 +147,20 @@ public override byte[] EncryptData(byte[] plainText) return this.EncryptData(plainText, hasAuthenticationTag: true); } + /// + /// Encryption Algorithm + /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + /// + /// Plaintext data to be encrypted + /// Returns the ciphertext corresponding to the plaintext. + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + /// /// Encryption Algorithm /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits @@ -418,5 +442,26 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length); return authenticationTag; } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + + public override int GetEncryptByteCount(int plainTextLength) + { + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + throw new NotImplementedException(); + } + + private static int GetCipherTextLength(int inputSize) + { + return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 462bd56a1f..0288ee79db 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,22 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + /// + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + /// public override async Task EncryptAsync( byte[] plainText, @@ -67,5 +83,53 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + /// + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + /// + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetEncryptByteCount(plainTextLength); + } + + /// + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetDecryptByteCount(cipherTextLength); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index bcee51d0e1..65890ec941 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -29,6 +29,24 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract byte[] EncryptData(byte[] plainText); + /// + /// Encrypts the plainText with a data encryption key. + /// + /// Plain text value to be encrypted. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Encrypted value. + public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Size of input when encrypted. + public abstract int GetEncryptByteCount(int plainTextLength); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -36,6 +54,24 @@ public abstract class DataEncryptionKey /// Plain text. public abstract byte[] DecryptData(byte[] cipherText); + /// + /// Decrypts the cipherText with a data encryption key. + /// + /// Ciphertext value to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Plain text. + public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Upper bound size of the input when decrypted. + public abstract int GetDecryptByteCount(int cipherTextLength); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index af10de5e11..8b4da86a93 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -99,19 +99,28 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - cipherText = await encryptor.EncryptAsync( + int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( + plainText.Length, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm); + + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + cipherTextWithTypeMarker[0] = (byte)typeMarker; + + int encryptedBytesCount = await encryptor.EncryptAsync( plainText, + plainTextOffset: 0, + plainTextLength: plainText.Length, + cipherTextWithTypeMarker, + outputOffset: 1, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - if (cipherText == null) + if (encryptedBytesCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1]; - cipherTextWithTypeMarker[0] = (byte)typeMarker; - Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length); itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 1e7f272ce5..469d069313 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -27,6 +27,42 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Encrypts the plainText using the key and algorithm provided. + /// + /// Plain text. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Cipher text. + public abstract Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Size of input when encrypted. + public abstract Task GetEncryptBytesCountAsync( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -40,5 +76,41 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Decrypts the cipherText using the key and algorithm provided. + /// + /// Ciphertext to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Plain text. + public abstract Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Upper bound size of the input when decrypted. + public abstract Task GetDecryptBytesCountAsync( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 68d863114e..ba606e9412 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -12,6 +12,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey { + private const byte Version = 1; + private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm; private readonly byte[] unwrapKey; @@ -65,7 +67,8 @@ public MdeEncryptionAlgorithm( dekProperties.WrappedDataEncryptionKey); this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( protectedDataEncryptionKey, - encryptionType); + encryptionType, + Version); } else { @@ -80,11 +83,9 @@ public MdeEncryptionAlgorithm( this.RawKey = rawKey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( plaintextDataEncryptionKey, - encryptionType); - + encryptionType, + Version); } - - } /// @@ -103,7 +104,8 @@ public MdeEncryptionAlgorithm( this.RawKey = rawkey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( dataEncryptionKey, - encryptionType); + encryptionType, + Version); } /// @@ -125,5 +127,25 @@ public override byte[] DecryptData(byte[] cipherText) { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText); } + + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Encrypt(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + + public override int GetEncryptByteCount(int plainTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetEncryptByteCount(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetDecryptByteCount(cipherTextLength); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index fd83ef528d..9150a739cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1763,6 +1763,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -1776,6 +1796,36 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 85b7bf3f36..cb20c71c58 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(32, unwrapcount); + Assert.AreEqual(48, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); @@ -2248,6 +2248,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2261,6 +2281,36 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2d388b505d..a4921705bc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **44.46 μs** | **0.661 μs** | **0.969 μs** | **4.2114** | **1.0376** | **-** | **51.96 KB** | -| Decrypt | 1 | 56.00 μs | 1.062 μs | 1.589 μs | 5.0049 | 1.2817 | - | 61.57 KB | -| **Encrypt** | **10** | **131.08 μs** | **0.893 μs** | **1.309 μs** | **16.3574** | **3.1738** | **-** | **200.9 KB** | -| Decrypt | 10 | 174.88 μs | 3.443 μs | 4.938 μs | 24.6582 | 4.8828 | - | 303.41 KB | -| **Encrypt** | **100** | **2,052.43 μs** | **230.487 μs** | **344.982 μs** | **160.1563** | **107.4219** | **83.9844** | **1891.44 KB** | -| Decrypt | 100 | 2,791.54 μs | 284.376 μs | 425.641 μs | 234.3750 | 166.0156 | 140.6250 | 3066.91 KB | +| **Encrypt** | **1** | **60.61 μs** | **0.554 μs** | **0.758 μs** | **4.6997** | **1.5869** | **-** | **57.72 KB** | +| Decrypt | 1 | 61.02 μs | 0.984 μs | 1.473 μs | 5.0049 | 1.2817 | - | 61.57 KB | +| **Encrypt** | **10** | **158.70 μs** | **3.114 μs** | **4.565 μs** | **15.8691** | **3.9063** | **-** | **196.51 KB** | +| Decrypt | 10 | 189.60 μs | 1.248 μs | 1.829 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **1,773.67 μs** | **135.590 μs** | **202.944 μs** | **117.1875** | **50.7813** | **41.0156** | **1784.35 KB** | +| Decrypt | 100 | 2,787.37 μs | 264.329 μs | 395.636 μs | 230.4688 | 158.2031 | 136.7188 | 3067.03 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs new file mode 100644 index 0000000000..329e06e7c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -0,0 +1,43 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System.Linq; + using System.Text; + + [TestClass] + public class AeadAes256CbcHmac256AlgorithmTests + { + private static readonly byte[] RootKey = new byte[32]; + + private static AeadAes256CbcHmac256EncryptionKey key; + private static AeadAes256CbcHmac256Algorithm algorithm; + + [ClassInitialize] + public static void ClassInitialize(TestContext testContext) + { + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); + AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); + } + + [TestMethod] + public void EncryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; + + int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); + byte[] cipherTextBytes = new byte[cipherTextLength]; + + int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); + Assert.Equals(encrypted, cipherTextLength); + + byte[] decrypted = algorithm.DecryptData(cipherTextBytes); + + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 1396e7e06e..95a5b66d39 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -24,7 +24,7 @@ public class MdeEncryptionProcessorTests private const string dekId = "dekId"; [ClassInitialize] - public static void ClassInitilize(TestContext testContext) + public static void ClassInitialize(TestContext testContext) { _ = testContext; MdeEncryptionProcessorTests.encryptionOptions = new EncryptionOptions() @@ -38,9 +38,22 @@ public static void ClassInitilize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => + .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] @@ -108,7 +121,7 @@ await EncryptionProcessor.EncryptAsync( [TestMethod] public async Task EncryptDecryptPropertyWithNullValue() - { + { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index 63968723e6..1d90625bc9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,11 +27,26 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } + internal static int EncryptData(byte[] plainText, int inputOffset, int inputLength, byte[] output, int outputOffset) + { + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, cipherText.Length); + + return cipherText.Length; + } + internal static byte[] DecryptData(byte[] cipherText) { return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - + + internal static int DecryptData(byte[] cipherText, int inputOffset, int inputLength, byte[] output, int outputOffset) + { + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); + return plainText.Length; + } + internal static Stream ToStream(T input) { string s = JsonConvert.SerializeObject(input); @@ -48,14 +63,9 @@ internal static T FromStream(Stream stream) } } - private static JObject ParseStream(Stream stream) - { - return JObject.Load(new JsonTextReader(new StreamReader(stream))); - } - internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt" }; + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveArr", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } @@ -68,17 +78,12 @@ internal class TestDoc public int SensitiveInt { get; set; } - public TestDoc() - { - } + public string[] SensitiveArr { get; set; } - public TestDoc(TestDoc other) + public Dictionary SensitiveDict { get; set; } + + public TestDoc() { - this.Id = other.Id; - this.PK = other.PK; - this.NonSensitive = other.NonSensitive; - this.SensitiveStr = other.SensitiveStr; - this.SensitiveInt = other.SensitiveInt; } public override bool Equals(object obj) @@ -88,7 +93,9 @@ public override bool Equals(object obj) && this.PK == doc.PK && this.NonSensitive == doc.NonSensitive && this.SensitiveInt == doc.SensitiveInt - && this.SensitiveStr == this.SensitiveStr; + && this.SensitiveStr == doc.SensitiveStr + && this.SensitiveArr?.Equals(doc.SensitiveArr) == true + && this.SensitiveDict?.Equals(doc.SensitiveDict) == true; } public override int GetHashCode() @@ -99,6 +106,8 @@ public override int GetHashCode() hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveStr); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveInt); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveArr); + hashCode = (hashCode * -1521134295) + EqualityComparer>.Default.GetHashCode(this.SensitiveDict); return hashCode; } @@ -110,7 +119,20 @@ public static TestDoc Create(string partitionKey = null) PK = partitionKey ?? Guid.NewGuid().ToString(), NonSensitive = Guid.NewGuid().ToString(), SensitiveStr = Guid.NewGuid().ToString(), - SensitiveInt = new Random().Next() + SensitiveInt = new Random().Next(), + SensitiveArr = new string[] + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }, + SensitiveDict = new Dictionary + { + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + } }; } From 14bce371afb7208d4381bd0312e01154ca154313 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:16:05 +0200 Subject: [PATCH 14/85] ~ drop repeated DEK calls ~ reduce validations overhead --- .../src/CosmosEncryptor.cs | 104 +++++++++--------- .../src/EncryptionProcessor.cs | 19 ++-- .../src/Encryptor.cs | 10 ++ 3 files changed, 71 insertions(+), 62 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 0288ee79db..2ded90893b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -29,8 +29,7 @@ public CosmosEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) } /// - public override async Task DecryptAsync( - byte[] cipherText, + public override async Task GetEncryptionKeyAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) @@ -45,91 +44,94 @@ public override async Task DecryptAsync( throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); } - return dek.DecryptData(cipherText); + return dek; } /// - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.")] + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + return dek.DecryptData(cipherText); + } + + /// + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.")] + public override async Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); } /// + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.")] public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.EncryptData(plainText); } /// - public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.")] + public override async Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } /// - public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + public override async Task GetEncryptBytesCountAsync( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetEncryptByteCount(plainTextLength); } /// - public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + public override async Task GetDecryptBytesCountAsync( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetDecryptByteCount(cipherTextLength); } + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index a0ea6fa213..a964a04d6d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -55,19 +55,19 @@ public static async Task EncryptAsync( return input; } - if (!encryptionOptions.PathsToEncrypt.Distinct().SequenceEqual(encryptionOptions.PathsToEncrypt)) + if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } foreach (string path in encryptionOptions.PathsToEncrypt) { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf('/') != 0) + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf(',', 1) != -1) { throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); } - if (string.Equals(path.Substring(1), "id")) + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.InvariantCulture)) { throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); } @@ -99,22 +99,19 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( - plainText.Length, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + + int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; cipherTextWithTypeMarker[0] = (byte)typeMarker; - int encryptedBytesCount = await encryptor.EncryptAsync( + int encryptedBytesCount = encryptionKey.EncryptData( plainText, plainTextOffset: 0, plainTextLength: plainText.Length, cipherTextWithTypeMarker, - outputOffset: 1, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm); + outputOffset: 1); if (encryptedBytesCount < 0) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 469d069313..2fd857eeb4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -49,12 +49,22 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Retrieve Data Encryption Key. + /// + /// Identifier of the data encryption key. + /// Identifier of the encryption algorithm. + /// Token for cancellation. + /// Data Encryption Key + public abstract Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// /// Calculate size of input after encryption. /// /// Input data size. /// Identifier of the data encryption key. /// Identifier for the encryption algorithm. + /// Data Encryption Key used. /// Token for cancellation. /// Size of input when encrypted. public abstract Task GetEncryptBytesCountAsync( From 4ff1601869dbc2801c6f57e24573d00a1dde4ab8 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:22:48 +0200 Subject: [PATCH 15/85] ! typo --- Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 2ded90893b..7e705453ce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -121,7 +121,7 @@ public override async Task GetEncryptBytesCountAsync( } /// - [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.")] public override async Task GetDecryptBytesCountAsync( int cipherTextLength, string dataEncryptionKeyId, From d8a345cc26c744a07ef68f22c69e0668679f8d1c Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:24:47 +0200 Subject: [PATCH 16/85] ~ update benchmark --- .../Readme.md | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index f0166dd8b8..dd87b34c36 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,20 +1,19 @@ -``` ini +``` ini BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.108 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **36.44 μs** | **0.518 μs** | **0.759 μs** | **36.40 μs** | **4.0894** | **1.0376** | **-** | **50.81 KB** | -| Decrypt | 1 | 44.51 μs | 0.900 μs | 1.291 μs | 43.83 μs | 4.8828 | 1.2207 | - | 60.1 KB | -| **Encrypt** | **10** | **114.63 μs** | **3.234 μs** | **4.841 μs** | **113.80 μs** | **16.2354** | **3.2959** | **-** | **199.75 KB** | -| Decrypt | 10 | 146.56 μs | 1.205 μs | 1.766 μs | 146.08 μs | 24.4141 | 4.6387 | - | 301.94 KB | -| **Encrypt** | **100** | **1,784.59 μs** | **180.928 μs** | **270.805 μs** | **1,743.86 μs** | **158.2031** | **95.7031** | **82.0313** | **1890.27 KB** | -| Decrypt | 100 | 2,772.68 μs | 261.923 μs | 392.035 μs | 2,648.81 μs | 236.3281 | 158.2031 | 142.5781 | 3064.91 KB | - +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| +| **Encrypt** | **1** | **37.87 μs** | **0.699 μs** | **0.979 μs** | **3.7842** | **0.9766** | **-** | **47.03 KB** | +| Decrypt | 1 | 47.27 μs | 1.148 μs | 1.683 μs | 4.3945 | 1.0986 | - | 54.17 KB | +| **Encrypt** | **10** | **113.49 μs** | **1.380 μs** | **1.934 μs** | **15.1367** | **3.0518** | **-** | **185.81 KB** | +| Decrypt | 10 | 146.93 μs | 1.724 μs | 2.581 μs | 19.5313 | 2.1973 | - | 239.94 KB | +| **Encrypt** | **100** | **1,610.57 μs** | **161.738 μs** | **242.081 μs** | **150.3906** | **107.4219** | **74.2188** | **1773.64 KB** | +| Decrypt | 100 | 2,058.97 μs | 202.464 μs | 303.039 μs | 160.1563 | 107.4219 | 76.1719 | 2042.61 KB | From 03c06e05ce457d3a0fc5c24d09d0d5ec53cc8c79 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 16:22:22 +0200 Subject: [PATCH 17/85] ~ fix tests ~ update api file --- .../src/AeadAes256CbcHmac256Algorithm.cs | 10 +- .../EmulatorTests/LegacyEncryptionTests.cs | 39 ++++- .../EmulatorTests/MdeCustomEncryptionTests.cs | 104 ++++++------- .../AeadAes256CbcHmac256AlgorithmTests.cs | 2 +- .../DotNetSDKEncryptionCustomAPI.json | 139 +++++++++++++++++- .../MdeEncryptionProcessorTests.cs | 13 ++ .../Contracts/ContractEnforcement.cs | 2 +- 7 files changed, 235 insertions(+), 74 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index a51e09726b..903e620207 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -158,7 +158,15 @@ public override byte[] EncryptData(byte[] plainText) /// Returns the ciphertext corresponding to the plaintext. public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) { - throw new NotImplementedException(); + var buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); + + if (buffer.Length > output.Length-outputOffset) + { + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + } + + buffer.CopyTo(output, outputOffset); + return buffer.Length; } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 9150a739cf..6f5b798ed3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -922,7 +922,7 @@ public async Task EncryptionTransactionalBatchWithCustomSerializer() await LegacyEncryptionTests.VerifyItemByReadAsync(LegacyEncryptionTests.itemContainer, doc1ToReplace); } - [TestMethod] + [TestMethod] public async Task VerifyDekOperationWithSystemTextSerializer() { System.Text.Json.JsonSerializerOptions jsonSerializerOptions = new System.Text.Json.JsonSerializerOptions() @@ -1037,6 +1037,23 @@ await LegacyEncryptionTests.IterateDekFeedAsync( isResultOrderExpected: false, "SELECT * from c", itemCountInPage: 3); + + + //clean up + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } } [TestMethod] @@ -1750,7 +1767,7 @@ public override async Task DecryptAsync( throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1770,7 +1787,7 @@ public override async Task DecryptAsync(byte[] cipherText, int cipherTextOf throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1789,7 +1806,7 @@ public override async Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1799,7 +1816,7 @@ public override async Task EncryptAsync( public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1809,7 +1826,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1819,13 +1836,21 @@ public override async Task GetEncryptBytesCountAsync(int plainTextLength, s public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetDecryptByteCount(cipherTextLength); } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + return await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index cb20c71c58..d5f27926fd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -69,9 +69,9 @@ public static async Task ClassInitialize(TestContext context) MdeCustomEncryptionTests.encryptor = new TestEncryptor(MdeCustomEncryptionTests.dekProvider); MdeCustomEncryptionTests.encryptionContainer = MdeCustomEncryptionTests.itemContainer.WithEncryptor(encryptor); MdeCustomEncryptionTests.encryptionContainerForChangeFeed = MdeCustomEncryptionTests.itemContainerForChangeFeed.WithEncryptor(encryptor); + await MdeCustomEncryptionTests.dekProvider.InitializeAsync(MdeCustomEncryptionTests.database, MdeCustomEncryptionTests.keyContainer.Id); MdeCustomEncryptionTests.dekProperties = await MdeCustomEncryptionTests.CreateDekAsync(MdeCustomEncryptionTests.dekProvider, MdeCustomEncryptionTests.dekId); - } [ClassCleanup] @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(48, unwrapcount); + Assert.AreEqual(32, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); @@ -1153,6 +1153,22 @@ await MdeCustomEncryptionTests.IterateDekFeedAsync( isResultOrderExpected: false, "SELECT * from c", itemCountInPage: 3); + + // cleanup + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } } [TestMethod] @@ -2218,54 +2234,36 @@ private class TestEncryptor : Encryptor public DataEncryptionKeyProvider DataEncryptionKeyProvider { get; } public bool FailDecryption { get; set; } + private readonly CosmosEncryptor encryptor; + public TestEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) { - this.DataEncryptionKeyProvider = dataEncryptionKeyProvider; + this.encryptor = new CosmosEncryptor(dataEncryptionKeyProvider); this.FailDecryption = false; } - public override async Task DecryptAsync( - byte[] cipherText, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default) + private void ThrowIfFail(string dataEncryptionKeyId) { if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) { throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } + } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText); + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task EncryptAsync( @@ -2274,42 +2272,32 @@ public override async Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, plainTextOffset, plainTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetEncryptByteCount(plainTextLength); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptBytesCountAsync(plainTextLength, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetDecryptBytesCountAsync(cipherTextLength, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); + } - return dek.GetDecryptByteCount(cipherTextLength); + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 329e06e7c5..9c2125f903 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -33,7 +33,7 @@ public void EncryptUsingBufferDecryptsSuccessfully() byte[] cipherTextBytes = new byte[cipherTextLength]; int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); - Assert.Equals(encrypted, cipherTextLength); + Assert.AreEqual(encrypted, cipherTextLength); byte[] decrypted = algorithm.DecryptData(cipherTextBytes); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json index 1912d16407..ccb4b56fff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json @@ -113,20 +113,61 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider get_DataEncryptionKeyProvider();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ "AsyncStateMachineAttribute" ], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -163,6 +204,26 @@ "Attributes": [], "MethodInfo": "Byte[] RawKey;CanRead:True;CanWrite:False;Byte[] get_RawKey();IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Int32 DecryptData(Byte[], Int32, Int32, Byte[], Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 DecryptData(Byte[], Int32, Int32, Byte[], Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 EncryptData(Byte[], Int32, Int32, Byte[], Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 EncryptData(Byte[], Int32, Int32, Byte[], Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 GetDecryptByteCount(Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 GetDecryptByteCount(Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 GetEncryptByteCount(Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 GetEncryptByteCount(Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey Create(Byte[], System.String)": { "Type": "Method", "Attributes": [], @@ -1064,20 +1125,61 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider get_DataEncryptionKeyProvider();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ "AsyncStateMachineAttribute" ], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1088,6 +1190,11 @@ } }, "Members": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)": { "Type": "Method", "Attributes": [], @@ -1097,6 +1204,26 @@ "Type": "Method", "Attributes": [], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, "NestedTypes": {} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 95a5b66d39..aec838a413 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -34,7 +34,20 @@ public static void ClassInitialize(TestContext testContext) PathsToEncrypt = TestDoc.PathsToEncrypt }; + var DekMock = new Mock(); + DekMock.Setup(m => m.EncryptData(It.IsAny())) + .Returns((byte[] plainText) => TestCommon.EncryptData(plainText)); + DekMock.Setup(m => m.GetEncryptByteCount(It.IsAny())) + .Returns((int plainTextLength) => plainTextLength); + DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + + MdeEncryptionProcessorTests.mockEncryptor = new Mock(); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string dekId, string algorithm, CancellationToken token) => + dekId == MdeEncryptionProcessorTests.dekId ? DekMock.Object : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index f0e127495a..c58109d65b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -200,7 +200,7 @@ public static void ValidateContractContainBreakingChanges( File.WriteAllText($"Contracts/{breakingChangesPath}", localJson); string baselineJson = GetBaselineContract(baselinePath); - ContractEnforcement.ValidateJsonAreSame(localJson, baselineJson); + ContractEnforcement.ValidateJsonAreSame(baselineJson, localJson); } public static void ValidateTelemetryContractContainBreakingChanges( From 3bf77c89a6c571095d6b95b40e8649afee7796b6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:27:54 +0200 Subject: [PATCH 18/85] ~ cleanup --- .../src/AeadAes256CbcHmac256Algorithm.cs | 24 +++++++++++++++---- .../src/EncryptionProcessor.cs | 6 ++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index 903e620207..e50c60e8fd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -158,11 +158,11 @@ public override byte[] EncryptData(byte[] plainText) /// Returns the ciphertext corresponding to the plaintext. public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) { - var buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); + byte[] buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); - if (buffer.Length > output.Length-outputOffset) + if (buffer.Length > output.Length - outputOffset) { - throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes."); } buffer.CopyTo(output, outputOffset); @@ -453,7 +453,15 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) { - throw new NotImplementedException(); + byte[] buffer = this.DecryptData(cipherText.AsSpan(cipherTextOffset, cipherTextLength).ToArray(), true); + + if (buffer.Length > output.Length - outputOffset) + { + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + } + + buffer.CopyTo(output, outputOffset); + return buffer.Length; } public override int GetEncryptByteCount(int plainTextLength) @@ -464,7 +472,13 @@ public override int GetEncryptByteCount(int plainTextLength) public override int GetDecryptByteCount(int cipherTextLength) { - throw new NotImplementedException(); + int value = cipherTextLength - (sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes); + if (value < BlockSizeInBytes) + { + throw new ArgumentOutOfRangeException(nameof(cipherTextLength), $"Cipher text length is too short."); + } + + return value; } private static int GetCipherTextLength(int inputSize) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index a964a04d6d..a6ee23b4e5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -55,19 +55,19 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (!encryptionOptions.PathsToEncrypt.Distinct().SequenceEqual(encryptionOptions.PathsToEncrypt)) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } foreach (string path in encryptionOptions.PathsToEncrypt) { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf(',', 1) != -1) + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf('/') != 0) { throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); } - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.InvariantCulture)) + if (string.Equals(path.Substring(1), "id")) { throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); } From ceaa8b5ee346a3ad779a79521cf95da90176f4d9 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:39:52 +0200 Subject: [PATCH 19/85] + refresh benchmark --- .../Readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index dd87b34c36..64a686244c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-rc.1.24452.12 +.NET SDK=8.0.400 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| -| **Encrypt** | **1** | **37.87 μs** | **0.699 μs** | **0.979 μs** | **3.7842** | **0.9766** | **-** | **47.03 KB** | -| Decrypt | 1 | 47.27 μs | 1.148 μs | 1.683 μs | 4.3945 | 1.0986 | - | 54.17 KB | -| **Encrypt** | **10** | **113.49 μs** | **1.380 μs** | **1.934 μs** | **15.1367** | **3.0518** | **-** | **185.81 KB** | -| Decrypt | 10 | 146.93 μs | 1.724 μs | 2.581 μs | 19.5313 | 2.1973 | - | 239.94 KB | -| **Encrypt** | **100** | **1,610.57 μs** | **161.738 μs** | **242.081 μs** | **150.3906** | **107.4219** | **74.2188** | **1773.64 KB** | -| Decrypt | 100 | 2,058.97 μs | 202.464 μs | 303.039 μs | 160.1563 | 107.4219 | 76.1719 | 2042.61 KB | +| **Encrypt** | **1** | **37.15 μs** | **0.683 μs** | **1.002 μs** | **3.8452** | **0.9766** | **-** | **47.28 KB** | +| Decrypt | 1 | 45.29 μs | 0.757 μs | 1.062 μs | 4.3945 | 1.0986 | - | 54.17 KB | +| **Encrypt** | **10** | **111.83 μs** | **1.252 μs** | **1.874 μs** | **15.1367** | **3.0518** | **-** | **186.07 KB** | +| Decrypt | 10 | 151.46 μs | 2.259 μs | 3.311 μs | 19.5313 | 2.1973 | - | 239.94 KB | +| **Encrypt** | **100** | **1,567.24 μs** | **153.944 μs** | **230.416 μs** | **152.3438** | **109.3750** | **76.1719** | **1773.95 KB** | +| Decrypt | 100 | 2,088.77 μs | 232.084 μs | 347.372 μs | 160.1563 | 113.2813 | 76.1719 | 2042.61 KB | From 611b3ac485adc698df5b3a7fe062cf8c4e42af0e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:55:26 +0200 Subject: [PATCH 20/85] + unit test --- .../AeadAes256CbcHmac256AlgorithmTests.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 9c2125f903..900fac6798 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests { using Microsoft.Azure.Cosmos.Encryption.Custom; using Microsoft.VisualStudio.TestTools.UnitTesting; + using System; using System.Linq; using System.Text; @@ -20,6 +21,8 @@ public class AeadAes256CbcHmac256AlgorithmTests [ClassInitialize] public static void ClassInitialize(TestContext testContext) { + _ = testContext; + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); } @@ -39,5 +42,20 @@ public void EncryptUsingBufferDecryptsSuccessfully() Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); } + + [TestMethod] + public void DecryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 }; + byte[] encrypted = algorithm.EncryptData(plainTextBytes); + + int plainTextMaxLength = algorithm.GetDecryptByteCount(encrypted.Length); + byte[] decrypted = new byte[plainTextMaxLength]; + + int decryptedBytes = algorithm.DecryptData(encrypted, 0, encrypted.Length, decrypted, 0); + + Assert.AreEqual(plainTextBytes.Length, decryptedBytes); + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted.AsSpan(0, decryptedBytes).ToArray())); + } } } From 8a78fe87cd516919f88782d686e1b2b1e5f85e4b Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 14:35:47 +0200 Subject: [PATCH 21/85] ~ merge fixes and initial cleanup --- .../src/EncryptionProcessor.cs | 50 +++++++++---------- .../src/Encryptor.cs | 36 ------------- .../Readme.md | 16 +++--- 3 files changed, 31 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b657bf40ed..245e99b69d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -49,6 +49,8 @@ public static async Task EncryptAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + EncryptionProcessor.ValidateInputForEncrypt( input, encryptor, @@ -118,7 +120,7 @@ public static async Task EncryptAsync( cipherTextWithTypeMarker[0] = (byte)typeMarker; - int encryptedBytesCount = await encryptionKey.EncryptAsync( + int encryptedBytesCount = encryptionKey.EncryptData( plainText, plainTextOffset: 0, plainTextLength, @@ -287,6 +289,13 @@ private static async Task MdeEncAlgoDecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); JObject plainTextJObj = new JObject(); @@ -304,22 +313,18 @@ private static async Task MdeEncAlgoDecryptObjectAsync( continue; } - int plainTextLength = await encryptor.GetDecryptBytesCountAsync( - cipherTextWithTypeMarker.Length - 1, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( - encryptionProperties, + int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, - plainText, - encryptor, - diagnosticsContext, - cancellationToken); + plainText); EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], @@ -357,30 +362,19 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static async Task MdeEncAlgoDecryptPropertyAsync( - EncryptionProperties encryptionProperties, + private static int MdeEncAlgoDecryptPropertyAsync( + DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextOffset, int cipherTextLength, - byte[] buffer, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + byte[] buffer) { - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - int decryptedCount = await encryptor.DecryptAsync( + int decryptedCount = encryptionKey.DecryptData( cipherText, cipherTextOffset, cipherTextLength, buffer, - outputOffset: 0, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm, - cancellationToken); + outputOffset: 0); if (decryptedCount < 0) { @@ -397,6 +391,8 @@ private static async Task LegacyEncAlgoDecryptContentAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + if (encryptionProperties.EncryptionFormatVersion != 2) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 51ce14a7cd..2fd857eeb4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -122,41 +122,5 @@ public abstract Task GetDecryptBytesCountAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); - - /// - /// Decrypts the cipherText using the key and algorithm provided. - /// - /// Ciphertext to be decrypted. - /// Offset in the cipherText array at which to begin using data from. - /// Number of bytes in the cipherText array to use as input. - /// Output buffer to write the decrypted data to. - /// Offset in the output array at which to begin writing data to. - /// Identifier of the data encryption key. - /// Identifier for the encryption algorithm. - /// Token for cancellation. - /// Plain text. - public abstract Task DecryptAsync( - byte[] cipherText, - int cipherTextOffset, - int cipherTextLength, - byte[] output, - int outputOffset, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default); - - /// - /// Calculate upper bound size of the input after decryption. - /// - /// Input data size. - /// Identifier of the data encryption key. - /// Identifier for the encryption algorithm. - /// Token for cancellation. - /// Upper bound size of the input when decrypted. - public abstract Task GetDecryptBytesCountAsync( - int cipherTextLength, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 64a686244c..01e1e3a41a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| -| **Encrypt** | **1** | **37.15 μs** | **0.683 μs** | **1.002 μs** | **3.8452** | **0.9766** | **-** | **47.28 KB** | -| Decrypt | 1 | 45.29 μs | 0.757 μs | 1.062 μs | 4.3945 | 1.0986 | - | 54.17 KB | -| **Encrypt** | **10** | **111.83 μs** | **1.252 μs** | **1.874 μs** | **15.1367** | **3.0518** | **-** | **186.07 KB** | -| Decrypt | 10 | 151.46 μs | 2.259 μs | 3.311 μs | 19.5313 | 2.1973 | - | 239.94 KB | -| **Encrypt** | **100** | **1,567.24 μs** | **153.944 μs** | **230.416 μs** | **152.3438** | **109.3750** | **76.1719** | **1773.95 KB** | -| Decrypt | 100 | 2,088.77 μs | 232.084 μs | 347.372 μs | 160.1563 | 113.2813 | 76.1719 | 2042.61 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **37.10 μs** | **0.527 μs** | **0.788 μs** | **36.82 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | +| Decrypt | 1 | 43.43 μs | 0.686 μs | 1.027 μs | 43.06 μs | 3.9673 | 1.0376 | - | 49.16 KB | +| **Encrypt** | **10** | **115.97 μs** | **2.256 μs** | **3.377 μs** | **117.36 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | +| Decrypt | 10 | 144.93 μs | 2.238 μs | 3.350 μs | 146.64 μs | 15.6250 | 3.1738 | - | 194.3 KB | +| **Encrypt** | **100** | **1,604.78 μs** | **150.864 μs** | **225.806 μs** | **1,557.92 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | +| Decrypt | 100 | 1,943.81 μs | 193.233 μs | 289.223 μs | 1,872.14 μs | 126.9531 | 78.1250 | 42.9688 | 1586.28 KB | From 8ed21357a28d393f6148462dac9ae37ff437e70d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 15:06:56 +0200 Subject: [PATCH 22/85] ~ write directly to output document instead of copying --- .../src/EncryptionProcessor.cs | 21 +++----- .../EmulatorTests/MdeCustomEncryptionTests.cs | 50 ------------------- .../Readme.md | 16 +++--- 3 files changed, 16 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 245e99b69d..cd95f1540b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -298,7 +298,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - JObject plainTextJObj = new JObject(); + List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); @@ -329,15 +329,10 @@ private static async Task MdeEncAlgoDecryptObjectAsync( EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), - plainTextJObj, + document, propertyName); - } - List pathsDecrypted = new List(); - foreach (JProperty property in plainTextJObj.Properties()) - { - document[property.Name] = property.Value; - pathsDecrypted.Add("/" + property.Name); + pathsDecrypted.Add(path); } DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( @@ -557,16 +552,16 @@ private static void DeserializeAndAddProperty( switch (typeMarker) { case TypeMarker.Boolean: - jObject.Add(key, SqlBoolSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); break; case TypeMarker.Double: - jObject.Add(key, SqlDoubleSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); break; case TypeMarker.Long: - jObject.Add(key, SqlLongSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); break; case TypeMarker.String: - jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); break; case TypeMarker.Array: DeserializeAndAddProperty(serializedBytes); @@ -592,7 +587,7 @@ void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); using JsonTextReader reader = new JsonTextReader(memoryTextReader); - jObject.Add(key, serializer.Deserialize(reader)); + jObject[key] = serializer.Deserialize(reader); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 277bacbe89..df51ab0cc5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2266,26 +2266,6 @@ public override async Task DecryptAsync(byte[] cipherText, int cipherTextOf return await this.encryptor.DecryptAsync(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); - } - public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2319,36 +2299,6 @@ public override async Task GetEncryptionKeyAsync(string dataE this.ThrowIfFail(dataEncryptionKeyId); return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } - - public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); - } - - public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetEncryptByteCount(plainTextLength); - } - - public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetDecryptByteCount(cipherTextLength); - } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 01e1e3a41a..7ffdbdcd60 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **37.10 μs** | **0.527 μs** | **0.788 μs** | **36.82 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | -| Decrypt | 1 | 43.43 μs | 0.686 μs | 1.027 μs | 43.06 μs | 3.9673 | 1.0376 | - | 49.16 KB | -| **Encrypt** | **10** | **115.97 μs** | **2.256 μs** | **3.377 μs** | **117.36 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | -| Decrypt | 10 | 144.93 μs | 2.238 μs | 3.350 μs | 146.64 μs | 15.6250 | 3.1738 | - | 194.3 KB | -| **Encrypt** | **100** | **1,604.78 μs** | **150.864 μs** | **225.806 μs** | **1,557.92 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | -| Decrypt | 100 | 1,943.81 μs | 193.233 μs | 289.223 μs | 1,872.14 μs | 126.9531 | 78.1250 | 42.9688 | 1586.28 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **37.28 μs** | **0.676 μs** | **0.991 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | +| Decrypt | 1 | 43.12 μs | 1.753 μs | 2.623 μs | 3.6011 | 1.1597 | - | 44.63 KB | +| **Encrypt** | **10** | **115.55 μs** | **1.717 μs** | **2.570 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | +| Decrypt | 10 | 121.81 μs | 2.127 μs | 3.050 μs | 12.9395 | 3.1738 | - | 159.56 KB | +| **Encrypt** | **100** | **1,571.06 μs** | **128.737 μs** | **192.687 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | +| Decrypt | 100 | 1,687.55 μs | 143.998 μs | 215.529 μs | 101.5625 | 62.5000 | 44.9219 | 1253.19 KB | From bbe98458081b72b5a53b8e70d2ca9ba873610b2e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 15:22:54 +0200 Subject: [PATCH 23/85] ! tests --- .../EmulatorTests/LegacyEncryptionTests.cs | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index c39441fb80..d63f24755c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -103,7 +103,7 @@ public async Task EncryptionCreateDekWithMdeAlgorithmFails() { Assert.AreEqual("For use of 'MdeAeadAes256CbcHmac256Randomized' algorithm, Encryptor or CosmosDataEncryptionKeyProvider needs to be initialized with EncryptionKeyStoreProvider.", ex.Message); } - } + } [TestMethod] public async Task EncryptionRewrapDek() @@ -217,7 +217,7 @@ await LegacyEncryptionTests.IterateDekFeedAsync( QueryDefinition queryDefinition = new QueryDefinition("SELECT * from c where c.id >= @startId and c.id <= @endId ORDER BY c.id ASC") .WithParameter("@startId", "Contoso_v000") .WithParameter("@endId", "Contoso_v999"); - + await LegacyEncryptionTests.IterateDekFeedAsync( dekProvider, new List { contosoV1, contosoV2 }, @@ -405,7 +405,7 @@ public async Task EncryptionChangeFeedDecryptionSuccessful() TestDoc testDoc1 = await LegacyEncryptionTests.CreateItemAsync(LegacyEncryptionTests.encryptionContainer, LegacyEncryptionTests.dekId, TestDoc.PathsToEncrypt); TestDoc testDoc2 = await LegacyEncryptionTests.CreateItemAsync(LegacyEncryptionTests.encryptionContainer, dek2, TestDoc.PathsToEncrypt); - + // change feed iterator await this.ValidateChangeFeedIteratorResponse(LegacyEncryptionTests.encryptionContainer, testDoc1, testDoc2); @@ -597,7 +597,7 @@ public async Task EncryptionRudItemLazyDecryption() { TestDoc testDoc = TestDoc.Create(); // Upsert (item doesn't exist) - ItemResponse > upsertResponse = await LegacyEncryptionTests.encryptionContainer.UpsertItemAsync( + ItemResponse> upsertResponse = await LegacyEncryptionTests.encryptionContainer.UpsertItemAsync( new EncryptableItem(testDoc), new PartitionKey(testDoc.PK), LegacyEncryptionTests.GetRequestOptions(LegacyEncryptionTests.dekId, TestDoc.PathsToEncrypt)); @@ -660,7 +660,7 @@ public async Task EncryptionResourceTokenAuthRestricted() restrictedUserPermission.Token); Database databaseForRestrictedUser = clientForRestrictedUser.GetDatabase(LegacyEncryptionTests.database.Id); - Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(LegacyEncryptionTests.itemContainer.Id); + Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(LegacyEncryptionTests.itemContainer.Id); Container encryptionContainerForRestrictedUser = containerForRestrictedUser.WithEncryptor(encryptor); await LegacyEncryptionTests.PerformForbiddenOperationAsync(() => @@ -939,15 +939,15 @@ public async Task VerifyDekOperationWithSystemTextSerializer() // get database and container Database databaseWithCosmosSystemTextJsonSerializer = clientWithCosmosSystemTextJsonSerializer.GetDatabase(LegacyEncryptionTests.database.Id); Container containerWithCosmosSystemTextJsonSerializer = databaseWithCosmosSystemTextJsonSerializer.GetContainer(LegacyEncryptionTests.itemContainer.Id); - + // create the Dek container Container dekContainerWithCosmosSystemTextJsonSerializer = await databaseWithCosmosSystemTextJsonSerializer.CreateContainerAsync(Guid.NewGuid().ToString(), "/id", 400); - + CosmosDataEncryptionKeyProvider dekProviderWithCosmosSystemTextJsonSerializer = new CosmosDataEncryptionKeyProvider(new TestKeyWrapProvider()); await dekProviderWithCosmosSystemTextJsonSerializer.InitializeAsync(databaseWithCosmosSystemTextJsonSerializer, dekContainerWithCosmosSystemTextJsonSerializer.Id); - - TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new TestEncryptor(dekProviderWithCosmosSystemTextJsonSerializer); - + + TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new TestEncryptor(dekProviderWithCosmosSystemTextJsonSerializer); + // enable encryption on container Container encryptionContainerWithCosmosSystemTextJsonSerializer = containerWithCosmosSystemTextJsonSerializer.WithEncryptor(encryptorWithCosmosSystemTextJsonSerializer); @@ -991,7 +991,7 @@ public async Task VerifyDekOperationWithSystemTextSerializer() ItemResponse createTestDoc = await encryptionContainerWithCosmosSystemTextJsonSerializer.CreateItemAsync( testDocSystemText, new PartitionKey(testDocSystemText.PartitionKey), - LegacyEncryptionTests.GetRequestOptions(dekId, new List() { "/status"})); + LegacyEncryptionTests.GetRequestOptions(dekId, new List() { "/status" })); Assert.AreEqual(HttpStatusCode.Created, createTestDoc.StatusCode); @@ -1074,7 +1074,7 @@ public async Task EncryptionTransactionalBatchConflictResponse() Assert.AreEqual(HttpStatusCode.Conflict, batchResponse.StatusCode); Assert.AreEqual(1, batchResponse.Count); } - + private static async Task ValidateSprocResultsAsync(Container container, TestDoc expectedDoc) { string sprocId = Guid.NewGuid().ToString(); @@ -1129,7 +1129,7 @@ private static async Task ValidateQueryResultsAsync( queryResponseIterator = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); } - + FeedResponse readDocs = await queryResponseIterator.ReadNextAsync(); Assert.AreEqual(null, readDocs.ContinuationToken); @@ -1373,7 +1373,7 @@ private static async Task IterateDekFeedAsync( : dekProvider.DataEncryptionKeyContainer.GetDataEncryptionKeyQueryIterator( query, requestOptions: requestOptions); - + Assert.IsTrue(dekIterator.HasMoreResults); List readDekIds = new List(); @@ -1588,7 +1588,7 @@ private static async Task CreateDekAsync(CosmosData LegacyEncryptionTests.metadata1); Assert.AreEqual(HttpStatusCode.Created, dekResponse.StatusCode); - + return VerifyDekResponse(dekResponse, dekId); } @@ -1607,7 +1607,7 @@ private static DataEncryptionKeyProperties VerifyDekResponse( Assert.IsNotNull(dekProperties.SelfLink); Assert.IsNotNull(dekProperties.CreatedTime); Assert.IsNotNull(dekProperties.LastModified); - + return dekProperties; } @@ -1737,7 +1737,7 @@ public override Task WrapKeyAsync(byte[] key, Encryptio { this.WrapKeyCallsCount[metadata.Value]++; } - + EncryptionKeyWrapMetadata responseMetadata = new EncryptionKeyWrapMetadata(metadata.Value + LegacyEncryptionTests.metadataUpdateSuffix); int moveBy = metadata.Value == LegacyEncryptionTests.metadata1.Value ? 1 : 2; return Task.FromResult(new EncryptionKeyWrapResult(key.Select(b => (byte)(b + moveBy)).ToArray(), responseMetadata)); @@ -1813,7 +1813,45 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } - } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + return await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + } + } internal class CustomSerializer : CosmosSerializer { From a107f62b848c4a9eba5c00c94cc6acf05375d1ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 16:02:55 +0200 Subject: [PATCH 24/85] ~ retrieve DataEncryptionKey only once per document --- .../src/EncryptionProcessor.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index cd95f1540b..0370b94771 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -92,6 +92,8 @@ public static async Task EncryptAsync( { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { string propertyName = pathToEncrypt.Substring(1); @@ -112,8 +114,6 @@ public static async Task EncryptAsync( continue; } - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); @@ -298,6 +298,8 @@ private static async Task MdeEncAlgoDecryptObjectAsync( using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { @@ -313,8 +315,6 @@ private static async Task MdeEncAlgoDecryptObjectAsync( continue; } - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); From a1ad02b9095dae60db6e9ccc9df098d0962f9df7 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 16:16:27 +0200 Subject: [PATCH 25/85] ! fix tests ~ bump benchmark --- .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- .../Readme.md | 16 ++++++++-------- .../MdeEncryptionProcessorTests.cs | 5 ++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index df51ab0cc5..ec98b39567 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(64, unwrapcount); + Assert.AreEqual(4, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 7ffdbdcd60..4df982a29d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **37.28 μs** | **0.676 μs** | **0.991 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | -| Decrypt | 1 | 43.12 μs | 1.753 μs | 2.623 μs | 3.6011 | 1.1597 | - | 44.63 KB | -| **Encrypt** | **10** | **115.55 μs** | **1.717 μs** | **2.570 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | -| Decrypt | 10 | 121.81 μs | 2.127 μs | 3.050 μs | 12.9395 | 3.1738 | - | 159.56 KB | -| **Encrypt** | **100** | **1,571.06 μs** | **128.737 μs** | **192.687 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | -| Decrypt | 100 | 1,687.55 μs | 143.998 μs | 215.529 μs | 101.5625 | 62.5000 | 44.9219 | 1253.19 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **29.20 μs** | **0.607 μs** | **0.871 μs** | **29.26 μs** | **3.3875** | **2.5940** | **-** | **41.51 KB** | +| Decrypt | 1 | 32.23 μs | 0.528 μs | 0.790 μs | 32.78 μs | 3.2349 | 0.7935 | - | 39.71 KB | +| **Encrypt** | **10** | **107.83 μs** | **1.975 μs** | **2.956 μs** | **107.93 μs** | **13.7939** | **0.6104** | **-** | **170.14 KB** | +| Decrypt | 10 | 116.25 μs | 1.537 μs | 2.301 μs | 117.15 μs | 12.5732 | 1.2207 | - | 154.63 KB | +| **Encrypt** | **100** | **1,493.01 μs** | **314.932 μs** | **471.376 μs** | **1,482.68 μs** | **214.8438** | **173.8281** | **140.6250** | **1655.51 KB** | +| Decrypt | 100 | 1,406.56 μs | 126.286 μs | 189.019 μs | 1,408.25 μs | 138.6719 | 103.5156 | 82.0313 | 1248.58 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index aec838a413..3e626a7453 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -41,7 +41,10 @@ public static void ClassInitialize(TestContext testContext) .Returns((int plainTextLength) => plainTextLength); DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); - + DekMock.Setup(m => m.DecryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.DecryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.GetDecryptByteCount(It.IsAny())) + .Returns((int cipherTextLength) => cipherTextLength); MdeEncryptionProcessorTests.mockEncryptor = new Mock(); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) From 4f2f072d2c9b3f8025203b425c1801c5302ae538 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 2 Oct 2024 11:40:26 +0200 Subject: [PATCH 26/85] ~ update Aes algorithm to reuse GetEncryptedByteCount --- .../src/AeadAes256CbcHmac256Algorithm.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index e50c60e8fd..c5ae780a1f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -201,12 +201,11 @@ protected byte[] EncryptData(byte[] plainText, bool hasAuthenticationTag) // Final blob we return = version + HMAC + iv + cipherText const int hmacStartIndex = 1; - int authenticationTagLen = hasAuthenticationTag ? KeySizeInBytes : 0; + int authenticationTagLen = hasAuthenticationTag ? AuthenticationTagSizeInBytes : 0; int ivStartIndex = hmacStartIndex + authenticationTagLen; int cipherStartIndex = ivStartIndex + BlockSizeInBytes; // this is where hmac starts. - // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. - int outputBufSize = sizeof(byte) + authenticationTagLen + iv.Length + (numBlocks * BlockSizeInBytes); + int outputBufSize = this.GetEncryptByteCount(plainText.Length) - (hasAuthenticationTag ? 0 : authenticationTagLen); byte[] outBuffer = new byte[outputBufSize]; // Store the version and IV rightaway From cbbeee2f664fd63a2db5da70c49044d1e025423f Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 13:54:06 +0200 Subject: [PATCH 27/85] ~ refactor EncryptionProcessor --- .../src/AeAesEncryptionProcessor.cs | 116 +++++ .../src/EncryptionProcessor.cs | 458 ++---------------- .../src/MdeEncryptionProcessor.cs | 124 +++++ .../Transformation/JObjectSqlSerializer.cs | 121 +++++ .../src/Transformation/MdeEncryptor.cs | 55 +++ .../src/Transformation/TypeMarker.cs | 17 + 6 files changed, 472 insertions(+), 419 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs new file mode 100644 index 0000000000..7b7d366295 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs @@ -0,0 +1,116 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal static class AeAesEncryptionProcessor + { + public static async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken cancellationToken) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + EncryptionProperties encryptionProperties = null; + byte[] plainText = null; + byte[] cipherText = null; + + JObject toEncryptJObj = new (); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + toEncryptJObj.Add(propertyName, propertyValue.Value()); + itemJObj.Remove(propertyName); + } + + MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); + Debug.Assert(memoryStream != null); + Debug.Assert(memoryStream.TryGetBuffer(out _)); + plainText = memoryStream.ToArray(); + + cipherText = await encryptor.EncryptAsync( + plainText, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm, + cancellationToken); + + if (cipherText == null) + { + throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); + } + + encryptionProperties = new EncryptionProperties( + encryptionFormatVersion: 2, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: cipherText, + encryptionOptions.PathsToEncrypt); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal static async Task LegacyEncAlgoDecryptContentAsync( + JObject document, + EncryptionProperties encryptionProperties, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 2) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + byte[] plainText = await encryptor.DecryptAsync( + encryptionProperties.EncryptedData, + encryptionProperties.DataEncryptionKeyId, + encryptionProperties.EncryptionAlgorithm, + cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(EncryptionProcessor.DecryptAsync)}."); + JObject plainTextJObj; + using (MemoryStream memoryStream = new (plainText)) + using (StreamReader streamReader = new (memoryStream)) + using (JsonTextReader jsonTextReader = new (streamReader)) + { + jsonTextReader.ArrayPool = JsonArrayPool.Instance; + plainTextJObj = JObject.Load(jsonTextReader); + } + + List pathsDecrypted = new (); + foreach (JProperty property in plainTextJObj.Properties()) + { + document.Add(property.Name, property.Value); + pathsDecrypted.Add("/" + property.Name); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + + return decryptionContext; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 0370b94771..1c2c78fe24 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; - using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -13,7 +12,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Text; using System.Threading; using System.Threading.Tasks; - using Microsoft.Data.Encryption.Cryptography.Serializers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -22,20 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal static class EncryptionProcessor { - private static readonly SqlSerializerFactory SqlSerializerFactory = new SqlSerializerFactory(); - - // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); - private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); - private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); - - private static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings() + internal static readonly JsonSerializerSettings JsonSerializerSettings = new () { DateParseHandling = DateParseHandling.None, }; - internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new CosmosJsonDotNetSerializer(JsonSerializerSettings); + internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); /// /// If there isn't any PathsToEncrypt, input stream will be returned without any modification. @@ -51,7 +41,7 @@ public static async Task EncryptAsync( { _ = diagnosticsContext; - EncryptionProcessor.ValidateInputForEncrypt( + ValidateInputForEncrypt( input, encryptor, encryptionOptions); @@ -79,118 +69,14 @@ public static async Task EncryptAsync( } } - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); - List pathsEncrypted = new List(); - EncryptionProperties encryptionProperties = null; - byte[] plainText = null; - byte[] cipherText = null; - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - - switch (encryptionOptions.EncryptionAlgorithm) +#pragma warning disable CS0618 // Type or member is obsolete + return encryptionOptions.EncryptionAlgorithm switch { - case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { - string propertyName = pathToEncrypt.Substring(1); - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - if (propertyValue.Type == JTokenType.Null) - { - continue; - } - - (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); - - if (plainText == null) - { - continue; - } - - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); - - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); - - cipherTextWithTypeMarker[0] = (byte)typeMarker; - - int encryptedBytesCount = encryptionKey.EncryptData( - plainText, - plainTextOffset: 0, - plainTextLength, - cipherTextWithTypeMarker, - outputOffset: 1); - - if (encryptedBytesCount < 0) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); - } - - itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); - pathsEncrypted.Add(pathToEncrypt); - } - - encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - break; - - case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: - - JObject toEncryptJObj = new JObject(); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { - string propertyName = pathToEncrypt.Substring(1); - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - toEncryptJObj.Add(propertyName, propertyValue.Value()); - itemJObj.Remove(propertyName); - } - - MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); - Debug.Assert(memoryStream != null); - Debug.Assert(memoryStream.TryGetBuffer(out _)); - plainText = memoryStream.ToArray(); - - cipherText = await encryptor.EncryptAsync( - plainText, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm, - cancellationToken); - - if (cipherText == null) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); - } - - encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 2, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: cipherText, - encryptionOptions.PathsToEncrypt); - break; - - default: - throw new NotSupportedException($"Encryption Algorithm : {encryptionOptions.EncryptionAlgorithm} is not supported."); - } - - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); - input.Dispose(); - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken), + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken), + _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionOptions.EncryptionAlgorithm} is not supported."), + }; +#pragma warning restore CS0618 // Type or member is obsolete } /// @@ -213,8 +99,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = EncryptionProcessor.RetrieveItem(input); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = RetrieveItem(input); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -222,26 +108,9 @@ public static async Task EncryptAsync( return (input, null); } - EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); - DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch - { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( - itemJObj, - encryptor, - encryptionProperties, - diagnosticsContext, - cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( - itemJObj, - encryptionProperties, - encryptor, - diagnosticsContext, - cancellationToken), - _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), - }; - + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); input.Dispose(); - return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext); + return (BaseSerializer.ToStream(itemJObj), decryptionContext); } public static async Task<(JObject, DecryptionContext)> DecryptAsync( @@ -254,175 +123,56 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { return (document, null); } + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionPropertiesJObj, cancellationToken); + + return (document, decryptionContext); + } + + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) + { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); +#pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( - document, + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( - document, + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + itemJObj, encryptionProperties, encryptor, diagnosticsContext, cancellationToken), _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), }; - - return (document, decryptionContext); - } - - private static async Task MdeEncAlgoDecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); - foreach (string path in encryptionProperties.EncryptedPaths) - { - string propertyName = path.Substring(1); - if (!document.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) - { - continue; - } - - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); - - byte[] plainText = arrayPoolManager.Rent(plainTextLength); - - int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( - encryptionKey, - cipherTextWithTypeMarker, - cipherTextOffset: 1, - cipherTextWithTypeMarker.Length - 1, - plainText); - - EncryptionProcessor.DeserializeAndAddProperty( - (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - document, - propertyName); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); +#pragma warning restore CS0618 // Type or member is obsolete return decryptionContext; } - private static DecryptionContext CreateDecryptionContext( + internal static DecryptionContext CreateDecryptionContext( List pathsDecrypted, string dataEncryptionKeyId) { - DecryptionInfo decryptionInfo = new DecryptionInfo( + DecryptionInfo decryptionInfo = new ( pathsDecrypted, dataEncryptionKeyId); - DecryptionContext decryptionContext = new DecryptionContext( + DecryptionContext decryptionContext = new ( new List() { decryptionInfo }); return decryptionContext; } - private static int MdeEncAlgoDecryptPropertyAsync( - DataEncryptionKey encryptionKey, - byte[] cipherText, - int cipherTextOffset, - int cipherTextLength, - byte[] buffer) - { - int decryptedCount = encryptionKey.DecryptData( - cipherText, - cipherTextOffset, - cipherTextLength, - buffer, - outputOffset: 0); - - if (decryptedCount < 0) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); - } - - return decryptedCount; - } - - private static async Task LegacyEncAlgoDecryptContentAsync( - JObject document, - EncryptionProperties encryptionProperties, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 2) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - byte[] plainText = await encryptor.DecryptAsync( - encryptionProperties.EncryptedData, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm, - cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); - JObject plainTextJObj; - using (MemoryStream memoryStream = new MemoryStream(plainText)) - using (StreamReader streamReader = new StreamReader(memoryStream)) - using (JsonTextReader jsonTextReader = new JsonTextReader(streamReader)) - { - jsonTextReader.ArrayPool = JsonArrayPool.Instance; - plainTextJObj = JObject.Load(jsonTextReader); - } - - List pathsDecrypted = new List(); - foreach (JProperty property in plainTextJObj.Properties()) - { - document.Add(property.Name, property.Value); - pathsDecrypted.Add("/" + property.Name); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); - - return decryptionContext; - } - private static void ValidateInputForEncrypt( Stream input, Encryptor encryptor, @@ -465,11 +215,11 @@ private static JObject RetrieveItem( Debug.Assert(input != null); JObject itemJObj; - using (StreamReader sr = new StreamReader(input, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true)) - using (JsonTextReader jsonTextReader = new JsonTextReader(sr)) + using (StreamReader sr = new (input, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true)) + using (JsonTextReader jsonTextReader = new (sr)) { jsonTextReader.ArrayPool = JsonArrayPool.Instance; - JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings() + JsonSerializerSettings jsonSerializerSettings = new () { DateParseHandling = DateParseHandling.None, MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr @@ -494,129 +244,21 @@ private static JObject RetrieveEncryptionProperties( return encryptionPropertiesJObj; } - private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) - { - byte[] buffer; - int length; - switch (propertyValue.Type) - { - case JTokenType.Undefined: - Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null, -1); - case JTokenType.Null: - Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null, -1); - case JTokenType.Boolean: - (buffer, length) = SerializeFixed(SqlBoolSerializer); - return (TypeMarker.Boolean, buffer, length); - case JTokenType.Float: - (buffer, length) = SerializeFixed(SqlDoubleSerializer); - return (TypeMarker.Double, buffer, length); - case JTokenType.Integer: - (buffer, length) = SerializeFixed(SqlLongSerializer); - return (TypeMarker.Long, buffer, length); - case JTokenType.String: - (buffer, length) = SerializeString(propertyValue.ToObject()); - return (TypeMarker.String, buffer, length); - case JTokenType.Array: - (buffer, length) = SerializeString(propertyValue.ToString()); - return (TypeMarker.Array, buffer, length); - case JTokenType.Object: - (buffer, length) = SerializeString(propertyValue.ToString()); - return (TypeMarker.Object, buffer, length); - default: - throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); - } - - (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) - { - byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int length = serializer.Serialize(propertyValue.ToObject(), buffer); - return (buffer, length); - } - - (byte[], int) SerializeString(string value) - { - byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int length = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, length); - } - } - - private static void DeserializeAndAddProperty( - TypeMarker typeMarker, - ReadOnlySpan serializedBytes, - JObject jObject, - string key) - { - switch (typeMarker) - { - case TypeMarker.Boolean: - jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Double: - jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Long: - jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.String: - jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Array: - DeserializeAndAddProperty(serializedBytes); - break; - case TypeMarker.Object: - DeserializeAndAddProperty(serializedBytes); - break; - default: - Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); - break; - } - - void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) - where T : JToken - { - using ArrayPoolManager manager = new ArrayPoolManager(); - - char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); - int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); - - JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); - - using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); - using JsonTextReader reader = new JsonTextReader(memoryTextReader); - - jObject[key] = serializer.Deserialize(reader); - } - } - - private enum TypeMarker : byte - { - Null = 1, // not used - String = 2, - Double = 3, - Long = 4, - Boolean = 5, - Array = 6, - Object = 7, - } - internal static async Task DeserializeAndDecryptResponseAsync( Stream content, Encryptor encryptor, CancellationToken cancellationToken) { - JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); + JObject contentJObj = BaseSerializer.FromStream(content); - if (!(contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents)) + if (contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is not JArray documents) { throw new InvalidOperationException("Feed Response body contract was violated. Feed response did not have an array of Documents"); } foreach (JToken value in documents) { - if (!(value is JObject document)) + if (value is not JObject document) { continue; } @@ -624,7 +266,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await EncryptionProcessor.DecryptAsync( + await DecryptAsync( document, encryptor, diagnosticsContext, @@ -634,29 +276,7 @@ await EncryptionProcessor.DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); - } - - internal static int GetOriginalBase64Length(string base64string) - { - if (string.IsNullOrEmpty(base64string)) - { - return 0; - } - - int paddingCount = 0; - int characterCount = base64string.Length; - if (base64string[characterCount - 1] == '=') - { - paddingCount++; - } - - if (base64string[characterCount - 2] == '=') - { - paddingCount++; - } - - return (3 * (characterCount / 4)) - paddingCount; + return BaseSerializer.ToStream(contentJObj); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs new file mode 100644 index 0000000000..e15dbf88f8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Newtonsoft.Json.Linq; + + internal static class MdeEncryptionProcessor + { + public static async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = JObjectSqlSerializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedLength) = MdeEncryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + + itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal static async Task MdeEncAlgoDecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { + string propertyName = path.Substring(1); + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = MdeEncryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + JObjectSqlSerializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs new file mode 100644 index 0000000000..5056b4645c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs @@ -0,0 +1,121 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using Microsoft.Data.Encryption.Cryptography.Serializers; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal static class JObjectSqlSerializer + { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + + private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; + + internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) + { + byte[] buffer; + int length; + switch (propertyValue.Type) + { + case JTokenType.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null, -1); + case JTokenType.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null, -1); + case JTokenType.Boolean: + (buffer, length) = SerializeFixed(SqlBoolSerializer); + return (TypeMarker.Boolean, buffer, length); + case JTokenType.Float: + (buffer, length) = SerializeFixed(SqlDoubleSerializer); + return (TypeMarker.Double, buffer, length); + case JTokenType.Integer: + (buffer, length) = SerializeFixed(SqlLongSerializer); + return (TypeMarker.Long, buffer, length); + case JTokenType.String: + (buffer, length) = SerializeString(propertyValue.ToObject()); + return (TypeMarker.String, buffer, length); + case JTokenType.Array: + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Array, buffer, length); + case JTokenType.Object: + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Object, buffer, length); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); + } + + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int length = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, length); + } + + (byte[], int) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); + } + } + + internal static void DeserializeAndAddProperty( + TypeMarker typeMarker, + ReadOnlySpan serializedBytes, + JObject jObject, + string key) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.String: + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + DeserializeAndAddProperty(serializedBytes); + break; + case TypeMarker.Object: + DeserializeAndAddProperty(serializedBytes); + break; + default: + Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + break; + } + + void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + where T : JToken + { + using ArrayPoolManager manager = new (); + + char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); + + JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); + + using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); + using JsonTextReader reader = new (memoryTextReader); + + jObject[key] = serializer.Deserialize(reader); + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs new file mode 100644 index 0000000000..325c1b5f03 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + + internal static class MdeEncryptor + { + internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + { + int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; + + byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + + encryptedText[0] = (byte)typeMarker; + + int encryptedLength = encryptionKey.EncryptData( + plainText, + plainTextOffset: 0, + plainTextLength, + encryptedText, + outputOffset: 1); + + if (encryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); + } + + return (encryptedText, encryptedLength + 1); + } + + internal static (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + { + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); + + byte[] plainText = arrayPoolManager.Rent(plainTextLength); + + int decryptedLength = encryptionKey.DecryptData( + cipherText, + cipherTextOffset: 1, + cipherTextLength: cipherText.Length - 1, + plainText, + outputOffset: 0); + + if (decryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null plainText from {nameof(DataEncryptionKey.DecryptData)}."); + } + + return (plainText, decryptedLength); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs new file mode 100644 index 0000000000..f2f31c0927 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + internal enum TypeMarker : byte + { + Null = 1, // not used + String = 2, + Double = 3, + Long = 4, + Boolean = 5, + Array = 6, + Object = 7, + } +} From b6c851cbda5a5fd974e0f7f95d24c6cea683efd7 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 13:59:46 +0200 Subject: [PATCH 28/85] ! names --- .../src/AeAesEncryptionProcessor.cs | 2 +- .../src/EncryptionProcessor.cs | 4 ++-- .../src/MdeEncryptionProcessor.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs index 7b7d366295..0944714aac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs @@ -69,7 +69,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task LegacyEncAlgoDecryptContentAsync( + internal static async Task DecryptContentAsync( JObject document, EncryptionProperties encryptionProperties, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 1c2c78fe24..d274d1d899 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -141,13 +141,13 @@ private static async Task DecryptInternalAsync(Encryptor encr #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.DecryptContentAsync( itemJObj, encryptionProperties, encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs index e15dbf88f8..fc601a0916 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -68,7 +68,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task MdeEncAlgoDecryptObjectAsync( + internal static async Task DecryptObjectAsync( JObject document, Encryptor encryptor, EncryptionProperties encryptionProperties, From 72ccae7a87c9fcd49663c8ba82ccb0f1a3be5b7e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 14:30:38 +0200 Subject: [PATCH 29/85] ~ less static --- .../src/EncryptionProcessor.cs | 2 ++ .../src/MdeEncryptionProcessor.cs | 18 +++++++++++------- .../src/Transformation/JObjectSqlSerializer.cs | 16 +++++++++------- .../src/Transformation/MdeEncryptor.cs | 6 +++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index d274d1d899..3a97b051a9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -27,6 +27,8 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); + /// /// If there isn't any PathsToEncrypt, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs index fc601a0916..571c5b4ac7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -13,9 +13,13 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; - internal static class MdeEncryptionProcessor + internal class MdeEncryptionProcessor { - public static async Task EncryptAsync( + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( Stream input, Encryptor encryptor, EncryptionOptions encryptionOptions, @@ -43,14 +47,14 @@ public static async Task EncryptAsync( } byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = JObjectSqlSerializer.Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); if (plainText == null) { continue; } - (byte[] encryptedBytes, int encryptedLength) = MdeEncryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); pathsEncrypted.Add(pathToEncrypt); @@ -68,7 +72,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task DecryptObjectAsync( + internal async Task DecryptObjectAsync( JObject document, Encryptor encryptor, EncryptionProperties encryptionProperties, @@ -102,9 +106,9 @@ internal static async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = MdeEncryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); - JObjectSqlSerializer.DeserializeAndAddProperty( + this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs index 5056b4645c..53ee238481 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using Newtonsoft.Json; using Newtonsoft.Json.Linq; - internal static class JObjectSqlSerializer + internal class JObjectSqlSerializer { private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); @@ -21,7 +21,8 @@ internal static class JObjectSqlSerializer private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; - internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) +#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed + internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) { byte[] buffer; int length; @@ -69,8 +70,9 @@ internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBy return (buffer, length); } } +#pragma warning restore SA1101 // Prefix local calls with this - internal static void DeserializeAndAddProperty( + internal virtual void DeserializeAndAddProperty( TypeMarker typeMarker, ReadOnlySpan serializedBytes, JObject jObject, @@ -91,17 +93,17 @@ internal static void DeserializeAndAddProperty( jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); break; case TypeMarker.Array: - DeserializeAndAddProperty(serializedBytes); + jObject[key] = Deserialize(serializedBytes); break; case TypeMarker.Object: - DeserializeAndAddProperty(serializedBytes); + jObject[key] = Deserialize(serializedBytes); break; default: Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); break; } - void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + static T Deserialize(ReadOnlySpan serializedBytes) where T : JToken { using ArrayPoolManager manager = new (); @@ -114,7 +116,7 @@ void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); using JsonTextReader reader = new (memoryTextReader); - jObject[key] = serializer.Deserialize(reader); + return serializer.Deserialize(reader); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index 325c1b5f03..9243c632d6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -6,9 +6,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; - internal static class MdeEncryptor + internal class MdeEncryptor { - internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) { int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; @@ -31,7 +31,7 @@ internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncrypti return (encryptedText, encryptedLength + 1); } - internal static (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) { int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); From 5554aa0fa308910b501f07219af1860f2da0f3df Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 13:44:16 +0200 Subject: [PATCH 30/85] ~ merge fixes --- .../src/ArrayPoolManager.cs | 4 +- .../src/CosmosEncryptor.cs | 1 - .../src/EncryptionProcessor.cs | 73 ++++++++++--------- .../src/Encryptor.cs | 1 - .../src/MemoryTextReader.cs | 4 +- .../EmulatorTests/LegacyEncryptionTests.cs | 20 ----- .../MdeEncryptionProcessorTests.cs | 18 ++--- 7 files changed, 53 insertions(+), 68 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index 8cafd07c5d..9679adc1c4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -8,9 +8,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Buffers; using System.Collections.Generic; +#pragma warning disable SA1402 // File may only contain a single type internal class ArrayPoolManager : IDisposable +#pragma warning restore SA1402 // File may only contain a single type { - private List rentedBuffers = new List(); + private List rentedBuffers = new (); private bool disposedValue; public T[] Rent(int minimumLength) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 6e6be7822d..454440f5ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -64,6 +64,5 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } - } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 7222180f76..b36747b960 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; - using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -25,10 +24,10 @@ internal static class EncryptionProcessor private static readonly SqlSerializerFactory SqlSerializerFactory = new (); // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); - private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); - private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); private static readonly JsonSerializerSettings JsonSerializerSettings = new () { @@ -79,15 +78,16 @@ public static async Task EncryptAsync( } } - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + JObject itemJObj = BaseSerializer.FromStream(input); List pathsEncrypted = new (encryptionOptions.PathsToEncrypt.Count()); EncryptionProperties encryptionProperties = null; byte[] plainText = null; byte[] cipherText = null; TypeMarker typeMarker; - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + using ArrayPoolManager arrayPoolManager = new (); +#pragma warning disable CS0618 // Type or member is obsolete switch (encryptionOptions.EncryptionAlgorithm) { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: @@ -107,7 +107,7 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = Serialize(propertyValue, arrayPoolManager); if (plainText == null) { @@ -160,7 +160,7 @@ public static async Task EncryptAsync( itemJObj.Remove(propertyName); } - MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); + MemoryStream memoryStream = BaseSerializer.ToStream(toEncryptJObj); Debug.Assert(memoryStream != null); Debug.Assert(memoryStream.TryGetBuffer(out _)); plainText = memoryStream.ToArray(); @@ -191,7 +191,7 @@ public static async Task EncryptAsync( itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); input.Dispose(); - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + return BaseSerializer.ToStream(itemJObj); } /// @@ -214,8 +214,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = EncryptionProcessor.RetrieveItem(input); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = RetrieveItem(input); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -227,13 +227,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( itemJObj, encryptionProperties, encryptor, @@ -244,7 +244,7 @@ public static async Task EncryptAsync( #pragma warning restore CS0618 // Type or member is obsolete input.Dispose(); - return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext); + return (BaseSerializer.ToStream(itemJObj), decryptionContext); } public static async Task<(JObject, DecryptionContext)> DecryptAsync( @@ -257,7 +257,7 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { @@ -268,13 +268,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( document, encryptionProperties, encryptor, @@ -301,11 +301,11 @@ private static async Task MdeEncAlgoDecryptObjectAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); @@ -324,14 +324,14 @@ private static async Task MdeEncAlgoDecryptObjectAsync( byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + int decryptedCount = MdeEncAlgoDecryptProperty( encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, plainText); - EncryptionProcessor.DeserializeAndAddProperty( + DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, @@ -340,7 +340,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( pathsDecrypted.Add(path); } - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + DecryptionContext decryptionContext = CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -362,14 +362,21 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static int MdeEncAlgoDecryptPropertyAsync( + private static int MdeEncAlgoDecryptProperty( DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] buffer) { - if (encryptionProperties.EncryptionFormatVersion != 3) + int decryptedCount = encryptionKey.DecryptData( + cipherText, + cipherTextOffset, + cipherTextLength, + buffer, + outputOffset: 0); + + if (decryptedCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); } @@ -412,7 +419,7 @@ private static async Task LegacyEncAlgoDecryptContentAsync( pathsDecrypted.Add("/" + property.Name); } - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + DecryptionContext decryptionContext = CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -575,15 +582,15 @@ private static void DeserializeAndAddProperty( void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) where T : JToken { - using ArrayPoolManager manager = new ArrayPoolManager(); + using ArrayPoolManager manager = new (); char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); - using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); - using JsonTextReader reader = new JsonTextReader(memoryTextReader); + using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); + using JsonTextReader reader = new (memoryTextReader); jObject[key] = serializer.Deserialize(reader); } @@ -605,7 +612,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( Encryptor encryptor, CancellationToken cancellationToken) { - JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); + JObject contentJObj = BaseSerializer.FromStream(content); if (contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is not JArray documents) { @@ -622,7 +629,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await EncryptionProcessor.DecryptAsync( + await DecryptAsync( document, encryptor, diagnosticsContext, @@ -632,7 +639,7 @@ await EncryptionProcessor.DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); + return BaseSerializer.ToStream(contentJObj); } internal static int GetOriginalBase64Length(string base64string) @@ -657,4 +664,4 @@ internal static int GetOriginalBase64Length(string base64string) return (3 * (characterCount / 4)) - paddingCount; } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index a517c5fea6..27c152d491 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -13,7 +13,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// public abstract class Encryptor { - /// /// Retrieve Data Encryption Key. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs index b8996ca802..9326fbbf0e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs @@ -135,7 +135,7 @@ public override string ReadLine() char ch = this.chars.Span[i]; if (ch == '\r' || ch == '\n') { - string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); this.pos = i + 1; if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') { @@ -150,7 +150,7 @@ public override string ReadLine() if (i > this.pos) { - string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); this.pos = i; return result; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 3672f808c6..30f6364a32 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1768,26 +1768,6 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.GetEncryptionKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); - } - public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 757245a83b..69a9c7afb0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -41,26 +41,24 @@ public static void ClassInitialize(TestContext testContext) .Returns((int plainTextLength) => plainTextLength); DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.DecryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.DecryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.GetDecryptByteCount(It.IsAny())) + .Returns((int cipherTextLength) => cipherTextLength); - MdeEncryptionProcessorTests.mockEncryptor = new Mock(); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor = new Mock(); + mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string dekId, string algorithm, CancellationToken token) => dekId == MdeEncryptionProcessorTests.dekId ? DekMock.Object : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] From 28620edf2961e70384f29d75476dd90f18e9f592 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:18:17 +0200 Subject: [PATCH 31/85] ~ cleanup --- .../src/EncryptionProcessor.cs | 6 ++---- .../Readme.md | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b36747b960..b36619b11f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -21,8 +21,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal static class EncryptionProcessor { - private static readonly SqlSerializerFactory SqlSerializerFactory = new (); - // UTF-8 encoding. private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); private static readonly SqlBitSerializer SqlBoolSerializer = new (); @@ -92,7 +90,7 @@ public static async Task EncryptAsync( { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, cancellationToken); foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { @@ -575,7 +573,7 @@ private static void DeserializeAndAddProperty( DeserializeAndAddProperty(serializedBytes); break; default: - Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + Debug.Fail($"Unexpected type marker {typeMarker}"); break; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 4df982a29d..f8c40c6bce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **29.20 μs** | **0.607 μs** | **0.871 μs** | **29.26 μs** | **3.3875** | **2.5940** | **-** | **41.51 KB** | -| Decrypt | 1 | 32.23 μs | 0.528 μs | 0.790 μs | 32.78 μs | 3.2349 | 0.7935 | - | 39.71 KB | -| **Encrypt** | **10** | **107.83 μs** | **1.975 μs** | **2.956 μs** | **107.93 μs** | **13.7939** | **0.6104** | **-** | **170.14 KB** | -| Decrypt | 10 | 116.25 μs | 1.537 μs | 2.301 μs | 117.15 μs | 12.5732 | 1.2207 | - | 154.63 KB | -| **Encrypt** | **100** | **1,493.01 μs** | **314.932 μs** | **471.376 μs** | **1,482.68 μs** | **214.8438** | **173.8281** | **140.6250** | **1655.51 KB** | -| Decrypt | 100 | 1,406.56 μs | 126.286 μs | 189.019 μs | 1,408.25 μs | 138.6719 | 103.5156 | 82.0313 | 1248.58 KB | +| **Encrypt** | **1** | **29.20 μs** | **0.559 μs** | **0.836 μs** | **28.66 μs** | **3.3569** | **1.0681** | **-** | **41.24 KB** | +| Decrypt | 1 | 33.10 μs | 0.630 μs | 0.904 μs | 32.57 μs | 3.2349 | 0.7935 | - | 39.7 KB | +| **Encrypt** | **10** | **107.36 μs** | **1.942 μs** | **2.907 μs** | **107.29 μs** | **13.7939** | **0.8545** | **-** | **169.87 KB** | +| Decrypt | 10 | 117.12 μs | 1.939 μs | 2.843 μs | 118.31 μs | 12.5732 | 1.2207 | - | 154.62 KB | +| **Encrypt** | **100** | **1,489.27 μs** | **335.506 μs** | **502.170 μs** | **1,486.77 μs** | **214.8438** | **175.7813** | **140.6250** | **1655.29 KB** | +| Decrypt | 100 | 1,404.43 μs | 149.158 μs | 223.253 μs | 1,408.35 μs | 142.5781 | 107.4219 | 85.9375 | 1248.36 KB | From eb059c84917e280e22d3f7fe441ec83e518ec9c6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:35:29 +0200 Subject: [PATCH 32/85] ~ unwanted changes --- Directory.Build.props | 1 - Microsoft.Azure.Cosmos.lutconfig | 6 ------ 2 files changed, 7 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.lutconfig diff --git a/Directory.Build.props b/Directory.Build.props index 3f8f26ab75..dfbe8b10fc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,7 +12,6 @@ 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) $(DefineConstants);PREVIEW;ENCRYPTIONPREVIEW - NU1903 diff --git a/Microsoft.Azure.Cosmos.lutconfig b/Microsoft.Azure.Cosmos.lutconfig deleted file mode 100644 index 596a860301..0000000000 --- a/Microsoft.Azure.Cosmos.lutconfig +++ /dev/null @@ -1,6 +0,0 @@ - - - true - true - 180000 - \ No newline at end of file From cc2eab5cc063a446628155f3123ed80fe731377d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:40:22 +0200 Subject: [PATCH 33/85] - unused method --- .../src/EncryptionProcessor.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b36619b11f..c223f44cae 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -639,27 +639,5 @@ await DecryptAsync( // and corresponding decrypted properties are added back in the documents. return BaseSerializer.ToStream(contentJObj); } - - internal static int GetOriginalBase64Length(string base64string) - { - if (string.IsNullOrEmpty(base64string)) - { - return 0; - } - - int paddingCount = 0; - int characterCount = base64string.Length; - if (base64string[characterCount - 1] == '=') - { - paddingCount++; - } - - if (base64string[characterCount - 2] == '=') - { - paddingCount++; - } - - return (3 * (characterCount / 4)) - paddingCount; - } } } \ No newline at end of file From c9ba3002002dd2f75247b1f9e5212f306fbcad75 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 19:50:25 +0200 Subject: [PATCH 34/85] ~ updates (PR) --- .../src/ArrayPoolManager.cs | 3 +- .../src/EncryptionProcessor.cs | 34 +++++++++---------- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../Readme.md | 12 +++---- .../AeadAes256CbcHmac256AlgorithmTests.cs | 1 - 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index 9679adc1c4..3546b7155b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -32,9 +32,10 @@ protected virtual void Dispose(bool disposing) { ArrayPool.Shared.Return(buffer, clearArray: true); } + + this.rentedBuffers = null; } - this.rentedBuffers = null; this.disposedValue = true; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index c223f44cae..1c7f692fe2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -105,16 +105,16 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText, int plainTextLength) = Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); if (plainText == null) { continue; } - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); + int cipherTextLength = encryptionKey.GetEncryptByteCount(plainTextLength); - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; cipherTextWithTypeMarker[0] = (byte)typeMarker; @@ -130,7 +130,7 @@ public static async Task EncryptAsync( throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } @@ -212,8 +212,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = RetrieveItem(input); - JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = EncryptionProcessor.RetrieveItem(input); + JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -225,13 +225,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( itemJObj, encryptionProperties, encryptor, @@ -255,7 +255,7 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { @@ -266,13 +266,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( document, encryptionProperties, encryptor, @@ -322,14 +322,14 @@ private static async Task MdeEncAlgoDecryptObjectAsync( byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = MdeEncAlgoDecryptProperty( + int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptProperty( encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, plainText); - DeserializeAndAddProperty( + EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, @@ -338,7 +338,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( pathsDecrypted.Add(path); } - DecryptionContext decryptionContext = CreateDecryptionContext( + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -417,7 +417,7 @@ private static async Task LegacyEncAlgoDecryptContentAsync( pathsDecrypted.Add("/" + property.Name); } - DecryptionContext decryptionContext = CreateDecryptionContext( + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -627,7 +627,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await DecryptAsync( + await EncryptionProcessor.DecryptAsync( document, encryptor, diagnosticsContext, @@ -637,7 +637,7 @@ await DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return BaseSerializer.ToStream(contentJObj); + return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 4c0fae78f1..8769abdb89 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index f8c40c6bce..ced6fc95ed 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **29.20 μs** | **0.559 μs** | **0.836 μs** | **28.66 μs** | **3.3569** | **1.0681** | **-** | **41.24 KB** | -| Decrypt | 1 | 33.10 μs | 0.630 μs | 0.904 μs | 32.57 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **107.36 μs** | **1.942 μs** | **2.907 μs** | **107.29 μs** | **13.7939** | **0.8545** | **-** | **169.87 KB** | -| Decrypt | 10 | 117.12 μs | 1.939 μs | 2.843 μs | 118.31 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,489.27 μs** | **335.506 μs** | **502.170 μs** | **1,486.77 μs** | **214.8438** | **175.7813** | **140.6250** | **1655.29 KB** | -| Decrypt | 100 | 1,404.43 μs | 149.158 μs | 223.253 μs | 1,408.35 μs | 142.5781 | 107.4219 | 85.9375 | 1248.36 KB | +| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | +| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | +| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | +| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | +| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | +| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 900fac6798..c2fd0551ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; - using System.Text; [TestClass] public class AeadAes256CbcHmac256AlgorithmTests From 9f9cbcafec0b109bffc2bfb471df34da97be3742 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 13:43:29 +0200 Subject: [PATCH 35/85] ~ add stable vs preview release duplicity --- .../{ => AeadAes}/AeAesEncryptionProcessor.cs | 0 .../AeadAes256CbcHmac256Algorithm.cs | 0 .../AeadAes256CbcHmac256EncryptionKey.cs | 0 .../src/{ => AeadAes}/SymmetricKey.cs | 0 .../src/EncryptionContainer.cs | 18 +++ .../src/EncryptionProcessor.cs | 1 + ...soft.Azure.Cosmos.Encryption.Custom.csproj | 7 +- ...zer.cs => JObjectSqlSerializer.Preview.cs} | 4 + .../JObjectSqlSerializer.Stable.cs | 85 ++++++++++++ .../MdeEncryptionProcessor.Preview.cs} | 6 +- .../MdeEncryptionProcessor.Stable.cs | 130 ++++++++++++++++++ .../{ => Transformation}/MemoryTextReader.cs | 0 12 files changed, 247 insertions(+), 4 deletions(-) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeAesEncryptionProcessor.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeadAes256CbcHmac256Algorithm.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeadAes256CbcHmac256EncryptionKey.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/SymmetricKey.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/{JObjectSqlSerializer.cs => JObjectSqlSerializer.Preview.cs} (99%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{MdeEncryptionProcessor.cs => Transformation/MdeEncryptionProcessor.Preview.cs} (97%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => Transformation}/MemoryTextReader.cs (100%) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index b898a77179..d4026cd216 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -1083,5 +1083,23 @@ private async Task> DecryptChangeFeedDocumentsAsync( return decryptItems; } + +#if IS_PREVIEW + public override Task DeleteAllItemsByPartitionKeyStreamAsync(PartitionKey partitionKey, RequestOptions requestOptions = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task> GetPartitionKeyRangesAsync(FeedRange feedRange, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(string processorName, ChangeFeedHandler> onChangesDelegate) + { + throw new NotImplementedException(); + } +#endif + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3a97b051a9..a2b4d8fb4f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -12,6 +12,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Text; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 4c0fae78f1..a192203753 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + IS_PREVIEW @@ -24,20 +25,22 @@ + + - + - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs similarity index 99% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 53ee238481..9e2e934303 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -2,6 +2,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ +#if IS_PREVIEW + namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; @@ -121,3 +123,5 @@ static T Deserialize(ReadOnlySpan serializedBytes) } } } + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs new file mode 100644 index 0000000000..315fd82289 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -0,0 +1,85 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if !IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using Microsoft.Data.Encryption.Cryptography.Serializers; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal class JObjectSqlSerializer + { + private static readonly SqlSerializerFactory SqlSerializerFactory = new(); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new(size: -1, codePageCharacterEncoding: 65001); + + private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; + + internal (TypeMarker, byte[]) Serialize(JToken propertyValue) + { + switch (propertyValue.Type) + { + case JTokenType.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null); + case JTokenType.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null); + case JTokenType.Boolean: + return (TypeMarker.Boolean, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.Float: + return (TypeMarker.Double, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.Integer: + return (TypeMarker.Long, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.String: + return (TypeMarker.String, SqlVarCharSerializer.Serialize(propertyValue.ToObject())); + case JTokenType.Array: + return (TypeMarker.Array, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + case JTokenType.Object: + return (TypeMarker.Object, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); + } + } + + internal virtual void DeserializeAndAddProperty( + TypeMarker typeMarker, + byte[] serializedBytes, + JObject jObject, + string key) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.String: + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + jObject[key] = JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings); + break; + case TypeMarker.Object: + jObject[key] = JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings); + break; + default: + Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + break; + } + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs similarity index 97% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 571c5b4ac7..3b34113b6a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -2,7 +2,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Encryption.Custom +#if IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Collections.Generic; @@ -10,7 +12,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Linq; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; internal class MdeEncryptionProcessor @@ -126,3 +127,4 @@ internal async Task DecryptObjectAsync( } } } +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs new file mode 100644 index 0000000000..0eeb1f1865 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -0,0 +1,130 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if !IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + + internal class MdeEncryptionProcessor + { + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText) = this.Serializer.Serialize(propertyValue); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length, arrayPoolManager); + + itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal async Task DecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new(encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { + string propertyName = path.Substring(1); + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount).ToArray(), + document, + propertyName); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs From 64172b8cc616962539765e360e9c2cd14ff99887 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 14:21:37 +0200 Subject: [PATCH 36/85] ~ cleanup and parent branch merge --- .../src/Transformation/JObjectSqlSerializer.Stable.cs | 4 ++-- .../src/Transformation/MdeEncryptionProcessor.Stable.cs | 6 +++--- .../src/Transformation/MemoryTextReader.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs index 315fd82289..f1332baea9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -14,10 +14,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class JObjectSqlSerializer { - private static readonly SqlSerializerFactory SqlSerializerFactory = new(); + private static readonly SqlSerializerFactory SqlSerializerFactory = new (); // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new(size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 0eeb1f1865..fc2b282d86 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -30,7 +30,7 @@ public async Task EncryptAsync( List pathsEncrypted = new (); TypeMarker typeMarker; - using ArrayPoolManager arrayPoolManager = new(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); @@ -87,11 +87,11 @@ internal async Task DecryptObjectAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - using ArrayPoolManager arrayPoolManager = new(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - List pathsDecrypted = new(encryptionProperties.EncryptedPaths.Count()); + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs index 9326fbbf0e..ec88ccd100 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Encryption.Custom +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Diagnostics.Contracts; From 326b1bec05e5848f3b1a11a8e15a0e062d7ac91e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 19:08:35 +0200 Subject: [PATCH 37/85] ~ master merges --- .../src/Transformation/MdeEncryptionProcessor.Preview.cs | 4 ++-- .../src/Transformation/MdeEncryptionProcessor.Stable.cs | 4 ++-- .../src/Transformation/MdeEncryptor.cs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 3b34113b6a..a5427b015b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -55,9 +55,9 @@ public async Task EncryptAsync( continue; } - (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); - itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + itemJObj[propertyName] = encryptedBytes; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index fc2b282d86..a861a05996 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -55,9 +55,9 @@ public async Task EncryptAsync( continue; } - (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length, arrayPoolManager); + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length); - itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + itemJObj[propertyName] = encryptedBytes.ToArray(); pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index 9243c632d6..cf5a0ffdf6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -8,11 +8,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class MdeEncryptor { - internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + internal virtual byte[] Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength) { int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; - byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + byte[] encryptedText = new byte[encryptedTextLength]; encryptedText[0] = (byte)typeMarker; @@ -28,7 +28,7 @@ internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncrypt throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); } - return (encryptedText, encryptedLength + 1); + return encryptedText; } internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) From c520e16a4861aaf2d507da191500b2c436919830 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 19:15:08 +0200 Subject: [PATCH 38/85] - duplicate --- .../src/MemoryTextReader.cs | 161 ------------------ 1 file changed, 161 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs deleted file mode 100644 index 9326fbbf0e..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs +++ /dev/null @@ -1,161 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System; - using System.Diagnostics.Contracts; - using System.IO; - - /// - /// Adjusted implementation of .Net StringReader reading from a Memory instead of a string. - /// - internal class MemoryTextReader : TextReader - { - private Memory chars; - private int length; - private int pos; - private bool closed; - - public MemoryTextReader(Memory chars) - { - this.chars = chars; - this.length = chars.Length; - } - - public override void Close() - { - this.Dispose(true); - } - - protected override void Dispose(bool disposing) - { - this.chars = null; - this.pos = 0; - this.length = 0; - this.closed = true; - base.Dispose(disposing); - } - - [Pure] - public override int Peek() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - if (this.pos == this.length) - { - return -1; - } - - return this.chars.Span[this.pos]; - } - - public override int Read() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - if (this.pos == this.length) - { - return -1; - } - - return this.chars.Span[this.pos++]; - } - - public override int Read(char[] buffer, int index, int count) - { - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - if (index < 0) - { - throw new ArgumentOutOfRangeException(nameof(index)); - } - - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count)); - } - - if (buffer.Length - index < count) - { - throw new ArgumentOutOfRangeException(); - } - - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - int n = this.length - this.pos; - if (n > 0) - { - if (n > count) - { - n = count; - } - - this.chars.Span.Slice(this.pos, n).CopyTo(buffer.AsSpan(index, n)); - this.pos += n; - } - - return n; - } - - public override string ReadToEnd() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - this.pos = this.length; - return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); - } - - public override string ReadLine() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - int i = this.pos; - while (i < this.length) - { - char ch = this.chars.Span[i]; - if (ch == '\r' || ch == '\n') - { - string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); - this.pos = i + 1; - if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') - { - this.pos++; - } - - return result; - } - - i++; - } - - if (i > this.pos) - { - string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); - this.pos = i; - return result; - } - - return null; - } - } -} From 5c408214623b3509ca7d4b87d380f21700e3c909 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 20:02:16 +0200 Subject: [PATCH 39/85] ~ cleanup --- .../Transformation/JObjectSqlSerializer.Preview.cs | 11 +++++------ .../Transformation/MdeEncryptionProcessor.Preview.cs | 4 +++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 9e2e934303..2c706aa361 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -72,13 +72,13 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB return (buffer, length); } } -#pragma warning restore SA1101 // Prefix local calls with this internal virtual void DeserializeAndAddProperty( TypeMarker typeMarker, ReadOnlySpan serializedBytes, JObject jObject, - string key) + string key, + ArrayPoolManager arrayPoolManager) { switch (typeMarker) { @@ -105,12 +105,10 @@ internal virtual void DeserializeAndAddProperty( break; } - static T Deserialize(ReadOnlySpan serializedBytes) + T Deserialize(ReadOnlySpan serializedBytes) where T : JToken { - using ArrayPoolManager manager = new (); - - char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + char[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); @@ -121,6 +119,7 @@ static T Deserialize(ReadOnlySpan serializedBytes) return serializer.Deserialize(reader); } } +#pragma warning restore SA1101 // Prefix local calls with this } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index a5427b015b..ccb59b0b40 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -88,6 +88,7 @@ internal async Task DecryptObjectAsync( } using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); @@ -113,7 +114,8 @@ internal async Task DecryptObjectAsync( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, - propertyName); + propertyName, + charPoolManager); pathsDecrypted.Add(path); } From 99c7a754d96a7f8c42adae0474b66850599982d0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 09:23:27 +0200 Subject: [PATCH 40/85] + Add .NET8.0 target for Cosmos.Encryption.Custom + Address new compiler complaints --- .../src/AeadAes/AeAesEncryptionProcessor.cs | 6 ++++ .../AeadAes/AeadAes256CbcHmac256Algorithm.cs | 4 +++ .../src/Common/CosmosJsonDotNetSerializer.cs | 4 +++ .../src/CosmosDataEncryptionKeyProvider.cs | 4 +++ .../src/DataEncryptionKey.cs | 4 +++ .../src/DataEncryptionKeyContainerCore.cs | 8 ++++++ .../DataEncryptionKeyContainerInlineCore.cs | 4 +++ .../src/DataEncryptionKeyFeedIterator{T}.cs | 6 ++-- .../src/DecryptableFeedResponse.cs | 4 +++ .../src/EncryptionContainer.cs | 28 ++++++++++++------- .../src/EncryptionExceptionFactory.cs | 2 ++ .../src/EncryptionFeedIterator{T}.cs | 5 ++-- .../src/EncryptionKeyWrapMetadata.cs | 13 --------- .../src/EncryptionProcessor.cs | 12 ++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 5 ++++ .../src/MdeServices/MdeKeyWrapProvider.cs | 8 ++++++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 3 +- .../src/SecurityUtility.cs | 6 ++++ .../JObjectSqlSerializer.Stable.cs | 2 +- .../MdeEncryptionProcessor.Preview.cs | 13 +++++++++ .../MdeEncryptionProcessor.Stable.cs | 12 ++++++++ .../src/Transformation/MemoryTextReader.cs | 19 +++++++++++++ .../EmulatorTests/LegacyEncryptionTests.cs | 14 ++++------ .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- ...mos.Encryption.Custom.EmulatorTests.csproj | 2 +- ...Encryption.Custom.Performance.Tests.csproj | 2 +- ...zure.Cosmos.Encryption.Custom.Tests.csproj | 2 +- .../Contracts/ContractEnforcement.cs | 16 +++++++---- 28 files changed, 162 insertions(+), 48 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs index 0944714aac..c7eb3658d4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs @@ -13,6 +13,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using Newtonsoft.Json; using Newtonsoft.Json.Linq; +#pragma warning disable IDE0057 // Use range operator +#pragma warning disable VSTHRD103 // Call async methods when in an async method internal static class AeAesEncryptionProcessor { public static async Task EncryptAsync( @@ -65,6 +67,7 @@ public static async Task EncryptAsync( encryptionOptions.PathsToEncrypt); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -113,4 +116,7 @@ internal static async Task DecryptContentAsync( return decryptionContext; } } + +#pragma warning restore IDE0057 // Use range operator +#pragma warning restore VSTHRD103 // Call async methods when in an async method } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs index fab34ec143..3bc0486487 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs @@ -10,6 +10,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Security.Cryptography; +#pragma warning disable SYSLIB0021 // Type or member is obsolete + /// /// This class implements authenticated encryption algorithm with associated data as described in /// http://tools.ietf.org/html/draft-mcgrew-aead-aes-cbc-hmac-sha2-05 - specifically this implements @@ -484,4 +486,6 @@ private static int GetCipherTextLength(int inputSize) return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; } } + +#pragma warning restore SYSLIB0021 // Type or member is obsolete } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs index b00c004476..9ed8056360 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs @@ -40,10 +40,14 @@ internal CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSetting /// The object representing the deserialized stream public T FromStream(Stream stream) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(stream); +#else if (stream == null) { throw new ArgumentNullException(nameof(stream)); } +#endif if (typeof(Stream).IsAssignableFrom(typeof(T))) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs index cd2fe84fbc..c6f1ffb66c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs @@ -148,10 +148,14 @@ public async Task InitializeAsync( throw new InvalidOperationException($"{nameof(CosmosDataEncryptionKeyProvider)} has already been initialized."); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(database); +#else if (database == null) { throw new ArgumentNullException(nameof(database)); } +#endif ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync( containerId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index b764debfd5..810227479e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -102,10 +102,14 @@ public static DataEncryptionKey Create( byte[] rawKey, string encryptionAlgorithm) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(rawKey); +#else if (rawKey == null) { throw new ArgumentNullException(nameof(rawKey)); } +#endif #pragma warning disable CS0618 // Type or member is obsolete if (!string.Equals(encryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized)) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs index 90cf907abc..1fcdd8c783 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs @@ -67,10 +67,14 @@ public override async Task> CreateData throw new ArgumentException(string.Format("Unsupported Encryption Algorithm {0}", encryptionAlgorithm), nameof(encryptionAlgorithm)); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(encryptionKeyWrapMetadata); +#else if (encryptionKeyWrapMetadata == null) { throw new ArgumentNullException(nameof(encryptionKeyWrapMetadata)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); @@ -155,10 +159,14 @@ public override async Task> RewrapData ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(newWrapMetadata); +#else if (newWrapMetadata == null) { throw new ArgumentNullException(nameof(newWrapMetadata)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs index d918d85d4a..bab5cadd62 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs @@ -78,10 +78,14 @@ public override Task> RewrapDataEncryp throw new ArgumentNullException(nameof(id)); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(newWrapMetadata); +#else if (newWrapMetadata == null) { throw new ArgumentNullException(nameof(newWrapMetadata)); } +#endif return TaskHelper.RunInlineIfNeededAsync(() => this.dataEncryptionKeyContainerCore.RewrapDataEncryptionKeyAsync(id, newWrapMetadata, encryptionAlgorithm, requestOptions, cancellationToken)); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs index 18e4246fdc..d01422c81b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs @@ -13,7 +13,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal sealed class DataEncryptionKeyFeedIterator : FeedIterator { - private readonly FeedIterator feedIterator; + private readonly DataEncryptionKeyFeedIterator feedIterator; private readonly CosmosResponseFactory responseFactory; public DataEncryptionKeyFeedIterator( @@ -57,7 +57,7 @@ public override async Task> ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { - dataEncryptionKeyPropertiesList = this.ConvertResponseToDataEncryptionKeyPropertiesList( + dataEncryptionKeyPropertiesList = DataEncryptionKeyFeedIterator.ConvertResponseToDataEncryptionKeyPropertiesList( responseMessage.Content); return (responseMessage, dataEncryptionKeyPropertiesList); @@ -67,7 +67,7 @@ public override async Task> ReadNextAsync(CancellationToken canc } } - private List ConvertResponseToDataEncryptionKeyPropertiesList( + private static List ConvertResponseToDataEncryptionKeyPropertiesList( Stream content) { JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs index c16497ac20..24c07c71e4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs @@ -45,10 +45,14 @@ internal static DecryptableFeedResponse CreateResponse( ResponseMessage responseMessage, IReadOnlyCollection resource) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(responseMessage); +#else if (responseMessage == null) { throw new ArgumentNullException(nameof(responseMessage)); } +#endif using (responseMessage) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index d870f1b835..d13c1c16fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -122,10 +122,14 @@ public override async Task CreateItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (streamPayload == null) { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("CreateItemStream")) @@ -304,6 +308,10 @@ public override async Task> ReplaceItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(item); +#else if (id == null) { throw new ArgumentNullException(nameof(id)); @@ -313,6 +321,7 @@ public override async Task> ReplaceItemAsync( { throw new ArgumentNullException(nameof(item)); } +#endif if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || encryptionItemRequestOptions.EncryptionOptions == null) @@ -384,6 +393,10 @@ public override async Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (id == null) { throw new ArgumentNullException(nameof(id)); @@ -393,6 +406,7 @@ public override async Task ReplaceItemStreamAsync( { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("ReplaceItemStream")) @@ -428,11 +442,6 @@ private async Task ReplaceItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, @@ -536,10 +545,14 @@ public override async Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (streamPayload == null) { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("UpsertItemStream")) @@ -572,11 +585,6 @@ private async Task UpsertItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs index 55884f1093..3b50114cdb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal static class EncryptionExceptionFactory { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly internal static ArgumentException InvalidKeySize(string algorithmName, int actualKeylength, int expectedLength) { return new ArgumentException( @@ -28,6 +29,7 @@ internal static ArgumentException InvalidAlgorithmVersion(byte actual, byte expe $"Invalid encryption algorithm version; actual: {actual:X2}, expected: {expected:X2}.", "cipherText"); } +#pragma warning restore CA2208 // Instantiate argument exceptions correctly internal static ArgumentException InvalidAuthenticationTag() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs index ee22f330ac..81f1516e1b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs @@ -11,7 +11,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal sealed class EncryptionFeedIterator : FeedIterator { - private readonly FeedIterator feedIterator; + private readonly EncryptionFeedIterator feedIterator; private readonly CosmosResponseFactory responseFactory; public EncryptionFeedIterator( @@ -31,8 +31,7 @@ public override async Task> ReadNextAsync(CancellationToken canc if (typeof(T) == typeof(DecryptableItem)) { IReadOnlyCollection resource; - EncryptionFeedIterator encryptionFeedIterator = this.feedIterator as EncryptionFeedIterator; - (responseMessage, resource) = await encryptionFeedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); + (responseMessage, resource) = await this.feedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); return DecryptableFeedResponse.CreateResponse( responseMessage, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs index e3c285531d..5855a011cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs @@ -114,18 +114,5 @@ public bool Equals(EncryptionKeyWrapMetadata other) this.Value == other.Value && this.Name == other.Name; } - - internal string GetName(EncryptionKeyWrapMetadata encryptionKeyWrapMetadata) - { - /* A legacy DEK may not have a Name value in meta-data*/ - if (string.IsNullOrWhiteSpace(encryptionKeyWrapMetadata.Name)) - { - return encryptionKeyWrapMetadata.Value; - } - else - { - return encryptionKeyWrapMetadata.Name; - } - } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e1e384d156..53d74390f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -112,7 +112,11 @@ public static async Task EncryptAsync( } DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return (BaseSerializer.ToStream(itemJObj), decryptionContext); } @@ -181,6 +185,11 @@ private static void ValidateInputForEncrypt( Encryptor encryptor, EncryptionOptions encryptionOptions) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(encryptor); + ArgumentNullException.ThrowIfNull(encryptionOptions); +#else if (input == null) { throw new ArgumentNullException(nameof(input)); @@ -195,7 +204,9 @@ private static void ValidateInputForEncrypt( { throw new ArgumentNullException(nameof(encryptionOptions)); } +#endif +#pragma warning disable CA2208 // Instantiate argument exceptions correctly if (string.IsNullOrWhiteSpace(encryptionOptions.DataEncryptionKeyId)) { throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId)); @@ -210,6 +221,7 @@ private static void ValidateInputForEncrypt( { throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); } +#pragma warning restore CA2208 // Instantiate argument exceptions correctly } private static JObject RetrieveItem( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 963ab8705a..ff543dfc0a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -37,6 +37,10 @@ public MdeEncryptionAlgorithm( TimeSpan? cacheTimeToLive, bool withRawKey = false) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(dekProperties); + ArgumentNullException.ThrowIfNull(encryptionKeyStoreProvider); +#else if (dekProperties == null) { throw new ArgumentNullException(nameof(dekProperties)); @@ -46,6 +50,7 @@ public MdeEncryptionAlgorithm( { throw new ArgumentNullException(nameof(encryptionKeyStoreProvider)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( dekProperties.EncryptionKeyWrapMetadata.Name, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs index 6520ecd67e..297a4dac7e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs @@ -28,10 +28,14 @@ public override Task UnwrapKeyAsync( EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(metadata); +#else if (metadata == null) { throw new ArgumentNullException(nameof(metadata)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( metadata.Name, @@ -47,10 +51,14 @@ public override Task WrapKeyAsync( EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(metadata); +#else if (metadata == null) { throw new ArgumentNullException(nameof(metadata)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( metadata.Name, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index a192203753..d17c1bc48b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -1,6 +1,6 @@ - netstandard2.0 + netstandard2.0;net8.0 Microsoft.Azure.Cosmos.Encryption.Custom Microsoft.Azure.Cosmos.Encryption.Custom $(LangVersion) @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + True IS_PREVIEW diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs index 632f1904d5..9a7ad82bbe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs @@ -42,7 +42,9 @@ internal static string GetSHA256Hash(byte[] input) using (SHA256 sha256 = SHA256.Create()) { +#pragma warning disable CA1850 // Prefer static 'HashData' method over 'ComputeHash' byte[] hashValue = sha256.ComputeHash(input); +#pragma warning restore CA1850 // Prefer static 'HashData' method over 'ComputeHash' return GetHexString(hashValue); } } @@ -54,8 +56,12 @@ internal static string GetSHA256Hash(byte[] input) internal static void GenerateRandomBytes(byte[] randomBytes) { // Generate random bytes cryptographically. +#if NET8_0_OR_GREATER + RandomNumberGenerator.Fill(randomBytes); +#else using RNGCryptoServiceProvider rngCsp = new (); rngCsp.GetBytes(randomBytes); +#endif } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs index f1332baea9..209048f907 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -21,7 +21,7 @@ internal class JObjectSqlSerializer private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; - internal (TypeMarker, byte[]) Serialize(JToken propertyValue) + internal virtual (TypeMarker, byte[]) Serialize(JToken propertyValue) { switch (propertyValue.Type) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index ccb59b0b40..b0f293a592 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -36,7 +36,11 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -69,7 +73,11 @@ public async Task EncryptAsync( pathsEncrypted); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -95,7 +103,12 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif + if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index a861a05996..d4222b6f1d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -36,7 +36,11 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -69,7 +73,11 @@ public async Task EncryptAsync( pathsEncrypted); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -94,7 +102,11 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs index ec88ccd100..893d31b864 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs @@ -71,6 +71,12 @@ public override int Read() public override int Read(char[] buffer, int index, int count) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); + ArgumentOutOfRangeException.ThrowIfNegative(index); + ArgumentOutOfRangeException.ThrowIfNegative(count); + ArgumentOutOfRangeException.ThrowIfGreaterThan(count, buffer.Length - index); +#else if (buffer == null) { throw new ArgumentNullException(nameof(buffer)); @@ -90,6 +96,7 @@ public override int Read(char[] buffer, int index, int count) { throw new ArgumentOutOfRangeException(); } +#endif if (this.closed) { @@ -119,7 +126,11 @@ public override string ReadToEnd() } this.pos = this.length; +#if NET8_0_OR_GREATER + return new string(this.chars[this.pos..this.length].Span); +#else return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); +#endif } public override string ReadLine() @@ -135,7 +146,11 @@ public override string ReadLine() char ch = this.chars.Span[i]; if (ch == '\r' || ch == '\n') { +#if NET8_0_OR_GREATER + string result = new (this.chars[this.pos..i].Span); +#else string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); +#endif this.pos = i + 1; if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') { @@ -150,7 +165,11 @@ public override string ReadLine() if (i > this.pos) { +#if NET8_0_OR_GREATER + string result = new (this.chars[this.pos..i].Span); +#else string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); +#endif this.pos = i; return result; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 30f6364a32..9ee43b8b07 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1491,7 +1491,7 @@ private static EncryptionItemRequestOptions GetRequestOptions( }; } - private static TransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( string dekId, List pathsToEncrypt, string ifMatchEtag = null) @@ -1812,13 +1812,11 @@ public override Stream ToStream(T input) MemoryStream streamPayload = new(); using (StreamWriter streamWriter = new(streamPayload, encoding: Encoding.UTF8, bufferSize: 1024, leaveOpen: true)) { - using (JsonWriter writer = new JsonTextWriter(streamWriter)) - { - writer.Formatting = Formatting.None; - this.serializer.Serialize(writer, input); - writer.Flush(); - streamWriter.Flush(); - } + using JsonTextWriter writer = new (streamWriter); + writer.Formatting = Formatting.None; + this.serializer.Serialize(writer, input); + writer.Flush(); + streamWriter.Flush(); } streamPayload.Position = 0; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 661b876907..14b1915abb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -1822,7 +1822,7 @@ private static EncryptionItemRequestOptions GetRequestOptions( } } - private static TransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( string dekId, List pathsToEncrypt, string ifMatchEtag = null) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj index c5dfa8db0c..915be6c3f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj @@ -3,7 +3,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 4deb4d0edf..821014c014 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -3,7 +3,7 @@ Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests Exe - net6 + net8.0 enable enable diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index eee64aada9..7e1f7d48fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -4,7 +4,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Tests diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index c58109d65b..84bd8c63c1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -14,7 +14,7 @@ public class ContractEnforcement { - private static readonly InvariantComparer invariantComparer = new InvariantComparer(); + private static readonly InvariantComparer invariantComparer = new (); private static Assembly GetAssemblyLocally(string name) { @@ -91,7 +91,10 @@ private static string GenerateNameWithClassAttributes(Type type) $"{nameof(type.IsValueType)}:{(type.IsValueType ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsNested)}:{(type.IsNested ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsGenericType)}:{(type.IsGenericType ? bool.TrueString : bool.FalseString)};" + +#pragma warning disable SYSLIB0050 // 'Type.IsSerializable' is obsolete: 'Formatter-based serialization is obsolete and should not be used. $"{nameof(type.IsSerializable)}:{(type.IsSerializable ? bool.TrueString : bool.FalseString)}"; +#pragma warning restore SYSLIB0050 // 'Type.IsSerializable' is obsolete: 'Formatter-based serialization is obsolete and should not be used. + } private static string GenerateNameWithMethodAttributes(MethodInfo methodInfo) @@ -241,7 +244,7 @@ public static void ValidatePreviewContractContainBreakingChanges( public static string GetCurrentContract(string dllName) { - TypeTree locally = new TypeTree(typeof(object)); + TypeTree locally = new (typeof(object)); Assembly assembly = ContractEnforcement.GetAssemblyLocally(dllName); Type[] exportedTypes = assembly.GetExportedTypes(); ContractEnforcement.BuildTypeTree(locally, exportedTypes); @@ -252,13 +255,13 @@ public static string GetCurrentContract(string dllName) public static string GetCurrentTelemetryContract(string dllName) { - List nonTelemetryModels = new List + List nonTelemetryModels = new() { "AzureVMMetadata", "Compute" }; - TypeTree locally = new TypeTree(typeof(object)); + TypeTree locally = new (typeof(object)); Assembly assembly = ContractEnforcement.GetAssemblyLocally(dllName); Type[] exportedTypes = assembly.GetTypes().Where(t => t!= null && @@ -328,7 +331,10 @@ public static void ValidateJsonAreSame(string baselineJson, string currentJson) private class InvariantComparer : IComparer { - public int Compare(string a, string b) => Comparer.DefaultInvariant.Compare(a, b); + public int Compare(string a, string b) + { + return Comparer.DefaultInvariant.Compare(a, b); + } } } } From 31c20e74721c88cda94b1a58633b382409fde480 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 09:30:13 +0200 Subject: [PATCH 41/85] - remove implicit IsPreview from csproj --- .../src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index d17c1bc48b..83f2758040 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,7 +17,6 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok - True IS_PREVIEW From 90b2cecf0352aa7d1b9ebc15053635bb2f2c9b68 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 11:14:57 +0200 Subject: [PATCH 42/85] + JsonNodeSqlSerializer ! JObjectSqlSerializer remove formatting on serializing inner arrays/objects to string --- .../JObjectSqlSerializer.Preview.cs | 4 +- .../JsonNodeSqlSerializer.Preview.cs | 93 +++++++++++++++++++ .../JsonNodeSqlSerializerTests.cs | 92 ++++++++++++++++++ 3 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 2c706aa361..b1ccb5b88c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -49,10 +49,10 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB (buffer, length) = SerializeString(propertyValue.ToObject()); return (TypeMarker.String, buffer, length); case JTokenType.Array: - (buffer, length) = SerializeString(propertyValue.ToString()); + (buffer, length) = SerializeString(propertyValue.ToString(Formatting.None)); return (TypeMarker.Array, buffer, length); case JTokenType.Object: - (buffer, length) = SerializeString(propertyValue.ToString()); + (buffer, length) = SerializeString(propertyValue.ToString(Formatting.None)); return (TypeMarker.Object, buffer, length); default: throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs new file mode 100644 index 0000000000..e86915700b --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -0,0 +1,93 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if IS_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using System.Text.Json; + using System.Text.Json.Nodes; + using Microsoft.Data.Encryption.Cryptography.Serializers; + + internal class JsonNodeSqlSerializer + { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + +#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed + internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JsonNode propertyValue, ArrayPoolManager arrayPoolManager) + { + byte[] buffer; + int length; + + if (propertyValue == null) + { + return (TypeMarker.Null, null, -1); + } + + switch (propertyValue.GetValueKind()) + { + case JsonValueKind.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null, -1); + case JsonValueKind.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null, -1); + case JsonValueKind.True: + (buffer, length) = SerializeFixed(SqlBoolSerializer, true); + return (TypeMarker.Boolean, buffer, length); + case JsonValueKind.False: + (buffer, length) = SerializeFixed(SqlBoolSerializer, false); + return (TypeMarker.Boolean, buffer, length); + case JsonValueKind.Number: + if (long.TryParse(propertyValue.ToJsonString(), out long longValue)) + { + (buffer, length) = SerializeFixed(SqlLongSerializer, longValue); + return (TypeMarker.Long, buffer, length); + } + else if (double.TryParse(propertyValue.ToJsonString(), out double doubleValue)) + { + (buffer, length) = SerializeFixed(SqlDoubleSerializer, doubleValue); + return (TypeMarker.Double, buffer, length); + } + else + { + throw new InvalidOperationException("Unsupported Number type"); + } + + case JsonValueKind.String: + (buffer, length) = SerializeString(propertyValue.GetValue()); + return (TypeMarker.String, buffer, length); + case JsonValueKind.Array: + (buffer, length) = SerializeString(propertyValue.ToJsonString()); + return (TypeMarker.Array, buffer, length); + case JsonValueKind.Object: + (buffer, length) = SerializeString(propertyValue.ToJsonString()); + return (TypeMarker.Object, buffer, length); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.GetValueKind()}"); + } + + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer, T value) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int length = serializer.Serialize(value, buffer); + return (buffer, length); + } + + (byte[], int) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); + } + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs new file mode 100644 index 0000000000..f8f46fea2f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs @@ -0,0 +1,92 @@ +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text.Json.Nodes; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json.Linq; + + [TestClass] + public class JsonNodeSqlSerializerTests + { + private static ArrayPoolManager _poolManager; + + [ClassInitialize] + public static void ClassInitialize(TestContext context) + { + _ = context; + _poolManager = new ArrayPoolManager(); + } + + [TestMethod] + [DynamicData(nameof(SerializationSamples))] + public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[] expectedBytes, int expectedLength) + { + JsonNodeSqlSerializer serializer = new(); + + (TypeMarker serializedType, byte[] serializedBytes, int serializedBytesCount) = serializer.Serialize(testNode, _poolManager); + + Assert.AreEqual((TypeMarker)expectedType, serializedType); + Assert.AreEqual(expectedLength, serializedBytesCount); + if (expectedLength == -1) + { + Assert.IsTrue(serializedBytes == null); + } + else + { + Assert.IsTrue(expectedBytes.SequenceEqual(serializedBytes.AsSpan(0, serializedBytesCount).ToArray())); + } + } + + public static IEnumerable SerializationSamples + { + get + { + List values = new() + { + new object[] {JsonValue.Create((string)null), (byte)TypeMarker.Null, null, -1 }, + new object[] {JsonValue.Create(true), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), 8}, + new object[] {JsonValue.Create(false), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), 8}, + new object[] {JsonValue.Create(192), (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), 8}, + new object[] {JsonValue.Create(192.5), (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), 8}, + new object[] {JsonValue.Create(testString), (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), 11}, + new object[] {JsonValue.Create(testArray), (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), 10}, + new object[] {JsonValue.Create(testClass), (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), 33} + }; + + return values; + } + } + + private static readonly string testString = "Hello world"; + private static readonly int[] testArray = new[] {10, 18, 19}; + private static readonly TestClass testClass = new() { SomeInt = 1, SomeString = "asdf" }; + + private class TestClass + { + public int SomeInt { get; set; } + public string SomeString { get; set; } + } + + private static byte[] GetNewtonsoftValueEquivalent(T value) + { + JObjectSqlSerializer serializer = new (); + JToken token = value switch + { + int[] => new JArray(value), + TestClass => JObject.FromObject(value), + _ => new JValue(value), + }; + (TypeMarker _, byte[] bytes, int lenght) = serializer.Serialize(token, _poolManager); + return bytes.AsSpan(0, lenght).ToArray(); + } + + } +} + +#endif \ No newline at end of file From 6f27d9d6a2c087521dc30ddd6084ff738c303666 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 14:10:54 +0200 Subject: [PATCH 43/85] + initial commit --- .../src/EncryptionOptions.cs | 25 +++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 1 + .../MdeEncryptionProcessor.Preview.cs | 115 ++----------- .../src/Transformation/MdeEncryptor.cs | 23 +++ .../MdeJObjectEncryptionProcessor.Preview.cs | 160 ++++++++++++++++++ .../MdeJsonNodeEncryptionProcessor.Preview.cs | 115 +++++++++++++ .../SystemTextJson/JsonBytes.cs | 34 ++++ .../SystemTextJson/JsonBytesConverter.cs | 26 +++ .../EncryptionBenchmark.cs | 10 +- ...Encryption.Custom.Performance.Tests.csproj | 1 + .../Transformation/JsonBytesConverterTests.cs | 40 +++++ .../Transformation/JsonBytesTests.cs | 33 ++++ 12 files changed, 475 insertions(+), 108 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index a5dff8f12b..2d9ad7aee6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -6,6 +6,25 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System.Collections.Generic; + /// + /// API for JSON processing + /// + public enum JsonProcessor + { + /// + /// Newtonsoft.Json + /// + Newtonsoft, + +#if NET8_0_OR_GREATER + /// + /// System.Text.Json + /// + /// Available with .NET8.0 package only. + SystemTextJson, +#endif + } + /// /// Options for encryption of data. /// @@ -35,5 +54,11 @@ public sealed class EncryptionOptions /// Example of a path specification: /sensitive /// public IEnumerable PathsToEncrypt { get; set; } + + /// + /// Gets or sets API used for Json processing + /// + /// Setting only applies with Mde encryption is used. + public JsonProcessor JsonProcessor { get; set; } = JsonProcessor.Newtonsoft; } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 83f2758040..d17c1bc48b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + True IS_PREVIEW diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index b0f293a592..530be77e14 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -7,18 +7,18 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; - using System.Collections.Generic; using System.IO; - using System.Linq; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; internal class MdeEncryptionProcessor { - internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + internal MdeJObjectEncryptionProcessor JObjectEncryptionProcessor { get; set; } = new MdeJObjectEncryptionProcessor(); - internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); +#if NET8_0_OR_GREATER + internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor(); +#endif public async Task EncryptAsync( Stream input, @@ -26,59 +26,14 @@ public async Task EncryptAsync( EncryptionOptions encryptionOptions, CancellationToken token) { - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); - List pathsEncrypted = new (); - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new (); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + return encryptionOptions.JsonProcessor switch { + JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #if NET8_0_OR_GREATER - string propertyName = pathToEncrypt[1..]; -#else - string propertyName = pathToEncrypt.Substring(1); + JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #endif - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - if (propertyValue.Type == JTokenType.Null) - { - continue; - } - - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - - if (plainText == null) - { - continue; - } - - byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); - - itemJObj[propertyName] = encryptedBytes; - pathsEncrypted.Add(pathToEncrypt); - } - - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); -#if NET8_0_OR_GREATER - await input.DisposeAsync(); -#else - input.Dispose(); -#endif - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + _ => throw new InvalidOperationException("Unsupported JsonProcessor") + }; } internal async Task DecryptObjectAsync( @@ -88,57 +43,7 @@ internal async Task DecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new (); - using ArrayPoolManager charPoolManager = new (); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); - foreach (string path in encryptionProperties.EncryptedPaths) - { -#if NET8_0_OR_GREATER - string propertyName = path[1..]; -#else - string propertyName = path.Substring(1); -#endif - - if (!document.TryGetValue(propertyName, out JToken propertyValue)) - { - // malformed document, such record shouldn't be there at all - continue; - } - - byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) - { - continue; - } - - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); - - this.Serializer.DeserializeAndAddProperty( - (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - document, - propertyName, - charPoolManager); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); - return decryptionContext; + return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index cf5a0ffdf6..bb1608a466 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -31,6 +31,29 @@ internal virtual byte[] Encrypt(DataEncryptionKey encryptionKey, TypeMarker type return encryptedText; } + internal virtual (byte[], int) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + { + int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; + + byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + + encryptedText[0] = (byte)typeMarker; + + int encryptedLength = encryptionKey.EncryptData( + plainText, + plainTextOffset: 0, + plainTextLength, + encryptedText, + outputOffset: 1); + + if (encryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); + } + + return (encryptedText, encryptedTextLength); + } + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) { int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs new file mode 100644 index 0000000000..d606e2fc28 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -0,0 +1,160 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + + internal class MdeJObjectEncryptionProcessor + { + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + + Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); + +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else + input.Dispose(); +#endif + + return result; + } + + public async Task EncryptAsync( + JObject input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else + string propertyName = pathToEncrypt.Substring(1); +#endif + if (!input.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); + + input[propertyName] = encryptedBytes; + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + input.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + + return EncryptionProcessor.BaseSerializer.ToStream(input); + } + + internal async Task DecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else + string propertyName = path.Substring(1); +#endif + + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName, + charPoolManager); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs new file mode 100644 index 0000000000..45a886d3ba --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -0,0 +1,115 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if IS_PREVIEW && NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System.Collections.Generic; + using System.IO; + using System.Text.Json; + using System.Text.Json.Nodes; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + + internal class MdeJsonNodeEncryptionProcessor + { + internal JsonNodeSqlSerializer Serializer { get; set; } = new JsonNodeSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + internal JsonSerializerOptions JsonSerializerOptions { get; set; } + + private JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; + + public MdeJsonNodeEncryptionProcessor() + { + this.JsonSerializerOptions = new JsonSerializerOptions(); + this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); + } + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JsonNode itemJObj = JsonNode.Parse(input); + + Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); + + await input.DisposeAsync(); + return result; + } + + public async Task EncryptAsync( + JsonNode document, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + JsonObject itemObj = document.AsObject(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else + string propertyName = pathToEncrypt.Substring(1); +#endif + if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) + { + continue; + } + + if (propertyValue == null || propertyValue.GetValueKind() == JsonValueKind.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + + itemObj[propertyName] = JsonValue.Create(new JsonBytes(encryptedBytes, 0, encryptedBytesCount)); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + JsonNode propertiesNode = JsonSerializer.SerializeToNode(encryptionProperties); + + itemObj.Add(Constants.EncryptedInfo, propertiesNode); + + MemoryStream ms = new (); + Utf8JsonWriter writer = new (ms, this.jsonWriterOptions); + + JsonSerializer.Serialize(writer, document, this.JsonSerializerOptions); + + ms.Position = 0; + return ms; + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs new file mode 100644 index 0000000000..a8bc77f75b --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson +{ + using System; + + internal class JsonBytes + { + internal byte[] Bytes { get; private set; } + + internal int Offset { get; private set; } + + internal int Length { get; private set; } + + public JsonBytes(byte[] bytes, int offset, int length) + { + ArgumentNullException.ThrowIfNull(bytes); + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegative(length); + if (bytes.Length < offset + length) + { + throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); + } + + this.Bytes = bytes; + this.Offset = offset; + this.Length = length; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs new file mode 100644 index 0000000000..d9048375c0 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson +{ + using System; + using System.Text.Json; + using System.Text.Json.Serialization; + + internal class JsonBytesConverter : JsonConverter + { + public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + throw new NotImplementedException(); + } + + public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) + { + writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 638222785f..1e84efccc7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -25,6 +25,9 @@ public partial class EncryptionBenchmark [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } + [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson)] + public JsonProcessor JsonProcessor { get; set; } + [GlobalSetup] public async Task Setup() { @@ -38,7 +41,7 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.encryptionOptions = CreateEncryptionOptions(); + this.encryptionOptions = this.CreateEncryptionOptions(); this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( @@ -74,13 +77,14 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private static EncryptionOptions CreateEncryptionOptions() + private EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() { DataEncryptionKeyId = "dekId", EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = this.JsonProcessor, }; return options; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 821014c014..a36482571b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -6,6 +6,7 @@ net8.0 enable enable + true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs new file mode 100644 index 0000000000..fbef47e872 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs @@ -0,0 +1,40 @@ +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ + using System; + using System.IO; + using System.Text.Json; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class JsonBytesConverterTests + { + [TestMethod] + public void Write_Results_IdenticalToNewtonsoft() + { + byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; + + JsonBytes jsonBytes = new (bytes, 5, 5); + + using MemoryStream ms = new (); + using Utf8JsonWriter writer = new (ms); + + JsonBytesConverter jsonConverter = new (); + jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); + + writer.Flush(); + ms.Flush(); + ms.Position = 0; + StreamReader sr = new(ms); + string systemTextResult = sr.ReadToEnd(); + + byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); + string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); + + Assert.AreEqual(systemTextResult, newtonsoftResult); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs new file mode 100644 index 0000000000..a29f378ff2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs @@ -0,0 +1,33 @@ +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ + using System; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class JsonBytesTests + { + [TestMethod] + public void Ctor_ThrowsForInvalidInputs() + { + Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); + Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); + Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); + Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); + } + + [TestMethod] + public void Properties_AreSetCorrectly() + { + byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; + JsonBytes jsonBytes = new (bytes, 1, 5); + + Assert.AreEqual(1, jsonBytes.Offset); + Assert.AreEqual(5, jsonBytes.Length); + Assert.AreSame(bytes, jsonBytes.Bytes); + } + } +} +#endif From d7c06d656315b2974af1826f7f9184656b6ad0ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 14:24:30 +0200 Subject: [PATCH 44/85] ! EncryptioProperties System.Text.Json annotations + tests for MdeEncryptionProperties --- .../src/EncryptionProperties.cs | 6 +++ .../MdeEncryptionProcessorTests.cs | 45 +++++++++++++------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs index 8e0ccaf84a..5108cfa157 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs @@ -5,23 +5,29 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System.Collections.Generic; + using System.Text.Json.Serialization; using Newtonsoft.Json; internal class EncryptionProperties { [JsonProperty(PropertyName = Constants.EncryptionFormatVersion)] + [JsonPropertyName(Constants.EncryptionFormatVersion)] public int EncryptionFormatVersion { get; } [JsonProperty(PropertyName = Constants.EncryptionDekId)] + [JsonPropertyName(Constants.EncryptionDekId)] public string DataEncryptionKeyId { get; } [JsonProperty(PropertyName = Constants.EncryptionAlgorithm)] + [JsonPropertyName(Constants.EncryptionAlgorithm)] public string EncryptionAlgorithm { get; } [JsonProperty(PropertyName = Constants.EncryptedData)] + [JsonPropertyName(Constants.EncryptedData)] public byte[] EncryptedData { get; } [JsonProperty(PropertyName = Constants.EncryptedPaths)] + [JsonPropertyName(Constants.EncryptedPaths)] public IEnumerable EncryptedPaths { get; } public EncryptionProperties( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 69a9c7afb0..35722f40eb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -20,19 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests public class MdeEncryptionProcessorTests { private static Mock mockEncryptor; - private static EncryptionOptions encryptionOptions; private const string dekId = "dekId"; [ClassInitialize] public static void ClassInitialize(TestContext testContext) { _ = testContext; - encryptionOptions = new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt - }; Mock DekMock = new(); DekMock.Setup(m => m.EncryptData(It.IsAny())) @@ -125,12 +118,13 @@ await EncryptionProcessor.EncryptAsync( } [TestMethod] - public async Task EncryptDecryptPropertyWithNullValue() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -146,11 +140,12 @@ public async Task EncryptDecryptPropertyWithNullValue() } [TestMethod] - public async Task ValidateEncryptDecryptDocument() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -166,7 +161,8 @@ public async Task ValidateEncryptDecryptDocument() } [TestMethod] - public async Task ValidateDecryptStream() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); @@ -209,7 +205,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() Assert.IsNull(decryptionContext); } - private static async Task VerifyEncryptionSucceeded(TestDoc testDoc) + private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions) { Stream encryptedStream = await EncryptionProcessor.EncryptAsync( testDoc.ToStream(), @@ -287,5 +283,26 @@ private static void VerifyDecryptionSucceeded( } } } + + public static IEnumerable EncryptionOptionsCombinations => new[] { + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft + } + }, +#if NET8_0_OR_GREATER + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.SystemTextJson + } + }, +#endif + }; } -} +} \ No newline at end of file From cb9d1665dd563edd6f5b633a6983899bdd9069ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 14:51:57 +0200 Subject: [PATCH 45/85] + bump benchmark --- .../Readme.md | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index ced6fc95ed..8be5532ca4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -3,17 +3,23 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores .NET SDK=8.0.400 - [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 + [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | -| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | -| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | -| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | +| Method | DocumentSizeInKb | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **Newtonsoft** | **22.40 μs** | **0.342 μs** | **0.501 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | +| Decrypt | 1 | Newtonsoft | 25.81 μs | 0.305 μs | 0.427 μs | 0.1526 | 0.0305 | - | 39.19 KB | +| **Encrypt** | **1** | **SystemTextJson** | **14.72 μs** | **0.275 μs** | **0.411 μs** | **0.0916** | **0.0153** | **-** | **22.55 KB** | +| Decrypt | 1 | SystemTextJson | 25.69 μs | 0.290 μs | 0.396 μs | 0.1526 | 0.0305 | - | 39.19 KB | +| **Encrypt** | **10** | **Newtonsoft** | **83.98 μs** | **0.437 μs** | **0.613 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | +| Decrypt | 10 | Newtonsoft | 99.39 μs | 0.553 μs | 0.827 μs | 0.6104 | 0.1221 | - | 152.45 KB | +| **Encrypt** | **10** | **SystemTextJson** | **41.92 μs** | **0.212 μs** | **0.304 μs** | **0.4272** | **0.0610** | **-** | **103.06 KB** | +| Decrypt | 10 | SystemTextJson | 99.43 μs | 0.558 μs | 0.835 μs | 0.6104 | 0.1221 | - | 152.45 KB | +| **Encrypt** | **100** | **Newtonsoft** | **1,074.93 μs** | **11.946 μs** | **17.510 μs** | **25.3906** | **23.4375** | **21.4844** | **1638.32 KB** | +| Decrypt | 100 | Newtonsoft | 1,133.11 μs | 20.544 μs | 29.463 μs | 17.5781 | 15.6250 | 15.6250 | 1229.43 KB | +| **Encrypt** | **100** | **SystemTextJson** | **797.64 μs** | **15.574 μs** | **22.828 μs** | **26.3672** | **26.3672** | **26.3672** | **942.81 KB** | +| Decrypt | 100 | SystemTextJson | 1,120.97 μs | 14.956 μs | 22.386 μs | 19.5313 | 17.5781 | 17.5781 | 1229.45 KB | From 5fc51650fd36a42d64553ba626b4825b2fba46eb Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 16:07:35 +0200 Subject: [PATCH 46/85] ~ wip --- .../JsonNodeSqlSerializer.Preview.cs | 34 +++++++++ .../MdeJObjectEncryptionProcessor.Preview.cs | 10 +-- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 69 +++++++++++++++++++ 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs index e86915700b..283e91bc3b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -88,6 +88,40 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB return (buffer, length); } } + + internal virtual void DeserializeAndAddProperty( + TypeMarker typeMarker, + ReadOnlySpan serializedBytes, + JsonNode jsonNode, + string key, + ArrayPoolManager arrayPoolManager) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jsonNode[key] = SqlBoolSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jsonNode[key] = SqlDoubleSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jsonNode[key] = SqlLongSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.String: + jsonNode[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + jsonNode[key] = JsonNode.Parse(serializedBytes); + break; + case TypeMarker.Object: + jsonNode[key] = JsonNode.Parse(serializedBytes); + break; + default: + Debug.Fail($"Unexpected type marker {typeMarker}"); + break; + } + } + } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index d606e2fc28..189bd34e42 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -96,11 +96,11 @@ public async Task EncryptAsync( } internal async Task DecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) { _ = diagnosticsContext; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 45a886d3ba..197e044e3b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -6,13 +6,16 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { + using System; using System.Collections.Generic; using System.IO; + using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + using Newtonsoft.Json.Linq; internal class MdeJsonNodeEncryptionProcessor { @@ -109,6 +112,72 @@ public async Task EncryptAsync( ms.Position = 0; return ms; } + + internal async Task DecryptObjectAsync( + JsonNode document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + + JsonObject itemObj = document.AsObject(); + + foreach (string path in encryptionProperties.EncryptedPaths) + { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else + string propertyName = path.Substring(1); +#endif + + if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + //TODO: figure out if we can get char span out from JsonNode + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(propertyValue.) + + byte[] cipherTextWithTypeMarker = Convert.FromBase64String.TryFromBase64StringBase64Encoder propertyValue.propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName, + charPoolManager); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } } } From c9e7f306cb0ec39026add9d19871860b9fc09c6a Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 00:34:58 +0200 Subject: [PATCH 47/85] + initial for JsonNode decryption - drop unnecessary classes --- .../src/EncryptionProcessor.cs | 102 +++++++++++++++++- .../MdeEncryptionProcessor.Preview.cs | 15 +++ .../MdeJsonNodeEncryptionProcessor.Preview.cs | 31 ++---- .../SystemTextJson/JsonBytes.cs | 34 ------ .../SystemTextJson/JsonBytesConverter.cs | 26 ----- .../EncryptionBenchmark.cs | 1 + .../Readme.md | 24 ++--- .../Transformation/JsonBytesConverterTests.cs | 40 ------- .../Transformation/JsonBytesTests.cs | 33 ------ 9 files changed, 135 insertions(+), 171 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 53d74390f8..c2c6abd498 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,6 +10,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; + using System.Text.Json; +#if NET8_0_OR_GREATER + using System.Text.Json.Nodes; +#endif using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; @@ -27,6 +31,7 @@ internal static class EncryptionProcessor }; internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -120,6 +125,54 @@ public static async Task EncryptAsync( return (BaseSerializer.ToStream(itemJObj), decryptionContext); } + public static async Task<(Stream, DecryptionContext)> DecryptAsync( + Stream input, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + JsonProcessor jsonProcessor, + CancellationToken cancellationToken) + { + return jsonProcessor switch + { + JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), +#if NET8_0_OR_GREATER + JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), +#endif + _ => throw new InvalidOperationException("Unsupported Json Processor") + }; + } + +#if NET8_0_OR_GREATER + public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( + Stream input, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (input == null) + { + return (input, null); + } + + Debug.Assert(input.CanSeek); + Debug.Assert(encryptor != null); + Debug.Assert(diagnosticsContext != null); + + JsonNode document = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken); + + (JsonNode decryptedDocument, DecryptionContext context) = await DecryptAsync(document, encryptor, diagnosticsContext, cancellationToken); + await input.DisposeAsync(); + + MemoryStream ms = new (); + Utf8JsonWriter writer = new (ms, EncryptionProcessor.JsonWriterOptions); + + System.Text.Json.JsonSerializer.Serialize(writer, decryptedDocument); + + ms.Position = 0; + return (ms, context); + } +#endif + public static async Task<(JObject, DecryptionContext)> DecryptAsync( JObject document, Encryptor encryptor, @@ -142,6 +195,53 @@ public static async Task EncryptAsync( return (document, decryptionContext); } +#if NET8_0_OR_GREATER + public static async Task<(JsonNode, DecryptionContext)> DecryptAsync( + JsonNode document, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + Debug.Assert(document != null); + + Debug.Assert(encryptor != null); + + if (!document.AsObject().TryGetPropertyValue(Constants.EncryptedInfo, out JsonNode encryptionPropertiesNode)) + { + throw new InvalidOperationException("Encryption properties deserialization failed."); + } + + EncryptionProperties encryptionProperties; + try + { + encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(encryptionPropertiesNode); + } + catch (Exception) + { + return (document, null); + } + + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionProperties, cancellationToken); + + return (document, decryptionContext); + } + + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JsonNode itemNode, EncryptionProperties encryptionProperties, CancellationToken cancellationToken) + { + DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch + { + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( + itemNode, + encryptor, + encryptionProperties, + diagnosticsContext, + cancellationToken), + _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), + }; + return decryptionContext; + } +#endif + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); @@ -240,7 +340,7 @@ private static JObject RetrieveItem( MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr }; - itemJObj = JsonSerializer.Create(jsonSerializerSettings).Deserialize(jsonTextReader); + itemJObj = Newtonsoft.Json.JsonSerializer.Create(jsonSerializerSettings).Deserialize(jsonTextReader); } return itemJObj; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 530be77e14..ecca230b6a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -8,6 +8,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.IO; +#if NET8_0_OR_GREATER + using System.Text.Json.Nodes; +#endif using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; @@ -45,6 +48,18 @@ internal async Task DecryptObjectAsync( { return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); } + +#if NET8_0_OR_GREATER + internal async Task DecryptObjectAsync( + JsonNode document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + return await this.JsonNodeEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); + } +#endif } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 197e044e3b..188bcbff21 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -10,12 +10,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Collections.Generic; using System.IO; using System.Linq; + using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Newtonsoft.Json.Linq; internal class MdeJsonNodeEncryptionProcessor { @@ -23,16 +22,8 @@ internal class MdeJsonNodeEncryptionProcessor internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - internal JsonSerializerOptions JsonSerializerOptions { get; set; } - private JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; - public MdeJsonNodeEncryptionProcessor() - { - this.JsonSerializerOptions = new JsonSerializerOptions(); - this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); - } - public async Task EncryptAsync( Stream input, Encryptor encryptor, @@ -64,11 +55,7 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { -#if NET8_0_OR_GREATER string propertyName = pathToEncrypt[1..]; -#else - string propertyName = pathToEncrypt.Substring(1); -#endif if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) { continue; @@ -89,7 +76,7 @@ public async Task EncryptAsync( (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); - itemObj[propertyName] = JsonValue.Create(new JsonBytes(encryptedBytes, 0, encryptedBytesCount)); + itemObj[propertyName] = JsonValue.Create(new Memory(encryptedBytes, 0, encryptedBytesCount)); pathsEncrypted.Add(pathToEncrypt); } @@ -107,7 +94,7 @@ public async Task EncryptAsync( MemoryStream ms = new (); Utf8JsonWriter writer = new (ms, this.jsonWriterOptions); - JsonSerializer.Serialize(writer, document, this.JsonSerializerOptions); + JsonSerializer.Serialize(writer, document); ms.Position = 0; return ms; @@ -138,11 +125,7 @@ internal async Task DecryptObjectAsync( foreach (string path in encryptionProperties.EncryptedPaths) { -#if NET8_0_OR_GREATER string propertyName = path[1..]; -#else - string propertyName = path.Substring(1); -#endif if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) { @@ -150,10 +133,8 @@ internal async Task DecryptObjectAsync( continue; } - //TODO: figure out if we can get char span out from JsonNode - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(propertyValue.) - - byte[] cipherTextWithTypeMarker = Convert.FromBase64String.TryFromBase64StringBase64Encoder propertyValue.propertyValue.ToObject(); + // can we get to internal JsonNode buffers to avoid string allocation here? + byte[] cipherTextWithTypeMarker = Convert.FromBase64String(propertyValue.GetValue()); if (cipherTextWithTypeMarker == null) { continue; @@ -175,7 +156,7 @@ internal async Task DecryptObjectAsync( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); - document.Remove(Constants.EncryptedInfo); + itemObj.Remove(Constants.EncryptedInfo); return decryptionContext; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs deleted file mode 100644 index a8bc77f75b..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - - internal class JsonBytes - { - internal byte[] Bytes { get; private set; } - - internal int Offset { get; private set; } - - internal int Length { get; private set; } - - public JsonBytes(byte[] bytes, int offset, int length) - { - ArgumentNullException.ThrowIfNull(bytes); - ArgumentOutOfRangeException.ThrowIfNegative(offset); - ArgumentOutOfRangeException.ThrowIfNegative(length); - if (bytes.Length < offset + length) - { - throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); - } - - this.Bytes = bytes; - this.Offset = offset; - this.Length = length; - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs deleted file mode 100644 index d9048375c0..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - using System.Text.Json; - using System.Text.Json.Serialization; - - internal class JsonBytesConverter : JsonConverter - { - public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) - { - writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 1e84efccc7..0d28361369 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -74,6 +74,7 @@ await EncryptionProcessor.DecryptAsync( new MemoryStream(this.encryptedData!), this.encryptor, new CosmosDiagnosticsContext(), + this.JsonProcessor, CancellationToken.None); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 8be5532ca4..b2bcc93f17 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,15 +11,15 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **Newtonsoft** | **22.40 μs** | **0.342 μs** | **0.501 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | -| Decrypt | 1 | Newtonsoft | 25.81 μs | 0.305 μs | 0.427 μs | 0.1526 | 0.0305 | - | 39.19 KB | -| **Encrypt** | **1** | **SystemTextJson** | **14.72 μs** | **0.275 μs** | **0.411 μs** | **0.0916** | **0.0153** | **-** | **22.55 KB** | -| Decrypt | 1 | SystemTextJson | 25.69 μs | 0.290 μs | 0.396 μs | 0.1526 | 0.0305 | - | 39.19 KB | -| **Encrypt** | **10** | **Newtonsoft** | **83.98 μs** | **0.437 μs** | **0.613 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | -| Decrypt | 10 | Newtonsoft | 99.39 μs | 0.553 μs | 0.827 μs | 0.6104 | 0.1221 | - | 152.45 KB | -| **Encrypt** | **10** | **SystemTextJson** | **41.92 μs** | **0.212 μs** | **0.304 μs** | **0.4272** | **0.0610** | **-** | **103.06 KB** | -| Decrypt | 10 | SystemTextJson | 99.43 μs | 0.558 μs | 0.835 μs | 0.6104 | 0.1221 | - | 152.45 KB | -| **Encrypt** | **100** | **Newtonsoft** | **1,074.93 μs** | **11.946 μs** | **17.510 μs** | **25.3906** | **23.4375** | **21.4844** | **1638.32 KB** | -| Decrypt | 100 | Newtonsoft | 1,133.11 μs | 20.544 μs | 29.463 μs | 17.5781 | 15.6250 | 15.6250 | 1229.43 KB | -| **Encrypt** | **100** | **SystemTextJson** | **797.64 μs** | **15.574 μs** | **22.828 μs** | **26.3672** | **26.3672** | **26.3672** | **942.81 KB** | -| Decrypt | 100 | SystemTextJson | 1,120.97 μs | 14.956 μs | 22.386 μs | 19.5313 | 17.5781 | 17.5781 | 1229.45 KB | +| **Encrypt** | **1** | **Newtonsoft** | **23.24 μs** | **0.606 μs** | **0.888 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | +| Decrypt | 1 | Newtonsoft | 24.95 μs | 0.230 μs | 0.344 μs | 0.1526 | 0.0305 | - | 39.27 KB | +| **Encrypt** | **1** | **SystemTextJson** | **14.46 μs** | **0.165 μs** | **0.242 μs** | **0.0916** | **0.0153** | **-** | **22.48 KB** | +| Decrypt | 1 | SystemTextJson | 16.05 μs | 0.284 μs | 0.416 μs | 0.0916 | 0.0305 | - | 22.1 KB | +| **Encrypt** | **10** | **Newtonsoft** | **84.60 μs** | **0.908 μs** | **1.359 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | +| Decrypt | 10 | Newtonsoft | 98.39 μs | 0.856 μs | 1.255 μs | 0.6104 | 0.1221 | - | 152.53 KB | +| **Encrypt** | **10** | **SystemTextJson** | **41.72 μs** | **0.341 μs** | **0.501 μs** | **0.4272** | **0.0610** | **-** | **102.99 KB** | +| Decrypt | 10 | SystemTextJson | 46.91 μs | 0.430 μs | 0.630 μs | 0.4272 | 0.0610 | - | 105.06 KB | +| **Encrypt** | **100** | **Newtonsoft** | **1,072.91 μs** | **13.802 μs** | **20.231 μs** | **25.3906** | **21.4844** | **21.4844** | **1638.34 KB** | +| Decrypt | 100 | Newtonsoft | 1,107.93 μs | 12.955 μs | 18.990 μs | 17.5781 | 15.6250 | 15.6250 | 1229.52 KB | +| **Encrypt** | **100** | **SystemTextJson** | **794.00 μs** | **25.274 μs** | **37.047 μs** | **24.4141** | **24.4141** | **24.4141** | **942.73 KB** | +| Decrypt | 100 | SystemTextJson | 819.31 μs | 14.159 μs | 20.754 μs | 22.4609 | 22.4609 | 22.4609 | 1037.04 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs deleted file mode 100644 index fbef47e872..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.IO; - using System.Text.Json; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesConverterTests - { - [TestMethod] - public void Write_Results_IdenticalToNewtonsoft() - { - byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - - JsonBytes jsonBytes = new (bytes, 5, 5); - - using MemoryStream ms = new (); - using Utf8JsonWriter writer = new (ms); - - JsonBytesConverter jsonConverter = new (); - jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); - - writer.Flush(); - ms.Flush(); - ms.Position = 0; - StreamReader sr = new(ms); - string systemTextResult = sr.ReadToEnd(); - - byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); - string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); - - Assert.AreEqual(systemTextResult, newtonsoftResult); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs deleted file mode 100644 index a29f378ff2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesTests - { - [TestMethod] - public void Ctor_ThrowsForInvalidInputs() - { - Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); - } - - [TestMethod] - public void Properties_AreSetCorrectly() - { - byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - JsonBytes jsonBytes = new (bytes, 1, 5); - - Assert.AreEqual(1, jsonBytes.Offset); - Assert.AreEqual(5, jsonBytes.Length); - Assert.AreSame(bytes, jsonBytes.Bytes); - } - } -} -#endif From fed6a3a21d8af9f958147cc45bc5264152d9c89c Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 13:35:32 +0200 Subject: [PATCH 48/85] ~ polishing and benchmark refresh --- .../JsonNodeSqlSerializer.Preview.cs | 25 ++++++----------- .../src/Transformation/MdeEncryptor.cs | 6 ++-- .../MdeJObjectEncryptionProcessor.Preview.cs | 2 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 11 ++++---- .../Readme.md | 28 +++++++++---------- 5 files changed, 31 insertions(+), 41 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs index 283e91bc3b..3fff2166de 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -89,39 +89,30 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB } } - internal virtual void DeserializeAndAddProperty( + internal virtual JsonNode Deserialize( TypeMarker typeMarker, ReadOnlySpan serializedBytes, - JsonNode jsonNode, - string key, ArrayPoolManager arrayPoolManager) { switch (typeMarker) { case TypeMarker.Boolean: - jsonNode[key] = SqlBoolSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlBoolSerializer.Deserialize(serializedBytes)); case TypeMarker.Double: - jsonNode[key] = SqlDoubleSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlDoubleSerializer.Deserialize(serializedBytes)); case TypeMarker.Long: - jsonNode[key] = SqlLongSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlLongSerializer.Deserialize(serializedBytes)); case TypeMarker.String: - jsonNode[key] = SqlVarCharSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlVarCharSerializer.Deserialize(serializedBytes)); case TypeMarker.Array: - jsonNode[key] = JsonNode.Parse(serializedBytes); - break; + return JsonNode.Parse(serializedBytes); case TypeMarker.Object: - jsonNode[key] = JsonNode.Parse(serializedBytes); - break; + return JsonNode.Parse(serializedBytes); default: Debug.Fail($"Unexpected type marker {typeMarker}"); - break; + return null; } } - } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index bb1608a466..3b7054611e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -54,16 +54,16 @@ internal virtual (byte[], int) Encrypt(DataEncryptionKey encryptionKey, TypeMark return (encryptedText, encryptedTextLength); } - internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextLength, ArrayPoolManager arrayPoolManager) { - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextLength - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); int decryptedLength = encryptionKey.DecryptData( cipherText, cipherTextOffset: 1, - cipherTextLength: cipherText.Length - 1, + cipherTextLength: cipherTextLength - 1, plainText, outputOffset: 0); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 189bd34e42..15260aa4dc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -135,7 +135,7 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 188bcbff21..02fc91466e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -134,19 +134,18 @@ internal async Task DecryptObjectAsync( } // can we get to internal JsonNode buffers to avoid string allocation here? - byte[] cipherTextWithTypeMarker = Convert.FromBase64String(propertyValue.GetValue()); - if (cipherTextWithTypeMarker == null) + string base64String = propertyValue.GetValue(); + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent((base64String.Length * sizeof(char) * 3 / 4) + 4); + if (!Convert.TryFromBase64Chars(base64String, cipherTextWithTypeMarker, out int cipherTextLength)) { continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); - this.Serializer.DeserializeAndAddProperty( + document[propertyName] = this.Serializer.Deserialize( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), - document, - propertyName, charPoolManager); pathsDecrypted.Add(path); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index b2bcc93f17..e59c09ed0a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,8 +1,8 @@ ``` ini -BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.26100.2033) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.400 +.NET SDK=8.0.403 [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,15 +11,15 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **Newtonsoft** | **23.24 μs** | **0.606 μs** | **0.888 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | -| Decrypt | 1 | Newtonsoft | 24.95 μs | 0.230 μs | 0.344 μs | 0.1526 | 0.0305 | - | 39.27 KB | -| **Encrypt** | **1** | **SystemTextJson** | **14.46 μs** | **0.165 μs** | **0.242 μs** | **0.0916** | **0.0153** | **-** | **22.48 KB** | -| Decrypt | 1 | SystemTextJson | 16.05 μs | 0.284 μs | 0.416 μs | 0.0916 | 0.0305 | - | 22.1 KB | -| **Encrypt** | **10** | **Newtonsoft** | **84.60 μs** | **0.908 μs** | **1.359 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | -| Decrypt | 10 | Newtonsoft | 98.39 μs | 0.856 μs | 1.255 μs | 0.6104 | 0.1221 | - | 152.53 KB | -| **Encrypt** | **10** | **SystemTextJson** | **41.72 μs** | **0.341 μs** | **0.501 μs** | **0.4272** | **0.0610** | **-** | **102.99 KB** | -| Decrypt | 10 | SystemTextJson | 46.91 μs | 0.430 μs | 0.630 μs | 0.4272 | 0.0610 | - | 105.06 KB | -| **Encrypt** | **100** | **Newtonsoft** | **1,072.91 μs** | **13.802 μs** | **20.231 μs** | **25.3906** | **21.4844** | **21.4844** | **1638.34 KB** | -| Decrypt | 100 | Newtonsoft | 1,107.93 μs | 12.955 μs | 18.990 μs | 17.5781 | 15.6250 | 15.6250 | 1229.52 KB | -| **Encrypt** | **100** | **SystemTextJson** | **794.00 μs** | **25.274 μs** | **37.047 μs** | **24.4141** | **24.4141** | **24.4141** | **942.73 KB** | -| Decrypt | 100 | SystemTextJson | 819.31 μs | 14.159 μs | 20.754 μs | 22.4609 | 22.4609 | 22.4609 | 1037.04 KB | +| **Encrypt** | **1** | **Newtonsoft** | **22.23 μs** | **0.392 μs** | **0.562 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | +| Decrypt | 1 | Newtonsoft | 25.65 μs | 0.482 μs | 0.691 μs | 0.1221 | - | - | 39.27 KB | +| **Encrypt** | **1** | **SystemTextJson** | **15.61 μs** | **0.886 μs** | **1.242 μs** | **0.0916** | **0.0305** | **-** | **22.48 KB** | +| Decrypt | 1 | SystemTextJson | 14.68 μs | 0.334 μs | 0.479 μs | 0.0763 | 0.0153 | - | 20.83 KB | +| **Encrypt** | **10** | **Newtonsoft** | **83.23 μs** | **1.608 μs** | **2.147 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | +| Decrypt | 10 | Newtonsoft | 101.62 μs | 1.638 μs | 2.349 μs | 0.6104 | 0.1221 | - | 152.53 KB | +| **Encrypt** | **10** | **SystemTextJson** | **41.49 μs** | **0.317 μs** | **0.464 μs** | **0.4272** | **0.0610** | **-** | **102.99 KB** | +| Decrypt | 10 | SystemTextJson | 41.53 μs | 0.505 μs | 0.725 μs | 0.3662 | 0.0610 | - | 94.09 KB | +| **Encrypt** | **100** | **Newtonsoft** | **1,081.23 μs** | **13.538 μs** | **18.978 μs** | **25.3906** | **23.4375** | **21.4844** | **1638.32 KB** | +| Decrypt | 100 | Newtonsoft | 1,135.00 μs | 32.719 μs | 45.867 μs | 17.5781 | 15.6250 | 15.6250 | 1229.52 KB | +| **Encrypt** | **100** | **SystemTextJson** | **819.27 μs** | **22.564 μs** | **33.074 μs** | **25.3906** | **25.3906** | **25.3906** | **942.76 KB** | +| Decrypt | 100 | SystemTextJson | 698.30 μs | 20.402 μs | 29.905 μs | 21.4844 | 21.4844 | 21.4844 | 927.92 KB | From 38cc928b7bde9298688a532f2ac231be2e751e06 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 17:03:27 +0200 Subject: [PATCH 49/85] - remove explicit System.Text.Json 8.0.5 --- .../src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index da7c2053e9..cca4d719bf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -38,11 +38,11 @@ - + - + @@ -57,10 +57,6 @@ - - - - From 2af9ddb0e3e4bbd7930efd199d4e6ac819feaced Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 19:37:29 +0200 Subject: [PATCH 50/85] ~ propagate changes from master --- .../src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs | 2 +- .../Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index d606e2fc28..110cf35eaa 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if IS_PREVIEW +#if ENCRYPTION_CUSTOM_PREVIEW namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 45a886d3ba..365fb7de9e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if IS_PREVIEW && NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { From 54b0ad9b1b07c085ff5cd5dd90b398359969e9b0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 19:48:53 +0200 Subject: [PATCH 51/85] ~ fixes for preview x non-preview branching ~ cleanup ~ merge master followup fixes --- .../src/EncryptionProcessor.cs | 11 +++++++---- .../Transformation/MdeEncryptionProcessor.Stable.cs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index c2c6abd498..843eb5b3d7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,8 +10,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; - using System.Text.Json; #if NET8_0_OR_GREATER + using System.Text.Json; using System.Text.Json.Nodes; #endif using System.Threading; @@ -31,7 +31,10 @@ internal static class EncryptionProcessor }; internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + +#if NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; +#endif private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -135,14 +138,14 @@ public static async Task EncryptAsync( return jsonProcessor switch { JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") }; } -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( Stream input, Encryptor encryptor, @@ -195,7 +198,7 @@ public static async Task EncryptAsync( return (document, decryptionContext); } -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(JsonNode, DecryptionContext)> DecryptAsync( JsonNode document, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 1f9b628617..659ca74c91 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -119,7 +119,7 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], From ad15ceb6894977b3e6ab9f86b7725b1d304a7acd Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 11 Oct 2024 11:00:52 +0200 Subject: [PATCH 52/85] + tests ! no encryption info scenario behaves same on all paths ~ cleanup --- .../src/EncryptionProcessor.cs | 12 +- .../JsonNodeSqlSerializer.Preview.cs | 3 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 9 +- .../MdeEncryptionProcessorTests.cs | 255 +++++++++++++++++- .../JsonNodeSqlSerializerTests.cs | 79 ++++++ 5 files changed, 336 insertions(+), 22 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 843eb5b3d7..fd52424331 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using System.Text.Json; using System.Text.Json.Nodes; #endif @@ -32,7 +32,7 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; #endif @@ -164,6 +164,12 @@ public static async Task EncryptAsync( JsonNode document = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken); (JsonNode decryptedDocument, DecryptionContext context) = await DecryptAsync(document, encryptor, diagnosticsContext, cancellationToken); + if (context == null) + { + input.Position = 0; + return (input, null); + } + await input.DisposeAsync(); MemoryStream ms = new (); @@ -211,7 +217,7 @@ public static async Task EncryptAsync( if (!document.AsObject().TryGetPropertyValue(Constants.EncryptedInfo, out JsonNode encryptionPropertiesNode)) { - throw new InvalidOperationException("Encryption properties deserialization failed."); + return (document, null); } EncryptionProperties encryptionProperties; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs index 0b98d254b8..c96b76a731 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -91,8 +91,7 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB internal virtual JsonNode Deserialize( TypeMarker typeMarker, - ReadOnlySpan serializedBytes, - ArrayPoolManager arrayPoolManager) + ReadOnlySpan serializedBytes) { switch (typeMarker) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index e46f7c05e7..f90d9938f9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -10,7 +10,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Collections.Generic; using System.IO; using System.Linq; - using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; @@ -18,12 +17,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class MdeJsonNodeEncryptionProcessor { + private readonly JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; + internal JsonNodeSqlSerializer Serializer { get; set; } = new JsonNodeSqlSerializer(); internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - private JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; - public async Task EncryptAsync( Stream input, Encryptor encryptor, @@ -115,7 +114,6 @@ internal async Task DecryptObjectAsync( } using ArrayPoolManager arrayPoolManager = new (); - using ArrayPoolManager charPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); @@ -145,8 +143,7 @@ internal async Task DecryptObjectAsync( document[propertyName] = this.Serializer.Deserialize( (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - charPoolManager); + plainText.AsSpan(0, decryptedCount)); pathsDecrypted.Add(path); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 35722f40eb..9e20bd249c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -8,6 +8,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using System.Collections.Generic; using System.IO; using System.Linq; +#if NET8_0_OR_GREATER + using System.Text.Json.Nodes; +#endif using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom; @@ -55,14 +58,19 @@ public static void ClassInitialize(TestContext testContext) } [TestMethod] - public async Task InvalidPathToEncrypt() + [DataRow(JsonProcessor.Newtonsoft)] +#if NET8_0_OR_GREATER + [DataRow(JsonProcessor.SystemTextJson)] +#endif + public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) { TestDoc testDoc = TestDoc.Create(); EncryptionOptions encryptionOptionsWithInvalidPathToEncrypt = new() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = new List() { "/SensitiveStr", "/Invalid" } + PathsToEncrypt = new List() { "/SensitiveStr", "/Invalid" }, + JsonProcessor = jsonProcessor, }; Stream encryptedStream = await EncryptionProcessor.EncryptAsync( @@ -90,14 +98,19 @@ public async Task InvalidPathToEncrypt() } [TestMethod] - public async Task DuplicatePathToEncrypt() + [DataRow(JsonProcessor.Newtonsoft)] +#if NET8_0_OR_GREATER + [DataRow(JsonProcessor.SystemTextJson)] +#endif + public async Task DuplicatePathToEncrypt(JsonProcessor jsonProcessor) { TestDoc testDoc = TestDoc.Create(); EncryptionOptions encryptionOptionsWithDuplicatePathToEncrypt = new() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = new List() { "/SensitiveStr", "/SensitiveStr" } + PathsToEncrypt = new List() { "/SensitiveStr", "/SensitiveStr" }, + JsonProcessor = jsonProcessor, }; try @@ -119,12 +132,12 @@ await EncryptionProcessor.EncryptAsync( [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions) + public async Task EncryptDecryptPropertyWithNullValue_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); + JObject encryptedDoc = await VerifyEncryptionSucceededNewtonsoft(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -139,13 +152,37 @@ public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encrypti decryptionContext); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [TestMethod] + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task EncryptDecryptPropertyWithNullValue_VerifyBySystemText(EncryptionOptions encryptionOptions) + { + TestDoc testDoc = TestDoc.Create(); + testDoc.SensitiveStr = null; + + JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); + + (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedDoc, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } +#endif + [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions) + public async Task ValidateEncryptDecryptDocument_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); + JObject encryptedDoc = await VerifyEncryptionSucceededNewtonsoft(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -160,9 +197,60 @@ public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOpt decryptionContext); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [TestMethod] + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateEncryptDecryptDocument_VerifyBySystemText(EncryptionOptions encryptionOptions) + { + TestDoc testDoc = TestDoc.Create(); + + JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); + + (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedDoc, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } +#endif + [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) + public async Task ValidateDecryptByNewtonsoftStream_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) + { + TestDoc testDoc = TestDoc.Create(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + testDoc.ToStream(), + mockEncryptor.Object, + encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + (Stream decryptedStream, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedStream, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + JsonProcessor.Newtonsoft, + CancellationToken.None); + + JObject decryptedDoc = EncryptionProcessor.BaseSerializer.FromStream(decryptedStream); + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } + + [TestMethod] + [DynamicData(nameof(EncryptionOptionsStreamTestCombinations))] + public async Task ValidateDecryptBySystemTextStream_VerifyByNewtonsoft(EncryptionOptions encryptionOptions, JsonProcessor decryptionJsonProcessor) { TestDoc testDoc = TestDoc.Create(); @@ -177,6 +265,7 @@ public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) encryptedStream, mockEncryptor.Object, new CosmosDiagnosticsContext(), + decryptionJsonProcessor, CancellationToken.None); JObject decryptedDoc = EncryptionProcessor.BaseSerializer.FromStream(decryptedStream); @@ -187,8 +276,42 @@ public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) decryptionContext); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [TestMethod] + [DynamicData(nameof(EncryptionOptionsStreamTestCombinations))] + public async Task ValidateDecryptBySystemTextStream_VerifyBySystemText(EncryptionOptions encryptionOptions, JsonProcessor decryptionJsonProcessor) + { + TestDoc testDoc = TestDoc.Create(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + testDoc.ToStream(), + mockEncryptor.Object, + encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + (Stream decryptedStream, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedStream, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + decryptionJsonProcessor, + CancellationToken.None); + + JsonNode decryptedDoc = JsonNode.Parse(decryptedStream); + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } +#endif + [TestMethod] - public async Task DecryptStreamWithoutEncryptedProperty() + [DataRow(JsonProcessor.Newtonsoft)] +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [DataRow(JsonProcessor.SystemTextJson)] +#endif + public async Task DecryptStreamWithoutEncryptedProperty(JsonProcessor processor) { TestDoc testDoc = TestDoc.Create(); Stream docStream = testDoc.ToStream(); @@ -197,6 +320,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() docStream, mockEncryptor.Object, new CosmosDiagnosticsContext(), + processor, CancellationToken.None); Assert.IsTrue(decryptedStream.CanSeek); @@ -205,7 +329,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() Assert.IsNull(decryptionContext); } - private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions) + private static async Task VerifyEncryptionSucceededNewtonsoft(TestDoc testDoc, EncryptionOptions encryptionOptions) { Stream encryptedStream = await EncryptionProcessor.EncryptAsync( testDoc.ToStream(), @@ -249,6 +373,51 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, En return encryptedDoc; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private static async Task VerifyEncryptionSucceededSystemText(TestDoc testDoc, EncryptionOptions encryptionOptions) + { + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + testDoc.ToStream(), + mockEncryptor.Object, + encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + JsonNode encryptedDoc = JsonNode.Parse(encryptedStream, documentOptions: new System.Text.Json.JsonDocumentOptions() { }); + + Assert.AreEqual(testDoc.Id, encryptedDoc["id"].GetValue()); + Assert.AreEqual(testDoc.PK, encryptedDoc[nameof(TestDoc.PK)].GetValue()); + Assert.AreEqual(testDoc.NonSensitive, encryptedDoc[nameof(TestDoc.NonSensitive)].GetValue()); + Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); + Assert.AreNotEqual(testDoc.SensitiveInt, encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); // not equal since value is encrypted + + JsonNode eiJProp = encryptedDoc[Constants.EncryptedInfo]; + Assert.IsNotNull(eiJProp); + Assert.IsNotNull(eiJProp.AsObject()); + EncryptionProperties encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(eiJProp); + + Assert.IsNotNull(encryptionProperties); + Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); + Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion); + Assert.IsNull(encryptionProperties.EncryptedData); + Assert.IsNotNull(encryptionProperties.EncryptedPaths); + + if (testDoc.SensitiveStr == null) + { + AssertNullableValueKind(null, encryptedDoc, nameof(TestDoc.SensitiveStr)); // since null value is not encrypted + Assert.AreEqual(TestDoc.PathsToEncrypt.Count - 1, encryptionProperties.EncryptedPaths.Count()); + } + else + { + Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); + Assert.AreNotEqual(testDoc.SensitiveStr, encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); // not equal since value is encrypted + Assert.AreEqual(TestDoc.PathsToEncrypt.Count, encryptionProperties.EncryptedPaths.Count()); + } + + return encryptedDoc; + } +#endif + private static void VerifyDecryptionSucceeded( JObject decryptedDoc, TestDoc expectedDoc, @@ -284,6 +453,56 @@ private static void VerifyDecryptionSucceeded( } } +#if NET8_0_OR_GREATER + private static void VerifyDecryptionSucceeded( + JsonNode decryptedDoc, + TestDoc expectedDoc, + int pathCount, + DecryptionContext decryptionContext, + bool invalidPathsConfigured = false) + { + AssertNullableValueKind(expectedDoc.SensitiveStr, decryptedDoc, nameof(TestDoc.SensitiveStr)); + Assert.AreEqual(expectedDoc.SensitiveInt, decryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); + Assert.IsNull(decryptedDoc[Constants.EncryptedInfo]); + + Assert.IsNotNull(decryptionContext); + Assert.IsNotNull(decryptionContext.DecryptionInfoList); + DecryptionInfo decryptionInfo = decryptionContext.DecryptionInfoList[0]; + Assert.AreEqual(dekId, decryptionInfo.DataEncryptionKeyId); + if (expectedDoc.SensitiveStr == null) + { + Assert.AreEqual(pathCount - 1, decryptionInfo.PathsDecrypted.Count); + Assert.IsTrue(TestDoc.PathsToEncrypt.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + else + { + Assert.AreEqual(pathCount, decryptionInfo.PathsDecrypted.Count); + + if (!invalidPathsConfigured) + { + Assert.IsFalse(TestDoc.PathsToEncrypt.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + else + { + Assert.IsTrue(TestDoc.PathsToEncrypt.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + } + } + + private static void AssertNullableValueKind(T expectedValue, JsonNode node, string propertyName) where T : class + { + if (expectedValue == null) + { + Assert.IsTrue(node.AsObject().ContainsKey(propertyName)); + Assert.AreEqual(null, node[propertyName]); + } + else + { + Assert.AreEqual(expectedValue, node[propertyName].GetValue()); + } + } +#endif + public static IEnumerable EncryptionOptionsCombinations => new[] { new object[] { new EncryptionOptions() { @@ -304,5 +523,19 @@ private static void VerifyDecryptionSucceeded( }, #endif }; + + public static IEnumerable EncryptionOptionsStreamTestCombinations + { + get + { + foreach (object[] encryptionOptions in EncryptionOptionsCombinations) + { + yield return new object[] { encryptionOptions[0], JsonProcessor.Newtonsoft }; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + yield return new object[] { encryptionOptions[0], JsonProcessor.SystemTextJson }; +#endif + } + } + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs index deb6bb7b35..e28bd409a1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation using System; using System.Collections.Generic; using System.Linq; + using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Azure.Cosmos.Encryption.Custom; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; @@ -43,6 +44,79 @@ public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[ } } + [TestMethod] + [DynamicData(nameof(DeserializationSamples))] + public void Deserialize_SupportedValue(byte typeMarkerByte, byte[] serializedBytes, JsonNode expectedNode) + { + JsonNodeSqlSerializer serializer = new(); + TypeMarker typeMarker = (TypeMarker)typeMarkerByte; + JsonNode deserializedNode = serializer.Deserialize(typeMarker, serializedBytes); + + if ((expectedNode as JsonValue) != null) + { + AssertValueNodeEquality(expectedNode, deserializedNode); + return; + } + + if ((expectedNode as JsonArray) != null) + { + Assert.IsNotNull(deserializedNode as JsonArray); + + JsonArray expectedArray = expectedNode.AsArray(); + JsonArray deserializedArray = deserializedNode.AsArray(); + + Assert.AreEqual(expectedArray.Count, deserializedArray.Count); + + for (int i = 0; i < deserializedNode.AsArray().Count; i++) + { + AssertValueNodeEquality(expectedArray[i], deserializedArray[i]); + } + return; + } + + if ((expectedNode as JsonObject) != null) + { + Assert.IsNotNull(deserializedNode as JsonObject); + + JsonObject expectedObject = expectedNode.AsObject(); + JsonObject deserializedObject = deserializedNode.AsObject(); + + Assert.AreEqual(expectedObject.Count, deserializedObject.Count); + + foreach (KeyValuePair expected in expectedObject) + { + Assert.IsTrue(deserializedObject.ContainsKey(expected.Key)); + AssertValueNodeEquality(expected.Value, deserializedObject[expected.Key]); + } + return; + } + + Assert.Fail("Attempt to validate unsupported JsonNode type"); + } + + private static void AssertValueNodeEquality(JsonNode expectedNode, JsonNode actualNode) + { + JsonValue expectedValueNode = expectedNode.AsValue(); + JsonValue actualValueNode = actualNode.AsValue(); + + Assert.AreEqual(expectedValueNode.GetValueKind(), actualValueNode.GetValueKind()); + Assert.AreEqual(expectedValueNode.ToString(), actualValueNode.ToString()); + } + + public static IEnumerable DeserializationSamples + { + get + { + yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), JsonValue.Create(true) }; + yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), JsonValue.Create(false) }; + yield return new object[] { (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), JsonValue.Create(192) }; + yield return new object[] { (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), JsonValue.Create(192.5) }; + yield return new object[] { (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), JsonValue.Create(testString) }; + yield return new object[] { (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), JsonNode.Parse("[10,18,19]") }; + yield return new object[] { (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), JsonNode.Parse(testClass.ToJson()) }; + } + } + public static IEnumerable SerializationSamples { get @@ -71,6 +145,11 @@ private class TestClass { public int SomeInt { get; set; } public string SomeString { get; set; } + + public string ToJson() + { + return JsonSerializer.Serialize(this); + } } private static byte[] GetNewtonsoftValueEquivalent(T value) From d9c315d8f76ccb528a9ced24857d7adf3ef31da6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 13 Oct 2024 17:27:35 +0200 Subject: [PATCH 53/85] ~ complete merge from master & update benchmark --- .../MdeJObjectEncryptionProcessor.Preview.cs | 82 +++++++++++++++---- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 33 ++++++-- .../Readme.md | 34 ++++---- .../MdeEncryptionProcessorTests.cs | 40 ++++++--- 4 files changed, 144 insertions(+), 45 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 110cf35eaa..4d70ec8a2c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -52,6 +52,14 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + +#if NET8_0_OR_GREATER + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; +#endif + Dictionary compressedPaths = new (); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { #if NET8_0_OR_GREATER @@ -69,26 +77,41 @@ public async Task EncryptAsync( continue; } - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + byte[] processedBytes = null; + (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - if (plainText == null) + if (processedBytes == null) { continue; } - byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); +#if NET8_0_OR_GREATER + if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); + processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytes = compressedBytes; + } +#endif + + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength); input[propertyName] = encryptedBytes; + pathsEncrypted.Add(pathToEncrypt); } +#if NET8_0_OR_GREATER + compressor?.Dispose(); +#endif EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, + encryptionFormatVersion: compressionEnabled ? 4 : 3, encryptionOptions.EncryptionAlgorithm, encryptionOptions.DataEncryptionKeyId, encryptedData: null, - pathsEncrypted); + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); input.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); @@ -96,15 +119,15 @@ public async Task EncryptAsync( } internal async Task DecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3) + if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -115,6 +138,24 @@ internal async Task DecryptObjectAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + +#if NET8_0_OR_GREATER + BrotliCompressor decompressor = null; + if (encryptionProperties.EncryptionFormatVersion == 4) + { + bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; + if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) + { + throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}"); + } + + if (containsCompressed) + { + decompressor = new (); + } + } +#endif + foreach (string path in encryptionProperties.EncryptedPaths) { #if NET8_0_OR_GREATER @@ -135,11 +176,24 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + +#if NET8_0_OR_GREATER + if (decompressor != null) + { + if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) + { + byte[] buffer = arrayPoolManager.Rent(decompressedSize); + processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); + + bytes = buffer; + } + } +#endif this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), + bytes.AsSpan(0, processedBytes), document, propertyName, charPoolManager); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 365fb7de9e..63b2428832 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -59,6 +59,14 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + +#if NET8_0_OR_GREATER + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; +#endif + Dictionary compressedPaths = new (); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { #if NET8_0_OR_GREATER @@ -76,26 +84,39 @@ public async Task EncryptAsync( continue; } - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + byte[] processedBytes = null; + (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - if (plainText == null) + if (processedBytes == null) { continue; } - (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); +#if NET8_0_OR_GREATER + if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); + processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytes = compressedBytes; + } +#endif + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); itemObj[propertyName] = JsonValue.Create(new JsonBytes(encryptedBytes, 0, encryptedBytesCount)); pathsEncrypted.Add(pathToEncrypt); } +#if NET8_0_OR_GREATER + compressor?.Dispose(); +#endif EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, + encryptionFormatVersion: compressionEnabled ? 4 : 3, encryptionOptions.EncryptionAlgorithm, encryptionOptions.DataEncryptionKeyId, encryptedData: null, - pathsEncrypted); + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); JsonNode propertiesNode = JsonSerializer.SerializeToNode(encryptionProperties); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2913528571..b7110fd6eb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,17 +9,23 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **None** | **22.45 μs** | **0.447 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **40.94 KB** | -| Decrypt | 1 | None | 26.65 μs | 0.165 μs | 0.247 μs | 0.1526 | 0.0305 | - | 40.32 KB | -| **Encrypt** | **1** | **Brotli** | **27.92 μs** | **0.212 μs** | **0.317 μs** | **0.1526** | **0.0305** | **-** | **37.3 KB** | -| Decrypt | 1 | Brotli | 33.45 μs | 0.718 μs | 1.075 μs | 0.1221 | - | - | 39.95 KB | -| **Encrypt** | **10** | **None** | **83.56 μs** | **0.358 μs** | **0.502 μs** | **0.6104** | **0.1221** | **-** | **167.12 KB** | -| Decrypt | 10 | None | 100.88 μs | 0.437 μs | 0.627 μs | 0.6104 | 0.1221 | - | 153.59 KB | -| **Encrypt** | **10** | **Brotli** | **111.94 μs** | **0.456 μs** | **0.669 μs** | **0.6104** | **0.1221** | **-** | **164.26 KB** | -| Decrypt | 10 | Brotli | 121.06 μs | 2.794 μs | 4.182 μs | 0.4883 | - | - | 141.31 KB | -| **Encrypt** | **100** | **None** | **1,194.10 μs** | **37.744 μs** | **56.494 μs** | **23.4375** | **23.4375** | **21.4844** | **1638.78 KB** | -| Decrypt | 100 | None | 1,247.89 μs | 32.037 μs | 47.952 μs | 17.5781 | 15.6250 | 15.6250 | 1230.56 KB | -| **Encrypt** | **100** | **Brotli** | **1,199.53 μs** | **30.018 μs** | **44.930 μs** | **13.6719** | **11.7188** | **9.7656** | **1347 KB** | -| Decrypt | 100 | Brotli | 1,196.64 μs | 22.702 μs | 33.979 μs | 11.7188 | 9.7656 | 9.7656 | 1097.75 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.40 μs** | **0.387 μs** | **0.580 μs** | **0.1526** | **0.0305** | **-** | **41.08 KB** | +| Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.215 μs | 0.322 μs | 0.1526 | 0.0305 | - | 40.39 KB | +| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.84 μs** | **0.245 μs** | **0.358 μs** | **0.0916** | **0.0305** | **-** | **22.75 KB** | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **27.89 μs** | **0.220 μs** | **0.329 μs** | **0.1526** | **0.0305** | **-** | **37.45 KB** | +| Decrypt | 1 | Brotli | Newtonsoft | 33.39 μs | 0.612 μs | 0.878 μs | 0.1221 | - | - | 40.02 KB | +| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **21.43 μs** | **0.168 μs** | **0.251 μs** | **0.0916** | **0.0305** | **-** | **21.82 KB** | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **82.79 μs** | **0.439 μs** | **0.643 μs** | **0.6104** | **0.1221** | **-** | **167.26 KB** | +| Decrypt | 10 | None | Newtonsoft | 98.98 μs | 0.499 μs | 0.747 μs | 0.6104 | 0.1221 | - | 153.66 KB | +| **Encrypt** | **10** | **None** | **SystemTextJson** | **40.74 μs** | **0.214 μs** | **0.321 μs** | **0.4272** | **0.0610** | **-** | **103.26 KB** | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **112.08 μs** | **1.172 μs** | **1.681 μs** | **0.6104** | **0.1221** | **-** | **164.4 KB** | +| Decrypt | 10 | Brotli | Newtonsoft | 117.21 μs | 0.920 μs | 1.349 μs | 0.4883 | - | - | 141.38 KB | +| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **69.51 μs** | **0.491 μs** | **0.719 μs** | **0.2441** | **-** | **-** | **84.58 KB** | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,165.87 μs** | **41.512 μs** | **62.133 μs** | **23.4375** | **21.4844** | **19.5313** | **1638.94 KB** | +| Decrypt | 100 | None | Newtonsoft | 1,166.04 μs | 32.206 μs | 48.204 μs | 17.5781 | 15.6250 | 15.6250 | 1230.62 KB | +| **Encrypt** | **100** | **None** | **SystemTextJson** | **854.64 μs** | **35.123 μs** | **51.482 μs** | **21.4844** | **21.4844** | **21.4844** | **942.96 KB** | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,121.21 μs** | **24.814 μs** | **37.141 μs** | **13.6719** | **11.7188** | **9.7656** | **1347.12 KB** | +| Decrypt | 100 | Brotli | Newtonsoft | 1,135.33 μs | 11.013 μs | 16.483 μs | 11.7188 | 9.7656 | 9.7656 | 1097.84 KB | +| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **986.73 μs** | **19.142 μs** | **28.058 μs** | **21.4844** | **21.4844** | **21.4844** | **749.06 KB** | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 59a5a215f1..4a28e93aac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -295,6 +295,7 @@ private static void VerifyDecryptionSucceeded( DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft, CompressionOptions = new CompressionOptions() { Algorithm = CompressionOptions.CompressionAlgorithm.None @@ -307,6 +308,19 @@ private static void VerifyDecryptionSucceeded( DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.SystemTextJson, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.None + } + } + }, + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft, CompressionOptions = new CompressionOptions() { Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, @@ -319,36 +333,40 @@ private static void VerifyDecryptionSucceeded( DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft, CompressionOptions = new CompressionOptions() { Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, } } - } -#endif - }; - } - } - - public static IEnumerable EncryptionOptionsCombinations => new[] { + }, new object[] { new EncryptionOptions() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft + JsonProcessor = JsonProcessor.SystemTextJson, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.Fastest + } } }, -#if NET8_0_OR_GREATER new object[] { new EncryptionOptions() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson + JsonProcessor = JsonProcessor.SystemTextJson, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, + } } - }, + } #endif }; } From 996252bc9e9ca7a1a431e496e498cefc59034b2d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 11:20:10 +0200 Subject: [PATCH 54/85] + initial --- .../src/EncryptionOptions.cs | 6 + .../src/EncryptionProcessor.cs | 38 +++ .../src/EncryptionPropertiesWrapper.cs | 19 ++ .../MdeEncryptionProcessor.Preview.cs | 1 + .../src/Transformation/StreamProcessor.cs | 236 ++++++++++++++++++ .../EncryptionBenchmark.cs | 5 +- 6 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index 0559e988c3..aadb2dbbd3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -22,6 +22,12 @@ public enum JsonProcessor /// /// Available with .NET8.0 package only. SystemTextJson, + + /// + /// Ut8JsonReader/Writer + /// + /// Available with .NET8.0 package only. + Stream, #endif } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 6f9d87f064..8922432a34 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -34,6 +34,7 @@ internal static class EncryptionProcessor #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; + private static readonly StreamProcessor StreamProcessor = new (); #endif private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -140,6 +141,7 @@ public static async Task EncryptAsync( JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), + JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") }; @@ -182,6 +184,42 @@ public static async Task EncryptAsync( } #endif +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( + Stream input, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (input == null) + { + return (input, null); + } + + Debug.Assert(input.CanSeek); + Debug.Assert(encryptor != null); + Debug.Assert(diagnosticsContext != null); + input.Position = 0; + + EncryptionPropertiesWrapper properties = System.Text.Json.JsonSerializer.Deserialize(input); + input.Position = 0; + if (properties?.EncryptionProperties == null) + { + return (input, null); + } + + (Stream decryptedDocument, DecryptionContext context) = await StreamProcessor.DecryptStreamAsync(input, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + if (context == null) + { + input.Position = 0; + return (input, null); + } + + await input.DisposeAsync(); + return (decryptedDocument, context); + } +#endif + public static async Task<(JObject, DecryptionContext)> DecryptAsync( JObject document, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs new file mode 100644 index 0000000000..4c75c16032 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.Text.Json.Serialization; + + internal class EncryptionPropertiesWrapper + { + [JsonPropertyName(Constants.EncryptedInfo)] + public EncryptionProperties EncryptionProperties { get; } + + public EncryptionPropertiesWrapper(EncryptionProperties properties) + { + this.EncryptionProperties = properties; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 635901552d..8649b36079 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -34,6 +34,7 @@ public async Task EncryptAsync( JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #if NET8_0_OR_GREATER JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), + JsonProcessor.Stream => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #endif _ => throw new InvalidOperationException("Unsupported JsonProcessor") }; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs new file mode 100644 index 0000000000..c2ae7040c2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -0,0 +1,236 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class StreamProcessor + { + private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + internal async Task<(Stream, DecryptionContext)> DecryptStreamAsync( + Stream inputStream, + Encryptor encryptor, + EncryptionProperties properties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (properties.EncryptionFormatVersion != 3 && properties.EncryptionFormatVersion != 4) + { + throw new NotSupportedException($"Unknown encryption format version: {properties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(properties.DataEncryptionKeyId, properties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (properties.EncryptedPaths.Count()); + + MemoryStream outputStream = new (); + Utf8JsonWriter writer = new (outputStream); + + // we determine initial buffer size based on max uncompressed path length, it still might need scale out in case there is large non-encrypted object, but likehood is rather low + int bufferSize = 16384; // Math.Max(16384, properties.CompressedEncryptedPaths.Values.Max()); + byte[] buffer = arrayPoolManager.Rent(bufferSize); + + JsonReaderState state = new (this.jsonReaderOptions); + + int leftOver = 0; + + bool isFinalBlock = false; + + while (!isFinalBlock) + { + int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + int dataSize = dataLength + leftOver; + isFinalBlock = dataSize == 0; + long bytesConsumed = 0; + + // processing itself here + bytesConsumed = this.TransformReadBuffer( + buffer.AsSpan(0, dataSize), + isFinalBlock, + writer, + ref state, + pathsDecrypted, + properties, + arrayPoolManager, + encryptionKey); + + leftOver = dataSize - (int)bytesConsumed; + + // we need to scale out buffer + if (leftOver == dataSize) + { + bufferSize *= 2; + byte[] newBuffer = arrayPoolManager.Rent(bufferSize); + buffer.AsSpan(0, leftOver).CopyTo(newBuffer); + buffer = newBuffer; + } + else if (leftOver != 0) + { + buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + } + } + + writer.Flush(); + inputStream.Position = 0; + outputStream.Position = 0; + + return ( + outputStream, + EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId)); + } + + /* + private static Dictionary GetUtf8DecryptionList(EncryptionProperties properties) + { + Dictionary output = new (properties.EncryptedPaths.Count()); + foreach (KeyValuePair compressedPath in properties.CompressedEncryptedPaths) + { + byte[] utf8String = Encoding.UTF8.GetBytes(compressedPath.Key, 1, compressedPath.Key.Length - 1); + output[utf8String] = compressedPath.Value; + } + + foreach (string encryptedPath in properties.EncryptedPaths) + { + byte[] utf8String = Encoding.UTF8.GetBytes(encryptedPath, 1, encryptedPath.Length - 1); + output.TryAdd(utf8String, -1); + } + + return output; + }*/ + + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + { + Utf8JsonReader json = new (buffer, isFinalBlock, state); + + string decryptPropertyName = null; + + while (json.Read()) + { + JsonTokenType tokenType = json.TokenType; + + switch (tokenType) + { + case JsonTokenType.String: + if (decryptPropertyName == null) + { + writer.WriteRawValue(json.ValueSpan); + } + else + { + this.TransformDecryptProperty( + json.GetBytesFromBase64(), + writer, + decryptPropertyName, + properties, + encryptionKey, + arrayPoolManager); + + pathsDecrypted.Add("/" + decryptPropertyName); + } + + decryptPropertyName = null; + break; + case JsonTokenType.Number: + decryptPropertyName = null; + writer.WriteRawValue(json.ValueSpan); + break; + case JsonTokenType.None: + decryptPropertyName = null; + break; + case JsonTokenType.StartObject: + decryptPropertyName = null; + writer.WriteStartObject(); + break; + case JsonTokenType.EndObject: + decryptPropertyName = null; + writer.WriteEndObject(); + break; + case JsonTokenType.StartArray: + decryptPropertyName = null; + writer.WriteStartArray(); + break; + case JsonTokenType.EndArray: + decryptPropertyName = null; + writer.WriteEndArray(); + break; + case JsonTokenType.PropertyName: + string propertyName = json.GetString(); + if (properties.EncryptedPaths.Contains("/" + propertyName)) + { + decryptPropertyName = propertyName; + } + + writer.WritePropertyName(json.ValueSpan); + break; + case JsonTokenType.Comment: + break; + case JsonTokenType.True: + decryptPropertyName = null; + writer.WriteBooleanValue(true); + break; + case JsonTokenType.False: + decryptPropertyName = null; + writer.WriteBooleanValue(false); + break; + case JsonTokenType.Null: + decryptPropertyName = null; + writer.WriteNullValue(); + break; + } + } + + state = json.CurrentState; + return json.BytesConsumed; + } + + private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) + { + BrotliCompressor decompressor = null; + if (properties.EncryptionFormatVersion == 4) + { + bool containsCompressed = properties.CompressedEncryptedPaths?.Any() == true; + if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) + { + throw new NotSupportedException($"Unknown compression algorithm {properties.CompressionAlgorithm}"); + } + + if (containsCompressed) + { + decompressor = new (); + } + } + + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); + + if (decompressor != null) + { + if (properties.CompressedEncryptedPaths?.TryGetValue(decryptPropertyName, out int decompressedSize) == true) + { + byte[] buffer = arrayPoolManager.Rent(decompressedSize); + processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); + + bytes = buffer; + } + } + + writer.WriteRawValue(bytes.AsSpan(0, processedBytes)); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 4bdcdc2941..051b87d69d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -28,7 +28,7 @@ public partial class EncryptionBenchmark [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } - [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson)] + [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson,*/ JsonProcessor.Stream)] public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -59,6 +59,7 @@ public async Task Setup() this.encryptedData = memoryStream.ToArray(); } + /* [Benchmark] public async Task Encrypt() { @@ -68,7 +69,7 @@ await EncryptionProcessor.EncryptAsync( this.encryptionOptions, new CosmosDiagnosticsContext(), CancellationToken.None); - } + }*/ [Benchmark] public async Task Decrypt() From 758e98d900bc3e005533c5e89f1e427355ce1c44 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 16:21:40 +0200 Subject: [PATCH 55/85] + tests, fixes and benchmark --- .../src/EncryptionProcessor.cs | 48 +++++++++- .../src/EncryptionPropertiesWrapper.cs | 4 +- .../src/Transformation/StreamProcessor.cs | 96 +++++++++++++++---- .../EncryptionBenchmark.cs | 17 +++- .../Readme.md | 40 +++----- .../MdeEncryptionProcessorTests.cs | 4 + 6 files changed, 157 insertions(+), 52 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 8922432a34..59a43f2206 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -147,6 +147,45 @@ public static async Task EncryptAsync( }; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public static async Task DecryptAsync( + Stream input, + Stream output, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (input == null) + { + return null; + } + + Debug.Assert(input.CanSeek); + Debug.Assert(output.CanWrite); + Debug.Assert(output.CanSeek); + Debug.Assert(encryptor != null); + Debug.Assert(diagnosticsContext != null); + input.Position = 0; + + EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); + input.Position = 0; + if (properties?.EncryptionProperties == null) + { + return null; + } + + DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + if (context == null) + { + input.Position = 0; + return null; + } + + await input.DisposeAsync(); + return context; + } +#endif + #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( Stream input, @@ -201,14 +240,16 @@ public static async Task EncryptAsync( Debug.Assert(diagnosticsContext != null); input.Position = 0; - EncryptionPropertiesWrapper properties = System.Text.Json.JsonSerializer.Deserialize(input); + EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); input.Position = 0; if (properties?.EncryptionProperties == null) { return (input, null); } - (Stream decryptedDocument, DecryptionContext context) = await StreamProcessor.DecryptStreamAsync(input, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + MemoryStream ms = new MemoryStream(); + + DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, ms, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) { input.Position = 0; @@ -216,8 +257,9 @@ public static async Task EncryptAsync( } await input.DisposeAsync(); - return (decryptedDocument, context); + return (ms, context); } + #endif public static async Task<(JObject, DecryptionContext)> DecryptAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs index 4c75c16032..8c343a12de 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs @@ -11,9 +11,9 @@ internal class EncryptionPropertiesWrapper [JsonPropertyName(Constants.EncryptedInfo)] public EncryptionProperties EncryptionProperties { get; } - public EncryptionPropertiesWrapper(EncryptionProperties properties) + public EncryptionPropertiesWrapper(EncryptionProperties encryptionProperties) { - this.EncryptionProperties = properties; + this.EncryptionProperties = encryptionProperties; } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs index c2ae7040c2..b24b8d9d89 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -6,21 +6,30 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; + using System.Buffers; + using System.Buffers.Text; using System.Collections.Generic; using System.IO; using System.Linq; + using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; + using Microsoft.Data.Encryption.Cryptography.Serializers; internal class StreamProcessor { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - internal async Task<(Stream, DecryptionContext)> DecryptStreamAsync( + internal async Task DecryptStreamAsync( Stream inputStream, + Stream outputStream, Encryptor encryptor, EncryptionProperties properties, CosmosDiagnosticsContext diagnosticsContext, @@ -39,7 +48,6 @@ internal class StreamProcessor List pathsDecrypted = new (properties.EncryptedPaths.Count()); - MemoryStream outputStream = new (); Utf8JsonWriter writer = new (outputStream); // we determine initial buffer size based on max uncompressed path length, it still might need scale out in case there is large non-encrypted object, but likehood is rather low @@ -51,6 +59,7 @@ internal class StreamProcessor int leftOver = 0; bool isFinalBlock = false; + bool isIgnoredBlock = false; while (!isFinalBlock) { @@ -65,6 +74,7 @@ internal class StreamProcessor isFinalBlock, writer, ref state, + ref isIgnoredBlock, pathsDecrypted, properties, arrayPoolManager, @@ -90,9 +100,7 @@ internal class StreamProcessor inputStream.Position = 0; outputStream.Position = 0; - return ( - outputStream, - EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId)); + return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } /* @@ -114,27 +122,37 @@ private static Dictionary GetUtf8DecryptionList(EncryptionPropertie return output; }*/ - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { - Utf8JsonReader json = new (buffer, isFinalBlock, state); + Utf8JsonReader reader = new (buffer, isFinalBlock, state); string decryptPropertyName = null; - while (json.Read()) + while (reader.Read()) { - JsonTokenType tokenType = json.TokenType; + JsonTokenType tokenType = reader.TokenType; + + if (isIgnoredBlock && reader.CurrentDepth == 1 && tokenType == JsonTokenType.EndObject) + { + isIgnoredBlock = false; + continue; + } + else if (isIgnoredBlock) + { + continue; + } switch (tokenType) { case JsonTokenType.String: if (decryptPropertyName == null) { - writer.WriteRawValue(json.ValueSpan); + writer.WriteStringValue(reader.ValueSpan); } else { this.TransformDecryptProperty( - json.GetBytesFromBase64(), + ref reader, writer, decryptPropertyName, properties, @@ -148,7 +166,7 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW break; case JsonTokenType.Number: decryptPropertyName = null; - writer.WriteRawValue(json.ValueSpan); + writer.WriteRawValue(reader.ValueSpan); break; case JsonTokenType.None: decryptPropertyName = null; @@ -170,13 +188,19 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW writer.WriteEndArray(); break; case JsonTokenType.PropertyName: - string propertyName = json.GetString(); + string propertyName = reader.GetString(); if (properties.EncryptedPaths.Contains("/" + propertyName)) { decryptPropertyName = propertyName; } - writer.WritePropertyName(json.ValueSpan); + if (propertyName == Constants.EncryptedInfo) + { + isIgnoredBlock = true; + break; + } + + writer.WritePropertyName(reader.ValueSpan); break; case JsonTokenType.Comment: break; @@ -195,11 +219,11 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW } } - state = json.CurrentState; - return json.BytesConsumed; + state = reader.CurrentState; + return reader.BytesConsumed; } - private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) + private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) { BrotliCompressor decompressor = null; if (properties.EncryptionFormatVersion == 4) @@ -216,11 +240,22 @@ private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonW } } - (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(reader.ValueSpan.Length); + + // necessary for proper un-escaping + int initialLength = reader.CopyString(cipherTextWithTypeMarker); + + OperationStatus status = Base64.DecodeFromUtf8InPlace(cipherTextWithTypeMarker.AsSpan(0, initialLength), out int cipherTextLength); + if (status != OperationStatus.Done) + { + throw new InvalidOperationException($"Base64 decoding failed: {status}"); + } + + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); if (decompressor != null) { - if (properties.CompressedEncryptedPaths?.TryGetValue(decryptPropertyName, out int decompressedSize) == true) + if (properties.CompressedEncryptedPaths?.TryGetValue("/" + decryptPropertyName, out int decompressedSize) == true) { byte[] buffer = arrayPoolManager.Rent(decompressedSize); processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); @@ -229,7 +264,28 @@ private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonW } } - writer.WriteRawValue(bytes.AsSpan(0, processedBytes)); + Span bytesToWrite = bytes.AsSpan(0, processedBytes); + switch ((TypeMarker)cipherTextWithTypeMarker[0]) + { + case TypeMarker.String: + writer.WriteStringValue(bytesToWrite); + break; + case TypeMarker.Long: + writer.WriteNumberValue(SqlLongSerializer.Deserialize(bytesToWrite)); + break; + case TypeMarker.Double: + writer.WriteNumberValue(SqlDoubleSerializer.Deserialize(bytesToWrite)); + break; + case TypeMarker.Boolean: + writer.WriteBooleanValue(SqlBoolSerializer.Deserialize(bytesToWrite)); + break; + case TypeMarker.Null: + writer.WriteNullValue(); + break; + default: + writer.WriteRawValue(bytes.AsSpan(0, processedBytes), true); + break; + } } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 051b87d69d..395eed045d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -22,13 +22,15 @@ public partial class EncryptionBenchmark private byte[]? encryptedData; private byte[]? plaintext; + private static readonly MemoryStream recycledStream = new (); + [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } - [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson,*/ JsonProcessor.Stream)] + [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, */JsonProcessor.Stream)] public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -82,6 +84,19 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } + [Benchmark] + public async Task DecryptToProvidedStream() + { + await EncryptionProcessor.DecryptAsync( + new MemoryStream(this.encryptedData!), + EncryptionBenchmark.recycledStream, + this.encryptor, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + EncryptionBenchmark.recycledStream.Position = 0; + } + private EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 523b62413f..d872d7f1a1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,29 +9,17 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.84 μs** | **0.676 μs** | **1.013 μs** | **0.1526** | **0.0305** | **-** | **41.08 KB** | -| Decrypt | 1 | None | Newtonsoft | 26.85 μs | 0.273 μs | 0.409 μs | 0.1526 | 0.0305 | - | 40.47 KB | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **15.41 μs** | **0.224 μs** | **0.335 μs** | **0.0916** | **0.0305** | **-** | **22.64 KB** | -| Decrypt | 1 | None | SystemTextJson | 14.41 μs | 0.121 μs | 0.174 μs | 0.0763 | 0.0153 | - | 20.95 KB | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **28.44 μs** | **0.246 μs** | **0.369 μs** | **0.1526** | **0.0305** | **-** | **37.45 KB** | -| Decrypt | 1 | Brotli | Newtonsoft | 34.21 μs | 0.795 μs | 1.189 μs | 0.1221 | - | - | 40.1 KB | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **21.68 μs** | **0.264 μs** | **0.378 μs** | **0.0610** | **-** | **-** | **21.71 KB** | -| Decrypt | 1 | Brotli | SystemTextJson | 20.41 μs | 0.167 μs | 0.249 μs | 0.0610 | 0.0305 | - | 20.01 KB | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **82.84 μs** | **0.495 μs** | **0.725 μs** | **0.6104** | **0.1221** | **-** | **167.26 KB** | -| Decrypt | 10 | None | Newtonsoft | 100.04 μs | 0.733 μs | 1.096 μs | 0.6104 | 0.1221 | - | 153.74 KB | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.34 μs** | **0.245 μs** | **0.351 μs** | **0.4272** | **0.0610** | **-** | **103.15 KB** | -| Decrypt | 10 | None | SystemTextJson | 41.09 μs | 0.264 μs | 0.395 μs | 0.3662 | 0.0610 | - | 94.2 KB | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **112.09 μs** | **0.821 μs** | **1.203 μs** | **0.6104** | **0.1221** | **-** | **164.4 KB** | -| Decrypt | 10 | Brotli | Newtonsoft | 119.50 μs | 1.371 μs | 1.966 μs | 0.4883 | - | - | 141.45 KB | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **70.75 μs** | **0.423 μs** | **0.620 μs** | **0.2441** | **-** | **-** | **84.47 KB** | -| Decrypt | 10 | Brotli | SystemTextJson | 64.51 μs | 1.042 μs | 1.560 μs | 0.2441 | - | - | 80.27 KB | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,142.95 μs** | **36.247 μs** | **54.253 μs** | **23.4375** | **21.4844** | **19.5313** | **1638.94 KB** | -| Decrypt | 100 | None | Newtonsoft | 1,160.91 μs | 26.561 μs | 39.755 μs | 17.5781 | 15.6250 | 15.6250 | 1230.71 KB | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **835.31 μs** | **25.982 μs** | **38.084 μs** | **26.3672** | **26.3672** | **26.3672** | **942.9 KB** | -| Decrypt | 100 | None | SystemTextJson | 731.05 μs | 23.379 μs | 33.530 μs | 18.5547 | 18.5547 | 18.5547 | 928 KB | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,138.53 μs** | **21.347 μs** | **31.952 μs** | **13.6719** | **11.7188** | **9.7656** | **1347.1 KB** | -| Decrypt | 100 | Brotli | Newtonsoft | 1,150.43 μs | 15.475 μs | 22.684 μs | 11.7188 | 9.7656 | 9.7656 | 1097.91 KB | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **994.72 μs** | **26.940 μs** | **39.489 μs** | **19.5313** | **19.5313** | **19.5313** | **748.94 KB** | -| Decrypt | 100 | Brotli | SystemTextJson | 886.36 μs | 14.437 μs | 21.162 μs | 17.5781 | 17.5781 | 17.5781 | 782.67 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |----------:|---------:|---------:|-------:|-------:|-------:|----------:| +| **Decrypt** | **1** | **None** | **Stream** | **12.63 μs** | **0.146 μs** | **0.213 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | +| DecryptToProvidedStream | 1 | None | Stream | 12.58 μs | 0.082 μs | 0.115 μs | 0.0458 | 0.0153 | - | 10.9 KB | +| **Decrypt** | **1** | **Brotli** | **Stream** | **19.30 μs** | **0.744 μs** | **1.066 μs** | **0.0305** | **-** | **-** | **12.99 KB** | +| DecryptToProvidedStream | 1 | Brotli | Stream | 18.39 μs | 0.320 μs | 0.449 μs | 0.0305 | - | - | 11.58 KB | +| **Decrypt** | **10** | **None** | **Stream** | **26.64 μs** | **0.150 μs** | **0.220 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | +| DecryptToProvidedStream | 10 | None | Stream | 25.71 μs | 0.138 μs | 0.203 μs | 0.0610 | 0.0305 | - | 17.65 KB | +| **Decrypt** | **10** | **Brotli** | **Stream** | **53.89 μs** | **0.631 μs** | **0.945 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | +| DecryptToProvidedStream | 10 | Brotli | Stream | 54.60 μs | 0.605 μs | 0.887 μs | 0.0610 | - | - | 18.33 KB | +| **Decrypt** | **100** | **None** | **Stream** | **450.58 μs** | **6.572 μs** | **9.837 μs** | **8.3008** | **8.3008** | **8.3008** | **320.02 KB** | +| DecryptToProvidedStream | 100 | None | Stream | 379.11 μs | 4.473 μs | 6.415 μs | 4.3945 | 4.3945 | 4.3945 | 163 KB | +| **Decrypt** | **100** | **Brotli** | **Stream** | **303.48 μs** | **5.305 μs** | **7.940 μs** | **5.8594** | **5.8594** | **5.8594** | **215.95 KB** | +| DecryptToProvidedStream | 100 | Brotli | Stream | 253.71 μs | 3.448 μs | 5.054 μs | 2.9297 | 2.9297 | 2.9297 | 111.63 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 271ab6671a..95a0658821 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -61,6 +61,7 @@ public static void ClassInitialize(TestContext testContext) [DataRow(JsonProcessor.Newtonsoft)] #if NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] + [DataRow(JsonProcessor.Stream)] #endif public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) { @@ -101,6 +102,7 @@ public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) [DataRow(JsonProcessor.Newtonsoft)] #if NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] + [DataRow(JsonProcessor.Stream)] #endif public async Task DuplicatePathToEncrypt(JsonProcessor jsonProcessor) { @@ -310,6 +312,7 @@ public async Task ValidateDecryptBySystemTextStream_VerifyBySystemText(Encryptio [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] + [DataRow(JsonProcessor.Stream)] #endif public async Task DecryptStreamWithoutEncryptedProperty(JsonProcessor processor) { @@ -600,6 +603,7 @@ public static IEnumerable EncryptionOptionsStreamTestCombinations yield return new object[] { encryptionOptions[0], JsonProcessor.Newtonsoft }; #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER yield return new object[] { encryptionOptions[0], JsonProcessor.SystemTextJson }; + yield return new object[] { encryptionOptions[0], JsonProcessor.Stream }; #endif } } From 420a35d8f8d45f7d1816bf1e6670c7fffca85da0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 18:28:42 +0200 Subject: [PATCH 56/85] ! bugfix --- .../src/EncryptionProcessor.cs | 2 +- .../src/Transformation/StreamProcessor.cs | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 59a43f2206..776fe9d304 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -247,7 +247,7 @@ public static async Task DecryptAsync( return (input, null); } - MemoryStream ms = new MemoryStream(); + MemoryStream ms = new (); DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, ms, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs index b24b8d9d89..cba10ff0bb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -50,9 +50,9 @@ internal async Task DecryptStreamAsync( Utf8JsonWriter writer = new (outputStream); - // we determine initial buffer size based on max uncompressed path length, it still might need scale out in case there is large non-encrypted object, but likehood is rather low - int bufferSize = 16384; // Math.Max(16384, properties.CompressedEncryptedPaths.Values.Max()); + int bufferSize = 16384; byte[] buffer = arrayPoolManager.Rent(bufferSize); + bufferSize = buffer.Length; JsonReaderState state = new (this.jsonReaderOptions); @@ -61,6 +61,8 @@ internal async Task DecryptStreamAsync( bool isFinalBlock = false; bool isIgnoredBlock = false; + string decryptPropertyName = null; + while (!isFinalBlock) { int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); @@ -75,6 +77,7 @@ internal async Task DecryptStreamAsync( writer, ref state, ref isIgnoredBlock, + ref decryptPropertyName, pathsDecrypted, properties, arrayPoolManager, @@ -87,7 +90,7 @@ internal async Task DecryptStreamAsync( { bufferSize *= 2; byte[] newBuffer = arrayPoolManager.Rent(bufferSize); - buffer.AsSpan(0, leftOver).CopyTo(newBuffer); + buffer.AsSpan().CopyTo(newBuffer); buffer = newBuffer; } else if (leftOver != 0) @@ -122,12 +125,10 @@ private static Dictionary GetUtf8DecryptionList(EncryptionPropertie return output; }*/ - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); - string decryptPropertyName = null; - while (reader.Read()) { JsonTokenType tokenType = reader.TokenType; From d08be05dcfcfb3e3c25f18daf9fc8b31d86b2a69 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 18:45:56 +0200 Subject: [PATCH 57/85] ~ benchmark update --- .../Readme.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index d872d7f1a1..aaf21846e5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,17 +9,17 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |-------------- |----------:|---------:|---------:|-------:|-------:|-------:|----------:| -| **Decrypt** | **1** | **None** | **Stream** | **12.63 μs** | **0.146 μs** | **0.213 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | -| DecryptToProvidedStream | 1 | None | Stream | 12.58 μs | 0.082 μs | 0.115 μs | 0.0458 | 0.0153 | - | 10.9 KB | -| **Decrypt** | **1** | **Brotli** | **Stream** | **19.30 μs** | **0.744 μs** | **1.066 μs** | **0.0305** | **-** | **-** | **12.99 KB** | -| DecryptToProvidedStream | 1 | Brotli | Stream | 18.39 μs | 0.320 μs | 0.449 μs | 0.0305 | - | - | 11.58 KB | -| **Decrypt** | **10** | **None** | **Stream** | **26.64 μs** | **0.150 μs** | **0.220 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | -| DecryptToProvidedStream | 10 | None | Stream | 25.71 μs | 0.138 μs | 0.203 μs | 0.0610 | 0.0305 | - | 17.65 KB | -| **Decrypt** | **10** | **Brotli** | **Stream** | **53.89 μs** | **0.631 μs** | **0.945 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | -| DecryptToProvidedStream | 10 | Brotli | Stream | 54.60 μs | 0.605 μs | 0.887 μs | 0.0610 | - | - | 18.33 KB | -| **Decrypt** | **100** | **None** | **Stream** | **450.58 μs** | **6.572 μs** | **9.837 μs** | **8.3008** | **8.3008** | **8.3008** | **320.02 KB** | -| DecryptToProvidedStream | 100 | None | Stream | 379.11 μs | 4.473 μs | 6.415 μs | 4.3945 | 4.3945 | 4.3945 | 163 KB | -| **Decrypt** | **100** | **Brotli** | **Stream** | **303.48 μs** | **5.305 μs** | **7.940 μs** | **5.8594** | **5.8594** | **5.8594** | **215.95 KB** | -| DecryptToProvidedStream | 100 | Brotli | Stream | 253.71 μs | 3.448 μs | 5.054 μs | 2.9297 | 2.9297 | 2.9297 | 111.63 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |----------:|----------:|----------:|-------:|-------:|-------:|----------:| +| **Decrypt** | **1** | **None** | **Stream** | **12.80 μs** | **0.161 μs** | **0.237 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | +| DecryptToProvidedStream | 1 | None | Stream | 12.82 μs | 0.075 μs | 0.108 μs | 0.0458 | 0.0153 | - | 10.9 KB | +| **Decrypt** | **1** | **Brotli** | **Stream** | **19.41 μs** | **0.613 μs** | **0.899 μs** | **0.0305** | **-** | **-** | **12.99 KB** | +| DecryptToProvidedStream | 1 | Brotli | Stream | 18.52 μs | 0.206 μs | 0.288 μs | 0.0305 | - | - | 11.58 KB | +| **Decrypt** | **10** | **None** | **Stream** | **26.96 μs** | **0.103 μs** | **0.148 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | +| DecryptToProvidedStream | 10 | None | Stream | 25.94 μs | 0.104 μs | 0.149 μs | 0.0610 | 0.0305 | - | 17.65 KB | +| **Decrypt** | **10** | **Brotli** | **Stream** | **53.24 μs** | **0.602 μs** | **0.882 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | +| DecryptToProvidedStream | 10 | Brotli | Stream | 54.38 μs | 0.471 μs | 0.691 μs | 0.0610 | - | - | 18.33 KB | +| **Decrypt** | **100** | **None** | **Stream** | **336.31 μs** | **7.637 μs** | **11.194 μs** | **5.8594** | **5.8594** | **5.8594** | **225.27 KB** | +| DecryptToProvidedStream | 100 | None | Stream | 283.21 μs | 2.668 μs | 3.993 μs | 2.9297 | 2.9297 | 2.9297 | 115.98 KB | +| **Decrypt** | **100** | **Brotli** | **Stream** | **487.48 μs** | **7.638 μs** | **11.433 μs** | **6.8359** | **6.8359** | **6.8359** | **225.84 KB** | +| DecryptToProvidedStream | 100 | Brotli | Stream | 457.04 μs | 10.030 μs | 14.384 μs | 3.4180 | 3.4180 | 3.4180 | 116.52 KB | From 0bf29b82eb21c499f5cd19ea11e59c43aa6cc7cc Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 08:56:50 +0200 Subject: [PATCH 58/85] + cleanup --- .../src/Transformation/StreamProcessor.cs | 54 +++++++------------ .../MdeEncryptionProcessorTests.cs | 7 +++ 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs index cba10ff0bb..2f95a8ce33 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -19,12 +19,15 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class StreamProcessor { + private const string EncryptionPropertiesPath = "/" + Constants.EncryptedInfo; private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); private static readonly SqlBigIntSerializer SqlLongSerializer = new (); private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + internal static int InitialBufferSize { get; set; } = 16384; + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); internal async Task DecryptStreamAsync( @@ -48,11 +51,9 @@ internal async Task DecryptStreamAsync( List pathsDecrypted = new (properties.EncryptedPaths.Count()); - Utf8JsonWriter writer = new (outputStream); + using Utf8JsonWriter writer = new (outputStream); - int bufferSize = 16384; - byte[] buffer = arrayPoolManager.Rent(bufferSize); - bufferSize = buffer.Length; + byte[] buffer = arrayPoolManager.Rent(InitialBufferSize); JsonReaderState state = new (this.jsonReaderOptions); @@ -63,6 +64,8 @@ internal async Task DecryptStreamAsync( string decryptPropertyName = null; + bool containsCompressed = properties.CompressedEncryptedPaths?.Count > 0; + while (!isFinalBlock) { int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); @@ -80,6 +83,7 @@ internal async Task DecryptStreamAsync( ref decryptPropertyName, pathsDecrypted, properties, + containsCompressed, arrayPoolManager, encryptionKey); @@ -88,8 +92,7 @@ internal async Task DecryptStreamAsync( // we need to scale out buffer if (leftOver == dataSize) { - bufferSize *= 2; - byte[] newBuffer = arrayPoolManager.Rent(bufferSize); + byte[] newBuffer = arrayPoolManager.Rent(buffer.Length * 2); buffer.AsSpan().CopyTo(newBuffer); buffer = newBuffer; } @@ -100,32 +103,12 @@ internal async Task DecryptStreamAsync( } writer.Flush(); - inputStream.Position = 0; outputStream.Position = 0; return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } - /* - private static Dictionary GetUtf8DecryptionList(EncryptionProperties properties) - { - Dictionary output = new (properties.EncryptedPaths.Count()); - foreach (KeyValuePair compressedPath in properties.CompressedEncryptedPaths) - { - byte[] utf8String = Encoding.UTF8.GetBytes(compressedPath.Key, 1, compressedPath.Key.Length - 1); - output[utf8String] = compressedPath.Value; - } - - foreach (string encryptedPath in properties.EncryptedPaths) - { - byte[] utf8String = Encoding.UTF8.GetBytes(encryptedPath, 1, encryptedPath.Length - 1); - output.TryAdd(utf8String, -1); - } - - return output; - }*/ - - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); @@ -158,9 +141,10 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW decryptPropertyName, properties, encryptionKey, + containsCompressed, arrayPoolManager); - pathsDecrypted.Add("/" + decryptPropertyName); + pathsDecrypted.Add(decryptPropertyName); } decryptPropertyName = null; @@ -189,13 +173,12 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW writer.WriteEndArray(); break; case JsonTokenType.PropertyName: - string propertyName = reader.GetString(); - if (properties.EncryptedPaths.Contains("/" + propertyName)) + string propertyName = "/" + reader.GetString(); + if (properties.EncryptedPaths.Contains(propertyName)) { decryptPropertyName = propertyName; } - - if (propertyName == Constants.EncryptedInfo) + else if (propertyName == StreamProcessor.EncryptionPropertiesPath) { isIgnoredBlock = true; break; @@ -224,12 +207,11 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW return reader.BytesConsumed; } - private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) + private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, bool containsCompressed, ArrayPoolManager arrayPoolManager) { BrotliCompressor decompressor = null; if (properties.EncryptionFormatVersion == 4) { - bool containsCompressed = properties.CompressedEncryptedPaths?.Any() == true; if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) { throw new NotSupportedException($"Unknown compression algorithm {properties.CompressionAlgorithm}"); @@ -254,9 +236,9 @@ private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); - if (decompressor != null) + if (containsCompressed) { - if (properties.CompressedEncryptedPaths?.TryGetValue("/" + decryptPropertyName, out int decompressedSize) == true) + if (properties.CompressedEncryptedPaths?.TryGetValue(decryptPropertyName, out int decompressedSize) == true) { byte[] buffer = arrayPoolManager.Rent(decompressedSize); processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 95a0658821..404bdc9928 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -14,6 +14,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom; +#if NET8_0_OR_GREATER + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; +#endif using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json.Linq; @@ -30,6 +33,10 @@ public static void ClassInitialize(TestContext testContext) { _ = testContext; +#if NET8_0_OR_GREATER + StreamProcessor.InitialBufferSize = 16; //we force smallest possible initial buffer to make sure both secondary reads and resize paths are executed +#endif + Mock DekMock = new(); DekMock.Setup(m => m.EncryptData(It.IsAny())) .Returns((byte[] plainText) => TestCommon.EncryptData(plainText)); From 2b2209f165c18ab3fdaa3f30f2808973f061209d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 15:56:02 +0200 Subject: [PATCH 59/85] + stream serializer + tests + Stream provided benchmarks via RecyclableMemoryStream --- .../src/EncryptionProcessor.cs | 67 ++++ .../src/RentArrayBufferWriter.cs | 203 +++++++++++ .../MdeEncryptionProcessor.Preview.cs | 26 +- ...cessor.cs => StreamProcessor.Decryptor.cs} | 11 +- .../StreamProcessor.Encryptor.cs | 344 ++++++++++++++++++ .../EncryptionBenchmark.cs | 30 +- ...Encryption.Custom.Performance.Tests.csproj | 1 + .../Readme.md | 88 ++++- .../MdeEncryptionProcessorTests.cs | 100 ++--- 9 files changed, 762 insertions(+), 108 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/{StreamProcessor.cs => StreamProcessor.Decryptor.cs} (94%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 776fe9d304..10bc00a0f0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -91,6 +91,60 @@ public static async Task EncryptAsync( #pragma warning restore CS0618 // Type or member is obsolete } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public static async Task EncryptAsync( + Stream input, + Stream output, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + ValidateInputForEncrypt( + input, + encryptor, + encryptionOptions); + + if (!encryptionOptions.PathsToEncrypt.Any()) + { + await input.CopyToAsync(output, cancellationToken); + return; + } + + if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + { + throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); + } + + foreach (string path in encryptionOptions.PathsToEncrypt) + { + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) + { + throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); + } + + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) + { + throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); + } + } + + if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + { + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); + } + + if (encryptionOptions.JsonProcessor != JsonProcessor.Stream) + { + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); + } + + await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); + } +#endif + /// /// If there isn't any data that needs to be decrypted, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. @@ -153,6 +207,7 @@ public static async Task DecryptAsync( Stream output, Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, + JsonProcessor jsonProcessor, CancellationToken cancellationToken) { if (input == null) @@ -160,6 +215,11 @@ public static async Task DecryptAsync( return null; } + if (jsonProcessor != JsonProcessor.Stream) + { + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); + } + Debug.Assert(input.CanSeek); Debug.Assert(output.CanWrite); Debug.Assert(output.CanSeek); @@ -171,9 +231,16 @@ public static async Task DecryptAsync( input.Position = 0; if (properties?.EncryptionProperties == null) { + await input.CopyToAsync(output, cancellationToken: cancellationToken); return null; } + if (properties.EncryptionProperties.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + { + input.Position = 0; + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); + } + DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs new file mode 100644 index 0000000000..cfd342d12a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs @@ -0,0 +1,203 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom; + +#if NET8_0_OR_GREATER + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +/// +/// https://gist.github.com/ahsonkhan/c76a1cc4dc7107537c3fdc0079a68b35 +/// Standard ArrayBufferWriter is not using pooled memory +/// +internal class RentArrayBufferWriter : IBufferWriter, IDisposable +{ + private const int MinimumBufferSize = 256; + + private byte[] rentedBuffer; + private int written; + private long committed; + + public RentArrayBufferWriter(int initialCapacity = MinimumBufferSize) + { + if (initialCapacity <= 0) + { + throw new ArgumentException(null, nameof(initialCapacity)); + } + + this.rentedBuffer = ArrayPool.Shared.Rent(initialCapacity); + this.written = 0; + this.committed = 0; + } + + public (byte[], int) WrittenBuffer => (this.rentedBuffer, this.written); + + public Memory WrittenMemory + { + get + { + this.CheckIfDisposed(); + + return this.rentedBuffer.AsMemory(0, this.written); + } + } + + public Span WrittenSpan + { + get + { + this.CheckIfDisposed(); + + return this.rentedBuffer.AsSpan(0, this.written); + } + } + + public int BytesWritten + { + get + { + this.CheckIfDisposed(); + + return this.written; + } + } + + public long BytesCommitted + { + get + { + this.CheckIfDisposed(); + + return this.committed; + } + } + + public void Clear() + { + this.CheckIfDisposed(); + + this.ClearHelper(); + } + + private void ClearHelper() + { + this.rentedBuffer.AsSpan(0, this.written).Clear(); + this.written = 0; + } + + public async Task CopyToAsync(Stream stream, CancellationToken cancellationToken = default) + { + this.CheckIfDisposed(); + + ArgumentNullException.ThrowIfNull(stream); + + await stream.WriteAsync(new Memory(this.rentedBuffer, 0, this.written), cancellationToken).ConfigureAwait(false); + this.committed += this.written; + + this.ClearHelper(); + } + + public void CopyTo(Stream stream) + { + this.CheckIfDisposed(); + + ArgumentNullException.ThrowIfNull(stream); + + stream.Write(this.rentedBuffer, 0, this.written); + this.committed += this.written; + + this.ClearHelper(); + } + + public void Advance(int count) + { + this.CheckIfDisposed(); + + ArgumentOutOfRangeException.ThrowIfLessThan(count, 0); + + if (this.written > this.rentedBuffer.Length - count) + { + throw new InvalidOperationException("Cannot advance past the end of the buffer."); + } + + this.written += count; + } + + // Returns the rented buffer back to the pool + public void Dispose() + { + if (this.rentedBuffer == null) + { + return; + } + + ArrayPool.Shared.Return(this.rentedBuffer, clearArray: true); + this.rentedBuffer = null; + this.written = 0; + } + + private void CheckIfDisposed() + { + ObjectDisposedException.ThrowIf(this.rentedBuffer == null, this); + } + + public Memory GetMemory(int sizeHint = 0) + { + this.CheckIfDisposed(); + + ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); + + this.CheckAndResizeBuffer(sizeHint); + return this.rentedBuffer.AsMemory(this.written); + } + + public Span GetSpan(int sizeHint = 0) + { + this.CheckIfDisposed(); + + ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); + + this.CheckAndResizeBuffer(sizeHint); + return this.rentedBuffer.AsSpan(this.written); + } + + private void CheckAndResizeBuffer(int sizeHint) + { + Debug.Assert(sizeHint >= 0); + + if (sizeHint == 0) + { + sizeHint = MinimumBufferSize; + } + + int availableSpace = this.rentedBuffer.Length - this.written; + + if (sizeHint > availableSpace) + { + int growBy = sizeHint > this.rentedBuffer.Length ? sizeHint : this.rentedBuffer.Length; + + int newSize = checked(this.rentedBuffer.Length + growBy); + + byte[] oldBuffer = this.rentedBuffer; + + this.rentedBuffer = ArrayPool.Shared.Rent(newSize); + + Debug.Assert(oldBuffer.Length >= this.written); + Debug.Assert(this.rentedBuffer.Length >= this.written); + + oldBuffer.AsSpan(0, this.written).CopyTo(this.rentedBuffer); + ArrayPool.Shared.Return(oldBuffer, clearArray: true); + } + + Debug.Assert(this.rentedBuffer.Length - this.written > 0); + Debug.Assert(this.rentedBuffer.Length - this.written >= sizeHint); + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 8649b36079..2dc3eb7e10 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -21,6 +21,8 @@ internal class MdeEncryptionProcessor #if NET8_0_OR_GREATER internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor(); + + internal StreamProcessor StreamProcessor { get; set; } = new StreamProcessor(); #endif public async Task EncryptAsync( @@ -29,15 +31,29 @@ public async Task EncryptAsync( EncryptionOptions encryptionOptions, CancellationToken token) { +#if NET8_0_OR_GREATER + switch (encryptionOptions.JsonProcessor) + { + case JsonProcessor.Newtonsoft: + return await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); + + case JsonProcessor.SystemTextJson: + return await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); + case JsonProcessor.Stream: + MemoryStream ms = new (); + await this.StreamProcessor.EncryptStreamAsync(input, ms, encryptor, encryptionOptions, token); + return ms; + + default: + throw new InvalidOperationException("Unsupported JsonProcessor"); + } +#else return encryptionOptions.JsonProcessor switch { JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), -#if NET8_0_OR_GREATER - JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), - JsonProcessor.Stream => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), -#endif - _ => throw new InvalidOperationException("Unsupported JsonProcessor") + _ => throw new InvalidOperationException("Unsupported JsonProcessor"), }; +#endif } internal async Task DecryptObjectAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs similarity index 94% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 2f95a8ce33..6489a1f3e3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -11,20 +11,19 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Collections.Generic; using System.IO; using System.Linq; - using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Encryption.Cryptography.Serializers; - internal class StreamProcessor + internal partial class StreamProcessor { private const string EncryptionPropertiesPath = "/" + Constants.EncryptedInfo; private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); private static readonly SqlBigIntSerializer SqlLongSerializer = new (); - private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + private static readonly JsonReaderOptions JsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; internal static int InitialBufferSize { get; set; } = 16384; @@ -55,7 +54,7 @@ internal async Task DecryptStreamAsync( byte[] buffer = arrayPoolManager.Rent(InitialBufferSize); - JsonReaderState state = new (this.jsonReaderOptions); + JsonReaderState state = new (StreamProcessor.JsonReaderOptions); int leftOver = 0; @@ -74,7 +73,7 @@ internal async Task DecryptStreamAsync( long bytesConsumed = 0; // processing itself here - bytesConsumed = this.TransformReadBuffer( + bytesConsumed = this.TransformDecryptBuffer( buffer.AsSpan(0, dataSize), isFinalBlock, writer, @@ -108,7 +107,7 @@ internal async Task DecryptStreamAsync( return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformDecryptBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs new file mode 100644 index 0000000000..9f521cf9ef --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -0,0 +1,344 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal partial class StreamProcessor + { + private readonly byte[] encryptionPropertiesNameBytes = Encoding.UTF8.GetBytes(Constants.EncryptedInfo); + + internal async Task EncryptStreamAsync( + Stream inputStream, + Stream outputStream, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken cancellationToken) + { + List pathsEncrypted = new (); + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, cancellationToken); + + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; + + Dictionary compressedPaths = new (); + + using Utf8JsonWriter writer = new (outputStream); + + byte[] buffer = arrayPoolManager.Rent(InitialBufferSize); + + JsonReaderState state = new (StreamProcessor.JsonReaderOptions); + + int leftOver = 0; + + bool isFinalBlock = false; + + Utf8JsonWriter encryptionPayloadWriter = null; + string encryptPropertyName = null; + RentArrayBufferWriter bufferWriter = null; + + while (!isFinalBlock) + { + int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + int dataSize = dataLength + leftOver; + isFinalBlock = dataSize == 0; + long bytesConsumed = 0; + + bytesConsumed = this.TransformEncryptBuffer( + buffer.AsSpan(0, dataSize), + isFinalBlock, + writer, + ref encryptionPayloadWriter, + ref bufferWriter, + ref state, + ref encryptPropertyName, + pathsEncrypted, + compressedPaths, + compressor, + arrayPoolManager, + encryptionKey, + encryptionOptions); + + leftOver = dataSize - (int)bytesConsumed; + + // we need to scale out buffer + if (leftOver == dataSize) + { + byte[] newBuffer = arrayPoolManager.Rent(buffer.Length * 2); + buffer.AsSpan().CopyTo(newBuffer); + buffer = newBuffer; + } + else if (leftOver != 0) + { + buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + } + } + + await inputStream.DisposeAsync(); + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: compressionEnabled ? 4 : 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); + + writer.WritePropertyName(this.encryptionPropertiesNameBytes); + JsonSerializer.Serialize(writer, encryptionProperties); + writer.WriteEndObject(); + + writer.Flush(); + outputStream.Position = 0; + } + + private long TransformEncryptBuffer( + Span buffer, + bool isFinalBlock, + Utf8JsonWriter writer, + ref Utf8JsonWriter encryptionPayloadWriter, + ref RentArrayBufferWriter bufferWriter, + ref JsonReaderState state, + ref string encryptPropertyName, + List pathsEncrypted, + Dictionary compressedPaths, + BrotliCompressor compressor, + ArrayPoolManager arrayPoolManager, + DataEncryptionKey encryptionKey, + EncryptionOptions encryptionOptions) + { + Utf8JsonReader reader = new (buffer, isFinalBlock, state); + + while (reader.Read()) + { + Utf8JsonWriter currentWriter = encryptionPayloadWriter ?? writer; + + JsonTokenType tokenType = reader.TokenType; + + switch (tokenType) + { + case JsonTokenType.None: + break; + case JsonTokenType.StartObject: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartObject(); + } + else + { + currentWriter.WriteStartObject(); + } + + break; + case JsonTokenType.EndObject: + if (reader.CurrentDepth == 0) + { + continue; + } + + currentWriter.WriteEndObject(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; + encryptionPayloadWriter.Dispose(); + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.StartArray: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartArray(); + } + else + { + currentWriter.WriteStartArray(); + } + + break; + case JsonTokenType.EndArray: + currentWriter.WriteEndArray(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; + encryptionPayloadWriter.Dispose(); + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.PropertyName: + string propertyName = "/" + reader.GetString(); + if (encryptionOptions.PathsToEncrypt.Contains(propertyName, StringComparer.Ordinal)) + { + encryptPropertyName = propertyName; + } + + currentWriter.WritePropertyName(reader.ValueSpan); + break; + case JsonTokenType.Comment: + currentWriter.WriteCommentValue(reader.ValueSpan); + break; + case JsonTokenType.String: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); + int length = reader.CopyString(bytes); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteStringValue(reader.ValueSpan); + } + + break; + case JsonTokenType.Number: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteRawValue(reader.ValueSpan); + } + + break; + case JsonTokenType.True: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(true, arrayPoolManager); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(true); + } + + break; + case JsonTokenType.False: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(false, arrayPoolManager); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(false); + } + + break; + case JsonTokenType.Null: + currentWriter.WriteNullValue(); + break; + } + } + + state = reader.CurrentState; + return reader.BytesConsumed; + } + + private static (byte[] buffer, int length) Serialize(bool value, ArrayPoolManager arrayPoolManager) + { + int byteCount = StreamProcessor.SqlBoolSerializer.GetSerializedMaxByteCount(); + byte[] buffer = arrayPoolManager.Rent(byteCount); + int length = StreamProcessor.SqlBoolSerializer.Serialize(value, buffer); + + return (buffer, length); + } + + private static (TypeMarker typeMarker, byte[] buffer, int length) SerializeNumber(ReadOnlySpan utf8bytes, ArrayPoolManager arrayPoolManager) + { + if (long.TryParse(utf8bytes, out long longValue)) + { + return Serialize(longValue, arrayPoolManager); + } + else if (double.TryParse(utf8bytes, out double doubleValue)) + { + return Serialize(doubleValue, arrayPoolManager); + } + else + { + throw new InvalidOperationException("Unsupported Number type"); + } + } + + private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(long value, ArrayPoolManager arrayPoolManager) + { + int byteCount = StreamProcessor.SqlLongSerializer.GetSerializedMaxByteCount(); + byte[] buffer = arrayPoolManager.Rent(byteCount); + int length = StreamProcessor.SqlLongSerializer.Serialize(value, buffer); + + return (TypeMarker.Long, buffer, length); + } + + private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(double value, ArrayPoolManager arrayPoolManager) + { + int byteCount = StreamProcessor.SqlDoubleSerializer.GetSerializedMaxByteCount(); + byte[] buffer = arrayPoolManager.Rent(byteCount); + int length = StreamProcessor.SqlDoubleSerializer.Serialize(value, buffer); + + return (TypeMarker.Double, buffer, length); + } + + private Span TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) + { + byte[] processedBytes = payload; + int processedBytesLength = payloadSize; + + if (compressor != null && payloadSize >= options.CompressionOptions.MinimalCompressedLength) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(payloadSize)); + processedBytesLength = compressor.Compress(pathsCompressed, encryptName, processedBytes, payloadSize, compressedBytes); + processedBytes = compressedBytes; + } + + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); + + pathsEncrypted.Add(encryptName); + return encryptedBytes.AsSpan(0, encryptedBytesCount); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 395eed045d..ad8931c305 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -3,6 +3,7 @@ using System.IO; using BenchmarkDotNet.Attributes; using Microsoft.Data.Encryption.Cryptography; + using Microsoft.IO; using Moq; [RPlotExporter] @@ -16,21 +17,21 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); + private readonly RecyclableMemoryStreamManager recyclableMemoryStreamManager = new (); + private CosmosEncryptor? encryptor; private EncryptionOptions? encryptionOptions; private byte[]? encryptedData; private byte[]? plaintext; - private static readonly MemoryStream recycledStream = new (); - [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } - [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, */JsonProcessor.Stream)] + [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, JsonProcessor.Stream)] public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -61,7 +62,7 @@ public async Task Setup() this.encryptedData = memoryStream.ToArray(); } - /* + [Benchmark] public async Task Encrypt() { @@ -71,7 +72,20 @@ await EncryptionProcessor.EncryptAsync( this.encryptionOptions, new CosmosDiagnosticsContext(), CancellationToken.None); - }*/ + } + + [Benchmark] + public async Task EncryptToProvidedStream() + { + using RecyclableMemoryStream rms = new (this.recyclableMemoryStreamManager); + await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext!), + rms, + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } [Benchmark] public async Task Decrypt() @@ -87,14 +101,14 @@ await EncryptionProcessor.DecryptAsync( [Benchmark] public async Task DecryptToProvidedStream() { + using RecyclableMemoryStream rms = new(this.recyclableMemoryStreamManager); await EncryptionProcessor.DecryptAsync( new MemoryStream(this.encryptedData!), - EncryptionBenchmark.recycledStream, + rms, this.encryptor, new CosmosDiagnosticsContext(), + this.JsonProcessor, CancellationToken.None); - - EncryptionBenchmark.recycledStream.Position = 0; } private EncryptionOptions CreateEncryptionOptions() diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index edcdac998b..2adf26ad02 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index aaf21846e5..6d65279c26 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,17 +9,77 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |-------------- |----------:|----------:|----------:|-------:|-------:|-------:|----------:| -| **Decrypt** | **1** | **None** | **Stream** | **12.80 μs** | **0.161 μs** | **0.237 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | -| DecryptToProvidedStream | 1 | None | Stream | 12.82 μs | 0.075 μs | 0.108 μs | 0.0458 | 0.0153 | - | 10.9 KB | -| **Decrypt** | **1** | **Brotli** | **Stream** | **19.41 μs** | **0.613 μs** | **0.899 μs** | **0.0305** | **-** | **-** | **12.99 KB** | -| DecryptToProvidedStream | 1 | Brotli | Stream | 18.52 μs | 0.206 μs | 0.288 μs | 0.0305 | - | - | 11.58 KB | -| **Decrypt** | **10** | **None** | **Stream** | **26.96 μs** | **0.103 μs** | **0.148 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | -| DecryptToProvidedStream | 10 | None | Stream | 25.94 μs | 0.104 μs | 0.149 μs | 0.0610 | 0.0305 | - | 17.65 KB | -| **Decrypt** | **10** | **Brotli** | **Stream** | **53.24 μs** | **0.602 μs** | **0.882 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | -| DecryptToProvidedStream | 10 | Brotli | Stream | 54.38 μs | 0.471 μs | 0.691 μs | 0.0610 | - | - | 18.33 KB | -| **Decrypt** | **100** | **None** | **Stream** | **336.31 μs** | **7.637 μs** | **11.194 μs** | **5.8594** | **5.8594** | **5.8594** | **225.27 KB** | -| DecryptToProvidedStream | 100 | None | Stream | 283.21 μs | 2.668 μs | 3.993 μs | 2.9297 | 2.9297 | 2.9297 | 115.98 KB | -| **Decrypt** | **100** | **Brotli** | **Stream** | **487.48 μs** | **7.638 μs** | **11.433 μs** | **6.8359** | **6.8359** | **6.8359** | **225.84 KB** | -| DecryptToProvidedStream | 100 | Brotli | Stream | 457.04 μs | 10.030 μs | 14.384 μs | 3.4180 | 3.4180 | 3.4180 | 116.52 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **23.89 μs** | **0.489 μs** | **0.702 μs** | **0.1526** | **0.0305** | **-** | **42064 B** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | Newtonsoft | 28.65 μs | 0.340 μs | 0.488 μs | 0.1221 | - | - | 41440 B | +| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **SystemTextJson** | **15.67 μs** | **0.206 μs** | **0.309 μs** | **0.0916** | **0.0305** | **-** | **23184 B** | +| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | SystemTextJson | 15.46 μs | 0.124 μs | 0.186 μs | 0.0610 | 0.0305 | - | 21448 B | +| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **Stream** | **14.29 μs** | **0.159 μs** | **0.237 μs** | **0.0610** | **0.0153** | **-** | **18408 B** | +| EncryptToProvidedStream | 1 | None | Stream | 14.03 μs | 0.225 μs | 0.323 μs | 0.0458 | 0.0153 | - | 12272 B | +| Decrypt | 1 | None | Stream | 13.75 μs | 0.258 μs | 0.370 μs | 0.0305 | - | - | 12456 B | +| DecryptToProvidedStream | 1 | None | Stream | 14.10 μs | 0.056 μs | 0.082 μs | 0.0458 | 0.0153 | - | 11288 B | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **29.79 μs** | **0.287 μs** | **0.429 μs** | **0.1526** | **0.0305** | **-** | **38344 B** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | Newtonsoft | 36.13 μs | 0.684 μs | 1.003 μs | 0.1221 | - | - | 41064 B | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **22.27 μs** | **0.206 μs** | **0.309 μs** | **0.0610** | **-** | **-** | **22232 B** | +| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | SystemTextJson | 21.63 μs | 0.161 μs | 0.240 μs | 0.0610 | 0.0305 | - | 20488 B | +| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **Stream** | **22.12 μs** | **0.287 μs** | **0.420 μs** | **0.0610** | **0.0305** | **-** | **17464 B** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 22.00 μs | 0.212 μs | 0.305 μs | 0.0305 | - | - | 12552 B | +| Decrypt | 1 | Brotli | Stream | 19.76 μs | 0.131 μs | 0.196 μs | 0.0305 | - | - | 13000 B | +| DecryptToProvidedStream | 1 | Brotli | Stream | 20.27 μs | 0.194 μs | 0.290 μs | 0.0305 | - | - | 11832 B | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **90.20 μs** | **1.743 μs** | **2.609 μs** | **0.6104** | **0.1221** | **-** | **171273 B** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | Newtonsoft | 105.28 μs | 1.740 μs | 2.551 μs | 0.6104 | 0.1221 | - | 157425 B | +| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **SystemTextJson** | **42.25 μs** | **0.164 μs** | **0.245 μs** | **0.4272** | **0.0610** | **-** | **105625 B** | +| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | SystemTextJson | 42.52 μs | 0.270 μs | 0.395 μs | 0.3662 | 0.0610 | - | 96464 B | +| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **Stream** | **43.70 μs** | **0.568 μs** | **0.850 μs** | **0.3052** | **0.0610** | **-** | **87488 B** | +| EncryptToProvidedStream | 10 | None | Stream | 40.54 μs | 0.283 μs | 0.424 μs | 0.1221 | - | - | 41608 B | +| Decrypt | 10 | None | Stream | 28.14 μs | 0.105 μs | 0.150 μs | 0.0916 | 0.0305 | - | 29304 B | +| DecryptToProvidedStream | 10 | None | Stream | 27.86 μs | 0.111 μs | 0.163 μs | 0.0610 | 0.0305 | - | 18200 B | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.64 μs** | **0.974 μs** | **1.397 μs** | **0.6104** | **0.1221** | **-** | **168345 B** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | Newtonsoft | 124.43 μs | 0.985 μs | 1.475 μs | 0.4883 | - | - | 144849 B | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **72.28 μs** | **0.366 μs** | **0.514 μs** | **0.2441** | **-** | **-** | **86497 B** | +| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | SystemTextJson | 66.36 μs | 0.830 μs | 1.242 μs | 0.2441 | - | - | 82201 B | +| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **Stream** | **91.08 μs** | **3.161 μs** | **4.731 μs** | **0.2441** | **-** | **-** | **68369 B** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 97.09 μs | 1.725 μs | 2.581 μs | 0.1221 | - | - | 37025 B | +| Decrypt | 10 | Brotli | Stream | 57.83 μs | 0.657 μs | 0.963 μs | 0.1221 | 0.0610 | - | 29848 B | +| DecryptToProvidedStream | 10 | Brotli | Stream | 57.69 μs | 0.554 μs | 0.830 μs | 0.0610 | - | - | 18744 B | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,167.44 μs** | **45.257 μs** | **67.739 μs** | **25.3906** | **23.4375** | **21.4844** | **1678336 B** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | Newtonsoft | 1,182.35 μs | 23.743 μs | 34.803 μs | 17.5781 | 15.6250 | 15.6250 | 1260244 B | +| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **SystemTextJson** | **830.10 μs** | **27.127 μs** | **40.603 μs** | **25.3906** | **25.3906** | **25.3906** | **965525 B** | +| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | SystemTextJson | 749.29 μs | 19.292 μs | 28.279 μs | 18.5547 | 18.5547 | 18.5547 | 950282 B | +| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **Stream** | **637.45 μs** | **34.205 μs** | **51.196 μs** | **14.6484** | **14.6484** | **14.6484** | **719521 B** | +| EncryptToProvidedStream | 100 | None | Stream | 385.08 μs | 5.025 μs | 7.521 μs | 4.8828 | 4.3945 | 4.3945 | 271565 B | +| Decrypt | 100 | None | Stream | 380.02 μs | 11.443 μs | 17.128 μs | 6.3477 | 6.3477 | 6.3477 | 230536 B | +| DecryptToProvidedStream | 100 | None | Stream | 304.54 μs | 8.678 μs | 12.989 μs | 2.9297 | 2.9297 | 2.9297 | 118897 B | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,172.02 μs** | **19.488 μs** | **29.169 μs** | **13.6719** | **11.7188** | **9.7656** | **1379452 B** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | Newtonsoft | 1,153.94 μs | 12.008 μs | 17.602 μs | 11.7188 | 9.7656 | 9.7656 | 1124251 B | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **995.73 μs** | **27.736 μs** | **40.655 μs** | **21.4844** | **21.4844** | **21.4844** | **766965 B** | +| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | SystemTextJson | 892.93 μs | 16.896 μs | 25.288 μs | 17.5781 | 17.5781 | 17.5781 | 801458 B | +| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **Stream** | **770.06 μs** | **11.317 μs** | **16.939 μs** | **10.7422** | **10.7422** | **10.7422** | **520929 B** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 606.21 μs | 9.107 μs | 13.631 μs | 2.9297 | 2.9297 | 2.9297 | 222081 B | +| Decrypt | 100 | Brotli | Stream | 537.74 μs | 8.192 μs | 12.007 μs | 6.3477 | 6.3477 | 6.3477 | 230938 B | +| DecryptToProvidedStream | 100 | Brotli | Stream | 464.80 μs | 4.408 μs | 6.461 μs | 3.4180 | 3.4180 | 3.4180 | 119300 B | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 404bdc9928..0f19a8ebc6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using System; using System.Collections.Generic; using System.IO; + using System.IO.Compression; using System.Linq; #if NET8_0_OR_GREATER using System.Text.Json.Nodes; @@ -520,84 +521,33 @@ private static void AssertNullableValueKind(T expectedValue, JsonNode node, s } #endif - public static IEnumerable EncryptionOptionsCombinations => new[] { - new object[] { new EncryptionOptions() + private static EncryptionOptions CreateEncryptionOptions(JsonProcessor processor, CompressionOptions.CompressionAlgorithm compressionAlgorithm, CompressionLevel compressionLevel) + { + return new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = processor, + CompressionOptions = new CompressionOptions() { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.None - } + Algorithm = compressionAlgorithm, + CompressionLevel = compressionLevel } - }, + }; + } + + public static IEnumerable EncryptionOptionsCombinations => new[] { + new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, #if NET8_0_OR_GREATER - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.None - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.Fastest - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.Fastest - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, - } - } - } + new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, + new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, #endif }; From 42462d5670d78c9a5b563d8748122a2f4e900bbe Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 19:01:55 +0200 Subject: [PATCH 60/85] ~ preview branching fixes --- .../src/EncryptionOptions.cs | 2 +- .../src/Transformation/StreamProcessor.Decryptor.cs | 2 +- .../src/Transformation/StreamProcessor.Encryptor.cs | 2 +- .../EncryptionBenchmark.cs | 12 ++++++++++++ ...Cosmos.Encryption.Custom.Performance.Tests.csproj | 5 ++++- .../MdeEncryptionProcessorTests.cs | 10 +++++----- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index aadb2dbbd3..af8ab293e9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -16,7 +16,7 @@ public enum JsonProcessor /// Newtonsoft, -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER /// /// System.Text.Json /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 6489a1f3e3..6459dd235d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index 9f521cf9ef..e19b802800 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index ad8931c305..c851750c53 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -3,7 +3,9 @@ using System.IO; using BenchmarkDotNet.Attributes; using Microsoft.Data.Encryption.Cryptography; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using Microsoft.IO; +#endif using Moq; [RPlotExporter] @@ -17,7 +19,9 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private readonly RecyclableMemoryStreamManager recyclableMemoryStreamManager = new (); +#endif private CosmosEncryptor? encryptor; @@ -31,7 +35,11 @@ public partial class EncryptionBenchmark [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, JsonProcessor.Stream)] +#else + [Params(JsonProcessor.Newtonsoft)] +#endif public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -74,6 +82,7 @@ await EncryptionProcessor.EncryptAsync( CancellationToken.None); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [Benchmark] public async Task EncryptToProvidedStream() { @@ -86,6 +95,7 @@ await EncryptionProcessor.EncryptAsync( new CosmosDiagnosticsContext(), CancellationToken.None); } +#endif [Benchmark] public async Task Decrypt() @@ -98,6 +108,7 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [Benchmark] public async Task DecryptToProvidedStream() { @@ -110,6 +121,7 @@ await EncryptionProcessor.DecryptAsync( this.JsonProcessor, CancellationToken.None); } +#endif private EncryptionOptions CreateEncryptionOptions() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 2adf26ad02..b04c78e2a8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -17,10 +17,13 @@ - + + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 0f19a8ebc6..c7f3595fc4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -34,7 +34,7 @@ public static void ClassInitialize(TestContext testContext) { _ = testContext; -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER StreamProcessor.InitialBufferSize = 16; //we force smallest possible initial buffer to make sure both secondary reads and resize paths are executed #endif @@ -67,7 +67,7 @@ public static void ClassInitialize(TestContext testContext) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif @@ -108,7 +108,7 @@ public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif @@ -471,7 +471,7 @@ private static void VerifyDecryptionSucceeded( } } -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static void VerifyDecryptionSucceeded( JsonNode decryptedDoc, TestDoc expectedDoc, @@ -539,7 +539,7 @@ private static EncryptionOptions CreateEncryptionOptions(JsonProcessor processor public static IEnumerable EncryptionOptionsCombinations => new[] { new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, From 703830f91877375ad01f84c86d208cee9759d27f Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 19:15:56 +0200 Subject: [PATCH 61/85] + add support for Stream deserialization of obsoleted encryption algorithm --- .../src/EncryptionProcessor.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 10bc00a0f0..e373fc9e67 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -235,13 +235,25 @@ public static async Task DecryptAsync( return null; } - if (properties.EncryptionProperties.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + DecryptionContext context; +#pragma warning disable CS0618 // Type or member is obsolete + if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + { + context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + } + else if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized) + { + (Stream stream, context) = await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken); + await stream.CopyToAsync(output, cancellationToken); + output.Position = 0; + } + else { input.Position = 0; - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); + throw new NotSupportedException($"Streaming mode is not supported for encryption algorithm {properties.EncryptionProperties.EncryptionAlgorithm}"); } +#pragma warning restore CS0618 // Type or member is obsolete - DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) { input.Position = 0; From 431952402d0c7f429abc494da75c9f82b5fadd28 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 16 Oct 2024 12:10:12 +0200 Subject: [PATCH 62/85] ~ final touches --- .../src/EncryptionProcessor.cs | 4 +- .../src/RentArrayBufferWriter.cs | 10 +- .../StreamProcessor.Decryptor.cs | 17 +- .../StreamProcessor.Encryptor.cs | 25 +-- ...Encryption.Custom.Performance.Tests.csproj | 3 +- .../Readme.md | 148 +++++++++--------- .../TestDoc.cs | 2 +- 7 files changed, 114 insertions(+), 95 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e373fc9e67..93209d2450 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -63,7 +63,7 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } @@ -113,7 +113,7 @@ public static async Task EncryptAsync( return; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs index cfd342d12a..5b37ded4fb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs @@ -37,7 +37,15 @@ public RentArrayBufferWriter(int initialCapacity = MinimumBufferSize) this.committed = 0; } - public (byte[], int) WrittenBuffer => (this.rentedBuffer, this.written); + public (byte[], int) WrittenBuffer + { + get + { + this.CheckIfDisposed(); + + return (this.rentedBuffer, this.written); + } + } public Memory WrittenMemory { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 6459dd235d..442800d6e0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -56,6 +56,8 @@ internal async Task DecryptStreamAsync( JsonReaderState state = new (StreamProcessor.JsonReaderOptions); + HashSet encryptedPaths = properties.EncryptedPaths as HashSet ?? new (properties.EncryptedPaths, StringComparer.Ordinal); + int leftOver = 0; bool isFinalBlock = false; @@ -80,6 +82,7 @@ internal async Task DecryptStreamAsync( ref state, ref isIgnoredBlock, ref decryptPropertyName, + encryptedPaths, pathsDecrypted, properties, containsCompressed, @@ -107,7 +110,7 @@ internal async Task DecryptStreamAsync( return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } - private long TransformDecryptBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformDecryptBuffer(ReadOnlySpan buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, HashSet encryptedPaths, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); @@ -173,13 +176,17 @@ private long TransformDecryptBuffer(Span buffer, bool isFinalBlock, Utf8Js break; case JsonTokenType.PropertyName: string propertyName = "/" + reader.GetString(); - if (properties.EncryptedPaths.Contains(propertyName)) + if (encryptedPaths.Contains(propertyName)) { decryptPropertyName = propertyName; } else if (propertyName == StreamProcessor.EncryptionPropertiesPath) { - isIgnoredBlock = true; + if (!reader.TrySkip()) + { + isIgnoredBlock = true; + } + break; } @@ -246,7 +253,7 @@ private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter } } - Span bytesToWrite = bytes.AsSpan(0, processedBytes); + ReadOnlySpan bytesToWrite = bytes.AsSpan(0, processedBytes); switch ((TypeMarker)cipherTextWithTypeMarker[0]) { case TypeMarker.String: @@ -265,7 +272,7 @@ private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer.WriteNullValue(); break; default: - writer.WriteRawValue(bytes.AsSpan(0, processedBytes), true); + writer.WriteRawValue(bytesToWrite, true); break; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index e19b802800..3fd4b0e4f9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System; using System.Collections.Generic; using System.IO; - using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -36,6 +35,8 @@ internal async Task EncryptStreamAsync( BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; + HashSet pathsToEncrypt = encryptionOptions.PathsToEncrypt as HashSet ?? new (encryptionOptions.PathsToEncrypt, StringComparer.Ordinal); + Dictionary compressedPaths = new (); using Utf8JsonWriter writer = new (outputStream); @@ -67,6 +68,7 @@ internal async Task EncryptStreamAsync( ref bufferWriter, ref state, ref encryptPropertyName, + pathsToEncrypt, pathsEncrypted, compressedPaths, compressor, @@ -109,13 +111,14 @@ internal async Task EncryptStreamAsync( } private long TransformEncryptBuffer( - Span buffer, + ReadOnlySpan buffer, bool isFinalBlock, Utf8JsonWriter writer, ref Utf8JsonWriter encryptionPayloadWriter, ref RentArrayBufferWriter bufferWriter, ref JsonReaderState state, ref string encryptPropertyName, + HashSet pathsToEncrypt, List pathsEncrypted, Dictionary compressedPaths, BrotliCompressor compressor, @@ -159,7 +162,7 @@ private long TransformEncryptBuffer( { currentWriter.Flush(); (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; @@ -189,7 +192,7 @@ private long TransformEncryptBuffer( { currentWriter.Flush(); (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; @@ -202,7 +205,7 @@ private long TransformEncryptBuffer( break; case JsonTokenType.PropertyName: string propertyName = "/" + reader.GetString(); - if (encryptionOptions.PathsToEncrypt.Contains(propertyName, StringComparer.Ordinal)) + if (pathsToEncrypt.Contains(propertyName)) { encryptPropertyName = propertyName; } @@ -217,7 +220,7 @@ private long TransformEncryptBuffer( { byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); int length = reader.CopyString(bytes); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } @@ -231,13 +234,13 @@ private long TransformEncryptBuffer( if (encryptPropertyName != null && encryptionPayloadWriter == null) { (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } else { - currentWriter.WriteRawValue(reader.ValueSpan); + currentWriter.WriteRawValue(reader.ValueSpan, true); } break; @@ -245,7 +248,7 @@ private long TransformEncryptBuffer( if (encryptPropertyName != null && encryptionPayloadWriter == null) { (byte[] bytes, int length) = Serialize(true, arrayPoolManager); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } @@ -259,7 +262,7 @@ private long TransformEncryptBuffer( if (encryptPropertyName != null && encryptionPayloadWriter == null) { (byte[] bytes, int length) = Serialize(false, arrayPoolManager); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } @@ -322,7 +325,7 @@ private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(doub return (TypeMarker.Double, buffer, length); } - private Span TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) + private ReadOnlySpan TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) { byte[] processedBytes = payload; int processedBytesLength = payloadSize; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index b04c78e2a8..5c154e0a9a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -7,6 +7,7 @@ enable enable true + $(DefineConstants);ENCRYPTION_CUSTOM_PREVIEW @@ -20,7 +21,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 6d65279c26..22cb620269 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,77 +9,77 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **23.89 μs** | **0.489 μs** | **0.702 μs** | **0.1526** | **0.0305** | **-** | **42064 B** | -| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | Newtonsoft | 28.65 μs | 0.340 μs | 0.488 μs | 0.1221 | - | - | 41440 B | -| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **15.67 μs** | **0.206 μs** | **0.309 μs** | **0.0916** | **0.0305** | **-** | **23184 B** | -| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | SystemTextJson | 15.46 μs | 0.124 μs | 0.186 μs | 0.0610 | 0.0305 | - | 21448 B | -| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **Stream** | **14.29 μs** | **0.159 μs** | **0.237 μs** | **0.0610** | **0.0153** | **-** | **18408 B** | -| EncryptToProvidedStream | 1 | None | Stream | 14.03 μs | 0.225 μs | 0.323 μs | 0.0458 | 0.0153 | - | 12272 B | -| Decrypt | 1 | None | Stream | 13.75 μs | 0.258 μs | 0.370 μs | 0.0305 | - | - | 12456 B | -| DecryptToProvidedStream | 1 | None | Stream | 14.10 μs | 0.056 μs | 0.082 μs | 0.0458 | 0.0153 | - | 11288 B | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **29.79 μs** | **0.287 μs** | **0.429 μs** | **0.1526** | **0.0305** | **-** | **38344 B** | -| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | Newtonsoft | 36.13 μs | 0.684 μs | 1.003 μs | 0.1221 | - | - | 41064 B | -| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **22.27 μs** | **0.206 μs** | **0.309 μs** | **0.0610** | **-** | **-** | **22232 B** | -| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | SystemTextJson | 21.63 μs | 0.161 μs | 0.240 μs | 0.0610 | 0.0305 | - | 20488 B | -| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **Stream** | **22.12 μs** | **0.287 μs** | **0.420 μs** | **0.0610** | **0.0305** | **-** | **17464 B** | -| EncryptToProvidedStream | 1 | Brotli | Stream | 22.00 μs | 0.212 μs | 0.305 μs | 0.0305 | - | - | 12552 B | -| Decrypt | 1 | Brotli | Stream | 19.76 μs | 0.131 μs | 0.196 μs | 0.0305 | - | - | 13000 B | -| DecryptToProvidedStream | 1 | Brotli | Stream | 20.27 μs | 0.194 μs | 0.290 μs | 0.0305 | - | - | 11832 B | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **90.20 μs** | **1.743 μs** | **2.609 μs** | **0.6104** | **0.1221** | **-** | **171273 B** | -| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | Newtonsoft | 105.28 μs | 1.740 μs | 2.551 μs | 0.6104 | 0.1221 | - | 157425 B | -| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **42.25 μs** | **0.164 μs** | **0.245 μs** | **0.4272** | **0.0610** | **-** | **105625 B** | -| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | SystemTextJson | 42.52 μs | 0.270 μs | 0.395 μs | 0.3662 | 0.0610 | - | 96464 B | -| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **Stream** | **43.70 μs** | **0.568 μs** | **0.850 μs** | **0.3052** | **0.0610** | **-** | **87488 B** | -| EncryptToProvidedStream | 10 | None | Stream | 40.54 μs | 0.283 μs | 0.424 μs | 0.1221 | - | - | 41608 B | -| Decrypt | 10 | None | Stream | 28.14 μs | 0.105 μs | 0.150 μs | 0.0916 | 0.0305 | - | 29304 B | -| DecryptToProvidedStream | 10 | None | Stream | 27.86 μs | 0.111 μs | 0.163 μs | 0.0610 | 0.0305 | - | 18200 B | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.64 μs** | **0.974 μs** | **1.397 μs** | **0.6104** | **0.1221** | **-** | **168345 B** | -| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | Newtonsoft | 124.43 μs | 0.985 μs | 1.475 μs | 0.4883 | - | - | 144849 B | -| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **72.28 μs** | **0.366 μs** | **0.514 μs** | **0.2441** | **-** | **-** | **86497 B** | -| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | SystemTextJson | 66.36 μs | 0.830 μs | 1.242 μs | 0.2441 | - | - | 82201 B | -| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **Stream** | **91.08 μs** | **3.161 μs** | **4.731 μs** | **0.2441** | **-** | **-** | **68369 B** | -| EncryptToProvidedStream | 10 | Brotli | Stream | 97.09 μs | 1.725 μs | 2.581 μs | 0.1221 | - | - | 37025 B | -| Decrypt | 10 | Brotli | Stream | 57.83 μs | 0.657 μs | 0.963 μs | 0.1221 | 0.0610 | - | 29848 B | -| DecryptToProvidedStream | 10 | Brotli | Stream | 57.69 μs | 0.554 μs | 0.830 μs | 0.0610 | - | - | 18744 B | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,167.44 μs** | **45.257 μs** | **67.739 μs** | **25.3906** | **23.4375** | **21.4844** | **1678336 B** | -| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | Newtonsoft | 1,182.35 μs | 23.743 μs | 34.803 μs | 17.5781 | 15.6250 | 15.6250 | 1260244 B | -| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **830.10 μs** | **27.127 μs** | **40.603 μs** | **25.3906** | **25.3906** | **25.3906** | **965525 B** | -| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | SystemTextJson | 749.29 μs | 19.292 μs | 28.279 μs | 18.5547 | 18.5547 | 18.5547 | 950282 B | -| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **Stream** | **637.45 μs** | **34.205 μs** | **51.196 μs** | **14.6484** | **14.6484** | **14.6484** | **719521 B** | -| EncryptToProvidedStream | 100 | None | Stream | 385.08 μs | 5.025 μs | 7.521 μs | 4.8828 | 4.3945 | 4.3945 | 271565 B | -| Decrypt | 100 | None | Stream | 380.02 μs | 11.443 μs | 17.128 μs | 6.3477 | 6.3477 | 6.3477 | 230536 B | -| DecryptToProvidedStream | 100 | None | Stream | 304.54 μs | 8.678 μs | 12.989 μs | 2.9297 | 2.9297 | 2.9297 | 118897 B | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,172.02 μs** | **19.488 μs** | **29.169 μs** | **13.6719** | **11.7188** | **9.7656** | **1379452 B** | -| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | Newtonsoft | 1,153.94 μs | 12.008 μs | 17.602 μs | 11.7188 | 9.7656 | 9.7656 | 1124251 B | -| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **995.73 μs** | **27.736 μs** | **40.655 μs** | **21.4844** | **21.4844** | **21.4844** | **766965 B** | -| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | SystemTextJson | 892.93 μs | 16.896 μs | 25.288 μs | 17.5781 | 17.5781 | 17.5781 | 801458 B | -| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **Stream** | **770.06 μs** | **11.317 μs** | **16.939 μs** | **10.7422** | **10.7422** | **10.7422** | **520929 B** | -| EncryptToProvidedStream | 100 | Brotli | Stream | 606.21 μs | 9.107 μs | 13.631 μs | 2.9297 | 2.9297 | 2.9297 | 222081 B | -| Decrypt | 100 | Brotli | Stream | 537.74 μs | 8.192 μs | 12.007 μs | 6.3477 | 6.3477 | 6.3477 | 230938 B | -| DecryptToProvidedStream | 100 | Brotli | Stream | 464.80 μs | 4.408 μs | 6.461 μs | 3.4180 | 3.4180 | 3.4180 | 119300 B | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.53 μs** | **0.511 μs** | **0.733 μs** | **22.29 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.224 μs | 0.322 μs | 26.23 μs | 0.1526 | 0.0305 | - | 41440 B | +| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.33 μs** | **0.137 μs** | **0.201 μs** | **14.32 μs** | **0.0916** | **0.0153** | **-** | **22904 B** | +| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | SystemTextJson | 14.54 μs | 0.124 μs | 0.186 μs | 14.52 μs | 0.0610 | 0.0305 | - | 21448 B | +| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **Stream** | **12.85 μs** | **0.095 μs** | **0.143 μs** | **12.84 μs** | **0.0610** | **0.0153** | **-** | **17528 B** | +| EncryptToProvidedStream | 1 | None | Stream | 13.00 μs | 0.096 μs | 0.141 μs | 12.98 μs | 0.0458 | 0.0153 | - | 11392 B | +| Decrypt | 1 | None | Stream | 13.01 μs | 0.152 μs | 0.228 μs | 13.05 μs | 0.0458 | 0.0153 | - | 12672 B | +| DecryptToProvidedStream | 1 | None | Stream | 13.48 μs | 0.132 μs | 0.197 μs | 13.45 μs | 0.0458 | 0.0153 | - | 11504 B | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **27.94 μs** | **0.226 μs** | **0.338 μs** | **27.96 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | Newtonsoft | 33.49 μs | 0.910 μs | 1.335 μs | 33.99 μs | 0.1221 | - | - | 41064 B | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **20.92 μs** | **0.136 μs** | **0.199 μs** | **20.95 μs** | **0.0610** | **-** | **-** | **21952 B** | +| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | SystemTextJson | 20.53 μs | 0.136 μs | 0.200 μs | 20.52 μs | 0.0610 | 0.0305 | - | 20488 B | +| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **Stream** | **21.15 μs** | **1.037 μs** | **1.521 μs** | **20.52 μs** | **0.0610** | **0.0305** | **-** | **16584 B** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 20.57 μs | 0.213 μs | 0.292 μs | 20.57 μs | 0.0305 | - | - | 11672 B | +| Decrypt | 1 | Brotli | Stream | 21.14 μs | 2.212 μs | 3.311 μs | 19.46 μs | 0.0305 | - | - | 13216 B | +| DecryptToProvidedStream | 1 | Brotli | Stream | 19.60 μs | 0.439 μs | 0.600 μs | 19.52 μs | 0.0305 | - | - | 12048 B | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **84.82 μs** | **3.002 μs** | **4.208 μs** | **83.32 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | Newtonsoft | 112.98 μs | 15.294 μs | 21.934 μs | 100.38 μs | 0.6104 | 0.1221 | - | 157425 B | +| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.85 μs** | **0.868 μs** | **1.272 μs** | **41.40 μs** | **0.4272** | **0.0610** | **-** | **105345 B** | +| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | SystemTextJson | 41.79 μs | 0.501 μs | 0.718 μs | 41.64 μs | 0.3662 | 0.0610 | - | 96464 B | +| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **Stream** | **39.63 μs** | **0.658 μs** | **0.923 μs** | **39.41 μs** | **0.3052** | **0.0610** | **-** | **82928 B** | +| EncryptToProvidedStream | 10 | None | Stream | 36.59 μs | 0.272 μs | 0.399 μs | 36.57 μs | 0.1221 | - | - | 37048 B | +| Decrypt | 10 | None | Stream | 28.64 μs | 0.378 μs | 0.517 μs | 28.59 μs | 0.1221 | 0.0305 | - | 29520 B | +| DecryptToProvidedStream | 10 | None | Stream | 27.61 μs | 0.237 μs | 0.332 μs | 27.64 μs | 0.0610 | 0.0305 | - | 18416 B | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **115.28 μs** | **3.336 μs** | **4.677 μs** | **113.71 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | Newtonsoft | 118.98 μs | 1.530 μs | 2.195 μs | 118.76 μs | 0.4883 | - | - | 144849 B | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **71.40 μs** | **0.799 μs** | **1.145 μs** | **71.23 μs** | **0.2441** | **-** | **-** | **86217 B** | +| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | SystemTextJson | 73.37 μs | 7.283 μs | 10.676 μs | 67.12 μs | 0.2441 | - | - | 82201 B | +| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **Stream** | **90.10 μs** | **3.136 μs** | **4.693 μs** | **88.92 μs** | **0.2441** | **-** | **-** | **63809 B** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 97.27 μs | 1.885 μs | 2.703 μs | 97.35 μs | 0.1221 | - | - | 32465 B | +| Decrypt | 10 | Brotli | Stream | 58.48 μs | 0.956 μs | 1.372 μs | 58.59 μs | 0.1221 | 0.0610 | - | 30064 B | +| DecryptToProvidedStream | 10 | Brotli | Stream | 59.12 μs | 1.160 μs | 1.664 μs | 59.14 μs | 0.0610 | - | - | 18960 B | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,199.74 μs** | **42.805 μs** | **64.069 μs** | **1,206.48 μs** | **23.4375** | **21.4844** | **21.4844** | **1677978 B** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | Newtonsoft | 1,177.48 μs | 25.746 μs | 38.535 μs | 1,172.04 μs | 17.5781 | 15.6250 | 15.6250 | 1260228 B | +| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **SystemTextJson** | **824.48 μs** | **31.605 μs** | **47.305 μs** | **812.80 μs** | **25.3906** | **25.3906** | **25.3906** | **965259 B** | +| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | SystemTextJson | 814.40 μs | 50.865 μs | 76.132 μs | 811.34 μs | 21.4844 | 21.4844 | 21.4844 | 950333 B | +| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **Stream** | **636.72 μs** | **31.468 μs** | **47.099 μs** | **630.15 μs** | **16.6016** | **16.6016** | **16.6016** | **678066 B** | +| EncryptToProvidedStream | 100 | None | Stream | 383.33 μs | 7.441 μs | 10.671 μs | 384.69 μs | 4.3945 | 4.3945 | 4.3945 | 230133 B | +| Decrypt | 100 | None | Stream | 384.93 μs | 12.519 μs | 18.738 μs | 383.59 μs | 5.8594 | 5.8594 | 5.8594 | 230753 B | +| DecryptToProvidedStream | 100 | None | Stream | 295.19 μs | 7.094 μs | 10.618 μs | 296.11 μs | 3.4180 | 3.4180 | 3.4180 | 119116 B | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,178.06 μs** | **63.246 μs** | **94.664 μs** | **1,152.03 μs** | **13.6719** | **11.7188** | **9.7656** | **1379183 B** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | Newtonsoft | 1,175.01 μs | 41.917 μs | 61.441 μs | 1,156.01 μs | 11.7188 | 9.7656 | 9.7656 | 1124274 B | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **1,050.27 μs** | **31.128 μs** | **46.591 μs** | **1,052.70 μs** | **17.5781** | **17.5781** | **17.5781** | **766642 B** | +| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | SystemTextJson | 926.80 μs | 28.605 μs | 41.025 μs | 925.73 μs | 18.5547 | 18.5547 | 18.5547 | 801460 B | +| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **Stream** | **757.11 μs** | **19.549 μs** | **29.260 μs** | **754.55 μs** | **10.7422** | **10.7422** | **10.7422** | **479493 B** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 563.46 μs | 9.960 μs | 14.284 μs | 561.60 μs | 2.9297 | 2.9297 | 2.9297 | 180637 B | +| Decrypt | 100 | Brotli | Stream | 542.34 μs | 14.514 μs | 21.724 μs | 542.04 μs | 6.8359 | 6.8359 | 6.8359 | 231162 B | +| DecryptToProvidedStream | 100 | Brotli | Stream | 463.69 μs | 9.130 μs | 12.800 μs | 460.71 μs | 3.4180 | 3.4180 | 3.4180 | 119506 B | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs index 4c4d42a6ee..9d6713e5f0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs @@ -7,7 +7,7 @@ public partial class EncryptionBenchmark { internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; + public static HashSet PathsToEncrypt { get; } = new HashSet{ "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } = default!; From da92d2225f4acfd9a08c1247d7aade1e2f84c897 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 17 Oct 2024 12:03:53 +0200 Subject: [PATCH 63/85] + initial draft for new DecryptableItem --- .../src/DecryptableItem.cs | 9 ++ .../src/DecryptableItemCore.cs | 10 +- .../src/StreamManager.cs | 25 +++++ .../StreamProcessing/DecryptableItemStream.cs | 101 ++++++++++++++++++ 4 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs index f9cf7c86ef..8406f84fe4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { + using System.Threading; using System.Threading.Tasks; /// @@ -79,5 +80,13 @@ public abstract class DecryptableItem /// The type of item to be returned. /// The requested item and the decryption related context. public abstract Task<(T, DecryptionContext)> GetItemAsync(); + + /// + /// Decrypts and deserializes the content. + /// + /// Cancellation token. + /// The type of item to be returned. + /// The requested item and the decryption related context. + public abstract Task<(T, DecryptionContext)> GetItemAsync(CancellationToken cancellationToken); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs index 4c1f9c4698..d309155c60 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; + using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; @@ -27,7 +28,12 @@ public DecryptableItemCore( this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); } - public override async Task<(T, DecryptionContext)> GetItemAsync() + public override Task<(T, DecryptionContext)> GetItemAsync() + { + return this.GetItemAsync(CancellationToken.None); + } + + public override async Task<(T, DecryptionContext)> GetItemAsync(CancellationToken cancellationToken) { if (this.decryptableContent is not JObject document) { @@ -40,7 +46,7 @@ public DecryptableItemCore( document, this.encryptor, new CosmosDiagnosticsContext(), - cancellationToken: default); + cancellationToken: cancellationToken); return (this.cosmosSerializer.FromStream(EncryptionProcessor.BaseSerializer.ToStream(decryptedItem)), decryptionContext); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs new file mode 100644 index 0000000000..d6c73dab52 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO; + using System.Threading.Tasks; + + // Recyclable memory stream should be here + internal class StreamManager + { + public Stream CreateStream(int hintSize = 0) + { + return new MemoryStream(hintSize); + } + + public async ValueTask ReturnStreamAsync(Stream stream) + { + await stream.DisposeAsync(); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs new file mode 100644 index 0000000000..0db9e95ff4 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs @@ -0,0 +1,101 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing +{ + using System; + using System.IO; + using System.Text.Json; + using System.Text.Json.Nodes; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + + internal sealed class DecryptableItemStream : DecryptableItem, IAsyncDisposable + { + private readonly Stream encryptedStream; // this stream should be recyclable + private readonly Encryptor encryptor; + private readonly JsonProcessor jsonProcessor; + private readonly StreamManager streamManager; + private readonly CosmosSerializer cosmosSerializer; + + private Stream decryptedStream; // this stream should be recyclable + private DecryptionContext decryptionContext; + + public DecryptableItemStream( + Stream encryptedStream, + Encryptor encryptor, + JsonProcessor processor, + CosmosSerializer cosmosSerializer, + StreamManager streamManager) + { + this.encryptedStream = encryptedStream; + this.encryptor = encryptor; + this.jsonProcessor = processor; + this.cosmosSerializer = cosmosSerializer; + this.streamManager = streamManager; + } + + public override Task<(T, DecryptionContext)> GetItemAsync() + { + return this.GetItemAsync(CancellationToken.None); + } + + public override async Task<(T, DecryptionContext)> GetItemAsync(CancellationToken cancellationToken) + { + if (this.decryptedStream == null) + { + this.decryptedStream = this.streamManager.CreateStream(); + + this.decryptionContext = await EncryptionProcessor.DecryptAsync( + this.encryptedStream, + this.decryptedStream, + this.encryptor, + new CosmosDiagnosticsContext(), + this.jsonProcessor, + cancellationToken); + + await this.encryptedStream.DisposeAsync(); + } + + // class is not generic, so we cannot reasonably cache deserialized content + + T selector = default; + switch (selector) + { + case Stream: // consumer doesn't need payload deserialized + // should we make deep copy here? handing out 'Recyclable' memory stream + return ((T)(object)this.decryptedStream, this.decryptionContext); + + case JsonNode: // Read/Write System.Text.Json DOM + // we don't have anywhere to get settings from atm + JsonNode jsonNode = await JsonNode.ParseAsync(this.decryptedStream, cancellationToken: cancellationToken); + return ((T)(object)jsonNode, this.decryptionContext); + + case JsonDocument: // Read only System.Text.Json DOM + // we don't have anywhere to get settings from atm + JsonDocument jsonDocument = await JsonDocument.ParseAsync(this.decryptedStream, cancellationToken: cancellationToken); + return ((T)(object)jsonDocument, this.decryptionContext); + + case JObject: // We must call explicit Newtonsoft implementation otherwise result would be nonsense + return ((T)(object)EncryptionProcessor.BaseSerializer.FromStream(decryptedStream)) + else: // Either Newtonsoft DOM or direct object mapping + // this API is missing Async => should not be used + // we have no chance to + return (this.cosmosSerializer.FromStream(this.decryptedStream), this.decryptionContext); + } + } + + public async ValueTask DisposeAsync() + { + if (this.decryptedStream != null) + { + await this.streamManager.ReturnStreamAsync(this.decryptedStream); + this.decryptedStream = null; + } + } + } +} +#endif \ No newline at end of file From 71c674fb349afc25b90d47262549400e69e13020 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 17 Oct 2024 12:38:28 +0200 Subject: [PATCH 64/85] + unsaved comments --- .../src/StreamProcessing/DecryptableItemStream.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs index 0db9e95ff4..f9248fbd1d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs @@ -79,11 +79,10 @@ public DecryptableItemStream( JsonDocument jsonDocument = await JsonDocument.ParseAsync(this.decryptedStream, cancellationToken: cancellationToken); return ((T)(object)jsonDocument, this.decryptionContext); - case JObject: // We must call explicit Newtonsoft implementation otherwise result would be nonsense + case JObject: // We must call explicit Newtonsoft implementation otherwise result would be nonsense if cosmosSerializer is not Newtonsoft and we have no chance to tell return ((T)(object)EncryptionProcessor.BaseSerializer.FromStream(decryptedStream)) - else: // Either Newtonsoft DOM or direct object mapping + else: // Direct object mapping // this API is missing Async => should not be used - // we have no chance to return (this.cosmosSerializer.FromStream(this.decryptedStream), this.decryptionContext); } } From 4bf4a6bdb3b8f53a120a5a6b3978e5113c2d7b69 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 17 Oct 2024 14:58:23 +0200 Subject: [PATCH 65/85] + new DecryptableItem --- .../src/DecryptableItem.cs | 8 ++- .../src/DecryptableItemCore.cs | 4 ++ .../StreamProcessing/DecryptableItemStream.cs | 61 +++++++++++-------- 3 files changed, 47 insertions(+), 26 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs index 8406f84fe4..f0b1b2660e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItem.cs @@ -4,6 +4,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { + using System; using System.Threading; using System.Threading.Tasks; @@ -72,7 +73,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// ]]> /// /// - public abstract class DecryptableItem + public abstract class DecryptableItem : IDisposable { /// /// Decrypts and deserializes the content. @@ -88,5 +89,10 @@ public abstract class DecryptableItem /// The type of item to be returned. /// The requested item and the decryption related context. public abstract Task<(T, DecryptionContext)> GetItemAsync(CancellationToken cancellationToken); + + /// + /// Dispose unmanaged resources. + /// + public abstract void Dispose(); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs index d309155c60..de247e8652 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableItemCore.cs @@ -28,6 +28,10 @@ public DecryptableItemCore( this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); } + public override void Dispose() + { + } + public override Task<(T, DecryptionContext)> GetItemAsync() { return this.GetItemAsync(CancellationToken.None); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs index f9248fbd1d..55e992dd4c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs @@ -7,23 +7,22 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing { using System; using System.IO; - using System.Text.Json; - using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; - using Newtonsoft.Json.Linq; - internal sealed class DecryptableItemStream : DecryptableItem, IAsyncDisposable + internal sealed class DecryptableItemStream : DecryptableItem { - private readonly Stream encryptedStream; // this stream should be recyclable private readonly Encryptor encryptor; private readonly JsonProcessor jsonProcessor; private readonly StreamManager streamManager; private readonly CosmosSerializer cosmosSerializer; + private Stream encryptedStream; // this stream should be recyclable private Stream decryptedStream; // this stream should be recyclable private DecryptionContext decryptionContext; + private bool isDisposed; + public DecryptableItemStream( Stream encryptedStream, Encryptor encryptor, @@ -45,6 +44,8 @@ public DecryptableItemStream( public override async Task<(T, DecryptionContext)> GetItemAsync(CancellationToken cancellationToken) { + ObjectDisposedException.ThrowIf(this.isDisposed, this); + if (this.decryptedStream == null) { this.decryptedStream = this.streamManager.CreateStream(); @@ -58,32 +59,19 @@ public DecryptableItemStream( cancellationToken); await this.encryptedStream.DisposeAsync(); + this.encryptedStream = null; } - // class is not generic, so we cannot reasonably cache deserialized content - T selector = default; switch (selector) { case Stream: // consumer doesn't need payload deserialized - // should we make deep copy here? handing out 'Recyclable' memory stream - return ((T)(object)this.decryptedStream, this.decryptionContext); - - case JsonNode: // Read/Write System.Text.Json DOM - // we don't have anywhere to get settings from atm - JsonNode jsonNode = await JsonNode.ParseAsync(this.decryptedStream, cancellationToken: cancellationToken); - return ((T)(object)jsonNode, this.decryptionContext); - - case JsonDocument: // Read only System.Text.Json DOM - // we don't have anywhere to get settings from atm - JsonDocument jsonDocument = await JsonDocument.ParseAsync(this.decryptedStream, cancellationToken: cancellationToken); - return ((T)(object)jsonDocument, this.decryptionContext); - - case JObject: // We must call explicit Newtonsoft implementation otherwise result would be nonsense if cosmosSerializer is not Newtonsoft and we have no chance to tell - return ((T)(object)EncryptionProcessor.BaseSerializer.FromStream(decryptedStream)) - else: // Direct object mapping - // this API is missing Async => should not be used - return (this.cosmosSerializer.FromStream(this.decryptedStream), this.decryptionContext); + MemoryStream ms = new ((int)this.decryptedStream.Length); + await this.decryptedStream.CopyToAsync(ms, cancellationToken); + return ((T)(object)ms, this.decryptionContext); + default: + // this API is missing Async => should not be used + return (this.cosmosSerializer.FromStream(this.decryptedStream), this.decryptionContext); } } @@ -95,6 +83,29 @@ public async ValueTask DisposeAsync() this.decryptedStream = null; } } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.encryptedStream?.Dispose(); + this.decryptedStream?.Dispose(); + this.encryptedStream = null; + this.decryptedStream = null; + } + + this.isDisposed = true; + } + } + + public override void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } } } #endif \ No newline at end of file From 03a4e9bfc191a37a25a5b58821c2d0f6e05f3d14 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 17 Oct 2024 15:42:45 +0200 Subject: [PATCH 66/85] +EncryptableItemStream --- .../src/EncryptableItem.cs | 35 ++++++++- .../src/MemoryStreamManager.cs | 38 ++++++++++ .../src/StreamManager.cs | 26 ++++--- .../StreamProcessing/EncryptableItemStream.cs | 76 +++++++++++++++++++ 4 files changed, 164 insertions(+), 11 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs index 5e6e173cf7..c27a29fb87 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs @@ -4,13 +4,17 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { + using System; using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; /// /// Input type should implement this abstract class for lazy decryption and to retrieve the details in the write path. /// - public abstract class EncryptableItem + public abstract class EncryptableItem : IDisposable { /// /// Gets DecryptableItem @@ -22,17 +26,46 @@ public abstract class EncryptableItem /// /// Cosmos Serializer /// Input payload in stream format + [Obsolete("Use overload with outputStream")] protected internal abstract Stream ToStream(CosmosSerializer serializer); + /// + /// Gets the input payload in stream format. + /// + /// Cosmos Serializer + /// Output stream + /// CancellationToken + /// A representing the asynchronous operation. + protected internal abstract Task ToStreamAsync(CosmosSerializer serializer, Stream outputStream, CancellationToken cancellationToken); + /// /// Populates the DecryptableItem that can be used getting the decryption result. /// /// The encrypted content which is yet to be decrypted. /// Encryptor instance which will be used for decryption. /// Serializer instance which will be used for deserializing the content after decryption. + [Obsolete("Use overload with decryptableStream")] protected internal abstract void SetDecryptableItem( JToken decryptableContent, Encryptor encryptor, CosmosSerializer cosmosSerializer); + + /// + /// Populates the DecryptableItem that can be used getting the decryption result. + /// + /// The encrypted content stream which is yet to be decrypted. + /// Encryptor instance which will be used for decryption. + /// Json processor for decryption. + /// Serializer instance which will be used for deserializing the content after decryption. + protected internal abstract void SetDecryptableStream( + Stream decryptableStream, + Encryptor encryptor, + JsonProcessor jsonProcessor, + CosmosSerializer cosmosSerializer); + + /// + /// Release unmananaged resources + /// + public abstract void Dispose(); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs new file mode 100644 index 0000000000..ac068223de --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs @@ -0,0 +1,38 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO; + using System.Threading.Tasks; + + /// + /// Memory Stream manager + /// + /// Placeholder + internal class MemoryStreamManager : StreamManager + { + /// + /// Create stream + /// + /// Desired minimal size of stream. + /// Instance of stream. + public override Stream CreateStream(int hintSize = 0) + { + return new MemoryStream(hintSize); + } + + /// + /// Dispose of used Stream (return to pool) + /// + /// Stream to dispose. + /// ValueTask.CompletedTask + public async override ValueTask ReturnStreamAsync(Stream stream) + { + await stream.DisposeAsync(); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs index d6c73dab52..ed7831b783 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs @@ -8,18 +8,24 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Threading.Tasks; - // Recyclable memory stream should be here - internal class StreamManager + /// + /// Abstraction for pooling streams + /// + public abstract class StreamManager { - public Stream CreateStream(int hintSize = 0) - { - return new MemoryStream(hintSize); - } + /// + /// Create stream + /// + /// Desired minimal size of stream. + /// Instance of stream. + public abstract Stream CreateStream(int hintSize = 0); - public async ValueTask ReturnStreamAsync(Stream stream) - { - await stream.DisposeAsync(); - } + /// + /// Dispose of used Stream (return to pool) + /// + /// Stream to dispose. + /// ValueTask.CompletedTask + public abstract ValueTask ReturnStreamAsync(Stream stream); } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs new file mode 100644 index 0000000000..77eb53b4b2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs @@ -0,0 +1,76 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos; + using Newtonsoft.Json.Linq; + + public sealed class EncryptableItemStream : EncryptableItem + { + private DecryptableItemStream decryptableItem = null; + + /// + /// Gets the input item + /// + public T Item { get; } + + private readonly StreamManager streamManager; + + /// + public override DecryptableItem DecryptableItem => this.decryptableItem ?? throw new InvalidOperationException("Decryptable content is not initialized."); + + /// + /// Initializes a new instance of the class. + /// + /// Item to be written. + /// Stream manager to provide output streams. + /// Thrown when input is null. + public EncryptableItemStream(T input, StreamManager streamManager = null) + { + this.Item = input ?? throw new ArgumentNullException(nameof(input)); + this.streamManager = streamManager ?? new MemoryStreamManager(); + } + +#pragma warning disable CS0672 // Member overrides obsolete member + /// + protected internal override void SetDecryptableItem(JToken decryptableContent, Encryptor encryptor, CosmosSerializer cosmosSerializer) +#pragma warning restore CS0672 // Member overrides obsolete member + { + throw new NotImplementedException(); + } + + /// + protected internal override void SetDecryptableStream(Stream decryptableStream, Encryptor encryptor, JsonProcessor jsonProcessor, CosmosSerializer cosmosSerializer) + { + ArgumentNullException.ThrowIfNull(decryptableStream); + + this.decryptableItem = new DecryptableItemStream(decryptableStream, encryptor, jsonProcessor, cosmosSerializer, this.streamManager); + } + + /// +#pragma warning disable CS0672 // Member overrides obsolete member + protected internal override Stream ToStream(CosmosSerializer serializer) +#pragma warning restore CS0672 // Member overrides obsolete member + { + return serializer.ToStream(this.Item); + } + + /// + protected internal override async Task ToStreamAsync(CosmosSerializer serializer, Stream outputStream, CancellationToken cancellationToken) + { + // TODO: CosmosSerializer is lacking suitable methods + Stream cosmosSerializerOutput = serializer.ToStream(this.Item); + await cosmosSerializerOutput.CopyToAsync(outputStream, cancellationToken); + } + } +} +#endif \ No newline at end of file From 0ed76e66501a401b4e1efb9b511667fcac6869fa Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 18 Oct 2024 14:25:25 +0200 Subject: [PATCH 67/85] + update EncryptionContainer to work with new EncryptionProcessor --- .../src/EncryptableItem.cs | 6 +- .../src/EncryptionContainer.cs | 308 +++- .../src/EncryptionProcessor.cs | 19 + .../src/MemoryStreamManager.cs | 7 +- .../RecyclableMemoryStreamMirror/README.md | 3 + .../RecyclableMemoryStream.cs | 1585 +++++++++++++++++ ...RecyclableMemoryStreamManager.EventArgs.cs | 456 +++++ .../RecyclableMemoryStreamManager.Events.cs | 288 +++ .../RecyclableMemoryStreamManager.cs | 989 ++++++++++ .../StreamProcessing/DecryptableItemStream.cs | 15 +- .../StreamProcessing/EncryptableItemStream.cs | 53 +- .../Transformation/ArrayStreamProcessor.cs | 199 +++ .../SystemTextJson/JsonBytes.cs | 34 - .../SystemTextJson/JsonBytesConverter.cs | 26 - .../src/Serializer/CosmosSerializer.cs | 34 + .../CosmosSystemTextJsonSerializer.cs | 31 + 16 files changed, 3969 insertions(+), 84 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs index c27a29fb87..5351967880 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs @@ -50,6 +50,7 @@ protected internal abstract void SetDecryptableItem( Encryptor encryptor, CosmosSerializer cosmosSerializer); +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER /// /// Populates the DecryptableItem that can be used getting the decryption result. /// @@ -57,11 +58,14 @@ protected internal abstract void SetDecryptableItem( /// Encryptor instance which will be used for decryption. /// Json processor for decryption. /// Serializer instance which will be used for deserializing the content after decryption. + /// Stream manager providing output streams. protected internal abstract void SetDecryptableStream( Stream decryptableStream, Encryptor encryptor, JsonProcessor jsonProcessor, - CosmosSerializer cosmosSerializer); + CosmosSerializer cosmosSerializer, + StreamManager streamManager); +#endif /// /// Release unmananaged resources diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index f769e1504a..2b8473ba62 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -10,6 +10,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Linq; using System.Threading; using System.Threading.Tasks; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; +#endif using Newtonsoft.Json.Linq; internal sealed class EncryptionContainer : Container @@ -22,6 +25,10 @@ internal sealed class EncryptionContainer : Container public CosmosResponseFactory ResponseFactory { get; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private readonly StreamManager streamManager; +#endif + /// /// All the operations / requests for exercising client-side encryption functionality need to be made using this EncryptionContainer instance. /// @@ -35,6 +42,9 @@ public EncryptionContainer( this.Encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); this.ResponseFactory = this.Database.Client.ResponseFactory; this.CosmosSerializer = this.Database.Client.ClientOptions.Serializer; +#if NET8_0_OR_GREATER + this.streamManager = new MemoryStreamManager(); +#endif } public override string Id => this.container.Id; @@ -76,8 +86,27 @@ public override async Task> CreateItemAsync( { ResponseMessage responseMessage; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (item is EncryptableItemStream encryptableItemStream) + { + using Stream rms = this.streamManager.CreateStream(); + await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); + responseMessage = await this.CreateItemHelperAsync( + rms, + partitionKey.Value, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); + + return new EncryptionItemResponse(responseMessage, item); + } +#endif if (item is EncryptableItem encryptableItem) { +#pragma warning disable CS0618 // Type or member is obsolete using (Stream streamPayload = encryptableItem.ToStream(this.CosmosSerializer)) { responseMessage = await this.CreateItemHelperAsync( @@ -88,11 +117,14 @@ public override async Task> CreateItemAsync( diagnosticsContext, cancellationToken); } +#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning disable CS0618 // Type or member is obsolete encryptableItem.SetDecryptableItem( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); +#pragma warning restore CS0618 // Type or member is obsolete return new EncryptionItemResponse( responseMessage, @@ -223,7 +255,30 @@ public override async Task> ReadItemAsync( using (diagnosticsContext.CreateScope("ReadItem")) { ResponseMessage responseMessage; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (typeof(T) == typeof(DecryptableItemStream)) + { + responseMessage = await this.ReadItemHelperAsync( + id, + partitionKey, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + EncryptionItemRequestOptions options = requestOptions as EncryptionItemRequestOptions; + DecryptableItem decryptableItem = new DecryptableItemStream( + responseMessage.Content, + this.Encryptor, + options.EncryptionOptions.JsonProcessor, + this.CosmosSerializer, + this.streamManager); + return new EncryptionItemResponse( + responseMessage, + (T)(object)decryptableItem); + } +#endif if (typeof(T) == typeof(DecryptableItem)) { responseMessage = await this.ReadItemHelperAsync( @@ -291,11 +346,31 @@ private async Task ReadItemHelperAsync( if (decryptResponse) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + using Stream rms = this.streamManager.CreateStream(); + + EncryptionItemRequestOptions options = requestOptions as EncryptionItemRequestOptions; + + if (options?.EncryptionOptions != null) + { + _ = await EncryptionProcessor.DecryptAsync(responseMessage.Content, rms, this.Encryptor, diagnosticsContext, options.EncryptionOptions.JsonProcessor, cancellationToken); + responseMessage.Content = rms; + } + else + { + (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( + responseMessage.Content, + this.Encryptor, + diagnosticsContext, + cancellationToken); + } +#else (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( responseMessage.Content, this.Encryptor, diagnosticsContext, cancellationToken); +#endif } return responseMessage; @@ -344,8 +419,28 @@ public override async Task> ReplaceItemAsync( { ResponseMessage responseMessage; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (item is EncryptableItemStream encryptableItemStream) + { + using Stream rms = this.streamManager.CreateStream(); + await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); + responseMessage = await this.CreateItemHelperAsync( + rms, + partitionKey.Value, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); + + return new EncryptionItemResponse(responseMessage, item); + } +#endif + if (item is EncryptableItem encryptableItem) { +#pragma warning disable CS0618 // Type or member is obsolete using (Stream streamPayload = encryptableItem.ToStream(this.CosmosSerializer)) { responseMessage = await this.ReplaceItemHelperAsync( @@ -357,11 +452,14 @@ public override async Task> ReplaceItemAsync( diagnosticsContext, cancellationToken); } +#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning disable CS0618 // Type or member is obsolete encryptableItem.SetDecryptableItem( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); +#pragma warning restore CS0618 // Type or member is obsolete return new EncryptionItemResponse( responseMessage, @@ -442,12 +540,24 @@ private async Task ReplaceItemHelperAsync( cancellationToken); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream encryptedStream = this.streamManager.CreateStream(); + await EncryptionProcessor.EncryptAsync( + streamPayload, + encryptedStream, + this.Encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken); + streamPayload = encryptedStream; +#else streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken); +#endif ResponseMessage responseMessage = await this.container.ReplaceItemStreamAsync( streamPayload, @@ -458,11 +568,23 @@ private async Task ReplaceItemHelperAsync( if (decryptResponse) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + responseMessage.Content, + decryptedStream, + this.Encryptor, + diagnosticsContext, + encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, + cancellationToken); + responseMessage.Content = decryptedStream; +#else (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( responseMessage.Content, this.Encryptor, diagnosticsContext, cancellationToken); +#endif } return responseMessage; @@ -499,8 +621,28 @@ public override async Task> UpsertItemAsync( { ResponseMessage responseMessage; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (item is EncryptableItemStream encryptableItemStream) + { + using Stream rms = this.streamManager.CreateStream(); + await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); + responseMessage = await this.UpsertItemHelperAsync( + rms, + partitionKey.Value, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); + + return new EncryptionItemResponse(responseMessage, item); + } +#endif + if (item is EncryptableItem encryptableItem) { +#pragma warning disable CS0618 // Type or member is obsolete using (Stream streamPayload = encryptableItem.ToStream(this.CosmosSerializer)) { responseMessage = await this.UpsertItemHelperAsync( @@ -511,11 +653,14 @@ public override async Task> UpsertItemAsync( diagnosticsContext, cancellationToken); } +#pragma warning restore CS0618 // Type or member is obsolete +#pragma warning disable CS0618 // Type or member is obsolete encryptableItem.SetDecryptableItem( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); +#pragma warning restore CS0618 // Type or member is obsolete return new EncryptionItemResponse( responseMessage, @@ -585,12 +730,24 @@ private async Task UpsertItemHelperAsync( cancellationToken); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream rms = this.streamManager.CreateStream(); + await EncryptionProcessor.EncryptAsync( + streamPayload, + rms, + this.Encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken); + streamPayload = rms; +#else streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken); +#endif ResponseMessage responseMessage = await this.container.UpsertItemStreamAsync( streamPayload, @@ -600,11 +757,23 @@ private async Task UpsertItemHelperAsync( if (decryptResponse) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream decryptStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + responseMessage.Content, + decryptStream, + this.Encryptor, + diagnosticsContext, + encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, + cancellationToken); + responseMessage.Content = decryptStream; +#else (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( responseMessage.Content, this.Encryptor, diagnosticsContext, cancellationToken); +#endif } return responseMessage; @@ -883,6 +1052,26 @@ public override Task PatchItemStreamAsync( throw new NotImplementedException(); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( + string processorName, + ChangesHandler onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilder( + processorName, + async ( + IReadOnlyCollection documents, + CancellationToken cancellationToken) => + { + List decryptItems = await this.DecryptChangeFeedDocumentsAsync( + documents, + cancellationToken); + + // Call the original passed in delegate + await onChangesDelegate(decryptItems, cancellationToken); + }); + } +#else public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( string processorName, ChangesHandler onChangesDelegate) @@ -901,7 +1090,29 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( await onChangesDelegate(decryptItems, cancellationToken); }); } +#endif + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( + string processorName, + ChangeFeedHandler onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilder( + processorName, + async ( + ChangeFeedProcessorContext context, + IReadOnlyCollection documents, + CancellationToken cancellationToken) => + { + List decryptItems = await this.DecryptChangeFeedDocumentsAsync( + documents, + cancellationToken); + // Call the original passed in delegate + await onChangesDelegate(context, decryptItems, cancellationToken); + }); + } +#else public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( string processorName, ChangeFeedHandler onChangesDelegate) @@ -921,7 +1132,30 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( await onChangesDelegate(context, decryptItems, cancellationToken); }); } +#endif + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( + string processorName, + ChangeFeedHandlerWithManualCheckpoint onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilderWithManualCheckpoint( + processorName, + async ( + ChangeFeedProcessorContext context, + IReadOnlyCollection documents, + Func tryCheckpointAsync, + CancellationToken cancellationToken) => + { + List decryptItems = await this.DecryptChangeFeedDocumentsAsync( + documents, + cancellationToken); + // Call the original passed in delegate + await onChangesDelegate(context, decryptItems, tryCheckpointAsync, cancellationToken); + }); + } +#else public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( string processorName, ChangeFeedHandlerWithManualCheckpoint onChangesDelegate) @@ -942,6 +1176,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu await onChangesDelegate(context, decryptItems, tryCheckpointAsync, cancellationToken); }); } +#endif public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( string processorName, @@ -954,10 +1189,20 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( Stream changes, CancellationToken cancellationToken) => { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream decryptedChanges = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync( + changes, + decryptedChanges, + this.Encryptor, + this.streamManager, + cancellationToken); +#else Stream decryptedChanges = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( changes, this.Encryptor, cancellationToken); +#endif // Call the original passed in delegate await onChangesDelegate(context, decryptedChanges, cancellationToken); @@ -976,10 +1221,20 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu Func tryCheckpointAsync, CancellationToken cancellationToken) => { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream decryptedChanges = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync( + changes, + decryptedChanges, + this.Encryptor, + this.streamManager, + cancellationToken); +#else Stream decryptedChanges = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( changes, this.Encryptor, cancellationToken); +#endif // Call the original passed in delegate await onChangesDelegate(context, decryptedChanges, tryCheckpointAsync, cancellationToken); @@ -1105,5 +1360,56 @@ private async Task> DecryptChangeFeedDocumentsAsync( return decryptItems; } - } + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private async Task> DecryptChangeFeedDocumentsAsync( + IReadOnlyCollection documents, + CancellationToken cancellationToken) + { + List decryptItems = new (documents.Count); + if (typeof(T) == typeof(DecryptableItemStream) || typeof(T) == typeof(DecryptableItem)) + { + foreach (Stream documentStream in documents) + { + DecryptableItemStream item = new ( + documentStream, + this.Encryptor, + JsonProcessor.Stream, + this.CosmosSerializer, + this.streamManager); + + decryptItems.Add((T)(object)item); + } + } + else + { + foreach (Stream document in documents) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); + using (diagnosticsContext.CreateScope("DecryptChangeFeedDocumentsAsync<")) + { + Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + document, + decryptedStream, + this.Encryptor, + diagnosticsContext, + JsonProcessor.Stream, + cancellationToken); + +#if SDKPROJECTREF + decryptItems.Add(await this.CosmosSerializer.FromStreamAsync(decryptedStream, cancellationToken)); +#else + decryptItems.Add(this.CosmosSerializer.FromStream(decryptedStream)); +#endif + + await decryptedStream.DisposeAsync(); + } + } + } + + return decryptItems; + } +#endif + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 93209d2450..370355b558 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -35,6 +35,7 @@ internal static class EncryptionProcessor #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); + private static readonly ArrayStreamProcessor ArrayStreamProcessor = new (); #endif private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -546,5 +547,23 @@ await DecryptAsync( // and corresponding decrypted properties are added back in the documents. return BaseSerializer.ToStream(contentJObj); } + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + internal static async Task DeserializeAndDecryptResponseAsync( + Stream inputStream, + Stream outputStream, + Encryptor encryptor, + StreamManager streamManager, + CancellationToken cancellationToken) + { + await ArrayStreamProcessor.DeserializeAndDecryptCollectionAsync( + inputStream, + outputStream, + encryptor, + streamManager, + cancellationToken); + } +#endif + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs index ac068223de..2403e4803b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System.IO; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; /// /// Memory Stream manager @@ -14,14 +15,16 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// Placeholder internal class MemoryStreamManager : StreamManager { + private readonly RecyclableMemoryStreamManager streamManager = new RecyclableMemoryStreamManager(); + /// /// Create stream /// - /// Desired minimal size of stream. + /// Desired minimal capacity of stream. /// Instance of stream. public override Stream CreateStream(int hintSize = 0) { - return new MemoryStream(hintSize); + return new RecyclableMemoryStream(this.streamManager, null, hintSize); } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md new file mode 100644 index 0000000000..eed315c783 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md @@ -0,0 +1,3 @@ +# Microsoft.IO.RecyclableMemoryStream 3.0.1 + +Mirrored from https://github.com/microsoft/Microsoft.IO.RecyclableMemoryStream/tree/master/src diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs new file mode 100644 index 0000000000..ca6c61c8a2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs @@ -0,0 +1,1585 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// The MIT License (MIT) +// +// Copyright (c) 2015-2016 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Buffers; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + + /// + /// MemoryStream implementation that deals with pooling and managing memory streams which use potentially large + /// buffers. + /// + /// + /// This class works in tandem with the to supply MemoryStream-derived + /// objects to callers, while avoiding these specific problems: + /// + /// + /// LOH allocations + /// Since all large buffers are pooled, they will never incur a Gen2 GC + /// + /// + /// Memory wasteA standard memory stream doubles its size when it runs out of room. This + /// leads to continual memory growth as each stream approaches the maximum allowed size. + /// + /// + /// Memory copying + /// Each time a MemoryStream grows, all the bytes are copied into new buffers. + /// This implementation only copies the bytes when is called. + /// + /// + /// Memory fragmentation + /// By using homogeneous buffer sizes, it ensures that blocks of memory + /// can be easily reused. + /// + /// + /// + /// + /// The stream is implemented on top of a series of uniformly-sized blocks. As the stream's length grows, + /// additional blocks are retrieved from the memory manager. It is these blocks that are pooled, not the stream + /// object itself. + /// + /// + /// The biggest wrinkle in this implementation is when is called. This requires a single + /// contiguous buffer. If only a single block is in use, then that block is returned. If multiple blocks + /// are in use, we retrieve a larger buffer from the memory manager. These large buffers are also pooled, + /// split by size--they are multiples/exponentials of a chunk size (1 MB by default). + /// + /// + /// Once a large buffer is assigned to the stream the small blocks are NEVER again used for this stream. All operations take place on the + /// large buffer. The large buffer can be replaced by a larger buffer from the pool as needed. All blocks and large buffers + /// are maintained in the stream until the stream is disposed (unless AggressiveBufferReturn is enabled in the stream manager). + /// + /// + /// A further wrinkle is what happens when the stream is longer than the maximum allowable array length under .NET. This is allowed + /// when only blocks are in use, and only the Read/Write APIs are used. Once a stream grows to this size, any attempt to convert it + /// to a single buffer will result in an exception. Similarly, if a stream is already converted to use a single larger buffer, then + /// it cannot grow beyond the limits of the maximum allowable array size. + /// + /// + /// Any method that modifies the stream has the potential to throw an OutOfMemoryException, either because + /// the stream is beyond the limits set in RecyclableStreamManager, or it would result in a buffer larger than + /// the maximum array size supported by .NET. + /// + /// + public sealed class RecyclableMemoryStream : MemoryStream, IBufferWriter + { + /// + /// All of these blocks must be the same size. + /// + private readonly List blocks; + + private readonly Guid id; + + private readonly RecyclableMemoryStreamManager memoryManager; + + private readonly string tag; + + private readonly long creationTimestamp; + + /// + /// This list is used to store buffers once they're replaced by something larger. + /// This is for the cases where you have users of this class that may hold onto the buffers longer + /// than they should and you want to prevent race conditions which could corrupt the data. + /// + private List dirtyBuffers; + + private bool disposed; + + /// + /// This is only set by GetBuffer() if the necessary buffer is larger than a single block size, or on + /// construction if the caller immediately requests a single large buffer. + /// + /// If this field is non-null, it contains the concatenation of the bytes found in the individual + /// blocks. Once it is created, this (or a larger) largeBuffer will be used for the life of the stream. + /// + private byte[] largeBuffer; + + /// + /// Gets unique identifier for this stream across its entire lifetime. + /// + /// Object has been disposed. + internal Guid Id + { + get + { + this.CheckDisposed(); + return this.id; + } + } + + /// + /// Gets a temporary identifier for the current usage of this stream. + /// + /// Object has been disposed. + internal string Tag + { + get + { + this.CheckDisposed(); + return this.tag; + } + } + + /// + /// Gets the memory manager being used by this stream. + /// + /// Object has been disposed. + internal RecyclableMemoryStreamManager MemoryManager + { + get + { + this.CheckDisposed(); + return this.memoryManager; + } + } + + /// + /// Gets call stack of the constructor. It is only set if is true, + /// which should only be in debugging situations. + /// + internal string AllocationStack { get; } + + /// + /// Gets call stack of the call. It is only set if is true, + /// which should only be in debugging situations. + /// + internal string DisposeStack { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager) + : this(memoryManager, Guid.NewGuid(), null, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id) + : this(memoryManager, id, null, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A string identifying this stream for logging and debugging purposes. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag) + : this(memoryManager, Guid.NewGuid(), tag, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag) + : this(memoryManager, id, tag, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag, long requestedSize) + : this(memoryManager, Guid.NewGuid(), tag, requestedSize, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize) + : this(memoryManager, id, tag, requestedSize, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + /// An initial buffer to use. This buffer will be owned by the stream and returned to the memory manager upon Dispose. + internal RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize, byte[] initialLargeBuffer) + : base(Array.Empty()) + { + this.memoryManager = memoryManager; + this.id = id; + this.tag = tag; + this.blocks = new List(); + this.creationTimestamp = Stopwatch.GetTimestamp(); + + long actualRequestedSize = Math.Max(requestedSize, this.memoryManager.OptionsValue.BlockSize); + + if (initialLargeBuffer == null) + { + this.EnsureCapacity(actualRequestedSize); + } + else + { + this.largeBuffer = initialLargeBuffer; + } + + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + this.AllocationStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamCreated(this.id, this.tag, requestedSize, actualRequestedSize); + this.memoryManager.ReportUsageReport(); + } + + /// + /// Finalizes an instance of the RecyclableMemoryStream class. + /// + /// Failing to dispose indicates a bug in the code using streams. Care should be taken to properly account for stream lifetime. + ~RecyclableMemoryStream() + { + this.Dispose(false); + } + + /// + /// Returns the memory used by this stream back to the pool. + /// + /// Whether we're disposing (true), or being called by the finalizer (false). + protected override void Dispose(bool disposing) + { + if (this.disposed) + { + string doubleDisposeStack = null; + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + doubleDisposeStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamDoubleDisposed(this.id, this.tag, this.AllocationStack, this.DisposeStack, doubleDisposeStack); + return; + } + + this.disposed = true; + TimeSpan lifetime = TimeSpan.FromTicks((Stopwatch.GetTimestamp() - this.creationTimestamp) * TimeSpan.TicksPerSecond / Stopwatch.Frequency); + + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + this.DisposeStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamDisposed(this.id, this.tag, lifetime, this.AllocationStack, this.DisposeStack); + + if (disposing) + { + GC.SuppressFinalize(this); + } + else + { + // We're being finalized. + this.memoryManager.ReportStreamFinalized(this.id, this.tag, this.AllocationStack); + + if (AppDomain.CurrentDomain.IsFinalizingForUnload()) + { + // If we're being finalized because of a shutdown, don't go any further. + // We have no idea what's already been cleaned up. Triggering events may cause + // a crash. + base.Dispose(disposing); + return; + } + } + + this.memoryManager.ReportStreamLength(this.length); + + if (this.largeBuffer != null) + { + this.memoryManager.ReturnLargeBuffer(this.largeBuffer, this.id, this.tag); + } + + if (this.dirtyBuffers != null) + { + foreach (byte[] buffer in this.dirtyBuffers) + { + this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); + } + } + + this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); + this.memoryManager.ReportUsageReport(); + this.blocks.Clear(); + + base.Dispose(disposing); + } + + /// + /// Equivalent to Dispose. + /// + public override void Close() + { + this.Dispose(true); + } + + /// + /// Gets or sets the capacity. + /// + /// + /// + /// Capacity is always in multiples of the memory manager's block size, unless + /// the large buffer is in use. Capacity never decreases during a stream's lifetime. + /// Explicitly setting the capacity to a lower value than the current value will have no effect. + /// This is because the buffers are all pooled by chunks and there's little reason to + /// allow stream truncation. + /// + /// + /// Writing past the current capacity will cause to automatically increase, until MaximumStreamCapacity is reached. + /// + /// + /// If the capacity is larger than int.MaxValue, then InvalidOperationException will be thrown. If you anticipate using + /// larger streams, use the property instead. + /// + /// + /// Object has been disposed. + /// Capacity is larger than int.MaxValue. + public override int Capacity + { + get + { + this.CheckDisposed(); + if (this.largeBuffer != null) + { + return this.largeBuffer.Length; + } + + long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; + if (size > int.MaxValue) + { + throw new InvalidOperationException($"{nameof(this.Capacity)} is larger than int.MaxValue. Use {nameof(this.Capacity64)} instead."); + } + + return (int)size; + } + + set => this.Capacity64 = value; + } + + /// + /// Gets or sets returns a 64-bit version of capacity, for streams larger than int.MaxValue in length. + /// + public long Capacity64 + { + get + { + this.CheckDisposed(); + if (this.largeBuffer != null) + { + return this.largeBuffer.Length; + } + + long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; + return size; + } + + set + { + this.CheckDisposed(); + this.EnsureCapacity(value); + } + } + + private long length; + + /// + /// Gets the number of bytes written to this stream. + /// + /// Object has been disposed. + /// If the buffer has already been converted to a large buffer, then the maximum length is limited by the maximum allowed array length in .NET. + public override long Length + { + get + { + this.CheckDisposed(); + return this.length; + } + } + + private long position; + + /// + /// Gets or sets the current position in the stream. + /// + /// Object has been disposed. + /// A negative value was passed. + /// Stream is in large-buffer mode, but an attempt was made to set the position past the maximum allowed array length. + /// If the buffer has already been converted to a large buffer, then the maximum length (and thus position) is limited by the maximum allowed array length in .NET. + public override long Position + { + get + { + this.CheckDisposed(); + return this.position; + } + + set + { + this.CheckDisposed(); + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); + } + + if (this.largeBuffer != null && value > RecyclableMemoryStreamManager.MaxArrayLength) + { + throw new InvalidOperationException($"Once the stream is converted to a single large buffer, position cannot be set past {RecyclableMemoryStreamManager.MaxArrayLength}."); + } + + this.position = value; + } + } + + /// + /// Gets a value indicating whether whether the stream can currently read. + /// + public override bool CanRead => !this.disposed; + + /// + /// Gets a value indicating whether whether the stream can currently seek. + /// + public override bool CanSeek => !this.disposed; + + /// + /// Gets a value indicating whether the steram can timeout. + /// + /// Always false + public override bool CanTimeout => false; + + /// + /// Gets a value indicating whether whether the stream can currently write. + /// + public override bool CanWrite => !this.disposed; + + /// + /// Returns a single buffer containing the contents of the stream. + /// The buffer may be longer than the stream length. + /// + /// A byte[] buffer. + /// IMPORTANT: Doing a after calling GetBuffer invalidates the buffer. The old buffer is held onto + /// until is called, but the next time GetBuffer is called, a new buffer from the pool will be required. + /// Object has been disposed. + /// stream is too large for a contiguous buffer. + public override byte[] GetBuffer() + { + this.CheckDisposed(); + + if (this.largeBuffer != null) + { + return this.largeBuffer; + } + + if (this.blocks.Count == 1) + { + return this.blocks[0]; + } + + // Buffer needs to reflect the capacity, not the length, because + // it's possible that people will manipulate the buffer directly + // and set the length afterward. Capacity sets the expectation + // for the size of the buffer. + byte[] newBuffer = this.memoryManager.GetLargeBuffer(this.Capacity64, this.id, this.tag); + + // InternalRead will check for existence of largeBuffer, so make sure we + // don't set it until after we've copied the data. + this.AssertLengthIsSmall(); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + this.largeBuffer = newBuffer; + + if (this.blocks.Count > 0 && this.memoryManager.OptionsValue.AggressiveBufferReturn) + { + this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); + this.blocks.Clear(); + } + + return this.largeBuffer; + } + +#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + /// + public override void CopyTo(Stream destination, int bufferSize) + { + this.WriteTo(destination, this.position, this.length - this.position); + } +#endif + + /// Asynchronously reads all the bytes from the current position in this stream and writes them to another stream. + /// The stream to which the contents of the current stream will be copied. + /// This parameter is ignored. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous copy operation. + /// + /// is . + /// Either the current stream or the destination stream is disposed. + /// The current stream does not support reading, or the destination stream does not support writing. + /// Similarly to MemoryStream's behavior, CopyToAsync will adjust the source stream's position by the number of bytes written to the destination stream, as a Read would do. + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(destination); +#else + if (destination == null) + { + throw new ArgumentNullException(nameof(destination)); + } +#endif + + this.CheckDisposed(); + + if (this.length == 0) + { + return Task.CompletedTask; + } + + long startPos = this.position; + long count = this.length - startPos; + this.position += count; + + if (destination is MemoryStream destinationRMS) + { + this.WriteTo(destinationRMS, startPos, count); + return Task.CompletedTask; + } + else + { + if (this.largeBuffer == null) + { + if (this.blocks.Count == 1) + { + this.AssertLengthIsSmall(); + return destination.WriteAsync(this.blocks[0], (int)startPos, (int)count, cancellationToken); + } + else + { + return CopyToAsyncImpl(destination, this.GetBlockAndRelativeOffset(startPos), count, this.blocks, cancellationToken); + } + } + else + { + this.AssertLengthIsSmall(); + return destination.WriteAsync(this.largeBuffer, (int)startPos, (int)count, cancellationToken); + } + } + + static async Task CopyToAsyncImpl(Stream destination, BlockAndOffset blockAndOffset, long count, List blocks, CancellationToken cancellationToken) + { + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + while (bytesRemaining > 0) + { + byte[] block = blocks[currentBlock]; + int amountToCopy = (int)Math.Min(block.Length - currentOffset, bytesRemaining); +#if NET8_0_OR_GREATER + await destination.WriteAsync(block.AsMemory(currentOffset, amountToCopy), cancellationToken); +#else + await destination.WriteAsync(block, currentOffset, amountToCopy, cancellationToken); +#endif + bytesRemaining -= amountToCopy; + ++currentBlock; + currentOffset = 0; + } + } + } + + private byte[] bufferWriterTempBuffer; + + /// + /// Notifies the stream that bytes were written to the buffer returned by or . + /// Seeks forward by bytes. + /// + /// + /// You must request a new buffer after calling Advance to continue writing more data and cannot write to a previously acquired buffer. + /// + /// How many bytes to advance. + /// Object has been disposed. + /// is negative. + /// is larger than the size of the previously requested buffer. + public void Advance(int count) + { + this.CheckDisposed(); + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} must be non-negative."); + } + + byte[] buffer = this.bufferWriterTempBuffer; + if (buffer != null) + { + if (count > buffer.Length) + { + throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {buffer.Length}."); + } + + this.Write(buffer, 0, count); + this.ReturnTempBuffer(buffer); + this.bufferWriterTempBuffer = null; + } + else + { + long bufferSize = this.largeBuffer == null + ? this.memoryManager.OptionsValue.BlockSize - this.GetBlockAndRelativeOffset(this.position).Offset + : this.largeBuffer.Length - this.position; + + if (count > bufferSize) + { + throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {bufferSize}."); + } + + this.position += count; + this.length = Math.Max(this.position, this.length); + } + } + + private void ReturnTempBuffer(byte[] buffer) + { + if (buffer.Length == this.memoryManager.OptionsValue.BlockSize) + { + this.memoryManager.ReturnBlock(buffer, this.id, this.tag); + } + else + { + this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); + } + } + + /// + /// + /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), + /// or setting Position after calling GetMemory() invalidates the memory. + /// + public Memory GetMemory(int sizeHint = 0) + { + return this.GetWritableBuffer(sizeHint); + } + + /// + /// + /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), + /// or setting Position after calling GetSpan() invalidates the span. + /// + public Span GetSpan(int sizeHint = 0) + { + return this.GetWritableBuffer(sizeHint); + } + + /// + /// When callers to GetSpan() or GetMemory() request a buffer that is larger than the remaining size of the current block + /// this method return a temp buffer. When Advance() is called, that temp buffer is then copied into the stream. + /// + private ArraySegment GetWritableBuffer(int sizeHint) + { + this.CheckDisposed(); + if (sizeHint < 0) + { + throw new ArgumentOutOfRangeException(nameof(sizeHint), $"{nameof(sizeHint)} must be non-negative."); + } + + int minimumBufferSize = Math.Max(sizeHint, 1); + + this.EnsureCapacity(this.position + minimumBufferSize); + if (this.bufferWriterTempBuffer != null) + { + this.ReturnTempBuffer(this.bufferWriterTempBuffer); + this.bufferWriterTempBuffer = null; + } + + if (this.largeBuffer != null) + { + return new ArraySegment(this.largeBuffer, (int)this.position, this.largeBuffer.Length - (int)this.position); + } + + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + int remainingBytesInBlock = this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset; + if (remainingBytesInBlock >= minimumBufferSize) + { + return new ArraySegment(this.blocks[blockAndOffset.Block], blockAndOffset.Offset, this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset); + } + + this.bufferWriterTempBuffer = minimumBufferSize > this.memoryManager.OptionsValue.BlockSize ? + this.memoryManager.GetLargeBuffer(minimumBufferSize, this.id, this.tag) : + this.memoryManager.GetBlock(); + + return new ArraySegment(this.bufferWriterTempBuffer); + } + + /// + /// Returns a sequence containing the contents of the stream. + /// + /// A ReadOnlySequence of bytes. + /// IMPORTANT: Calling Write(), GetMemory(), GetSpan(), Dispose(), or Close() after calling GetReadOnlySequence() invalidates the sequence. + /// Object has been disposed. + public ReadOnlySequence GetReadOnlySequence() + { + this.CheckDisposed(); + + if (this.largeBuffer != null) + { + this.AssertLengthIsSmall(); + return new ReadOnlySequence(this.largeBuffer, 0, (int)this.length); + } + + if (this.blocks.Count == 1) + { + this.AssertLengthIsSmall(); + return new ReadOnlySequence(this.blocks[0], 0, (int)this.length); + } + + BlockSegment first = new (this.blocks[0]); + BlockSegment last = first; + + for (int blockIdx = 1; last.RunningIndex + last.Memory.Length < this.length; blockIdx++) + { + last = last.Append(this.blocks[blockIdx]); + } + + return new ReadOnlySequence(first, 0, last, (int)(this.length - last.RunningIndex)); + } + + private sealed class BlockSegment : ReadOnlySequenceSegment + { + public BlockSegment(Memory memory) + { + this.Memory = memory; + } + + public BlockSegment Append(Memory memory) + { + BlockSegment nextSegment = new (memory) { RunningIndex = this.RunningIndex + this.Memory.Length }; + this.Next = nextSegment; + return nextSegment; + } + } + + /// + /// Returns an ArraySegment that wraps a single buffer containing the contents of the stream. + /// + /// An ArraySegment containing a reference to the underlying bytes. + /// Returns if a buffer can be returned; otherwise, . + public override bool TryGetBuffer(out ArraySegment buffer) + { + this.CheckDisposed(); + + try + { + if (this.length <= RecyclableMemoryStreamManager.MaxArrayLength) + { + buffer = new ArraySegment(this.GetBuffer(), 0, (int)this.Length); + return true; + } + } + catch (OutOfMemoryException) + { + } + +#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + buffer = ArraySegment.Empty; +#else + buffer = default; +#endif + return false; + } + + /// + /// Returns a new array with a copy of the buffer's contents. You should almost certainly be using combined with the to + /// access the bytes in this stream. Calling ToArray will destroy the benefits of pooled buffers, but it is included + /// for the sake of completeness. + /// + /// Object has been disposed. + /// The current object disallows ToArray calls. + /// The length of the stream is too long for a contiguous array. + /// Array of bytes +#pragma warning disable CS0809 + [Obsolete("This method has degraded performance vs. GetBuffer and should be avoided.")] + public override byte[] ToArray() + { + this.CheckDisposed(); + + string stack = this.memoryManager.OptionsValue.GenerateCallStacks ? Environment.StackTrace : null; + this.memoryManager.ReportStreamToArray(this.id, this.tag, stack, this.length); + + if (this.memoryManager.OptionsValue.ThrowExceptionOnToArray) + { + throw new NotSupportedException("The underlying RecyclableMemoryStreamManager is configured to not allow calls to ToArray."); + } + + byte[] newBuffer = new byte[this.Length]; + + Debug.Assert(this.length <= int.MaxValue); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + + return newBuffer; + } +#pragma warning restore CS0809 + + /// + /// Reads from the current position into the provided buffer. + /// + /// Destination buffer. + /// Offset into buffer at which to start placing the read bytes. + /// Number of bytes to read. + /// The number of bytes read. + /// buffer is null. + /// offset or count is less than 0. + /// offset subtracted from the buffer length is less than count. + /// Object has been disposed. + public override int Read(byte[] buffer, int offset, int count) + { + return this.SafeRead(buffer, offset, count, ref this.position); + } + + /// + /// Reads from the specified position into the provided buffer. + /// + /// Destination buffer. + /// Offset into buffer at which to start placing the read bytes. + /// Number of bytes to read. + /// Position in the stream to start reading from. + /// The number of bytes read. + /// is null. + /// or is less than 0. + /// subtracted from the buffer length is less than . + /// Object has been disposed. + public int SafeRead(byte[] buffer, int offset, int count, ref long streamPosition) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset), $"{nameof(offset)} cannot be negative."); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} cannot be negative."); + } + + if (offset + count > buffer.Length) + { + throw new ArgumentException($"{nameof(buffer)} length must be at least {nameof(offset)} + {nameof(count)}."); + } + + int amountRead = this.InternalRead(buffer, offset, count, streamPosition); + streamPosition += amountRead; + return amountRead; + } + + /// + /// Reads from the current position into the provided buffer. + /// + /// Destination buffer. + /// The number of bytes read. + /// Object has been disposed. +#if NETSTANDARD2_0 + public int Read(Span buffer) +#else + public override int Read(Span buffer) +#endif + { + return this.SafeRead(buffer, ref this.position); + } + + /// + /// Reads from the specified position into the provided buffer. + /// + /// Destination buffer. + /// Position in the stream to start reading from. + /// The number of bytes read. + /// Object has been disposed. + public int SafeRead(Span buffer, ref long streamPosition) + { + this.CheckDisposed(); + + int amountRead = this.InternalRead(buffer, streamPosition); + streamPosition += amountRead; + return amountRead; + } + + /// + /// Writes the buffer to the stream. + /// + /// Source buffer. + /// Start position. + /// Number of bytes to write. + /// buffer is null. + /// offset or count is negative. + /// buffer.Length - offset is not less than count. + /// Object has been disposed. + public override void Write(byte[] buffer, int offset, int count) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0) + { + throw new ArgumentOutOfRangeException( + nameof(offset), + offset, + $"{nameof(offset)} must be in the range of 0 - {nameof(buffer)}.{nameof(buffer.Length)}-1."); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), count, $"{nameof(count)} must be non-negative."); + } + + if (count + offset > buffer.Length) + { + throw new ArgumentException($"{nameof(count)} must be greater than {nameof(buffer)}.{nameof(buffer.Length)} - {nameof(offset)}."); + } + + int blockSize = this.memoryManager.OptionsValue.BlockSize; + long end = this.position + count; + + this.EnsureCapacity(end); + + if (this.largeBuffer == null) + { + int bytesRemaining = count; + int bytesWritten = 0; + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + + while (bytesRemaining > 0) + { + byte[] currentBlock = this.blocks[blockAndOffset.Block]; + int remainingInBlock = blockSize - blockAndOffset.Offset; + int amountToWriteInBlock = Math.Min(remainingInBlock, bytesRemaining); + + Buffer.BlockCopy( + buffer, + offset + bytesWritten, + currentBlock, + blockAndOffset.Offset, + amountToWriteInBlock); + + bytesRemaining -= amountToWriteInBlock; + bytesWritten += amountToWriteInBlock; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + } + else + { + Buffer.BlockCopy(buffer, offset, this.largeBuffer, (int)this.position, count); + } + + this.position = end; + this.length = Math.Max(this.position, this.length); + } + + /// + /// Writes the buffer to the stream. + /// + /// Source buffer. + /// buffer is null. + /// Object has been disposed. +#if NETSTANDARD2_0 + public void Write(ReadOnlySpan source) +#else + public override void Write(ReadOnlySpan source) +#endif + { + this.CheckDisposed(); + + int blockSize = this.memoryManager.OptionsValue.BlockSize; + long end = this.position + source.Length; + + this.EnsureCapacity(end); + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + + while (source.Length > 0) + { + byte[] currentBlock = this.blocks[blockAndOffset.Block]; + int remainingInBlock = blockSize - blockAndOffset.Offset; + int amountToWriteInBlock = Math.Min(remainingInBlock, source.Length); +#if NET8_0_OR_GREATER + source[..amountToWriteInBlock] + .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); + + source = source[amountToWriteInBlock..]; +#else + source.Slice(0, amountToWriteInBlock) + .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); + + source = source.Slice(amountToWriteInBlock); +#endif + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + } + else + { + source.CopyTo(this.largeBuffer.AsSpan((int)this.position)); + } + + this.position = end; + this.length = Math.Max(this.position, this.length); + } + + /// + /// Returns a useful string for debugging. This should not normally be called in actual production code. + /// + /// String with debug data. + public override string ToString() + { + if (!this.disposed) + { + return $"Id = {this.Id}, Tag = {this.Tag}, Length = {this.Length:N0} bytes"; + } + else + { + // Avoid properties because of the dispose check, but the fields themselves are not cleared. + return $"Disposed: Id = {this.id}, Tag = {this.tag}, Final Length: {this.length:N0} bytes"; + } + } + + /// + /// Writes a single byte to the current position in the stream. + /// + /// byte value to write. + /// Object has been disposed. + public override void WriteByte(byte value) + { + this.CheckDisposed(); + + long end = this.position + 1; + + if (this.largeBuffer == null) + { + int blockSize = this.memoryManager.OptionsValue.BlockSize; + + int block = (int)Math.DivRem(this.position, blockSize, out long index); + + if (block >= this.blocks.Count) + { + this.EnsureCapacity(end); + } + + this.blocks[block][index] = value; + } + else + { + if (this.position >= this.largeBuffer.Length) + { + this.EnsureCapacity(end); + } + + this.largeBuffer[this.position] = value; + } + + this.position = end; + + if (this.position > this.length) + { + this.length = this.position; + } + } + + /// + /// Reads a single byte from the current position in the stream. + /// + /// The byte at the current position, or -1 if the position is at the end of the stream. + /// Object has been disposed. + public override int ReadByte() + { + return this.SafeReadByte(ref this.position); + } + + /// + /// Reads a single byte from the specified position in the stream. + /// + /// The position in the stream to read from. + /// The byte at the current position, or -1 if the position is at the end of the stream. + /// Object has been disposed. + public int SafeReadByte(ref long streamPosition) + { + this.CheckDisposed(); + if (streamPosition == this.length) + { + return -1; + } + + byte value; + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(streamPosition); + value = this.blocks[blockAndOffset.Block][blockAndOffset.Offset]; + } + else + { + value = this.largeBuffer[streamPosition]; + } + + streamPosition++; + return value; + } + + /// + /// Sets the length of the stream. + /// + /// length of the stream + /// value is negative or larger than . + /// Object has been disposed. + public override void SetLength(long value) + { + this.CheckDisposed(); + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); + } + + this.EnsureCapacity(value); + + this.length = value; + if (this.position > value) + { + this.position = value; + } + } + + /// + /// Sets the position to the offset from the seek location. + /// + /// How many bytes to move. + /// From where. + /// The new position. + /// Object has been disposed. + /// is larger than . + /// Invalid seek origin. + /// Attempt to set negative position. + public override long Seek(long offset, SeekOrigin loc) + { + this.CheckDisposed(); + long newPosition = loc switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => offset + this.position, + SeekOrigin.End => offset + this.length, + _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), + }; + if (newPosition < 0) + { + throw new IOException("Seek before beginning."); + } + + this.position = newPosition; + return this.position; + } + + /// + /// Synchronously writes this stream's bytes to the argument stream. + /// + /// Destination stream. + /// Important: This does a synchronous write, which may not be desired in some situations. + /// is null. + /// Object has been disposed. + public override void WriteTo(Stream stream) + { + this.WriteTo(stream, 0, this.length); + } + + /// + /// Synchronously writes this stream's bytes, starting at offset, for count bytes, to the argument stream. + /// + /// Destination stream. + /// Offset in source. + /// Number of bytes to write. + /// is null. + /// + /// is less than 0, or + is beyond this 's length. + /// + /// Object has been disposed. + public void WriteTo(Stream stream, long offset, long count) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(stream); +#else + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } +#endif + + if (offset < 0 || offset + count > this.length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the {nameof(stream)}.", + innerException: null); + } + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[currentBlock]; + int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); + stream.Write(block, currentOffset, amountToCopy); + + bytesRemaining -= amountToCopy; + + ++currentBlock; + currentOffset = 0; + } + } + else + { + stream.Write(this.largeBuffer, (int)offset, (int)count); + } + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// The entire stream is written to the target array. + /// > is null. + /// Object has been disposed. + public void WriteTo(byte[] buffer) + { + this.WriteTo(buffer, 0, this.Length); + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// Offset in the source stream, from which to start. + /// Number of bytes to write. + /// > is null. + /// + /// is less than 0, or + is beyond this stream's length. + /// + /// Object has been disposed. + public void WriteTo(byte[] buffer, long offset, long count) + { + this.WriteTo(buffer, offset, count, 0); + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// Offset in the source stream, from which to start. + /// Number of bytes to write. + /// Offset in the target byte array to start writing + /// buffer is null + /// + /// is less than 0, or + is beyond this stream's length. + /// + /// + /// is less than 0, or + is beyond the target 's length. + /// + /// Object has been disposed. + public void WriteTo(byte[] buffer, long offset, long count, int targetOffset) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0 || offset + count > this.length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the stream.", + innerException: null); + } + + if (targetOffset < 0 || count + targetOffset > buffer.Length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(targetOffset)} must not be negative and {nameof(targetOffset)} + {nameof(count)} must not exceed the length of the target {nameof(buffer)}.", + innerException: null); + } + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + int currentTargetOffset = targetOffset; + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[currentBlock]; + int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); + Buffer.BlockCopy(block, currentOffset, buffer, currentTargetOffset, amountToCopy); + + bytesRemaining -= amountToCopy; + + ++currentBlock; + currentOffset = 0; + currentTargetOffset += amountToCopy; + } + } + else + { + this.AssertLengthIsSmall(); + Buffer.BlockCopy(this.largeBuffer, (int)offset, buffer, targetOffset, (int)count); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CheckDisposed() + { + if (this.disposed) + { + this.ThrowDisposedException(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void ThrowDisposedException() + { + throw new ObjectDisposedException($"The stream with Id {this.id} and Tag {this.tag} is disposed."); + } + + private int InternalRead(byte[] buffer, int offset, int count, long fromPosition) + { + if (this.length - fromPosition <= 0) + { + return 0; + } + + int amountToCopy; + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); + int bytesWritten = 0; + int bytesRemaining = (int)Math.Min(count, this.length - fromPosition); + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[blockAndOffset.Block]; + amountToCopy = Math.Min( + block.Length - blockAndOffset.Offset, + bytesRemaining); + Buffer.BlockCopy( + block, + blockAndOffset.Offset, + buffer, + bytesWritten + offset, + amountToCopy); + + bytesWritten += amountToCopy; + bytesRemaining -= amountToCopy; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + + return bytesWritten; + } + + amountToCopy = (int)Math.Min(count, this.length - fromPosition); + Buffer.BlockCopy(this.largeBuffer, (int)fromPosition, buffer, offset, amountToCopy); + return amountToCopy; + } + + private int InternalRead(Span buffer, long fromPosition) + { + if (this.length - fromPosition <= 0) + { + return 0; + } + + int amountToCopy; + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); + int bytesWritten = 0; + int bytesRemaining = (int)Math.Min(buffer.Length, this.length - fromPosition); + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[blockAndOffset.Block]; + amountToCopy = Math.Min( + block.Length - blockAndOffset.Offset, + bytesRemaining); +#if NET8_0_OR_GREATER + block.AsSpan(blockAndOffset.Offset, amountToCopy) + .CopyTo(buffer[bytesWritten..]); +#else + block.AsSpan(blockAndOffset.Offset, amountToCopy) + .CopyTo(buffer.Slice(bytesWritten)); +#endif + + bytesWritten += amountToCopy; + bytesRemaining -= amountToCopy; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + + return bytesWritten; + } + + amountToCopy = (int)Math.Min(buffer.Length, this.length - fromPosition); + this.largeBuffer.AsSpan((int)fromPosition, amountToCopy).CopyTo(buffer); + return amountToCopy; + } + + private struct BlockAndOffset + { + public int Block; + public int Offset; + + public BlockAndOffset(int block, int offset) + { + this.Block = block; + this.Offset = offset; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private BlockAndOffset GetBlockAndRelativeOffset(long offset) + { + int blockSize = this.memoryManager.OptionsValue.BlockSize; + int blockIndex = (int)Math.DivRem(offset, blockSize, out long offsetIndex); + return new BlockAndOffset(blockIndex, (int)offsetIndex); + } + + private void EnsureCapacity(long newCapacity) + { + if (newCapacity > this.memoryManager.OptionsValue.MaximumStreamCapacity && this.memoryManager.OptionsValue.MaximumStreamCapacity > 0) + { + this.memoryManager.ReportStreamOverCapacity(this.id, this.tag, newCapacity, this.AllocationStack); + + throw new OutOfMemoryException($"Requested capacity is too large: {newCapacity}. Limit is {this.memoryManager.OptionsValue.MaximumStreamCapacity}."); + } + + if (this.largeBuffer != null) + { + if (newCapacity > this.largeBuffer.Length) + { + byte[] newBuffer = this.memoryManager.GetLargeBuffer(newCapacity, this.id, this.tag); + Debug.Assert(this.length <= int.MaxValue); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + this.ReleaseLargeBuffer(); + this.largeBuffer = newBuffer; + } + } + else + { + // Let's save some re-allocation of the blocks list + long blocksRequired = (newCapacity / this.memoryManager.OptionsValue.BlockSize) + 1; + if (this.blocks.Capacity < blocksRequired) + { + this.blocks.Capacity = (int)blocksRequired; + } + + while (this.Capacity64 < newCapacity) + { + this.blocks.Add(this.memoryManager.GetBlock()); + } + } + } + + /// + /// Release the large buffer (either stores it for eventual release or returns it immediately). + /// + private void ReleaseLargeBuffer() + { + Debug.Assert(this.largeBuffer != null); + + if (this.memoryManager.OptionsValue.AggressiveBufferReturn) + { + this.memoryManager.ReturnLargeBuffer(this.largeBuffer!, this.id, this.tag); + } + else + { + // We most likely will only ever need space for one + this.dirtyBuffers ??= new List(1); + this.dirtyBuffers.Add(this.largeBuffer!); + } + + this.largeBuffer = null; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void AssertLengthIsSmall() + { + Debug.Assert(this.length <= int.MaxValue, "this.length was assumed to be <= Int32.MaxValue, but was larger."); + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs new file mode 100644 index 0000000000..7c450a418a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs @@ -0,0 +1,456 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + + /// + /// Wrapper for EventArgs + /// + public sealed partial class RecyclableMemoryStreamManager + { + /// + /// Arguments for the event. + /// + public sealed class StreamCreatedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets requested stream size. + /// + public long RequestedSize { get; } + + /// + /// Gets actual stream size. + /// + public long ActualSize { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// The requested stream size. + /// The actual stream size. + public StreamCreatedEventArgs(Guid guid, string tag, long requestedSize, long actualSize) + { + this.Id = guid; + this.Tag = tag; + this.RequestedSize = requestedSize; + this.ActualSize = actualSize; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamDisposedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Gets stack where stream was disposed. + /// + public string DisposeStack { get; } + + /// + /// Gets lifetime of the stream. + /// + public TimeSpan Lifetime { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Lifetime of the stream + /// Stack of original allocation. + /// Dispose stack. + public StreamDisposedEventArgs(Guid guid, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) + { + this.Id = guid; + this.Tag = tag; + this.Lifetime = lifetime; + this.AllocationStack = allocationStack; + this.DisposeStack = disposeStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamDoubleDisposedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Gets first dispose stack. + /// + public string DisposeStack1 { get; } + + /// + /// Gets second dispose stack. + /// + public string DisposeStack2 { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of original allocation. + /// First dispose stack. + /// Second dispose stack. + public StreamDoubleDisposedEventArgs(Guid guid, string tag, string allocationStack, string disposeStack1, string disposeStack2) + { + this.Id = guid; + this.Tag = tag; + this.AllocationStack = allocationStack; + this.DisposeStack1 = disposeStack1; + this.DisposeStack2 = disposeStack2; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamFinalizedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of original allocation. + public StreamFinalizedEventArgs(Guid guid, string tag, string allocationStack) + { + this.Id = guid; + this.Tag = tag; + this.AllocationStack = allocationStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamConvertedToArrayEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where ToArray was called. + /// + public string Stack { get; } + + /// + /// Gets length of stack. + /// + public long Length { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of ToArray call. + /// Length of stream. + public StreamConvertedToArrayEventArgs(Guid guid, string tag, string stack, long length) + { + this.Id = guid; + this.Tag = tag; + this.Stack = stack; + this.Length = length; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamOverCapacityEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets original allocation stack. + /// + public string AllocationStack { get; } + + /// + /// Gets requested capacity. + /// + public long RequestedCapacity { get; } + + /// + /// Gets maximum capacity. + /// + public long MaximumCapacity { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Requested capacity. + /// Maximum stream capacity of the manager. + /// Original allocation stack. + internal StreamOverCapacityEventArgs(Guid guid, string tag, long requestedCapacity, long maximumCapacity, string allocationStack) + { + this.Id = guid; + this.Tag = tag; + this.RequestedCapacity = requestedCapacity; + this.MaximumCapacity = maximumCapacity; + this.AllocationStack = allocationStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class BlockCreatedEventArgs : EventArgs + { + /// + /// Gets how many bytes are currently in use from the small pool. + /// + public long SmallPoolInUse { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Number of bytes currently in use from the small pool. + internal BlockCreatedEventArgs(long smallPoolInUse) + { + this.SmallPoolInUse = smallPoolInUse; + } + } + + /// + /// Arguments for the events. + /// + public sealed class LargeBufferCreatedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets a value indicating whether whether the buffer was satisfied from the pool or not. + /// + public bool Pooled { get; } + + /// + /// Gets required buffer size. + /// + public long RequiredSize { get; } + + /// + /// Gets how many bytes are in use from the large pool. + /// + public long LargePoolInUse { get; } + + /// + /// Gets if the buffer was not satisfied from the pool, and is turned on, then. + /// this will contain the call stack of the allocation request. + /// + public string CallStack { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Required size of the new buffer. + /// How many bytes from the large pool are currently in use. + /// Whether the buffer was satisfied from the pool or not. + /// Call stack of the allocation, if it wasn't pooled. + internal LargeBufferCreatedEventArgs(Guid guid, string tag, long requiredSize, long largePoolInUse, bool pooled, string callStack) + { + this.RequiredSize = requiredSize; + this.LargePoolInUse = largePoolInUse; + this.Pooled = pooled; + this.Id = guid; + this.Tag = tag; + this.CallStack = callStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class BufferDiscardedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets type of the buffer. + /// + public Events.MemoryStreamBufferType BufferType { get; } + + /// + /// Gets the reason this buffer was discarded. + /// + public Events.MemoryStreamDiscardReason Reason { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Type of buffer being discarded. + /// The reason for the discard. + internal BufferDiscardedEventArgs(Guid guid, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) + { + this.Id = guid; + this.Tag = tag; + this.BufferType = bufferType; + this.Reason = reason; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamLengthEventArgs : EventArgs + { + /// + /// Gets length of the stream. + /// + public long Length { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Length of the strength. + public StreamLengthEventArgs(long length) + { + this.Length = length; + } + } + + /// + /// Arguments for the event. + /// + public sealed class UsageReportEventArgs : EventArgs + { + /// + /// Gets bytes from the small pool currently in use. + /// + public long SmallPoolInUseBytes { get; } + + /// + /// Gets bytes from the small pool currently available. + /// + public long SmallPoolFreeBytes { get; } + + /// + /// Gets bytes from the large pool currently in use. + /// + public long LargePoolInUseBytes { get; } + + /// + /// Gets bytes from the large pool currently available. + /// + public long LargePoolFreeBytes { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Bytes from the small pool currently in use. + /// Bytes from the small pool currently available. + /// Bytes from the large pool currently in use. + /// Bytes from the large pool currently available. + public UsageReportEventArgs( + long smallPoolInUseBytes, + long smallPoolFreeBytes, + long largePoolInUseBytes, + long largePoolFreeBytes) + { + this.SmallPoolInUseBytes = smallPoolInUseBytes; + this.SmallPoolFreeBytes = smallPoolFreeBytes; + this.LargePoolInUseBytes = largePoolInUseBytes; + this.LargePoolFreeBytes = largePoolFreeBytes; + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs new file mode 100644 index 0000000000..81a24f9de8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs @@ -0,0 +1,288 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// --------------------------------------------------------------------- +// Copyright (c) 2015 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// --------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Diagnostics.Tracing; + + /// + /// Holder for Events + /// + public sealed partial class RecyclableMemoryStreamManager + { + /// + /// ETW events for RecyclableMemoryStream. + /// + [EventSource(Name = "Microsoft-IO-RecyclableMemoryStream", Guid = "{B80CD4E4-890E-468D-9CBA-90EB7C82DFC7}")] + public sealed class Events : EventSource + { + /// + /// Static log object, through which all events are written. + /// +#pragma warning disable SA1401 // Fields should be private +#pragma warning disable CA2211 // Non-constant fields should not be visible + public static Events Writer = new (); +#pragma warning restore CA2211 // Non-constant fields should not be visible +#pragma warning restore SA1401 // Fields should be private + + /// + /// Type of buffer. + /// + public enum MemoryStreamBufferType + { + /// + /// Small block buffer. + /// + Small, + + /// + /// Large pool buffer. + /// + Large, + } + + /// + /// The possible reasons for discarding a buffer. + /// + public enum MemoryStreamDiscardReason + { + /// + /// Buffer was too large to be re-pooled. + /// + TooLarge, + + /// + /// There are enough free bytes in the pool. + /// + EnoughFree, + } + + /// + /// Logged when a stream object is created. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Requested size of the stream. + /// Actual size given to the stream from the pool. + [Event(1, Level = EventLevel.Verbose, Version = 2)] + public void MemoryStreamCreated(Guid guid, string tag, long requestedSize, long actualSize) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(1, guid, tag ?? string.Empty, requestedSize, actualSize); + } + } + + /// + /// Logged when the stream is disposed. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Lifetime in milliseconds of the stream + /// Call stack of initial allocation. + /// Call stack of the dispose. + [Event(2, Level = EventLevel.Verbose, Version = 3)] + public void MemoryStreamDisposed(Guid guid, string tag, long lifetimeMs, string allocationStack, string disposeStack) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(2, guid, tag ?? string.Empty, lifetimeMs, allocationStack ?? string.Empty, disposeStack ?? string.Empty); + } + } + + /// + /// Logged when the stream is disposed for the second time. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of initial allocation. + /// Call stack of the first dispose. + /// Call stack of the second dispose. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(3, Level = EventLevel.Critical)] + public void MemoryStreamDoubleDispose( + Guid guid, + string tag, + string allocationStack, + string disposeStack1, + string disposeStack2) + { + if (this.IsEnabled()) + { + this.WriteEvent( + 3, + guid, + tag ?? string.Empty, + allocationStack ?? string.Empty, + disposeStack1 ?? string.Empty, + disposeStack2 ?? string.Empty); + } + } + + /// + /// Logged when a stream is finalized. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of initial allocation. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(4, Level = EventLevel.Error)] + public void MemoryStreamFinalized(Guid guid, string tag, string allocationStack) + { + if (this.IsEnabled()) + { + this.WriteEvent(4, guid, tag ?? string.Empty, allocationStack ?? string.Empty); + } + } + + /// + /// Logged when ToArray is called on a stream. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of the ToArray call. + /// Length of stream. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(5, Level = EventLevel.Verbose, Version = 2)] + public void MemoryStreamToArray(Guid guid, string tag, string stack, long size) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(5, guid, tag ?? string.Empty, stack ?? string.Empty, size); + } + } + + /// + /// Logged when the RecyclableMemoryStreamManager is initialized. + /// + /// Size of blocks, in bytes. + /// Size of the large buffer multiple, in bytes. + /// Maximum buffer size, in bytes. + [Event(6, Level = EventLevel.Informational)] + public void MemoryStreamManagerInitialized(int blockSize, int largeBufferMultiple, int maximumBufferSize) + { + if (this.IsEnabled()) + { + this.WriteEvent(6, blockSize, largeBufferMultiple, maximumBufferSize); + } + } + + /// + /// Logged when a new block is created. + /// + /// Number of bytes in the small pool currently in use. + [Event(7, Level = EventLevel.Warning, Version = 2)] + public void MemoryStreamNewBlockCreated(long smallPoolInUseBytes) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(7, smallPoolInUseBytes); + } + } + + /// + /// Logged when a new large buffer is created. + /// + /// Requested size. + /// Number of bytes in the large pool in use. + [Event(8, Level = EventLevel.Warning, Version = 3)] + public void MemoryStreamNewLargeBufferCreated(long requiredSize, long largePoolInUseBytes) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(8, requiredSize, largePoolInUseBytes); + } + } + + /// + /// Logged when a buffer is created that is too large to pool. + /// + /// Unique stream ID. + /// A temporary ID for this stream, usually indicates current usage. + /// Size requested by the caller. + /// Call stack of the requested stream. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(9, Level = EventLevel.Verbose, Version = 3)] + public void MemoryStreamNonPooledLargeBufferCreated(Guid guid, string tag, long requiredSize, string allocationStack) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(9, guid, tag ?? string.Empty, requiredSize, allocationStack ?? string.Empty); + } + } + + /// + /// Logged when a buffer is discarded (not put back in the pool, but given to GC to clean up). + /// + /// Unique stream ID. + /// A temporary ID for this stream, usually indicates current usage. + /// Type of the buffer being discarded. + /// Reason for the discard. + /// Number of free small pool blocks. + /// Bytes free in the small pool. + /// Bytes in use from the small pool. + /// Number of free large pool blocks. + /// Bytes free in the large pool. + /// Bytes in use from the large pool. + [Event(10, Level = EventLevel.Warning, Version = 2)] + public void MemoryStreamDiscardBuffer( + Guid guid, + string tag, + MemoryStreamBufferType bufferType, + MemoryStreamDiscardReason reason, + long smallBlocksFree, + long smallPoolBytesFree, + long smallPoolBytesInUse, + long largeBlocksFree, + long largePoolBytesFree, + long largePoolBytesInUse) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(10, guid, tag ?? string.Empty, bufferType, reason, smallBlocksFree, smallPoolBytesFree, smallPoolBytesInUse, largeBlocksFree, largePoolBytesFree, largePoolBytesInUse); + } + } + + /// + /// Logged when a stream grows beyond the maximum capacity. + /// + /// Unique stream ID + /// A temporary ID for this stream, usually indicates current usage. + /// The requested capacity. + /// Maximum capacity, as configured by RecyclableMemoryStreamManager. + /// Call stack for the capacity request. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(11, Level = EventLevel.Error, Version = 3)] + public void MemoryStreamOverCapacity(Guid guid, string tag, long requestedCapacity, long maxCapacity, string allocationStack) + { + if (this.IsEnabled()) + { + this.WriteEvent(11, guid, tag ?? string.Empty, requestedCapacity, maxCapacity, allocationStack ?? string.Empty); + } + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs new file mode 100644 index 0000000000..8348ccc809 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs @@ -0,0 +1,989 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// --------------------------------------------------------------------- +// Copyright (c) 2015-2016 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// --------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Runtime.CompilerServices; + using System.Threading; + using Microsoft.Extensions.Logging; + + /// + /// Manages pools of objects. + /// + /// + /// + /// There are two pools managed in here. The small pool contains same-sized buffers that are handed to streams + /// as they write more data. + /// + /// + /// For scenarios that need to call , the large pool contains buffers of various sizes, all + /// multiples/exponentials of (1 MB by default). They are split by size to avoid overly-wasteful buffer + /// usage. There should be far fewer 8 MB buffers than 1 MB buffers, for example. + /// + /// + public partial class RecyclableMemoryStreamManager + { + /// + /// Maximum length of a single array. + /// + /// See documentation at https://docs.microsoft.com/dotnet/api/system.array?view=netcore-3.1 + /// + internal const int MaxArrayLength = 0X7FFFFFC7; + + /// + /// Default block size, in bytes. + /// + public const int DefaultBlockSize = 128 * 1024; + + /// + /// Default large buffer multiple, in bytes. + /// + public const int DefaultLargeBufferMultiple = 1024 * 1024; + + /// + /// Default maximum buffer size, in bytes. + /// + public const int DefaultMaximumBufferSize = 128 * 1024 * 1024; + + // 0 to indicate unbounded + private const long DefaultMaxSmallPoolFreeBytes = 0L; + private const long DefaultMaxLargePoolFreeBytes = 0L; + + private readonly long[] largeBufferFreeSize; + private readonly long[] largeBufferInUseSize; + + private readonly ConcurrentStack[] largePools; + + private readonly ConcurrentStack smallPool; + +#pragma warning disable SA1401 // Fields should be private - performance reasons + internal readonly Options OptionsValue; +#pragma warning restore SA1401 // Fields should be private + + private long smallPoolFreeSize; + private long smallPoolInUseSize; + + /// + /// Gets settings for controlling the behavior of RecyclableMemoryStream + /// + public Options Settings => this.OptionsValue; + + /// + /// Gets number of bytes in small pool not currently in use. + /// + public long SmallPoolFreeSize => this.smallPoolFreeSize; + + /// + /// Gets number of bytes currently in use by stream from the small pool. + /// + public long SmallPoolInUseSize => this.smallPoolInUseSize; + + /// + /// Gets number of bytes in large pool not currently in use. + /// + public long LargePoolFreeSize + { + get + { + long sum = 0; + foreach (long freeSize in this.largeBufferFreeSize) + { + sum += freeSize; + } + + return sum; + } + } + + /// + /// Gets number of bytes currently in use by streams from the large pool. + /// + public long LargePoolInUseSize + { + get + { + long sum = 0; + foreach (long inUseSize in this.largeBufferInUseSize) + { + sum += inUseSize; + } + + return sum; + } + } + + /// + /// Gets how many blocks are in the small pool. + /// + public long SmallBlocksFree => this.smallPool.Count; + + /// + /// Gets how many buffers are in the large pool. + /// + public long LargeBuffersFree + { + get + { + long free = 0; + foreach (ConcurrentStack pool in this.largePools) + { + free += pool.Count; + } + + return free; + } + } + + /// + /// Parameters for customizing the behavior of + /// + public class Options + { + /// + /// Gets or sets the size of the pooled blocks. This must be greater than 0. + /// + /// The default size 131,072 (128KB) + public int BlockSize { get; set; } = DefaultBlockSize; + + /// + /// Gets or sets each large buffer will be a multiple exponential of this value + /// + /// The default value is 1,048,576 (1MB) + public int LargeBufferMultiple { get; set; } = DefaultLargeBufferMultiple; + + /// + /// Gets or sets buffer beyond this length are not pooled. + /// + /// The default value is 134,217,728 (128MB) + public int MaximumBufferSize { get; set; } = DefaultMaximumBufferSize; + + /// + /// Gets or sets maximum number of bytes to keep available in the small pool. + /// + /// + /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. + /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. + /// + public long MaximumSmallPoolFreeBytes { get; set; } + + /// + /// Gets or sets maximum number of bytes to keep available in the large pools. + /// + /// + /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. + /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. + /// + public long MaximumLargePoolFreeBytes { get; set; } + + /// + /// Gets or sets a value indicating whether whether to use the exponential allocation strategy (see documentation). + /// + /// The default value is false. + public bool UseExponentialLargeBuffer { get; set; } = false; + + /// + /// Gets or sets maximum stream capacity in bytes. Attempts to set a larger capacity will + /// result in an exception. + /// + /// The default value of 0 indicates no limit. + public long MaximumStreamCapacity { get; set; } = 0; + + /// + /// Gets or sets a value indicating whether whether to save call stacks for stream allocations. This can help in debugging. + /// It should NEVER be turned on generally in production. + /// + public bool GenerateCallStacks { get; set; } = false; + + /// + /// Gets or sets a value indicating whether whether dirty buffers can be immediately returned to the buffer pool. + /// + /// + /// + /// When is called on a stream and creates a single large buffer, if this setting is enabled, the other blocks will be returned + /// to the buffer pool immediately. + /// + /// + /// Note when enabling this setting that the user is responsible for ensuring that any buffer previously + /// retrieved from a stream which is subsequently modified is not used after modification (as it may no longer + /// be valid). + /// + /// + public bool AggressiveBufferReturn { get; set; } = false; + + /// + /// Gets or sets a value indicating whether causes an exception to be thrown if is ever called. + /// + /// Calling defeats the purpose of a pooled buffer. Use this property to discover code that is calling . If this is + /// set and is called, a NotSupportedException will be thrown. + public bool ThrowExceptionOnToArray { get; set; } = false; + + /// + /// Gets or sets a value indicating whether zero out buffers on allocation and before returning them to the pool. + /// + /// Setting this to true causes a performance hit and should only be set if one wants to avoid accidental data leaks. + public bool ZeroOutBuffer { get; set; } = false; + + /// + /// Creates a new object. + /// + public Options() + { + } + + /// + /// Creates a new object with the most common options. + /// + /// Size of the blocks in the small pool. + /// Size of the large buffer multiple + /// Maximum poolable buffer size. + /// Maximum bytes to hold in the small pool. + /// Maximum bytes to hold in each of the large pools. + public Options(int blockSize, int largeBufferMultiple, int maximumBufferSize, long maximumSmallPoolFreeBytes, long maximumLargePoolFreeBytes) + { + this.BlockSize = blockSize; + this.LargeBufferMultiple = largeBufferMultiple; + this.MaximumBufferSize = maximumBufferSize; + this.MaximumSmallPoolFreeBytes = maximumSmallPoolFreeBytes; + this.MaximumLargePoolFreeBytes = maximumLargePoolFreeBytes; + } + } + + /// + /// Initializes the memory manager with the default block/buffer specifications. This pool may have unbounded growth unless you modify . + /// + public RecyclableMemoryStreamManager() + : this(new Options()) + { + } + + /// + /// Initializes the memory manager with the given block requiredSize. + /// + /// Object specifying options for stream behavior. + /// + /// is not a positive number, + /// or is not a positive number, + /// or is less than options.BlockSize, + /// or is negative, + /// or is negative, + /// or is not a multiple/exponential of . + /// + public RecyclableMemoryStreamManager(Options options) + { + if (options.BlockSize <= 0) + { + throw new InvalidOperationException($"{nameof(options.BlockSize)} must be a positive number"); + } + + if (options.LargeBufferMultiple <= 0) + { + throw new InvalidOperationException($"{nameof(options.LargeBufferMultiple)} must be a positive number"); + } + + if (options.MaximumBufferSize < options.BlockSize) + { + throw new InvalidOperationException($"{nameof(options.MaximumBufferSize)} must be at least {nameof(options.BlockSize)}"); + } + + if (options.MaximumSmallPoolFreeBytes < 0) + { + throw new InvalidOperationException($"{nameof(options.MaximumSmallPoolFreeBytes)} must be non-negative"); + } + + if (options.MaximumLargePoolFreeBytes < 0) + { + throw new InvalidOperationException($"{nameof(options.MaximumLargePoolFreeBytes)} must be non-negative"); + } + + this.OptionsValue = options; + + if (!this.IsLargeBufferSize(options.MaximumBufferSize)) + { + throw new InvalidOperationException( + $"{nameof(options.MaximumBufferSize)} is not {(options.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {nameof(options.LargeBufferMultiple)}."); + } + + this.smallPool = new ConcurrentStack(); + int numLargePools = options.UseExponentialLargeBuffer + ? (int)Math.Log(options.MaximumBufferSize / options.LargeBufferMultiple, 2) + 1 + : options.MaximumBufferSize / options.LargeBufferMultiple; + + // +1 to store size of bytes in use that are too large to be pooled + this.largeBufferInUseSize = new long[numLargePools + 1]; + this.largeBufferFreeSize = new long[numLargePools]; + + this.largePools = new ConcurrentStack[numLargePools]; + + for (int i = 0; i < this.largePools.Length; ++i) + { + this.largePools[i] = new ConcurrentStack(); + } + + Events.Writer.MemoryStreamManagerInitialized(options.BlockSize, options.LargeBufferMultiple, options.MaximumBufferSize); + } + + /// + /// Removes and returns a single block from the pool. + /// + /// A byte[] array. + internal byte[] GetBlock() + { + Interlocked.Add(ref this.smallPoolInUseSize, this.OptionsValue.BlockSize); + + if (!this.smallPool.TryPop(out byte[] block)) + { + // We'll add this back to the pool when the stream is disposed + // (unless our free pool is too large) +#if NET6_0_OR_GREATER + block = this.OptionsValue.ZeroOutBuffer ? GC.AllocateArray(this.OptionsValue.BlockSize) : GC.AllocateUninitializedArray(this.OptionsValue.BlockSize); +#else + block = new byte[this.OptionsValue.BlockSize]; +#endif + this.ReportBlockCreated(); + } + else + { + Interlocked.Add(ref this.smallPoolFreeSize, -this.OptionsValue.BlockSize); + } + + return block; + } + + /// + /// Returns a buffer of arbitrary size from the large buffer pool. This buffer + /// will be at least the requiredSize and always be a multiple/exponential of largeBufferMultiple. + /// + /// The minimum length of the buffer. + /// Unique ID for the stream. + /// The tag of the stream returning this buffer, for logging if necessary. + /// A buffer of at least the required size. + /// Requested array size is larger than the maximum allowed. + internal byte[] GetLargeBuffer(long requiredSize, Guid id, string tag) + { + requiredSize = this.RoundToLargeBufferSize(requiredSize); + + if (requiredSize > MaxArrayLength) + { + throw new OutOfMemoryException($"Required buffer size exceeds maximum array length of {MaxArrayLength}."); + } + + int poolIndex = this.GetPoolIndex(requiredSize); + + bool createdNew = false; + bool pooled = true; + string callStack = null; + + byte[] buffer; + if (poolIndex < this.largePools.Length) + { + if (!this.largePools[poolIndex].TryPop(out buffer)) + { + buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); + createdNew = true; + } + else + { + Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], -buffer.Length); + } + } + else + { + // Buffer is too large to pool. They get a new buffer. + + // We still want to track the size, though, and we've reserved a slot + // in the end of the in-use array for non-pooled bytes in use. + poolIndex = this.largeBufferInUseSize.Length - 1; + + // We still want to round up to reduce heap fragmentation. + buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); + if (this.OptionsValue.GenerateCallStacks) + { + // Grab the stack -- we want to know who requires such large buffers + callStack = Environment.StackTrace; + } + + createdNew = true; + pooled = false; + } + + Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], buffer.Length); + if (createdNew) + { + this.ReportLargeBufferCreated(id, tag, requiredSize, pooled: pooled, callStack); + } + + return buffer; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static byte[] AllocateArray(long requiredSize, bool zeroInitializeArray) => +#if NET6_0_OR_GREATER + zeroInitializeArray ? GC.AllocateArray((int)requiredSize) : GC.AllocateUninitializedArray((int)requiredSize); +#else + new byte[requiredSize]; +#endif + } + + private long RoundToLargeBufferSize(long requiredSize) + { + if (this.OptionsValue.UseExponentialLargeBuffer) + { + long pow = 1; + while (this.OptionsValue.LargeBufferMultiple * pow < requiredSize) + { + pow <<= 1; + } + + return this.OptionsValue.LargeBufferMultiple * pow; + } + else + { + return (requiredSize + this.OptionsValue.LargeBufferMultiple - 1) / this.OptionsValue.LargeBufferMultiple * this.OptionsValue.LargeBufferMultiple; + } + } + + private bool IsLargeBufferSize(int value) + { + return value != 0 && (this.OptionsValue.UseExponentialLargeBuffer + ? value == this.RoundToLargeBufferSize(value) + : value % this.OptionsValue.LargeBufferMultiple == 0); + } + + private int GetPoolIndex(long length) + { + if (this.OptionsValue.UseExponentialLargeBuffer) + { + int index = 0; + while (this.OptionsValue.LargeBufferMultiple << index < length) + { + ++index; + } + + return index; + } + else + { + return (int)((length / this.OptionsValue.LargeBufferMultiple) - 1); + } + } + + /// + /// Returns the buffer to the large pool. + /// + /// The buffer to return. + /// Unique stream ID. + /// The tag of the stream returning this buffer, for logging if necessary. + /// is null. + /// buffer.Length is not a multiple/exponential of (it did not originate from this pool). + internal void ReturnLargeBuffer(byte[] buffer, Guid id, string tag) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (!this.IsLargeBufferSize(buffer.Length)) + { + throw new ArgumentException($"{nameof(buffer)} did not originate from this memory manager. The size is not " + + $"{(this.OptionsValue.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {this.OptionsValue.LargeBufferMultiple}."); + } + + this.ZeroOutMemoryIfEnabled(buffer); + int poolIndex = this.GetPoolIndex(buffer.Length); + if (poolIndex < this.largePools.Length) + { + if ((this.largePools[poolIndex].Count + 1) * buffer.Length <= this.OptionsValue.MaximumLargePoolFreeBytes || + this.OptionsValue.MaximumLargePoolFreeBytes == 0) + { + this.largePools[poolIndex].Push(buffer); + Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], buffer.Length); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.EnoughFree); + } + } + else + { + // This is a non-poolable buffer, but we still want to track its size for in-use + // analysis. We have space in the InUse array for this. + poolIndex = this.largeBufferInUseSize.Length - 1; + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.TooLarge); + } + + Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], -buffer.Length); + } + + /// + /// Returns the blocks to the pool. + /// + /// Collection of blocks to return to the pool. + /// Unique Stream ID. + /// The tag of the stream returning these blocks, for logging if necessary. + /// is null. + /// contains buffers that are the wrong size (or null) for this memory manager. + internal void ReturnBlocks(List blocks, Guid id, string tag) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(blocks); +#else + if (blocks == null) + { + throw new ArgumentNullException(nameof(blocks)); + } +#endif + + long bytesToReturn = blocks.Count * (long)this.OptionsValue.BlockSize; + Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); + + foreach (byte[] block in blocks) + { + if (block == null || block.Length != this.OptionsValue.BlockSize) + { + throw new ArgumentException($"{nameof(blocks)} contains buffers that are not {nameof(this.OptionsValue.BlockSize)} in length.", nameof(blocks)); + } + } + + foreach (byte[] block in blocks) + { + this.ZeroOutMemoryIfEnabled(block); + if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) + { + Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); + this.smallPool.Push(block); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); + break; + } + } + } + + /// + /// Returns a block to the pool. + /// + /// Block to return to the pool. + /// Unique Stream ID. + /// The tag of the stream returning this, for logging if necessary. + /// is null. + /// is the wrong size for this memory manager. + internal void ReturnBlock(byte[] block, Guid id, string tag) + { + int bytesToReturn = this.OptionsValue.BlockSize; + Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); + +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(block); +#else + if (block == null) + { + throw new ArgumentNullException(nameof(block)); + } +#endif + + if (block.Length != this.OptionsValue.BlockSize) + { + throw new ArgumentException($"{nameof(block)} is not not {nameof(this.OptionsValue.BlockSize)} in length."); + } + + this.ZeroOutMemoryIfEnabled(block); + if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) + { + Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); + this.smallPool.Push(block); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ZeroOutMemoryIfEnabled(byte[] buffer) + { + if (this.OptionsValue.ZeroOutBuffer) + { +#if NET6_0_OR_GREATER + Array.Clear(buffer); +#else + Array.Clear(buffer, 0, buffer.Length); +#endif + } + } + + internal void ReportBlockCreated() + { + Events.Writer.MemoryStreamNewBlockCreated(this.smallPoolInUseSize); + this.BlockCreated?.Invoke(this, new BlockCreatedEventArgs(this.smallPoolInUseSize)); + } + + internal void ReportLargeBufferCreated(Guid id, string tag, long requiredSize, bool pooled, string callStack) + { + if (pooled) + { + Events.Writer.MemoryStreamNewLargeBufferCreated(requiredSize, this.LargePoolInUseSize); + } + else + { + Events.Writer.MemoryStreamNonPooledLargeBufferCreated(id, tag, requiredSize, callStack); + } + + this.LargeBufferCreated?.Invoke(this, new LargeBufferCreatedEventArgs(id, tag, requiredSize, this.LargePoolInUseSize, pooled, callStack)); + } + + internal void ReportBufferDiscarded(Guid id, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) + { + Events.Writer.MemoryStreamDiscardBuffer( + id, + tag, + bufferType, + reason, + this.SmallBlocksFree, + this.smallPoolFreeSize, + this.smallPoolInUseSize, + this.LargeBuffersFree, + this.LargePoolFreeSize, + this.LargePoolInUseSize); + this.BufferDiscarded?.Invoke(this, new BufferDiscardedEventArgs(id, tag, bufferType, reason)); + } + + internal void ReportStreamCreated(Guid id, string tag, long requestedSize, long actualSize) + { + Events.Writer.MemoryStreamCreated(id, tag, requestedSize, actualSize); + this.StreamCreated?.Invoke(this, new StreamCreatedEventArgs(id, tag, requestedSize, actualSize)); + } + + internal void ReportStreamDisposed(Guid id, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) + { + Events.Writer.MemoryStreamDisposed(id, tag, (long)lifetime.TotalMilliseconds, allocationStack, disposeStack); + this.StreamDisposed?.Invoke(this, new StreamDisposedEventArgs(id, tag, lifetime, allocationStack, disposeStack)); + } + + internal void ReportStreamDoubleDisposed(Guid id, string tag, string allocationStack, string disposeStack1, string disposeStack2) + { + Events.Writer.MemoryStreamDoubleDispose(id, tag, allocationStack, disposeStack1, disposeStack2); + this.StreamDoubleDisposed?.Invoke(this, new StreamDoubleDisposedEventArgs(id, tag, allocationStack, disposeStack1, disposeStack2)); + } + + internal void ReportStreamFinalized(Guid id, string tag, string allocationStack) + { + Events.Writer.MemoryStreamFinalized(id, tag, allocationStack); + this.StreamFinalized?.Invoke(this, new StreamFinalizedEventArgs(id, tag, allocationStack)); + } + + internal void ReportStreamLength(long bytes) + { + this.StreamLength?.Invoke(this, new StreamLengthEventArgs(bytes)); + } + + internal void ReportStreamToArray(Guid id, string tag, string stack, long length) + { + Events.Writer.MemoryStreamToArray(id, tag, stack, length); + this.StreamConvertedToArray?.Invoke(this, new StreamConvertedToArrayEventArgs(id, tag, stack, length)); + } + + internal void ReportStreamOverCapacity(Guid id, string tag, long requestedCapacity, string allocationStack) + { + Events.Writer.MemoryStreamOverCapacity(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack); + this.StreamOverCapacity?.Invoke(this, new StreamOverCapacityEventArgs(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack)); + } + + internal void ReportUsageReport() + { + this.UsageReport?.Invoke(this, new UsageReportEventArgs(this.smallPoolInUseSize, this.smallPoolFreeSize, this.LargePoolInUseSize, this.LargePoolFreeSize)); + } + + /// + /// Retrieve a new object with no tag and a default initial capacity. + /// + /// A . + public RecyclableMemoryStream GetStream() + { + return new RecyclableMemoryStream(this); + } + + /// + /// Retrieve a new object with no tag and a default initial capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id) + { + return new RecyclableMemoryStream(this, id); + } + + /// + /// Retrieve a new object with the given tag and a default initial capacity. + /// + /// A tag which can be used to track the source of the stream. + /// A . + public RecyclableMemoryStream GetStream(string tag) + { + return new RecyclableMemoryStream(this, tag); + } + + /// + /// Retrieve a new object with the given tag and a default initial capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag) + { + return new RecyclableMemoryStream(this, id, tag); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity. + /// + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// A . + public RecyclableMemoryStream GetStream(string tag, long requiredSize) + { + return new RecyclableMemoryStream(this, tag, requiredSize); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize) + { + return new RecyclableMemoryStream(this, id, tag, requiredSize); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity, possibly using + /// a single contiguous underlying buffer. + /// + /// Retrieving a which provides a single contiguous buffer can be useful in situations + /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying + /// buffers to a single large one. This is most helpful when you know that you will always call + /// on the underlying stream. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// Whether to attempt to use a single contiguous buffer. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize, bool asContiguousBuffer) + { + if (!asContiguousBuffer || requiredSize <= this.OptionsValue.BlockSize) + { + return this.GetStream(id, tag, requiredSize); + } + + return new RecyclableMemoryStream(this, id, tag, requiredSize, this.GetLargeBuffer(requiredSize, id, tag)); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity, possibly using + /// a single contiguous underlying buffer. + /// + /// Retrieving a which provides a single contiguous buffer can be useful in situations + /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying + /// buffers to a single large one. This is most helpful when you know that you will always call + /// on the underlying stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// Whether to attempt to use a single contiguous buffer. + /// A . + public RecyclableMemoryStream GetStream(string tag, long requiredSize, bool asContiguousBuffer) + { + return this.GetStream(Guid.NewGuid(), tag, requiredSize, asContiguousBuffer); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// The offset from the start of the buffer to copy from. + /// The number of bytes to copy from the buffer. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, byte[] buffer, int offset, int count) + { + RecyclableMemoryStream stream = null; + try + { + stream = new RecyclableMemoryStream(this, id, tag, count); + stream.Write(buffer, offset, count); + stream.Position = 0; + return stream; + } + catch + { + stream?.Dispose(); + throw; + } + } + + /// + /// Retrieve a new object with the contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(byte[] buffer) + { + return this.GetStream(null, buffer, 0, buffer.Length); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// The offset from the start of the buffer to copy from. + /// The number of bytes to copy from the buffer. + /// A . + public RecyclableMemoryStream GetStream(string tag, byte[] buffer, int offset, int count) + { + return this.GetStream(Guid.NewGuid(), tag, buffer, offset, count); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, ReadOnlySpan buffer) + { + RecyclableMemoryStream stream = null; + try + { + stream = new RecyclableMemoryStream(this, id, tag, buffer.Length); + stream.Write(buffer); + stream.Position = 0; + return stream; + } + catch + { + stream?.Dispose(); + throw; + } + } + + /// + /// Retrieve a new object with the contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(ReadOnlySpan buffer) + { + return this.GetStream(null, buffer); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(string tag, ReadOnlySpan buffer) + { + return this.GetStream(Guid.NewGuid(), tag, buffer); + } + + /// + /// Triggered when a new block is created. + /// + public event EventHandler BlockCreated; + + /// + /// Triggered when a new large buffer is created. + /// + public event EventHandler LargeBufferCreated; + + /// + /// Triggered when a new stream is created. + /// + public event EventHandler StreamCreated; + + /// + /// Triggered when a stream is disposed. + /// + public event EventHandler StreamDisposed; + + /// + /// Triggered when a stream is disposed of twice (an error). + /// + public event EventHandler StreamDoubleDisposed; + + /// + /// Triggered when a stream is finalized. + /// + public event EventHandler StreamFinalized; + + /// + /// Triggered when a stream is disposed to report the stream's length. + /// + public event EventHandler StreamLength; + + /// + /// Triggered when a user converts a stream to array. + /// + public event EventHandler StreamConvertedToArray; + + /// + /// Triggered when a stream is requested to expand beyond the maximum length specified by the responsible RecyclableMemoryStreamManager. + /// + public event EventHandler StreamOverCapacity; + + /// + /// Triggered when a buffer of either type is discarded, along with the reason for the discard. + /// + public event EventHandler BufferDiscarded; + + /// + /// Periodically triggered to report usage statistics. + /// + public event EventHandler UsageReport; + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs index 55e992dd4c..73c1a87aa5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs @@ -9,13 +9,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing using System.IO; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; internal sealed class DecryptableItemStream : DecryptableItem { private readonly Encryptor encryptor; private readonly JsonProcessor jsonProcessor; - private readonly StreamManager streamManager; private readonly CosmosSerializer cosmosSerializer; + private readonly StreamManager streamManager; private Stream encryptedStream; // this stream should be recyclable private Stream decryptedStream; // this stream should be recyclable @@ -70,17 +71,13 @@ public DecryptableItemStream( await this.decryptedStream.CopyToAsync(ms, cancellationToken); return ((T)(object)ms, this.decryptionContext); default: +#if SDKPROJECTREF + return (await this.CosmosSerializer.FromStreamAsync(this.decryptedStream, cancellationToken), this.decryptionContext); +#else // this API is missing Async => should not be used return (this.cosmosSerializer.FromStream(this.decryptedStream), this.decryptionContext); - } - } +#endif - public async ValueTask DisposeAsync() - { - if (this.decryptedStream != null) - { - await this.streamManager.ReturnStreamAsync(this.decryptedStream); - this.decryptedStream = null; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs index 77eb53b4b2..c8f02b20aa 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs @@ -6,38 +6,44 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing { using System; - using System.Collections.Generic; using System.IO; - using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; using Newtonsoft.Json.Linq; - public sealed class EncryptableItemStream : EncryptableItem + /// + /// Input type that can be used to allow for lazy decryption in the write path. + /// + /// Type of item. + public sealed class EncryptableItemStream : EncryptableItem, IDisposable { private DecryptableItemStream decryptableItem = null; + private bool isDisposed; /// /// Gets the input item /// public T Item { get; } - private readonly StreamManager streamManager; - /// - public override DecryptableItem DecryptableItem => this.decryptableItem ?? throw new InvalidOperationException("Decryptable content is not initialized."); + public override DecryptableItem DecryptableItem + { + get + { + ObjectDisposedException.ThrowIf(this.isDisposed, this); + return this.decryptableItem ?? throw new InvalidOperationException("Decryptable content is not initialized."); + } + } /// /// Initializes a new instance of the class. /// /// Item to be written. - /// Stream manager to provide output streams. /// Thrown when input is null. - public EncryptableItemStream(T input, StreamManager streamManager = null) + public EncryptableItemStream(T input) { this.Item = input ?? throw new ArgumentNullException(nameof(input)); - this.streamManager = streamManager ?? new MemoryStreamManager(); } #pragma warning disable CS0672 // Member overrides obsolete member @@ -49,11 +55,11 @@ protected internal override void SetDecryptableItem(JToken decryptableContent, E } /// - protected internal override void SetDecryptableStream(Stream decryptableStream, Encryptor encryptor, JsonProcessor jsonProcessor, CosmosSerializer cosmosSerializer) + protected internal override void SetDecryptableStream(Stream decryptableStream, Encryptor encryptor, JsonProcessor jsonProcessor, CosmosSerializer cosmosSerializer, StreamManager streamManager) { ArgumentNullException.ThrowIfNull(decryptableStream); - this.decryptableItem = new DecryptableItemStream(decryptableStream, encryptor, jsonProcessor, cosmosSerializer, this.streamManager); + this.decryptableItem = new DecryptableItemStream(decryptableStream, encryptor, jsonProcessor, cosmosSerializer, streamManager); } /// @@ -67,9 +73,34 @@ protected internal override Stream ToStream(CosmosSerializer serializer) /// protected internal override async Task ToStreamAsync(CosmosSerializer serializer, Stream outputStream, CancellationToken cancellationToken) { +#if SDKPROJECTREF + await serializer.ToStreamAsync(this.Item, outputStream, cancellationToken); +#else // TODO: CosmosSerializer is lacking suitable methods Stream cosmosSerializerOutput = serializer.ToStream(this.Item); await cosmosSerializerOutput.CopyToAsync(outputStream, cancellationToken); +#endif + } + + private void Dispose(bool disposing) + { + if (!this.isDisposed) + { + if (disposing) + { + this.DecryptableItem?.Dispose(); + } + + this.isDisposed = true; + } + } + + /// + public override void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs new file mode 100644 index 0000000000..c6d3160bb5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs @@ -0,0 +1,199 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Buffers; + using System.IO; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; + + internal class ArrayStreamProcessor + { + internal int InitialBufferSize { get; set; } = 16384; + + private static readonly JsonReaderOptions JsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + + private static readonly ReadOnlyMemory DocumentsPropertyUtf8Bytes; + + static ArrayStreamProcessor() + { + DocumentsPropertyUtf8Bytes = new Memory(Encoding.UTF8.GetBytes(Constants.DocumentsResourcePropertyName)); + } + + internal async Task DeserializeAndDecryptCollectionAsync( + Stream input, + Stream output, + Encryptor encryptor, + StreamManager manager, + CancellationToken cancellationToken) + { + Stream readStream = input; + if (!input.CanSeek) + { + Stream temp = manager.CreateStream(); + await input.CopyToAsync(temp, cancellationToken); + temp.Position = 0; + readStream = temp; + } + + using ArrayPoolManager arrayPoolManager = new (); + using Utf8JsonWriter writer = new (output); + + byte[] buffer = arrayPoolManager.Rent(this.InitialBufferSize); + + Utf8JsonWriter chunkWriter = null; + + int leftOver = 0; + bool isFinalBlock = false; + bool isDocumentsArray = false; + RecyclableMemoryStream bufferWriter = null; + bool isDocumentsProperty = false; + + RecyclableMemoryStreamManager recyclableMemoryStreamManager = new (); + + JsonReaderState state = new (ArrayStreamProcessor.JsonReaderOptions); + + while (!isFinalBlock) + { + int dataLength = await readStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + int dataSize = dataLength + leftOver; + isFinalBlock = dataSize == 0; + long bytesConsumed = 0; + + bytesConsumed = this.TransformBuffer( + buffer.AsSpan(0, dataSize), + isFinalBlock, + writer, + ref bufferWriter, + ref chunkWriter, + ref state, + ref isDocumentsProperty, + ref isDocumentsArray, + arrayPoolManager, + encryptor, + manager, + recyclableMemoryStreamManager); + + leftOver = dataSize - (int)bytesConsumed; + + // we need to scale out buffer + if (leftOver == dataSize) + { + byte[] newBuffer = arrayPoolManager.Rent(buffer.Length * 2); + buffer.AsSpan().CopyTo(newBuffer); + buffer = newBuffer; + } + else if (leftOver != 0) + { + buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + } + } + + await readStream.DisposeAsync(); + output.Position = 0; + } + + private long TransformBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref RecyclableMemoryStream bufferWriter, ref Utf8JsonWriter chunkWriter, ref JsonReaderState state, ref bool isDocumentsProperty, ref bool isDocumentsArray, ArrayPoolManager arrayPoolManager, Encryptor encryptor, StreamManager streamManager, RecyclableMemoryStreamManager manager) + { + Utf8JsonReader reader = new Utf8JsonReader(buffer, isFinalBlock, state); + + while (reader.Read()) + { + Utf8JsonWriter currentWriter = chunkWriter ?? writer; + + JsonTokenType tokenType = reader.TokenType; + + switch (tokenType) + { + case JsonTokenType.None: + break; + case JsonTokenType.StartObject: + if (isDocumentsArray && chunkWriter == null) + { + bufferWriter = new RecyclableMemoryStream(manager); + chunkWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); + chunkWriter.WriteStartObject(); + } + else + { + currentWriter.WriteStartObject(); + } + + break; + case JsonTokenType.EndObject: + currentWriter.WriteEndObject(); + if (reader.CurrentDepth == 2 && chunkWriter != null) + { + currentWriter.Flush(); + Stream transformStream = streamManager.CreateStream(); + bufferWriter.Position = 0; +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits - we cannot make this call async + _ = EncryptionProcessor.DecryptAsync(bufferWriter, transformStream, encryptor, new CosmosDiagnosticsContext(), JsonProcessor.Stream, CancellationToken.None).GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + + byte[] copyBuffer = arrayPoolManager.Rent(16384); + Span copySpan = copyBuffer.AsSpan(); + int readBytes = 16384; + while (readBytes == 16384) + { + readBytes = transformStream.Read(copySpan); + + if (readBytes > 0) + { + writer.WriteRawValue(copySpan[..readBytes], false); + } + } + + transformStream.Dispose(); + chunkWriter.Dispose(); + bufferWriter.Dispose(); + chunkWriter = null; + } + + break; + case JsonTokenType.StartArray: + if (isDocumentsProperty && reader.CurrentDepth == 1) + { + isDocumentsArray = true; + } + + currentWriter.WriteStartArray(); + break; + + case JsonTokenType.EndArray: + currentWriter.WriteEndArray(); + if (isDocumentsArray && reader.CurrentDepth == 1) + { + isDocumentsArray = false; + isDocumentsProperty = false; + } + + break; + + case JsonTokenType.PropertyName: + if (reader.ValueTextEquals(DocumentsPropertyUtf8Bytes.Span)) + { + isDocumentsProperty = true; + } + + currentWriter.WritePropertyName(reader.ValueSpan); + break; + default: + currentWriter.WriteRawValue(reader.ValueSpan, true); + break; + } + } + + state = reader.CurrentState; + return reader.BytesConsumed; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs deleted file mode 100644 index a8bc77f75b..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - - internal class JsonBytes - { - internal byte[] Bytes { get; private set; } - - internal int Offset { get; private set; } - - internal int Length { get; private set; } - - public JsonBytes(byte[] bytes, int offset, int length) - { - ArgumentNullException.ThrowIfNull(bytes); - ArgumentOutOfRangeException.ThrowIfNegative(offset); - ArgumentOutOfRangeException.ThrowIfNegative(length); - if (bytes.Length < offset + length) - { - throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); - } - - this.Bytes = bytes; - this.Offset = offset; - this.Length = length; - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs deleted file mode 100644 index d9048375c0..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - using System.Text.Json; - using System.Text.Json.Serialization; - - internal class JsonBytesConverter : JsonConverter - { - public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) - { - writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializer.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializer.cs index e6bd34d1f5..01494e4b4d 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializer.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSerializer.cs @@ -5,6 +5,8 @@ namespace Microsoft.Azure.Cosmos { using System.IO; + using System.Threading; + using System.Threading.Tasks; /// /// This abstract class can be implemented to allow a custom serializer to be used by the CosmosClient. @@ -32,5 +34,37 @@ public abstract class CosmosSerializer /// Any type passed to . /// A readable Stream containing JSON of the serialized object. public abstract Stream ToStream(T input); + + /// + /// Convert a Stream of JSON to an object. + /// The implementation is responsible for Disposing of the stream, + /// including when an exception is thrown, to avoid memory leaks. + /// + /// Any type passed to . + /// The Stream response containing JSON from Cosmos DB. + /// Cancellation token. + /// The object deserialized from the stream. + public virtual Task FromStreamAsync(Stream stream, CancellationToken cancellationToken) + { + _ = cancellationToken; + return Task.FromResult(this.FromStream(stream)); + } + + /// + /// Convert the object to a Stream. + /// The caller provides the Stream and has full control over it. + /// Stream.CanRead must be true. + /// + /// Any type passed to . + /// Output stream. + /// Cancellation token. + /// A readable Stream containing JSON of the serialized object. + public virtual async Task ToStreamAsync(T input, Stream output, CancellationToken cancellationToken) + { + Stream temp = this.ToStream(input); + + //80kiB is default value + await temp.CopyToAsync(output, 81920, cancellationToken); + } } } diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs index ac5550b128..e776d4454b 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosSystemTextJsonSerializer.cs @@ -9,6 +9,8 @@ namespace Microsoft.Azure.Cosmos using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; + using System.Threading; + using System.Threading.Tasks; /// /// This class provides a default implementation of System.Text.Json Cosmos Linq Serializer. @@ -54,6 +56,28 @@ public override T FromStream(Stream stream) } } + /// + public override async Task FromStreamAsync(Stream stream, CancellationToken cancellationToken) + { + if (stream == null) + throw new ArgumentNullException(nameof(stream)); + + if (typeof(Stream).IsAssignableFrom(typeof(T))) + { + return (T)(object)stream; + } + + if (stream.CanSeek && stream.Length == 0) + { + return default; + } + + using (stream) + { + return await JsonSerializer.DeserializeAsync(stream, this.jsonSerializerOptions, cancellationToken); + } + } + /// public override Stream ToStream(T input) { @@ -66,6 +90,13 @@ public override Stream ToStream(T input) return streamPayload; } + /// + public override async Task ToStreamAsync(T input, Stream output, CancellationToken cancellationToken) + { + await JsonSerializer.SerializeAsync(output, input, this.jsonSerializerOptions, cancellationToken); + output.Position = 0; + } + /// /// Convert a MemberInfo to a string for use in LINQ query translation. /// From b55a857b1549e683c31d5e65c804496f424330c9 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 18 Oct 2024 14:29:49 +0200 Subject: [PATCH 68/85] ~ cleanup --- .../src/EncryptionContainer.cs | 25 +++++-------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index 2b8473ba62..060fb79bdc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -255,8 +255,8 @@ public override async Task> ReadItemAsync( using (diagnosticsContext.CreateScope("ReadItem")) { ResponseMessage responseMessage; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (typeof(T) == typeof(DecryptableItemStream)) + + if (typeof(T) == typeof(DecryptableItem)) { responseMessage = await this.ReadItemHelperAsync( id, @@ -266,6 +266,7 @@ public override async Task> ReadItemAsync( diagnosticsContext, cancellationToken); +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER EncryptionItemRequestOptions options = requestOptions as EncryptionItemRequestOptions; DecryptableItem decryptableItem = new DecryptableItemStream( responseMessage.Content, @@ -273,26 +274,12 @@ public override async Task> ReadItemAsync( options.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); - - return new EncryptionItemResponse( - responseMessage, - (T)(object)decryptableItem); - } -#endif - if (typeof(T) == typeof(DecryptableItem)) - { - responseMessage = await this.ReadItemHelperAsync( - id, - partitionKey, - requestOptions, - decryptResponse: false, - diagnosticsContext, - cancellationToken); - +#else DecryptableItemCore decryptableItem = new ( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); +#endif return new EncryptionItemResponse( responseMessage, @@ -1367,7 +1354,7 @@ private async Task> DecryptChangeFeedDocumentsAsync( CancellationToken cancellationToken) { List decryptItems = new (documents.Count); - if (typeof(T) == typeof(DecryptableItemStream) || typeof(T) == typeof(DecryptableItem)) + if (typeof(T) == typeof(DecryptableItem)) { foreach (Stream documentStream in documents) { From 46b3ca421a4363d1133454ed177166c5f6908f9e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 18 Oct 2024 14:57:24 +0200 Subject: [PATCH 69/85] + EncryptionFeedIterator --- .../src/EncryptionFeedIterator.cs | 66 ++++++- .../src/Transformation/ArrayStreamSplitter.cs | 180 ++++++++++++++++++ 2 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs index 202a84428e..be32751829 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs @@ -4,12 +4,17 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { - using System; using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; +#else + using System; using Newtonsoft.Json.Linq; +#endif internal sealed class EncryptionFeedIterator : FeedIterator { @@ -17,6 +22,25 @@ internal sealed class EncryptionFeedIterator : FeedIterator private readonly Encryptor encryptor; private readonly CosmosSerializer cosmosSerializer; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private readonly StreamManager streamManager; + + private static readonly ArrayStreamSplitter StreamSplitter = new ArrayStreamSplitter(); +#endif + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public EncryptionFeedIterator( + FeedIterator feedIterator, + Encryptor encryptor, + CosmosSerializer cosmosSerializer, + StreamManager streamManager) + { + this.feedIterator = feedIterator; + this.encryptor = encryptor; + this.cosmosSerializer = cosmosSerializer; + this.streamManager = streamManager; + } +#else public EncryptionFeedIterator( FeedIterator feedIterator, Encryptor encryptor, @@ -26,6 +50,7 @@ public EncryptionFeedIterator( this.encryptor = encryptor; this.cosmosSerializer = cosmosSerializer; } +#endif public override bool HasMoreResults => this.feedIterator.HasMoreResults; @@ -38,10 +63,20 @@ public override async Task ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream decryptedContent = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync( + responseMessage.Content, + decryptedContent, + this.encryptor, + this.streamManager, + cancellationToken); +#else Stream decryptedContent = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( responseMessage.Content, this.encryptor, cancellationToken); +#endif return new DecryptedResponseMessage(responseMessage, decryptedContent); } @@ -60,8 +95,14 @@ public override async Task ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + decryptableContent = await this.ConvertResponseToDecryptableItemsAsync( + responseMessage.Content, + cancellationToken); +#else decryptableContent = this.ConvertResponseToDecryptableItems( responseMessage.Content); +#endif return (responseMessage, decryptableContent); } @@ -70,6 +111,28 @@ public override async Task ReadNextAsync(CancellationToken canc } } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private async Task> ConvertResponseToDecryptableItemsAsync( + Stream content, + CancellationToken token) + { + List decryptableStreams = await StreamSplitter.SplitCollectionAsync(content, this.streamManager, token); + List decryptableItems = new List(); + + foreach (Stream item in decryptableStreams) + { + decryptableItems.Add( + (T)(object)new DecryptableItemStream( + item, + this.encryptor, + JsonProcessor.Stream, + this.cosmosSerializer, + this.streamManager)); + } + + return decryptableItems; + } +#else private List ConvertResponseToDecryptableItems( Stream content) { @@ -94,5 +157,6 @@ private List ConvertResponseToDecryptableItems( return decryptableItems; } +#endif } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs new file mode 100644 index 0000000000..39e12edabd --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs @@ -0,0 +1,180 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Buffers; + using System.Collections.Generic; + using System.IO; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; + + internal class ArrayStreamSplitter + { + internal int InitialBufferSize { get; set; } = 16384; + + private static readonly JsonReaderOptions JsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + + private static readonly ReadOnlyMemory DocumentsPropertyUtf8Bytes; + + static ArrayStreamSplitter() + { + DocumentsPropertyUtf8Bytes = new Memory(Encoding.UTF8.GetBytes(Constants.DocumentsResourcePropertyName)); + } + + internal async Task> SplitCollectionAsync( + Stream input, + StreamManager manager, + CancellationToken cancellationToken) + { + Stream readStream = input; + if (!input.CanSeek) + { + Stream temp = manager.CreateStream(); + await input.CopyToAsync(temp, cancellationToken); + temp.Position = 0; + readStream = temp; + } + + using ArrayPoolManager arrayPoolManager = new (); + + byte[] buffer = arrayPoolManager.Rent(this.InitialBufferSize); + + Utf8JsonWriter chunkWriter = null; + + int leftOver = 0; + bool isFinalBlock = false; + bool isDocumentsArray = false; + RecyclableMemoryStream bufferWriter = null; + bool isDocumentsProperty = false; + + RecyclableMemoryStreamManager recyclableMemoryStreamManager = new (); + + JsonReaderState state = new (ArrayStreamSplitter.JsonReaderOptions); + List outputList = new List(); + + while (!isFinalBlock) + { + int dataLength = await readStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + int dataSize = dataLength + leftOver; + isFinalBlock = dataSize == 0; + long bytesConsumed = 0; + + bytesConsumed = this.TransformBuffer( + buffer.AsSpan(0, dataSize), + outputList, + isFinalBlock, + ref bufferWriter, + ref chunkWriter, + ref state, + ref isDocumentsProperty, + ref isDocumentsArray, + recyclableMemoryStreamManager); + + leftOver = dataSize - (int)bytesConsumed; + + // we need to scale out buffer + if (leftOver == dataSize) + { + byte[] newBuffer = arrayPoolManager.Rent(buffer.Length * 2); + buffer.AsSpan().CopyTo(newBuffer); + buffer = newBuffer; + } + else if (leftOver != 0) + { + buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + } + } + + await readStream.DisposeAsync(); + + return outputList; + } + + private long TransformBuffer(Span buffer, List outputList, bool isFinalBlock, ref RecyclableMemoryStream bufferWriter, ref Utf8JsonWriter chunkWriter, ref JsonReaderState state, ref bool isDocumentsProperty, ref bool isDocumentsArray, RecyclableMemoryStreamManager manager) + { + Utf8JsonReader reader = new Utf8JsonReader(buffer, isFinalBlock, state); + + while (reader.Read()) + { + Utf8JsonWriter currentWriter = chunkWriter; + + JsonTokenType tokenType = reader.TokenType; + + switch (tokenType) + { + case JsonTokenType.None: + break; + case JsonTokenType.StartObject: + if (isDocumentsArray && chunkWriter == null) + { + bufferWriter = new RecyclableMemoryStream(manager); + chunkWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); + chunkWriter?.WriteStartObject(); + } + else + { + currentWriter?.WriteStartObject(); + } + + break; + case JsonTokenType.EndObject: + currentWriter?.WriteEndObject(); + if (reader.CurrentDepth == 2 && chunkWriter != null) + { + currentWriter.Flush(); + bufferWriter.Position = 0; + outputList.Add(bufferWriter); + + bufferWriter = null; + + chunkWriter.Dispose(); + chunkWriter = null; + } + + break; + case JsonTokenType.StartArray: + if (isDocumentsProperty && reader.CurrentDepth == 1) + { + isDocumentsArray = true; + } + + currentWriter?.WriteStartArray(); + break; + + case JsonTokenType.EndArray: + currentWriter?.WriteEndArray(); + if (isDocumentsArray && reader.CurrentDepth == 1) + { + isDocumentsArray = false; + isDocumentsProperty = false; + } + + break; + + case JsonTokenType.PropertyName: + if (reader.ValueTextEquals(DocumentsPropertyUtf8Bytes.Span)) + { + isDocumentsProperty = true; + } + + currentWriter?.WritePropertyName(reader.ValueSpan); + break; + default: + currentWriter?.WriteRawValue(reader.ValueSpan, true); + break; + } + } + + state = reader.CurrentState; + return reader.BytesConsumed; + } + } +} +#endif \ No newline at end of file From 5ec68baf4fae2a881f61c1bad9445adeed9b14df Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 18 Oct 2024 15:20:20 +0200 Subject: [PATCH 70/85] ~ update EncryptionTransactionalBatch --- .../src/EncryptionTransactionalBatch.cs | 81 ++++++++++++++++--- .../src/MemoryStreamManager.cs | 2 +- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs index 0755ec0cf6..6fa0b3d6a0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs @@ -16,8 +16,26 @@ internal sealed class EncryptionTransactionalBatch : TransactionalBatch { private readonly Encryptor encryptor; private readonly CosmosSerializer cosmosSerializer; + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private readonly StreamManager streamManager; +#endif + private TransactionalBatch transactionalBatch; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public EncryptionTransactionalBatch( + TransactionalBatch transactionalBatch, + Encryptor encryptor, + CosmosSerializer cosmosSerializer, + StreamManager streamManager) + { + this.transactionalBatch = transactionalBatch ?? throw new ArgumentNullException(nameof(transactionalBatch)); + this.encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); + this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); + this.streamManager = streamManager ?? throw new ArgumentNullException(nameof(streamManager)); + } +#else public EncryptionTransactionalBatch( TransactionalBatch transactionalBatch, Encryptor encryptor, @@ -27,6 +45,7 @@ public EncryptionTransactionalBatch( this.encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); } +#endif public override TransactionalBatch CreateItem( T item, @@ -58,12 +77,24 @@ public override TransactionalBatch CreateItemStream( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("EncryptItemStream")) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream temp = this.streamManager.CreateStream(); + EncryptionProcessor.EncryptAsync( + streamPayload, + temp, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).GetAwaiter().GetResult(); + streamPayload = temp; +#else streamPayload = EncryptionProcessor.EncryptAsync( streamPayload, this.encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken: default).Result; +#endif } } @@ -128,15 +159,24 @@ public override TransactionalBatch ReplaceItemStream( encryptionItemRequestOptions.EncryptionOptions != null) { CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); - using (diagnosticsContext.CreateScope("EncryptItemStream")) - { - streamPayload = EncryptionProcessor.EncryptAsync( - streamPayload, - this.encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken: default).Result; - } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream temp = this.streamManager.CreateStream(); + EncryptionProcessor.EncryptAsync( + streamPayload, + temp, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).GetAwaiter().GetResult(); + streamPayload = temp; +#else + streamPayload = EncryptionProcessor.EncryptAsync( + streamPayload, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).Result; +#endif } this.transactionalBatch = this.transactionalBatch.ReplaceItemStream( @@ -177,12 +217,24 @@ public override TransactionalBatch UpsertItemStream( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("EncryptItemStream")) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream temp = this.streamManager.CreateStream(); + EncryptionProcessor.EncryptAsync( + streamPayload, + temp, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).GetAwaiter().GetResult(); + streamPayload = temp; +#else streamPayload = EncryptionProcessor.EncryptAsync( streamPayload, this.encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken: default).Result; +#endif } } @@ -233,11 +285,22 @@ private async Task DecryptTransactionalBatchResponse { if (response.IsSuccessStatusCode && result.ResourceStream != null) { +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + result.ResourceStream, + decryptedStream, + this.encryptor, + diagnosticsContext, + JsonProcessor.Stream, + cancellationToken); +#else (Stream decryptedStream, _) = await EncryptionProcessor.DecryptAsync( result.ResourceStream, this.encryptor, diagnosticsContext, cancellationToken); +#endif decryptedTransactionalBatchOperationResults.Add(new EncryptionTransactionalBatchOperationResult(result, decryptedStream)); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs index 2403e4803b..0edbdf52b2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs @@ -15,7 +15,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// Placeholder internal class MemoryStreamManager : StreamManager { - private readonly RecyclableMemoryStreamManager streamManager = new RecyclableMemoryStreamManager(); + private readonly RecyclableMemoryStreamManager streamManager = new (); /// /// Create stream From fa0020c66bcf8d92379604d271a0b635277d81dd Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 18 Oct 2024 20:28:55 +0200 Subject: [PATCH 71/85] ~ PR based changes --- .../src/AeadAes/AeAesEncryptionProcessor.cs | 2 +- .../src/EncryptionFormatVersion.cs | 13 + .../src/EncryptionOptionsExtensions.cs | 20 + .../src/EncryptionProcessor.cs | 38 +- .../MdeEncryptionProcessor.Stable.cs | 2 +- .../MdeJObjectEncryptionProcessor.Preview.cs | 4 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 4 +- .../StreamProcessor.Decryptor.cs | 202 +++++----- .../StreamProcessor.Encryptor.cs | 372 ++++++++---------- 9 files changed, 309 insertions(+), 348 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs index c7eb3658d4..0822ffc0f9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs @@ -81,7 +81,7 @@ internal static async Task DecryptContentAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 2) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.AeAes) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs new file mode 100644 index 0000000000..035df18c61 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + internal static class EncryptionFormatVersion + { + public const int AeAes = 2; + public const int Mde = 3; + public const int MdeWithCompression = 4; + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs index 1297652a25..09f02d71d0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs @@ -5,6 +5,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; + using System.Collections.Generic; + using System.Linq; internal static class EncryptionOptionsExtensions { @@ -31,6 +33,24 @@ internal static void Validate(this EncryptionOptions options) #pragma warning restore CA2208 // Instantiate argument exceptions correctly } + if (options.PathsToEncrypt is not HashSet && options.PathsToEncrypt.Distinct().Count() != options.PathsToEncrypt.Count()) + { + throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); + } + + foreach (string path in options.PathsToEncrypt) + { + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) + { + throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(options.PathsToEncrypt)}"); + } + + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) + { + throw new InvalidOperationException($"{nameof(options.PathsToEncrypt)} includes a invalid path: '{path}'."); + } + } + options.CompressionOptions?.Validate(); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 93209d2450..8612eb3958 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -63,24 +63,6 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) - { - throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); - } - - foreach (string path in encryptionOptions.PathsToEncrypt) - { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) - { - throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); - } - - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) - { - throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); - } - } - #pragma warning disable CS0618 // Type or member is obsolete return encryptionOptions.EncryptionAlgorithm switch { @@ -113,24 +95,6 @@ public static async Task EncryptAsync( return; } - if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) - { - throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); - } - - foreach (string path in encryptionOptions.PathsToEncrypt) - { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) - { - throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); - } - - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) - { - throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); - } - } - if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) { throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); @@ -250,7 +214,7 @@ public static async Task DecryptAsync( else { input.Position = 0; - throw new NotSupportedException($"Streaming mode is not supported for encryption algorithm {properties.EncryptionProperties.EncryptionAlgorithm}"); + throw new NotSupportedException($"Encryption Algorithm: {properties.EncryptionProperties.EncryptionAlgorithm} is not supported."); } #pragma warning restore CS0618 // Type or member is obsolete diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 659ca74c91..fc8a20f718 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -90,7 +90,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 6d7b49ac04..a3b60dee57 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -127,7 +127,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -141,7 +141,7 @@ internal async Task DecryptObjectAsync( #if NET8_0_OR_GREATER BrotliCompressor decompressor = null; - if (encryptionProperties.EncryptionFormatVersion == 4) + if (encryptionProperties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) { bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index abee2721fb..b7fcda6b43 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -142,7 +142,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -157,7 +157,7 @@ internal async Task DecryptObjectAsync( #if NET8_0_OR_GREATER BrotliCompressor decompressor = null; - if (encryptionProperties.EncryptionFormatVersion == 4) + if (encryptionProperties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) { bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 442800d6e0..342fe3bebf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -39,7 +39,7 @@ internal async Task DecryptStreamAsync( { _ = diagnosticsContext; - if (properties.EncryptionFormatVersion != 3 && properties.EncryptionFormatVersion != 4) + if (properties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && properties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) { throw new NotSupportedException($"Unknown encryption format version: {properties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -75,19 +75,7 @@ internal async Task DecryptStreamAsync( long bytesConsumed = 0; // processing itself here - bytesConsumed = this.TransformDecryptBuffer( - buffer.AsSpan(0, dataSize), - isFinalBlock, - writer, - ref state, - ref isIgnoredBlock, - ref decryptPropertyName, - encryptedPaths, - pathsDecrypted, - properties, - containsCompressed, - arrayPoolManager, - encryptionKey); + bytesConsumed = TransformDecryptBuffer(buffer.AsSpan(0, dataSize)); leftOver = dataSize - (int)bytesConsumed; @@ -108,115 +96,115 @@ internal async Task DecryptStreamAsync( outputStream.Position = 0; return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); - } - - private long TransformDecryptBuffer(ReadOnlySpan buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, HashSet encryptedPaths, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) - { - Utf8JsonReader reader = new (buffer, isFinalBlock, state); - while (reader.Read()) + long TransformDecryptBuffer(ReadOnlySpan buffer) { - JsonTokenType tokenType = reader.TokenType; + Utf8JsonReader reader = new (buffer, isFinalBlock, state); - if (isIgnoredBlock && reader.CurrentDepth == 1 && tokenType == JsonTokenType.EndObject) + while (reader.Read()) { - isIgnoredBlock = false; - continue; - } - else if (isIgnoredBlock) - { - continue; - } + JsonTokenType tokenType = reader.TokenType; + + if (isIgnoredBlock && reader.CurrentDepth == 1 && tokenType == JsonTokenType.EndObject) + { + isIgnoredBlock = false; + continue; + } + else if (isIgnoredBlock) + { + continue; + } + + switch (tokenType) + { + case JsonTokenType.String: + if (decryptPropertyName == null) + { + writer.WriteStringValue(reader.ValueSpan); + } + else + { + this.TransformDecryptProperty( + ref reader, + writer, + decryptPropertyName, + properties, + encryptionKey, + containsCompressed, + arrayPoolManager); + + pathsDecrypted.Add(decryptPropertyName); + } - switch (tokenType) - { - case JsonTokenType.String: - if (decryptPropertyName == null) - { - writer.WriteStringValue(reader.ValueSpan); - } - else - { - this.TransformDecryptProperty( - ref reader, - writer, - decryptPropertyName, - properties, - encryptionKey, - containsCompressed, - arrayPoolManager); - - pathsDecrypted.Add(decryptPropertyName); - } - - decryptPropertyName = null; - break; - case JsonTokenType.Number: - decryptPropertyName = null; - writer.WriteRawValue(reader.ValueSpan); - break; - case JsonTokenType.None: - decryptPropertyName = null; - break; - case JsonTokenType.StartObject: - decryptPropertyName = null; - writer.WriteStartObject(); - break; - case JsonTokenType.EndObject: - decryptPropertyName = null; - writer.WriteEndObject(); - break; - case JsonTokenType.StartArray: - decryptPropertyName = null; - writer.WriteStartArray(); - break; - case JsonTokenType.EndArray: - decryptPropertyName = null; - writer.WriteEndArray(); - break; - case JsonTokenType.PropertyName: - string propertyName = "/" + reader.GetString(); - if (encryptedPaths.Contains(propertyName)) - { - decryptPropertyName = propertyName; - } - else if (propertyName == StreamProcessor.EncryptionPropertiesPath) - { - if (!reader.TrySkip()) + decryptPropertyName = null; + break; + case JsonTokenType.Number: + decryptPropertyName = null; + writer.WriteRawValue(reader.ValueSpan); + break; + case JsonTokenType.None: + decryptPropertyName = null; + break; + case JsonTokenType.StartObject: + decryptPropertyName = null; + writer.WriteStartObject(); + break; + case JsonTokenType.EndObject: + decryptPropertyName = null; + writer.WriteEndObject(); + break; + case JsonTokenType.StartArray: + decryptPropertyName = null; + writer.WriteStartArray(); + break; + case JsonTokenType.EndArray: + decryptPropertyName = null; + writer.WriteEndArray(); + break; + case JsonTokenType.PropertyName: + string propertyName = "/" + reader.GetString(); + if (encryptedPaths.Contains(propertyName)) { - isIgnoredBlock = true; + decryptPropertyName = propertyName; + } + else if (propertyName == StreamProcessor.EncryptionPropertiesPath) + { + if (!reader.TrySkip()) + { + isIgnoredBlock = true; + } + + break; } + writer.WritePropertyName(reader.ValueSpan); break; - } - - writer.WritePropertyName(reader.ValueSpan); - break; - case JsonTokenType.Comment: - break; - case JsonTokenType.True: - decryptPropertyName = null; - writer.WriteBooleanValue(true); - break; - case JsonTokenType.False: - decryptPropertyName = null; - writer.WriteBooleanValue(false); - break; - case JsonTokenType.Null: - decryptPropertyName = null; - writer.WriteNullValue(); - break; + case JsonTokenType.Comment: + break; + case JsonTokenType.True: + decryptPropertyName = null; + writer.WriteBooleanValue(true); + break; + case JsonTokenType.False: + decryptPropertyName = null; + writer.WriteBooleanValue(false); + break; + case JsonTokenType.Null: + decryptPropertyName = null; + writer.WriteNullValue(); + break; + } } - } - state = reader.CurrentState; - return reader.BytesConsumed; + state = reader.CurrentState; + return reader.BytesConsumed; + } } private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, bool containsCompressed, ArrayPoolManager arrayPoolManager) { BrotliCompressor decompressor = null; - if (properties.EncryptionFormatVersion == 4) + if (properties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) { if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index 3fd4b0e4f9..a3980ae56a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -60,21 +60,7 @@ internal async Task EncryptStreamAsync( isFinalBlock = dataSize == 0; long bytesConsumed = 0; - bytesConsumed = this.TransformEncryptBuffer( - buffer.AsSpan(0, dataSize), - isFinalBlock, - writer, - ref encryptionPayloadWriter, - ref bufferWriter, - ref state, - ref encryptPropertyName, - pathsToEncrypt, - pathsEncrypted, - compressedPaths, - compressor, - arrayPoolManager, - encryptionKey, - encryptionOptions); + bytesConsumed = TransformEncryptBuffer(buffer.AsSpan(0, dataSize)); leftOver = dataSize - (int)bytesConsumed; @@ -108,178 +94,186 @@ internal async Task EncryptStreamAsync( writer.Flush(); outputStream.Position = 0; - } - - private long TransformEncryptBuffer( - ReadOnlySpan buffer, - bool isFinalBlock, - Utf8JsonWriter writer, - ref Utf8JsonWriter encryptionPayloadWriter, - ref RentArrayBufferWriter bufferWriter, - ref JsonReaderState state, - ref string encryptPropertyName, - HashSet pathsToEncrypt, - List pathsEncrypted, - Dictionary compressedPaths, - BrotliCompressor compressor, - ArrayPoolManager arrayPoolManager, - DataEncryptionKey encryptionKey, - EncryptionOptions encryptionOptions) - { - Utf8JsonReader reader = new (buffer, isFinalBlock, state); - while (reader.Read()) + long TransformEncryptBuffer(ReadOnlySpan buffer) { - Utf8JsonWriter currentWriter = encryptionPayloadWriter ?? writer; + Utf8JsonReader reader = new (buffer, isFinalBlock, state); - JsonTokenType tokenType = reader.TokenType; - - switch (tokenType) + while (reader.Read()) { - case JsonTokenType.None: - break; - case JsonTokenType.StartObject: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); - encryptionPayloadWriter.WriteStartObject(); - } - else - { - currentWriter.WriteStartObject(); - } - - break; - case JsonTokenType.EndObject: - if (reader.CurrentDepth == 0) - { - continue; - } - - currentWriter.WriteEndObject(); - if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) - { - currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - writer.WriteBase64StringValue(encryptedBytes); - - encryptPropertyName = null; - encryptionPayloadWriter.Dispose(); - encryptionPayloadWriter = null; - bufferWriter.Dispose(); - bufferWriter = null; - } - - break; - case JsonTokenType.StartArray: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); - encryptionPayloadWriter.WriteStartArray(); - } - else - { - currentWriter.WriteStartArray(); - } - - break; - case JsonTokenType.EndArray: - currentWriter.WriteEndArray(); - if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) - { - currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - writer.WriteBase64StringValue(encryptedBytes); - - encryptPropertyName = null; - encryptionPayloadWriter.Dispose(); - encryptionPayloadWriter = null; - bufferWriter.Dispose(); - bufferWriter = null; - } - - break; - case JsonTokenType.PropertyName: - string propertyName = "/" + reader.GetString(); - if (pathsToEncrypt.Contains(propertyName)) - { - encryptPropertyName = propertyName; - } - - currentWriter.WritePropertyName(reader.ValueSpan); - break; - case JsonTokenType.Comment: - currentWriter.WriteCommentValue(reader.ValueSpan); - break; - case JsonTokenType.String: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); - int length = reader.CopyString(bytes); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteStringValue(reader.ValueSpan); - } - - break; - case JsonTokenType.Number: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteRawValue(reader.ValueSpan, true); - } - - break; - case JsonTokenType.True: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - (byte[] bytes, int length) = Serialize(true, arrayPoolManager); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteBooleanValue(true); - } - - break; - case JsonTokenType.False: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - (byte[] bytes, int length) = Serialize(false, arrayPoolManager); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteBooleanValue(false); - } - - break; - case JsonTokenType.Null: - currentWriter.WriteNullValue(); - break; + Utf8JsonWriter currentWriter = encryptionPayloadWriter ?? writer; + + JsonTokenType tokenType = reader.TokenType; + + switch (tokenType) + { + case JsonTokenType.None: + break; + case JsonTokenType.StartObject: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartObject(); + } + else + { + currentWriter.WriteStartObject(); + } + + break; + case JsonTokenType.EndObject: + if (reader.CurrentDepth == 0) + { + continue; + } + + currentWriter.WriteEndObject(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Object); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; +#pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct + encryptionPayloadWriter.Dispose(); +#pragma warning restore VSTHRD103 // Call async methods when in an async method + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.StartArray: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartArray(); + } + else + { + currentWriter.WriteStartArray(); + } + + break; + case JsonTokenType.EndArray: + currentWriter.WriteEndArray(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Array); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; +#pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct + encryptionPayloadWriter.Dispose(); +#pragma warning restore VSTHRD103 // Call async methods when in an async method + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.PropertyName: + string propertyName = "/" + reader.GetString(); + if (pathsToEncrypt.Contains(propertyName)) + { + encryptPropertyName = propertyName; + } + + currentWriter.WritePropertyName(reader.ValueSpan); + break; + case JsonTokenType.Comment: + currentWriter.WriteCommentValue(reader.ValueSpan); + break; + case JsonTokenType.String: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); + int length = reader.CopyString(bytes); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.String); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteStringValue(reader.ValueSpan); + } + + break; + case JsonTokenType.Number: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, typeMarker); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteRawValue(reader.ValueSpan, true); + } + + break; + case JsonTokenType.True: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(true, arrayPoolManager); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Boolean); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(true); + } + + break; + case JsonTokenType.False: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(false, arrayPoolManager); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Boolean); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(false); + } + + break; + case JsonTokenType.Null: + currentWriter.WriteNullValue(); + break; + } } + + state = reader.CurrentState; + return reader.BytesConsumed; } - state = reader.CurrentState; - return reader.BytesConsumed; + ReadOnlySpan TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker) + { + byte[] processedBytes = payload; + int processedBytesLength = payloadSize; + + if (compressor != null && payloadSize >= encryptionOptions.CompressionOptions.MinimalCompressedLength) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(payloadSize)); + processedBytesLength = compressor.Compress(compressedPaths, encryptPropertyName, processedBytes, payloadSize, compressedBytes); + processedBytes = compressedBytes; + } + + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); + + pathsEncrypted.Add(encryptPropertyName); + return encryptedBytes.AsSpan(0, encryptedBytesCount); + } } private static (byte[] buffer, int length) Serialize(bool value, ArrayPoolManager arrayPoolManager) @@ -324,24 +318,6 @@ private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(doub return (TypeMarker.Double, buffer, length); } - - private ReadOnlySpan TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) - { - byte[] processedBytes = payload; - int processedBytesLength = payloadSize; - - if (compressor != null && payloadSize >= options.CompressionOptions.MinimalCompressedLength) - { - byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(payloadSize)); - processedBytesLength = compressor.Compress(pathsCompressed, encryptName, processedBytes, payloadSize, compressedBytes); - processedBytes = compressedBytes; - } - - (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); - - pathsEncrypted.Add(encryptName); - return encryptedBytes.AsSpan(0, encryptedBytesCount); - } } } #endif \ No newline at end of file From 14281189e08d293bac6443e40079242cfa5955c0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 10:31:45 +0200 Subject: [PATCH 72/85] + wip --- .../src/EncryptionContainerExtensions.cs | 36 + .../EncryptionContainerStream.cs | 1102 +++++++++++++++++ .../EncryptionFeedIteratorStream.cs | 106 ++ .../EncryptionFeedIteratorStream{T}.cs | 50 + .../EncryptionTransactionalBatchStream.cs | 293 +++++ 5 files changed, 1587 insertions(+) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream{T}.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs index fdde796298..cfdc4cd151 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs @@ -23,6 +23,13 @@ public static Container WithEncryptor( this Container container, Encryptor encryptor) { +#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (container.Database.Client.ClientOptions.SerializerOptions.UseSystemTextJsonSerializerWithOptions) + { + return new EncryptionContainerStream(container, encryptor); + } +#endif + return new EncryptionContainer( container, encryptor); @@ -50,6 +57,19 @@ public static FeedIterator ToEncryptionFeedIterator( this Container container, IQueryable query) { +#if SDKPROJECTREF + if (container.Database.Client.ClientOptions.SerializerOptions.UseSystemTextJsonSerializerWithOptions) + { + if (container is not EncryptionContainerStream encryptionContainer) + { + throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainerStream)}."); + } + + return new EncryptionFeedIteratorStream( + (EncryptionFeedIteratorStream)encryptionContainer.ToEncryptionStreamIterator(query), + encryptionContainer.ResponseFactory); + } +#endif if (container is not EncryptionContainer encryptionContainer) { throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainer)}."); @@ -82,6 +102,22 @@ public static FeedIterator ToEncryptionStreamIterator( this Container container, IQueryable query) { +#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (container.Database.Client.ClientOptions.SerializerOptions.UseSystemTextJsonSerializerWithOptions) + { + if (container is not EncryptionContainerStream encryptionContainer) + { + throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainerStream)}."); + } + + return new EncryptionFeedIteratorStream( + query.ToStreamIterator(), + encryptionContainer.Encryptor, + encryptionContainer.CosmosSerializer, + new MemoryStreamManager()); + } +#endif + if (container is not EncryptionContainer encryptionContainer) { throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionStreamIterator)} is only supported with {nameof(EncryptionContainer)}."); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs new file mode 100644 index 0000000000..72dbeaca01 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs @@ -0,0 +1,1102 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + + using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; + + using Newtonsoft.Json.Linq; + + internal sealed class EncryptionContainerStream : Container + { + private readonly Container container; + + public CosmosSerializer CosmosSerializer { get; } + + public Encryptor Encryptor { get; } + + public CosmosResponseFactory ResponseFactory { get; } + + private readonly StreamManager streamManager; + + /// + /// All the operations / requests for exercising client-side encryption functionality need to be made using this EncryptionContainer instance. + /// + /// Regular cosmos container. + /// Provider that allows encrypting and decrypting data. + public EncryptionContainerStream( + Container container, + Encryptor encryptor) + { + this.container = container ?? throw new ArgumentNullException(nameof(container)); + this.Encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); + this.ResponseFactory = this.Database.Client.ResponseFactory; + this.CosmosSerializer = this.Database.Client.ClientOptions.Serializer; + this.streamManager = new MemoryStreamManager(); + } + + public override string Id => this.container.Id; + + public override Conflicts Conflicts => this.container.Conflicts; + + public override Scripts.Scripts Scripts => this.container.Scripts; + + public override Database Database => this.container.Database; + + public override async Task> CreateItemAsync( + T item, + PartitionKey? partitionKey = null, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + return await this.container.CreateItemAsync( + item, + partitionKey, + requestOptions, + cancellationToken); + } + + if (partitionKey == null) + { + throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); + } + + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("CreateItem")) + { + ResponseMessage responseMessage; + + if (item is EncryptableItemStream encryptableItemStream) + { + using Stream rms = this.streamManager.CreateStream(); + await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); + responseMessage = await this.CreateItemHelperAsync( + rms, + partitionKey.Value, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); + + return new EncryptionItemResponse(responseMessage, item); + } + else + { + using (Stream itemStream = this.CosmosSerializer.ToStream(item)) + { + responseMessage = await this.CreateItemHelperAsync( + itemStream, + partitionKey.Value, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + + return this.ResponseFactory.CreateItemResponse(responseMessage); + } + } + } + + public override async Task CreateItemStreamAsync( + Stream streamPayload, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(streamPayload); + + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("CreateItemStream")) + { + return await this.CreateItemHelperAsync( + streamPayload, + partitionKey, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + } + + private async Task CreateItemHelperAsync( + Stream streamPayload, + PartitionKey partitionKey, + ItemRequestOptions requestOptions, + bool decryptResponse, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + return await this.container.CreateItemStreamAsync( + streamPayload, + partitionKey, + requestOptions, + cancellationToken); + } + + using Stream encryptedStream = this.streamManager.CreateStream(); + await EncryptionProcessor.EncryptAsync( + streamPayload, + encryptedStream, + this.Encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken); + + ResponseMessage responseMessage = await this.container.CreateItemStreamAsync( + streamPayload, + partitionKey, + requestOptions, + cancellationToken); + + if (decryptResponse) + { + using Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + responseMessage.Content, + decryptedStream, + this.Encryptor, + diagnosticsContext, + encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, + cancellationToken); + responseMessage.Content = decryptedStream; + } + + return responseMessage; + } + + public override Task> DeleteItemAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.DeleteItemAsync( + id, + partitionKey, + requestOptions, + cancellationToken); + } + + public override Task DeleteItemStreamAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.DeleteItemStreamAsync( + id, + partitionKey, + requestOptions, + cancellationToken); + } + + public override async Task> ReadItemAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("ReadItem")) + { + ResponseMessage responseMessage; + + if (typeof(T) == typeof(DecryptableItem)) + { + responseMessage = await this.ReadItemHelperAsync( + id, + partitionKey, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + EncryptionItemRequestOptions options = requestOptions as EncryptionItemRequestOptions; + DecryptableItem decryptableItem = new DecryptableItemStream( + responseMessage.Content, + this.Encryptor, + options.EncryptionOptions.JsonProcessor, + this.CosmosSerializer, + this.streamManager); + + return new EncryptionItemResponse( + responseMessage, + (T)(object)decryptableItem); + } + + responseMessage = await this.ReadItemHelperAsync( + id, + partitionKey, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + + return this.ResponseFactory.CreateItemResponse(responseMessage); + } + } + + public override async Task ReadItemStreamAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("ReadItemStream")) + { + return await this.ReadItemHelperAsync( + id, + partitionKey, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + } + + private async Task ReadItemHelperAsync( + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions, + bool decryptResponse, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + ResponseMessage responseMessage = await this.container.ReadItemStreamAsync( + id, + partitionKey, + requestOptions, + cancellationToken); + + if (decryptResponse && requestOptions is EncryptionItemRequestOptions options) + { + using Stream rms = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync(responseMessage.Content, rms, this.Encryptor, diagnosticsContext, options.EncryptionOptions.JsonProcessor, cancellationToken); + responseMessage.Content = rms; + } + + return responseMessage; + } + + public override async Task> ReplaceItemAsync( + T item, + string id, + PartitionKey? partitionKey = null, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(item); + + if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + return await this.container.ReplaceItemAsync( + item, + id, + partitionKey, + requestOptions, + cancellationToken); + } + + if (partitionKey == null) + { + throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); + } + + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("ReplaceItem")) + { + ResponseMessage responseMessage; + + if (item is EncryptableItemStream encryptableItemStream) + { + using Stream rms = this.streamManager.CreateStream(); + await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); + responseMessage = await this.CreateItemHelperAsync( + rms, + partitionKey.Value, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); + + return new EncryptionItemResponse(responseMessage, item); + } + else + { + using (Stream itemStream = this.CosmosSerializer.ToStream(item)) + { + responseMessage = await this.ReplaceItemHelperAsync( + itemStream, + id, + partitionKey.Value, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + + return this.ResponseFactory.CreateItemResponse(responseMessage); + } + } + } + + public override async Task ReplaceItemStreamAsync( + Stream streamPayload, + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(streamPayload); + + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("ReplaceItemStream")) + { + return await this.ReplaceItemHelperAsync( + streamPayload, + id, + partitionKey, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + } + + private async Task ReplaceItemHelperAsync( + Stream streamPayload, + string id, + PartitionKey partitionKey, + ItemRequestOptions requestOptions, + bool decryptResponse, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + return await this.container.ReplaceItemStreamAsync( + streamPayload, + id, + partitionKey, + requestOptions, + cancellationToken); + } + + using Stream encryptedStream = this.streamManager.CreateStream(); + await EncryptionProcessor.EncryptAsync( + streamPayload, + encryptedStream, + this.Encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken); + streamPayload = encryptedStream; + + ResponseMessage responseMessage = await this.container.ReplaceItemStreamAsync( + streamPayload, + id, + partitionKey, + requestOptions, + cancellationToken); + + if (decryptResponse) + { + using Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + responseMessage.Content, + decryptedStream, + this.Encryptor, + diagnosticsContext, + encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, + cancellationToken); + responseMessage.Content = decryptedStream; + } + + return responseMessage; + } + + public override async Task> UpsertItemAsync( + T item, + PartitionKey? partitionKey = null, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + return await this.container.UpsertItemAsync( + item, + partitionKey, + requestOptions, + cancellationToken); + } + + if (partitionKey == null) + { + throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); + } + + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("UpsertItem")) + { + ResponseMessage responseMessage; + + if (item is EncryptableItemStream encryptableItemStream) + { + using Stream rms = this.streamManager.CreateStream(); + await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); + responseMessage = await this.UpsertItemHelperAsync( + rms, + partitionKey.Value, + requestOptions, + decryptResponse: false, + diagnosticsContext, + cancellationToken); + + encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); + + return new EncryptionItemResponse(responseMessage, item); + } + else + { + using (Stream itemStream = this.CosmosSerializer.ToStream(item)) + { + responseMessage = await this.UpsertItemHelperAsync( + itemStream, + partitionKey.Value, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + + return this.ResponseFactory.CreateItemResponse(responseMessage); + } + } + } + + public override async Task UpsertItemStreamAsync( + Stream streamPayload, + PartitionKey partitionKey, + ItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(streamPayload); + + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("UpsertItemStream")) + { + return await this.UpsertItemHelperAsync( + streamPayload, + partitionKey, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); + } + } + + private async Task UpsertItemHelperAsync( + Stream streamPayload, + PartitionKey partitionKey, + ItemRequestOptions requestOptions, + bool decryptResponse, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + return await this.container.UpsertItemStreamAsync( + streamPayload, + partitionKey, + requestOptions, + cancellationToken); + } + + using Stream rms = this.streamManager.CreateStream(); + await EncryptionProcessor.EncryptAsync( + streamPayload, + rms, + this.Encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken); + streamPayload = rms; + + ResponseMessage responseMessage = await this.container.UpsertItemStreamAsync( + streamPayload, + partitionKey, + requestOptions, + cancellationToken); + + if (decryptResponse) + { + using Stream decryptStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + responseMessage.Content, + decryptStream, + this.Encryptor, + diagnosticsContext, + encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, + cancellationToken); + responseMessage.Content = decryptStream; + } + + return responseMessage; + } + + public override TransactionalBatch CreateTransactionalBatch( + PartitionKey partitionKey) + { + return new EncryptionTransactionalBatchStream( + this.container.CreateTransactionalBatch(partitionKey), + this.Encryptor, + this.CosmosSerializer, + this.streamManager); + } + + public override Task DeleteContainerAsync( + ContainerRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.DeleteContainerAsync( + requestOptions, + cancellationToken); + } + + public override Task DeleteContainerStreamAsync( + ContainerRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.DeleteContainerStreamAsync( + requestOptions, + cancellationToken); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedEstimatorBuilder( + string processorName, + ChangesEstimationHandler estimationDelegate, + TimeSpan? estimationPeriod = null) + { + return this.container.GetChangeFeedEstimatorBuilder( + processorName, + estimationDelegate, + estimationPeriod); + } + + public override IOrderedQueryable GetItemLinqQueryable( + bool allowSynchronousQueryExecution = false, + string continuationToken = null, + QueryRequestOptions requestOptions = null, + CosmosLinqSerializerOptions linqSerializerOptions = null) + { + return this.container.GetItemLinqQueryable( + allowSynchronousQueryExecution, + continuationToken, + requestOptions, + linqSerializerOptions); + } + + public override FeedIterator GetItemQueryIterator( + QueryDefinition queryDefinition, + string continuationToken = null, + QueryRequestOptions requestOptions = null) + { + return new EncryptionFeedIteratorStream( + (EncryptionFeedIteratorStream)this.GetItemQueryStreamIterator( + queryDefinition, + continuationToken, + requestOptions), + this.ResponseFactory); + } + + public override FeedIterator GetItemQueryIterator( + string queryText = null, + string continuationToken = null, + QueryRequestOptions requestOptions = null) + { + return new EncryptionFeedIteratorStream( + (EncryptionFeedIteratorStream)this.GetItemQueryStreamIterator( + queryText, + continuationToken, + requestOptions), + this.ResponseFactory); + } + + public override Task ReadContainerAsync( + ContainerRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.ReadContainerAsync( + requestOptions, + cancellationToken); + } + + public override Task ReadContainerStreamAsync( + ContainerRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.ReadContainerStreamAsync( + requestOptions, + cancellationToken); + } + + public override Task ReadThroughputAsync( + CancellationToken cancellationToken = default) + { + return this.container.ReadThroughputAsync(cancellationToken); + } + + public override Task ReadThroughputAsync( + RequestOptions requestOptions, + CancellationToken cancellationToken = default) + { + return this.container.ReadThroughputAsync( + requestOptions, + cancellationToken); + } + + public override Task ReplaceContainerAsync( + ContainerProperties containerProperties, + ContainerRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.ReplaceContainerAsync( + containerProperties, + requestOptions, + cancellationToken); + } + + public override Task ReplaceContainerStreamAsync( + ContainerProperties containerProperties, + ContainerRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.ReplaceContainerStreamAsync( + containerProperties, + requestOptions, + cancellationToken); + } + + public override Task ReplaceThroughputAsync( + int throughput, + RequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.ReplaceThroughputAsync( + throughput, + requestOptions, + cancellationToken); + } + + public override FeedIterator GetItemQueryStreamIterator( + QueryDefinition queryDefinition, + string continuationToken = null, + QueryRequestOptions requestOptions = null) + { + return new EncryptionFeedIteratorStream( + this.container.GetItemQueryStreamIterator( + queryDefinition, + continuationToken, + requestOptions), + this.Encryptor, + this.CosmosSerializer, + this.streamManager); + } + + public override FeedIterator GetItemQueryStreamIterator( + string queryText = null, + string continuationToken = null, + QueryRequestOptions requestOptions = null) + { + return new EncryptionFeedIteratorStream( + this.container.GetItemQueryStreamIterator( + queryText, + continuationToken, + requestOptions), + this.Encryptor, + this.CosmosSerializer, + this.streamManager); + } + + public override Task ReplaceThroughputAsync( + ThroughputProperties throughputProperties, + RequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.ReplaceThroughputAsync( + throughputProperties, + requestOptions, + cancellationToken); + } + + public override Task> GetFeedRangesAsync( + CancellationToken cancellationToken = default) + { + return this.container.GetFeedRangesAsync(cancellationToken); + } + + public override FeedIterator GetItemQueryStreamIterator( + FeedRange feedRange, + QueryDefinition queryDefinition, + string continuationToken, + QueryRequestOptions requestOptions = null) + { + return new EncryptionFeedIteratorStream( + this.container.GetItemQueryStreamIterator( + feedRange, + queryDefinition, + continuationToken, + requestOptions), + this.Encryptor, + this.CosmosSerializer, + this.streamManager); + } + + public override FeedIterator GetItemQueryIterator( + FeedRange feedRange, + QueryDefinition queryDefinition, + string continuationToken = null, + QueryRequestOptions requestOptions = null) + { + return new EncryptionFeedIteratorStream( + (EncryptionFeedIteratorStream)this.GetItemQueryStreamIterator( + feedRange, + queryDefinition, + continuationToken, + requestOptions), + this.ResponseFactory); + } + + public override ChangeFeedEstimator GetChangeFeedEstimator( + string processorName, + Container leaseContainer) + { + return this.container.GetChangeFeedEstimator(processorName, leaseContainer); + } + + public override FeedIterator GetChangeFeedStreamIterator( + ChangeFeedStartFrom changeFeedStartFrom, + ChangeFeedMode changeFeedMode, + ChangeFeedRequestOptions changeFeedRequestOptions = null) + { + return new EncryptionFeedIteratorStream( + this.container.GetChangeFeedStreamIterator( + changeFeedStartFrom, + changeFeedMode, + changeFeedRequestOptions), + this.Encryptor, + this.CosmosSerializer, + this.streamManager); + } + + public override FeedIterator GetChangeFeedIterator( + ChangeFeedStartFrom changeFeedStartFrom, + ChangeFeedMode changeFeedMode, + ChangeFeedRequestOptions changeFeedRequestOptions = null) + { + return new EncryptionFeedIteratorStream( + (EncryptionFeedIteratorStream)this.GetChangeFeedStreamIterator( + changeFeedStartFrom, + changeFeedMode, + changeFeedRequestOptions), + this.ResponseFactory); + } + + public override Task> PatchItemAsync( + string id, + PartitionKey partitionKey, + IReadOnlyList patchOperations, + PatchItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task PatchItemStreamAsync( + string id, + PartitionKey partitionKey, + IReadOnlyList patchOperations, + PatchItemRequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( + string processorName, + ChangesHandler onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilder( + processorName, + async ( + IReadOnlyCollection documents, + CancellationToken cancellationToken) => + { + List decryptItems = await this.DecryptChangeFeedDocumentsAsync( + documents, + cancellationToken); + + // Call the original passed in delegate + await onChangesDelegate(decryptItems, cancellationToken); + }); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( + string processorName, + ChangeFeedHandler onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilder( + processorName, + async ( + ChangeFeedProcessorContext context, + IReadOnlyCollection documents, + CancellationToken cancellationToken) => + { + List decryptItems = await this.DecryptChangeFeedDocumentsAsync( + documents, + cancellationToken); + + // Call the original passed in delegate + await onChangesDelegate(context, decryptItems, cancellationToken); + }); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( + string processorName, + ChangeFeedHandlerWithManualCheckpoint onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilderWithManualCheckpoint( + processorName, + async ( + ChangeFeedProcessorContext context, + IReadOnlyCollection documents, + Func tryCheckpointAsync, + CancellationToken cancellationToken) => + { + List decryptItems = await this.DecryptChangeFeedDocumentsAsync( + documents, + cancellationToken); + + // Call the original passed in delegate + await onChangesDelegate(context, decryptItems, tryCheckpointAsync, cancellationToken); + }); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( + string processorName, + ChangeFeedStreamHandler onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilder( + processorName, + async ( + ChangeFeedProcessorContext context, + Stream changes, + CancellationToken cancellationToken) => + { + using Stream decryptedChanges = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync( + changes, + decryptedChanges, + this.Encryptor, + this.streamManager, + cancellationToken); + + // Call the original passed in delegate + await onChangesDelegate(context, decryptedChanges, cancellationToken); + }); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( + string processorName, + ChangeFeedStreamHandlerWithManualCheckpoint onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilderWithManualCheckpoint( + processorName, + async ( + ChangeFeedProcessorContext context, + Stream changes, + Func tryCheckpointAsync, + CancellationToken cancellationToken) => + { + using Stream decryptedChanges = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync( + changes, + decryptedChanges, + this.Encryptor, + this.streamManager, + cancellationToken); + + // Call the original passed in delegate + await onChangesDelegate(context, decryptedChanges, tryCheckpointAsync, cancellationToken); + }); + } + + public override Task ReadManyItemsStreamAsync( + IReadOnlyList<(string id, PartitionKey partitionKey)> items, + ReadManyRequestOptions readManyRequestOptions = null, + CancellationToken cancellationToken = default) + { + return this.ReadManyItemsHelperAsync( + items, + readManyRequestOptions, + cancellationToken); + } + + public override async Task> ReadManyItemsAsync( + IReadOnlyList<(string id, PartitionKey partitionKey)> items, + ReadManyRequestOptions readManyRequestOptions = null, + CancellationToken cancellationToken = default) + { + ResponseMessage responseMessage = await this.ReadManyItemsHelperAsync( + items, + readManyRequestOptions, + cancellationToken); + + return this.ResponseFactory.CreateItemFeedResponse(responseMessage); + } + +#if ENCRYPTIONPREVIEW + public override Task> GetPartitionKeyRangesAsync( + FeedRange feedRange, + CancellationToken cancellationToken = default) + { + return this.container.GetPartitionKeyRangesAsync(feedRange, cancellationToken); + } + + public override Task DeleteAllItemsByPartitionKeyStreamAsync( + Cosmos.PartitionKey partitionKey, + RequestOptions requestOptions = null, + CancellationToken cancellationToken = default) + { + return this.container.DeleteAllItemsByPartitionKeyStreamAsync( + partitionKey, + requestOptions, + cancellationToken); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(string processorName, ChangeFeedHandler> onChangesDelegate) + { + return this.container.GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes( + processorName, + onChangesDelegate); + } +#endif + +#if SDKPROJECTREF + public override Task IsFeedRangePartOfAsync( + Cosmos.FeedRange x, + Cosmos.FeedRange y, + CancellationToken cancellationToken = default) + { + return this.container.IsFeedRangePartOfAsync( + x, + y, + cancellationToken); + } +#endif + + private async Task ReadManyItemsHelperAsync( + IReadOnlyList<(string id, PartitionKey partitionKey)> items, + ReadManyRequestOptions readManyRequestOptions = null, + CancellationToken cancellationToken = default) + { + ResponseMessage responseMessage = await this.container.ReadManyItemsStreamAsync( + items, + readManyRequestOptions, + cancellationToken); + + using Stream decryptedStream = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync(responseMessage.Content, decryptedStream, this.Encryptor, this.streamManager, cancellationToken); + + return new DecryptedResponseMessage(responseMessage, decryptedStream); + } + + private async Task> DecryptChangeFeedDocumentsAsync( + IReadOnlyCollection documents, + CancellationToken cancellationToken) + { + List decryptItems = new (documents.Count); + if (typeof(T) == typeof(DecryptableItem)) + { + foreach (Stream documentStream in documents) + { + DecryptableItemStream item = new ( + documentStream, + this.Encryptor, + JsonProcessor.Stream, + this.CosmosSerializer, + this.streamManager); + + decryptItems.Add((T)(object)item); + } + } + else + { + foreach (Stream document in documents) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); + using (diagnosticsContext.CreateScope("DecryptChangeFeedDocumentsAsync<")) + { + using Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + document, + decryptedStream, + this.Encryptor, + diagnosticsContext, + JsonProcessor.Stream, + cancellationToken); + +#if SDKPROJECTREF + decryptItems.Add(await this.CosmosSerializer.FromStreamAsync(decryptedStream, cancellationToken)); +#else + decryptItems.Add(this.CosmosSerializer.FromStream(decryptedStream)); +#endif + + await decryptedStream.DisposeAsync(); + } + } + } + + return decryptItems; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs new file mode 100644 index 0000000000..6de6c1aeae --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs @@ -0,0 +1,106 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.Collections.Generic; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + + internal sealed class EncryptionFeedIteratorStream : FeedIterator + { + private readonly FeedIterator feedIterator; + private readonly Encryptor encryptor; + private readonly CosmosSerializer cosmosSerializer; + private readonly StreamManager streamManager; + + private static readonly ArrayStreamSplitter StreamSplitter = new (); + + public EncryptionFeedIteratorStream( + FeedIterator feedIterator, + Encryptor encryptor, + CosmosSerializer cosmosSerializer, + StreamManager streamManager) + { + this.feedIterator = feedIterator; + this.encryptor = encryptor; + this.cosmosSerializer = cosmosSerializer; + this.streamManager = streamManager; + } + + public override bool HasMoreResults => this.feedIterator.HasMoreResults; + + public override async Task ReadNextAsync(CancellationToken cancellationToken = default) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null); + using (diagnosticsContext.CreateScope("FeedIterator.ReadNext")) + { + ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken); + + if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) + { + using Stream decryptedContent = this.streamManager.CreateStream(); + await EncryptionProcessor.DeserializeAndDecryptResponseAsync( + responseMessage.Content, + decryptedContent, + this.encryptor, + this.streamManager, + cancellationToken); + + return new DecryptedResponseMessage(responseMessage, decryptedContent); + } + + return responseMessage; + } + } + + public async Task<(ResponseMessage, List)> ReadNextWithoutDecryptionAsync(CancellationToken cancellationToken = default) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null); + using (diagnosticsContext.CreateScope("FeedIterator.ReadNextWithoutDecryption")) + { + ResponseMessage responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken); + List decryptableContent = null; + + if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) + { + decryptableContent = await this.ConvertResponseToDecryptableItemsAsync( + responseMessage.Content, + cancellationToken); + + return (responseMessage, decryptableContent); + } + + return (responseMessage, decryptableContent); + } + } + + private async Task> ConvertResponseToDecryptableItemsAsync( + Stream content, + CancellationToken token) + { + List decryptableStreams = await StreamSplitter.SplitCollectionAsync(content, this.streamManager, token); + List decryptableItems = new (); + + foreach (Stream item in decryptableStreams) + { + decryptableItems.Add( + (T)(object)new DecryptableItemStream( + item, + this.encryptor, + JsonProcessor.Stream, + this.cosmosSerializer, + this.streamManager)); + } + + return decryptableItems; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream{T}.cs new file mode 100644 index 0000000000..a2b050e529 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream{T}.cs @@ -0,0 +1,50 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + + internal sealed class EncryptionFeedIteratorStream : FeedIterator + { + private readonly EncryptionFeedIteratorStream feedIterator; + private readonly CosmosResponseFactory responseFactory; + + public EncryptionFeedIteratorStream( + EncryptionFeedIteratorStream feedIterator, + CosmosResponseFactory responseFactory) + { + this.feedIterator = feedIterator ?? throw new ArgumentNullException(nameof(feedIterator)); + this.responseFactory = responseFactory ?? throw new ArgumentNullException(nameof(responseFactory)); + } + + public override bool HasMoreResults => this.feedIterator.HasMoreResults; + + public override async Task> ReadNextAsync(CancellationToken cancellationToken = default) + { + ResponseMessage responseMessage; + + if (typeof(T) == typeof(DecryptableItem)) + { + IReadOnlyCollection resource; + (responseMessage, resource) = await this.feedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); + + return DecryptableFeedResponse.CreateResponse( + responseMessage, + resource); + } + else + { + responseMessage = await this.feedIterator.ReadNextAsync(cancellationToken); + } + + return this.responseFactory.CreateItemFeedResponse(responseMessage); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs new file mode 100644 index 0000000000..284eb390b5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs @@ -0,0 +1,293 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos; + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits", Justification = "To be fixed, tracked in issue #1575")] + internal sealed class EncryptionTransactionalBatchStream : TransactionalBatch + { + private readonly Encryptor encryptor; + private readonly CosmosSerializer cosmosSerializer; + private readonly StreamManager streamManager; + + private TransactionalBatch transactionalBatch; + + public EncryptionTransactionalBatchStream( + TransactionalBatch transactionalBatch, + Encryptor encryptor, + CosmosSerializer cosmosSerializer, + StreamManager streamManager) + { + this.transactionalBatch = transactionalBatch ?? throw new ArgumentNullException(nameof(transactionalBatch)); + this.encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); + this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); + this.streamManager = streamManager ?? throw new ArgumentNullException(nameof(streamManager)); + } + + public override TransactionalBatch CreateItem( + T item, + TransactionalBatchItemRequestOptions requestOptions = null) + { + if (requestOptions is not EncryptionTransactionalBatchItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + this.transactionalBatch = this.transactionalBatch.CreateItem( + item, + requestOptions); + + return this; + } + +#if SDKPROJECTREF + using Stream itemStream = this.streamManager.CreateStream(); + this.cosmosSerializer.ToStreamAsync(item, itemStream, CancellationToken.None).GetAwaiter().GetResult(); +#else + Stream itemStream = this.cosmosSerializer.ToStream(item); +#endif + return this.CreateItemStream( + itemStream, + requestOptions); + } + + public override TransactionalBatch CreateItemStream( + Stream streamPayload, + TransactionalBatchItemRequestOptions requestOptions = null) + { + if (requestOptions is EncryptionTransactionalBatchItemRequestOptions encryptionItemRequestOptions && + encryptionItemRequestOptions.EncryptionOptions != null) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("EncryptItemStream")) + { + using Stream temp = this.streamManager.CreateStream(); + EncryptionProcessor.EncryptAsync( + streamPayload, + temp, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).GetAwaiter().GetResult(); + streamPayload = temp; + } + } + + this.transactionalBatch = this.transactionalBatch.CreateItemStream( + streamPayload, + requestOptions); + + return this; + } + + public override TransactionalBatch DeleteItem( + string id, + TransactionalBatchItemRequestOptions requestOptions = null) + { + this.transactionalBatch = this.transactionalBatch.DeleteItem( + id, + requestOptions); + + return this; + } + + public override TransactionalBatch ReadItem( + string id, + TransactionalBatchItemRequestOptions requestOptions = null) + { + this.transactionalBatch = this.transactionalBatch.ReadItem( + id, + requestOptions); + + return this; + } + + public override TransactionalBatch ReplaceItem( + string id, + T item, + TransactionalBatchItemRequestOptions requestOptions = null) + { + if (requestOptions is not EncryptionTransactionalBatchItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + this.transactionalBatch = this.transactionalBatch.ReplaceItem( + id, + item, + requestOptions); + + return this; + } +#if SDKPROJECTREF + using Stream itemStream = this.streamManager.CreateStream(); + this.cosmosSerializer.ToStreamAsync(item, itemStream, CancellationToken.None).GetAwaiter().GetResult(); +#else + Stream itemStream = this.cosmosSerializer.ToStream(item); +#endif + return this.ReplaceItemStream( + id, + itemStream, + requestOptions); + } + + public override TransactionalBatch ReplaceItemStream( + string id, + Stream streamPayload, + TransactionalBatchItemRequestOptions requestOptions = null) + { + if (requestOptions is EncryptionTransactionalBatchItemRequestOptions encryptionItemRequestOptions && + encryptionItemRequestOptions.EncryptionOptions != null) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + Stream temp = this.streamManager.CreateStream(); + EncryptionProcessor.EncryptAsync( + streamPayload, + temp, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).GetAwaiter().GetResult(); + streamPayload = temp; + } + + this.transactionalBatch = this.transactionalBatch.ReplaceItemStream( + id, + streamPayload, + requestOptions); + + return this; + } + + public override TransactionalBatch UpsertItem( + T item, + TransactionalBatchItemRequestOptions requestOptions = null) + { + if (requestOptions is not EncryptionTransactionalBatchItemRequestOptions encryptionItemRequestOptions || + encryptionItemRequestOptions.EncryptionOptions == null) + { + this.transactionalBatch = this.transactionalBatch.UpsertItem( + item, + requestOptions); + + return this; + } + +#if SDKPROJECTREF + using Stream itemStream = this.streamManager.CreateStream(); + this.cosmosSerializer.ToStreamAsync(item, itemStream, CancellationToken.None).GetAwaiter().GetResult(); +#else + Stream itemStream = this.cosmosSerializer.ToStream(item); +#endif + return this.UpsertItemStream( + itemStream, + requestOptions); + } + + public override TransactionalBatch UpsertItemStream( + Stream streamPayload, + TransactionalBatchItemRequestOptions requestOptions = null) + { + if (requestOptions is EncryptionTransactionalBatchItemRequestOptions encryptionItemRequestOptions && + encryptionItemRequestOptions.EncryptionOptions != null) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); + using (diagnosticsContext.CreateScope("EncryptItemStream")) + { + Stream temp = this.streamManager.CreateStream(); + EncryptionProcessor.EncryptAsync( + streamPayload, + temp, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).GetAwaiter().GetResult(); + streamPayload = temp; + } + } + + this.transactionalBatch = this.transactionalBatch.UpsertItemStream( + streamPayload, + requestOptions); + + return this; + } + + public override async Task ExecuteAsync( + CancellationToken cancellationToken = default) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null); + using (diagnosticsContext.CreateScope("TransactionalBatch.ExecuteAsync")) + { + TransactionalBatchResponse response = await this.transactionalBatch.ExecuteAsync(cancellationToken); + return await this.DecryptTransactionalBatchResponseAsync( + response, + diagnosticsContext, + cancellationToken); + } + } + + public override async Task ExecuteAsync( + TransactionalBatchRequestOptions requestOptions, + CancellationToken cancellationToken = default) + { + CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(options: null); + using (diagnosticsContext.CreateScope("TransactionalBatch.ExecuteAsync.WithRequestOptions")) + { + TransactionalBatchResponse response = await this.transactionalBatch.ExecuteAsync(requestOptions, cancellationToken); + return await this.DecryptTransactionalBatchResponseAsync( + response, + diagnosticsContext, + cancellationToken); + } + } + + private async Task DecryptTransactionalBatchResponseAsync( + TransactionalBatchResponse response, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + List decryptedTransactionalBatchOperationResults = new (); + + foreach (TransactionalBatchOperationResult result in response) + { + if (response.IsSuccessStatusCode && result.ResourceStream != null) + { + Stream decryptedStream = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync( + result.ResourceStream, + decryptedStream, + this.encryptor, + diagnosticsContext, + JsonProcessor.Stream, + cancellationToken); + + decryptedTransactionalBatchOperationResults.Add(new EncryptionTransactionalBatchOperationResult(result, decryptedStream)); + } + else + { + decryptedTransactionalBatchOperationResults.Add(result); + } + } + + return new EncryptionTransactionalBatchResponse( + decryptedTransactionalBatchOperationResults, + response, + this.cosmosSerializer); + } + + public override TransactionalBatch PatchItem( + string id, + IReadOnlyList patchOperations, + TransactionalBatchPatchItemRequestOptions requestOptions = null) + { + throw new NotImplementedException(); + } + } +} +#endif \ No newline at end of file From a7a66b1cfe7649d2f9e50102324601f16ed09711 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 12:32:46 +0200 Subject: [PATCH 73/85] ~ cleanup --- .../src/EncryptableItem.cs | 2 - .../src/EncryptableItemStream.cs | 35 ++- .../src/EncryptableItem{T}.cs | 29 ++ .../src/EncryptionContainer.cs | 295 +----------------- .../src/EncryptionFeedIterator.cs | 66 +--- .../src/EncryptionTransactionalBatch.cs | 81 +---- .../RecyclableMemoryStream.cs | 2 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 9 - .../Transformation/JsonBytesConverterTests.cs | 40 --- .../Transformation/JsonBytesTests.cs | 33 -- 10 files changed, 74 insertions(+), 518 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs index 5351967880..174cf504e8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem.cs @@ -26,7 +26,6 @@ public abstract class EncryptableItem : IDisposable /// /// Cosmos Serializer /// Input payload in stream format - [Obsolete("Use overload with outputStream")] protected internal abstract Stream ToStream(CosmosSerializer serializer); /// @@ -44,7 +43,6 @@ public abstract class EncryptableItem : IDisposable /// The encrypted content which is yet to be decrypted. /// Encryptor instance which will be used for decryption. /// Serializer instance which will be used for deserializing the content after decryption. - [Obsolete("Use overload with decryptableStream")] protected internal abstract void SetDecryptableItem( JToken decryptableContent, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItemStream.cs index f96f4172d0..5a794d4817 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItemStream.cs @@ -6,6 +6,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; using System.IO; + using System.Threading; + using System.Threading.Tasks; using Newtonsoft.Json.Linq; /// @@ -69,10 +71,19 @@ protected internal override void SetDecryptableItem( cosmosSerializer); } + private void Dispose(bool disposing) + { + if (disposing) + { + this.StreamPayload?.Dispose(); + this.DecryptableItem?.Dispose(); + } + } + /// - public void Dispose() + public override void Dispose() { - this.StreamPayload.Dispose(); + this.Dispose(true); } /// @@ -80,5 +91,25 @@ protected internal override Stream ToStream(CosmosSerializer serializer) { return this.StreamPayload; } + + /// + /// This solution is not performant with Newtonsoft.Json. + protected internal override async Task ToStreamAsync(CosmosSerializer serializer, Stream outputStream, CancellationToken cancellationToken) + { +#if NET8_0_OR_GREATER + await this.StreamPayload.CopyToAsync(outputStream, cancellationToken); +#else + await this.StreamPayload.CopyToAsync(outputStream, 81920, cancellationToken); +#endif + } + +#if NET8_0_OR_GREATER + /// + /// Direct stream based item is not supported with Newtonsoft.Json. + protected internal override void SetDecryptableStream(Stream decryptableStream, Encryptor encryptor, JsonProcessor jsonProcessor, CosmosSerializer cosmosSerializer, StreamManager streamManager) + { + throw new NotImplementedException("Stream based item is only allowed for EncryptionContainerStream"); + } +#endif } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem{T}.cs index d6d6531e1c..7aef884c8a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptableItem{T}.cs @@ -6,6 +6,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; using System.IO; + using System.Threading; + using System.Threading.Tasks; using Newtonsoft.Json.Linq; /// @@ -85,5 +87,32 @@ protected internal override Stream ToStream(CosmosSerializer serializer) { return serializer.ToStream(this.Item); } + + /// + /// This solution is not performant with Newtonsoft.Json. + protected internal override async Task ToStreamAsync(CosmosSerializer serializer, Stream outputStream, CancellationToken cancellationToken) + { + Stream temp = serializer.ToStream(this.Item); +#if NET8_0_OR_GREATER + await temp.CopyToAsync(outputStream, cancellationToken); +#else + await temp.CopyToAsync(outputStream, 81920, cancellationToken); +#endif + } + +#if NET8_0_OR_GREATER + /// + /// Direct stream based item is not supported with Newtonsoft.Json. + protected internal override void SetDecryptableStream(Stream decryptableStream, Encryptor encryptor, JsonProcessor jsonProcessor, CosmosSerializer cosmosSerializer, StreamManager streamManager) + { + throw new NotImplementedException("Stream based item is only allowed for EncryptionContainerStream"); + } +#endif + + /// + /// Does nothing with Newtonsoft based EncryptableItem. + public override void Dispose() + { + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index 060fb79bdc..f769e1504a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -10,9 +10,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Linq; using System.Threading; using System.Threading.Tasks; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; -#endif using Newtonsoft.Json.Linq; internal sealed class EncryptionContainer : Container @@ -25,10 +22,6 @@ internal sealed class EncryptionContainer : Container public CosmosResponseFactory ResponseFactory { get; } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private readonly StreamManager streamManager; -#endif - /// /// All the operations / requests for exercising client-side encryption functionality need to be made using this EncryptionContainer instance. /// @@ -42,9 +35,6 @@ public EncryptionContainer( this.Encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); this.ResponseFactory = this.Database.Client.ResponseFactory; this.CosmosSerializer = this.Database.Client.ClientOptions.Serializer; -#if NET8_0_OR_GREATER - this.streamManager = new MemoryStreamManager(); -#endif } public override string Id => this.container.Id; @@ -86,27 +76,8 @@ public override async Task> CreateItemAsync( { ResponseMessage responseMessage; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (item is EncryptableItemStream encryptableItemStream) - { - using Stream rms = this.streamManager.CreateStream(); - await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); - responseMessage = await this.CreateItemHelperAsync( - rms, - partitionKey.Value, - requestOptions, - decryptResponse: false, - diagnosticsContext, - cancellationToken); - - encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); - - return new EncryptionItemResponse(responseMessage, item); - } -#endif if (item is EncryptableItem encryptableItem) { -#pragma warning disable CS0618 // Type or member is obsolete using (Stream streamPayload = encryptableItem.ToStream(this.CosmosSerializer)) { responseMessage = await this.CreateItemHelperAsync( @@ -117,14 +88,11 @@ public override async Task> CreateItemAsync( diagnosticsContext, cancellationToken); } -#pragma warning restore CS0618 // Type or member is obsolete -#pragma warning disable CS0618 // Type or member is obsolete encryptableItem.SetDecryptableItem( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); -#pragma warning restore CS0618 // Type or member is obsolete return new EncryptionItemResponse( responseMessage, @@ -266,20 +234,10 @@ public override async Task> ReadItemAsync( diagnosticsContext, cancellationToken); -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - EncryptionItemRequestOptions options = requestOptions as EncryptionItemRequestOptions; - DecryptableItem decryptableItem = new DecryptableItemStream( - responseMessage.Content, - this.Encryptor, - options.EncryptionOptions.JsonProcessor, - this.CosmosSerializer, - this.streamManager); -#else DecryptableItemCore decryptableItem = new ( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); -#endif return new EncryptionItemResponse( responseMessage, @@ -333,31 +291,11 @@ private async Task ReadItemHelperAsync( if (decryptResponse) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - using Stream rms = this.streamManager.CreateStream(); - - EncryptionItemRequestOptions options = requestOptions as EncryptionItemRequestOptions; - - if (options?.EncryptionOptions != null) - { - _ = await EncryptionProcessor.DecryptAsync(responseMessage.Content, rms, this.Encryptor, diagnosticsContext, options.EncryptionOptions.JsonProcessor, cancellationToken); - responseMessage.Content = rms; - } - else - { - (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( - responseMessage.Content, - this.Encryptor, - diagnosticsContext, - cancellationToken); - } -#else (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( responseMessage.Content, this.Encryptor, diagnosticsContext, cancellationToken); -#endif } return responseMessage; @@ -406,28 +344,8 @@ public override async Task> ReplaceItemAsync( { ResponseMessage responseMessage; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (item is EncryptableItemStream encryptableItemStream) - { - using Stream rms = this.streamManager.CreateStream(); - await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); - responseMessage = await this.CreateItemHelperAsync( - rms, - partitionKey.Value, - requestOptions, - decryptResponse: false, - diagnosticsContext, - cancellationToken); - - encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); - - return new EncryptionItemResponse(responseMessage, item); - } -#endif - if (item is EncryptableItem encryptableItem) { -#pragma warning disable CS0618 // Type or member is obsolete using (Stream streamPayload = encryptableItem.ToStream(this.CosmosSerializer)) { responseMessage = await this.ReplaceItemHelperAsync( @@ -439,14 +357,11 @@ public override async Task> ReplaceItemAsync( diagnosticsContext, cancellationToken); } -#pragma warning restore CS0618 // Type or member is obsolete -#pragma warning disable CS0618 // Type or member is obsolete encryptableItem.SetDecryptableItem( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); -#pragma warning restore CS0618 // Type or member is obsolete return new EncryptionItemResponse( responseMessage, @@ -527,24 +442,12 @@ private async Task ReplaceItemHelperAsync( cancellationToken); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream encryptedStream = this.streamManager.CreateStream(); - await EncryptionProcessor.EncryptAsync( - streamPayload, - encryptedStream, - this.Encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken); - streamPayload = encryptedStream; -#else streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken); -#endif ResponseMessage responseMessage = await this.container.ReplaceItemStreamAsync( streamPayload, @@ -555,23 +458,11 @@ await EncryptionProcessor.EncryptAsync( if (decryptResponse) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream decryptedStream = this.streamManager.CreateStream(); - _ = await EncryptionProcessor.DecryptAsync( - responseMessage.Content, - decryptedStream, - this.Encryptor, - diagnosticsContext, - encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, - cancellationToken); - responseMessage.Content = decryptedStream; -#else (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( responseMessage.Content, this.Encryptor, diagnosticsContext, cancellationToken); -#endif } return responseMessage; @@ -608,28 +499,8 @@ public override async Task> UpsertItemAsync( { ResponseMessage responseMessage; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (item is EncryptableItemStream encryptableItemStream) - { - using Stream rms = this.streamManager.CreateStream(); - await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); - responseMessage = await this.UpsertItemHelperAsync( - rms, - partitionKey.Value, - requestOptions, - decryptResponse: false, - diagnosticsContext, - cancellationToken); - - encryptableItemStream.SetDecryptableStream(responseMessage.Content, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, this.CosmosSerializer, this.streamManager); - - return new EncryptionItemResponse(responseMessage, item); - } -#endif - if (item is EncryptableItem encryptableItem) { -#pragma warning disable CS0618 // Type or member is obsolete using (Stream streamPayload = encryptableItem.ToStream(this.CosmosSerializer)) { responseMessage = await this.UpsertItemHelperAsync( @@ -640,14 +511,11 @@ public override async Task> UpsertItemAsync( diagnosticsContext, cancellationToken); } -#pragma warning restore CS0618 // Type or member is obsolete -#pragma warning disable CS0618 // Type or member is obsolete encryptableItem.SetDecryptableItem( EncryptionProcessor.BaseSerializer.FromStream(responseMessage.Content), this.Encryptor, this.CosmosSerializer); -#pragma warning restore CS0618 // Type or member is obsolete return new EncryptionItemResponse( responseMessage, @@ -717,24 +585,12 @@ private async Task UpsertItemHelperAsync( cancellationToken); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream rms = this.streamManager.CreateStream(); - await EncryptionProcessor.EncryptAsync( - streamPayload, - rms, - this.Encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken); - streamPayload = rms; -#else streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken); -#endif ResponseMessage responseMessage = await this.container.UpsertItemStreamAsync( streamPayload, @@ -744,23 +600,11 @@ await EncryptionProcessor.EncryptAsync( if (decryptResponse) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream decryptStream = this.streamManager.CreateStream(); - _ = await EncryptionProcessor.DecryptAsync( - responseMessage.Content, - decryptStream, - this.Encryptor, - diagnosticsContext, - encryptionItemRequestOptions.EncryptionOptions.JsonProcessor, - cancellationToken); - responseMessage.Content = decryptStream; -#else (responseMessage.Content, _) = await EncryptionProcessor.DecryptAsync( responseMessage.Content, this.Encryptor, diagnosticsContext, cancellationToken); -#endif } return responseMessage; @@ -1039,26 +883,6 @@ public override Task PatchItemStreamAsync( throw new NotImplementedException(); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( - string processorName, - ChangesHandler onChangesDelegate) - { - return this.container.GetChangeFeedProcessorBuilder( - processorName, - async ( - IReadOnlyCollection documents, - CancellationToken cancellationToken) => - { - List decryptItems = await this.DecryptChangeFeedDocumentsAsync( - documents, - cancellationToken); - - // Call the original passed in delegate - await onChangesDelegate(decryptItems, cancellationToken); - }); - } -#else public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( string processorName, ChangesHandler onChangesDelegate) @@ -1077,29 +901,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( await onChangesDelegate(decryptItems, cancellationToken); }); } -#endif -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( - string processorName, - ChangeFeedHandler onChangesDelegate) - { - return this.container.GetChangeFeedProcessorBuilder( - processorName, - async ( - ChangeFeedProcessorContext context, - IReadOnlyCollection documents, - CancellationToken cancellationToken) => - { - List decryptItems = await this.DecryptChangeFeedDocumentsAsync( - documents, - cancellationToken); - - // Call the original passed in delegate - await onChangesDelegate(context, decryptItems, cancellationToken); - }); - } -#else public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( string processorName, ChangeFeedHandler onChangesDelegate) @@ -1119,30 +921,7 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( await onChangesDelegate(context, decryptItems, cancellationToken); }); } -#endif -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( - string processorName, - ChangeFeedHandlerWithManualCheckpoint onChangesDelegate) - { - return this.container.GetChangeFeedProcessorBuilderWithManualCheckpoint( - processorName, - async ( - ChangeFeedProcessorContext context, - IReadOnlyCollection documents, - Func tryCheckpointAsync, - CancellationToken cancellationToken) => - { - List decryptItems = await this.DecryptChangeFeedDocumentsAsync( - documents, - cancellationToken); - - // Call the original passed in delegate - await onChangesDelegate(context, decryptItems, tryCheckpointAsync, cancellationToken); - }); - } -#else public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManualCheckpoint( string processorName, ChangeFeedHandlerWithManualCheckpoint onChangesDelegate) @@ -1163,7 +942,6 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu await onChangesDelegate(context, decryptItems, tryCheckpointAsync, cancellationToken); }); } -#endif public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( string processorName, @@ -1176,20 +954,10 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilder( Stream changes, CancellationToken cancellationToken) => { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream decryptedChanges = this.streamManager.CreateStream(); - await EncryptionProcessor.DeserializeAndDecryptResponseAsync( - changes, - decryptedChanges, - this.Encryptor, - this.streamManager, - cancellationToken); -#else Stream decryptedChanges = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( changes, this.Encryptor, cancellationToken); -#endif // Call the original passed in delegate await onChangesDelegate(context, decryptedChanges, cancellationToken); @@ -1208,20 +976,10 @@ public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithManu Func tryCheckpointAsync, CancellationToken cancellationToken) => { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream decryptedChanges = this.streamManager.CreateStream(); - await EncryptionProcessor.DeserializeAndDecryptResponseAsync( - changes, - decryptedChanges, - this.Encryptor, - this.streamManager, - cancellationToken); -#else Stream decryptedChanges = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( changes, this.Encryptor, cancellationToken); -#endif // Call the original passed in delegate await onChangesDelegate(context, decryptedChanges, tryCheckpointAsync, cancellationToken); @@ -1347,56 +1105,5 @@ private async Task> DecryptChangeFeedDocumentsAsync( return decryptItems; } - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private async Task> DecryptChangeFeedDocumentsAsync( - IReadOnlyCollection documents, - CancellationToken cancellationToken) - { - List decryptItems = new (documents.Count); - if (typeof(T) == typeof(DecryptableItem)) - { - foreach (Stream documentStream in documents) - { - DecryptableItemStream item = new ( - documentStream, - this.Encryptor, - JsonProcessor.Stream, - this.CosmosSerializer, - this.streamManager); - - decryptItems.Add((T)(object)item); - } - } - else - { - foreach (Stream document in documents) - { - CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); - using (diagnosticsContext.CreateScope("DecryptChangeFeedDocumentsAsync<")) - { - Stream decryptedStream = this.streamManager.CreateStream(); - _ = await EncryptionProcessor.DecryptAsync( - document, - decryptedStream, - this.Encryptor, - diagnosticsContext, - JsonProcessor.Stream, - cancellationToken); - -#if SDKPROJECTREF - decryptItems.Add(await this.CosmosSerializer.FromStreamAsync(decryptedStream, cancellationToken)); -#else - decryptItems.Add(this.CosmosSerializer.FromStream(decryptedStream)); -#endif - - await decryptedStream.DisposeAsync(); - } - } - } - - return decryptItems; - } -#endif - } + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs index be32751829..202a84428e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator.cs @@ -4,17 +4,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { + using System; using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; -#else - using System; using Newtonsoft.Json.Linq; -#endif internal sealed class EncryptionFeedIterator : FeedIterator { @@ -22,25 +17,6 @@ internal sealed class EncryptionFeedIterator : FeedIterator private readonly Encryptor encryptor; private readonly CosmosSerializer cosmosSerializer; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private readonly StreamManager streamManager; - - private static readonly ArrayStreamSplitter StreamSplitter = new ArrayStreamSplitter(); -#endif - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public EncryptionFeedIterator( - FeedIterator feedIterator, - Encryptor encryptor, - CosmosSerializer cosmosSerializer, - StreamManager streamManager) - { - this.feedIterator = feedIterator; - this.encryptor = encryptor; - this.cosmosSerializer = cosmosSerializer; - this.streamManager = streamManager; - } -#else public EncryptionFeedIterator( FeedIterator feedIterator, Encryptor encryptor, @@ -50,7 +26,6 @@ public EncryptionFeedIterator( this.encryptor = encryptor; this.cosmosSerializer = cosmosSerializer; } -#endif public override bool HasMoreResults => this.feedIterator.HasMoreResults; @@ -63,20 +38,10 @@ public override async Task ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream decryptedContent = this.streamManager.CreateStream(); - await EncryptionProcessor.DeserializeAndDecryptResponseAsync( - responseMessage.Content, - decryptedContent, - this.encryptor, - this.streamManager, - cancellationToken); -#else Stream decryptedContent = await EncryptionProcessor.DeserializeAndDecryptResponseAsync( responseMessage.Content, this.encryptor, cancellationToken); -#endif return new DecryptedResponseMessage(responseMessage, decryptedContent); } @@ -95,14 +60,8 @@ await EncryptionProcessor.DeserializeAndDecryptResponseAsync( if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - decryptableContent = await this.ConvertResponseToDecryptableItemsAsync( - responseMessage.Content, - cancellationToken); -#else decryptableContent = this.ConvertResponseToDecryptableItems( responseMessage.Content); -#endif return (responseMessage, decryptableContent); } @@ -111,28 +70,6 @@ await EncryptionProcessor.DeserializeAndDecryptResponseAsync( } } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private async Task> ConvertResponseToDecryptableItemsAsync( - Stream content, - CancellationToken token) - { - List decryptableStreams = await StreamSplitter.SplitCollectionAsync(content, this.streamManager, token); - List decryptableItems = new List(); - - foreach (Stream item in decryptableStreams) - { - decryptableItems.Add( - (T)(object)new DecryptableItemStream( - item, - this.encryptor, - JsonProcessor.Stream, - this.cosmosSerializer, - this.streamManager)); - } - - return decryptableItems; - } -#else private List ConvertResponseToDecryptableItems( Stream content) { @@ -157,6 +94,5 @@ private List ConvertResponseToDecryptableItems( return decryptableItems; } -#endif } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs index 6fa0b3d6a0..0755ec0cf6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionTransactionalBatch.cs @@ -16,26 +16,8 @@ internal sealed class EncryptionTransactionalBatch : TransactionalBatch { private readonly Encryptor encryptor; private readonly CosmosSerializer cosmosSerializer; - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private readonly StreamManager streamManager; -#endif - private TransactionalBatch transactionalBatch; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public EncryptionTransactionalBatch( - TransactionalBatch transactionalBatch, - Encryptor encryptor, - CosmosSerializer cosmosSerializer, - StreamManager streamManager) - { - this.transactionalBatch = transactionalBatch ?? throw new ArgumentNullException(nameof(transactionalBatch)); - this.encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); - this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); - this.streamManager = streamManager ?? throw new ArgumentNullException(nameof(streamManager)); - } -#else public EncryptionTransactionalBatch( TransactionalBatch transactionalBatch, Encryptor encryptor, @@ -45,7 +27,6 @@ public EncryptionTransactionalBatch( this.encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); this.cosmosSerializer = cosmosSerializer ?? throw new ArgumentNullException(nameof(cosmosSerializer)); } -#endif public override TransactionalBatch CreateItem( T item, @@ -77,24 +58,12 @@ public override TransactionalBatch CreateItemStream( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("EncryptItemStream")) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream temp = this.streamManager.CreateStream(); - EncryptionProcessor.EncryptAsync( - streamPayload, - temp, - this.encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken: default).GetAwaiter().GetResult(); - streamPayload = temp; -#else streamPayload = EncryptionProcessor.EncryptAsync( streamPayload, this.encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken: default).Result; -#endif } } @@ -159,24 +128,15 @@ public override TransactionalBatch ReplaceItemStream( encryptionItemRequestOptions.EncryptionOptions != null) { CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream temp = this.streamManager.CreateStream(); - EncryptionProcessor.EncryptAsync( - streamPayload, - temp, - this.encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken: default).GetAwaiter().GetResult(); - streamPayload = temp; -#else - streamPayload = EncryptionProcessor.EncryptAsync( - streamPayload, - this.encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken: default).Result; -#endif + using (diagnosticsContext.CreateScope("EncryptItemStream")) + { + streamPayload = EncryptionProcessor.EncryptAsync( + streamPayload, + this.encryptor, + encryptionItemRequestOptions.EncryptionOptions, + diagnosticsContext, + cancellationToken: default).Result; + } } this.transactionalBatch = this.transactionalBatch.ReplaceItemStream( @@ -217,24 +177,12 @@ public override TransactionalBatch UpsertItemStream( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("EncryptItemStream")) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream temp = this.streamManager.CreateStream(); - EncryptionProcessor.EncryptAsync( - streamPayload, - temp, - this.encryptor, - encryptionItemRequestOptions.EncryptionOptions, - diagnosticsContext, - cancellationToken: default).GetAwaiter().GetResult(); - streamPayload = temp; -#else streamPayload = EncryptionProcessor.EncryptAsync( streamPayload, this.encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken: default).Result; -#endif } } @@ -285,22 +233,11 @@ private async Task DecryptTransactionalBatchResponse { if (response.IsSuccessStatusCode && result.ResourceStream != null) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - Stream decryptedStream = this.streamManager.CreateStream(); - _ = await EncryptionProcessor.DecryptAsync( - result.ResourceStream, - decryptedStream, - this.encryptor, - diagnosticsContext, - JsonProcessor.Stream, - cancellationToken); -#else (Stream decryptedStream, _) = await EncryptionProcessor.DecryptAsync( result.ResourceStream, this.encryptor, diagnosticsContext, cancellationToken); -#endif decryptedTransactionalBatchOperationResults.Add(new EncryptionTransactionalBatchOperationResult(result, decryptedStream)); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs index ca6c61c8a2..92b003d71e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs @@ -275,7 +275,7 @@ internal RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Gui } /// - /// Finalizes an instance of the RecyclableMemoryStream class. + /// Finalizes an instance of the class. /// /// Failing to dispose indicates a bug in the code using streams. Care should be taken to properly account for stream lifetime. ~RecyclableMemoryStream() diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index abee2721fb..61f913cbc9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -14,7 +14,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; internal class MdeJsonNodeEncryptionProcessor { @@ -24,14 +23,6 @@ internal class MdeJsonNodeEncryptionProcessor internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - internal JsonSerializerOptions JsonSerializerOptions { get; set; } - - public MdeJsonNodeEncryptionProcessor() - { - this.JsonSerializerOptions = new JsonSerializerOptions(); - this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); - } - public async Task EncryptAsync( Stream input, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs deleted file mode 100644 index fbef47e872..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.IO; - using System.Text.Json; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesConverterTests - { - [TestMethod] - public void Write_Results_IdenticalToNewtonsoft() - { - byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - - JsonBytes jsonBytes = new (bytes, 5, 5); - - using MemoryStream ms = new (); - using Utf8JsonWriter writer = new (ms); - - JsonBytesConverter jsonConverter = new (); - jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); - - writer.Flush(); - ms.Flush(); - ms.Position = 0; - StreamReader sr = new(ms); - string systemTextResult = sr.ReadToEnd(); - - byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); - string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); - - Assert.AreEqual(systemTextResult, newtonsoftResult); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs deleted file mode 100644 index a29f378ff2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesTests - { - [TestMethod] - public void Ctor_ThrowsForInvalidInputs() - { - Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); - } - - [TestMethod] - public void Properties_AreSetCorrectly() - { - byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - JsonBytes jsonBytes = new (bytes, 1, 5); - - Assert.AreEqual(1, jsonBytes.Offset); - Assert.AreEqual(5, jsonBytes.Length); - Assert.AreSame(bytes, jsonBytes.Bytes); - } - } -} -#endif From eb1ec76a792734294327fa2969c07ad946e9d4fb Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 14:10:15 +0200 Subject: [PATCH 74/85] + tests for EncryptionContainerStream --- .../src/EncryptionContainerExtensions.cs | 20 +- .../StreamProcessing/DecryptableItemStream.cs | 2 +- .../StreamProcessing/EncryptableItemStream.cs | 4 - .../EncryptionContainerStream.cs | 16 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 27 +- .../MdeCustomEncryptionTestsWithSystemText.cs | 2707 +++++++++++++++++ ...mos.Encryption.Custom.EmulatorTests.csproj | 1 + 7 files changed, 2736 insertions(+), 41 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs index cfdc4cd151..f624960f65 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs @@ -24,7 +24,7 @@ public static Container WithEncryptor( Encryptor encryptor) { #if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (container.Database.Client.ClientOptions.SerializerOptions.UseSystemTextJsonSerializerWithOptions) + if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) { return new EncryptionContainerStream(container, encryptor); } @@ -57,17 +57,17 @@ public static FeedIterator ToEncryptionFeedIterator( this Container container, IQueryable query) { -#if SDKPROJECTREF - if (container.Database.Client.ClientOptions.SerializerOptions.UseSystemTextJsonSerializerWithOptions) +#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) { - if (container is not EncryptionContainerStream encryptionContainer) + if (container is not EncryptionContainerStream encryptionContainerStream) { throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainerStream)}."); } return new EncryptionFeedIteratorStream( - (EncryptionFeedIteratorStream)encryptionContainer.ToEncryptionStreamIterator(query), - encryptionContainer.ResponseFactory); + (EncryptionFeedIteratorStream)encryptionContainerStream.ToEncryptionStreamIterator(query), + encryptionContainerStream.ResponseFactory); } #endif if (container is not EncryptionContainer encryptionContainer) @@ -103,17 +103,17 @@ public static FeedIterator ToEncryptionStreamIterator( IQueryable query) { #if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (container.Database.Client.ClientOptions.SerializerOptions.UseSystemTextJsonSerializerWithOptions) + if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) { - if (container is not EncryptionContainerStream encryptionContainer) + if (container is not EncryptionContainerStream encryptionContainerStream) { throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainerStream)}."); } return new EncryptionFeedIteratorStream( query.ToStreamIterator(), - encryptionContainer.Encryptor, - encryptionContainer.CosmosSerializer, + encryptionContainerStream.Encryptor, + encryptionContainerStream.CosmosSerializer, new MemoryStreamManager()); } #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs index 73c1a87aa5..58f73923fa 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs @@ -72,7 +72,7 @@ public DecryptableItemStream( return ((T)(object)ms, this.decryptionContext); default: #if SDKPROJECTREF - return (await this.CosmosSerializer.FromStreamAsync(this.decryptedStream, cancellationToken), this.decryptionContext); + return (await this.cosmosSerializer.FromStreamAsync(this.decryptedStream, cancellationToken), this.decryptionContext); #else // this API is missing Async => should not be used return (this.cosmosSerializer.FromStream(this.decryptedStream), this.decryptionContext); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs index c8f02b20aa..45dc241a75 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs @@ -46,10 +46,8 @@ public EncryptableItemStream(T input) this.Item = input ?? throw new ArgumentNullException(nameof(input)); } -#pragma warning disable CS0672 // Member overrides obsolete member /// protected internal override void SetDecryptableItem(JToken decryptableContent, Encryptor encryptor, CosmosSerializer cosmosSerializer) -#pragma warning restore CS0672 // Member overrides obsolete member { throw new NotImplementedException(); } @@ -63,9 +61,7 @@ protected internal override void SetDecryptableStream(Stream decryptableStream, } /// -#pragma warning disable CS0672 // Member overrides obsolete member protected internal override Stream ToStream(CosmosSerializer serializer) -#pragma warning restore CS0672 // Member overrides obsolete member { return serializer.ToStream(this.Item); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs index 72dbeaca01..b3b920e608 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs @@ -34,15 +34,27 @@ internal sealed class EncryptionContainerStream : Container /// /// Regular cosmos container. /// Provider that allows encrypting and decrypting data. + public EncryptionContainerStream(Container container, Encryptor encryptor) + : this(container, encryptor, new MemoryStreamManager()) + { + } + + /// + /// All the operations / requests for exercising client-side encryption functionality need to be made using this EncryptionContainer instance. + /// + /// Regular cosmos container. + /// Provider that allows encrypting and decrypting data. + /// Custom stream manager instance. public EncryptionContainerStream( Container container, - Encryptor encryptor) + Encryptor encryptor, + StreamManager streamManager) { this.container = container ?? throw new ArgumentNullException(nameof(container)); this.Encryptor = encryptor ?? throw new ArgumentNullException(nameof(encryptor)); this.ResponseFactory = this.Database.Client.ResponseFactory; this.CosmosSerializer = this.Database.Client.ClientOptions.Serializer; - this.streamManager = new MemoryStreamManager(); + this.streamManager = streamManager; } public override string Id => this.container.Id; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 14b1915abb..e6b1b69966 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2177,14 +2177,7 @@ public TestEncryptionKeyStoreProvider() public override byte[] UnwrapKey(string masterKeyPath, KeyEncryptionKeyAlgorithm encryptionAlgorithm, byte[] encryptedKey) { - if (!this.UnWrapKeyCallsCount.ContainsKey(masterKeyPath)) - { - this.UnWrapKeyCallsCount[masterKeyPath] = 1; - } - else - { - this.UnWrapKeyCallsCount[masterKeyPath]++; - } + this.UnWrapKeyCallsCount[masterKeyPath] = !this.UnWrapKeyCallsCount.TryGetValue(masterKeyPath, out int value) ? 1 : ++value; this.keyinfo.TryGetValue(masterKeyPath, out int moveBy); byte[] plainkey = encryptedKey.Select(b => (byte)(b - moveBy)).ToArray(); @@ -2193,14 +2186,7 @@ public override byte[] UnwrapKey(string masterKeyPath, KeyEncryptionKeyAlgorithm public override byte[] WrapKey(string masterKeyPath, KeyEncryptionKeyAlgorithm encryptionAlgorithm, byte[] key) { - if (!this.WrapKeyCallsCount.ContainsKey(masterKeyPath)) - { - this.WrapKeyCallsCount[masterKeyPath] = 1; - } - else - { - this.WrapKeyCallsCount[masterKeyPath]++; - } + this.WrapKeyCallsCount[masterKeyPath] = !this.WrapKeyCallsCount.TryGetValue(masterKeyPath, out int value) ? 1 : ++value; this.keyinfo.TryGetValue(masterKeyPath, out int moveBy); byte[] encryptedkey = key.Select(b => (byte)(b + moveBy)).ToArray(); @@ -2648,14 +2634,7 @@ public override Task UnwrapKeyAsync(byte[] wrappedKey public override Task WrapKeyAsync(byte[] key, EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { - if (!this.WrapKeyCallsCount.ContainsKey(metadata.Value)) - { - this.WrapKeyCallsCount[metadata.Value] = 1; - } - else - { - this.WrapKeyCallsCount[metadata.Value]++; - } + this.WrapKeyCallsCount[metadata.Value] = !this.WrapKeyCallsCount.TryGetValue(metadata.Value, out int value) ? 1 : ++value; EncryptionKeyWrapMetadata responseMetadata = new(metadata.Value + metadataUpdateSuffix); int moveBy = metadata.Value == metadata1.Value ? 1 : 2; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs new file mode 100644 index 0000000000..35a714e950 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs @@ -0,0 +1,2707 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.Data.Encryption.Cryptography; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json; + using static Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.LegacyEncryptionTests; + using EncryptionKeyWrapMetadata = EncryptionKeyWrapMetadata; + using DataEncryptionKey = DataEncryptionKey; + using Newtonsoft.Json.Linq; + using Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.Utils; + + [TestClass] + public class MdeCustomEncryptionTestsWithSystemText + { + private static readonly Uri masterKeyUri1 = new("https://demo.keyvault.net/keys/samplekey1/03ded886623sss09bzc60351e536a111"); + private static readonly Uri masterKeyUri2 = new("https://demo.keyvault.net/keys/samplekey2/47d306aeaaeyyyaabs9467235460dc22"); + private static readonly EncryptionKeyWrapMetadata metadata1 = new(name: "metadata1", value: masterKeyUri1.ToString()); + private static readonly EncryptionKeyWrapMetadata metadata2 = new(name: "metadata2", value: masterKeyUri2.ToString()); + private const string dekId = "mydek"; + private const string legacydekId = "mylegacydek"; + private static CosmosClient client; + private static Database database; + private static DataEncryptionKeyProperties dekProperties; + private static Container itemContainer; + private static Container encryptionContainer; + private static Container itemContainerForChangeFeed; + private static Container encryptionContainerForChangeFeed; + private static Container keyContainer; + private static TestEncryptionKeyStoreProvider testKeyStoreProvider; + private static CosmosDataEncryptionKeyProvider dekProvider; + private static TestEncryptor encryptor; + + + private static TestKeyWrapProvider legacytestKeyWrapProvider; + private static CosmosDataEncryptionKeyProvider dualDekProvider; + private const string metadataUpdateSuffix = "updated"; + private static readonly TimeSpan cacheTTL = TimeSpan.FromDays(1); + private static TestEncryptor encryptorWithDualWrapProvider; + + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + _ = context; + + client = TestCommon.CreateCosmosClient(builder => builder.WithSystemTextJsonSerializerOptions(new System.Text.Json.JsonSerializerOptions())); + database = await client.CreateDatabaseAsync(Guid.NewGuid().ToString()); + keyContainer = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/id", 400); + itemContainer = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/PK", 400); + itemContainerForChangeFeed = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/PK", 400); + + testKeyStoreProvider = new TestEncryptionKeyStoreProvider(); + await LegacyClassInitializeAsync(); + + MdeCustomEncryptionTestsWithSystemText.encryptor = new TestEncryptor(MdeCustomEncryptionTestsWithSystemText.dekProvider); + MdeCustomEncryptionTestsWithSystemText.encryptionContainer = MdeCustomEncryptionTestsWithSystemText.itemContainer.WithEncryptor(encryptor); + MdeCustomEncryptionTestsWithSystemText.encryptionContainerForChangeFeed = MdeCustomEncryptionTestsWithSystemText.itemContainerForChangeFeed.WithEncryptor(encryptor); + + await MdeCustomEncryptionTestsWithSystemText.dekProvider.InitializeAsync(MdeCustomEncryptionTestsWithSystemText.database, MdeCustomEncryptionTestsWithSystemText.keyContainer.Id); + MdeCustomEncryptionTestsWithSystemText.dekProperties = await MdeCustomEncryptionTestsWithSystemText.CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, MdeCustomEncryptionTestsWithSystemText.dekId); + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (database != null) + { + using (await database.DeleteStreamAsync()) { } + } + + client?.Dispose(); + } + + [TestMethod] + public async Task EncryptionCreateDek() + { + string dekId = "anotherDek"; + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, dekId); + Assert.AreEqual( + new EncryptionKeyWrapMetadata(name: "metadata1", value: metadata1.Value), + dekProperties.EncryptionKeyWrapMetadata); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKeyProperties readProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + } + + [TestMethod] + public async Task FetchDataEncryptionKeyWithRawKey() + { + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKey k = await dekProvider.FetchDataEncryptionKeyAsync(dekProperties.Id, dekProperties.EncryptionAlgorithm, CancellationToken.None); + Assert.IsNotNull(k.RawKey); + } + + [TestMethod] + public async Task FetchDataEncryptionKeyWithoutRawKey() + { + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKey k = await dekProvider.FetchDataEncryptionKeyWithoutRawKeyAsync(dekProperties.Id, dekProperties.EncryptionAlgorithm, CancellationToken.None); + Assert.IsNull(k.RawKey); + } + + [TestMethod] + [Obsolete("Obsoleted algorithm")] + public async Task FetchDataEncryptionKeyMdeDEKAndLegacyBasedAlgorithm() + { + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKey k = await dekProvider.FetchDataEncryptionKeyAsync(dekProperties.Id, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized, CancellationToken.None); + Assert.IsNotNull(k.RawKey); + } + + [TestMethod] + [Obsolete("Obsoleted algorithm")] + public async Task FetchDataEncryptionKeyLegacyDEKAndMdeBasedAlgorithm() + { + string dekId = "legacyDEK"; + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, dekId, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestKeyWrapProvider(), new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKey k = await dekProvider.FetchDataEncryptionKeyAsync(dekProperties.Id, CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, CancellationToken.None); + Assert.IsNotNull(k.RawKey); + } + + [TestMethod] + public async Task EncryptionRewrapDek() + { + string dekId = "randomDek"; + + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, dekId); + Assert.AreEqual( + metadata1, + dekProperties.EncryptionKeyWrapMetadata); + + ItemResponse dekResponse = await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContainer.RewrapDataEncryptionKeyAsync( + dekId, + metadata2); + + Assert.AreEqual(HttpStatusCode.OK, dekResponse.StatusCode); + dekProperties = VerifyDekResponse( + dekResponse, + dekId); + Assert.AreEqual( + metadata2, + dekProperties.EncryptionKeyWrapMetadata); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKeyProperties readProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + } + + [TestMethod] + public async Task EncryptionRewrapDekEtagMismatch() + { + string dekId = "dummyDek"; + EncryptionKeyWrapMetadata newMetadata = new(name: "newMetadata", value: "newMetadataValue"); + + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, dekId); + Assert.AreEqual( + metadata1, + dekProperties.EncryptionKeyWrapMetadata); + + // modify dekProperties directly, which would lead to etag change + DataEncryptionKeyProperties updatedDekProperties = new( + dekProperties.Id, + dekProperties.EncryptionAlgorithm, + dekProperties.WrappedDataEncryptionKey, + dekProperties.EncryptionKeyWrapMetadata, + DateTime.UtcNow); + await keyContainer.ReplaceItemAsync( + updatedDekProperties, + dekProperties.Id, + new PartitionKey(dekProperties.Id)); + + // rewrap should succeed, despite difference in cached value + ItemResponse dekResponse = await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContainer.RewrapDataEncryptionKeyAsync( + dekId, + newMetadata); + + Assert.AreEqual(HttpStatusCode.OK, dekResponse.StatusCode); + dekProperties = VerifyDekResponse( + dekResponse, + dekId); + Assert.AreEqual( + newMetadata, + dekProperties.EncryptionKeyWrapMetadata); + + Assert.AreEqual(2, testKeyStoreProvider.WrapKeyCallsCount[newMetadata.Value]); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKeyProperties readProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + } + + [TestMethod] + public async Task EncryptionDekReadFeed() + { + Container newKeyContainer = await database.CreateContainerAsync(Guid.NewGuid().ToString(), "/id", 400); + try + { + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, newKeyContainer.Id); + + string contosoV1 = "Contoso_v001"; + string contosoV2 = "Contoso_v002"; + string fabrikamV1 = "Fabrikam_v001"; + string fabrikamV2 = "Fabrikam_v002"; + + await CreateDekAsync(dekProvider, contosoV1); + await CreateDekAsync(dekProvider, contosoV2); + await CreateDekAsync(dekProvider, fabrikamV1); + await CreateDekAsync(dekProvider, fabrikamV2); + + // Test getting all keys + await IterateDekFeedAsync( + dekProvider, + new List { contosoV1, contosoV2, fabrikamV1, fabrikamV2 }, + isExpectedDeksCompleteSetForRequest: true, + isResultOrderExpected: false, + "SELECT * from c"); + + // Test getting specific subset of keys + await IterateDekFeedAsync( + dekProvider, + new List { contosoV2 }, + isExpectedDeksCompleteSetForRequest: false, + isResultOrderExpected: true, + "SELECT TOP 1 * from c where c.id >= 'Contoso_v000' and c.id <= 'Contoso_v999' ORDER BY c.id DESC"); + + // Ensure only required results are returned + await IterateDekFeedAsync( + dekProvider, + new List { contosoV1, contosoV2 }, + isExpectedDeksCompleteSetForRequest: true, + isResultOrderExpected: true, + "SELECT * from c where c.id >= 'Contoso_v000' and c.id <= 'Contoso_v999' ORDER BY c.id ASC"); + + // Test pagination + await IterateDekFeedAsync( + dekProvider, + new List { contosoV1, contosoV2, fabrikamV1, fabrikamV2 }, + isExpectedDeksCompleteSetForRequest: true, + isResultOrderExpected: false, + "SELECT * from c", + itemCountInPage: 3); + } + finally + { + await newKeyContainer.DeleteContainerStreamAsync(); + } + } + + [TestMethod] + public async Task EncryptionCreateItemWithoutEncryptionOptions() + { + TestDoc testDoc = TestDoc.Create(); + ItemResponse createResponse = await encryptionContainer.CreateItemAsync( + testDoc, + new PartitionKey(testDoc.PK)); + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + VerifyExpectedDocResponse(testDoc, createResponse.Resource); + } + + [TestMethod] + public async Task EncryptionCreateItemWithNullEncryptionOptions() + { + TestDoc testDoc = TestDoc.Create(); + ItemResponse createResponse = await encryptionContainer.CreateItemAsync( + testDoc, + new PartitionKey(testDoc.PK), + new EncryptionItemRequestOptions() { }); + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + VerifyExpectedDocResponse(testDoc, createResponse.Resource); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionCreateItemWithoutPartitionKey(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = TestDoc.Create(); + try + { + await encryptionContainer.CreateItemAsync( + testDoc, + requestOptions: GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); + Assert.Fail("CreateItem should've failed because PartitionKey was not provided."); + } + catch (NotSupportedException ex) + { + Assert.AreEqual("partitionKey cannot be null for operations using EncryptionContainer.", ex.Message); + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionFailsWithUnknownDek(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string unknownDek = "unknownDek"; + + try + { + await CreateItemAsync(encryptionContainer, unknownDek, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + } + catch (ArgumentException ex) + { + Assert.AreEqual($"Failed to retrieve Data Encryption Key with id: '{unknownDek}'.", ex.Message); + Assert.IsTrue(ex.InnerException is CosmosException); + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task ValidateCachingOfProtectedDataEncryptionKey(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestEncryptionKeyStoreProvider testEncryptionKeyStoreProvider = new() + { + DataEncryptionKeyCacheTimeToLive = TimeSpan.FromMinutes(30) + }; + + string dekId = "pDekCache"; + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(dualDekProvider, dekId); + Assert.AreEqual( + new EncryptionKeyWrapMetadata(name: "metadata1", value: metadata1.Value), + dekProperties.EncryptionKeyWrapMetadata); + + // Caching for 30 min. + CosmosDataEncryptionKeyProvider dekProvider = new(testEncryptionKeyStoreProvider); + await dekProvider.InitializeAsync(database, keyContainer.Id); + + TestEncryptor encryptor = new(dekProvider); + Container encryptionContainer = itemContainer.WithEncryptor(encryptor); + for (int i = 0; i < 2; i++) + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out int unwrapcount); + Assert.AreEqual(1, unwrapcount); + + testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider + { + DataEncryptionKeyCacheTimeToLive = TimeSpan.Zero + }; + + // No caching + dekProvider = new CosmosDataEncryptionKeyProvider(testEncryptionKeyStoreProvider); + await dekProvider.InitializeAsync(database, keyContainer.Id); + + encryptor = new TestEncryptor(dekProvider); + encryptionContainer = itemContainer.WithEncryptor(encryptor); + for (int i = 0; i < 2; i++) + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); + Assert.AreEqual(4, unwrapcount); + + // 2 hours default + testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); + + dekProvider = new CosmosDataEncryptionKeyProvider(testEncryptionKeyStoreProvider); + await dekProvider.InitializeAsync(database, keyContainer.Id); + + encryptor = new TestEncryptor(dekProvider); + encryptionContainer = itemContainer.WithEncryptor(encryptor); + for (int i = 0; i < 2; i++) + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); + Assert.AreEqual(1, unwrapcount); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionReadManyItemAsync(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + TestDoc testDoc2 = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + List<(string, PartitionKey)> itemList = new() + { + (testDoc.Id, new PartitionKey(testDoc.PK)), + (testDoc2.Id, new PartitionKey(testDoc2.PK)) + }; + + FeedResponse response = await encryptionContainer.ReadManyItemsAsync(itemList); + + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + Assert.AreEqual(2, response.Count); + VerifyExpectedDocResponse(testDoc, response.Resource.ElementAt(0)); + VerifyExpectedDocResponse(testDoc2, response.Resource.ElementAt(1)); + + // stream test. + ResponseMessage responseStream = await encryptionContainer.ReadManyItemsStreamAsync(itemList); + + Assert.IsTrue(responseStream.IsSuccessStatusCode); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + + JObject contentJObjects = TestCommon.FromStream(responseStream.Content); + + if (contentJObjects.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents) + { + VerifyExpectedDocResponse(testDoc, documents.ElementAt(0).ToObject()); + VerifyExpectedDocResponse(testDoc2, documents.ElementAt(1).ToObject()); + } + else + { + Assert.Fail("ResponseMessage from ReadManyItemsStreamAsync did not have a valid response. "); + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionCreateItem(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + await VerifyItemByReadAsync(encryptionContainer, testDoc); + + await VerifyItemByReadStreamAsync(encryptionContainer, testDoc); + + TestDoc expectedDoc = new(testDoc); + + // Read feed (null query) + await MdeCustomEncryptionTestsWithSystemText.ValidateQueryResultsAsync( + MdeCustomEncryptionTestsWithSystemText.encryptionContainer, + query: null, + expectedDoc); + + await ValidateQueryResultsAsync( + encryptionContainer, + "SELECT * FROM c", + expectedDoc); + + await ValidateQueryResultsAsync( + encryptionContainer, + string.Format( + "SELECT * FROM c where c.PK = '{0}' and c.id = '{1}' and c.NonSensitive = '{2}'", + expectedDoc.PK, + expectedDoc.Id, + expectedDoc.NonSensitive), + expectedDoc); + + await ValidateQueryResultsAsync( + encryptionContainer, + string.Format("SELECT * FROM c where c.Sensitive_IntFormat = '{0}'", testDoc.Sensitive_IntFormat), + expectedDoc: null); + + await ValidateQueryResultsAsync( + encryptionContainer, + queryDefinition: new QueryDefinition( + "select * from c where c.id = @theId and c.PK = @thePK") + .WithParameter("@theId", expectedDoc.Id) + .WithParameter("@thePK", expectedDoc.PK), + expectedDoc: expectedDoc); + + expectedDoc.Sensitive_NestedObjectFormatL1 = null; + expectedDoc.Sensitive_ArrayFormat = null; + expectedDoc.Sensitive_DecimalFormat = 0; + expectedDoc.Sensitive_IntFormat = 0; + expectedDoc.Sensitive_FloatFormat = 0; + expectedDoc.Sensitive_BoolFormat = false; + expectedDoc.Sensitive_StringFormat = null; + expectedDoc.Sensitive_DateFormat = new DateTime(); + + await ValidateQueryResultsAsync( + encryptionContainer, + "SELECT c.id, c.PK, c.NonSensitive FROM c", + expectedDoc); + } + + [TestMethod] + [ExpectedException(typeof(InvalidOperationException), "Decryptable content is not initialized.")] + public void ValidateDecryptableContent() + { + TestDoc testDoc = TestDoc.Create(); + EncryptableItem encryptableItem = new(testDoc); + encryptableItem.DecryptableItem.GetItemAsync(); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionCreateItemWithLazyDecryption(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = TestDoc.Create(); + ItemResponse> createResponse = await encryptionContainer.CreateItemAsync( + new EncryptableItem(testDoc), + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); + + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + Assert.IsNotNull(createResponse.Resource); + + await ValidateDecryptableItem(createResponse.Resource.DecryptableItem, testDoc); + + // stream + TestDoc testDoc1 = TestDoc.Create(); + ItemResponse createResponseStream = await encryptionContainer.CreateItemAsync( + new EncryptableItemStream(TestCommon.ToStream(testDoc1)), + new PartitionKey(testDoc1.PK), + GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); + + Assert.AreEqual(HttpStatusCode.Created, createResponseStream.StatusCode); + Assert.IsNotNull(createResponseStream.Resource); + + await ValidateDecryptableItem(createResponseStream.Resource.DecryptableItem, testDoc1); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionChangeFeedDecryptionSuccessful(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string dek2 = "dek2ForChangeFeed"; + await CreateDekAsync(dekProvider, dek2); + + TestDoc testDoc1 = await CreateItemAsync(encryptionContainerForChangeFeed, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + TestDoc testDoc2 = await CreateItemAsync(encryptionContainerForChangeFeed, dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + // change feed iterator + await ValidateChangeFeedIteratorResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + + // change feed processor + await ValidateChangeFeedProcessorResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + + // change feed processor with feed handler + await ValidateChangeFeedProcessorWithFeedHandlerResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + + // change feed processor with manual checkpoint + await ValidateChangeFeedProcessorWithManualCheckpointResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + + // change feed processor with feed stream handler + await ValidateChangeFeedProcessorWithFeedStreamHandlerResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + + // change feed processor manual checkpoint with feed stream handler + await ValidateChangeFeedProcessorStreamWithManualCheckpointResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionHandleDecryptionFailure(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string dek2 = "failDek"; + await CreateDekAsync(dekProvider, dek2); + + TestDoc testDoc1 = await CreateItemAsync(encryptionContainer, dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + TestDoc testDoc2 = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + string query = $"SELECT * FROM c WHERE c.PK in ('{testDoc1.PK}', '{testDoc2.PK}')"; + + // success + await ValidateQueryResultsMultipleDocumentsAsync(encryptionContainer, testDoc1, testDoc2, query); + + // induce failure + encryptor.FailDecryption = true; + + FeedIterator queryResponseIterator = encryptionContainer.GetItemQueryIterator(query); + FeedResponse readDocsLazily = await queryResponseIterator.ReadNextAsync(); + await ValidateLazyDecryptionResponse(readDocsLazily.GetEnumerator(), dek2); + + // validate changeFeed handling + FeedIterator changeIterator = encryptionContainer.GetChangeFeedIterator( + ChangeFeedStartFrom.Beginning(), + ChangeFeedMode.Incremental); + + while (changeIterator.HasMoreResults) + { + readDocsLazily = await changeIterator.ReadNextAsync(); + if (readDocsLazily.StatusCode == HttpStatusCode.NotModified) + { + break; + } + + if (readDocsLazily.Resource != null) + { + await ValidateLazyDecryptionResponse(readDocsLazily.GetEnumerator(), dek2); + } + } + + // validate changeFeedProcessor handling + Container leaseContainer = await database.CreateContainerIfNotExistsAsync( + new ContainerProperties(id: "leasesContainer", partitionKeyPath: "/id")); + + List changeFeedReturnedDocs = new(); + ChangeFeedProcessor cfp = encryptionContainer.GetChangeFeedProcessorBuilder( + "testCFPFailure", + (IReadOnlyCollection changes, CancellationToken cancellationToken) => + { + changeFeedReturnedDocs.AddRange(changes); + return Task.CompletedTask; + }) + .WithInstanceName("dummy") + .WithLeaseContainer(leaseContainer) + .WithStartTime(DateTime.MinValue.ToUniversalTime()) + .Build(); + + await cfp.StartAsync(); + await Task.Delay(2000); + await cfp.StopAsync(); + + Assert.IsTrue(changeFeedReturnedDocs.Count >= 2); + await ValidateLazyDecryptionResponse(changeFeedReturnedDocs.GetEnumerator(), dek2); + + encryptor.FailDecryption = false; + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionDecryptQueryResultMultipleDocs(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc1 = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + TestDoc testDoc2 = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + // test GetItemLinqQueryable + await ValidateQueryResultsMultipleDocumentsAsync(encryptionContainer, testDoc1, testDoc2, null); + + string query = $"SELECT * FROM c WHERE c.PK in ('{testDoc1.PK}', '{testDoc2.PK}')"; + await ValidateQueryResultsMultipleDocumentsAsync(encryptionContainer, testDoc1, testDoc2, query); + + // ORDER BY query + query += " ORDER BY c._ts"; + await ValidateQueryResultsMultipleDocumentsAsync(encryptionContainer, testDoc1, testDoc2, query); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionDecryptQueryResultMultipleEncryptedProperties(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + List pathsEncrypted = new() { "/Sensitive_StringFormat", "/NonSensitive" }; + TestDoc testDoc = await CreateItemAsync( + encryptionContainer, + dekId, + pathsEncrypted, + jsonProcessor, + compressionAlgorithm); + + TestDoc expectedDoc = new(testDoc); + + await ValidateQueryResultsAsync( + encryptionContainer, + "SELECT * FROM c", + expectedDoc, + pathsEncrypted: pathsEncrypted); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionDecryptQueryValueResponse(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + string query = "SELECT VALUE COUNT(1) FROM c"; + + await ValidateQueryResponseAsync(encryptionContainer, query); + await ValidateQueryResponseWithLazyDecryptionAsync(encryptionContainer, query); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionDecryptGroupByQueryResultTest(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string partitionKey = Guid.NewGuid().ToString(); + + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + + string query = $"SELECT COUNT(c.Id), c.PK " + + $"FROM c WHERE c.PK = '{partitionKey}' " + + $"GROUP BY c.PK "; + + await ValidateQueryResponseAsync(encryptionContainer, query); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionStreamIteratorValidation(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + // test GetItemLinqQueryable with ToEncryptionStreamIterator extension + await ValidateQueryResponseAsync(encryptionContainer); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionRudItem(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = await UpsertItemAsync( + encryptionContainer, + TestDoc.Create(), + dekId, + TestDoc.PathsToEncrypt, + HttpStatusCode.Created, + jsonProcessor, + compressionAlgorithm); + + await VerifyItemByReadAsync(encryptionContainer, testDoc); + + testDoc.NonSensitive = Guid.NewGuid().ToString(); + testDoc.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + ItemResponse upsertResponse = await UpsertItemAsync( + encryptionContainer, + testDoc, + dekId, + TestDoc.PathsToEncrypt, + HttpStatusCode.OK, + jsonProcessor, + compressionAlgorithm); + TestDoc updatedDoc = upsertResponse.Resource; + + await VerifyItemByReadAsync(encryptionContainer, updatedDoc); + + updatedDoc.NonSensitive = Guid.NewGuid().ToString(); + updatedDoc.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc replacedDoc = await ReplaceItemAsync( + encryptionContainer, + updatedDoc, + dekId, + TestDoc.PathsToEncrypt, + jsonProcessor, + compressionAlgorithm, + upsertResponse.ETag); + + await VerifyItemByReadAsync(encryptionContainer, replacedDoc); + + await DeleteItemAsync(encryptionContainer, replacedDoc); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionRudItemLazyDecryption(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = TestDoc.Create(); + // Upsert (item doesn't exist) + ItemResponse> upsertResponse = await encryptionContainer.UpsertItemAsync( + new EncryptableItem(testDoc), + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); + + Assert.AreEqual(HttpStatusCode.Created, upsertResponse.StatusCode); + Assert.IsNotNull(upsertResponse.Resource); + + await ValidateDecryptableItem(upsertResponse.Resource.DecryptableItem, testDoc); + await VerifyItemByReadAsync(encryptionContainer, testDoc); + + // Upsert with stream (item exists) + testDoc.NonSensitive = Guid.NewGuid().ToString(); + testDoc.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + ItemResponse upsertResponseStream = await encryptionContainer.UpsertItemAsync( + new EncryptableItemStream(TestCommon.ToStream(testDoc)), + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); + + Assert.AreEqual(HttpStatusCode.OK, upsertResponseStream.StatusCode); + Assert.IsNotNull(upsertResponseStream.Resource); + + await ValidateDecryptableItem(upsertResponseStream.Resource.DecryptableItem, testDoc); + await VerifyItemByReadAsync(encryptionContainer, testDoc); + + // replace + testDoc.NonSensitive = Guid.NewGuid().ToString(); + testDoc.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + ItemResponse replaceResponseStream = await encryptionContainer.ReplaceItemAsync( + new EncryptableItemStream(TestCommon.ToStream(testDoc)), + testDoc.Id, + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, upsertResponseStream.ETag)); + + Assert.AreEqual(HttpStatusCode.OK, replaceResponseStream.StatusCode); + Assert.IsNotNull(replaceResponseStream.Resource); + + await ValidateDecryptableItem(replaceResponseStream.Resource.DecryptableItem, testDoc); + await VerifyItemByReadAsync(encryptionContainer, testDoc); + + await DeleteItemAsync(encryptionContainer, testDoc); + } + + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionResourceTokenAuthRestricted(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + User restrictedUser = database.GetUser(Guid.NewGuid().ToString()); + await database.CreateUserAsync(restrictedUser.Id); + + PermissionProperties restrictedUserPermission = await restrictedUser.CreatePermissionAsync( + new PermissionProperties(Guid.NewGuid().ToString(), PermissionMode.All, itemContainer)); + + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + TestEncryptor encryptor = new(dekProvider); + + CosmosClient clientForRestrictedUser = TestCommon.CreateCosmosClient( + restrictedUserPermission.Token); + + Database databaseForRestrictedUser = clientForRestrictedUser.GetDatabase(database.Id); + Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(itemContainer.Id); + + Container encryptionContainerForRestrictedUser = containerForRestrictedUser.WithEncryptor(encryptor); + + await PerformForbiddenOperationAsync(() => + dekProvider.InitializeAsync(databaseForRestrictedUser, keyContainer.Id), "CosmosDekProvider.InitializeAsync"); + + await PerformOperationOnUninitializedDekProviderAsync(() => + dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId), "DEK.ReadAsync"); + + try + { + await encryptionContainerForRestrictedUser.ReadItemAsync(testDoc.Id, new PartitionKey(testDoc.PK)); + } + catch (InvalidOperationException ex) + { + Assert.AreEqual(ex.Message, "The CosmosDataEncryptionKeyProvider was not initialized."); + } + + try + { + await encryptionContainerForRestrictedUser.ReadItemStreamAsync(testDoc.Id, new PartitionKey(testDoc.PK)); + } + catch (InvalidOperationException ex) + { + Assert.AreEqual(ex.Message, "The CosmosDataEncryptionKeyProvider was not initialized."); + } + } + + [TestMethod] + public async Task EncryptionResourceTokenAuthAllowed() + { + User keyManagerUser = database.GetUser(Guid.NewGuid().ToString()); + await database.CreateUserAsync(keyManagerUser.Id); + + PermissionProperties keyManagerUserPermission = await keyManagerUser.CreatePermissionAsync( + new PermissionProperties(Guid.NewGuid().ToString(), PermissionMode.All, keyContainer)); + + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + TestEncryptor encryptor = new(dekProvider); + CosmosClient clientForKeyManagerUser = TestCommon.CreateCosmosClient(keyManagerUserPermission.Token); + + Database databaseForKeyManagerUser = clientForKeyManagerUser.GetDatabase(database.Id); + + await dekProvider.InitializeAsync(databaseForKeyManagerUser, keyContainer.Id); + + DataEncryptionKeyProperties readDekProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readDekProperties); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionRestrictedProperties(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + try + { + await CreateItemAsync(encryptionContainer, dekId, new List() { "/id" }, jsonProcessor, compressionAlgorithm); + Assert.Fail("Expected item creation with id specified to be encrypted to fail."); + } + catch (InvalidOperationException ex) + { + Assert.AreEqual("PathsToEncrypt includes a invalid path: '/id'.", ex.Message); + } + + try + { + await CreateItemAsync(encryptionContainer, dekId, new List() { "/PK" }, jsonProcessor, compressionAlgorithm); + Assert.Fail("Expected item creation with PK specified to be encrypted to fail."); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.BadRequest) + { + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionBulkCrud(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc docToReplace = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + docToReplace.NonSensitive = Guid.NewGuid().ToString(); + docToReplace.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc docToUpsert = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + docToUpsert.NonSensitive = Guid.NewGuid().ToString(); + docToUpsert.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc docToDelete = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + CosmosClient clientWithBulk = TestCommon.CreateCosmosClient(builder => builder + .WithBulkExecution(true) + .Build()); + + Database databaseWithBulk = clientWithBulk.GetDatabase(database.Id); + Container containerWithBulk = databaseWithBulk.GetContainer(itemContainer.Id); + Container encryptionContainerWithBulk = containerWithBulk.WithEncryptor(encryptor); + + List tasks = new() + { + CreateItemAsync(encryptionContainerWithBulk, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm), + UpsertItemAsync(encryptionContainerWithBulk, TestDoc.Create(), dekId, TestDoc.PathsToEncrypt, HttpStatusCode.Created, jsonProcessor, compressionAlgorithm), + ReplaceItemAsync(encryptionContainerWithBulk, docToReplace, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm), + UpsertItemAsync(encryptionContainerWithBulk, docToUpsert, dekId, TestDoc.PathsToEncrypt, HttpStatusCode.OK, jsonProcessor, compressionAlgorithm), + DeleteItemAsync(encryptionContainerWithBulk, docToDelete) + }; + + await Task.WhenAll(tasks); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionTransactionBatchCrud(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string partitionKey = "thePK"; + string dek1 = dekId; + string dek2 = "dek2Forbatch"; + await CreateDekAsync(dekProvider, dek2); + + TestDoc doc1ToCreate = TestDoc.Create(partitionKey); + TestDoc doc2ToCreate = TestDoc.Create(partitionKey); + TestDoc doc3ToCreate = TestDoc.Create(partitionKey); + TestDoc doc4ToCreate = TestDoc.Create(partitionKey); + + ItemResponse doc1ToReplaceCreateResponse = await CreateItemAsync(encryptionContainer, dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + TestDoc doc1ToReplace = doc1ToReplaceCreateResponse.Resource; + doc1ToReplace.NonSensitive = Guid.NewGuid().ToString(); + doc1ToReplace.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc doc2ToReplace = await CreateItemAsync(encryptionContainer, dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + doc2ToReplace.NonSensitive = Guid.NewGuid().ToString(); + doc2ToReplace.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc doc1ToUpsert = await CreateItemAsync(encryptionContainer, dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + doc1ToUpsert.NonSensitive = Guid.NewGuid().ToString(); + doc1ToUpsert.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc doc2ToUpsert = await CreateItemAsync(encryptionContainer, dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + doc2ToUpsert.NonSensitive = Guid.NewGuid().ToString(); + doc2ToUpsert.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TestDoc docToDelete = await CreateItemAsync(encryptionContainer, dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + + TransactionalBatchResponse batchResponse = await encryptionContainer.CreateTransactionalBatch(new PartitionKey(partitionKey)) + .CreateItem(doc1ToCreate, GetBatchItemRequestOptions(dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .CreateItemStream(doc2ToCreate.ToStream(), GetBatchItemRequestOptions(dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .ReplaceItem(doc1ToReplace.Id, doc1ToReplace, GetBatchItemRequestOptions(dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, doc1ToReplaceCreateResponse.ETag)) + .CreateItem(doc3ToCreate) + .CreateItem(doc4ToCreate, GetBatchItemRequestOptions(dek1, new List(), jsonProcessor, compressionAlgorithm)) // empty PathsToEncrypt list + .ReplaceItemStream(doc2ToReplace.Id, doc2ToReplace.ToStream(), GetBatchItemRequestOptions(dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .UpsertItem(doc1ToUpsert, GetBatchItemRequestOptions(dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .DeleteItem(docToDelete.Id) + .UpsertItemStream(doc2ToUpsert.ToStream(), GetBatchItemRequestOptions(dek2, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .ExecuteAsync(); + + Assert.AreEqual(HttpStatusCode.OK, batchResponse.StatusCode); + + TransactionalBatchOperationResult doc1 = batchResponse.GetOperationResultAtIndex(0); + VerifyExpectedDocResponse(doc1ToCreate, doc1.Resource); + + TransactionalBatchOperationResult doc2 = batchResponse.GetOperationResultAtIndex(1); + VerifyExpectedDocResponse(doc2ToCreate, doc2.Resource); + + TransactionalBatchOperationResult doc3 = batchResponse.GetOperationResultAtIndex(2); + VerifyExpectedDocResponse(doc1ToReplace, doc3.Resource); + + TransactionalBatchOperationResult doc4 = batchResponse.GetOperationResultAtIndex(3); + VerifyExpectedDocResponse(doc3ToCreate, doc4.Resource); + + TransactionalBatchOperationResult doc5 = batchResponse.GetOperationResultAtIndex(4); + VerifyExpectedDocResponse(doc4ToCreate, doc5.Resource); + + TransactionalBatchOperationResult doc6 = batchResponse.GetOperationResultAtIndex(5); + VerifyExpectedDocResponse(doc2ToReplace, doc6.Resource); + + TransactionalBatchOperationResult doc7 = batchResponse.GetOperationResultAtIndex(6); + VerifyExpectedDocResponse(doc1ToUpsert, doc7.Resource); + + TransactionalBatchOperationResult doc8 = batchResponse.GetOperationResultAtIndex(8); + VerifyExpectedDocResponse(doc2ToUpsert, doc8.Resource); + + await VerifyItemByReadAsync(encryptionContainer, doc1ToCreate); + await VerifyItemByReadAsync(encryptionContainer, doc2ToCreate, dekId: dek2); + await VerifyItemByReadAsync(encryptionContainer, doc3ToCreate, isDocDecrypted: false); + await VerifyItemByReadAsync(encryptionContainer, doc4ToCreate, isDocDecrypted: false); + await VerifyItemByReadAsync(encryptionContainer, doc1ToReplace, dekId: dek2); + await VerifyItemByReadAsync(encryptionContainer, doc2ToReplace, dekId: dek2); + await VerifyItemByReadAsync(encryptionContainer, doc1ToUpsert); + await VerifyItemByReadAsync(encryptionContainer, doc2ToUpsert, dekId: dek2); + + ResponseMessage readResponseMessage = await encryptionContainer.ReadItemStreamAsync(docToDelete.Id, new PartitionKey(docToDelete.PK)); + Assert.AreEqual(HttpStatusCode.NotFound, readResponseMessage.StatusCode); + + // doc3ToCreate, doc4ToCreate wasn't encrypted + await VerifyItemByReadAsync(itemContainer, doc3ToCreate); + await VerifyItemByReadAsync(itemContainer, doc4ToCreate); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionTransactionalBatchWithCustomSerializer(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + CustomSerializer customSerializer = new(); + CosmosClient clientWithCustomSerializer = TestCommon.CreateCosmosClient(builder => builder + .WithCustomSerializer(customSerializer) + .Build()); + + Database databaseWithCustomSerializer = clientWithCustomSerializer.GetDatabase(database.Id); + Container containerWithCustomSerializer = databaseWithCustomSerializer.GetContainer(itemContainer.Id); + Container encryptionContainerWithCustomSerializer = containerWithCustomSerializer.WithEncryptor(encryptor); + + string partitionKey = "thePK"; + string dek1 = dekId; + + TestDoc doc1ToCreate = TestDoc.Create(partitionKey); + + ItemResponse doc1ToReplaceCreateResponse = await CreateItemAsync(encryptionContainerWithCustomSerializer, dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + TestDoc doc1ToReplace = doc1ToReplaceCreateResponse.Resource; + doc1ToReplace.NonSensitive = Guid.NewGuid().ToString(); + doc1ToReplace.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TransactionalBatchResponse batchResponse = await encryptionContainerWithCustomSerializer.CreateTransactionalBatch(new PartitionKey(partitionKey)) + .CreateItem(doc1ToCreate, GetBatchItemRequestOptions(dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .ReplaceItem(doc1ToReplace.Id, doc1ToReplace, GetBatchItemRequestOptions(dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, doc1ToReplaceCreateResponse.ETag)) + .ExecuteAsync(); + + Assert.AreEqual(HttpStatusCode.OK, batchResponse.StatusCode); + // FromStream is called as part of CreateItem request + Assert.AreEqual(1, customSerializer.FromStreamCalled); + + TransactionalBatchOperationResult doc1 = batchResponse.GetOperationResultAtIndex(0); + VerifyExpectedDocResponse(doc1ToCreate, doc1.Resource); + Assert.AreEqual(2, customSerializer.FromStreamCalled); + + TransactionalBatchOperationResult doc2 = batchResponse.GetOperationResultAtIndex(1); + VerifyExpectedDocResponse(doc1ToReplace, doc2.Resource); + Assert.AreEqual(3, customSerializer.FromStreamCalled); + + await VerifyItemByReadAsync(encryptionContainerWithCustomSerializer, doc1ToCreate); + await VerifyItemByReadAsync(encryptionContainerWithCustomSerializer, doc1ToReplace); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task VerifyDekOperationWithSystemTextSerializer(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + System.Text.Json.JsonSerializerOptions jsonSerializerOptions = new() + { + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + CosmosSystemTextJsonSerializer cosmosSystemTextJsonSerializer = new(jsonSerializerOptions); + + CosmosClient clientWithCosmosSystemTextJsonSerializer = TestCommon.CreateCosmosClient(builder => builder + .WithCustomSerializer(cosmosSystemTextJsonSerializer) + .Build()); + + // get database and container + Database databaseWithCosmosSystemTextJsonSerializer = clientWithCosmosSystemTextJsonSerializer.GetDatabase(database.Id); + Container containerWithCosmosSystemTextJsonSerializer = databaseWithCosmosSystemTextJsonSerializer.GetContainer(itemContainer.Id); + + // create the Dek container + Container dekContainerWithCosmosSystemTextJsonSerializer = await databaseWithCosmosSystemTextJsonSerializer.CreateContainerAsync(Guid.NewGuid().ToString(), "/id", 400); + + CosmosDataEncryptionKeyProvider dekProviderWithCosmosSystemTextJsonSerializer = new(new TestEncryptionKeyStoreProvider()); + await dekProviderWithCosmosSystemTextJsonSerializer.InitializeAsync(databaseWithCosmosSystemTextJsonSerializer, dekContainerWithCosmosSystemTextJsonSerializer.Id); + + TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new(dekProviderWithCosmosSystemTextJsonSerializer); + + // enable encryption on container + Container encryptionContainerWithCosmosSystemTextJsonSerializer = containerWithCosmosSystemTextJsonSerializer.WithEncryptor(encryptorWithCosmosSystemTextJsonSerializer); + + string dekId = "dekWithSystemTextJson"; + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(dekProviderWithCosmosSystemTextJsonSerializer, dekId); + Assert.AreEqual( + new EncryptionKeyWrapMetadata(name: "metadata1", value: metadata1.Value), + dekProperties.EncryptionKeyWrapMetadata); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(databaseWithCosmosSystemTextJsonSerializer, dekContainerWithCosmosSystemTextJsonSerializer.Id); + DataEncryptionKeyProperties readProperties = await dekProviderWithCosmosSystemTextJsonSerializer.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + + // rewrap + ItemResponse dekResponse = await dekProviderWithCosmosSystemTextJsonSerializer.DataEncryptionKeyContainer.RewrapDataEncryptionKeyAsync( + dekId, + metadata2); + + Assert.AreEqual(HttpStatusCode.OK, dekResponse.StatusCode); + dekProperties = VerifyDekResponse( + dekResponse, + dekId); + Assert.AreEqual( + metadata2, + dekProperties.EncryptionKeyWrapMetadata); + + readProperties = await dekProviderWithCosmosSystemTextJsonSerializer.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + + TestDocSystemText testDocSystemText = new() + { + Id = Guid.NewGuid().ToString(), + ActivityId = Guid.NewGuid().ToString(), + PartitionKey = "myPartitionKey", + Status = "Active" + }; + + // Create items that use System.Text.Json serialization attributes + ItemResponse createTestDoc = await encryptionContainerWithCosmosSystemTextJsonSerializer.CreateItemAsync( + testDocSystemText, + new PartitionKey(testDocSystemText.PartitionKey), + GetRequestOptions(dekId, new List() { "/status" }, jsonProcessor, compressionAlgorithm, legacyAlgo: false)); + + Assert.AreEqual(HttpStatusCode.Created, createTestDoc.StatusCode); + + string contosoV1 = "Contoso_v001"; + string contosoV2 = "Contoso_v002"; + string fabrikamV1 = "Fabrikam_v001"; + string fabrikamV2 = "Fabrikam_v002"; + + await CreateDekAsync(dekProviderWithCosmosSystemTextJsonSerializer, contosoV1); + await CreateDekAsync(dekProviderWithCosmosSystemTextJsonSerializer, contosoV2); + await CreateDekAsync(dekProviderWithCosmosSystemTextJsonSerializer, fabrikamV1); + await CreateDekAsync(dekProviderWithCosmosSystemTextJsonSerializer, fabrikamV2); + + // Test getting all keys + await IterateDekFeedAsync( + dekProviderWithCosmosSystemTextJsonSerializer, + new List { dekId, contosoV1, contosoV2, fabrikamV1, fabrikamV2 }, + isExpectedDeksCompleteSetForRequest: true, + isResultOrderExpected: false, + "SELECT * from c"); + + // Test getting specific subset of keys + await IterateDekFeedAsync( + dekProviderWithCosmosSystemTextJsonSerializer, + new List { contosoV2 }, + isExpectedDeksCompleteSetForRequest: false, + isResultOrderExpected: true, + "SELECT TOP 1 * from c where c.id >= 'Contoso_v000' and c.id <= 'Contoso_v999' ORDER BY c.id DESC"); + + // Ensure only required results are returned + await IterateDekFeedAsync( + dekProviderWithCosmosSystemTextJsonSerializer, + new List { contosoV1, contosoV2 }, + isExpectedDeksCompleteSetForRequest: true, + isResultOrderExpected: true, + "SELECT * from c where c.id >= 'Contoso_v000' and c.id <= 'Contoso_v999' ORDER BY c.id ASC"); + + // Test pagination + await IterateDekFeedAsync( + dekProviderWithCosmosSystemTextJsonSerializer, + new List { dekId, contosoV1, contosoV2, fabrikamV1, fabrikamV2 }, + isExpectedDeksCompleteSetForRequest: true, + isResultOrderExpected: false, + "SELECT * from c", + itemCountInPage: 3); + + // cleanup + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionTransactionalBatchConflictResponse(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string partitionKey = "thePK"; + string dek1 = dekId; + + ItemResponse doc1CreatedResponse = await CreateItemAsync(encryptionContainer, dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, partitionKey); + TestDoc doc1ToCreateAgain = doc1CreatedResponse.Resource; + doc1ToCreateAgain.NonSensitive = Guid.NewGuid().ToString(); + doc1ToCreateAgain.Sensitive_StringFormat = Guid.NewGuid().ToString(); + + TransactionalBatchResponse batchResponse = await encryptionContainer.CreateTransactionalBatch(new PartitionKey(partitionKey)) + .CreateItem(doc1ToCreateAgain, GetBatchItemRequestOptions(dek1, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)) + .ExecuteAsync(); + + Assert.AreEqual(HttpStatusCode.Conflict, batchResponse.StatusCode); + Assert.AreEqual(1, batchResponse.Count); + } + + // One of query or queryDefinition is to be passed in non-null + private static async Task ValidateQueryResultsAsync( + Container container, + string query = null, + TestDoc expectedDoc = null, + QueryDefinition queryDefinition = null, + List pathsEncrypted = null, + bool legacyAlgo = false) + { + QueryRequestOptions requestOptions = expectedDoc != null + ? new QueryRequestOptions() + { + PartitionKey = new PartitionKey(expectedDoc.PK), + } + : null; + + FeedIterator queryResponseIterator; + FeedIterator queryResponseIteratorForLazyDecryption; + if (query != null) + { + queryResponseIterator = container.GetItemQueryIterator(query, requestOptions: requestOptions); + queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(query, requestOptions: requestOptions); + } + else + { + queryResponseIterator = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); + queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); + } + FeedResponse readDocs = await queryResponseIterator.ReadNextAsync(); + Assert.AreEqual(null, readDocs.ContinuationToken); + + FeedResponse readDocsLazily = await queryResponseIteratorForLazyDecryption.ReadNextAsync(); + Assert.AreEqual(null, readDocsLazily.ContinuationToken); + + if (expectedDoc != null) + { + Assert.AreEqual(1, readDocs.Count); + TestDoc readDoc = readDocs.Single(); + VerifyExpectedDocResponse(expectedDoc, readDoc); + + Assert.AreEqual(1, readDocsLazily.Count); + if (!legacyAlgo) + { + await ValidateDecryptableItem(readDocsLazily.First(), expectedDoc, pathsEncrypted: pathsEncrypted); + } + else + { + await ValidateDecryptableItem(readDocsLazily.First(), expectedDoc, dekId: legacydekId, pathsEncrypted: pathsEncrypted); + } + } + else + { + Assert.AreEqual(0, readDocs.Count); + } + } + + private static async Task ValidateQueryResultsMultipleDocumentsAsync( + Container container, + TestDoc testDoc1, + TestDoc testDoc2, + string query, + bool compareEncryptedProperty = true) + { + FeedIterator queryResponseIterator; + FeedIterator queryResponseIteratorForLazyDecryption; + + if (query == null) + { + IOrderedQueryable linqQueryable = container.GetItemLinqQueryable(); + queryResponseIterator = container.ToEncryptionFeedIterator(linqQueryable); + + IOrderedQueryable linqQueryableDecryptableItem = container.GetItemLinqQueryable(); + queryResponseIteratorForLazyDecryption = container.ToEncryptionFeedIterator(linqQueryableDecryptableItem); + } + else + { + queryResponseIterator = container.GetItemQueryIterator(query); + queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(query); + } + + FeedResponse readDocs = await queryResponseIterator.ReadNextAsync(); + Assert.AreEqual(null, readDocs.ContinuationToken); + + FeedResponse readDocsLazily = await queryResponseIteratorForLazyDecryption.ReadNextAsync(); + Assert.AreEqual(null, readDocsLazily.ContinuationToken); + + if (query == null) + { + Assert.IsTrue(readDocs.Count >= 2); + Assert.IsTrue(readDocsLazily.Count >= 2); + } + else + { + Assert.AreEqual(2, readDocs.Count); + Assert.AreEqual(2, readDocsLazily.Count); + } + + for (int index = 0; index < readDocs.Count; index++) + { + if (readDocs.ElementAt(index).Id.Equals(testDoc1.Id)) + { + if (compareEncryptedProperty) + { + VerifyExpectedDocResponse(readDocs.ElementAt(index), testDoc1); + } + else + { + testDoc1.EqualsExceptEncryptedProperty(readDocs.ElementAt(index)); + } + } + else if (readDocs.ElementAt(index).Id.Equals(testDoc2.Id)) + { + if (compareEncryptedProperty) + { + VerifyExpectedDocResponse(readDocs.ElementAt(index), testDoc2); + } + else + { + testDoc2.EqualsExceptEncryptedProperty(readDocs.ElementAt(index)); + } + } + } + } + + private static async Task ValidateQueryResponseAsync(Container container, + string query = null) + { + FeedIterator feedIterator; + if (query == null) + { + IOrderedQueryable linqQueryable = container.GetItemLinqQueryable(); + feedIterator = container.ToEncryptionStreamIterator(linqQueryable); + } + else + { + feedIterator = container.GetItemQueryStreamIterator(query); + } + + while (feedIterator.HasMoreResults) + { + ResponseMessage response = await feedIterator.ReadNextAsync(); + Assert.IsTrue(response.IsSuccessStatusCode); + Assert.IsNull(response.ErrorMessage); + } + } + + private static async Task ValidateQueryResponseWithLazyDecryptionAsync(Container container, + string query = null) + { + FeedIterator queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(query); + FeedResponse readDocsLazily = await queryResponseIteratorForLazyDecryption.ReadNextAsync(); + Assert.AreEqual(null, readDocsLazily.ContinuationToken); + Assert.AreEqual(1, readDocsLazily.Count); + (dynamic readDoc, DecryptionContext decryptionContext) = await readDocsLazily.First().GetItemAsync(); + Assert.IsTrue((long)readDoc >= 1); + Assert.IsNull(decryptionContext); + } + + private static async Task ValidateChangeFeedIteratorResponse( + Container container, + TestDoc testDoc1, + TestDoc testDoc2) + { + FeedIterator changeIterator = container.GetChangeFeedIterator( + ChangeFeedStartFrom.Beginning(), + ChangeFeedMode.Incremental); + + while (changeIterator.HasMoreResults) + { + FeedResponse testDocs = await changeIterator.ReadNextAsync(); + if (testDocs.StatusCode == HttpStatusCode.NotModified) + { + break; + } + + Assert.AreEqual(testDocs.Count, 2); + + VerifyExpectedDocResponse(testDoc1, testDocs.Resource.ElementAt(0)); + VerifyExpectedDocResponse(testDoc2, testDocs.Resource.ElementAt(1)); + } + } + + private static async Task ValidateChangeFeedProcessorResponse( + Container container, + TestDoc testDoc1, + TestDoc testDoc2) + { + Database leaseDatabase = await client.CreateDatabaseAsync(Guid.NewGuid().ToString()); + Container leaseContainer = await leaseDatabase.CreateContainerIfNotExistsAsync( + new ContainerProperties(id: "leases", partitionKeyPath: "/id")); + ManualResetEvent allDocsProcessed = new(false); + int processedDocCount = 0; + + List changeFeedReturnedDocs = new(); + ChangeFeedProcessor cfp = container.GetChangeFeedProcessorBuilder( + "testCFP", + (IReadOnlyCollection changes, CancellationToken cancellationToken) => + { + changeFeedReturnedDocs.AddRange(changes); + processedDocCount += changes.Count; + if (processedDocCount == 2) + { + allDocsProcessed.Set(); + } + + return Task.CompletedTask; + }) + .WithInstanceName("random") + .WithLeaseContainer(leaseContainer) + .WithStartTime(DateTime.MinValue.ToUniversalTime()) + .Build(); + + await cfp.StartAsync(); + bool isStartOk = allDocsProcessed.WaitOne(60000); + await cfp.StopAsync(); + + Assert.AreEqual(changeFeedReturnedDocs.Count, 2); + + VerifyExpectedDocResponse(testDoc1, changeFeedReturnedDocs[^2]); + VerifyExpectedDocResponse(testDoc2, changeFeedReturnedDocs[^1]); + + if (leaseDatabase != null) + { + using (await leaseDatabase.DeleteStreamAsync()) { } + } + } + + private static async Task ValidateChangeFeedProcessorWithFeedHandlerResponse( + Container container, + TestDoc testDoc1, + TestDoc testDoc2) + { + Database leaseDatabase = await client.CreateDatabaseAsync(Guid.NewGuid().ToString()); + Container leaseContainer = await leaseDatabase.CreateContainerIfNotExistsAsync( + new ContainerProperties(id: "leases", partitionKeyPath: "/id")); + ManualResetEvent allDocsProcessed = new(false); + int processedDocCount = 0; + + List changeFeedReturnedDocs = new(); + ChangeFeedProcessor cfp = container.GetChangeFeedProcessorBuilder( + "testCFPWithFeedHandler", + ( + ChangeFeedProcessorContext context, + IReadOnlyCollection changes, + CancellationToken cancellationToken) => + { + changeFeedReturnedDocs.AddRange(changes); + processedDocCount += changes.Count; + if (processedDocCount == 2) + { + allDocsProcessed.Set(); + } + + return Task.CompletedTask; + }) + .WithInstanceName("random") + .WithLeaseContainer(leaseContainer) + .WithStartTime(DateTime.MinValue.ToUniversalTime()) + .Build(); + + await cfp.StartAsync(); + bool isStartOk = allDocsProcessed.WaitOne(60000); + await cfp.StopAsync(); + + Assert.AreEqual(changeFeedReturnedDocs.Count, 2); + + VerifyExpectedDocResponse(testDoc1, changeFeedReturnedDocs[^2]); + VerifyExpectedDocResponse(testDoc2, changeFeedReturnedDocs[^1]); + + if (leaseDatabase != null) + { + using (await leaseDatabase.DeleteStreamAsync()) { } + } + } + + private static async Task ValidateChangeFeedProcessorWithManualCheckpointResponse( + Container container, + TestDoc testDoc1, + TestDoc testDoc2) + { + Database leaseDatabase = await client.CreateDatabaseAsync(Guid.NewGuid().ToString()); + Container leaseContainer = await leaseDatabase.CreateContainerIfNotExistsAsync( + new ContainerProperties(id: "leases", partitionKeyPath: "/id")); + ManualResetEvent allDocsProcessed = new(false); + int processedDocCount = 0; + + List changeFeedReturnedDocs = new(); + ChangeFeedProcessor cfp = container.GetChangeFeedProcessorBuilderWithManualCheckpoint( + "testCFPWithManualCheckpoint", + ( + ChangeFeedProcessorContext context, + IReadOnlyCollection changes, + Func tryCheckpointAsync, + CancellationToken cancellationToken) => + { + changeFeedReturnedDocs.AddRange(changes); + processedDocCount += changes.Count; + if (processedDocCount == 2) + { + allDocsProcessed.Set(); + } + + return Task.CompletedTask; + }) + .WithInstanceName("random") + .WithLeaseContainer(leaseContainer) + .WithStartTime(DateTime.MinValue.ToUniversalTime()) + .Build(); + + await cfp.StartAsync(); + bool isStartOk = allDocsProcessed.WaitOne(60000); + await cfp.StopAsync(); + + Assert.AreEqual(changeFeedReturnedDocs.Count, 2); + + VerifyExpectedDocResponse(testDoc1, changeFeedReturnedDocs[^2]); + VerifyExpectedDocResponse(testDoc2, changeFeedReturnedDocs[^1]); + + if (leaseDatabase != null) + { + using (await leaseDatabase.DeleteStreamAsync()) { } + } + } + + private static async Task ValidateChangeFeedProcessorWithFeedStreamHandlerResponse( + Container container, + TestDoc testDoc1, + TestDoc testDoc2) + { + Database leaseDatabase = await client.CreateDatabaseAsync(Guid.NewGuid().ToString()); + Container leaseContainer = await leaseDatabase.CreateContainerIfNotExistsAsync( + new ContainerProperties(id: "leases", partitionKeyPath: "/id")); + ManualResetEvent allDocsProcessed = new(false); + int processedDocCount = 0; + + ChangeFeedProcessor cfp = container.GetChangeFeedProcessorBuilder( + "testCFPWithFeedStreamHandler", + ( +context, +changes, +cancellationToken) => + { + string changeFeed = string.Empty; + using (StreamReader streamReader = new(changes)) + { + changeFeed = streamReader.ReadToEnd(); + } + + if (changeFeed.Contains(testDoc1.Id)) + { + processedDocCount++; + } + + if (changeFeed.Contains(testDoc2.Id)) + { + processedDocCount++; + } + + if (processedDocCount == 2) + { + allDocsProcessed.Set(); + } + + return Task.CompletedTask; + }) + .WithInstanceName("random") + .WithLeaseContainer(leaseContainer) + .WithStartTime(DateTime.MinValue.ToUniversalTime()) + .Build(); + + await cfp.StartAsync(); + bool isStartOk = allDocsProcessed.WaitOne(60000); + await cfp.StopAsync(); + + if (leaseDatabase != null) + { + using (await leaseDatabase.DeleteStreamAsync()) { } + } + } + + private static async Task ValidateChangeFeedProcessorStreamWithManualCheckpointResponse( + Container container, + TestDoc testDoc1, + TestDoc testDoc2) + { + Database leaseDatabase = await client.CreateDatabaseAsync(Guid.NewGuid().ToString()); + Container leaseContainer = await leaseDatabase.CreateContainerIfNotExistsAsync( + new ContainerProperties(id: "leases", partitionKeyPath: "/id")); + ManualResetEvent allDocsProcessed = new(false); + int processedDocCount = 0; + + ChangeFeedProcessor cfp = container.GetChangeFeedProcessorBuilderWithManualCheckpoint( + "testCFPStreamWithManualCheckpoint", + ( +context, +changes, +tryCheckpointAsync, +cancellationToken) => + { + string changeFeed = string.Empty; + using (StreamReader streamReader = new(changes)) + { + changeFeed = streamReader.ReadToEnd(); + } + + if (changeFeed.Contains(testDoc1.Id)) + { + processedDocCount++; + } + + if (changeFeed.Contains(testDoc2.Id)) + { + processedDocCount++; + } + + if (processedDocCount == 2) + { + allDocsProcessed.Set(); + } + + return Task.CompletedTask; + }) + .WithInstanceName("random") + .WithLeaseContainer(leaseContainer) + .WithStartTime(DateTime.MinValue.ToUniversalTime()) + .Build(); + + await cfp.StartAsync(); + bool isStartOk = allDocsProcessed.WaitOne(60000); + await cfp.StopAsync(); + + if (leaseDatabase != null) + { + using (await leaseDatabase.DeleteStreamAsync()) { } + } + } + + private static async Task ValidateLazyDecryptionResponse( + IEnumerator readDocsLazily, + string failureDek) + { + int decryptedDoc = 0; + int failedDoc = 0; + + while (readDocsLazily.MoveNext()) + { + try + { + (_, _) = await readDocsLazily.Current.GetItemAsync(); + decryptedDoc++; + } + catch (EncryptionException encryptionException) + { + failedDoc++; + ValidateEncryptionException(encryptionException, failureDek); + } + } + + Assert.IsTrue(decryptedDoc >= 1); + Assert.AreEqual(1, failedDoc); + } + + private static void ValidateEncryptionException( + EncryptionException encryptionException, + string failureDek) + { + Assert.AreEqual(failureDek, encryptionException.DataEncryptionKeyId); + Assert.IsNotNull(encryptionException.EncryptedContent); + Assert.IsNotNull(encryptionException.InnerException); + Assert.IsTrue(encryptionException.InnerException is InvalidOperationException); + Assert.AreEqual(encryptionException.InnerException.Message, "Null DataEncryptionKey returned."); + } + + private static async Task IterateDekFeedAsync( + CosmosDataEncryptionKeyProvider dekProvider, + List expectedDekIds, + bool isExpectedDeksCompleteSetForRequest, + bool isResultOrderExpected, + string query, + int? itemCountInPage = null, + QueryDefinition queryDefinition = null) + { + int remainingItemCount = expectedDekIds.Count; + QueryRequestOptions requestOptions = null; + if (itemCountInPage.HasValue) + { + requestOptions = new QueryRequestOptions() + { + MaxItemCount = itemCountInPage + }; + } + + FeedIterator dekIterator = queryDefinition != null + ? dekProvider.DataEncryptionKeyContainer.GetDataEncryptionKeyQueryIterator( + queryDefinition, + requestOptions: requestOptions) + : dekProvider.DataEncryptionKeyContainer.GetDataEncryptionKeyQueryIterator( + query, + requestOptions: requestOptions); + + Assert.IsTrue(dekIterator.HasMoreResults); + + List readDekIds = new(); + while (remainingItemCount > 0) + { + FeedResponse page = await dekIterator.ReadNextAsync(); + if (itemCountInPage.HasValue) + { + // last page + if (remainingItemCount < itemCountInPage.Value) + { + Assert.AreEqual(remainingItemCount, page.Count); + } + else + { + Assert.AreEqual(itemCountInPage.Value, page.Count); + } + } + else + { + Assert.AreEqual(expectedDekIds.Count, page.Count); + } + + remainingItemCount -= page.Count; + if (isExpectedDeksCompleteSetForRequest) + { + Assert.AreEqual(remainingItemCount > 0, dekIterator.HasMoreResults); + } + + foreach (DataEncryptionKeyProperties dek in page.Resource) + { + readDekIds.Add(dek.Id); + } + } + + if (isResultOrderExpected) + { + Assert.IsTrue(expectedDekIds.SequenceEqual(readDekIds)); + } + else + { + Assert.IsTrue(expectedDekIds.ToHashSet().SetEquals(readDekIds)); + } + } + + + private static async Task> UpsertItemAsync( + Container container, + TestDoc testDoc, + string dekId, + List pathsToEncrypt, + HttpStatusCode expectedStatusCode, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm + ) + { + ItemResponse upsertResponse = await container.UpsertItemAsync( + testDoc, + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm)); + Assert.AreEqual(expectedStatusCode, upsertResponse.StatusCode); + VerifyExpectedDocResponse(testDoc, upsertResponse.Resource); + return upsertResponse; + } + + private static async Task> CreateItemAsync( + Container container, + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm, + string partitionKey = null, + bool legacyAlgo = false) + { + TestDoc testDoc = TestDoc.Create(partitionKey); + ItemResponse createResponse = await container.CreateItemAsync( + testDoc, + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm, legacyAlgo: legacyAlgo)); + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + VerifyExpectedDocResponse(testDoc, createResponse.Resource); + return createResponse; + } + + private static async Task> ReplaceItemAsync( + Container encryptedContainer, + TestDoc testDoc, + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm, + string etag = null) + { + ItemResponse replaceResponse = await encryptedContainer.ReplaceItemAsync( + testDoc, + testDoc.Id, + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm, etag)); + + Assert.AreEqual(HttpStatusCode.OK, replaceResponse.StatusCode); + + VerifyExpectedDocResponse(testDoc, replaceResponse.Resource); + + return replaceResponse; + } + + private static async Task> DeleteItemAsync( + Container encryptedContainer, + TestDoc testDoc) + { + ItemResponse deleteResponse = await encryptedContainer.DeleteItemAsync( + testDoc.Id, + new PartitionKey(testDoc.PK)); + + Assert.AreEqual(HttpStatusCode.NoContent, deleteResponse.StatusCode); + Assert.IsNull(deleteResponse.Resource); + return deleteResponse; + } + + private static EncryptionItemRequestOptions GetRequestOptions( + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm, + string ifMatchEtag = null, + bool legacyAlgo = false) + { + if (!legacyAlgo) + { + return new EncryptionItemRequestOptions + { + EncryptionOptions = GetEncryptionOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm), + IfMatchEtag = ifMatchEtag + }; + } + else + { + return new EncryptionItemRequestOptions + { + EncryptionOptions = GetLegacyEncryptionOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm), + IfMatchEtag = ifMatchEtag + }; + } + } + + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm, + string ifMatchEtag = null) + { + return new EncryptionTransactionalBatchItemRequestOptions + { + EncryptionOptions = GetEncryptionOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm), + IfMatchEtag = ifMatchEtag + }; + } + + private static EncryptionOptions GetEncryptionOptions( + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm + ) + { + return new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = pathsToEncrypt, + JsonProcessor = jsonProcessor, + CompressionOptions = new CompressionOptions { Algorithm = compressionAlgorithm} + }; + } + + private static async Task ValidateDecryptableItem( + DecryptableItem decryptableItem, + TestDoc testDoc, + string dekId = null, + List pathsEncrypted = null, + bool isDocDecrypted = true) + { + (TestDoc readDoc, DecryptionContext decryptionContext) = await decryptableItem.GetItemAsync(); + VerifyExpectedDocResponse(testDoc, readDoc); + + if (isDocDecrypted && testDoc.Sensitive_StringFormat != null) + { + ValidateDecryptionContext(decryptionContext, dekId, pathsEncrypted); + } + else + { + Assert.IsNull(decryptionContext); + } + } + + private static void ValidateDecryptionContext( + DecryptionContext decryptionContext, + string dekId = null, + List pathsEncrypted = null) + { + Assert.IsNotNull(decryptionContext.DecryptionInfoList); + Assert.AreEqual(1, decryptionContext.DecryptionInfoList.Count); + DecryptionInfo decryptionInfo = decryptionContext.DecryptionInfoList[0]; + Assert.AreEqual(dekId ?? MdeCustomEncryptionTestsWithSystemText.dekId, decryptionInfo.DataEncryptionKeyId); + + pathsEncrypted ??= TestDoc.PathsToEncrypt; + + Assert.AreEqual(pathsEncrypted.Count, decryptionInfo.PathsDecrypted.Count); + Assert.IsFalse(pathsEncrypted.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + + + private static async Task VerifyItemByReadStreamAsync(Container container, TestDoc testDoc, ItemRequestOptions requestOptions = null, bool compareEncryptedProperty = true) + { + ResponseMessage readResponseMessage = await container.ReadItemStreamAsync(testDoc.Id, new PartitionKey(testDoc.PK), requestOptions); + Assert.AreEqual(HttpStatusCode.OK, readResponseMessage.StatusCode); + Assert.IsNotNull(readResponseMessage.Content); + TestDoc readDoc = TestCommon.FromStream(readResponseMessage.Content); + if (compareEncryptedProperty) + { + VerifyExpectedDocResponse(testDoc, readDoc); + } + else + { + testDoc.EqualsExceptEncryptedProperty(readDoc); + } + } + + private static async Task VerifyItemByReadAsync(Container container, TestDoc testDoc, ItemRequestOptions requestOptions = null, string dekId = null, bool isDocDecrypted = true, bool compareEncryptedProperty = true) + { + ItemResponse readResponse = await container.ReadItemAsync(testDoc.Id, new PartitionKey(testDoc.PK), requestOptions); + Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode); + if (compareEncryptedProperty) + { + VerifyExpectedDocResponse(testDoc, readResponse.Resource); + } + else + { + testDoc.EqualsExceptEncryptedProperty(readResponse.Resource); + } + + // ignore for reads via regular container.. + if (container == encryptionContainer) + { + ItemResponse readResponseDecryptableItem = await container.ReadItemAsync(testDoc.Id, new PartitionKey(testDoc.PK), requestOptions); + Assert.AreEqual(HttpStatusCode.OK, readResponse.StatusCode); + await ValidateDecryptableItem(readResponseDecryptableItem.Resource, testDoc, dekId, isDocDecrypted: isDocDecrypted); + } + } + + private static async Task CreateDekAsync(CosmosDataEncryptionKeyProvider dekProvider, string dekId, string algorithm = null) + { + ItemResponse dekResponse = await dekProvider.DataEncryptionKeyContainer.CreateDataEncryptionKeyAsync( + dekId, + algorithm ?? CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + metadata1); + + Assert.AreEqual(HttpStatusCode.Created, dekResponse.StatusCode); + + return VerifyDekResponse(dekResponse, + dekId); + } + + + private static DataEncryptionKeyProperties VerifyDekResponse( + ItemResponse dekResponse, + string dekId) + { + Assert.IsTrue(dekResponse.RequestCharge > 0); + Assert.IsNotNull(dekResponse.ETag); + + DataEncryptionKeyProperties dekProperties = dekResponse.Resource; + Assert.IsNotNull(dekProperties); + Assert.AreEqual(dekResponse.ETag, dekProperties.ETag); + Assert.AreEqual(dekId, dekProperties.Id); + Assert.IsNotNull(dekProperties.SelfLink); + Assert.IsNotNull(dekProperties.CreatedTime); + Assert.IsNotNull(dekProperties.LastModified); + + return dekProperties; + } + + private static async Task PerformForbiddenOperationAsync(Func func, string operationName) + { + try + { + await func(); + Assert.Fail($"Expected resource token based client to not be able to perform {operationName}"); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) + { + } + } + + private static async Task PerformOperationOnUninitializedDekProviderAsync(Func func, string operationName) + { + try + { + await func(); + Assert.Fail($"Expected {operationName} to not work on uninitialized CosmosDataEncryptionKeyProvider."); + } + catch (InvalidOperationException ex) + { + Assert.IsTrue(ex.Message.Contains("The CosmosDataEncryptionKeyProvider was not initialized.")); + } + } + + private static void VerifyExpectedDocResponse(TestDoc expectedDoc, TestDoc verifyDoc) + { + Assert.AreEqual(expectedDoc.Id, verifyDoc.Id); + Assert.AreEqual(expectedDoc.Sensitive_StringFormat, verifyDoc.Sensitive_StringFormat); + if (expectedDoc.Sensitive_ArrayFormat != null) + { + Assert.AreEqual(expectedDoc.Sensitive_ArrayFormat[0].Sensitive_ArrayDecimalFormat, verifyDoc.Sensitive_ArrayFormat[0].Sensitive_ArrayDecimalFormat); + Assert.AreEqual(expectedDoc.Sensitive_ArrayFormat[0].Sensitive_ArrayIntFormat, verifyDoc.Sensitive_ArrayFormat[0].Sensitive_ArrayIntFormat); + Assert.AreEqual(expectedDoc.Sensitive_NestedObjectFormatL1.Sensitive_IntFormatL1, verifyDoc.Sensitive_NestedObjectFormatL1.Sensitive_IntFormatL1); + Assert.AreEqual( + expectedDoc.Sensitive_NestedObjectFormatL1.Sensitive_NestedObjectFormatL2.Sensitive_IntFormatL2, + verifyDoc.Sensitive_NestedObjectFormatL1.Sensitive_NestedObjectFormatL2.Sensitive_IntFormatL2); + } + else + { + Assert.AreEqual(expectedDoc.Sensitive_ArrayFormat, verifyDoc.Sensitive_ArrayFormat); + Assert.AreEqual(expectedDoc.Sensitive_NestedObjectFormatL1, verifyDoc.Sensitive_NestedObjectFormatL1); + } + Assert.AreEqual(expectedDoc.Sensitive_DateFormat, verifyDoc.Sensitive_DateFormat); + Assert.AreEqual(expectedDoc.Sensitive_DecimalFormat, verifyDoc.Sensitive_DecimalFormat); + Assert.AreEqual(expectedDoc.Sensitive_IntFormat, verifyDoc.Sensitive_IntFormat); + Assert.AreEqual(expectedDoc.Sensitive_FloatFormat, verifyDoc.Sensitive_FloatFormat); + Assert.AreEqual(expectedDoc.Sensitive_BoolFormat, verifyDoc.Sensitive_BoolFormat); + Assert.AreEqual(expectedDoc.NonSensitive, verifyDoc.NonSensitive); + } + + public class TestDoc + { + public static List PathsToEncrypt { get; } = + new List() { + "/Sensitive_StringFormat", + "/Sensitive_ArrayFormat", + "/Sensitive_DecimalFormat", + "/Sensitive_IntFormat", + "/Sensitive_DateFormat", + "/Sensitive_BoolFormat", + "/Sensitive_FloatFormat", + "/Sensitive_NestedObjectFormatL1" + }; + + [JsonProperty("id")] + public string Id { get; set; } + + public string PK { get; set; } + + public string NonSensitive { get; set; } + + public string Sensitive_StringFormat { get; set; } + + public DateTime Sensitive_DateFormat { get; set; } + + public decimal Sensitive_DecimalFormat { get; set; } + + public bool Sensitive_BoolFormat { get; set; } + + public int Sensitive_IntFormat { get; set; } + + public float Sensitive_FloatFormat { get; set; } + + public Sensitive_ArrayData[] Sensitive_ArrayFormat { get; set; } + + public Sensitive_NestedObjectL1 Sensitive_NestedObjectFormatL1 { get; set; } + + public TestDoc() + { + } + + public class Sensitive_ArrayData + { + public int Sensitive_ArrayIntFormat { get; set; } + public decimal Sensitive_ArrayDecimalFormat { get; set; } + } + + public class Sensitive_NestedObjectL1 + { + public int Sensitive_IntFormatL1 { get; set; } + public Sensitive_NestedObjectL2 Sensitive_NestedObjectFormatL2 { get; set; } + } + + public class Sensitive_NestedObjectL2 + { + public int Sensitive_IntFormatL2 { get; set; } + } + + public TestDoc(TestDoc other) + { + this.Id = other.Id; + this.PK = other.PK; + this.NonSensitive = other.NonSensitive; + this.Sensitive_StringFormat = other.Sensitive_StringFormat; + this.Sensitive_DateFormat = other.Sensitive_DateFormat; + this.Sensitive_DecimalFormat = other.Sensitive_DecimalFormat; + this.Sensitive_IntFormat = other.Sensitive_IntFormat; + this.Sensitive_ArrayFormat = other.Sensitive_ArrayFormat; + this.Sensitive_BoolFormat = other.Sensitive_BoolFormat; + this.Sensitive_FloatFormat = other.Sensitive_FloatFormat; + this.Sensitive_NestedObjectFormatL1 = other.Sensitive_NestedObjectFormatL1; + } + + public override bool Equals(object obj) + { + return obj is TestDoc doc + && this.Id == doc.Id + && this.PK == doc.PK + && this.NonSensitive == doc.NonSensitive + && this.Sensitive_StringFormat == doc.Sensitive_StringFormat + && this.Sensitive_DateFormat == doc.Sensitive_DateFormat + && this.Sensitive_DecimalFormat == doc.Sensitive_DecimalFormat + && this.Sensitive_IntFormat == doc.Sensitive_IntFormat + && this.Sensitive_ArrayFormat == doc.Sensitive_ArrayFormat + && this.Sensitive_BoolFormat == doc.Sensitive_BoolFormat + && this.Sensitive_FloatFormat == doc.Sensitive_FloatFormat + && this.Sensitive_NestedObjectFormatL1 != doc.Sensitive_NestedObjectFormatL1; + } + + public bool EqualsExceptEncryptedProperty(object obj) + { + return obj is TestDoc doc + && this.Id == doc.Id + && this.PK == doc.PK + && this.NonSensitive == doc.NonSensitive + && this.Sensitive_StringFormat != doc.Sensitive_StringFormat; + } + + public override int GetHashCode() + { + int hashCode = 1652434776; + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Id); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.PK); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_StringFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_DateFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_DecimalFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_IntFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_ArrayFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_BoolFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_FloatFormat); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.Sensitive_NestedObjectFormatL1); + return hashCode; + } + + public static TestDoc Create(string partitionKey = null) + { + return new TestDoc() + { + Id = Guid.NewGuid().ToString(), + PK = partitionKey ?? Guid.NewGuid().ToString(), + NonSensitive = Guid.NewGuid().ToString(), + Sensitive_StringFormat = Guid.NewGuid().ToString(), + Sensitive_DateFormat = new DateTime(1987, 12, 25), + Sensitive_DecimalFormat = 472.3108m, + Sensitive_IntFormat = 1965, + Sensitive_BoolFormat = true, + Sensitive_FloatFormat = 8923.124f, + Sensitive_ArrayFormat = new Sensitive_ArrayData[] + { + new() { + Sensitive_ArrayIntFormat = 1999, + Sensitive_ArrayDecimalFormat = 472.3199m + } + }, + Sensitive_NestedObjectFormatL1 = new Sensitive_NestedObjectL1() + { + Sensitive_IntFormatL1 = 1999, + Sensitive_NestedObjectFormatL2 = new Sensitive_NestedObjectL2() + { + Sensitive_IntFormatL2 = 2000, + } + } + }; + } + + public Stream ToStream() + { + return TestCommon.ToStream(this); + } + } + + private class TestEncryptionKeyStoreProvider : EncryptionKeyStoreProvider + { + readonly Dictionary keyinfo = new() + { + {masterKeyUri1.ToString(), 1}, + {masterKeyUri2.ToString(), 2}, + }; + + public Dictionary WrapKeyCallsCount { get; set; } + public Dictionary UnWrapKeyCallsCount { get; set; } + + public TestEncryptionKeyStoreProvider() + { + this.WrapKeyCallsCount = new Dictionary(); + this.UnWrapKeyCallsCount = new Dictionary(); + } + + public override string ProviderName => "TESTKEYSTORE_VAULT"; + + public override byte[] UnwrapKey(string masterKeyPath, KeyEncryptionKeyAlgorithm encryptionAlgorithm, byte[] encryptedKey) + { + this.UnWrapKeyCallsCount[masterKeyPath] = !this.UnWrapKeyCallsCount.TryGetValue(masterKeyPath, out int value) ? 1 : ++value; + + this.keyinfo.TryGetValue(masterKeyPath, out int moveBy); + byte[] plainkey = encryptedKey.Select(b => (byte)(b - moveBy)).ToArray(); + return plainkey; + } + + public override byte[] WrapKey(string masterKeyPath, KeyEncryptionKeyAlgorithm encryptionAlgorithm, byte[] key) + { + this.WrapKeyCallsCount[masterKeyPath] = !this.WrapKeyCallsCount.TryGetValue(masterKeyPath, out int value) ? 1 : ++value; + + this.keyinfo.TryGetValue(masterKeyPath, out int moveBy); + byte[] encryptedkey = key.Select(b => (byte)(b + moveBy)).ToArray(); + return encryptedkey; + } + + public override byte[] Sign(string masterKeyPath, bool allowEnclaveComputations) + { + byte[] rawKey = new byte[32]; + SecurityUtility.GenerateRandomBytes(rawKey); + return rawKey; + } + + public override bool Verify(string masterKeyPath, bool allowEnclaveComputations, byte[] signature) + { + return true; + } + } + + // This class is same as CosmosEncryptor but copied so as to induce decryption failure easily for testing. + private class TestEncryptor : Encryptor + { + public DataEncryptionKeyProvider DataEncryptionKeyProvider { get; } + public bool FailDecryption { get; set; } + + private readonly CosmosEncryptor encryptor; + + public TestEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) + { + this.encryptor = new CosmosEncryptor(dataEncryptionKeyProvider); + this.FailDecryption = false; + } + + private void ThrowIfFail(string dataEncryptionKeyId) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + } + + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); + } + + public override async Task EncryptAsync( + byte[] plainText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); + } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); + } + } + + public static IEnumerable ProcessorAndCompressorCombinations => new[] { + new object[] { JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None }, + new object[] { JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None }, + new object[] { JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli }, + new object[] { JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli }, + }; + + #region Legacy +#pragma warning disable CS0618 // Type or member is obsolete + [TestMethod] + public async Task EncryptionCreateDekWithDualDekProvider() + { + string dekId = "dekWithDualDekProviderNewAlgo"; + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(dualDekProvider, dekId); + Assert.AreEqual( + new EncryptionKeyWrapMetadata(name: "metadata1", value: metadata1.Value), + dekProperties.EncryptionKeyWrapMetadata); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestKeyWrapProvider(), new TestEncryptionKeyStoreProvider(), TimeSpan.FromMinutes(30)); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKeyProperties readProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + + dekId = "dekWithDualDekProviderLegacyAlgo"; + dekProperties = await CreateLegacyDekAsync(dualDekProvider, dekId); + Assert.AreEqual( + new EncryptionKeyWrapMetadata(metadata1.Value + metadataUpdateSuffix), + dekProperties.EncryptionKeyWrapMetadata); + + readProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dekProperties, readProperties); + } + + [TestMethod] + public async Task EncryptionCreateDekWithNonMdeAlgorithmFails() + { + string dekId = "oldDek"; + TestEncryptionKeyStoreProvider testKeyStoreProvider = new() + { + DataEncryptionKeyCacheTimeToLive = TimeSpan.FromSeconds(3600) + }; + + CosmosDataEncryptionKeyProvider dekProvider = new(testKeyStoreProvider); + try + { + await CreateDekAsync(dekProvider, dekId, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + Assert.Fail("CreateDataEncryptionKeyAsync should not have succeeded. "); + } + catch (InvalidOperationException ex) + { + Assert.AreEqual("For use of 'AEAes256CbcHmacSha256Randomized' algorithm, Encryptor or CosmosDataEncryptionKeyProvider needs to be initialized with EncryptionKeyWrapProvider.", ex.Message); + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionCreateItemWithIncompatibleWrapProvider(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + Container legacyEncryptionContainer; + CosmosDataEncryptionKeyProvider legacydekProvider = new(new TestKeyWrapProvider()); + await legacydekProvider.InitializeAsync(database, keyContainer.Id); + TestEncryptor legacyEncryptor = new(legacydekProvider); + legacyEncryptionContainer = itemContainer.WithEncryptor(legacyEncryptor); + TestDoc testDoc = TestDoc.Create(null); + + try + { + ItemResponse createResponse = await legacyEncryptionContainer.CreateItemAsync( + testDoc, + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, legacyAlgo: true)); + Assert.Fail("CreateItemAsync should not have succeeded. "); + } + catch (InvalidOperationException ex) + { + Assert.AreEqual("For use of 'MdeAeadAes256CbcHmac256Randomized' algorithm based DEK, Encryptor or CosmosDataEncryptionKeyProvider needs to be initialized with EncryptionKeyStoreProvider.", ex.Message); + } + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionCreateItemUsingLegacyAlgoWithMdeDek(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + TestDoc testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, legacyAlgo: true); + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: dekId); + } + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionCreateItemUsingMDEAlgoWithLegacyDek(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + CosmosDataEncryptionKeyProvider legacydekProvider = new(new TestKeyWrapProvider()); + await legacydekProvider.InitializeAsync(database, keyContainer.Id); + + TestDoc testDoc = TestDoc.Create(null); + + ItemResponse createResponse = await encryptionContainer.CreateItemAsync( + testDoc, + new PartitionKey(testDoc.PK), + GetRequestOptions(legacydekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, legacyAlgo: false)); + + VerifyExpectedDocResponse(testDoc, createResponse); + + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: legacydekId); + } + + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task EncryptionRewrapLegacyDekToMdeWrap(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + string dekId = "rewrapLegacyAlgoDektoMdeAlgoDek"; + DataEncryptionKeyProperties dataEncryptionKeyProperties; + + dataEncryptionKeyProperties = await CreateLegacyDekAsync(dualDekProvider, dekId); + + Assert.AreEqual( + metadata1.Value + metadataUpdateSuffix, + dataEncryptionKeyProperties.EncryptionKeyWrapMetadata.Value); + + Assert.AreEqual(CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized, dataEncryptionKeyProperties.EncryptionAlgorithm); + + // use it to create item with Legacy Algo + TestDoc testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, legacyAlgo: true); + + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: dekId); + + // validate key with new Algo + testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: dekId); + + ItemResponse dekResponse = await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContainer.RewrapDataEncryptionKeyAsync( + dekId, + metadata2, + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized); + + Assert.AreEqual(HttpStatusCode.OK, dekResponse.StatusCode); + + dataEncryptionKeyProperties = VerifyDekResponse( + dekResponse, + dekId); + + Assert.AreEqual(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, dataEncryptionKeyProperties.EncryptionAlgorithm); + + Assert.AreEqual( + metadata2, + dataEncryptionKeyProperties.EncryptionKeyWrapMetadata); + + // Use different DEK provider to avoid (unintentional) cache impact + CosmosDataEncryptionKeyProvider dekProvider = new(new TestEncryptionKeyStoreProvider()); + await dekProvider.InitializeAsync(database, keyContainer.Id); + DataEncryptionKeyProperties readProperties = await dekProvider.DataEncryptionKeyContainer.ReadDataEncryptionKeyAsync(dekId); + Assert.AreEqual(dataEncryptionKeyProperties, readProperties); + + // validate key + testDoc = await CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: dekId); + + // rewrap from Mde Algo to Legacy algo should fail + dekId = "rewrapMdeAlgoDekToLegacyAlgoDek"; + + DataEncryptionKeyProperties dekProperties = await CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, dekId); + Assert.AreEqual( + metadata1, + dekProperties.EncryptionKeyWrapMetadata); + + try + { + await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContainer.RewrapDataEncryptionKeyAsync( + dekId, + metadata2, + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized); + + Assert.Fail("RewrapDataEncryptionKeyAsync should not have succeeded. "); + } + catch (InvalidOperationException ex) + { + Assert.AreEqual("Rewrap operation with EncryptionAlgorithm 'AEAes256CbcHmacSha256Randomized' is not supported on Data Encryption Keys which are configured with 'MdeAeadAes256CbcHmac256Randomized'. ", ex.Message); + } + + // rewrap Mde to Mde with Option + + // rewrap from Mde Algo to Legacy algo should fail + dekId = "rewrapMdeAlgoDekToMdeAlgoDek"; + + dekProperties = await CreateDekAsync(MdeCustomEncryptionTestsWithSystemText.dekProvider, dekId); + Assert.AreEqual( + metadata1, + dekProperties.EncryptionKeyWrapMetadata); + + dekResponse = await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContainer.RewrapDataEncryptionKeyAsync( + dekId, + metadata2, + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized); + + Assert.AreEqual(HttpStatusCode.OK, dekResponse.StatusCode); + + dataEncryptionKeyProperties = VerifyDekResponse( + dekResponse, + dekId); + + Assert.AreEqual(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, dataEncryptionKeyProperties.EncryptionAlgorithm); + + Assert.AreEqual( + metadata2, + dataEncryptionKeyProperties.EncryptionKeyWrapMetadata); + } + + + [TestMethod] + [DynamicData(nameof(ProcessorAndCompressorCombinations))] + public async Task ReadLegacyEncryptedDataWithMdeProcessor(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + // Setup the Container with a Dual Wrap Provider Container. + encryptionContainer = itemContainer.WithEncryptor(encryptorWithDualWrapProvider); + + TestDoc testDoc = await CreateItemAsyncUsingLegacyAlgorithm(encryptionContainer, legacydekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: legacydekId); + + await VerifyItemByReadStreamAsync(encryptionContainer, testDoc); + + TestDoc expectedDoc = new(testDoc); + + // Read feed (null query) + await MdeCustomEncryptionTestsWithSystemText.ValidateQueryResultsAsync( + MdeCustomEncryptionTestsWithSystemText.encryptionContainer, + query: null, + expectedDoc, + legacyAlgo: true); + + await ValidateQueryResultsAsync( + encryptionContainer, + "SELECT * FROM c", + expectedDoc, + legacyAlgo: true); + + await ValidateQueryResultsAsync( + encryptionContainer, + string.Format( + "SELECT * FROM c where c.PK = '{0}' and c.id = '{1}' and c.NonSensitive = '{2}'", + expectedDoc.PK, + expectedDoc.Id, + expectedDoc.NonSensitive), + expectedDoc, + legacyAlgo: true); + + await ValidateQueryResultsAsync( + encryptionContainer, + string.Format("SELECT * FROM c where c.Sensitive_IntFormat = '{0}'", testDoc.Sensitive_StringFormat), + expectedDoc: null, + legacyAlgo: true); + + await ValidateQueryResultsAsync( + encryptionContainer, + queryDefinition: new QueryDefinition( + "select * from c where c.id = @theId and c.PK = @thePK") + .WithParameter("@theId", expectedDoc.Id) + .WithParameter("@thePK", expectedDoc.PK), + expectedDoc: expectedDoc, + legacyAlgo: true); + + expectedDoc.Sensitive_NestedObjectFormatL1 = null; + expectedDoc.Sensitive_ArrayFormat = null; + expectedDoc.Sensitive_DecimalFormat = 0; + expectedDoc.Sensitive_IntFormat = 0; + expectedDoc.Sensitive_FloatFormat = 0; + expectedDoc.Sensitive_BoolFormat = false; + expectedDoc.Sensitive_StringFormat = null; + expectedDoc.Sensitive_DateFormat = new DateTime(); + + await ValidateQueryResultsAsync( + encryptionContainer, + "SELECT c.id, c.PK, c.NonSensitive FROM c", + expectedDoc); + + // create Items with New Algorithm + await this.EncryptionCreateItem(jsonProcessor, compressionAlgorithm); + + // read back Data Items encrypted with Old Algorithm + await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: legacydekId); + + await VerifyItemByReadStreamAsync(encryptionContainer, testDoc); + + // Create and read back Data Items encrypted with Old Algorithm + TestDoc testDoc2 = await CreateItemAsyncUsingLegacyAlgorithm(encryptionContainer, legacydekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm); + + await VerifyItemByReadAsync(encryptionContainer, testDoc2, dekId: legacydekId); + + await VerifyItemByReadStreamAsync(encryptionContainer, testDoc2); + + // create Items with New Algorithm + await this.EncryptionCreateItem(jsonProcessor, compressionAlgorithm); + + // read back Data Items encrypted with Old Algorithm + await VerifyItemByReadAsync(encryptionContainer, testDoc2, dekId: legacydekId); + + await VerifyItemByReadStreamAsync(encryptionContainer, testDoc2); + + // Reset the Container for Other Tests to be carried on regular Encryptor with Single Dek Provider. + encryptionContainer = itemContainer.WithEncryptor(encryptor); + } + + + private static async Task> CreateItemAsyncUsingLegacyAlgorithm( + Container container, + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm, + string partitionKey = null) + { + TestDoc testDoc = TestDoc.Create(partitionKey); + ItemResponse createResponse = await container.CreateItemAsync( + testDoc, + new PartitionKey(testDoc.PK), + GetRequestOptions(dekId, pathsToEncrypt, jsonProcessor, compressionAlgorithm, legacyAlgo: true)); + Assert.AreEqual(HttpStatusCode.Created, createResponse.StatusCode); + + VerifyExpectedDocResponse(testDoc, createResponse.Resource); + + return createResponse; + } + + private static async Task LegacyClassInitializeAsync() + { + MdeCustomEncryptionTestsWithSystemText.testKeyStoreProvider.DataEncryptionKeyCacheTimeToLive = TimeSpan.FromSeconds(3600); + + dekProvider = new CosmosDataEncryptionKeyProvider(new TestKeyWrapProvider(), MdeCustomEncryptionTestsWithSystemText.testKeyStoreProvider); + legacytestKeyWrapProvider = new TestKeyWrapProvider(); + + TestEncryptionKeyStoreProvider testKeyStoreProvider = new() + { + DataEncryptionKeyCacheTimeToLive = TimeSpan.Zero + }; + dualDekProvider = new CosmosDataEncryptionKeyProvider(legacytestKeyWrapProvider, testKeyStoreProvider); + await dualDekProvider.InitializeAsync(database, keyContainer.Id); + + _ = await CreateLegacyDekAsync(MdeCustomEncryptionTestsWithSystemText.dualDekProvider, MdeCustomEncryptionTestsWithSystemText.legacydekId); + encryptorWithDualWrapProvider = new TestEncryptor(dualDekProvider); + } + + private static EncryptionOptions GetLegacyEncryptionOptions( + string dekId, + List pathsToEncrypt, + JsonProcessor jsonProcessor, + CompressionOptions.CompressionAlgorithm compressionAlgorithm) + { + return new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized, + PathsToEncrypt = pathsToEncrypt, + JsonProcessor = jsonProcessor, + CompressionOptions = new CompressionOptions() { Algorithm = compressionAlgorithm } + }; + } + + private static async Task CreateLegacyDekAsync(CosmosDataEncryptionKeyProvider dekProvider, string dekId, string algorithm = null) + { + ItemResponse dekResponse = await dekProvider.DataEncryptionKeyContainer.CreateDataEncryptionKeyAsync( + dekId, + algorithm ?? CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized, + metadata1); + + Assert.AreEqual(HttpStatusCode.Created, dekResponse.StatusCode); + + return VerifyDekResponse(dekResponse, + dekId); + } + + + private class TestKeyWrapProvider : EncryptionKeyWrapProvider + { + public Dictionary WrapKeyCallsCount { get; private set; } + + public TestKeyWrapProvider() + { + this.WrapKeyCallsCount = new Dictionary(); + } + + public override Task UnwrapKeyAsync(byte[] wrappedKey, EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) + { + int moveBy = metadata.Value == metadata1.Value + metadataUpdateSuffix ? 1 : 2; + return Task.FromResult(new EncryptionKeyUnwrapResult(wrappedKey.Select(b => (byte)(b - moveBy)).ToArray(), cacheTTL)); + } + + public override Task WrapKeyAsync(byte[] key, EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) + { + this.WrapKeyCallsCount[metadata.Value] = !this.WrapKeyCallsCount.TryGetValue(metadata.Value, out int value) ? 1 : ++value; + + EncryptionKeyWrapMetadata responseMetadata = new(metadata.Value + metadataUpdateSuffix); + int moveBy = metadata.Value == metadata1.Value ? 1 : 2; + return Task.FromResult(new EncryptionKeyWrapResult(key.Select(b => (byte)(b + moveBy)).ToArray(), responseMetadata)); + } + } + +#pragma warning restore CS0618 // Type or member is obsolete + #endregion + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj index bab1697c57..c2e5ee53b5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj @@ -12,6 +12,7 @@ master True $(LangVersion) + $(DefineConstants);ENCRYPTION_CUSTOM_PREVIEW From c98f2bf7a5082f1863d91dff61b8082d054c0739 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 14:48:27 +0200 Subject: [PATCH 75/85] - drop JsonNode serializer/deserializer --- .../src/EncryptionOptions.cs | 6 - .../src/EncryptionProcessor.cs | 90 ------- .../JsonNodeSqlSerializer.Preview.cs | 117 --------- .../MdeEncryptionProcessor.Preview.cs | 17 -- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 224 ------------------ .../SystemTextJson/JsonBytes.cs | 34 --- .../SystemTextJson/JsonBytesConverter.cs | 26 -- .../EncryptionBenchmark.cs | 2 +- .../MdeEncryptionProcessorTests.cs | 101 -------- .../Transformation/JsonBytesConverterTests.cs | 40 ---- .../Transformation/JsonBytesTests.cs | 33 --- .../JsonNodeSqlSerializerTests.cs | 171 ------------- 12 files changed, 1 insertion(+), 860 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index af8ab293e9..525a8182c3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -17,12 +17,6 @@ public enum JsonProcessor Newtonsoft, #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - /// - /// System.Text.Json - /// - /// Available with .NET8.0 package only. - SystemTextJson, - /// /// Ut8JsonReader/Writer /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 8612eb3958..7bf05fb930 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,10 +10,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - using System.Text.Json; - using System.Text.Json.Nodes; -#endif using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; @@ -33,7 +29,6 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); #endif @@ -158,7 +153,6 @@ public static async Task EncryptAsync( { JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") @@ -229,43 +223,6 @@ public static async Task DecryptAsync( } #endif -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( - Stream input, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - if (input == null) - { - return (input, null); - } - - Debug.Assert(input.CanSeek); - Debug.Assert(encryptor != null); - Debug.Assert(diagnosticsContext != null); - - JsonNode document = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken); - - (JsonNode decryptedDocument, DecryptionContext context) = await DecryptAsync(document, encryptor, diagnosticsContext, cancellationToken); - if (context == null) - { - input.Position = 0; - return (input, null); - } - - await input.DisposeAsync(); - - MemoryStream ms = new (); - Utf8JsonWriter writer = new (ms, EncryptionProcessor.JsonWriterOptions); - - System.Text.Json.JsonSerializer.Serialize(writer, decryptedDocument); - - ms.Position = 0; - return (ms, context); - } -#endif - #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( Stream input, @@ -327,53 +284,6 @@ public static async Task DecryptAsync( return (document, decryptionContext); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(JsonNode, DecryptionContext)> DecryptAsync( - JsonNode document, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - Debug.Assert(document != null); - - Debug.Assert(encryptor != null); - - if (!document.AsObject().TryGetPropertyValue(Constants.EncryptedInfo, out JsonNode encryptionPropertiesNode)) - { - return (document, null); - } - - EncryptionProperties encryptionProperties; - try - { - encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(encryptionPropertiesNode); - } - catch (Exception) - { - return (document, null); - } - - DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionProperties, cancellationToken); - - return (document, decryptionContext); - } - - private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JsonNode itemNode, EncryptionProperties encryptionProperties, CancellationToken cancellationToken) - { - DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch - { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( - itemNode, - encryptor, - encryptionProperties, - diagnosticsContext, - cancellationToken), - _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), - }; - return decryptionContext; - } -#endif - private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs deleted file mode 100644 index c96b76a731..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ /dev/null @@ -1,117 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation -{ - using System; - using System.Diagnostics; - using System.Text.Json; - using System.Text.Json.Nodes; - using Microsoft.Data.Encryption.Cryptography.Serializers; - - internal class JsonNodeSqlSerializer - { - private static readonly SqlBitSerializer SqlBoolSerializer = new (); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); - private static readonly SqlBigIntSerializer SqlLongSerializer = new (); - - // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); - -#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed - internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JsonNode propertyValue, ArrayPoolManager arrayPoolManager) - { - byte[] buffer; - int length; - - if (propertyValue == null) - { - return (TypeMarker.Null, null, -1); - } - - switch (propertyValue.GetValueKind()) - { - case JsonValueKind.Undefined: - Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null, -1); - case JsonValueKind.Null: - Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null, -1); - case JsonValueKind.True: - (buffer, length) = SerializeFixed(SqlBoolSerializer, true); - return (TypeMarker.Boolean, buffer, length); - case JsonValueKind.False: - (buffer, length) = SerializeFixed(SqlBoolSerializer, false); - return (TypeMarker.Boolean, buffer, length); - case JsonValueKind.Number: - if (long.TryParse(propertyValue.ToJsonString(), out long longValue)) - { - (buffer, length) = SerializeFixed(SqlLongSerializer, longValue); - return (TypeMarker.Long, buffer, length); - } - else if (double.TryParse(propertyValue.ToJsonString(), out double doubleValue)) - { - (buffer, length) = SerializeFixed(SqlDoubleSerializer, doubleValue); - return (TypeMarker.Double, buffer, length); - } - else - { - throw new InvalidOperationException("Unsupported Number type"); - } - - case JsonValueKind.String: - (buffer, length) = SerializeString(propertyValue.GetValue()); - return (TypeMarker.String, buffer, length); - case JsonValueKind.Array: - (buffer, length) = SerializeString(propertyValue.ToJsonString()); - return (TypeMarker.Array, buffer, length); - case JsonValueKind.Object: - (buffer, length) = SerializeString(propertyValue.ToJsonString()); - return (TypeMarker.Object, buffer, length); - default: - throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.GetValueKind()}"); - } - - (byte[], int) SerializeFixed(IFixedSizeSerializer serializer, T value) - { - byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int length = serializer.Serialize(value, buffer); - return (buffer, length); - } - - (byte[], int) SerializeString(string value) - { - byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int length = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, length); - } - } - - internal virtual JsonNode Deserialize( - TypeMarker typeMarker, - ReadOnlySpan serializedBytes) - { - switch (typeMarker) - { - case TypeMarker.Boolean: - return JsonValue.Create(SqlBoolSerializer.Deserialize(serializedBytes)); - case TypeMarker.Double: - return JsonValue.Create(SqlDoubleSerializer.Deserialize(serializedBytes)); - case TypeMarker.Long: - return JsonValue.Create(SqlLongSerializer.Deserialize(serializedBytes)); - case TypeMarker.String: - return JsonValue.Create(SqlVarCharSerializer.Deserialize(serializedBytes)); - case TypeMarker.Array: - return JsonNode.Parse(serializedBytes); - case TypeMarker.Object: - return JsonNode.Parse(serializedBytes); - default: - Debug.Fail($"Unexpected type marker {typeMarker}"); - return null; - } - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 2dc3eb7e10..389a709207 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -20,8 +20,6 @@ internal class MdeEncryptionProcessor internal MdeJObjectEncryptionProcessor JObjectEncryptionProcessor { get; set; } = new MdeJObjectEncryptionProcessor(); #if NET8_0_OR_GREATER - internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor(); - internal StreamProcessor StreamProcessor { get; set; } = new StreamProcessor(); #endif @@ -36,9 +34,6 @@ public async Task EncryptAsync( { case JsonProcessor.Newtonsoft: return await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); - - case JsonProcessor.SystemTextJson: - return await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); case JsonProcessor.Stream: MemoryStream ms = new (); await this.StreamProcessor.EncryptStreamAsync(input, ms, encryptor, encryptionOptions, token); @@ -65,18 +60,6 @@ internal async Task DecryptObjectAsync( { return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); } - -#if NET8_0_OR_GREATER - internal async Task DecryptObjectAsync( - JsonNode document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - return await this.JsonNodeEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); - } -#endif } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs deleted file mode 100644 index b7fcda6b43..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ /dev/null @@ -1,224 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation -{ - using System; - using System.Collections.Generic; - using System.IO; - using System.Linq; - using System.Text.Json; - using System.Text.Json.Nodes; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - - internal class MdeJsonNodeEncryptionProcessor - { - private readonly JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; - - internal JsonNodeSqlSerializer Serializer { get; set; } = new JsonNodeSqlSerializer(); - - internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - - internal JsonSerializerOptions JsonSerializerOptions { get; set; } - - public MdeJsonNodeEncryptionProcessor() - { - this.JsonSerializerOptions = new JsonSerializerOptions(); - this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); - } - - public async Task EncryptAsync( - Stream input, - Encryptor encryptor, - EncryptionOptions encryptionOptions, - CancellationToken token) - { - JsonNode itemJObj = JsonNode.Parse(input); - - Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); - - await input.DisposeAsync(); - return result; - } - - public async Task EncryptAsync( - JsonNode document, - Encryptor encryptor, - EncryptionOptions encryptionOptions, - CancellationToken token) - { - List pathsEncrypted = new (); - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new (); - - JsonObject itemObj = document.AsObject(); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); - - bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; - -#if NET8_0_OR_GREATER - BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli - ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; -#endif - Dictionary compressedPaths = new (); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { -#if NET8_0_OR_GREATER - string propertyName = pathToEncrypt[1..]; -#else - string propertyName = pathToEncrypt.Substring(1); -#endif - if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) - { - continue; - } - - if (propertyValue == null || propertyValue.GetValueKind() == JsonValueKind.Null) - { - continue; - } - - byte[] processedBytes = null; - (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - - if (processedBytes == null) - { - continue; - } - -#if NET8_0_OR_GREATER - if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) - { - byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); - processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); - processedBytes = compressedBytes; - } -#endif - (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); - - itemObj[propertyName] = JsonValue.Create(new Memory(encryptedBytes, 0, encryptedBytesCount)); - pathsEncrypted.Add(pathToEncrypt); - } - -#if NET8_0_OR_GREATER - compressor?.Dispose(); -#endif - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: compressionEnabled ? 4 : 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted, - encryptionOptions.CompressionOptions.Algorithm, - compressedPaths); - - JsonNode propertiesNode = JsonSerializer.SerializeToNode(encryptionProperties); - - itemObj.Add(Constants.EncryptedInfo, propertiesNode); - - MemoryStream ms = new (); - Utf8JsonWriter writer = new (ms, this.jsonWriterOptions); - - JsonSerializer.Serialize(writer, document); - - ms.Position = 0; - return ms; - } - - internal async Task DecryptObjectAsync( - JsonNode document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new (); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); - - JsonObject itemObj = document.AsObject(); - -#if NET8_0_OR_GREATER - BrotliCompressor decompressor = null; - if (encryptionProperties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) - { - bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; - if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) - { - throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}"); - } - - if (containsCompressed) - { - decompressor = new (); - } - } -#endif - - foreach (string path in encryptionProperties.EncryptedPaths) - { - string propertyName = path[1..]; - - if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) - { - // malformed document, such record shouldn't be there at all - continue; - } - - // can we get to internal JsonNode buffers to avoid string allocation here? - string base64String = propertyValue.GetValue(); - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent((base64String.Length * sizeof(char) * 3 / 4) + 4); - if (!Convert.TryFromBase64Chars(base64String, cipherTextWithTypeMarker, out int cipherTextLength)) - { - continue; - } - - (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); - -#if NET8_0_OR_GREATER - if (decompressor != null) - { - if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) - { - byte[] buffer = arrayPoolManager.Rent(decompressedSize); - processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); - - bytes = buffer; - } - } -#endif - document[propertyName] = this.Serializer.Deserialize( - (TypeMarker)cipherTextWithTypeMarker[0], - bytes.AsSpan(0, processedBytes)); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - itemObj.Remove(Constants.EncryptedInfo); - return decryptionContext; - } - } -} - -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs deleted file mode 100644 index a8bc77f75b..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - - internal class JsonBytes - { - internal byte[] Bytes { get; private set; } - - internal int Offset { get; private set; } - - internal int Length { get; private set; } - - public JsonBytes(byte[] bytes, int offset, int length) - { - ArgumentNullException.ThrowIfNull(bytes); - ArgumentOutOfRangeException.ThrowIfNegative(offset); - ArgumentOutOfRangeException.ThrowIfNegative(length); - if (bytes.Length < offset + length) - { - throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); - } - - this.Bytes = bytes; - this.Offset = offset; - this.Length = length; - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs deleted file mode 100644 index d9048375c0..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - using System.Text.Json; - using System.Text.Json.Serialization; - - internal class JsonBytesConverter : JsonConverter - { - public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) - { - writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index c851750c53..b436edd547 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -36,7 +36,7 @@ public partial class EncryptionBenchmark public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, JsonProcessor.Stream)] + [Params(JsonProcessor.Newtonsoft, JsonProcessor.Stream)] #else [Params(JsonProcessor.Newtonsoft)] #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 9e05872546..46fde1350c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -68,7 +68,6 @@ public static void ClassInitialize(TestContext testContext) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) @@ -109,7 +108,6 @@ public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif public async Task DuplicatePathToEncrypt(JsonProcessor jsonProcessor) @@ -162,30 +160,6 @@ public async Task EncryptDecryptPropertyWithNullValue_VerifyByNewtonsoft(Encrypt decryptionContext); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [TestMethod] - [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task EncryptDecryptPropertyWithNullValue_VerifyBySystemText(EncryptionOptions encryptionOptions) - { - TestDoc testDoc = TestDoc.Create(); - testDoc.SensitiveStr = null; - - JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); - - (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( - encryptedDoc, - mockEncryptor.Object, - new CosmosDiagnosticsContext(), - CancellationToken.None); - - VerifyDecryptionSucceeded( - decryptedDoc, - testDoc, - TestDoc.PathsToEncrypt.Count, - decryptionContext); - } -#endif - [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] public async Task ValidateEncryptDecryptDocument_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) @@ -207,29 +181,6 @@ public async Task ValidateEncryptDecryptDocument_VerifyByNewtonsoft(EncryptionOp decryptionContext); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [TestMethod] - [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task ValidateEncryptDecryptDocument_VerifyBySystemText(EncryptionOptions encryptionOptions) - { - TestDoc testDoc = TestDoc.Create(); - - JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); - - (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( - encryptedDoc, - mockEncryptor.Object, - new CosmosDiagnosticsContext(), - CancellationToken.None); - - VerifyDecryptionSucceeded( - decryptedDoc, - testDoc, - TestDoc.PathsToEncrypt.Count, - decryptionContext); - } -#endif - [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] public async Task ValidateDecryptByNewtonsoftStream_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) @@ -319,7 +270,6 @@ public async Task ValidateDecryptBySystemTextStream_VerifyBySystemText(Encryptio [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif public async Task DecryptStreamWithoutEncryptedProperty(JsonProcessor processor) @@ -389,53 +339,6 @@ private static async Task VerifyEncryptionSucceededNewtonsoft(TestDoc t return encryptedDoc; } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static async Task VerifyEncryptionSucceededSystemText(TestDoc testDoc, EncryptionOptions encryptionOptions) - { - Stream encryptedStream = await EncryptionProcessor.EncryptAsync( - testDoc.ToStream(), - mockEncryptor.Object, - encryptionOptions, - new CosmosDiagnosticsContext(), - CancellationToken.None); - - JsonNode encryptedDoc = JsonNode.Parse(encryptedStream, documentOptions: new System.Text.Json.JsonDocumentOptions() { }); - - Assert.AreEqual(testDoc.Id, encryptedDoc["id"].GetValue()); - Assert.AreEqual(testDoc.PK, encryptedDoc[nameof(TestDoc.PK)].GetValue()); - Assert.AreEqual(testDoc.NonSensitive, encryptedDoc[nameof(TestDoc.NonSensitive)].GetValue()); - Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); - Assert.AreNotEqual(testDoc.SensitiveInt, encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); // not equal since value is encrypted - - JsonNode eiJProp = encryptedDoc[Constants.EncryptedInfo]; - Assert.IsNotNull(eiJProp); - Assert.IsNotNull(eiJProp.AsObject()); - EncryptionProperties encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(eiJProp); - - Assert.IsNotNull(encryptionProperties); - Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); - - int expectedVersion = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None ? 4 : 3; - Assert.AreEqual(expectedVersion, encryptionProperties.EncryptionFormatVersion); - Assert.IsNull(encryptionProperties.EncryptedData); - Assert.IsNotNull(encryptionProperties.EncryptedPaths); - - if (testDoc.SensitiveStr == null) - { - AssertNullableValueKind(null, encryptedDoc, nameof(TestDoc.SensitiveStr)); // since null value is not encrypted - Assert.AreEqual(TestDoc.PathsToEncrypt.Count - 1, encryptionProperties.EncryptedPaths.Count()); - } - else - { - Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); - Assert.AreNotEqual(testDoc.SensitiveStr, encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); // not equal since value is encrypted - Assert.AreEqual(TestDoc.PathsToEncrypt.Count, encryptionProperties.EncryptedPaths.Count()); - } - - return encryptedDoc; - } -#endif - private static void VerifyDecryptionSucceeded( JObject decryptedDoc, TestDoc expectedDoc, @@ -540,13 +443,10 @@ private static EncryptionOptions CreateEncryptionOptions(JsonProcessor processor public static IEnumerable EncryptionOptionsCombinations => new[] { new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, - new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, - new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, #endif }; @@ -559,7 +459,6 @@ public static IEnumerable EncryptionOptionsStreamTestCombinations { yield return new object[] { encryptionOptions[0], JsonProcessor.Newtonsoft }; #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - yield return new object[] { encryptionOptions[0], JsonProcessor.SystemTextJson }; yield return new object[] { encryptionOptions[0], JsonProcessor.Stream }; #endif } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs deleted file mode 100644 index fbef47e872..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.IO; - using System.Text.Json; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesConverterTests - { - [TestMethod] - public void Write_Results_IdenticalToNewtonsoft() - { - byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - - JsonBytes jsonBytes = new (bytes, 5, 5); - - using MemoryStream ms = new (); - using Utf8JsonWriter writer = new (ms); - - JsonBytesConverter jsonConverter = new (); - jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); - - writer.Flush(); - ms.Flush(); - ms.Position = 0; - StreamReader sr = new(ms); - string systemTextResult = sr.ReadToEnd(); - - byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); - string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); - - Assert.AreEqual(systemTextResult, newtonsoftResult); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs deleted file mode 100644 index a29f378ff2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesTests - { - [TestMethod] - public void Ctor_ThrowsForInvalidInputs() - { - Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); - } - - [TestMethod] - public void Properties_AreSetCorrectly() - { - byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - JsonBytes jsonBytes = new (bytes, 1, 5); - - Assert.AreEqual(1, jsonBytes.Offset); - Assert.AreEqual(5, jsonBytes.Length); - Assert.AreSame(bytes, jsonBytes.Bytes); - } - } -} -#endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs deleted file mode 100644 index e28bd409a1..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs +++ /dev/null @@ -1,171 +0,0 @@ -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Text.Json; - using System.Text.Json.Nodes; - using Microsoft.Azure.Cosmos.Encryption.Custom; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Newtonsoft.Json.Linq; - - [TestClass] - public class JsonNodeSqlSerializerTests - { - private static ArrayPoolManager _poolManager; - - [ClassInitialize] - public static void ClassInitialize(TestContext context) - { - _ = context; - _poolManager = new ArrayPoolManager(); - } - - [TestMethod] - [DynamicData(nameof(SerializationSamples))] - public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[] expectedBytes, int expectedLength) - { - JsonNodeSqlSerializer serializer = new(); - - (TypeMarker serializedType, byte[] serializedBytes, int serializedBytesCount) = serializer.Serialize(testNode, _poolManager); - - Assert.AreEqual((TypeMarker)expectedType, serializedType); - Assert.AreEqual(expectedLength, serializedBytesCount); - if (expectedLength == -1) - { - Assert.IsTrue(serializedBytes == null); - } - else - { - Assert.IsTrue(expectedBytes.SequenceEqual(serializedBytes.AsSpan(0, serializedBytesCount).ToArray())); - } - } - - [TestMethod] - [DynamicData(nameof(DeserializationSamples))] - public void Deserialize_SupportedValue(byte typeMarkerByte, byte[] serializedBytes, JsonNode expectedNode) - { - JsonNodeSqlSerializer serializer = new(); - TypeMarker typeMarker = (TypeMarker)typeMarkerByte; - JsonNode deserializedNode = serializer.Deserialize(typeMarker, serializedBytes); - - if ((expectedNode as JsonValue) != null) - { - AssertValueNodeEquality(expectedNode, deserializedNode); - return; - } - - if ((expectedNode as JsonArray) != null) - { - Assert.IsNotNull(deserializedNode as JsonArray); - - JsonArray expectedArray = expectedNode.AsArray(); - JsonArray deserializedArray = deserializedNode.AsArray(); - - Assert.AreEqual(expectedArray.Count, deserializedArray.Count); - - for (int i = 0; i < deserializedNode.AsArray().Count; i++) - { - AssertValueNodeEquality(expectedArray[i], deserializedArray[i]); - } - return; - } - - if ((expectedNode as JsonObject) != null) - { - Assert.IsNotNull(deserializedNode as JsonObject); - - JsonObject expectedObject = expectedNode.AsObject(); - JsonObject deserializedObject = deserializedNode.AsObject(); - - Assert.AreEqual(expectedObject.Count, deserializedObject.Count); - - foreach (KeyValuePair expected in expectedObject) - { - Assert.IsTrue(deserializedObject.ContainsKey(expected.Key)); - AssertValueNodeEquality(expected.Value, deserializedObject[expected.Key]); - } - return; - } - - Assert.Fail("Attempt to validate unsupported JsonNode type"); - } - - private static void AssertValueNodeEquality(JsonNode expectedNode, JsonNode actualNode) - { - JsonValue expectedValueNode = expectedNode.AsValue(); - JsonValue actualValueNode = actualNode.AsValue(); - - Assert.AreEqual(expectedValueNode.GetValueKind(), actualValueNode.GetValueKind()); - Assert.AreEqual(expectedValueNode.ToString(), actualValueNode.ToString()); - } - - public static IEnumerable DeserializationSamples - { - get - { - yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), JsonValue.Create(true) }; - yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), JsonValue.Create(false) }; - yield return new object[] { (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), JsonValue.Create(192) }; - yield return new object[] { (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), JsonValue.Create(192.5) }; - yield return new object[] { (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), JsonValue.Create(testString) }; - yield return new object[] { (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), JsonNode.Parse("[10,18,19]") }; - yield return new object[] { (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), JsonNode.Parse(testClass.ToJson()) }; - } - } - - public static IEnumerable SerializationSamples - { - get - { - List values = new() - { - new object[] {JsonValue.Create((string)null), (byte)TypeMarker.Null, null, -1 }, - new object[] {JsonValue.Create(true), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), 8}, - new object[] {JsonValue.Create(false), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), 8}, - new object[] {JsonValue.Create(192), (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), 8}, - new object[] {JsonValue.Create(192.5), (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), 8}, - new object[] {JsonValue.Create(testString), (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), 11}, - new object[] {JsonValue.Create(testArray), (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), 10}, - new object[] {JsonValue.Create(testClass), (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), 33} - }; - - return values; - } - } - - private static readonly string testString = "Hello world"; - private static readonly int[] testArray = new[] {10, 18, 19}; - private static readonly TestClass testClass = new() { SomeInt = 1, SomeString = "asdf" }; - - private class TestClass - { - public int SomeInt { get; set; } - public string SomeString { get; set; } - - public string ToJson() - { - return JsonSerializer.Serialize(this); - } - } - - private static byte[] GetNewtonsoftValueEquivalent(T value) - { - JObjectSqlSerializer serializer = new (); - JToken token = value switch - { - int[] => new JArray(value), - TestClass => JObject.FromObject(value), - _ => new JValue(value), - }; - (TypeMarker _, byte[] bytes, int lenght) = serializer.Serialize(token, _poolManager); - return bytes.AsSpan(0, lenght).ToArray(); - } - - } -} - -#endif \ No newline at end of file From 3be6843a30260e8fe980171f27cd9abab17681b0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 14:57:53 +0200 Subject: [PATCH 76/85] + rms --- .../RecyclableMemoryStreamMirror/README.md | 3 + .../RecyclableMemoryStream.cs | 1585 +++++++++++++++++ ...RecyclableMemoryStreamManager.EventArgs.cs | 456 +++++ .../RecyclableMemoryStreamManager.Events.cs | 288 +++ .../RecyclableMemoryStreamManager.cs | 989 ++++++++++ 5 files changed, 3321 insertions(+) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md new file mode 100644 index 0000000000..eed315c783 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md @@ -0,0 +1,3 @@ +# Microsoft.IO.RecyclableMemoryStream 3.0.1 + +Mirrored from https://github.com/microsoft/Microsoft.IO.RecyclableMemoryStream/tree/master/src diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs new file mode 100644 index 0000000000..92b003d71e --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs @@ -0,0 +1,1585 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// The MIT License (MIT) +// +// Copyright (c) 2015-2016 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Buffers; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + + /// + /// MemoryStream implementation that deals with pooling and managing memory streams which use potentially large + /// buffers. + /// + /// + /// This class works in tandem with the to supply MemoryStream-derived + /// objects to callers, while avoiding these specific problems: + /// + /// + /// LOH allocations + /// Since all large buffers are pooled, they will never incur a Gen2 GC + /// + /// + /// Memory wasteA standard memory stream doubles its size when it runs out of room. This + /// leads to continual memory growth as each stream approaches the maximum allowed size. + /// + /// + /// Memory copying + /// Each time a MemoryStream grows, all the bytes are copied into new buffers. + /// This implementation only copies the bytes when is called. + /// + /// + /// Memory fragmentation + /// By using homogeneous buffer sizes, it ensures that blocks of memory + /// can be easily reused. + /// + /// + /// + /// + /// The stream is implemented on top of a series of uniformly-sized blocks. As the stream's length grows, + /// additional blocks are retrieved from the memory manager. It is these blocks that are pooled, not the stream + /// object itself. + /// + /// + /// The biggest wrinkle in this implementation is when is called. This requires a single + /// contiguous buffer. If only a single block is in use, then that block is returned. If multiple blocks + /// are in use, we retrieve a larger buffer from the memory manager. These large buffers are also pooled, + /// split by size--they are multiples/exponentials of a chunk size (1 MB by default). + /// + /// + /// Once a large buffer is assigned to the stream the small blocks are NEVER again used for this stream. All operations take place on the + /// large buffer. The large buffer can be replaced by a larger buffer from the pool as needed. All blocks and large buffers + /// are maintained in the stream until the stream is disposed (unless AggressiveBufferReturn is enabled in the stream manager). + /// + /// + /// A further wrinkle is what happens when the stream is longer than the maximum allowable array length under .NET. This is allowed + /// when only blocks are in use, and only the Read/Write APIs are used. Once a stream grows to this size, any attempt to convert it + /// to a single buffer will result in an exception. Similarly, if a stream is already converted to use a single larger buffer, then + /// it cannot grow beyond the limits of the maximum allowable array size. + /// + /// + /// Any method that modifies the stream has the potential to throw an OutOfMemoryException, either because + /// the stream is beyond the limits set in RecyclableStreamManager, or it would result in a buffer larger than + /// the maximum array size supported by .NET. + /// + /// + public sealed class RecyclableMemoryStream : MemoryStream, IBufferWriter + { + /// + /// All of these blocks must be the same size. + /// + private readonly List blocks; + + private readonly Guid id; + + private readonly RecyclableMemoryStreamManager memoryManager; + + private readonly string tag; + + private readonly long creationTimestamp; + + /// + /// This list is used to store buffers once they're replaced by something larger. + /// This is for the cases where you have users of this class that may hold onto the buffers longer + /// than they should and you want to prevent race conditions which could corrupt the data. + /// + private List dirtyBuffers; + + private bool disposed; + + /// + /// This is only set by GetBuffer() if the necessary buffer is larger than a single block size, or on + /// construction if the caller immediately requests a single large buffer. + /// + /// If this field is non-null, it contains the concatenation of the bytes found in the individual + /// blocks. Once it is created, this (or a larger) largeBuffer will be used for the life of the stream. + /// + private byte[] largeBuffer; + + /// + /// Gets unique identifier for this stream across its entire lifetime. + /// + /// Object has been disposed. + internal Guid Id + { + get + { + this.CheckDisposed(); + return this.id; + } + } + + /// + /// Gets a temporary identifier for the current usage of this stream. + /// + /// Object has been disposed. + internal string Tag + { + get + { + this.CheckDisposed(); + return this.tag; + } + } + + /// + /// Gets the memory manager being used by this stream. + /// + /// Object has been disposed. + internal RecyclableMemoryStreamManager MemoryManager + { + get + { + this.CheckDisposed(); + return this.memoryManager; + } + } + + /// + /// Gets call stack of the constructor. It is only set if is true, + /// which should only be in debugging situations. + /// + internal string AllocationStack { get; } + + /// + /// Gets call stack of the call. It is only set if is true, + /// which should only be in debugging situations. + /// + internal string DisposeStack { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager) + : this(memoryManager, Guid.NewGuid(), null, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id) + : this(memoryManager, id, null, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A string identifying this stream for logging and debugging purposes. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag) + : this(memoryManager, Guid.NewGuid(), tag, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag) + : this(memoryManager, id, tag, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag, long requestedSize) + : this(memoryManager, Guid.NewGuid(), tag, requestedSize, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize) + : this(memoryManager, id, tag, requestedSize, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + /// An initial buffer to use. This buffer will be owned by the stream and returned to the memory manager upon Dispose. + internal RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize, byte[] initialLargeBuffer) + : base(Array.Empty()) + { + this.memoryManager = memoryManager; + this.id = id; + this.tag = tag; + this.blocks = new List(); + this.creationTimestamp = Stopwatch.GetTimestamp(); + + long actualRequestedSize = Math.Max(requestedSize, this.memoryManager.OptionsValue.BlockSize); + + if (initialLargeBuffer == null) + { + this.EnsureCapacity(actualRequestedSize); + } + else + { + this.largeBuffer = initialLargeBuffer; + } + + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + this.AllocationStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamCreated(this.id, this.tag, requestedSize, actualRequestedSize); + this.memoryManager.ReportUsageReport(); + } + + /// + /// Finalizes an instance of the class. + /// + /// Failing to dispose indicates a bug in the code using streams. Care should be taken to properly account for stream lifetime. + ~RecyclableMemoryStream() + { + this.Dispose(false); + } + + /// + /// Returns the memory used by this stream back to the pool. + /// + /// Whether we're disposing (true), or being called by the finalizer (false). + protected override void Dispose(bool disposing) + { + if (this.disposed) + { + string doubleDisposeStack = null; + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + doubleDisposeStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamDoubleDisposed(this.id, this.tag, this.AllocationStack, this.DisposeStack, doubleDisposeStack); + return; + } + + this.disposed = true; + TimeSpan lifetime = TimeSpan.FromTicks((Stopwatch.GetTimestamp() - this.creationTimestamp) * TimeSpan.TicksPerSecond / Stopwatch.Frequency); + + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + this.DisposeStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamDisposed(this.id, this.tag, lifetime, this.AllocationStack, this.DisposeStack); + + if (disposing) + { + GC.SuppressFinalize(this); + } + else + { + // We're being finalized. + this.memoryManager.ReportStreamFinalized(this.id, this.tag, this.AllocationStack); + + if (AppDomain.CurrentDomain.IsFinalizingForUnload()) + { + // If we're being finalized because of a shutdown, don't go any further. + // We have no idea what's already been cleaned up. Triggering events may cause + // a crash. + base.Dispose(disposing); + return; + } + } + + this.memoryManager.ReportStreamLength(this.length); + + if (this.largeBuffer != null) + { + this.memoryManager.ReturnLargeBuffer(this.largeBuffer, this.id, this.tag); + } + + if (this.dirtyBuffers != null) + { + foreach (byte[] buffer in this.dirtyBuffers) + { + this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); + } + } + + this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); + this.memoryManager.ReportUsageReport(); + this.blocks.Clear(); + + base.Dispose(disposing); + } + + /// + /// Equivalent to Dispose. + /// + public override void Close() + { + this.Dispose(true); + } + + /// + /// Gets or sets the capacity. + /// + /// + /// + /// Capacity is always in multiples of the memory manager's block size, unless + /// the large buffer is in use. Capacity never decreases during a stream's lifetime. + /// Explicitly setting the capacity to a lower value than the current value will have no effect. + /// This is because the buffers are all pooled by chunks and there's little reason to + /// allow stream truncation. + /// + /// + /// Writing past the current capacity will cause to automatically increase, until MaximumStreamCapacity is reached. + /// + /// + /// If the capacity is larger than int.MaxValue, then InvalidOperationException will be thrown. If you anticipate using + /// larger streams, use the property instead. + /// + /// + /// Object has been disposed. + /// Capacity is larger than int.MaxValue. + public override int Capacity + { + get + { + this.CheckDisposed(); + if (this.largeBuffer != null) + { + return this.largeBuffer.Length; + } + + long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; + if (size > int.MaxValue) + { + throw new InvalidOperationException($"{nameof(this.Capacity)} is larger than int.MaxValue. Use {nameof(this.Capacity64)} instead."); + } + + return (int)size; + } + + set => this.Capacity64 = value; + } + + /// + /// Gets or sets returns a 64-bit version of capacity, for streams larger than int.MaxValue in length. + /// + public long Capacity64 + { + get + { + this.CheckDisposed(); + if (this.largeBuffer != null) + { + return this.largeBuffer.Length; + } + + long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; + return size; + } + + set + { + this.CheckDisposed(); + this.EnsureCapacity(value); + } + } + + private long length; + + /// + /// Gets the number of bytes written to this stream. + /// + /// Object has been disposed. + /// If the buffer has already been converted to a large buffer, then the maximum length is limited by the maximum allowed array length in .NET. + public override long Length + { + get + { + this.CheckDisposed(); + return this.length; + } + } + + private long position; + + /// + /// Gets or sets the current position in the stream. + /// + /// Object has been disposed. + /// A negative value was passed. + /// Stream is in large-buffer mode, but an attempt was made to set the position past the maximum allowed array length. + /// If the buffer has already been converted to a large buffer, then the maximum length (and thus position) is limited by the maximum allowed array length in .NET. + public override long Position + { + get + { + this.CheckDisposed(); + return this.position; + } + + set + { + this.CheckDisposed(); + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); + } + + if (this.largeBuffer != null && value > RecyclableMemoryStreamManager.MaxArrayLength) + { + throw new InvalidOperationException($"Once the stream is converted to a single large buffer, position cannot be set past {RecyclableMemoryStreamManager.MaxArrayLength}."); + } + + this.position = value; + } + } + + /// + /// Gets a value indicating whether whether the stream can currently read. + /// + public override bool CanRead => !this.disposed; + + /// + /// Gets a value indicating whether whether the stream can currently seek. + /// + public override bool CanSeek => !this.disposed; + + /// + /// Gets a value indicating whether the steram can timeout. + /// + /// Always false + public override bool CanTimeout => false; + + /// + /// Gets a value indicating whether whether the stream can currently write. + /// + public override bool CanWrite => !this.disposed; + + /// + /// Returns a single buffer containing the contents of the stream. + /// The buffer may be longer than the stream length. + /// + /// A byte[] buffer. + /// IMPORTANT: Doing a after calling GetBuffer invalidates the buffer. The old buffer is held onto + /// until is called, but the next time GetBuffer is called, a new buffer from the pool will be required. + /// Object has been disposed. + /// stream is too large for a contiguous buffer. + public override byte[] GetBuffer() + { + this.CheckDisposed(); + + if (this.largeBuffer != null) + { + return this.largeBuffer; + } + + if (this.blocks.Count == 1) + { + return this.blocks[0]; + } + + // Buffer needs to reflect the capacity, not the length, because + // it's possible that people will manipulate the buffer directly + // and set the length afterward. Capacity sets the expectation + // for the size of the buffer. + byte[] newBuffer = this.memoryManager.GetLargeBuffer(this.Capacity64, this.id, this.tag); + + // InternalRead will check for existence of largeBuffer, so make sure we + // don't set it until after we've copied the data. + this.AssertLengthIsSmall(); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + this.largeBuffer = newBuffer; + + if (this.blocks.Count > 0 && this.memoryManager.OptionsValue.AggressiveBufferReturn) + { + this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); + this.blocks.Clear(); + } + + return this.largeBuffer; + } + +#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + /// + public override void CopyTo(Stream destination, int bufferSize) + { + this.WriteTo(destination, this.position, this.length - this.position); + } +#endif + + /// Asynchronously reads all the bytes from the current position in this stream and writes them to another stream. + /// The stream to which the contents of the current stream will be copied. + /// This parameter is ignored. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous copy operation. + /// + /// is . + /// Either the current stream or the destination stream is disposed. + /// The current stream does not support reading, or the destination stream does not support writing. + /// Similarly to MemoryStream's behavior, CopyToAsync will adjust the source stream's position by the number of bytes written to the destination stream, as a Read would do. + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(destination); +#else + if (destination == null) + { + throw new ArgumentNullException(nameof(destination)); + } +#endif + + this.CheckDisposed(); + + if (this.length == 0) + { + return Task.CompletedTask; + } + + long startPos = this.position; + long count = this.length - startPos; + this.position += count; + + if (destination is MemoryStream destinationRMS) + { + this.WriteTo(destinationRMS, startPos, count); + return Task.CompletedTask; + } + else + { + if (this.largeBuffer == null) + { + if (this.blocks.Count == 1) + { + this.AssertLengthIsSmall(); + return destination.WriteAsync(this.blocks[0], (int)startPos, (int)count, cancellationToken); + } + else + { + return CopyToAsyncImpl(destination, this.GetBlockAndRelativeOffset(startPos), count, this.blocks, cancellationToken); + } + } + else + { + this.AssertLengthIsSmall(); + return destination.WriteAsync(this.largeBuffer, (int)startPos, (int)count, cancellationToken); + } + } + + static async Task CopyToAsyncImpl(Stream destination, BlockAndOffset blockAndOffset, long count, List blocks, CancellationToken cancellationToken) + { + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + while (bytesRemaining > 0) + { + byte[] block = blocks[currentBlock]; + int amountToCopy = (int)Math.Min(block.Length - currentOffset, bytesRemaining); +#if NET8_0_OR_GREATER + await destination.WriteAsync(block.AsMemory(currentOffset, amountToCopy), cancellationToken); +#else + await destination.WriteAsync(block, currentOffset, amountToCopy, cancellationToken); +#endif + bytesRemaining -= amountToCopy; + ++currentBlock; + currentOffset = 0; + } + } + } + + private byte[] bufferWriterTempBuffer; + + /// + /// Notifies the stream that bytes were written to the buffer returned by or . + /// Seeks forward by bytes. + /// + /// + /// You must request a new buffer after calling Advance to continue writing more data and cannot write to a previously acquired buffer. + /// + /// How many bytes to advance. + /// Object has been disposed. + /// is negative. + /// is larger than the size of the previously requested buffer. + public void Advance(int count) + { + this.CheckDisposed(); + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} must be non-negative."); + } + + byte[] buffer = this.bufferWriterTempBuffer; + if (buffer != null) + { + if (count > buffer.Length) + { + throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {buffer.Length}."); + } + + this.Write(buffer, 0, count); + this.ReturnTempBuffer(buffer); + this.bufferWriterTempBuffer = null; + } + else + { + long bufferSize = this.largeBuffer == null + ? this.memoryManager.OptionsValue.BlockSize - this.GetBlockAndRelativeOffset(this.position).Offset + : this.largeBuffer.Length - this.position; + + if (count > bufferSize) + { + throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {bufferSize}."); + } + + this.position += count; + this.length = Math.Max(this.position, this.length); + } + } + + private void ReturnTempBuffer(byte[] buffer) + { + if (buffer.Length == this.memoryManager.OptionsValue.BlockSize) + { + this.memoryManager.ReturnBlock(buffer, this.id, this.tag); + } + else + { + this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); + } + } + + /// + /// + /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), + /// or setting Position after calling GetMemory() invalidates the memory. + /// + public Memory GetMemory(int sizeHint = 0) + { + return this.GetWritableBuffer(sizeHint); + } + + /// + /// + /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), + /// or setting Position after calling GetSpan() invalidates the span. + /// + public Span GetSpan(int sizeHint = 0) + { + return this.GetWritableBuffer(sizeHint); + } + + /// + /// When callers to GetSpan() or GetMemory() request a buffer that is larger than the remaining size of the current block + /// this method return a temp buffer. When Advance() is called, that temp buffer is then copied into the stream. + /// + private ArraySegment GetWritableBuffer(int sizeHint) + { + this.CheckDisposed(); + if (sizeHint < 0) + { + throw new ArgumentOutOfRangeException(nameof(sizeHint), $"{nameof(sizeHint)} must be non-negative."); + } + + int minimumBufferSize = Math.Max(sizeHint, 1); + + this.EnsureCapacity(this.position + minimumBufferSize); + if (this.bufferWriterTempBuffer != null) + { + this.ReturnTempBuffer(this.bufferWriterTempBuffer); + this.bufferWriterTempBuffer = null; + } + + if (this.largeBuffer != null) + { + return new ArraySegment(this.largeBuffer, (int)this.position, this.largeBuffer.Length - (int)this.position); + } + + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + int remainingBytesInBlock = this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset; + if (remainingBytesInBlock >= minimumBufferSize) + { + return new ArraySegment(this.blocks[blockAndOffset.Block], blockAndOffset.Offset, this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset); + } + + this.bufferWriterTempBuffer = minimumBufferSize > this.memoryManager.OptionsValue.BlockSize ? + this.memoryManager.GetLargeBuffer(minimumBufferSize, this.id, this.tag) : + this.memoryManager.GetBlock(); + + return new ArraySegment(this.bufferWriterTempBuffer); + } + + /// + /// Returns a sequence containing the contents of the stream. + /// + /// A ReadOnlySequence of bytes. + /// IMPORTANT: Calling Write(), GetMemory(), GetSpan(), Dispose(), or Close() after calling GetReadOnlySequence() invalidates the sequence. + /// Object has been disposed. + public ReadOnlySequence GetReadOnlySequence() + { + this.CheckDisposed(); + + if (this.largeBuffer != null) + { + this.AssertLengthIsSmall(); + return new ReadOnlySequence(this.largeBuffer, 0, (int)this.length); + } + + if (this.blocks.Count == 1) + { + this.AssertLengthIsSmall(); + return new ReadOnlySequence(this.blocks[0], 0, (int)this.length); + } + + BlockSegment first = new (this.blocks[0]); + BlockSegment last = first; + + for (int blockIdx = 1; last.RunningIndex + last.Memory.Length < this.length; blockIdx++) + { + last = last.Append(this.blocks[blockIdx]); + } + + return new ReadOnlySequence(first, 0, last, (int)(this.length - last.RunningIndex)); + } + + private sealed class BlockSegment : ReadOnlySequenceSegment + { + public BlockSegment(Memory memory) + { + this.Memory = memory; + } + + public BlockSegment Append(Memory memory) + { + BlockSegment nextSegment = new (memory) { RunningIndex = this.RunningIndex + this.Memory.Length }; + this.Next = nextSegment; + return nextSegment; + } + } + + /// + /// Returns an ArraySegment that wraps a single buffer containing the contents of the stream. + /// + /// An ArraySegment containing a reference to the underlying bytes. + /// Returns if a buffer can be returned; otherwise, . + public override bool TryGetBuffer(out ArraySegment buffer) + { + this.CheckDisposed(); + + try + { + if (this.length <= RecyclableMemoryStreamManager.MaxArrayLength) + { + buffer = new ArraySegment(this.GetBuffer(), 0, (int)this.Length); + return true; + } + } + catch (OutOfMemoryException) + { + } + +#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + buffer = ArraySegment.Empty; +#else + buffer = default; +#endif + return false; + } + + /// + /// Returns a new array with a copy of the buffer's contents. You should almost certainly be using combined with the to + /// access the bytes in this stream. Calling ToArray will destroy the benefits of pooled buffers, but it is included + /// for the sake of completeness. + /// + /// Object has been disposed. + /// The current object disallows ToArray calls. + /// The length of the stream is too long for a contiguous array. + /// Array of bytes +#pragma warning disable CS0809 + [Obsolete("This method has degraded performance vs. GetBuffer and should be avoided.")] + public override byte[] ToArray() + { + this.CheckDisposed(); + + string stack = this.memoryManager.OptionsValue.GenerateCallStacks ? Environment.StackTrace : null; + this.memoryManager.ReportStreamToArray(this.id, this.tag, stack, this.length); + + if (this.memoryManager.OptionsValue.ThrowExceptionOnToArray) + { + throw new NotSupportedException("The underlying RecyclableMemoryStreamManager is configured to not allow calls to ToArray."); + } + + byte[] newBuffer = new byte[this.Length]; + + Debug.Assert(this.length <= int.MaxValue); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + + return newBuffer; + } +#pragma warning restore CS0809 + + /// + /// Reads from the current position into the provided buffer. + /// + /// Destination buffer. + /// Offset into buffer at which to start placing the read bytes. + /// Number of bytes to read. + /// The number of bytes read. + /// buffer is null. + /// offset or count is less than 0. + /// offset subtracted from the buffer length is less than count. + /// Object has been disposed. + public override int Read(byte[] buffer, int offset, int count) + { + return this.SafeRead(buffer, offset, count, ref this.position); + } + + /// + /// Reads from the specified position into the provided buffer. + /// + /// Destination buffer. + /// Offset into buffer at which to start placing the read bytes. + /// Number of bytes to read. + /// Position in the stream to start reading from. + /// The number of bytes read. + /// is null. + /// or is less than 0. + /// subtracted from the buffer length is less than . + /// Object has been disposed. + public int SafeRead(byte[] buffer, int offset, int count, ref long streamPosition) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset), $"{nameof(offset)} cannot be negative."); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} cannot be negative."); + } + + if (offset + count > buffer.Length) + { + throw new ArgumentException($"{nameof(buffer)} length must be at least {nameof(offset)} + {nameof(count)}."); + } + + int amountRead = this.InternalRead(buffer, offset, count, streamPosition); + streamPosition += amountRead; + return amountRead; + } + + /// + /// Reads from the current position into the provided buffer. + /// + /// Destination buffer. + /// The number of bytes read. + /// Object has been disposed. +#if NETSTANDARD2_0 + public int Read(Span buffer) +#else + public override int Read(Span buffer) +#endif + { + return this.SafeRead(buffer, ref this.position); + } + + /// + /// Reads from the specified position into the provided buffer. + /// + /// Destination buffer. + /// Position in the stream to start reading from. + /// The number of bytes read. + /// Object has been disposed. + public int SafeRead(Span buffer, ref long streamPosition) + { + this.CheckDisposed(); + + int amountRead = this.InternalRead(buffer, streamPosition); + streamPosition += amountRead; + return amountRead; + } + + /// + /// Writes the buffer to the stream. + /// + /// Source buffer. + /// Start position. + /// Number of bytes to write. + /// buffer is null. + /// offset or count is negative. + /// buffer.Length - offset is not less than count. + /// Object has been disposed. + public override void Write(byte[] buffer, int offset, int count) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0) + { + throw new ArgumentOutOfRangeException( + nameof(offset), + offset, + $"{nameof(offset)} must be in the range of 0 - {nameof(buffer)}.{nameof(buffer.Length)}-1."); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), count, $"{nameof(count)} must be non-negative."); + } + + if (count + offset > buffer.Length) + { + throw new ArgumentException($"{nameof(count)} must be greater than {nameof(buffer)}.{nameof(buffer.Length)} - {nameof(offset)}."); + } + + int blockSize = this.memoryManager.OptionsValue.BlockSize; + long end = this.position + count; + + this.EnsureCapacity(end); + + if (this.largeBuffer == null) + { + int bytesRemaining = count; + int bytesWritten = 0; + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + + while (bytesRemaining > 0) + { + byte[] currentBlock = this.blocks[blockAndOffset.Block]; + int remainingInBlock = blockSize - blockAndOffset.Offset; + int amountToWriteInBlock = Math.Min(remainingInBlock, bytesRemaining); + + Buffer.BlockCopy( + buffer, + offset + bytesWritten, + currentBlock, + blockAndOffset.Offset, + amountToWriteInBlock); + + bytesRemaining -= amountToWriteInBlock; + bytesWritten += amountToWriteInBlock; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + } + else + { + Buffer.BlockCopy(buffer, offset, this.largeBuffer, (int)this.position, count); + } + + this.position = end; + this.length = Math.Max(this.position, this.length); + } + + /// + /// Writes the buffer to the stream. + /// + /// Source buffer. + /// buffer is null. + /// Object has been disposed. +#if NETSTANDARD2_0 + public void Write(ReadOnlySpan source) +#else + public override void Write(ReadOnlySpan source) +#endif + { + this.CheckDisposed(); + + int blockSize = this.memoryManager.OptionsValue.BlockSize; + long end = this.position + source.Length; + + this.EnsureCapacity(end); + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + + while (source.Length > 0) + { + byte[] currentBlock = this.blocks[blockAndOffset.Block]; + int remainingInBlock = blockSize - blockAndOffset.Offset; + int amountToWriteInBlock = Math.Min(remainingInBlock, source.Length); +#if NET8_0_OR_GREATER + source[..amountToWriteInBlock] + .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); + + source = source[amountToWriteInBlock..]; +#else + source.Slice(0, amountToWriteInBlock) + .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); + + source = source.Slice(amountToWriteInBlock); +#endif + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + } + else + { + source.CopyTo(this.largeBuffer.AsSpan((int)this.position)); + } + + this.position = end; + this.length = Math.Max(this.position, this.length); + } + + /// + /// Returns a useful string for debugging. This should not normally be called in actual production code. + /// + /// String with debug data. + public override string ToString() + { + if (!this.disposed) + { + return $"Id = {this.Id}, Tag = {this.Tag}, Length = {this.Length:N0} bytes"; + } + else + { + // Avoid properties because of the dispose check, but the fields themselves are not cleared. + return $"Disposed: Id = {this.id}, Tag = {this.tag}, Final Length: {this.length:N0} bytes"; + } + } + + /// + /// Writes a single byte to the current position in the stream. + /// + /// byte value to write. + /// Object has been disposed. + public override void WriteByte(byte value) + { + this.CheckDisposed(); + + long end = this.position + 1; + + if (this.largeBuffer == null) + { + int blockSize = this.memoryManager.OptionsValue.BlockSize; + + int block = (int)Math.DivRem(this.position, blockSize, out long index); + + if (block >= this.blocks.Count) + { + this.EnsureCapacity(end); + } + + this.blocks[block][index] = value; + } + else + { + if (this.position >= this.largeBuffer.Length) + { + this.EnsureCapacity(end); + } + + this.largeBuffer[this.position] = value; + } + + this.position = end; + + if (this.position > this.length) + { + this.length = this.position; + } + } + + /// + /// Reads a single byte from the current position in the stream. + /// + /// The byte at the current position, or -1 if the position is at the end of the stream. + /// Object has been disposed. + public override int ReadByte() + { + return this.SafeReadByte(ref this.position); + } + + /// + /// Reads a single byte from the specified position in the stream. + /// + /// The position in the stream to read from. + /// The byte at the current position, or -1 if the position is at the end of the stream. + /// Object has been disposed. + public int SafeReadByte(ref long streamPosition) + { + this.CheckDisposed(); + if (streamPosition == this.length) + { + return -1; + } + + byte value; + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(streamPosition); + value = this.blocks[blockAndOffset.Block][blockAndOffset.Offset]; + } + else + { + value = this.largeBuffer[streamPosition]; + } + + streamPosition++; + return value; + } + + /// + /// Sets the length of the stream. + /// + /// length of the stream + /// value is negative or larger than . + /// Object has been disposed. + public override void SetLength(long value) + { + this.CheckDisposed(); + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); + } + + this.EnsureCapacity(value); + + this.length = value; + if (this.position > value) + { + this.position = value; + } + } + + /// + /// Sets the position to the offset from the seek location. + /// + /// How many bytes to move. + /// From where. + /// The new position. + /// Object has been disposed. + /// is larger than . + /// Invalid seek origin. + /// Attempt to set negative position. + public override long Seek(long offset, SeekOrigin loc) + { + this.CheckDisposed(); + long newPosition = loc switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => offset + this.position, + SeekOrigin.End => offset + this.length, + _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), + }; + if (newPosition < 0) + { + throw new IOException("Seek before beginning."); + } + + this.position = newPosition; + return this.position; + } + + /// + /// Synchronously writes this stream's bytes to the argument stream. + /// + /// Destination stream. + /// Important: This does a synchronous write, which may not be desired in some situations. + /// is null. + /// Object has been disposed. + public override void WriteTo(Stream stream) + { + this.WriteTo(stream, 0, this.length); + } + + /// + /// Synchronously writes this stream's bytes, starting at offset, for count bytes, to the argument stream. + /// + /// Destination stream. + /// Offset in source. + /// Number of bytes to write. + /// is null. + /// + /// is less than 0, or + is beyond this 's length. + /// + /// Object has been disposed. + public void WriteTo(Stream stream, long offset, long count) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(stream); +#else + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } +#endif + + if (offset < 0 || offset + count > this.length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the {nameof(stream)}.", + innerException: null); + } + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[currentBlock]; + int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); + stream.Write(block, currentOffset, amountToCopy); + + bytesRemaining -= amountToCopy; + + ++currentBlock; + currentOffset = 0; + } + } + else + { + stream.Write(this.largeBuffer, (int)offset, (int)count); + } + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// The entire stream is written to the target array. + /// > is null. + /// Object has been disposed. + public void WriteTo(byte[] buffer) + { + this.WriteTo(buffer, 0, this.Length); + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// Offset in the source stream, from which to start. + /// Number of bytes to write. + /// > is null. + /// + /// is less than 0, or + is beyond this stream's length. + /// + /// Object has been disposed. + public void WriteTo(byte[] buffer, long offset, long count) + { + this.WriteTo(buffer, offset, count, 0); + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// Offset in the source stream, from which to start. + /// Number of bytes to write. + /// Offset in the target byte array to start writing + /// buffer is null + /// + /// is less than 0, or + is beyond this stream's length. + /// + /// + /// is less than 0, or + is beyond the target 's length. + /// + /// Object has been disposed. + public void WriteTo(byte[] buffer, long offset, long count, int targetOffset) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0 || offset + count > this.length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the stream.", + innerException: null); + } + + if (targetOffset < 0 || count + targetOffset > buffer.Length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(targetOffset)} must not be negative and {nameof(targetOffset)} + {nameof(count)} must not exceed the length of the target {nameof(buffer)}.", + innerException: null); + } + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + int currentTargetOffset = targetOffset; + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[currentBlock]; + int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); + Buffer.BlockCopy(block, currentOffset, buffer, currentTargetOffset, amountToCopy); + + bytesRemaining -= amountToCopy; + + ++currentBlock; + currentOffset = 0; + currentTargetOffset += amountToCopy; + } + } + else + { + this.AssertLengthIsSmall(); + Buffer.BlockCopy(this.largeBuffer, (int)offset, buffer, targetOffset, (int)count); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CheckDisposed() + { + if (this.disposed) + { + this.ThrowDisposedException(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void ThrowDisposedException() + { + throw new ObjectDisposedException($"The stream with Id {this.id} and Tag {this.tag} is disposed."); + } + + private int InternalRead(byte[] buffer, int offset, int count, long fromPosition) + { + if (this.length - fromPosition <= 0) + { + return 0; + } + + int amountToCopy; + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); + int bytesWritten = 0; + int bytesRemaining = (int)Math.Min(count, this.length - fromPosition); + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[blockAndOffset.Block]; + amountToCopy = Math.Min( + block.Length - blockAndOffset.Offset, + bytesRemaining); + Buffer.BlockCopy( + block, + blockAndOffset.Offset, + buffer, + bytesWritten + offset, + amountToCopy); + + bytesWritten += amountToCopy; + bytesRemaining -= amountToCopy; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + + return bytesWritten; + } + + amountToCopy = (int)Math.Min(count, this.length - fromPosition); + Buffer.BlockCopy(this.largeBuffer, (int)fromPosition, buffer, offset, amountToCopy); + return amountToCopy; + } + + private int InternalRead(Span buffer, long fromPosition) + { + if (this.length - fromPosition <= 0) + { + return 0; + } + + int amountToCopy; + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); + int bytesWritten = 0; + int bytesRemaining = (int)Math.Min(buffer.Length, this.length - fromPosition); + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[blockAndOffset.Block]; + amountToCopy = Math.Min( + block.Length - blockAndOffset.Offset, + bytesRemaining); +#if NET8_0_OR_GREATER + block.AsSpan(blockAndOffset.Offset, amountToCopy) + .CopyTo(buffer[bytesWritten..]); +#else + block.AsSpan(blockAndOffset.Offset, amountToCopy) + .CopyTo(buffer.Slice(bytesWritten)); +#endif + + bytesWritten += amountToCopy; + bytesRemaining -= amountToCopy; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + + return bytesWritten; + } + + amountToCopy = (int)Math.Min(buffer.Length, this.length - fromPosition); + this.largeBuffer.AsSpan((int)fromPosition, amountToCopy).CopyTo(buffer); + return amountToCopy; + } + + private struct BlockAndOffset + { + public int Block; + public int Offset; + + public BlockAndOffset(int block, int offset) + { + this.Block = block; + this.Offset = offset; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private BlockAndOffset GetBlockAndRelativeOffset(long offset) + { + int blockSize = this.memoryManager.OptionsValue.BlockSize; + int blockIndex = (int)Math.DivRem(offset, blockSize, out long offsetIndex); + return new BlockAndOffset(blockIndex, (int)offsetIndex); + } + + private void EnsureCapacity(long newCapacity) + { + if (newCapacity > this.memoryManager.OptionsValue.MaximumStreamCapacity && this.memoryManager.OptionsValue.MaximumStreamCapacity > 0) + { + this.memoryManager.ReportStreamOverCapacity(this.id, this.tag, newCapacity, this.AllocationStack); + + throw new OutOfMemoryException($"Requested capacity is too large: {newCapacity}. Limit is {this.memoryManager.OptionsValue.MaximumStreamCapacity}."); + } + + if (this.largeBuffer != null) + { + if (newCapacity > this.largeBuffer.Length) + { + byte[] newBuffer = this.memoryManager.GetLargeBuffer(newCapacity, this.id, this.tag); + Debug.Assert(this.length <= int.MaxValue); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + this.ReleaseLargeBuffer(); + this.largeBuffer = newBuffer; + } + } + else + { + // Let's save some re-allocation of the blocks list + long blocksRequired = (newCapacity / this.memoryManager.OptionsValue.BlockSize) + 1; + if (this.blocks.Capacity < blocksRequired) + { + this.blocks.Capacity = (int)blocksRequired; + } + + while (this.Capacity64 < newCapacity) + { + this.blocks.Add(this.memoryManager.GetBlock()); + } + } + } + + /// + /// Release the large buffer (either stores it for eventual release or returns it immediately). + /// + private void ReleaseLargeBuffer() + { + Debug.Assert(this.largeBuffer != null); + + if (this.memoryManager.OptionsValue.AggressiveBufferReturn) + { + this.memoryManager.ReturnLargeBuffer(this.largeBuffer!, this.id, this.tag); + } + else + { + // We most likely will only ever need space for one + this.dirtyBuffers ??= new List(1); + this.dirtyBuffers.Add(this.largeBuffer!); + } + + this.largeBuffer = null; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void AssertLengthIsSmall() + { + Debug.Assert(this.length <= int.MaxValue, "this.length was assumed to be <= Int32.MaxValue, but was larger."); + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs new file mode 100644 index 0000000000..7c450a418a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs @@ -0,0 +1,456 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + + /// + /// Wrapper for EventArgs + /// + public sealed partial class RecyclableMemoryStreamManager + { + /// + /// Arguments for the event. + /// + public sealed class StreamCreatedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets requested stream size. + /// + public long RequestedSize { get; } + + /// + /// Gets actual stream size. + /// + public long ActualSize { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// The requested stream size. + /// The actual stream size. + public StreamCreatedEventArgs(Guid guid, string tag, long requestedSize, long actualSize) + { + this.Id = guid; + this.Tag = tag; + this.RequestedSize = requestedSize; + this.ActualSize = actualSize; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamDisposedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Gets stack where stream was disposed. + /// + public string DisposeStack { get; } + + /// + /// Gets lifetime of the stream. + /// + public TimeSpan Lifetime { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Lifetime of the stream + /// Stack of original allocation. + /// Dispose stack. + public StreamDisposedEventArgs(Guid guid, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) + { + this.Id = guid; + this.Tag = tag; + this.Lifetime = lifetime; + this.AllocationStack = allocationStack; + this.DisposeStack = disposeStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamDoubleDisposedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Gets first dispose stack. + /// + public string DisposeStack1 { get; } + + /// + /// Gets second dispose stack. + /// + public string DisposeStack2 { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of original allocation. + /// First dispose stack. + /// Second dispose stack. + public StreamDoubleDisposedEventArgs(Guid guid, string tag, string allocationStack, string disposeStack1, string disposeStack2) + { + this.Id = guid; + this.Tag = tag; + this.AllocationStack = allocationStack; + this.DisposeStack1 = disposeStack1; + this.DisposeStack2 = disposeStack2; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamFinalizedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of original allocation. + public StreamFinalizedEventArgs(Guid guid, string tag, string allocationStack) + { + this.Id = guid; + this.Tag = tag; + this.AllocationStack = allocationStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamConvertedToArrayEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where ToArray was called. + /// + public string Stack { get; } + + /// + /// Gets length of stack. + /// + public long Length { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of ToArray call. + /// Length of stream. + public StreamConvertedToArrayEventArgs(Guid guid, string tag, string stack, long length) + { + this.Id = guid; + this.Tag = tag; + this.Stack = stack; + this.Length = length; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamOverCapacityEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets original allocation stack. + /// + public string AllocationStack { get; } + + /// + /// Gets requested capacity. + /// + public long RequestedCapacity { get; } + + /// + /// Gets maximum capacity. + /// + public long MaximumCapacity { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Requested capacity. + /// Maximum stream capacity of the manager. + /// Original allocation stack. + internal StreamOverCapacityEventArgs(Guid guid, string tag, long requestedCapacity, long maximumCapacity, string allocationStack) + { + this.Id = guid; + this.Tag = tag; + this.RequestedCapacity = requestedCapacity; + this.MaximumCapacity = maximumCapacity; + this.AllocationStack = allocationStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class BlockCreatedEventArgs : EventArgs + { + /// + /// Gets how many bytes are currently in use from the small pool. + /// + public long SmallPoolInUse { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Number of bytes currently in use from the small pool. + internal BlockCreatedEventArgs(long smallPoolInUse) + { + this.SmallPoolInUse = smallPoolInUse; + } + } + + /// + /// Arguments for the events. + /// + public sealed class LargeBufferCreatedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets a value indicating whether whether the buffer was satisfied from the pool or not. + /// + public bool Pooled { get; } + + /// + /// Gets required buffer size. + /// + public long RequiredSize { get; } + + /// + /// Gets how many bytes are in use from the large pool. + /// + public long LargePoolInUse { get; } + + /// + /// Gets if the buffer was not satisfied from the pool, and is turned on, then. + /// this will contain the call stack of the allocation request. + /// + public string CallStack { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Required size of the new buffer. + /// How many bytes from the large pool are currently in use. + /// Whether the buffer was satisfied from the pool or not. + /// Call stack of the allocation, if it wasn't pooled. + internal LargeBufferCreatedEventArgs(Guid guid, string tag, long requiredSize, long largePoolInUse, bool pooled, string callStack) + { + this.RequiredSize = requiredSize; + this.LargePoolInUse = largePoolInUse; + this.Pooled = pooled; + this.Id = guid; + this.Tag = tag; + this.CallStack = callStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class BufferDiscardedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets type of the buffer. + /// + public Events.MemoryStreamBufferType BufferType { get; } + + /// + /// Gets the reason this buffer was discarded. + /// + public Events.MemoryStreamDiscardReason Reason { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Type of buffer being discarded. + /// The reason for the discard. + internal BufferDiscardedEventArgs(Guid guid, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) + { + this.Id = guid; + this.Tag = tag; + this.BufferType = bufferType; + this.Reason = reason; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamLengthEventArgs : EventArgs + { + /// + /// Gets length of the stream. + /// + public long Length { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Length of the strength. + public StreamLengthEventArgs(long length) + { + this.Length = length; + } + } + + /// + /// Arguments for the event. + /// + public sealed class UsageReportEventArgs : EventArgs + { + /// + /// Gets bytes from the small pool currently in use. + /// + public long SmallPoolInUseBytes { get; } + + /// + /// Gets bytes from the small pool currently available. + /// + public long SmallPoolFreeBytes { get; } + + /// + /// Gets bytes from the large pool currently in use. + /// + public long LargePoolInUseBytes { get; } + + /// + /// Gets bytes from the large pool currently available. + /// + public long LargePoolFreeBytes { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Bytes from the small pool currently in use. + /// Bytes from the small pool currently available. + /// Bytes from the large pool currently in use. + /// Bytes from the large pool currently available. + public UsageReportEventArgs( + long smallPoolInUseBytes, + long smallPoolFreeBytes, + long largePoolInUseBytes, + long largePoolFreeBytes) + { + this.SmallPoolInUseBytes = smallPoolInUseBytes; + this.SmallPoolFreeBytes = smallPoolFreeBytes; + this.LargePoolInUseBytes = largePoolInUseBytes; + this.LargePoolFreeBytes = largePoolFreeBytes; + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs new file mode 100644 index 0000000000..81a24f9de8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs @@ -0,0 +1,288 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// --------------------------------------------------------------------- +// Copyright (c) 2015 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// --------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Diagnostics.Tracing; + + /// + /// Holder for Events + /// + public sealed partial class RecyclableMemoryStreamManager + { + /// + /// ETW events for RecyclableMemoryStream. + /// + [EventSource(Name = "Microsoft-IO-RecyclableMemoryStream", Guid = "{B80CD4E4-890E-468D-9CBA-90EB7C82DFC7}")] + public sealed class Events : EventSource + { + /// + /// Static log object, through which all events are written. + /// +#pragma warning disable SA1401 // Fields should be private +#pragma warning disable CA2211 // Non-constant fields should not be visible + public static Events Writer = new (); +#pragma warning restore CA2211 // Non-constant fields should not be visible +#pragma warning restore SA1401 // Fields should be private + + /// + /// Type of buffer. + /// + public enum MemoryStreamBufferType + { + /// + /// Small block buffer. + /// + Small, + + /// + /// Large pool buffer. + /// + Large, + } + + /// + /// The possible reasons for discarding a buffer. + /// + public enum MemoryStreamDiscardReason + { + /// + /// Buffer was too large to be re-pooled. + /// + TooLarge, + + /// + /// There are enough free bytes in the pool. + /// + EnoughFree, + } + + /// + /// Logged when a stream object is created. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Requested size of the stream. + /// Actual size given to the stream from the pool. + [Event(1, Level = EventLevel.Verbose, Version = 2)] + public void MemoryStreamCreated(Guid guid, string tag, long requestedSize, long actualSize) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(1, guid, tag ?? string.Empty, requestedSize, actualSize); + } + } + + /// + /// Logged when the stream is disposed. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Lifetime in milliseconds of the stream + /// Call stack of initial allocation. + /// Call stack of the dispose. + [Event(2, Level = EventLevel.Verbose, Version = 3)] + public void MemoryStreamDisposed(Guid guid, string tag, long lifetimeMs, string allocationStack, string disposeStack) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(2, guid, tag ?? string.Empty, lifetimeMs, allocationStack ?? string.Empty, disposeStack ?? string.Empty); + } + } + + /// + /// Logged when the stream is disposed for the second time. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of initial allocation. + /// Call stack of the first dispose. + /// Call stack of the second dispose. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(3, Level = EventLevel.Critical)] + public void MemoryStreamDoubleDispose( + Guid guid, + string tag, + string allocationStack, + string disposeStack1, + string disposeStack2) + { + if (this.IsEnabled()) + { + this.WriteEvent( + 3, + guid, + tag ?? string.Empty, + allocationStack ?? string.Empty, + disposeStack1 ?? string.Empty, + disposeStack2 ?? string.Empty); + } + } + + /// + /// Logged when a stream is finalized. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of initial allocation. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(4, Level = EventLevel.Error)] + public void MemoryStreamFinalized(Guid guid, string tag, string allocationStack) + { + if (this.IsEnabled()) + { + this.WriteEvent(4, guid, tag ?? string.Empty, allocationStack ?? string.Empty); + } + } + + /// + /// Logged when ToArray is called on a stream. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of the ToArray call. + /// Length of stream. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(5, Level = EventLevel.Verbose, Version = 2)] + public void MemoryStreamToArray(Guid guid, string tag, string stack, long size) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(5, guid, tag ?? string.Empty, stack ?? string.Empty, size); + } + } + + /// + /// Logged when the RecyclableMemoryStreamManager is initialized. + /// + /// Size of blocks, in bytes. + /// Size of the large buffer multiple, in bytes. + /// Maximum buffer size, in bytes. + [Event(6, Level = EventLevel.Informational)] + public void MemoryStreamManagerInitialized(int blockSize, int largeBufferMultiple, int maximumBufferSize) + { + if (this.IsEnabled()) + { + this.WriteEvent(6, blockSize, largeBufferMultiple, maximumBufferSize); + } + } + + /// + /// Logged when a new block is created. + /// + /// Number of bytes in the small pool currently in use. + [Event(7, Level = EventLevel.Warning, Version = 2)] + public void MemoryStreamNewBlockCreated(long smallPoolInUseBytes) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(7, smallPoolInUseBytes); + } + } + + /// + /// Logged when a new large buffer is created. + /// + /// Requested size. + /// Number of bytes in the large pool in use. + [Event(8, Level = EventLevel.Warning, Version = 3)] + public void MemoryStreamNewLargeBufferCreated(long requiredSize, long largePoolInUseBytes) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(8, requiredSize, largePoolInUseBytes); + } + } + + /// + /// Logged when a buffer is created that is too large to pool. + /// + /// Unique stream ID. + /// A temporary ID for this stream, usually indicates current usage. + /// Size requested by the caller. + /// Call stack of the requested stream. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(9, Level = EventLevel.Verbose, Version = 3)] + public void MemoryStreamNonPooledLargeBufferCreated(Guid guid, string tag, long requiredSize, string allocationStack) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(9, guid, tag ?? string.Empty, requiredSize, allocationStack ?? string.Empty); + } + } + + /// + /// Logged when a buffer is discarded (not put back in the pool, but given to GC to clean up). + /// + /// Unique stream ID. + /// A temporary ID for this stream, usually indicates current usage. + /// Type of the buffer being discarded. + /// Reason for the discard. + /// Number of free small pool blocks. + /// Bytes free in the small pool. + /// Bytes in use from the small pool. + /// Number of free large pool blocks. + /// Bytes free in the large pool. + /// Bytes in use from the large pool. + [Event(10, Level = EventLevel.Warning, Version = 2)] + public void MemoryStreamDiscardBuffer( + Guid guid, + string tag, + MemoryStreamBufferType bufferType, + MemoryStreamDiscardReason reason, + long smallBlocksFree, + long smallPoolBytesFree, + long smallPoolBytesInUse, + long largeBlocksFree, + long largePoolBytesFree, + long largePoolBytesInUse) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(10, guid, tag ?? string.Empty, bufferType, reason, smallBlocksFree, smallPoolBytesFree, smallPoolBytesInUse, largeBlocksFree, largePoolBytesFree, largePoolBytesInUse); + } + } + + /// + /// Logged when a stream grows beyond the maximum capacity. + /// + /// Unique stream ID + /// A temporary ID for this stream, usually indicates current usage. + /// The requested capacity. + /// Maximum capacity, as configured by RecyclableMemoryStreamManager. + /// Call stack for the capacity request. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(11, Level = EventLevel.Error, Version = 3)] + public void MemoryStreamOverCapacity(Guid guid, string tag, long requestedCapacity, long maxCapacity, string allocationStack) + { + if (this.IsEnabled()) + { + this.WriteEvent(11, guid, tag ?? string.Empty, requestedCapacity, maxCapacity, allocationStack ?? string.Empty); + } + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs new file mode 100644 index 0000000000..8348ccc809 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs @@ -0,0 +1,989 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// --------------------------------------------------------------------- +// Copyright (c) 2015-2016 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// --------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Runtime.CompilerServices; + using System.Threading; + using Microsoft.Extensions.Logging; + + /// + /// Manages pools of objects. + /// + /// + /// + /// There are two pools managed in here. The small pool contains same-sized buffers that are handed to streams + /// as they write more data. + /// + /// + /// For scenarios that need to call , the large pool contains buffers of various sizes, all + /// multiples/exponentials of (1 MB by default). They are split by size to avoid overly-wasteful buffer + /// usage. There should be far fewer 8 MB buffers than 1 MB buffers, for example. + /// + /// + public partial class RecyclableMemoryStreamManager + { + /// + /// Maximum length of a single array. + /// + /// See documentation at https://docs.microsoft.com/dotnet/api/system.array?view=netcore-3.1 + /// + internal const int MaxArrayLength = 0X7FFFFFC7; + + /// + /// Default block size, in bytes. + /// + public const int DefaultBlockSize = 128 * 1024; + + /// + /// Default large buffer multiple, in bytes. + /// + public const int DefaultLargeBufferMultiple = 1024 * 1024; + + /// + /// Default maximum buffer size, in bytes. + /// + public const int DefaultMaximumBufferSize = 128 * 1024 * 1024; + + // 0 to indicate unbounded + private const long DefaultMaxSmallPoolFreeBytes = 0L; + private const long DefaultMaxLargePoolFreeBytes = 0L; + + private readonly long[] largeBufferFreeSize; + private readonly long[] largeBufferInUseSize; + + private readonly ConcurrentStack[] largePools; + + private readonly ConcurrentStack smallPool; + +#pragma warning disable SA1401 // Fields should be private - performance reasons + internal readonly Options OptionsValue; +#pragma warning restore SA1401 // Fields should be private + + private long smallPoolFreeSize; + private long smallPoolInUseSize; + + /// + /// Gets settings for controlling the behavior of RecyclableMemoryStream + /// + public Options Settings => this.OptionsValue; + + /// + /// Gets number of bytes in small pool not currently in use. + /// + public long SmallPoolFreeSize => this.smallPoolFreeSize; + + /// + /// Gets number of bytes currently in use by stream from the small pool. + /// + public long SmallPoolInUseSize => this.smallPoolInUseSize; + + /// + /// Gets number of bytes in large pool not currently in use. + /// + public long LargePoolFreeSize + { + get + { + long sum = 0; + foreach (long freeSize in this.largeBufferFreeSize) + { + sum += freeSize; + } + + return sum; + } + } + + /// + /// Gets number of bytes currently in use by streams from the large pool. + /// + public long LargePoolInUseSize + { + get + { + long sum = 0; + foreach (long inUseSize in this.largeBufferInUseSize) + { + sum += inUseSize; + } + + return sum; + } + } + + /// + /// Gets how many blocks are in the small pool. + /// + public long SmallBlocksFree => this.smallPool.Count; + + /// + /// Gets how many buffers are in the large pool. + /// + public long LargeBuffersFree + { + get + { + long free = 0; + foreach (ConcurrentStack pool in this.largePools) + { + free += pool.Count; + } + + return free; + } + } + + /// + /// Parameters for customizing the behavior of + /// + public class Options + { + /// + /// Gets or sets the size of the pooled blocks. This must be greater than 0. + /// + /// The default size 131,072 (128KB) + public int BlockSize { get; set; } = DefaultBlockSize; + + /// + /// Gets or sets each large buffer will be a multiple exponential of this value + /// + /// The default value is 1,048,576 (1MB) + public int LargeBufferMultiple { get; set; } = DefaultLargeBufferMultiple; + + /// + /// Gets or sets buffer beyond this length are not pooled. + /// + /// The default value is 134,217,728 (128MB) + public int MaximumBufferSize { get; set; } = DefaultMaximumBufferSize; + + /// + /// Gets or sets maximum number of bytes to keep available in the small pool. + /// + /// + /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. + /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. + /// + public long MaximumSmallPoolFreeBytes { get; set; } + + /// + /// Gets or sets maximum number of bytes to keep available in the large pools. + /// + /// + /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. + /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. + /// + public long MaximumLargePoolFreeBytes { get; set; } + + /// + /// Gets or sets a value indicating whether whether to use the exponential allocation strategy (see documentation). + /// + /// The default value is false. + public bool UseExponentialLargeBuffer { get; set; } = false; + + /// + /// Gets or sets maximum stream capacity in bytes. Attempts to set a larger capacity will + /// result in an exception. + /// + /// The default value of 0 indicates no limit. + public long MaximumStreamCapacity { get; set; } = 0; + + /// + /// Gets or sets a value indicating whether whether to save call stacks for stream allocations. This can help in debugging. + /// It should NEVER be turned on generally in production. + /// + public bool GenerateCallStacks { get; set; } = false; + + /// + /// Gets or sets a value indicating whether whether dirty buffers can be immediately returned to the buffer pool. + /// + /// + /// + /// When is called on a stream and creates a single large buffer, if this setting is enabled, the other blocks will be returned + /// to the buffer pool immediately. + /// + /// + /// Note when enabling this setting that the user is responsible for ensuring that any buffer previously + /// retrieved from a stream which is subsequently modified is not used after modification (as it may no longer + /// be valid). + /// + /// + public bool AggressiveBufferReturn { get; set; } = false; + + /// + /// Gets or sets a value indicating whether causes an exception to be thrown if is ever called. + /// + /// Calling defeats the purpose of a pooled buffer. Use this property to discover code that is calling . If this is + /// set and is called, a NotSupportedException will be thrown. + public bool ThrowExceptionOnToArray { get; set; } = false; + + /// + /// Gets or sets a value indicating whether zero out buffers on allocation and before returning them to the pool. + /// + /// Setting this to true causes a performance hit and should only be set if one wants to avoid accidental data leaks. + public bool ZeroOutBuffer { get; set; } = false; + + /// + /// Creates a new object. + /// + public Options() + { + } + + /// + /// Creates a new object with the most common options. + /// + /// Size of the blocks in the small pool. + /// Size of the large buffer multiple + /// Maximum poolable buffer size. + /// Maximum bytes to hold in the small pool. + /// Maximum bytes to hold in each of the large pools. + public Options(int blockSize, int largeBufferMultiple, int maximumBufferSize, long maximumSmallPoolFreeBytes, long maximumLargePoolFreeBytes) + { + this.BlockSize = blockSize; + this.LargeBufferMultiple = largeBufferMultiple; + this.MaximumBufferSize = maximumBufferSize; + this.MaximumSmallPoolFreeBytes = maximumSmallPoolFreeBytes; + this.MaximumLargePoolFreeBytes = maximumLargePoolFreeBytes; + } + } + + /// + /// Initializes the memory manager with the default block/buffer specifications. This pool may have unbounded growth unless you modify . + /// + public RecyclableMemoryStreamManager() + : this(new Options()) + { + } + + /// + /// Initializes the memory manager with the given block requiredSize. + /// + /// Object specifying options for stream behavior. + /// + /// is not a positive number, + /// or is not a positive number, + /// or is less than options.BlockSize, + /// or is negative, + /// or is negative, + /// or is not a multiple/exponential of . + /// + public RecyclableMemoryStreamManager(Options options) + { + if (options.BlockSize <= 0) + { + throw new InvalidOperationException($"{nameof(options.BlockSize)} must be a positive number"); + } + + if (options.LargeBufferMultiple <= 0) + { + throw new InvalidOperationException($"{nameof(options.LargeBufferMultiple)} must be a positive number"); + } + + if (options.MaximumBufferSize < options.BlockSize) + { + throw new InvalidOperationException($"{nameof(options.MaximumBufferSize)} must be at least {nameof(options.BlockSize)}"); + } + + if (options.MaximumSmallPoolFreeBytes < 0) + { + throw new InvalidOperationException($"{nameof(options.MaximumSmallPoolFreeBytes)} must be non-negative"); + } + + if (options.MaximumLargePoolFreeBytes < 0) + { + throw new InvalidOperationException($"{nameof(options.MaximumLargePoolFreeBytes)} must be non-negative"); + } + + this.OptionsValue = options; + + if (!this.IsLargeBufferSize(options.MaximumBufferSize)) + { + throw new InvalidOperationException( + $"{nameof(options.MaximumBufferSize)} is not {(options.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {nameof(options.LargeBufferMultiple)}."); + } + + this.smallPool = new ConcurrentStack(); + int numLargePools = options.UseExponentialLargeBuffer + ? (int)Math.Log(options.MaximumBufferSize / options.LargeBufferMultiple, 2) + 1 + : options.MaximumBufferSize / options.LargeBufferMultiple; + + // +1 to store size of bytes in use that are too large to be pooled + this.largeBufferInUseSize = new long[numLargePools + 1]; + this.largeBufferFreeSize = new long[numLargePools]; + + this.largePools = new ConcurrentStack[numLargePools]; + + for (int i = 0; i < this.largePools.Length; ++i) + { + this.largePools[i] = new ConcurrentStack(); + } + + Events.Writer.MemoryStreamManagerInitialized(options.BlockSize, options.LargeBufferMultiple, options.MaximumBufferSize); + } + + /// + /// Removes and returns a single block from the pool. + /// + /// A byte[] array. + internal byte[] GetBlock() + { + Interlocked.Add(ref this.smallPoolInUseSize, this.OptionsValue.BlockSize); + + if (!this.smallPool.TryPop(out byte[] block)) + { + // We'll add this back to the pool when the stream is disposed + // (unless our free pool is too large) +#if NET6_0_OR_GREATER + block = this.OptionsValue.ZeroOutBuffer ? GC.AllocateArray(this.OptionsValue.BlockSize) : GC.AllocateUninitializedArray(this.OptionsValue.BlockSize); +#else + block = new byte[this.OptionsValue.BlockSize]; +#endif + this.ReportBlockCreated(); + } + else + { + Interlocked.Add(ref this.smallPoolFreeSize, -this.OptionsValue.BlockSize); + } + + return block; + } + + /// + /// Returns a buffer of arbitrary size from the large buffer pool. This buffer + /// will be at least the requiredSize and always be a multiple/exponential of largeBufferMultiple. + /// + /// The minimum length of the buffer. + /// Unique ID for the stream. + /// The tag of the stream returning this buffer, for logging if necessary. + /// A buffer of at least the required size. + /// Requested array size is larger than the maximum allowed. + internal byte[] GetLargeBuffer(long requiredSize, Guid id, string tag) + { + requiredSize = this.RoundToLargeBufferSize(requiredSize); + + if (requiredSize > MaxArrayLength) + { + throw new OutOfMemoryException($"Required buffer size exceeds maximum array length of {MaxArrayLength}."); + } + + int poolIndex = this.GetPoolIndex(requiredSize); + + bool createdNew = false; + bool pooled = true; + string callStack = null; + + byte[] buffer; + if (poolIndex < this.largePools.Length) + { + if (!this.largePools[poolIndex].TryPop(out buffer)) + { + buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); + createdNew = true; + } + else + { + Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], -buffer.Length); + } + } + else + { + // Buffer is too large to pool. They get a new buffer. + + // We still want to track the size, though, and we've reserved a slot + // in the end of the in-use array for non-pooled bytes in use. + poolIndex = this.largeBufferInUseSize.Length - 1; + + // We still want to round up to reduce heap fragmentation. + buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); + if (this.OptionsValue.GenerateCallStacks) + { + // Grab the stack -- we want to know who requires such large buffers + callStack = Environment.StackTrace; + } + + createdNew = true; + pooled = false; + } + + Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], buffer.Length); + if (createdNew) + { + this.ReportLargeBufferCreated(id, tag, requiredSize, pooled: pooled, callStack); + } + + return buffer; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static byte[] AllocateArray(long requiredSize, bool zeroInitializeArray) => +#if NET6_0_OR_GREATER + zeroInitializeArray ? GC.AllocateArray((int)requiredSize) : GC.AllocateUninitializedArray((int)requiredSize); +#else + new byte[requiredSize]; +#endif + } + + private long RoundToLargeBufferSize(long requiredSize) + { + if (this.OptionsValue.UseExponentialLargeBuffer) + { + long pow = 1; + while (this.OptionsValue.LargeBufferMultiple * pow < requiredSize) + { + pow <<= 1; + } + + return this.OptionsValue.LargeBufferMultiple * pow; + } + else + { + return (requiredSize + this.OptionsValue.LargeBufferMultiple - 1) / this.OptionsValue.LargeBufferMultiple * this.OptionsValue.LargeBufferMultiple; + } + } + + private bool IsLargeBufferSize(int value) + { + return value != 0 && (this.OptionsValue.UseExponentialLargeBuffer + ? value == this.RoundToLargeBufferSize(value) + : value % this.OptionsValue.LargeBufferMultiple == 0); + } + + private int GetPoolIndex(long length) + { + if (this.OptionsValue.UseExponentialLargeBuffer) + { + int index = 0; + while (this.OptionsValue.LargeBufferMultiple << index < length) + { + ++index; + } + + return index; + } + else + { + return (int)((length / this.OptionsValue.LargeBufferMultiple) - 1); + } + } + + /// + /// Returns the buffer to the large pool. + /// + /// The buffer to return. + /// Unique stream ID. + /// The tag of the stream returning this buffer, for logging if necessary. + /// is null. + /// buffer.Length is not a multiple/exponential of (it did not originate from this pool). + internal void ReturnLargeBuffer(byte[] buffer, Guid id, string tag) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (!this.IsLargeBufferSize(buffer.Length)) + { + throw new ArgumentException($"{nameof(buffer)} did not originate from this memory manager. The size is not " + + $"{(this.OptionsValue.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {this.OptionsValue.LargeBufferMultiple}."); + } + + this.ZeroOutMemoryIfEnabled(buffer); + int poolIndex = this.GetPoolIndex(buffer.Length); + if (poolIndex < this.largePools.Length) + { + if ((this.largePools[poolIndex].Count + 1) * buffer.Length <= this.OptionsValue.MaximumLargePoolFreeBytes || + this.OptionsValue.MaximumLargePoolFreeBytes == 0) + { + this.largePools[poolIndex].Push(buffer); + Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], buffer.Length); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.EnoughFree); + } + } + else + { + // This is a non-poolable buffer, but we still want to track its size for in-use + // analysis. We have space in the InUse array for this. + poolIndex = this.largeBufferInUseSize.Length - 1; + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.TooLarge); + } + + Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], -buffer.Length); + } + + /// + /// Returns the blocks to the pool. + /// + /// Collection of blocks to return to the pool. + /// Unique Stream ID. + /// The tag of the stream returning these blocks, for logging if necessary. + /// is null. + /// contains buffers that are the wrong size (or null) for this memory manager. + internal void ReturnBlocks(List blocks, Guid id, string tag) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(blocks); +#else + if (blocks == null) + { + throw new ArgumentNullException(nameof(blocks)); + } +#endif + + long bytesToReturn = blocks.Count * (long)this.OptionsValue.BlockSize; + Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); + + foreach (byte[] block in blocks) + { + if (block == null || block.Length != this.OptionsValue.BlockSize) + { + throw new ArgumentException($"{nameof(blocks)} contains buffers that are not {nameof(this.OptionsValue.BlockSize)} in length.", nameof(blocks)); + } + } + + foreach (byte[] block in blocks) + { + this.ZeroOutMemoryIfEnabled(block); + if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) + { + Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); + this.smallPool.Push(block); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); + break; + } + } + } + + /// + /// Returns a block to the pool. + /// + /// Block to return to the pool. + /// Unique Stream ID. + /// The tag of the stream returning this, for logging if necessary. + /// is null. + /// is the wrong size for this memory manager. + internal void ReturnBlock(byte[] block, Guid id, string tag) + { + int bytesToReturn = this.OptionsValue.BlockSize; + Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); + +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(block); +#else + if (block == null) + { + throw new ArgumentNullException(nameof(block)); + } +#endif + + if (block.Length != this.OptionsValue.BlockSize) + { + throw new ArgumentException($"{nameof(block)} is not not {nameof(this.OptionsValue.BlockSize)} in length."); + } + + this.ZeroOutMemoryIfEnabled(block); + if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) + { + Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); + this.smallPool.Push(block); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ZeroOutMemoryIfEnabled(byte[] buffer) + { + if (this.OptionsValue.ZeroOutBuffer) + { +#if NET6_0_OR_GREATER + Array.Clear(buffer); +#else + Array.Clear(buffer, 0, buffer.Length); +#endif + } + } + + internal void ReportBlockCreated() + { + Events.Writer.MemoryStreamNewBlockCreated(this.smallPoolInUseSize); + this.BlockCreated?.Invoke(this, new BlockCreatedEventArgs(this.smallPoolInUseSize)); + } + + internal void ReportLargeBufferCreated(Guid id, string tag, long requiredSize, bool pooled, string callStack) + { + if (pooled) + { + Events.Writer.MemoryStreamNewLargeBufferCreated(requiredSize, this.LargePoolInUseSize); + } + else + { + Events.Writer.MemoryStreamNonPooledLargeBufferCreated(id, tag, requiredSize, callStack); + } + + this.LargeBufferCreated?.Invoke(this, new LargeBufferCreatedEventArgs(id, tag, requiredSize, this.LargePoolInUseSize, pooled, callStack)); + } + + internal void ReportBufferDiscarded(Guid id, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) + { + Events.Writer.MemoryStreamDiscardBuffer( + id, + tag, + bufferType, + reason, + this.SmallBlocksFree, + this.smallPoolFreeSize, + this.smallPoolInUseSize, + this.LargeBuffersFree, + this.LargePoolFreeSize, + this.LargePoolInUseSize); + this.BufferDiscarded?.Invoke(this, new BufferDiscardedEventArgs(id, tag, bufferType, reason)); + } + + internal void ReportStreamCreated(Guid id, string tag, long requestedSize, long actualSize) + { + Events.Writer.MemoryStreamCreated(id, tag, requestedSize, actualSize); + this.StreamCreated?.Invoke(this, new StreamCreatedEventArgs(id, tag, requestedSize, actualSize)); + } + + internal void ReportStreamDisposed(Guid id, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) + { + Events.Writer.MemoryStreamDisposed(id, tag, (long)lifetime.TotalMilliseconds, allocationStack, disposeStack); + this.StreamDisposed?.Invoke(this, new StreamDisposedEventArgs(id, tag, lifetime, allocationStack, disposeStack)); + } + + internal void ReportStreamDoubleDisposed(Guid id, string tag, string allocationStack, string disposeStack1, string disposeStack2) + { + Events.Writer.MemoryStreamDoubleDispose(id, tag, allocationStack, disposeStack1, disposeStack2); + this.StreamDoubleDisposed?.Invoke(this, new StreamDoubleDisposedEventArgs(id, tag, allocationStack, disposeStack1, disposeStack2)); + } + + internal void ReportStreamFinalized(Guid id, string tag, string allocationStack) + { + Events.Writer.MemoryStreamFinalized(id, tag, allocationStack); + this.StreamFinalized?.Invoke(this, new StreamFinalizedEventArgs(id, tag, allocationStack)); + } + + internal void ReportStreamLength(long bytes) + { + this.StreamLength?.Invoke(this, new StreamLengthEventArgs(bytes)); + } + + internal void ReportStreamToArray(Guid id, string tag, string stack, long length) + { + Events.Writer.MemoryStreamToArray(id, tag, stack, length); + this.StreamConvertedToArray?.Invoke(this, new StreamConvertedToArrayEventArgs(id, tag, stack, length)); + } + + internal void ReportStreamOverCapacity(Guid id, string tag, long requestedCapacity, string allocationStack) + { + Events.Writer.MemoryStreamOverCapacity(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack); + this.StreamOverCapacity?.Invoke(this, new StreamOverCapacityEventArgs(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack)); + } + + internal void ReportUsageReport() + { + this.UsageReport?.Invoke(this, new UsageReportEventArgs(this.smallPoolInUseSize, this.smallPoolFreeSize, this.LargePoolInUseSize, this.LargePoolFreeSize)); + } + + /// + /// Retrieve a new object with no tag and a default initial capacity. + /// + /// A . + public RecyclableMemoryStream GetStream() + { + return new RecyclableMemoryStream(this); + } + + /// + /// Retrieve a new object with no tag and a default initial capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id) + { + return new RecyclableMemoryStream(this, id); + } + + /// + /// Retrieve a new object with the given tag and a default initial capacity. + /// + /// A tag which can be used to track the source of the stream. + /// A . + public RecyclableMemoryStream GetStream(string tag) + { + return new RecyclableMemoryStream(this, tag); + } + + /// + /// Retrieve a new object with the given tag and a default initial capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag) + { + return new RecyclableMemoryStream(this, id, tag); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity. + /// + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// A . + public RecyclableMemoryStream GetStream(string tag, long requiredSize) + { + return new RecyclableMemoryStream(this, tag, requiredSize); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize) + { + return new RecyclableMemoryStream(this, id, tag, requiredSize); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity, possibly using + /// a single contiguous underlying buffer. + /// + /// Retrieving a which provides a single contiguous buffer can be useful in situations + /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying + /// buffers to a single large one. This is most helpful when you know that you will always call + /// on the underlying stream. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// Whether to attempt to use a single contiguous buffer. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize, bool asContiguousBuffer) + { + if (!asContiguousBuffer || requiredSize <= this.OptionsValue.BlockSize) + { + return this.GetStream(id, tag, requiredSize); + } + + return new RecyclableMemoryStream(this, id, tag, requiredSize, this.GetLargeBuffer(requiredSize, id, tag)); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity, possibly using + /// a single contiguous underlying buffer. + /// + /// Retrieving a which provides a single contiguous buffer can be useful in situations + /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying + /// buffers to a single large one. This is most helpful when you know that you will always call + /// on the underlying stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// Whether to attempt to use a single contiguous buffer. + /// A . + public RecyclableMemoryStream GetStream(string tag, long requiredSize, bool asContiguousBuffer) + { + return this.GetStream(Guid.NewGuid(), tag, requiredSize, asContiguousBuffer); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// The offset from the start of the buffer to copy from. + /// The number of bytes to copy from the buffer. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, byte[] buffer, int offset, int count) + { + RecyclableMemoryStream stream = null; + try + { + stream = new RecyclableMemoryStream(this, id, tag, count); + stream.Write(buffer, offset, count); + stream.Position = 0; + return stream; + } + catch + { + stream?.Dispose(); + throw; + } + } + + /// + /// Retrieve a new object with the contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(byte[] buffer) + { + return this.GetStream(null, buffer, 0, buffer.Length); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// The offset from the start of the buffer to copy from. + /// The number of bytes to copy from the buffer. + /// A . + public RecyclableMemoryStream GetStream(string tag, byte[] buffer, int offset, int count) + { + return this.GetStream(Guid.NewGuid(), tag, buffer, offset, count); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, ReadOnlySpan buffer) + { + RecyclableMemoryStream stream = null; + try + { + stream = new RecyclableMemoryStream(this, id, tag, buffer.Length); + stream.Write(buffer); + stream.Position = 0; + return stream; + } + catch + { + stream?.Dispose(); + throw; + } + } + + /// + /// Retrieve a new object with the contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(ReadOnlySpan buffer) + { + return this.GetStream(null, buffer); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(string tag, ReadOnlySpan buffer) + { + return this.GetStream(Guid.NewGuid(), tag, buffer); + } + + /// + /// Triggered when a new block is created. + /// + public event EventHandler BlockCreated; + + /// + /// Triggered when a new large buffer is created. + /// + public event EventHandler LargeBufferCreated; + + /// + /// Triggered when a new stream is created. + /// + public event EventHandler StreamCreated; + + /// + /// Triggered when a stream is disposed. + /// + public event EventHandler StreamDisposed; + + /// + /// Triggered when a stream is disposed of twice (an error). + /// + public event EventHandler StreamDoubleDisposed; + + /// + /// Triggered when a stream is finalized. + /// + public event EventHandler StreamFinalized; + + /// + /// Triggered when a stream is disposed to report the stream's length. + /// + public event EventHandler StreamLength; + + /// + /// Triggered when a user converts a stream to array. + /// + public event EventHandler StreamConvertedToArray; + + /// + /// Triggered when a stream is requested to expand beyond the maximum length specified by the responsible RecyclableMemoryStreamManager. + /// + public event EventHandler StreamOverCapacity; + + /// + /// Triggered when a buffer of either type is discarded, along with the reason for the discard. + /// + public event EventHandler BufferDiscarded; + + /// + /// Periodically triggered to report usage statistics. + /// + public event EventHandler UsageReport; + } +} \ No newline at end of file From de555a83f2af260ca200be8a44e3aff9f096addb Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 16:03:37 +0200 Subject: [PATCH 77/85] - drop rentArrayBufferWriter --- .../src/MemoryStreamManager.cs | 41 ++++ .../src/RentArrayBufferWriter.cs | 211 ------------------ .../src/StreamManager.cs | 31 +++ .../StreamProcessor.Encryptor.cs | 24 +- .../Readme.md | 124 +++++----- 5 files changed, 137 insertions(+), 294 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs new file mode 100644 index 0000000000..0edbdf52b2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; + + /// + /// Memory Stream manager + /// + /// Placeholder + internal class MemoryStreamManager : StreamManager + { + private readonly RecyclableMemoryStreamManager streamManager = new (); + + /// + /// Create stream + /// + /// Desired minimal capacity of stream. + /// Instance of stream. + public override Stream CreateStream(int hintSize = 0) + { + return new RecyclableMemoryStream(this.streamManager, null, hintSize); + } + + /// + /// Dispose of used Stream (return to pool) + /// + /// Stream to dispose. + /// ValueTask.CompletedTask + public async override ValueTask ReturnStreamAsync(Stream stream) + { + await stream.DisposeAsync(); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs deleted file mode 100644 index 5b37ded4fb..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs +++ /dev/null @@ -1,211 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Custom; - -#if NET8_0_OR_GREATER - -using System; -using System.Buffers; -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -/// -/// https://gist.github.com/ahsonkhan/c76a1cc4dc7107537c3fdc0079a68b35 -/// Standard ArrayBufferWriter is not using pooled memory -/// -internal class RentArrayBufferWriter : IBufferWriter, IDisposable -{ - private const int MinimumBufferSize = 256; - - private byte[] rentedBuffer; - private int written; - private long committed; - - public RentArrayBufferWriter(int initialCapacity = MinimumBufferSize) - { - if (initialCapacity <= 0) - { - throw new ArgumentException(null, nameof(initialCapacity)); - } - - this.rentedBuffer = ArrayPool.Shared.Rent(initialCapacity); - this.written = 0; - this.committed = 0; - } - - public (byte[], int) WrittenBuffer - { - get - { - this.CheckIfDisposed(); - - return (this.rentedBuffer, this.written); - } - } - - public Memory WrittenMemory - { - get - { - this.CheckIfDisposed(); - - return this.rentedBuffer.AsMemory(0, this.written); - } - } - - public Span WrittenSpan - { - get - { - this.CheckIfDisposed(); - - return this.rentedBuffer.AsSpan(0, this.written); - } - } - - public int BytesWritten - { - get - { - this.CheckIfDisposed(); - - return this.written; - } - } - - public long BytesCommitted - { - get - { - this.CheckIfDisposed(); - - return this.committed; - } - } - - public void Clear() - { - this.CheckIfDisposed(); - - this.ClearHelper(); - } - - private void ClearHelper() - { - this.rentedBuffer.AsSpan(0, this.written).Clear(); - this.written = 0; - } - - public async Task CopyToAsync(Stream stream, CancellationToken cancellationToken = default) - { - this.CheckIfDisposed(); - - ArgumentNullException.ThrowIfNull(stream); - - await stream.WriteAsync(new Memory(this.rentedBuffer, 0, this.written), cancellationToken).ConfigureAwait(false); - this.committed += this.written; - - this.ClearHelper(); - } - - public void CopyTo(Stream stream) - { - this.CheckIfDisposed(); - - ArgumentNullException.ThrowIfNull(stream); - - stream.Write(this.rentedBuffer, 0, this.written); - this.committed += this.written; - - this.ClearHelper(); - } - - public void Advance(int count) - { - this.CheckIfDisposed(); - - ArgumentOutOfRangeException.ThrowIfLessThan(count, 0); - - if (this.written > this.rentedBuffer.Length - count) - { - throw new InvalidOperationException("Cannot advance past the end of the buffer."); - } - - this.written += count; - } - - // Returns the rented buffer back to the pool - public void Dispose() - { - if (this.rentedBuffer == null) - { - return; - } - - ArrayPool.Shared.Return(this.rentedBuffer, clearArray: true); - this.rentedBuffer = null; - this.written = 0; - } - - private void CheckIfDisposed() - { - ObjectDisposedException.ThrowIf(this.rentedBuffer == null, this); - } - - public Memory GetMemory(int sizeHint = 0) - { - this.CheckIfDisposed(); - - ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); - - this.CheckAndResizeBuffer(sizeHint); - return this.rentedBuffer.AsMemory(this.written); - } - - public Span GetSpan(int sizeHint = 0) - { - this.CheckIfDisposed(); - - ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); - - this.CheckAndResizeBuffer(sizeHint); - return this.rentedBuffer.AsSpan(this.written); - } - - private void CheckAndResizeBuffer(int sizeHint) - { - Debug.Assert(sizeHint >= 0); - - if (sizeHint == 0) - { - sizeHint = MinimumBufferSize; - } - - int availableSpace = this.rentedBuffer.Length - this.written; - - if (sizeHint > availableSpace) - { - int growBy = sizeHint > this.rentedBuffer.Length ? sizeHint : this.rentedBuffer.Length; - - int newSize = checked(this.rentedBuffer.Length + growBy); - - byte[] oldBuffer = this.rentedBuffer; - - this.rentedBuffer = ArrayPool.Shared.Rent(newSize); - - Debug.Assert(oldBuffer.Length >= this.written); - Debug.Assert(this.rentedBuffer.Length >= this.written); - - oldBuffer.AsSpan(0, this.written).CopyTo(this.rentedBuffer); - ArrayPool.Shared.Return(oldBuffer, clearArray: true); - } - - Debug.Assert(this.rentedBuffer.Length - this.written > 0); - Debug.Assert(this.rentedBuffer.Length - this.written >= sizeHint); - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs new file mode 100644 index 0000000000..ed7831b783 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO; + using System.Threading.Tasks; + + /// + /// Abstraction for pooling streams + /// + public abstract class StreamManager + { + /// + /// Create stream + /// + /// Desired minimal size of stream. + /// Instance of stream. + public abstract Stream CreateStream(int hintSize = 0); + + /// + /// Dispose of used Stream (return to pool) + /// + /// Stream to dispose. + /// ValueTask.CompletedTask + public abstract ValueTask ReturnStreamAsync(Stream stream); + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index a3980ae56a..905d640dd4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -6,17 +6,21 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; + using System.Buffers; using System.Collections.Generic; using System.IO; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; internal partial class StreamProcessor { private readonly byte[] encryptionPropertiesNameBytes = Encoding.UTF8.GetBytes(Constants.EncryptedInfo); + private readonly RecyclableMemoryStreamManager streamManager = new (); + internal async Task EncryptStreamAsync( Stream inputStream, Stream outputStream, @@ -51,7 +55,7 @@ internal async Task EncryptStreamAsync( Utf8JsonWriter encryptionPayloadWriter = null; string encryptPropertyName = null; - RentArrayBufferWriter bufferWriter = null; + RecyclableMemoryStream bufferWriter = null; while (!isFinalBlock) { @@ -112,8 +116,8 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) case JsonTokenType.StartObject: if (encryptPropertyName != null && encryptionPayloadWriter == null) { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + bufferWriter = new RecyclableMemoryStream(this.streamManager); + encryptionPayloadWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); encryptionPayloadWriter.WriteStartObject(); } else @@ -132,16 +136,17 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) { currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + byte[] bytes = bufferWriter.GetBuffer(); + int length = (int)bufferWriter.Length; ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Object); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; #pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct encryptionPayloadWriter.Dispose(); + bufferWriter.Dispose(); #pragma warning restore VSTHRD103 // Call async methods when in an async method encryptionPayloadWriter = null; - bufferWriter.Dispose(); bufferWriter = null; } @@ -149,8 +154,8 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) case JsonTokenType.StartArray: if (encryptPropertyName != null && encryptionPayloadWriter == null) { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + bufferWriter = new RecyclableMemoryStream(this.streamManager); + encryptionPayloadWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); encryptionPayloadWriter.WriteStartArray(); } else @@ -164,16 +169,17 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) { currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + byte[] bytes = bufferWriter.GetBuffer(); + int length = (int)bufferWriter.Length; ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Array); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; #pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct encryptionPayloadWriter.Dispose(); + bufferWriter.Dispose(); #pragma warning restore VSTHRD103 // Call async methods when in an async method encryptionPayloadWriter = null; - bufferWriter.Dispose(); bufferWriter = null; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 22cb620269..31a77807b2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,77 +9,53 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.53 μs** | **0.511 μs** | **0.733 μs** | **22.29 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | -| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.224 μs | 0.322 μs | 26.23 μs | 0.1526 | 0.0305 | - | 41440 B | -| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.33 μs** | **0.137 μs** | **0.201 μs** | **14.32 μs** | **0.0916** | **0.0153** | **-** | **22904 B** | -| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | SystemTextJson | 14.54 μs | 0.124 μs | 0.186 μs | 14.52 μs | 0.0610 | 0.0305 | - | 21448 B | -| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **Stream** | **12.85 μs** | **0.095 μs** | **0.143 μs** | **12.84 μs** | **0.0610** | **0.0153** | **-** | **17528 B** | -| EncryptToProvidedStream | 1 | None | Stream | 13.00 μs | 0.096 μs | 0.141 μs | 12.98 μs | 0.0458 | 0.0153 | - | 11392 B | -| Decrypt | 1 | None | Stream | 13.01 μs | 0.152 μs | 0.228 μs | 13.05 μs | 0.0458 | 0.0153 | - | 12672 B | -| DecryptToProvidedStream | 1 | None | Stream | 13.48 μs | 0.132 μs | 0.197 μs | 13.45 μs | 0.0458 | 0.0153 | - | 11504 B | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **27.94 μs** | **0.226 μs** | **0.338 μs** | **27.96 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | -| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | Newtonsoft | 33.49 μs | 0.910 μs | 1.335 μs | 33.99 μs | 0.1221 | - | - | 41064 B | -| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **20.92 μs** | **0.136 μs** | **0.199 μs** | **20.95 μs** | **0.0610** | **-** | **-** | **21952 B** | -| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | SystemTextJson | 20.53 μs | 0.136 μs | 0.200 μs | 20.52 μs | 0.0610 | 0.0305 | - | 20488 B | -| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **Stream** | **21.15 μs** | **1.037 μs** | **1.521 μs** | **20.52 μs** | **0.0610** | **0.0305** | **-** | **16584 B** | -| EncryptToProvidedStream | 1 | Brotli | Stream | 20.57 μs | 0.213 μs | 0.292 μs | 20.57 μs | 0.0305 | - | - | 11672 B | -| Decrypt | 1 | Brotli | Stream | 21.14 μs | 2.212 μs | 3.311 μs | 19.46 μs | 0.0305 | - | - | 13216 B | -| DecryptToProvidedStream | 1 | Brotli | Stream | 19.60 μs | 0.439 μs | 0.600 μs | 19.52 μs | 0.0305 | - | - | 12048 B | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **84.82 μs** | **3.002 μs** | **4.208 μs** | **83.32 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | -| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | Newtonsoft | 112.98 μs | 15.294 μs | 21.934 μs | 100.38 μs | 0.6104 | 0.1221 | - | 157425 B | -| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.85 μs** | **0.868 μs** | **1.272 μs** | **41.40 μs** | **0.4272** | **0.0610** | **-** | **105345 B** | -| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | SystemTextJson | 41.79 μs | 0.501 μs | 0.718 μs | 41.64 μs | 0.3662 | 0.0610 | - | 96464 B | -| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **Stream** | **39.63 μs** | **0.658 μs** | **0.923 μs** | **39.41 μs** | **0.3052** | **0.0610** | **-** | **82928 B** | -| EncryptToProvidedStream | 10 | None | Stream | 36.59 μs | 0.272 μs | 0.399 μs | 36.57 μs | 0.1221 | - | - | 37048 B | -| Decrypt | 10 | None | Stream | 28.64 μs | 0.378 μs | 0.517 μs | 28.59 μs | 0.1221 | 0.0305 | - | 29520 B | -| DecryptToProvidedStream | 10 | None | Stream | 27.61 μs | 0.237 μs | 0.332 μs | 27.64 μs | 0.0610 | 0.0305 | - | 18416 B | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **115.28 μs** | **3.336 μs** | **4.677 μs** | **113.71 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | -| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | Newtonsoft | 118.98 μs | 1.530 μs | 2.195 μs | 118.76 μs | 0.4883 | - | - | 144849 B | -| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **71.40 μs** | **0.799 μs** | **1.145 μs** | **71.23 μs** | **0.2441** | **-** | **-** | **86217 B** | -| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | SystemTextJson | 73.37 μs | 7.283 μs | 10.676 μs | 67.12 μs | 0.2441 | - | - | 82201 B | -| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **Stream** | **90.10 μs** | **3.136 μs** | **4.693 μs** | **88.92 μs** | **0.2441** | **-** | **-** | **63809 B** | -| EncryptToProvidedStream | 10 | Brotli | Stream | 97.27 μs | 1.885 μs | 2.703 μs | 97.35 μs | 0.1221 | - | - | 32465 B | -| Decrypt | 10 | Brotli | Stream | 58.48 μs | 0.956 μs | 1.372 μs | 58.59 μs | 0.1221 | 0.0610 | - | 30064 B | -| DecryptToProvidedStream | 10 | Brotli | Stream | 59.12 μs | 1.160 μs | 1.664 μs | 59.14 μs | 0.0610 | - | - | 18960 B | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,199.74 μs** | **42.805 μs** | **64.069 μs** | **1,206.48 μs** | **23.4375** | **21.4844** | **21.4844** | **1677978 B** | -| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | Newtonsoft | 1,177.48 μs | 25.746 μs | 38.535 μs | 1,172.04 μs | 17.5781 | 15.6250 | 15.6250 | 1260228 B | -| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **824.48 μs** | **31.605 μs** | **47.305 μs** | **812.80 μs** | **25.3906** | **25.3906** | **25.3906** | **965259 B** | -| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | SystemTextJson | 814.40 μs | 50.865 μs | 76.132 μs | 811.34 μs | 21.4844 | 21.4844 | 21.4844 | 950333 B | -| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **Stream** | **636.72 μs** | **31.468 μs** | **47.099 μs** | **630.15 μs** | **16.6016** | **16.6016** | **16.6016** | **678066 B** | -| EncryptToProvidedStream | 100 | None | Stream | 383.33 μs | 7.441 μs | 10.671 μs | 384.69 μs | 4.3945 | 4.3945 | 4.3945 | 230133 B | -| Decrypt | 100 | None | Stream | 384.93 μs | 12.519 μs | 18.738 μs | 383.59 μs | 5.8594 | 5.8594 | 5.8594 | 230753 B | -| DecryptToProvidedStream | 100 | None | Stream | 295.19 μs | 7.094 μs | 10.618 μs | 296.11 μs | 3.4180 | 3.4180 | 3.4180 | 119116 B | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,178.06 μs** | **63.246 μs** | **94.664 μs** | **1,152.03 μs** | **13.6719** | **11.7188** | **9.7656** | **1379183 B** | -| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | Newtonsoft | 1,175.01 μs | 41.917 μs | 61.441 μs | 1,156.01 μs | 11.7188 | 9.7656 | 9.7656 | 1124274 B | -| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **1,050.27 μs** | **31.128 μs** | **46.591 μs** | **1,052.70 μs** | **17.5781** | **17.5781** | **17.5781** | **766642 B** | -| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | SystemTextJson | 926.80 μs | 28.605 μs | 41.025 μs | 925.73 μs | 18.5547 | 18.5547 | 18.5547 | 801460 B | -| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **Stream** | **757.11 μs** | **19.549 μs** | **29.260 μs** | **754.55 μs** | **10.7422** | **10.7422** | **10.7422** | **479493 B** | -| EncryptToProvidedStream | 100 | Brotli | Stream | 563.46 μs | 9.960 μs | 14.284 μs | 561.60 μs | 2.9297 | 2.9297 | 2.9297 | 180637 B | -| Decrypt | 100 | Brotli | Stream | 542.34 μs | 14.514 μs | 21.724 μs | 542.04 μs | 6.8359 | 6.8359 | 6.8359 | 231162 B | -| DecryptToProvidedStream | 100 | Brotli | Stream | 463.69 μs | 9.130 μs | 12.800 μs | 460.71 μs | 3.4180 | 3.4180 | 3.4180 | 119506 B | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.51 μs** | **0.393 μs** | **0.576 μs** | **22.63 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | Newtonsoft | 27.10 μs | 0.124 μs | 0.174 μs | 27.07 μs | 0.1526 | 0.0305 | - | 41440 B | +| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **Stream** | **12.82 μs** | **0.063 μs** | **0.091 μs** | **12.78 μs** | **0.0610** | **0.0153** | **-** | **17768 B** | +| EncryptToProvidedStream | 1 | None | Stream | 12.86 μs | 0.127 μs | 0.190 μs | 12.86 μs | 0.0458 | 0.0153 | - | 11632 B | +| Decrypt | 1 | None | Stream | 12.90 μs | 0.169 μs | 0.253 μs | 12.89 μs | 0.0458 | 0.0153 | - | 12672 B | +| DecryptToProvidedStream | 1 | None | Stream | 13.60 μs | 0.189 μs | 0.271 μs | 13.58 μs | 0.0458 | 0.0153 | - | 11504 B | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **28.87 μs** | **0.346 μs** | **0.474 μs** | **28.74 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | Newtonsoft | 35.28 μs | 0.905 μs | 1.269 μs | 35.40 μs | 0.1221 | - | - | 41064 B | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **Stream** | **21.52 μs** | **0.750 μs** | **1.026 μs** | **21.21 μs** | **0.0610** | **0.0305** | **-** | **16824 B** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 20.80 μs | 0.228 μs | 0.312 μs | 20.76 μs | 0.0305 | - | - | 11912 B | +| Decrypt | 1 | Brotli | Stream | 19.55 μs | 0.443 μs | 0.636 μs | 19.33 μs | 0.0305 | - | - | 13216 B | +| DecryptToProvidedStream | 1 | Brotli | Stream | 19.86 μs | 0.192 μs | 0.270 μs | 19.82 μs | 0.0305 | - | - | 12048 B | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **96.62 μs** | **10.278 μs** | **15.384 μs** | **86.34 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | Newtonsoft | 106.98 μs | 3.407 μs | 5.100 μs | 104.40 μs | 0.6104 | 0.1221 | - | 157425 B | +| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **Stream** | **39.15 μs** | **0.200 μs** | **0.281 μs** | **39.16 μs** | **0.3052** | **0.0610** | **-** | **83168 B** | +| EncryptToProvidedStream | 10 | None | Stream | 39.27 μs | 2.127 μs | 2.982 μs | 39.00 μs | 0.1221 | - | - | 37288 B | +| Decrypt | 10 | None | Stream | 28.94 μs | 0.369 μs | 0.518 μs | 28.91 μs | 0.0916 | 0.0305 | - | 29520 B | +| DecryptToProvidedStream | 10 | None | Stream | 27.56 μs | 0.167 μs | 0.235 μs | 27.54 μs | 0.0610 | 0.0305 | - | 18416 B | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.87 μs** | **0.707 μs** | **0.991 μs** | **116.89 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | Newtonsoft | 144.59 μs | 14.519 μs | 21.282 μs | 139.95 μs | 0.4883 | - | - | 144849 B | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **Stream** | **91.28 μs** | **3.359 μs** | **5.027 μs** | **89.89 μs** | **0.2441** | **-** | **-** | **64049 B** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 98.61 μs | 1.831 μs | 2.741 μs | 99.15 μs | 0.1221 | - | - | 32705 B | +| Decrypt | 10 | Brotli | Stream | 60.11 μs | 1.366 μs | 2.044 μs | 59.71 μs | 0.1221 | 0.0610 | - | 30064 B | +| DecryptToProvidedStream | 10 | Brotli | Stream | 58.15 μs | 1.689 μs | 2.422 μs | 58.25 μs | - | - | - | 18960 B | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,087.44 μs** | **15.865 μs** | **23.254 μs** | **1,085.47 μs** | **21.4844** | **19.5313** | **19.5313** | **1677999 B** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | Newtonsoft | 1,124.12 μs | 15.278 μs | 22.395 μs | 1,123.48 μs | 17.5781 | 15.6250 | 15.6250 | 1260236 B | +| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **Stream** | **517.26 μs** | **7.106 μs** | **10.636 μs** | **520.35 μs** | **14.6484** | **14.6484** | **14.6484** | **678303 B** | +| EncryptToProvidedStream | 100 | None | Stream | 339.83 μs | 5.149 μs | 7.706 μs | 337.59 μs | 4.3945 | 4.3945 | 4.3945 | 230367 B | +| Decrypt | 100 | None | Stream | 346.38 μs | 10.316 μs | 15.440 μs | 343.34 μs | 6.3477 | 6.3477 | 6.3477 | 230757 B | +| DecryptToProvidedStream | 100 | None | Stream | 280.22 μs | 4.289 μs | 6.420 μs | 278.61 μs | 3.4180 | 3.4180 | 3.4180 | 119111 B | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,113.95 μs** | **15.209 μs** | **22.764 μs** | **1,103.81 μs** | **13.6719** | **9.7656** | **9.7656** | **1379180 B** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | Newtonsoft | 1,138.03 μs | 8.340 μs | 12.224 μs | 1,137.53 μs | 11.7188 | 9.7656 | 9.7656 | 1124260 B | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **Stream** | **723.60 μs** | **10.132 μs** | **15.165 μs** | **719.90 μs** | **11.7188** | **11.7188** | **11.7188** | **479748 B** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 551.93 μs | 7.420 μs | 10.641 μs | 550.24 μs | 2.9297 | 2.9297 | 2.9297 | 180882 B | +| Decrypt | 100 | Brotli | Stream | 540.31 μs | 12.842 μs | 19.222 μs | 542.34 μs | 6.8359 | 6.8359 | 6.8359 | 231164 B | +| DecryptToProvidedStream | 100 | Brotli | Stream | 452.60 μs | 3.476 μs | 5.203 μs | 452.38 μs | 3.4180 | 3.4180 | 3.4180 | 119509 B | From 6c8a9180e0442d1f94bcb4935fd79d2d50444079 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 23 Oct 2024 08:30:03 +0200 Subject: [PATCH 78/85] + update Azure.Cosmos references to 3.43.1 (stable), 3.44.0-preview.1 (preview) to get access to UseSystemTextJson + Stream mode support for Newtonsoft + tests and fixes --- .../src/Common/CosmosJsonDotNetSerializer.cs | 35 +++++++-- .../src/DataEncryptionKeyProperties.cs | 14 +++- .../src/EncryptionContainerExtensions.cs | 6 +- .../src/EncryptionProcessor.cs | 54 ++++++++++---- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 4 +- .../StreamProcessing/DecryptableItemStream.cs | 1 + .../StreamProcessing/EncryptableItemStream.cs | 15 ++++ .../EncryptionContainerStream.cs | 74 +++++++++---------- .../Transformation/ArrayStreamProcessor.cs | 5 +- .../MdeEncryptionProcessor.Preview.cs | 22 ++++++ .../MdeJObjectEncryptionProcessor.Preview.cs | 22 ++++-- .../StreamProcessor.Encryptor.cs | 17 ++++- .../MdeCustomEncryptionTestsWithSystemText.cs | 25 ++++--- 13 files changed, 207 insertions(+), 87 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs index 9ed8056360..0496752de9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs @@ -7,6 +7,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System; using System.IO; using System.Text; + using System.Threading; + using System.Threading.Tasks; using Newtonsoft.Json; /// @@ -40,6 +42,18 @@ internal CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSetting /// The object representing the deserialized stream public T FromStream(Stream stream) { + return this.FromStream(stream, false); + } + + /// + /// Convert a Stream to the passed in type. + /// + /// The type of object that should be deserialized + /// An open stream that is readable that contains JSON + /// True if input stream shouldn't be disposed + /// The object representing the deserialized stream + public T FromStream(Stream stream, bool leaveOpen) + { #if NET8_0_OR_GREATER ArgumentNullException.ThrowIfNull(stream); #else @@ -54,7 +68,7 @@ public T FromStream(Stream stream) return (T)(object)stream; } - using (StreamReader sr = new (stream)) + using (StreamReader sr = new (stream, Encoding.UTF8, true, 1024, leaveOpen)) using (JsonTextReader jsonTextReader = new (sr)) { jsonTextReader.ArrayPool = JsonArrayPool.Instance; @@ -72,7 +86,15 @@ public T FromStream(Stream stream) public MemoryStream ToStream(T input) { MemoryStream streamPayload = new (); - using (StreamWriter streamWriter = new (streamPayload, encoding: CosmosJsonDotNetSerializer.DefaultEncoding, bufferSize: 1024, leaveOpen: true)) +#pragma warning disable VSTHRD002 // Avoid problematic synchronous waits + this.ToStreamAsync(input, streamPayload, CancellationToken.None).GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 // Avoid problematic synchronous waits + return streamPayload; + } + + public async Task ToStreamAsync(T input, Stream output, CancellationToken cancellationToken) + { + using (StreamWriter streamWriter = new (output, encoding: CosmosJsonDotNetSerializer.DefaultEncoding, bufferSize: 1024, leaveOpen: true)) using (JsonTextWriter writer = new (streamWriter)) { writer.ArrayPool = JsonArrayPool.Instance; @@ -80,11 +102,14 @@ public MemoryStream ToStream(T input) JsonSerializer jsonSerializer = this.GetSerializer(); jsonSerializer.Serialize(writer, input); writer.Flush(); - streamWriter.Flush(); +#if NET8_0_OR_GREATER + await streamWriter.FlushAsync(cancellationToken); +#else + await streamWriter.FlushAsync(); +#endif } - streamPayload.Position = 0; - return streamPayload; + output.Position = 0; } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyProperties.cs index ceca9e3db5..0f0c11dd17 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyProperties.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System; using System.Collections.Generic; using System.Linq; + using System.Text.Json.Serialization; using Newtonsoft.Json; /// @@ -73,31 +74,36 @@ internal DataEncryptionKeyProperties(DataEncryptionKeyProperties source) /// /// [JsonProperty(PropertyName = "id")] + [JsonPropertyName("id")] public string Id { get; internal set; } /// /// Gets the Encryption algorithm that will be used along with this data encryption key to encrypt/decrypt data. /// [JsonProperty(PropertyName = "encryptionAlgorithm", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("encryptionAlgorithm")] public string EncryptionAlgorithm { get; internal set; } /// /// Gets wrapped form of the data encryption key. /// [JsonProperty(PropertyName = "wrappedDataEncryptionKey", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("wrappedDataEncryptionKey")] public byte[] WrappedDataEncryptionKey { get; internal set; } /// /// Gets metadata for the wrapping provider that can be used to unwrap the wrapped data encryption key. /// [JsonProperty(PropertyName = "keyWrapMetadata", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("keyWrapMetadata")] public EncryptionKeyWrapMetadata EncryptionKeyWrapMetadata { get; internal set; } /// /// Gets the creation time of the resource from the Azure Cosmos DB service. /// - [JsonConverter(typeof(UnixDateTimeConverter))] + [Newtonsoft.Json.JsonConverter(typeof(UnixDateTimeConverter))] [JsonProperty(PropertyName = "createTime", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("createTime")] public DateTime? CreatedTime { get; internal set; } /// @@ -110,14 +116,16 @@ internal DataEncryptionKeyProperties(DataEncryptionKeyProperties source) /// ETags are used for concurrency checking when updating resources. /// [JsonProperty(PropertyName = "_etag", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("_etag")] public string ETag { get; internal set; } /// /// Gets the last modified time stamp associated with the resource from the Azure Cosmos DB service. /// /// The last modified time stamp associated with the resource. - [JsonConverter(typeof(UnixDateTimeConverter))] + [Newtonsoft.Json.JsonConverter(typeof(UnixDateTimeConverter))] [JsonProperty(PropertyName = "_ts", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("_ts")] public DateTime? LastModified { get; internal set; } /// @@ -129,6 +137,7 @@ internal DataEncryptionKeyProperties(DataEncryptionKeyProperties source) /// E.g. a self-link for a document could be dbs/db_resourceid/colls/coll_resourceid/documents/doc_resourceid /// [JsonProperty(PropertyName = "_self", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("_self")] public virtual string SelfLink { get; internal set; } /// @@ -143,6 +152,7 @@ internal DataEncryptionKeyProperties(DataEncryptionKeyProperties source) /// These resource ids are used when building up SelfLinks, a static addressable Uri for each resource within a database account. /// [JsonProperty(PropertyName = "_rid", NullValueHandling = NullValueHandling.Ignore)] + [JsonPropertyName("_rid")] internal string ResourceId { get; set; } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs index f624960f65..529df174ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs @@ -23,7 +23,7 @@ public static Container WithEncryptor( this Container container, Encryptor encryptor) { -#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) { return new EncryptionContainerStream(container, encryptor); @@ -57,7 +57,7 @@ public static FeedIterator ToEncryptionFeedIterator( this Container container, IQueryable query) { -#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) { if (container is not EncryptionContainerStream encryptionContainerStream) @@ -102,7 +102,7 @@ public static FeedIterator ToEncryptionStreamIterator( this Container container, IQueryable query) { -#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) { if (container is not EncryptionContainerStream encryptionContainerStream) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 370355b558..0c1792733c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -132,17 +132,34 @@ public static async Task EncryptAsync( } } - if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + switch (encryptionOptions.EncryptionAlgorithm) { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); - } - - if (encryptionOptions.JsonProcessor != JsonProcessor.Stream) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); +#pragma warning disable CS0618 // Type or member is obsolete + case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: + { + Stream result = await AeAesEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken); + await result.CopyToAsync(output, cancellationToken); + output.Position = 0; + return; + } +#pragma warning restore CS0618 // Type or member is obsolete + case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: + switch (encryptionOptions.JsonProcessor) + { + case JsonProcessor.Stream: + await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); + break; + case JsonProcessor.Newtonsoft: + await EncryptionProcessor.MdeEncryptionProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); + break; + default: + throw new NotSupportedException($"Streaming mode is not supported for {encryptionOptions.JsonProcessor}"); + } + + break; + default: + throw new NotSupportedException($"Encryption algorithm {encryptionOptions.EncryptionAlgorithm} not supported."); } - - await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); } #endif @@ -216,11 +233,6 @@ public static async Task DecryptAsync( return null; } - if (jsonProcessor != JsonProcessor.Stream) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); - } - Debug.Assert(input.CanSeek); Debug.Assert(output.CanWrite); Debug.Assert(output.CanSeek); @@ -240,7 +252,19 @@ public static async Task DecryptAsync( #pragma warning disable CS0618 // Type or member is obsolete if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) { - context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + switch (jsonProcessor) + { + case JsonProcessor.Stream: + context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + break; + case JsonProcessor.Newtonsoft: + (Stream ms, context) = await EncryptionProcessor.DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken); + await ms.CopyToAsync(output, cancellationToken); + output.Position = 0; + break; + default: + throw new NotSupportedException($"Streaming mode is not supported for {jsonProcessor}"); + } } else if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index cca4d719bf..8717019eab 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -26,11 +26,11 @@ - + - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs index 58f73923fa..9e5d44c17a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/DecryptableItemStream.cs @@ -69,6 +69,7 @@ public DecryptableItemStream( case Stream: // consumer doesn't need payload deserialized MemoryStream ms = new ((int)this.decryptedStream.Length); await this.decryptedStream.CopyToAsync(ms, cancellationToken); + ms.Position = 0; return ((T)(object)ms, this.decryptionContext); default: #if SDKPROJECTREF diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs index 45dc241a75..87af42bfcd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptableItemStream.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing { using System; using System.IO; + using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos; @@ -63,18 +64,32 @@ protected internal override void SetDecryptableStream(Stream decryptableStream, /// protected internal override Stream ToStream(CosmosSerializer serializer) { + if (this.Item is Stream stream) + { + return stream; + } + return serializer.ToStream(this.Item); } /// protected internal override async Task ToStreamAsync(CosmosSerializer serializer, Stream outputStream, CancellationToken cancellationToken) { + if (this.Item is Stream stream) + { + await stream.CopyToAsync(outputStream, cancellationToken); + stream.Position = 0; + outputStream.Position = 0; + return; + } + #if SDKPROJECTREF await serializer.ToStreamAsync(this.Item, outputStream, cancellationToken); #else // TODO: CosmosSerializer is lacking suitable methods Stream cosmosSerializerOutput = serializer.ToStream(this.Item); await cosmosSerializerOutput.CopyToAsync(outputStream, cancellationToken); + outputStream.Position = 0; #endif } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs index b3b920e608..56d13f0819 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs @@ -96,7 +96,7 @@ public override async Task> CreateItemAsync( { ResponseMessage responseMessage; - if (item is EncryptableItemStream encryptableItemStream) + if (item is EncryptableItem encryptableItemStream) { using Stream rms = this.streamManager.CreateStream(); await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); @@ -114,16 +114,14 @@ public override async Task> CreateItemAsync( } else { - using (Stream itemStream = this.CosmosSerializer.ToStream(item)) - { - responseMessage = await this.CreateItemHelperAsync( - itemStream, - partitionKey.Value, - requestOptions, - decryptResponse: true, - diagnosticsContext, - cancellationToken); - } + using Stream itemStream = this.CosmosSerializer.ToStream(item); + responseMessage = await this.CreateItemHelperAsync( + itemStream, + partitionKey.Value, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); return this.ResponseFactory.CreateItemResponse(responseMessage); } @@ -179,14 +177,14 @@ await EncryptionProcessor.EncryptAsync( cancellationToken); ResponseMessage responseMessage = await this.container.CreateItemStreamAsync( - streamPayload, + encryptedStream, partitionKey, requestOptions, cancellationToken); if (decryptResponse) { - using Stream decryptedStream = this.streamManager.CreateStream(); + Stream decryptedStream = this.streamManager.CreateStream(); _ = await EncryptionProcessor.DecryptAsync( responseMessage.Content, decryptedStream, @@ -251,7 +249,7 @@ public override async Task> ReadItemAsync( DecryptableItem decryptableItem = new DecryptableItemStream( responseMessage.Content, this.Encryptor, - options.EncryptionOptions.JsonProcessor, + options?.EncryptionOptions?.JsonProcessor ?? JsonProcessor.Newtonsoft, this.CosmosSerializer, this.streamManager); @@ -305,10 +303,11 @@ private async Task ReadItemHelperAsync( requestOptions, cancellationToken); - if (decryptResponse && requestOptions is EncryptionItemRequestOptions options) + if (decryptResponse) { - using Stream rms = this.streamManager.CreateStream(); - _ = await EncryptionProcessor.DecryptAsync(responseMessage.Content, rms, this.Encryptor, diagnosticsContext, options.EncryptionOptions.JsonProcessor, cancellationToken); + JsonProcessor processor = (requestOptions as EncryptionItemRequestOptions)?.EncryptionOptions?.JsonProcessor ?? JsonProcessor.Newtonsoft; + Stream rms = this.streamManager.CreateStream(); + _ = await EncryptionProcessor.DecryptAsync(responseMessage.Content, rms, this.Encryptor, diagnosticsContext, processor, cancellationToken); responseMessage.Content = rms; } @@ -346,12 +345,13 @@ public override async Task> ReplaceItemAsync( { ResponseMessage responseMessage; - if (item is EncryptableItemStream encryptableItemStream) + if (item is EncryptableItem encryptableItemStream) { using Stream rms = this.streamManager.CreateStream(); await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); - responseMessage = await this.CreateItemHelperAsync( + responseMessage = await this.ReplaceItemHelperAsync( rms, + id, partitionKey.Value, requestOptions, decryptResponse: false, @@ -364,17 +364,15 @@ public override async Task> ReplaceItemAsync( } else { - using (Stream itemStream = this.CosmosSerializer.ToStream(item)) - { - responseMessage = await this.ReplaceItemHelperAsync( - itemStream, - id, - partitionKey.Value, - requestOptions, - decryptResponse: true, - diagnosticsContext, - cancellationToken); - } + using Stream itemStream = this.CosmosSerializer.ToStream(item); + responseMessage = await this.ReplaceItemHelperAsync( + itemStream, + id, + partitionKey.Value, + requestOptions, + decryptResponse: true, + diagnosticsContext, + cancellationToken); return this.ResponseFactory.CreateItemResponse(responseMessage); } @@ -433,10 +431,9 @@ await EncryptionProcessor.EncryptAsync( encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken); - streamPayload = encryptedStream; ResponseMessage responseMessage = await this.container.ReplaceItemStreamAsync( - streamPayload, + encryptedStream, id, partitionKey, requestOptions, @@ -444,7 +441,7 @@ await EncryptionProcessor.EncryptAsync( if (decryptResponse) { - using Stream decryptedStream = this.streamManager.CreateStream(); + Stream decryptedStream = this.streamManager.CreateStream(); _ = await EncryptionProcessor.DecryptAsync( responseMessage.Content, decryptedStream, @@ -489,7 +486,7 @@ public override async Task> UpsertItemAsync( { ResponseMessage responseMessage; - if (item is EncryptableItemStream encryptableItemStream) + if (item is EncryptableItem encryptableItemStream) { using Stream rms = this.streamManager.CreateStream(); await encryptableItemStream.ToStreamAsync(this.CosmosSerializer, rms, cancellationToken); @@ -562,25 +559,24 @@ private async Task UpsertItemHelperAsync( cancellationToken); } - using Stream rms = this.streamManager.CreateStream(); + Stream encryptedStream = this.streamManager.CreateStream(); await EncryptionProcessor.EncryptAsync( streamPayload, - rms, + encryptedStream, this.Encryptor, encryptionItemRequestOptions.EncryptionOptions, diagnosticsContext, cancellationToken); - streamPayload = rms; ResponseMessage responseMessage = await this.container.UpsertItemStreamAsync( - streamPayload, + encryptedStream, partitionKey, requestOptions, cancellationToken); if (decryptResponse) { - using Stream decryptStream = this.streamManager.CreateStream(); + Stream decryptStream = this.streamManager.CreateStream(); _ = await EncryptionProcessor.DecryptAsync( responseMessage.Content, decryptStream, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs index c6d3160bb5..df9553b336 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs @@ -178,13 +178,16 @@ private long TransformBuffer(Span buffer, bool isFinalBlock, Utf8JsonWrite break; case JsonTokenType.PropertyName: - if (reader.ValueTextEquals(DocumentsPropertyUtf8Bytes.Span)) + if (chunkWriter == null && reader.ValueTextEquals(DocumentsPropertyUtf8Bytes.Span)) { isDocumentsProperty = true; } currentWriter.WritePropertyName(reader.ValueSpan); break; + case JsonTokenType.String: + currentWriter.WriteStringValue(reader.ValueSpan); + break; default: currentWriter.WriteRawValue(reader.ValueSpan, true); break; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 2dc3eb7e10..ac50912c09 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -56,6 +56,28 @@ public async Task EncryptAsync( #endif } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public async Task EncryptStreamAsync( + Stream input, + Stream output, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + switch (encryptionOptions.JsonProcessor) + { + case JsonProcessor.Newtonsoft: + await this.JObjectEncryptionProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, token); + break; + case JsonProcessor.Stream: + await this.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, token); + break; + default: + throw new InvalidOperationException($"Unsupported JsonProcessor {encryptionOptions.JsonProcessor}"); + } + } +#endif + internal async Task DecryptObjectAsync( JObject document, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 6d7b49ac04..63944015f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -26,9 +26,8 @@ public async Task EncryptAsync( EncryptionOptions encryptionOptions, CancellationToken token) { - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); - - Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); + MemoryStream result = new (); + await this.EncryptStreamAsync(input, result, encryptor, encryptionOptions, token); #if NET8_0_OR_GREATER await input.DisposeAsync(); @@ -39,8 +38,21 @@ public async Task EncryptAsync( return result; } - public async Task EncryptAsync( + public async Task EncryptStreamAsync( + Stream input, + Stream output, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input, leaveOpen: true); + + await this.EncryptAsync(itemJObj, output, encryptor, encryptionOptions, token); + } + + public async Task EncryptAsync( JObject input, + Stream output, Encryptor encryptor, EncryptionOptions encryptionOptions, CancellationToken token) @@ -115,7 +127,7 @@ public async Task EncryptAsync( input.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); - return EncryptionProcessor.BaseSerializer.ToStream(input); + await EncryptionProcessor.BaseSerializer.ToStreamAsync(input, output, token); } internal async Task DecryptObjectAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index 3fd4b0e4f9..20e3d00a52 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -17,6 +17,8 @@ internal partial class StreamProcessor { private readonly byte[] encryptionPropertiesNameBytes = Encoding.UTF8.GetBytes(Constants.EncryptedInfo); + private static ReadOnlySpan Utf8Bom => new byte[] { 0xEF, 0xBB, 0xBF }; + internal async Task EncryptStreamAsync( Stream inputStream, Stream outputStream, @@ -52,16 +54,23 @@ internal async Task EncryptStreamAsync( Utf8JsonWriter encryptionPayloadWriter = null; string encryptPropertyName = null; RentArrayBufferWriter bufferWriter = null; + bool firstRead = true; while (!isFinalBlock) { + int offset = 0; int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + if (firstRead && buffer.AsSpan(0, Utf8Bom.Length).StartsWith(Utf8Bom)) + { + offset = Utf8Bom.Length; + } + int dataSize = dataLength + leftOver; isFinalBlock = dataSize == 0; long bytesConsumed = 0; bytesConsumed = this.TransformEncryptBuffer( - buffer.AsSpan(0, dataSize), + buffer.AsSpan(0 + offset, dataSize - offset), isFinalBlock, writer, ref encryptionPayloadWriter, @@ -76,7 +85,7 @@ internal async Task EncryptStreamAsync( encryptionKey, encryptionOptions); - leftOver = dataSize - (int)bytesConsumed; + leftOver = dataSize - ((int)bytesConsumed + offset); // we need to scale out buffer if (leftOver == dataSize) @@ -88,11 +97,10 @@ internal async Task EncryptStreamAsync( else if (leftOver != 0) { buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + firstRead = false; } } - await inputStream.DisposeAsync(); - EncryptionProperties encryptionProperties = new ( encryptionFormatVersion: compressionEnabled ? 4 : 3, encryptionOptions.EncryptionAlgorithm, @@ -107,6 +115,7 @@ internal async Task EncryptStreamAsync( writer.WriteEndObject(); writer.Flush(); + inputStream.Position = 0; outputStream.Position = 0; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs index 35a714e950..7f02ffb4c0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -#if SDKPROJECTREF && ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests { @@ -23,6 +23,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests using DataEncryptionKey = DataEncryptionKey; using Newtonsoft.Json.Linq; using Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.Utils; + using System.Text.Json.Serialization; + using Microsoft.Azure.Cosmos.Encryption.Custom.StreamProcessing; [TestClass] public class MdeCustomEncryptionTestsWithSystemText @@ -506,8 +508,8 @@ public void ValidateDecryptableContent() public async Task EncryptionCreateItemWithLazyDecryption(JsonProcessor jsonProcessor, CompressionOptions.CompressionAlgorithm compressionAlgorithm) { TestDoc testDoc = TestDoc.Create(); - ItemResponse> createResponse = await encryptionContainer.CreateItemAsync( - new EncryptableItem(testDoc), + ItemResponse> createResponse = await encryptionContainer.CreateItemAsync( + new EncryptableItemStream(testDoc), new PartitionKey(testDoc.PK), GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); @@ -518,8 +520,8 @@ public async Task EncryptionCreateItemWithLazyDecryption(JsonProcessor jsonProce // stream TestDoc testDoc1 = TestDoc.Create(); - ItemResponse createResponseStream = await encryptionContainer.CreateItemAsync( - new EncryptableItemStream(TestCommon.ToStream(testDoc1)), + ItemResponse> createResponseStream = await encryptionContainer.CreateItemAsync( + new EncryptableItemStream(TestCommon.ToStream(testDoc1)), new PartitionKey(testDoc1.PK), GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); @@ -756,8 +758,8 @@ public async Task EncryptionRudItemLazyDecryption(JsonProcessor jsonProcessor, C { TestDoc testDoc = TestDoc.Create(); // Upsert (item doesn't exist) - ItemResponse> upsertResponse = await encryptionContainer.UpsertItemAsync( - new EncryptableItem(testDoc), + ItemResponse> upsertResponse = await encryptionContainer.UpsertItemAsync( + new EncryptableItemStream(testDoc), new PartitionKey(testDoc.PK), GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); @@ -771,8 +773,8 @@ public async Task EncryptionRudItemLazyDecryption(JsonProcessor jsonProcessor, C testDoc.NonSensitive = Guid.NewGuid().ToString(); testDoc.Sensitive_StringFormat = Guid.NewGuid().ToString(); - ItemResponse upsertResponseStream = await encryptionContainer.UpsertItemAsync( - new EncryptableItemStream(TestCommon.ToStream(testDoc)), + ItemResponse> upsertResponseStream = await encryptionContainer.UpsertItemAsync( + new EncryptableItemStream(TestCommon.ToStream(testDoc)), new PartitionKey(testDoc.PK), GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm)); @@ -786,8 +788,8 @@ public async Task EncryptionRudItemLazyDecryption(JsonProcessor jsonProcessor, C testDoc.NonSensitive = Guid.NewGuid().ToString(); testDoc.Sensitive_StringFormat = Guid.NewGuid().ToString(); - ItemResponse replaceResponseStream = await encryptionContainer.ReplaceItemAsync( - new EncryptableItemStream(TestCommon.ToStream(testDoc)), + ItemResponse> replaceResponseStream = await encryptionContainer.ReplaceItemAsync( + new EncryptableItemStream(TestCommon.ToStream(testDoc)), testDoc.Id, new PartitionKey(testDoc.PK), GetRequestOptions(dekId, TestDoc.PathsToEncrypt, jsonProcessor, compressionAlgorithm, upsertResponseStream.ETag)); @@ -2064,6 +2066,7 @@ public class TestDoc }; [JsonProperty("id")] + [JsonPropertyName("id")] public string Id { get; set; } public string PK { get; set; } From b362826c607e6f9cfa7d8af1a298970072892d43 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 07:45:03 +0200 Subject: [PATCH 79/85] ! Utf8JsonReader/Writer correctly working with escaped strings ! more bugfixes on new EncryptionContainer --- .../src/EncryptionContainerExtensions.cs | 22 +++++++++++++++++++ .../src/EncryptionProcessor.cs | 2 ++ .../EncryptionFeedIteratorStream.cs | 2 +- .../EncryptionTransactionalBatchStream.cs | 2 +- .../Transformation/ArrayStreamProcessor.cs | 12 +++++++++- .../src/Transformation/ArrayStreamSplitter.cs | 18 +++++++++++++-- .../StreamProcessor.Decryptor.cs | 11 +++++++++- .../MdeCustomEncryptionTestsWithSystemText.cs | 13 ++++++++++- 8 files changed, 75 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs index 529df174ff..475ac316ad 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs @@ -35,6 +35,28 @@ public static Container WithEncryptor( encryptor); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + /// + /// Get container with for performing operations using client-side encryption. + /// + /// Regular cosmos container. + /// Provider that allows encrypting and decrypting data. + /// Json Processor used for the container. + /// Container to perform operations supporting client-side encryption / decryption. + public static Container WithEncryptor( + this Container container, + Encryptor encryptor, + JsonProcessor jsonProcessor) + { + return jsonProcessor switch + { + JsonProcessor.Stream => new EncryptionContainerStream(container, encryptor), + JsonProcessor.Newtonsoft => new EncryptionContainer(container, encryptor), + _ => throw new NotSupportedException($"Json Processor {jsonProcessor} is not supported.") + }; + } +#endif + /// /// This method gets the FeedIterator from LINQ IQueryable to execute query asynchronously. /// This will create the fresh new FeedIterator when called which will support decryption. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 0c1792733c..55917805fa 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -111,6 +111,7 @@ public static async Task EncryptAsync( if (!encryptionOptions.PathsToEncrypt.Any()) { await input.CopyToAsync(output, cancellationToken); + output.Position = 0; return; } @@ -245,6 +246,7 @@ public static async Task DecryptAsync( if (properties?.EncryptionProperties == null) { await input.CopyToAsync(output, cancellationToken: cancellationToken); + output.Position = 0; return null; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs index 6de6c1aeae..720d753995 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionFeedIteratorStream.cs @@ -45,7 +45,7 @@ public override async Task ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { - using Stream decryptedContent = this.streamManager.CreateStream(); + Stream decryptedContent = this.streamManager.CreateStream(); await EncryptionProcessor.DeserializeAndDecryptResponseAsync( responseMessage.Content, decryptedContent, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs index 284eb390b5..0f6df9cb0c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionTransactionalBatchStream.cs @@ -68,7 +68,7 @@ public override TransactionalBatch CreateItemStream( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("EncryptItemStream")) { - using Stream temp = this.streamManager.CreateStream(); + Stream temp = this.streamManager.CreateStream(); EncryptionProcessor.EncryptAsync( streamPayload, temp, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs index df9553b336..fe85442d79 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs @@ -186,7 +186,17 @@ private long TransformBuffer(Span buffer, bool isFinalBlock, Utf8JsonWrite currentWriter.WritePropertyName(reader.ValueSpan); break; case JsonTokenType.String: - currentWriter.WriteStringValue(reader.ValueSpan); + if (!reader.ValueIsEscaped) + { + currentWriter.WriteStringValue(reader.ValueSpan); + } + else + { + byte[] temp = arrayPoolManager.Rent(reader.ValueSpan.Length); + int tempBytes = reader.CopyString(temp); + currentWriter.WriteStringValue(temp.AsSpan(0, tempBytes)); + } + break; default: currentWriter.WriteRawValue(reader.ValueSpan, true); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs index 39e12edabd..c30241ba26 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs @@ -75,7 +75,8 @@ internal async Task> SplitCollectionAsync( ref state, ref isDocumentsProperty, ref isDocumentsArray, - recyclableMemoryStreamManager); + recyclableMemoryStreamManager, + arrayPoolManager); leftOver = dataSize - (int)bytesConsumed; @@ -97,7 +98,7 @@ internal async Task> SplitCollectionAsync( return outputList; } - private long TransformBuffer(Span buffer, List outputList, bool isFinalBlock, ref RecyclableMemoryStream bufferWriter, ref Utf8JsonWriter chunkWriter, ref JsonReaderState state, ref bool isDocumentsProperty, ref bool isDocumentsArray, RecyclableMemoryStreamManager manager) + private long TransformBuffer(Span buffer, List outputList, bool isFinalBlock, ref RecyclableMemoryStream bufferWriter, ref Utf8JsonWriter chunkWriter, ref JsonReaderState state, ref bool isDocumentsProperty, ref bool isDocumentsArray, RecyclableMemoryStreamManager manager, ArrayPoolManager arrayPoolManager) { Utf8JsonReader reader = new Utf8JsonReader(buffer, isFinalBlock, state); @@ -165,6 +166,19 @@ private long TransformBuffer(Span buffer, List outputList, bool is } currentWriter?.WritePropertyName(reader.ValueSpan); + break; + case JsonTokenType.String: + if (!reader.ValueIsEscaped) + { + currentWriter.WriteStringValue(reader.ValueSpan); + } + else + { + byte[] temp = arrayPoolManager.Rent(reader.ValueSpan.Length); + int tempBytes = reader.CopyString(temp); + currentWriter.WriteStringValue(temp.AsSpan(0, tempBytes)); + } + break; default: currentWriter?.WriteRawValue(reader.ValueSpan, true); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 442800d6e0..fdbf898923 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -133,7 +133,16 @@ private long TransformDecryptBuffer(ReadOnlySpan buffer, bool isFinalBlock case JsonTokenType.String: if (decryptPropertyName == null) { - writer.WriteStringValue(reader.ValueSpan); + if (!reader.ValueIsEscaped) + { + writer.WriteStringValue(reader.ValueSpan); + } + else + { + byte[] temp = arrayPoolManager.Rent(reader.ValueSpan.Length); + int tempBytes = reader.CopyString(temp); + writer.WriteStringValue(temp.AsSpan(0, tempBytes)); + } } else { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs index 7f02ffb4c0..df48efa858 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs @@ -392,6 +392,8 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey(JsonProcessor json testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); Assert.AreEqual(1, unwrapcount); + + await dekProvider.Container.DeleteItemAsync< DataEncryptionKeyProperties>(dekId, new PartitionKey(dekId)); } [TestMethod] @@ -1017,6 +1019,15 @@ public async Task EncryptionTransactionBatchCrud(JsonProcessor jsonProcessor, Co // doc3ToCreate, doc4ToCreate wasn't encrypted await VerifyItemByReadAsync(itemContainer, doc3ToCreate); await VerifyItemByReadAsync(itemContainer, doc4ToCreate); + + await encryptionContainer.DeleteItemAsync(doc1ToCreate.Id, new PartitionKey(doc1ToCreate.Id)); + await encryptionContainer.DeleteItemAsync(doc2ToCreate.Id, new PartitionKey(doc2ToCreate.Id)); + await encryptionContainer.DeleteItemAsync(doc1ToReplace.Id, new PartitionKey(doc1ToReplace.Id)); + await encryptionContainer.DeleteItemAsync(doc3ToCreate.Id, new PartitionKey(doc3ToCreate.Id)); + await encryptionContainer.DeleteItemAsync(doc4ToCreate.Id, new PartitionKey(doc4ToCreate.Id)); + await encryptionContainer.DeleteItemAsync(doc2ToReplace.Id, new PartitionKey(doc2ToReplace.Id)); + await encryptionContainer.DeleteItemAsync(doc1ToUpsert.Id, new PartitionKey(doc1ToUpsert.Id)); + await encryptionContainer.DeleteItemAsync(doc2ToUpsert.Id, new PartitionKey(doc2ToUpsert.Id)); } [TestMethod] @@ -1030,7 +1041,7 @@ public async Task EncryptionTransactionalBatchWithCustomSerializer(JsonProcessor Database databaseWithCustomSerializer = clientWithCustomSerializer.GetDatabase(database.Id); Container containerWithCustomSerializer = databaseWithCustomSerializer.GetContainer(itemContainer.Id); - Container encryptionContainerWithCustomSerializer = containerWithCustomSerializer.WithEncryptor(encryptor); + Container encryptionContainerWithCustomSerializer = containerWithCustomSerializer.WithEncryptor(encryptor, JsonProcessor.Stream); string partitionKey = "thePK"; string dek1 = dekId; From 8ce362205c636927889ab36f13fb95b9d3468dc6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:05:20 +0200 Subject: [PATCH 80/85] ~ fix the mess after merge --- .../src/EncryptionProcessor.cs | 110 ------------------ 1 file changed, 110 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 68ed36b4ee..3c031fb7b3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -29,7 +29,6 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static readonly StreamProcessor StreamProcessor = new (); private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); #endif @@ -114,10 +113,6 @@ public static async Task EncryptAsync( } #endif - await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); - } -#endif - /// /// If there isn't any data that needs to be decrypted, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. @@ -168,8 +163,6 @@ public static async Task EncryptAsync( JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), - JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), - JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") }; @@ -239,109 +232,6 @@ public static async Task DecryptAsync( } #endif -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( - Stream input, - Stream output, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - JsonProcessor jsonProcessor, - CancellationToken cancellationToken) - { - if (input == null) - { - return null; - } - - if (jsonProcessor != JsonProcessor.Stream) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); - } - - Debug.Assert(input.CanSeek); - Debug.Assert(output.CanWrite); - Debug.Assert(output.CanSeek); - Debug.Assert(encryptor != null); - Debug.Assert(diagnosticsContext != null); - input.Position = 0; - - EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); - input.Position = 0; - if (properties?.EncryptionProperties == null) - { - await input.CopyToAsync(output, cancellationToken: cancellationToken); - return null; - } - - DecryptionContext context; -#pragma warning disable CS0618 // Type or member is obsolete - if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) - { - context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); - } - else if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized) - { - (Stream stream, context) = await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken); - await stream.CopyToAsync(output, cancellationToken); - output.Position = 0; - } - else - { - input.Position = 0; - throw new NotSupportedException($"Encryption Algorithm: {properties.EncryptionProperties.EncryptionAlgorithm} is not supported."); - } -#pragma warning restore CS0618 // Type or member is obsolete - - if (context == null) - { - input.Position = 0; - return null; - } - - await input.DisposeAsync(); - return context; - } -#endif - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( - Stream input, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - if (input == null) - { - return (input, null); - } - - Debug.Assert(input.CanSeek); - Debug.Assert(encryptor != null); - Debug.Assert(diagnosticsContext != null); - input.Position = 0; - - EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); - input.Position = 0; - if (properties?.EncryptionProperties == null) - { - return (input, null); - } - - MemoryStream ms = new (); - - DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, ms, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); - if (context == null) - { - input.Position = 0; - return (input, null); - } - - await input.DisposeAsync(); - return (ms, context); - } - -#endif - #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( Stream input, From 10ad1d4dc9036e2d2fee8d6979445dfd76243611 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:08:08 +0200 Subject: [PATCH 81/85] ~ fix merge mess --- .../src/EncryptionProcessor.cs | 9 --------- .../StreamProcessor.Decryptor.cs | 18 ------------------ ....Encryption.Custom.Performance.Tests.csproj | 4 ---- 3 files changed, 31 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3c031fb7b3..7bf05fb930 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -29,7 +29,6 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); #endif @@ -91,10 +90,6 @@ public static async Task EncryptAsync( return; } - if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); - } if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) { throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); @@ -105,10 +100,6 @@ public static async Task EncryptAsync( throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); } - if (encryptionOptions.JsonProcessor != JsonProcessor.Stream) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); - } await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); } #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 00d7f7f224..d4ca1ccfdf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -72,8 +72,6 @@ internal async Task DecryptStreamAsync( string decryptPropertyName = null; - bool containsCompressed = properties.CompressedEncryptedPaths?.Count > 0; - while (!isFinalBlock) { int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); @@ -199,24 +197,9 @@ long TransformDecryptBuffer(ReadOnlySpan buffer) state = reader.CurrentState; return reader.BytesConsumed; } - } - - private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, bool containsCompressed, ArrayPoolManager arrayPoolManager) - { - BrotliCompressor decompressor = null; - if (properties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) - { - if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) - { - throw new NotSupportedException($"Unknown compression algorithm {properties.CompressionAlgorithm}"); - } void TransformDecryptProperty(ref Utf8JsonReader reader) { - decompressor = new (); - } - } - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(reader.ValueSpan.Length); // necessary for proper un-escaping @@ -238,7 +221,6 @@ void TransformDecryptProperty(ref Utf8JsonReader reader) bytes = buffer; } - } ReadOnlySpan bytesToWrite = bytes.AsSpan(0, processedBytes); switch ((TypeMarker)cipherTextWithTypeMarker[0]) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index c8669fd9a8..23f9b23530 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -26,10 +26,6 @@ - - - - From 81394b299e2468dbecd771b1df80c416e55f3524 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:18:46 +0200 Subject: [PATCH 82/85] - drop JsonNode based benchmarks --- .../Readme.md | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 22cb620269..691dbf0a5d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -15,10 +15,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.224 μs | 0.322 μs | 26.23 μs | 0.1526 | 0.0305 | - | 41440 B | | DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.33 μs** | **0.137 μs** | **0.201 μs** | **14.32 μs** | **0.0916** | **0.0153** | **-** | **22904 B** | -| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | SystemTextJson | 14.54 μs | 0.124 μs | 0.186 μs | 14.52 μs | 0.0610 | 0.0305 | - | 21448 B | -| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **1** | **None** | **Stream** | **12.85 μs** | **0.095 μs** | **0.143 μs** | **12.84 μs** | **0.0610** | **0.0153** | **-** | **17528 B** | | EncryptToProvidedStream | 1 | None | Stream | 13.00 μs | 0.096 μs | 0.141 μs | 12.98 μs | 0.0458 | 0.0153 | - | 11392 B | | Decrypt | 1 | None | Stream | 13.01 μs | 0.152 μs | 0.228 μs | 13.05 μs | 0.0458 | 0.0153 | - | 12672 B | @@ -27,10 +23,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 1 | Brotli | Newtonsoft | 33.49 μs | 0.910 μs | 1.335 μs | 33.99 μs | 0.1221 | - | - | 41064 B | | DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **20.92 μs** | **0.136 μs** | **0.199 μs** | **20.95 μs** | **0.0610** | **-** | **-** | **21952 B** | -| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | SystemTextJson | 20.53 μs | 0.136 μs | 0.200 μs | 20.52 μs | 0.0610 | 0.0305 | - | 20488 B | -| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **1** | **Brotli** | **Stream** | **21.15 μs** | **1.037 μs** | **1.521 μs** | **20.52 μs** | **0.0610** | **0.0305** | **-** | **16584 B** | | EncryptToProvidedStream | 1 | Brotli | Stream | 20.57 μs | 0.213 μs | 0.292 μs | 20.57 μs | 0.0305 | - | - | 11672 B | | Decrypt | 1 | Brotli | Stream | 21.14 μs | 2.212 μs | 3.311 μs | 19.46 μs | 0.0305 | - | - | 13216 B | @@ -39,10 +31,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 10 | None | Newtonsoft | 112.98 μs | 15.294 μs | 21.934 μs | 100.38 μs | 0.6104 | 0.1221 | - | 157425 B | | DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.85 μs** | **0.868 μs** | **1.272 μs** | **41.40 μs** | **0.4272** | **0.0610** | **-** | **105345 B** | -| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | SystemTextJson | 41.79 μs | 0.501 μs | 0.718 μs | 41.64 μs | 0.3662 | 0.0610 | - | 96464 B | -| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **10** | **None** | **Stream** | **39.63 μs** | **0.658 μs** | **0.923 μs** | **39.41 μs** | **0.3052** | **0.0610** | **-** | **82928 B** | | EncryptToProvidedStream | 10 | None | Stream | 36.59 μs | 0.272 μs | 0.399 μs | 36.57 μs | 0.1221 | - | - | 37048 B | | Decrypt | 10 | None | Stream | 28.64 μs | 0.378 μs | 0.517 μs | 28.59 μs | 0.1221 | 0.0305 | - | 29520 B | @@ -51,10 +39,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 10 | Brotli | Newtonsoft | 118.98 μs | 1.530 μs | 2.195 μs | 118.76 μs | 0.4883 | - | - | 144849 B | | DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **71.40 μs** | **0.799 μs** | **1.145 μs** | **71.23 μs** | **0.2441** | **-** | **-** | **86217 B** | -| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | SystemTextJson | 73.37 μs | 7.283 μs | 10.676 μs | 67.12 μs | 0.2441 | - | - | 82201 B | -| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **10** | **Brotli** | **Stream** | **90.10 μs** | **3.136 μs** | **4.693 μs** | **88.92 μs** | **0.2441** | **-** | **-** | **63809 B** | | EncryptToProvidedStream | 10 | Brotli | Stream | 97.27 μs | 1.885 μs | 2.703 μs | 97.35 μs | 0.1221 | - | - | 32465 B | | Decrypt | 10 | Brotli | Stream | 58.48 μs | 0.956 μs | 1.372 μs | 58.59 μs | 0.1221 | 0.0610 | - | 30064 B | @@ -63,10 +47,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 100 | None | Newtonsoft | 1,177.48 μs | 25.746 μs | 38.535 μs | 1,172.04 μs | 17.5781 | 15.6250 | 15.6250 | 1260228 B | | DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **824.48 μs** | **31.605 μs** | **47.305 μs** | **812.80 μs** | **25.3906** | **25.3906** | **25.3906** | **965259 B** | -| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | SystemTextJson | 814.40 μs | 50.865 μs | 76.132 μs | 811.34 μs | 21.4844 | 21.4844 | 21.4844 | 950333 B | -| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **100** | **None** | **Stream** | **636.72 μs** | **31.468 μs** | **47.099 μs** | **630.15 μs** | **16.6016** | **16.6016** | **16.6016** | **678066 B** | | EncryptToProvidedStream | 100 | None | Stream | 383.33 μs | 7.441 μs | 10.671 μs | 384.69 μs | 4.3945 | 4.3945 | 4.3945 | 230133 B | | Decrypt | 100 | None | Stream | 384.93 μs | 12.519 μs | 18.738 μs | 383.59 μs | 5.8594 | 5.8594 | 5.8594 | 230753 B | @@ -75,10 +55,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 100 | Brotli | Newtonsoft | 1,175.01 μs | 41.917 μs | 61.441 μs | 1,156.01 μs | 11.7188 | 9.7656 | 9.7656 | 1124274 B | | DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **1,050.27 μs** | **31.128 μs** | **46.591 μs** | **1,052.70 μs** | **17.5781** | **17.5781** | **17.5781** | **766642 B** | -| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | SystemTextJson | 926.80 μs | 28.605 μs | 41.025 μs | 925.73 μs | 18.5547 | 18.5547 | 18.5547 | 801460 B | -| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **100** | **Brotli** | **Stream** | **757.11 μs** | **19.549 μs** | **29.260 μs** | **754.55 μs** | **10.7422** | **10.7422** | **10.7422** | **479493 B** | | EncryptToProvidedStream | 100 | Brotli | Stream | 563.46 μs | 9.960 μs | 14.284 μs | 561.60 μs | 2.9297 | 2.9297 | 2.9297 | 180637 B | | Decrypt | 100 | Brotli | Stream | 542.34 μs | 14.514 μs | 21.724 μs | 542.04 μs | 6.8359 | 6.8359 | 6.8359 | 231162 B | From b4da46cfebae637c4150f8f6dbe039c3b926a204 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:33:53 +0200 Subject: [PATCH 83/85] - drop files we don't need yet --- .../src/MemoryStreamManager.cs | 41 ------------------- .../src/StreamManager.cs | 31 -------------- 2 files changed, 72 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs deleted file mode 100644 index 0edbdf52b2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs +++ /dev/null @@ -1,41 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System.IO; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; - - /// - /// Memory Stream manager - /// - /// Placeholder - internal class MemoryStreamManager : StreamManager - { - private readonly RecyclableMemoryStreamManager streamManager = new (); - - /// - /// Create stream - /// - /// Desired minimal capacity of stream. - /// Instance of stream. - public override Stream CreateStream(int hintSize = 0) - { - return new RecyclableMemoryStream(this.streamManager, null, hintSize); - } - - /// - /// Dispose of used Stream (return to pool) - /// - /// Stream to dispose. - /// ValueTask.CompletedTask - public async override ValueTask ReturnStreamAsync(Stream stream) - { - await stream.DisposeAsync(); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs deleted file mode 100644 index ed7831b783..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs +++ /dev/null @@ -1,31 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System.IO; - using System.Threading.Tasks; - - /// - /// Abstraction for pooling streams - /// - public abstract class StreamManager - { - /// - /// Create stream - /// - /// Desired minimal size of stream. - /// Instance of stream. - public abstract Stream CreateStream(int hintSize = 0); - - /// - /// Dispose of used Stream (return to pool) - /// - /// Stream to dispose. - /// ValueTask.CompletedTask - public abstract ValueTask ReturnStreamAsync(Stream stream); - } -} -#endif \ No newline at end of file From 147ecc87ca66c81c0193bef96d3fdeab72175584 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 12:44:32 +0200 Subject: [PATCH 84/85] ~ bugfixes ~ small improvements ~ cleanup --- .../EncryptionContainerStream.cs | 2 +- .../Transformation/ArrayStreamProcessor.cs | 8 ++ .../src/Transformation/ArrayStreamSplitter.cs | 33 +++--- .../StreamProcessor.Decryptor.cs | 6 ++ .../StreamProcessor.Encryptor.cs | 6 ++ .../MdeCustomEncryptionTestsWithSystemText.cs | 14 ++- .../Readme.md | 100 +++++++++--------- 7 files changed, 95 insertions(+), 74 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs index 56d13f0819..3f9bb6d3be 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamProcessing/EncryptionContainerStream.cs @@ -1051,7 +1051,7 @@ private async Task ReadManyItemsHelperAsync( readManyRequestOptions, cancellationToken); - using Stream decryptedStream = this.streamManager.CreateStream(); + Stream decryptedStream = this.streamManager.CreateStream(); await EncryptionProcessor.DeserializeAndDecryptResponseAsync(responseMessage.Content, decryptedStream, this.Encryptor, this.streamManager, cancellationToken); return new DecryptedResponseMessage(responseMessage, decryptedStream); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs index fe85442d79..8c017ed0ce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamProcessor.cs @@ -65,6 +65,12 @@ internal async Task DeserializeAndDecryptCollectionAsync( int dataLength = await readStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); int dataSize = dataLength + leftOver; isFinalBlock = dataSize == 0; + + if (isFinalBlock) + { + break; + } + long bytesConsumed = 0; bytesConsumed = this.TransformBuffer( @@ -96,6 +102,8 @@ internal async Task DeserializeAndDecryptCollectionAsync( } } + writer.Flush(); + await readStream.DisposeAsync(); output.Position = 0; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs index c30241ba26..adc697f9d2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/ArrayStreamSplitter.cs @@ -64,6 +64,11 @@ internal async Task> SplitCollectionAsync( int dataLength = await readStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); int dataSize = dataLength + leftOver; isFinalBlock = dataSize == 0; + if (isFinalBlock) + { + break; + } + long bytesConsumed = 0; bytesConsumed = this.TransformBuffer( @@ -104,8 +109,6 @@ private long TransformBuffer(Span buffer, List outputList, bool is while (reader.Read()) { - Utf8JsonWriter currentWriter = chunkWriter; - JsonTokenType tokenType = reader.TokenType; switch (tokenType) @@ -117,19 +120,16 @@ private long TransformBuffer(Span buffer, List outputList, bool is { bufferWriter = new RecyclableMemoryStream(manager); chunkWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); - chunkWriter?.WriteStartObject(); - } - else - { - currentWriter?.WriteStartObject(); } + chunkWriter?.WriteStartObject(); + break; case JsonTokenType.EndObject: - currentWriter?.WriteEndObject(); + chunkWriter?.WriteEndObject(); if (reader.CurrentDepth == 2 && chunkWriter != null) { - currentWriter.Flush(); + chunkWriter.Flush(); bufferWriter.Position = 0; outputList.Add(bufferWriter); @@ -146,11 +146,11 @@ private long TransformBuffer(Span buffer, List outputList, bool is isDocumentsArray = true; } - currentWriter?.WriteStartArray(); + chunkWriter?.WriteStartArray(); break; case JsonTokenType.EndArray: - currentWriter?.WriteEndArray(); + chunkWriter?.WriteEndArray(); if (isDocumentsArray && reader.CurrentDepth == 1) { isDocumentsArray = false; @@ -164,24 +164,27 @@ private long TransformBuffer(Span buffer, List outputList, bool is { isDocumentsProperty = true; } + else + { + chunkWriter?.WritePropertyName(reader.ValueSpan); + } - currentWriter?.WritePropertyName(reader.ValueSpan); break; case JsonTokenType.String: if (!reader.ValueIsEscaped) { - currentWriter.WriteStringValue(reader.ValueSpan); + chunkWriter?.WriteStringValue(reader.ValueSpan); } else { byte[] temp = arrayPoolManager.Rent(reader.ValueSpan.Length); int tempBytes = reader.CopyString(temp); - currentWriter.WriteStringValue(temp.AsSpan(0, tempBytes)); + chunkWriter?.WriteStringValue(temp.AsSpan(0, tempBytes)); } break; default: - currentWriter?.WriteRawValue(reader.ValueSpan, true); + chunkWriter?.WriteRawValue(reader.ValueSpan, true); break; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index f682fb4e01..82d5e713b2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -77,6 +77,12 @@ internal async Task DecryptStreamAsync( int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); int dataSize = dataLength + leftOver; isFinalBlock = dataSize == 0; + + if (isFinalBlock) + { + break; + } + long bytesConsumed = 0; // processing itself here diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index a472d76db2..1643c26ed0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -72,6 +72,12 @@ internal async Task EncryptStreamAsync( int dataSize = dataLength + leftOver; isFinalBlock = dataSize == 0; + + if (isFinalBlock) + { + break; + } + long bytesConsumed = 0; bytesConsumed = TransformEncryptBuffer(buffer.AsSpan(0 + offset, dataSize - offset)); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs index df48efa858..44368cf42a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs @@ -1020,14 +1020,7 @@ public async Task EncryptionTransactionBatchCrud(JsonProcessor jsonProcessor, Co await VerifyItemByReadAsync(itemContainer, doc3ToCreate); await VerifyItemByReadAsync(itemContainer, doc4ToCreate); - await encryptionContainer.DeleteItemAsync(doc1ToCreate.Id, new PartitionKey(doc1ToCreate.Id)); - await encryptionContainer.DeleteItemAsync(doc2ToCreate.Id, new PartitionKey(doc2ToCreate.Id)); - await encryptionContainer.DeleteItemAsync(doc1ToReplace.Id, new PartitionKey(doc1ToReplace.Id)); - await encryptionContainer.DeleteItemAsync(doc3ToCreate.Id, new PartitionKey(doc3ToCreate.Id)); - await encryptionContainer.DeleteItemAsync(doc4ToCreate.Id, new PartitionKey(doc4ToCreate.Id)); - await encryptionContainer.DeleteItemAsync(doc2ToReplace.Id, new PartitionKey(doc2ToReplace.Id)); - await encryptionContainer.DeleteItemAsync(doc1ToUpsert.Id, new PartitionKey(doc1ToUpsert.Id)); - await encryptionContainer.DeleteItemAsync(doc2ToUpsert.Id, new PartitionKey(doc2ToUpsert.Id)); + await dekProvider.Container.DeleteItemAsync(dek2, new PartitionKey(dek2)); } [TestMethod] @@ -2475,6 +2468,8 @@ public async Task EncryptionRewrapLegacyDekToMdeWrap(JsonProcessor jsonProcessor await VerifyItemByReadAsync(encryptionContainer, testDoc, dekId: dekId); + await dekProvider.Container.DeleteItemAsync(dekId, new PartitionKey(dekId)); + // rewrap from Mde Algo to Legacy algo should fail dekId = "rewrapMdeAlgoDekToLegacyAlgoDek"; @@ -2497,6 +2492,7 @@ await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContai Assert.AreEqual("Rewrap operation with EncryptionAlgorithm 'AEAes256CbcHmacSha256Randomized' is not supported on Data Encryption Keys which are configured with 'MdeAeadAes256CbcHmac256Randomized'. ", ex.Message); } + await dekProvider.Container.DeleteItemAsync(dekId, new PartitionKey(dekId)); // rewrap Mde to Mde with Option // rewrap from Mde Algo to Legacy algo should fail @@ -2523,6 +2519,8 @@ await MdeCustomEncryptionTestsWithSystemText.dekProvider.DataEncryptionKeyContai Assert.AreEqual( metadata2, dataEncryptionKeyProperties.EncryptionKeyWrapMetadata); + + await dekProvider.Container.DeleteItemAsync(dekId, new PartitionKey(dekId)); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 31a77807b2..a944d8cfa5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,53 +9,53 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |-------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.51 μs** | **0.393 μs** | **0.576 μs** | **22.63 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | -| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | Newtonsoft | 27.10 μs | 0.124 μs | 0.174 μs | 27.07 μs | 0.1526 | 0.0305 | - | 41440 B | -| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **Stream** | **12.82 μs** | **0.063 μs** | **0.091 μs** | **12.78 μs** | **0.0610** | **0.0153** | **-** | **17768 B** | -| EncryptToProvidedStream | 1 | None | Stream | 12.86 μs | 0.127 μs | 0.190 μs | 12.86 μs | 0.0458 | 0.0153 | - | 11632 B | -| Decrypt | 1 | None | Stream | 12.90 μs | 0.169 μs | 0.253 μs | 12.89 μs | 0.0458 | 0.0153 | - | 12672 B | -| DecryptToProvidedStream | 1 | None | Stream | 13.60 μs | 0.189 μs | 0.271 μs | 13.58 μs | 0.0458 | 0.0153 | - | 11504 B | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **28.87 μs** | **0.346 μs** | **0.474 μs** | **28.74 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | -| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | Newtonsoft | 35.28 μs | 0.905 μs | 1.269 μs | 35.40 μs | 0.1221 | - | - | 41064 B | -| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **Stream** | **21.52 μs** | **0.750 μs** | **1.026 μs** | **21.21 μs** | **0.0610** | **0.0305** | **-** | **16824 B** | -| EncryptToProvidedStream | 1 | Brotli | Stream | 20.80 μs | 0.228 μs | 0.312 μs | 20.76 μs | 0.0305 | - | - | 11912 B | -| Decrypt | 1 | Brotli | Stream | 19.55 μs | 0.443 μs | 0.636 μs | 19.33 μs | 0.0305 | - | - | 13216 B | -| DecryptToProvidedStream | 1 | Brotli | Stream | 19.86 μs | 0.192 μs | 0.270 μs | 19.82 μs | 0.0305 | - | - | 12048 B | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **96.62 μs** | **10.278 μs** | **15.384 μs** | **86.34 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | -| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | Newtonsoft | 106.98 μs | 3.407 μs | 5.100 μs | 104.40 μs | 0.6104 | 0.1221 | - | 157425 B | -| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **Stream** | **39.15 μs** | **0.200 μs** | **0.281 μs** | **39.16 μs** | **0.3052** | **0.0610** | **-** | **83168 B** | -| EncryptToProvidedStream | 10 | None | Stream | 39.27 μs | 2.127 μs | 2.982 μs | 39.00 μs | 0.1221 | - | - | 37288 B | -| Decrypt | 10 | None | Stream | 28.94 μs | 0.369 μs | 0.518 μs | 28.91 μs | 0.0916 | 0.0305 | - | 29520 B | -| DecryptToProvidedStream | 10 | None | Stream | 27.56 μs | 0.167 μs | 0.235 μs | 27.54 μs | 0.0610 | 0.0305 | - | 18416 B | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.87 μs** | **0.707 μs** | **0.991 μs** | **116.89 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | -| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | Newtonsoft | 144.59 μs | 14.519 μs | 21.282 μs | 139.95 μs | 0.4883 | - | - | 144849 B | -| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **Stream** | **91.28 μs** | **3.359 μs** | **5.027 μs** | **89.89 μs** | **0.2441** | **-** | **-** | **64049 B** | -| EncryptToProvidedStream | 10 | Brotli | Stream | 98.61 μs | 1.831 μs | 2.741 μs | 99.15 μs | 0.1221 | - | - | 32705 B | -| Decrypt | 10 | Brotli | Stream | 60.11 μs | 1.366 μs | 2.044 μs | 59.71 μs | 0.1221 | 0.0610 | - | 30064 B | -| DecryptToProvidedStream | 10 | Brotli | Stream | 58.15 μs | 1.689 μs | 2.422 μs | 58.25 μs | - | - | - | 18960 B | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,087.44 μs** | **15.865 μs** | **23.254 μs** | **1,085.47 μs** | **21.4844** | **19.5313** | **19.5313** | **1677999 B** | -| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | Newtonsoft | 1,124.12 μs | 15.278 μs | 22.395 μs | 1,123.48 μs | 17.5781 | 15.6250 | 15.6250 | 1260236 B | -| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **Stream** | **517.26 μs** | **7.106 μs** | **10.636 μs** | **520.35 μs** | **14.6484** | **14.6484** | **14.6484** | **678303 B** | -| EncryptToProvidedStream | 100 | None | Stream | 339.83 μs | 5.149 μs | 7.706 μs | 337.59 μs | 4.3945 | 4.3945 | 4.3945 | 230367 B | -| Decrypt | 100 | None | Stream | 346.38 μs | 10.316 μs | 15.440 μs | 343.34 μs | 6.3477 | 6.3477 | 6.3477 | 230757 B | -| DecryptToProvidedStream | 100 | None | Stream | 280.22 μs | 4.289 μs | 6.420 μs | 278.61 μs | 3.4180 | 3.4180 | 3.4180 | 119111 B | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,113.95 μs** | **15.209 μs** | **22.764 μs** | **1,103.81 μs** | **13.6719** | **9.7656** | **9.7656** | **1379180 B** | -| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | Newtonsoft | 1,138.03 μs | 8.340 μs | 12.224 μs | 1,137.53 μs | 11.7188 | 9.7656 | 9.7656 | 1124260 B | -| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **Stream** | **723.60 μs** | **10.132 μs** | **15.165 μs** | **719.90 μs** | **11.7188** | **11.7188** | **11.7188** | **479748 B** | -| EncryptToProvidedStream | 100 | Brotli | Stream | 551.93 μs | 7.420 μs | 10.641 μs | 550.24 μs | 2.9297 | 2.9297 | 2.9297 | 180882 B | -| Decrypt | 100 | Brotli | Stream | 540.31 μs | 12.842 μs | 19.222 μs | 542.34 μs | 6.8359 | 6.8359 | 6.8359 | 231164 B | -| DecryptToProvidedStream | 100 | Brotli | Stream | 452.60 μs | 3.476 μs | 5.203 μs | 452.38 μs | 3.4180 | 3.4180 | 3.4180 | 119509 B | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.97 μs** | **0.545 μs** | **0.816 μs** | **0.1526** | **0.0305** | **-** | **41.2 KB** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | 21.46 μs | 0.105 μs | 0.153 μs | 0.1221 | 0.0305 | - | 34.13 KB | +| Decrypt | 1 | None | Newtonsoft | 27.28 μs | 0.149 μs | 0.219 μs | 0.1526 | 0.0305 | - | 40.84 KB | +| DecryptToProvidedStream | 1 | None | Newtonsoft | 34.38 μs | 1.292 μs | 1.934 μs | 0.1221 | - | - | 42.3 KB | +| **Encrypt** | **1** | **None** | **Stream** | **12.10 μs** | **0.081 μs** | **0.116 μs** | **0.0610** | **0.0153** | **-** | **17.35 KB** | +| EncryptToProvidedStream | 1 | None | Stream | 12.72 μs | 0.829 μs | 1.215 μs | 0.0458 | 0.0153 | - | 11.36 KB | +| Decrypt | 1 | None | Stream | 12.21 μs | 0.180 μs | 0.264 μs | 0.0458 | 0.0153 | - | 12.38 KB | +| DecryptToProvidedStream | 1 | None | Stream | 12.55 μs | 0.152 μs | 0.213 μs | 0.0458 | 0.0153 | - | 11.23 KB | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **28.73 μs** | **0.579 μs** | **0.830 μs** | **0.1526** | **0.0305** | **-** | **37.59 KB** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | 28.58 μs | 0.293 μs | 0.411 μs | 0.1221 | 0.0305 | - | 34.54 KB | +| Decrypt | 1 | Brotli | Newtonsoft | 35.35 μs | 0.894 μs | 1.337 μs | 0.1221 | - | - | 40.49 KB | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | 38.17 μs | 0.409 μs | 0.574 μs | 0.1221 | - | - | 42.14 KB | +| **Encrypt** | **1** | **Brotli** | **Stream** | **19.77 μs** | **0.275 μs** | **0.395 μs** | **0.0610** | **0.0305** | **-** | **16.43 KB** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 19.40 μs | 0.188 μs | 0.264 μs | 0.0305 | - | - | 11.63 KB | +| Decrypt | 1 | Brotli | Stream | 17.73 μs | 0.138 μs | 0.206 μs | 0.0305 | - | - | 12.65 KB | +| DecryptToProvidedStream | 1 | Brotli | Stream | 18.05 μs | 0.120 μs | 0.180 μs | 0.0305 | - | - | 11.51 KB | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **84.60 μs** | **0.488 μs** | **0.699 μs** | **0.6104** | **0.1221** | **-** | **168.82 KB** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | 82.21 μs | 0.199 μs | 0.272 μs | 0.4883 | - | - | 137.7 KB | +| Decrypt | 10 | None | Newtonsoft | 101.88 μs | 0.452 μs | 0.676 μs | 0.6104 | 0.1221 | - | 155.55 KB | +| DecryptToProvidedStream | 10 | None | Newtonsoft | 107.81 μs | 0.595 μs | 0.890 μs | 0.6104 | 0.1221 | - | 157.01 KB | +| **Encrypt** | **10** | **None** | **Stream** | **37.80 μs** | **0.181 μs** | **0.266 μs** | **0.3052** | **0.0610** | **-** | **81.22 KB** | +| EncryptToProvidedStream | 10 | None | Stream | 34.84 μs | 0.326 μs | 0.488 μs | 0.1221 | - | - | 36.41 KB | +| Decrypt | 10 | None | Stream | 26.40 μs | 0.164 μs | 0.245 μs | 0.1221 | 0.0305 | - | 28.83 KB | +| DecryptToProvidedStream | 10 | None | Stream | 25.85 μs | 0.175 μs | 0.262 μs | 0.0610 | 0.0305 | - | 17.98 KB | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **113.23 μs** | **0.688 μs** | **0.986 μs** | **0.6104** | **0.1221** | **-** | **165.98 KB** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | 111.05 μs | 0.535 μs | 0.801 μs | 0.4883 | - | - | 134.86 KB | +| Decrypt | 10 | Brotli | Newtonsoft | 122.44 μs | 1.023 μs | 1.499 μs | 0.4883 | - | - | 143.28 KB | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | 127.32 μs | 0.892 μs | 1.308 μs | 0.4883 | - | - | 144.93 KB | +| **Encrypt** | **10** | **Brotli** | **Stream** | **84.20 μs** | **2.861 μs** | **4.193 μs** | **0.2441** | **-** | **-** | **62.55 KB** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 92.70 μs | 1.253 μs | 1.876 μs | 0.1221 | - | - | 31.94 KB | +| Decrypt | 10 | Brotli | Stream | 54.23 μs | 0.528 μs | 0.775 μs | 0.1221 | - | - | 29.1 KB | +| DecryptToProvidedStream | 10 | Brotli | Stream | 54.34 μs | 0.505 μs | 0.756 μs | 0.0610 | - | - | 18.26 KB | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,074.94 μs** | **17.781 μs** | **26.614 μs** | **21.4844** | **19.5313** | **19.5313** | **1654.89 KB** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | 908.83 μs | 44.365 μs | 62.193 μs | 11.7188 | 9.7656 | 9.7656 | 1143.65 KB | +| Decrypt | 100 | None | Newtonsoft | 1,126.75 μs | 21.460 μs | 32.120 μs | 17.5781 | 15.6250 | 15.6250 | 1246.93 KB | +| DecryptToProvidedStream | 100 | None | Newtonsoft | 1,183.13 μs | 19.585 μs | 29.314 μs | 15.6250 | 13.6719 | 13.6719 | 1248.4 KB | +| **Encrypt** | **100** | **None** | **Stream** | **513.68 μs** | **11.309 μs** | **16.927 μs** | **16.6016** | **16.6016** | **16.6016** | **662.42 KB** | +| EncryptToProvidedStream | 100 | None | Stream | 335.51 μs | 7.015 μs | 10.500 μs | 4.3945 | 4.3945 | 4.3945 | 224.97 KB | +| Decrypt | 100 | None | Stream | 310.34 μs | 5.028 μs | 7.525 μs | 6.3477 | 6.3477 | 6.3477 | 225.35 KB | +| DecryptToProvidedStream | 100 | None | Stream | 264.40 μs | 3.169 μs | 4.545 μs | 3.4180 | 3.4180 | 3.4180 | 116.32 KB | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,098.17 μs** | **10.860 μs** | **16.255 μs** | **13.6719** | **9.7656** | **9.7656** | **1363.12 KB** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | 1,012.03 μs | 9.265 μs | 13.581 μs | 7.8125 | 5.8594 | 5.8594 | 1107.87 KB | +| Decrypt | 100 | Brotli | Newtonsoft | 1,137.56 μs | 8.877 μs | 13.012 μs | 11.7188 | 9.7656 | 9.7656 | 1114.15 KB | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | 1,160.69 μs | 9.399 μs | 13.777 μs | 11.7188 | 9.7656 | 9.7656 | 1115.79 KB | +| **Encrypt** | **100** | **Brotli** | **Stream** | **726.91 μs** | **10.086 μs** | **15.097 μs** | **11.7188** | **11.7188** | **11.7188** | **468.53 KB** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 551.89 μs | 6.359 μs | 9.518 μs | 2.9297 | 2.9297 | 2.9297 | 176.64 KB | +| Decrypt | 100 | Brotli | Stream | 517.81 μs | 7.945 μs | 11.891 μs | 6.3477 | 6.3477 | 6.3477 | 225.62 KB | +| DecryptToProvidedStream | 100 | Brotli | Stream | 440.63 μs | 4.781 μs | 7.007 μs | 3.4180 | 3.4180 | 3.4180 | 116.6 KB | From 86e33ec16cef3fa2f6243b4bcd30b63a0e924600 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 16:01:02 +0200 Subject: [PATCH 85/85] ~ more fixes --- .../src/EncryptionContainerExtensions.cs | 57 +++++++------------ .../MdeCustomEncryptionTestsWithSystemText.cs | 9 ++- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs index 475ac316ad..8654a07133 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainerExtensions.cs @@ -79,27 +79,20 @@ public static FeedIterator ToEncryptionFeedIterator( this Container container, IQueryable query) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) + return container switch { - if (container is not EncryptionContainerStream encryptionContainerStream) - { - throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainerStream)}."); - } - - return new EncryptionFeedIteratorStream( +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + EncryptionContainerStream encryptionContainerStream => new EncryptionFeedIteratorStream( (EncryptionFeedIteratorStream)encryptionContainerStream.ToEncryptionStreamIterator(query), - encryptionContainerStream.ResponseFactory); - } + encryptionContainerStream.ResponseFactory), #endif - if (container is not EncryptionContainer encryptionContainer) - { - throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainer)}."); - } + EncryptionContainer encryptionContainer => new EncryptionFeedIterator( + (EncryptionFeedIterator)encryptionContainer.ToEncryptionStreamIterator(query), + encryptionContainer.ResponseFactory), + + _ => throw new ArgumentOutOfRangeException(nameof(container), $"Container type {container.GetType().Name} is not supported.") - return new EncryptionFeedIterator( - (EncryptionFeedIterator)encryptionContainer.ToEncryptionStreamIterator(query), - encryptionContainer.ResponseFactory); + }; } /// @@ -124,31 +117,21 @@ public static FeedIterator ToEncryptionStreamIterator( this Container container, IQueryable query) { -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - if (container.Database.Client.ClientOptions.UseSystemTextJsonSerializerWithOptions is not null) + return container switch { - if (container is not EncryptionContainerStream encryptionContainerStream) - { - throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionFeedIterator)} is only supported with {nameof(EncryptionContainerStream)}."); - } - - return new EncryptionFeedIteratorStream( +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + EncryptionContainerStream encryptionContainerStream => new EncryptionFeedIteratorStream( query.ToStreamIterator(), encryptionContainerStream.Encryptor, encryptionContainerStream.CosmosSerializer, - new MemoryStreamManager()); - } + new MemoryStreamManager()), #endif - - if (container is not EncryptionContainer encryptionContainer) - { - throw new ArgumentOutOfRangeException(nameof(query), $"{nameof(ToEncryptionStreamIterator)} is only supported with {nameof(EncryptionContainer)}."); - } - - return new EncryptionFeedIterator( - query.ToStreamIterator(), - encryptionContainer.Encryptor, - encryptionContainer.CosmosSerializer); + EncryptionContainer encryptionContainer => new EncryptionFeedIterator( + query.ToStreamIterator(), + encryptionContainer.Encryptor, + encryptionContainer.CosmosSerializer), + _ => throw new ArgumentOutOfRangeException(nameof(container), $"Container type {container.GetType().Name} is not supported.") + }; } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs index 44368cf42a..0b5e0cf4d9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTestsWithSystemText.cs @@ -560,6 +560,8 @@ public async Task EncryptionChangeFeedDecryptionSuccessful(JsonProcessor jsonPro // change feed processor manual checkpoint with feed stream handler await ValidateChangeFeedProcessorStreamWithManualCheckpointResponse(encryptionContainerForChangeFeed, testDoc1, testDoc2); + + await dekProvider.Container.DeleteItemAsync(dek2, new PartitionKey(dek2)); } [TestMethod] @@ -628,6 +630,8 @@ public async Task EncryptionHandleDecryptionFailure(JsonProcessor jsonProcessor, await ValidateLazyDecryptionResponse(changeFeedReturnedDocs.GetEnumerator(), dek2); encryptor.FailDecryption = false; + + await dekProvider.Container.DeleteItemAsync(dek2, new PartitionKey(dek2)); } [TestMethod] @@ -1345,7 +1349,8 @@ private static async Task ValidateQueryResultsMultipleDocumentsAsync( } } - private static async Task ValidateQueryResponseAsync(Container container, + private static async Task ValidateQueryResponseAsync( + Container container, string query = null) { FeedIterator feedIterator; @@ -1437,7 +1442,7 @@ private static async Task ValidateChangeFeedProcessorResponse( bool isStartOk = allDocsProcessed.WaitOne(60000); await cfp.StopAsync(); - Assert.AreEqual(changeFeedReturnedDocs.Count, 2); + Assert.AreEqual(2, changeFeedReturnedDocs.Count); VerifyExpectedDocResponse(testDoc1, changeFeedReturnedDocs[^2]); VerifyExpectedDocResponse(testDoc2, changeFeedReturnedDocs[^1]);