From d6f89ea6f298e6b7459fa8246a22d8efe1aeb045 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 13:41:09 +0200 Subject: [PATCH 01/71] Add baseline benchmarks for `Microsoft.Azure.Cosmos.Encryption.Custom` --- .../src/AssemblyInfo.cs | 3 +- .../EncryptionBenchmark.cs | 102 ++++++++++++++++++ ...Encryption.Custom.Performance.Tests.csproj | 26 +++++ .../Program.cs | 21 ++++ .../Readme.md | 19 ++++ .../TestDoc.cs | 62 +++++++++++ Microsoft.Azure.Cosmos.sln | 31 +++++- 7 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs index c9e211085f..aca9899cc7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AssemblyInfo.cs @@ -5,4 +5,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.Tests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] -[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] \ No newline at end of file +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] +[assembly: InternalsVisibleTo("Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests" + Microsoft.Azure.Cosmos.Encryption.Custom.AssemblyKeys.TestPublicKey)] \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs new file mode 100644 index 0000000000..2e92274e6c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -0,0 +1,102 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using System.IO; + using BenchmarkDotNet.Attributes; + using Microsoft.Data.Encryption.Cryptography; + using Moq; + + [RPlotExporter] + public partial class EncryptionBenchmark + { + private static readonly byte[] DekData = Enumerable.Repeat((byte)0, 32).ToArray(); + private static readonly DataEncryptionKeyProperties DekProperties = new( + "id", + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + DekData, + new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); + private static readonly Mock StoreProvider = new(); + + private TestDoc? testDoc; + private CosmosEncryptor? encryptor; + private Custom.DataEncryptionKey? dek; + + private EncryptionOptions? encryptionOptions; + private byte[]? encryptedData; + private byte[]? plaintext; + + [Params(1, 10, 100)] + public int DocumentSizeInKb { get; set; } + + [GlobalSetup] + public async Task Setup() + { + StoreProvider + .Setup(x => x.UnwrapKey(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(DekData); + + this.dek = this.CreateMdeDek(); + + Mock keyProvider = new(); + keyProvider + .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(() => this.dek); + + this.encryptor = new(keyProvider.Object); + this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); + + this.encryptionOptions = CreateEncryptionOptions(); + + this.plaintext = EncryptionProcessor.BaseSerializer.ToStream(this.testDoc).ToArray(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext), + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + using MemoryStream memoryStream = new MemoryStream(); + + encryptedStream.CopyTo(memoryStream); + this.encryptedData = memoryStream.ToArray(); + } + + [Benchmark] + public async Task Encrypt() + { + await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext!), + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } + + [Benchmark] + public async Task Decrypt() + { + await EncryptionProcessor.DecryptAsync( + new MemoryStream(this.encryptedData!), + this.encryptor, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } + + private Custom.DataEncryptionKey CreateMdeDek() + { + return new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue); + } + + private static EncryptionOptions CreateEncryptionOptions() + { + EncryptionOptions options = new() + { + DataEncryptionKeyId = "dekId", + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt + }; + + return options; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj new file mode 100644 index 0000000000..3599027dc2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -0,0 +1,26 @@ + + + + Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests + Exe + net6 + enable + enable + + + + + + + + + + + + + true + true + ..\..\..\testkey.snk + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs new file mode 100644 index 0000000000..5304787e46 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Program.cs @@ -0,0 +1,21 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using BenchmarkDotNet.Configs; + using BenchmarkDotNet.Diagnosers; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Running; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + internal class Program + { + public static void Main(string[] args) + { + ManualConfig dontRequireSlnToRunBenchmarks = ManualConfig + .Create(DefaultConfig.Instance) + .AddJob(Job.MediumRun.WithToolchain(InProcessEmitToolchain.Instance)) + .AddDiagnoser(MemoryDiagnoser.Default); + + BenchmarkRunner.Run(dontRequireSlnToRunBenchmarks, args); + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md new file mode 100644 index 0000000000..967eed7430 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -0,0 +1,19 @@ +``` ini + +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores +.NET SDK=9.0.100-preview.7.24407.12 + [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 + +Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 +LaunchCount=2 WarmupCount=10 + +``` +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **47.90 μs** | **1.284 μs** | **1.842 μs** | **4.5776** | **1.1597** | **-** | **56.22 KB** | +| Decrypt | 1 | 59.67 μs | 1.041 μs | 1.558 μs | 5.2490 | 1.3428 | - | 64.79 KB | +| **Encrypt** | **10** | **154.57 μs** | **2.728 μs** | **3.998 μs** | **20.7520** | **4.1504** | **-** | **255.95 KB** | +| Decrypt | 10 | 220.03 μs | 6.124 μs | 8.585 μs | 29.0527 | 5.8594 | - | 357.41 KB | +| **Encrypt** | **100** | **2,761.51 μs** | **213.677 μs** | **319.822 μs** | **218.7500** | **173.8281** | **142.5781** | **2459.89 KB** | +| Decrypt | 100 | 2,445.99 μs | 136.839 μs | 200.577 μs | 347.6563 | 300.7813 | 253.9063 | 3406.33 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs new file mode 100644 index 0000000000..824045c489 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs @@ -0,0 +1,62 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests +{ + using System.Text; + using Newtonsoft.Json; + + public partial class EncryptionBenchmark + { + internal class TestDoc + { + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; + + [JsonProperty("id")] + public string Id { get; set; } + + public string NonSensitive { get; set; } + + public string SensitiveStr { get; set; } + + public int SensitiveInt { get; set; } + + public Dictionary SensitiveDict { get; set; } + + public TestDoc() + { + } + + public static TestDoc Create(int approximateSize = -1) + { + return new TestDoc() + { + Id = Guid.NewGuid().ToString(), + NonSensitive = Guid.NewGuid().ToString(), + SensitiveStr = Guid.NewGuid().ToString(), + SensitiveInt = new Random().Next(), + SensitiveDict = GenerateBigDictionary(approximateSize), + }; + } + + private static Dictionary GenerateBigDictionary(int approximateSize) + { + const int stringSize = 100; + int items = Math.Max(1, approximateSize / stringSize); + + return Enumerable.Range(1, items).ToDictionary(x => x.ToString(), y => GenerateRandomString(stringSize)); + } + + private static string GenerateRandomString(int size) + { + Random rnd = new Random(); + const string characters = "abcdefghijklmnopqrstuvwxyz0123456789"; + + StringBuilder sb = new(); + for (int i = 0; i < size; i++) + { + sb.Append(characters[rnd.Next(0, characters.Length)]); + } + return sb.ToString(); + } + } + + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index d412905195..6fa5e4c3f2 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.29123.88 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35209.166 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject @@ -34,6 +34,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos.Encr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FaultInjection", "Microsoft.Azure.Cosmos\FaultInjection\src\FaultInjection.csproj", "{021DDC27-02EF-42C4-9A9E-AA600833C2EE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests", "Microsoft.Azure.Cosmos.Encryption.Custom\tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests\Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj", "{CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Cover|Any CPU = Cover|Any CPU @@ -164,6 +166,30 @@ Global {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|Any CPU.Build.0 = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.ActiveCfg = Release|Any CPU {B5B3631D-AC2F-4257-855D-D6FE12F20B60}.Release|x64.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Cover|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Debug|x64.Build.0 = Debug|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|Any CPU.Build.0 = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.ActiveCfg = Release|Any CPU + {021DDC27-02EF-42C4-9A9E-AA600833C2EE}.Release|x64.Build.0 = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|Any CPU.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Cover|x64.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Debug|x64.Build.0 = Debug|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|Any CPU.Build.0 = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|x64.ActiveCfg = Release|Any CPU + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -175,6 +201,7 @@ Global {D7C78D76-A740-4129-BAAE-894640F95D74} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} {F87719DB-BB52-4B12-9D9C-F6AE30BAB3D7} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} {B5B3631D-AC2F-4257-855D-D6FE12F20B60} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} + {CE4D6DA8-148D-4A98-943B-D8C2D532E1DC} = {51F858D8-707E-4F21-BCC6-4D6123832E4F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {C6A1D820-CB03-4DE6-87D1-46EF476F0040} From 8928b880d43b58aea3fe3bc78ff6ba0dc40198a2 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 14:05:57 +0200 Subject: [PATCH 02/71] Cleanup --- .../EncryptionBenchmark.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 2e92274e6c..7a1bb21549 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -18,7 +18,6 @@ public partial class EncryptionBenchmark private TestDoc? testDoc; private CosmosEncryptor? encryptor; - private Custom.DataEncryptionKey? dek; private EncryptionOptions? encryptionOptions; private byte[]? encryptedData; @@ -34,12 +33,10 @@ public async Task Setup() .Setup(x => x.UnwrapKey(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(DekData); - this.dek = this.CreateMdeDek(); - Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => this.dek); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); @@ -55,8 +52,7 @@ public async Task Setup() new CosmosDiagnosticsContext(), CancellationToken.None); - using MemoryStream memoryStream = new MemoryStream(); - + using MemoryStream memoryStream = new MemoryStream(); encryptedStream.CopyTo(memoryStream); this.encryptedData = memoryStream.ToArray(); } @@ -82,11 +78,6 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private Custom.DataEncryptionKey CreateMdeDek() - { - return new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue); - } - private static EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() From 9dc7d5416803b524a86374e21c14aa9bf75abe02 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 12 Sep 2024 15:02:31 +0200 Subject: [PATCH 03/71] Use set of static test data for benchmarks --- .../EncryptionBenchmark.cs | 17 ++++++++++++----- ...s.Encryption.Custom.Performance.Tests.csproj | 12 ++++++++++++ .../Readme.md | 12 ++++++------ .../sampledata/testdoc-100kb.json | 1 + .../sampledata/testdoc-10kb.json | 1 + .../sampledata/testdoc-1kb.json | 1 + 6 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 7a1bb21549..d8e4c7d8a2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -16,7 +16,6 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); - private TestDoc? testDoc; private CosmosEncryptor? encryptor; private EncryptionOptions? encryptionOptions; @@ -39,11 +38,8 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.testDoc = TestDoc.Create(approximateSize: this.DocumentSizeInKb * 1024); - this.encryptionOptions = CreateEncryptionOptions(); - - this.plaintext = EncryptionProcessor.BaseSerializer.ToStream(this.testDoc).ToArray(); + this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( new MemoryStream(this.plaintext), @@ -89,5 +85,16 @@ private static EncryptionOptions CreateEncryptionOptions() return options; } + + private byte[] LoadTestDoc() + { + string name = $"Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.sampledata.testdoc-{this.DocumentSizeInKb}kb.json"; + using Stream resourceStream = typeof(EncryptionBenchmark).Assembly.GetManifestResourceStream(name)!; + + byte[] buffer = new byte[resourceStream!.Length]; + resourceStream.Read(buffer, 0, buffer.Length); + + return buffer; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 3599027dc2..e32a42c36c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -8,6 +8,18 @@ enable + + + + + + + + + + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 967eed7430..bb4836273b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **47.90 μs** | **1.284 μs** | **1.842 μs** | **4.5776** | **1.1597** | **-** | **56.22 KB** | -| Decrypt | 1 | 59.67 μs | 1.041 μs | 1.558 μs | 5.2490 | 1.3428 | - | 64.79 KB | -| **Encrypt** | **10** | **154.57 μs** | **2.728 μs** | **3.998 μs** | **20.7520** | **4.1504** | **-** | **255.95 KB** | -| Decrypt | 10 | 220.03 μs | 6.124 μs | 8.585 μs | 29.0527 | 5.8594 | - | 357.41 KB | -| **Encrypt** | **100** | **2,761.51 μs** | **213.677 μs** | **319.822 μs** | **218.7500** | **173.8281** | **142.5781** | **2459.89 KB** | -| Decrypt | 100 | 2,445.99 μs | 136.839 μs | 200.577 μs | 347.6563 | 300.7813 | 253.9063 | 3406.33 KB | +| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | +| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | +| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | +| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | +| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | +| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json new file mode 100644 index 0000000000..21d40c5f05 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-100kb.json @@ -0,0 +1 @@ +{"id":"e010f76d-17ce-4165-8225-35ce05333ee3","NonSensitive":"bbdb1cda-3444-4cf1-9a0f-063ce785476e","SensitiveStr":"b4f1b63c-96f3-4739-a5c1-ef1ca06f458e","SensitiveInt":770489940,"SensitiveDict":{"1":"qwt2ya60f3t7vst0j3cs9fqdiv658do98f5dfmh55yq5kjk3zu7p9ogbr8ph8c7h0iuttixja7b30vq52z9wqujjkf43utdsirsg","2":"vjvr3z8ns12xrhyxfnh5aptwqb3ejtkoql1od8jdxurgfxk2xwavu88de98b5he7v08d77guujejjrui08czkn4k7arrr1ijaqy6","3":"7kg7uasrf2i10b3itl13lxpzlt2tefzh7tk5r2eyxeu98b39l0zm7p2rk6ivvwyeti6q8wlkwebemw5i6nrnq1uog5j0kzlvnem9","4":"qasrvpil3s88t8f3fmr9i8jie3tqji5pxo0iaui21v97ay7o9myfs00w56euwd3pz3y5311rul9g42wuro9rb5eal42k6zo5m897","5":"sgu94rueqoqrpiljdekiiezjp4s1ds6xsd37yvcm7ulqoeq9h1xasaqgv4u067bbzi2uhd2ixbqh2ol1v3y6ndwhznxcs1l80ynz","6":"8z9uw1q9lobl3vqzj9n9lolcjtgdmyhg6riiabrrn1cj7fozwgmtetvv5xfnh8mg5avdib5csgl6hrrpvs8xgrqfv9atz1rqfrfn","7":"y7mo0hhc12sr7unppcwkfkfkavu47ym1c4ogwx0em6t3ept6ksfhxgn80645cy4q0njaisjc76o0l74ek1wyu83n55zkycuc909d","8":"ml3gxghqanig46gb0t5x0eowc8x6lwkosh054jcypjkj0sdykaync2o4853k1cm1ttod7a5htzyw9tnl3ch6iypopdh3271a5b7i","9":"jd3pcvu99al4qt476yjfauw31la95wjm07vxv4g5b9y32i3holfe38921chp54h3qx21bb7h42ho51caslx4lvsq4a4uv0r6chhi","10":"8mfl983nto693ihpwy9y3mo3c6lba8f4epr2anq7r26g5loauf5jt204wpmid3vcw7wq4vxfss5pgdco9ipgp5pcpivlrn0f0e22","11":"3zf5k550nhtqah3lmwj0qn6z940np7jpu3wmra6i22sf6wce1tftz7u9uur5vjts9qvixj41f19f0afi39inw2icke3pz1zj33lf","12":"xiog07lz2g0fvctld0l8ltfmamd1jgd7mmxh3z1xmshkrhkwhc0joa3g2zj7mwn2s1dbeo9tfpcwa32cqutxp34924i5pxy80tjv","13":"fq7tmr6pofoppj4xloubu493zlldoyqreciapdsohlyp49731bp6kw7nk3hvsd5m90nyqvddgmkt7ku61yulpdzy7xn1tqq3bnba","14":"5p03agq9dpmtmtrdeojlkq1llb9oxxrbg07yhc4dt34po7nf0hf89epk38pfxzgl6l8ru3g9tavh346dnfmlagfgumaqvevy526x","15":"edi9zs677mgmnhzjvjmc7ormtzl4oa1yt97ei7n441to010w3hbrk0uj4rzulb7f39w8a20cbqcxrhzrhsta4bz1bnnv8rgyo4x5","16":"1z986f9aidg55rii7volmthfr48beb29l5ttabi684pgwlxnveyhq82v0zuv7i2mcx6mo8r17qh6wuqqasw4cyu7kmgumly2ckzd","17":"rlz20xqzaqdec6ec4hqaswz55f1ckop30vpgpm6g18ub3ksakdjg8azt2rgpnwg6ruyp28x2sniqt3zctasrc69p7txmt97s340k","18":"v80p03aa9x9sqz5qcl1iljzy3um0kxxwa1z1nfna60f6ma48c023y7c0833th99bgta4qjrwir0d1c6pud8gspz63yrn8av8fdct","19":"ijrq4hmjsv4binbc03anhfbnl418rfvcfgzyce08g5hhudpwtj7ii05ri64kcife7uh9jdl1gz9bzyzjpr2st0qbnjomihd1yg7w","20":"j3ipcp1gyzqizwbtumcv430jek7siwi978lgxij5yiru5q98xo3q5oiqx87v7d05bctwchk7oj8xfyj7194jsbz5z6kr6bvpi4hk","21":"5lwdli6md1iacxxbqp9zc12tp26w0qw4b5dg87aif54mb66111nip0jyherf5obz91n6v8bdnchiqfulwsdsu4kwfeud4z0uvaog","22":"zl6ztc3ncnc61kwd0crhk7fpwignjggdivhje1yrvgtv31ksk63ttxe9001fa03jxzmwogh2ombrgkvdokihndkcjhhxhrn2ruon","23":"p4uemv64y8497cnexs05ih3i3of1k9112cedrrvz1j7zzv7tg3i4a5avbcdvw23vjs8ivnec7zdjxzihnpm7d6k6m5fqt09fbnrx","24":"gtgky4vi7x6oskpnqlv8eymkru8j19fy9rubxspr6zkjpm1zin34c1hacu21cp4xfq5hsclhmf443avr8g9hn51cr3r9h55et3se","25":"iqp21mfuklwrix35ixwuwxqxwmt4jhywfb5h3t11q4e5p1p1zqptjbipvrbhilc5icbvltfcv7vfcnsncjqfz8yut4h9gaymj06e","26":"4x7gkhll44182spczbl4bmtyhzs9mmefwxrfx6v72owz16ug59nrgch8o8wle0x8fm0090uyck8bkgxxl8fssnrftaypf5uwx84f","27":"cdwh6okkrd4f3pgyvessf2i9hl6bq4vhzvwucr2eyu0l53vq86pdwr6qxoe1vq8iwtzbfu2nwe5eg23zeyas10c2p9z5o9pjpl4p","28":"k20tzkucfijvtux1087365s69lrsg6axm2f7v03g4riygpzxj4e54rxfvvcw49xbyrq9oaofrroxw5owpbnihv3d35b72xolycpd","29":"fjy7dhkw5w06fm7vzlpjg29vg03605p2hivrrndrs4su432heklth18ns080z9ek8ihr6bssxc4mn5lzrmim4077qo7i1njqxyjx","30":"ld3s0rpja4naxd72v9u7op8rfmojlrv13z15tg1vmquwo4dvm2z1qpr6qn0hm4npva740kj582ltfzl66qnx35oec4fkul7kdftw","31":"fxro95cche8o7lprcf1sbl9u0biasa2nj9xqgvzaogep46rx6mbxtjs99lnrzgqson4nxz8x2rpegb2l9g0k0kepz4s9lbkr2sn3","32":"8cqqhyjskso2lh5xnjzx5qpei5149hfb9951wqwojdnbn86gaq1i2rpof4k2rzqfib5qqisfahg3awi1ug6owmezzwh3jsb9k7di","33":"mt8alt08khh3yitrp47npz6bcnbu6oqr9aiwyocww87k0wwtyqtwz2hyan7vkmnyz8ldnbsxinxbx9kep8ajofladz5ni84l5xrw","34":"zvm7or3haiy5bv03njeo03vs0f5gk6c4nnloq5vmoaq3v89zgm94350n7o3t6p7cv64i3igklsylnd9beiqp9dfwzgphffsi6pnm","35":"f0wzkqyacsnlft7g8ytcz3v6itbvrssn08pq0jkiufoyarteuemej4qgfmad88h1pq1daph09ovmm5dgzotrti0lcj8zf64sgb3p","36":"cubf6nafgxt71f6lr6d1b887i4r6pwscllaslo7pfznjtz1ijwf1e3hl4idg1d27k1s0k4hy5k1snztzkqn535735e66ecbhxq8l","37":"exyo0bmxxrd7rbiq7wd4wrta6rlc9x4oyxa09ctq259acpauqa413cpmz28usjg2hijcvarug64vccyvg8h6o6fznrcex4mmu95z","38":"3xlf2u73l8yt2r2v1qpavh2e81yyejgahelmeje1ckowmtlf1jqxiddkpj120cw54yax77jdk724qyfluvjqdkbinjd5zn019x53","39":"nqle32172bknqrze6zwy4t4jaft6z14kmnufopk1dxygrrw248joqpd2mzcgb3wt82g7mn8kmfvy8qzdfz0pjf4eych9jyjeiau4","40":"aq32vdqky0tkvfii1vyxsh862czpdwi8g12j0vs8ssdqrqbyc535h356uuipp0sljwkn92qjqcj3g1miz6tnymm952uiy3yz01wa","41":"dmwfuq186j6dnedbp6rsj0ge3g07xg66g12udj3unku6va3h40c6bpmueq5cx00q0ysal22q0pjzia5pujokm83xs209eorz7tk4","42":"s4rv0mwelpvtg1o53vr9jlwkqgnb9cb0omfor8qxc77vgjlfluedwzbnzd3cgbgbbkvtyjfs1y2gkjafwq18mejsfvhu3kxt9kj9","43":"khxgqvw2q6wxpcokidcg88gpm610ig4cs6dephbeutyzg020jzabmnuitizz7unv3c8d0hbwe535y8flmargdvfs3r9az2r5myg9","44":"roc8qqxdmavb744qd6pe3d2cv2dti8og2wcr1yhxkyynsbld0kscrcc82cgsuay7x5xnu4qwni1774dymzxjvxwhtw99d61bngdo","45":"58l4eccws8nldagemuu0hcp4b1tftohobh07agj4yxk7a2czjycnu41ohxtn1mjl2aeht7u99cwo4qqp7wvxzlcmhrj84amu1l1u","46":"nash8eow6tb49k1m1nhzodmnvhmlz5af70n77yefa0g9xmhlh840g877ts6kyahedhjvugf3joui44ntpia9gzxt00z6jipfantt","47":"d7nhmhwgj7vghf6qggkxw2zev1xs2f95l7h990lxoyc6ktdvfe4gsqx0iscqa0t2mtclhjps7uhz1an4gnpe6a0ba0u527dgfawi","48":"vb6tbv7cazdydqvi8qcnj1fadfbqcic9s8a8p674oo5jjbe3wc05rlovz6covhac3qm3jxonbycv31wlktx9cspetuqoe93k6hcn","49":"vnrsuz93okofehv60xlv0f4wht3whoz4asenw87yakja2luioht825b9rg2ry64vvsyuzyt2bhoeqnz1wabyhytaqujbgjhr3sog","50":"fmckh5p1ku854hz1rah2poma0rw6i7y2ylyboskao82pd19h3vvrsd19xlw4rsgiubzfa8q6j6nskiznxomilsn82lsl394prxrx","51":"6lxfcvmismjli2x3tbte38gq0ee81zpen4lthbmgh5htvccyz8x82tjogmb1xcivmon4udo4a9uwr535ltxhobojjtej9yjv543a","52":"1s21v23ajpgy5ff51tie14oz0zvhrrvu19dmpfo6ycb6m4p478ihs1onut69ab6959bn1mmonmb5vqw71stp70vcsc5fo037xgbp","53":"wfwla1k144s3s2xxecdb7cs2ndm3t5w6y4zhem3mwv89qypwzi7z0tkcacx6jj73721jr8sqx6uplhwojooqv20162pw39k8y6i2","54":"cek6rjkudrcdbwkti0dd37a40d4t44gxz2kf096vr6ff3nyyh8h6mrh1fcfplkrck7u7c0pzasr4f17tx0zwrs4w9ca0hizxfumr","55":"oqt2q6qp31qrcl21qqfxq0j9v05885c3f7u0y2voyg2kotoyuhjzaj6xlj5kuyj7aclvyne8jirgwlghplpkb38tacyv1yqxhyda","56":"erc8ma2ql6nc8lxs9cezywd515dayhxujopnt8oj385qp9shxwca8zqqin88jabsiql1ceejufknu1gmzgp6h1cv4r0ukj7lw1m5","57":"gwwulnmocf1kcz35xho022tsy9ec1j3wa1sa1m9rlnwt6z7rq79pe1zehwk4v2f6p6gizlbyeaskbgufqps6lelubczgbv7haczy","58":"nawt8ac0cli65jfc44itfm6xw8rl9y3tvc4dg7rir67imzn0reli2lzl7rt0wqb2hx6bsj0fi6fdmisiume4kp66e3tymbkv5qvb","59":"dy80gxpm9sooohykpx8cejnvrn1bz5vqrpzwp3zqmuvj5lepavnx68j12v8ly4hygnoe31aqp16mu5r5tmvksum2cpl0afc0wyw2","60":"h1s7tg8733ikglzojmq4lla16g5mfixje4vxq0ajp7sik6wuw1ctpzm01xwa584is8bsrrmv5s2pvy3xnyrslnh2jtd09uqsp2xs","61":"qwn8ksa9vwucdfgl71dicvrsv4sgh4olbfvnigd62jolal1lu22ttjebweg3nabs0u3uzhbv5bdwmxmtmu5dcjmiymb197n5wurq","62":"xbg0byt7xmtw6gjcbsyo9lvzg7amfl46reuzpzpokrt5vo1gfle9q42m6vw4vaoqu1b0h9fuupwnebkl6ish7a901nnwh0b890ly","63":"lj0xzdhvtnii10ikqxwd7vtd1cz41z1cx9fcpbv494zltl88dj0g7mzhlbygt44bwqj4xoxt22e8txvrh3ocp18rodwznbgyt63g","64":"lvvnygyywhqqcspmbrn1eas0uqfta08pbjhqhmnxywwr10kyqb5tv654k9h6jdhvwehbdimqndb7e65roppp88vah2u9zqf1eqwe","65":"iwzxqnfa0we8s333i0qo1csyptswuzx8zstm93bi8u4trl2mme5rsxdogfsmoez7fpqr8zbka3pnet1a0z8eqiltfb55ag2wrdyc","66":"2bfivp8qdt3s58wtykddfys4j3ypossy3arlvjwbp2sj9gd4wctbjd8xmr8um3z2xihw9l5c6w0jlj373ks9wdjtb1wh8a7yhjb6","67":"nufqjuqrnzeyx4s16jv4kwavw4micmfevqvmi9uffqt4ft960pgibsscpmg5k410jmnv6eehflc4jqow87asxll4f4wi8h6x9lar","68":"kfb02ufcimu9k7x9h5u7t2t08kn65l7huhe4n25ukuzz50ik11xn3qk159hzfccfde86cc0j1gnv7w4mxokynv5o1augohcykogp","69":"2pq2flugd0gfs7mlwwkeikvwoegy3a1i2rqqmhae3z5g8plwonu25q9xm0bgnumfq8o2w1hkz39nd9zi0wqlx72fj8i8ykr3wfa0","70":"khgghtv7w7uw00l73xes0hf0auabq192tjpr0vjhkwrthexbgnoektv4yklvitb306wk0rs6hqr56sy6rc8c5wonfa7x5e0dksle","71":"67w6l85ln872nsleeet2vht06cq9b94zfoq44ilr48nqzef1c8j54xwtygwebwmwcn2yi4cxpazm7v0lyjs7ldvuc6kuvl7kb2ks","72":"we7ylh7k9hy1vx0v72ljolms34vetgzqotsuwv163viy2k5jf12tx1jb9lt5qhzyp48pa2gtolyck4b5k4v7vb5xf0ohbtk9a5r4","73":"ndg6ufbv635xi7ow63jqlgr4rqqdvh6h082rkz8ui911q8nlfxh1vwc1zat3hy642hepuqdfxzfe96bcvd2aano4c7izytayd17f","74":"71ploor208vumkdgt2xshtljz961tjep6248xrc5efpm5krijwqtisrxj9ms50n8uemiau26fhp3rdyopgzjneat11y02cnobrzh","75":"z9ebfmy95tikhengt79kztvht0n156dev010qzvw201vs9lg6f5vqrn3sbv7upl6l750a8co385xpfblwdlzpndigh0tl2005tvv","76":"t6rozbasnvi30kk9za0jieyy5ff60jdv80w7a3nn8m0mr3y8vc1hqu8rs1s302iwq5qn3am5sdmny0tg4uhrsc47chhfbk4s2xft","77":"kqpowkkv2pdq4c7y4kpqz3parbgv9v4pk2wesfzxr6ssnlhpdpdrh1zayke7gswtu7lgmd17tum34vazwgy6p8c2evwhzwtxdkix","78":"hw570z97ayivd746etnt9aatnne9g9cnfbtrn7qlfvwns6pfx5lhx811aslmjxy3z7ht3rivut96qb1rx0n0qvminwrbj8wgc4wt","79":"b33snb68uejym9bvckbto56vnyqvgss6axbfi619x948hn7nxvurltj4zf7iqub59585p5sasw3kno5t58ot2rm3qewgpfivr37g","80":"h20v5q35etwdsw4z05j8abjx5qny6uk79h241u0mmg2s1xdtyvtbpcd6i9w3b4napggmwhzla921i7m5ki05s7zftaeoo2vqba4g","81":"d9qiueg2lv78brwmnci1lloon61j307fhcsrnh4s1artjok4q8iat05foh0pczziuabx1c0g7sjzmh152n5ojbwgh1iz1z1xbf60","82":"ie2y7y8i2xshl1u72l9jex8ql2n56i5i27cogue4lcmlgjo06roxu7dbg8b2zw07bpptms7pq0ktkz09vyxdru6gdluk4225luga","83":"2mx1x5jc2eyvx9o9de4izikzz5cliihjpntdd1wmemzkhkufzieri9fvgjmuur9ky6vszw8j5z1pvq5vuqi2yjm1a4u7tb1l8efp","84":"5w2ufrqbi9pahyby6w1xqqsgtea53yyg4tdrnmo56fcb275gzrt3s8f9sctfrovjcg4exdodf29ksvyqnoh5je289jl92vcuc2qv","85":"7jd2nqzrong3f1xei3g0ytl3qap9j53ov3nn7hkvmyvrb2wnbrifmqrw3him28l4rfs2kr43r7kf9cwg7xacecgdhx5aw5to6c1w","86":"91y3qy89mitteonfotvif1vsa9lul5d0l7n4h5bm2rcu4lf7mnnt3tctt08y50412g88scm2n5rch377qg1rksjjus0atbbxefph","87":"lywssvp6ojt7pdrrqp0cqpph6d1qc0l4oopstqvgq3n520b9gq1ovlk15m298lizfont3m7g08o7w1zxq17uu1q97i9fo1bfhcfg","88":"w8ritbk5qlnv93f5kjaoojrlgyo0vkfluomx4qgqipznczn5lfy8bjqavclww6861d1on3ymn23x7j03kp0pgjrtxk5mii5iifft","89":"9q1rbjidwgzwtm46wd1ngnxjaq0jeolqnmb68hmotj3u3oui4fzmfmsj89c31tgdfsys9uf18p4nq9yvpyaynlzurn5zsbakua5m","90":"5aa97rpo3qs2nwsdzieiqzwtz0cheiqnwapivhq6t6slqwaki5e8utqc2lp91a0l3og7rgq48xvbrrynj1von1aa7l6cmfxu9lju","91":"if2mnsvvb5h2w0t1r4zmgx1ngph1n72drqp8evn7g40lo45zxsgszdvf50t8kk6rzd3ayge3i143rcmx43v7etbrx7lhsr5xvbug","92":"ev81wrtb95922kbs1ffgolvkmklz9na3ka9nz91chasajeuj38c23l02kpelh3bsuk9b53siffw2hw5d83n5cuoeugyrr6a6pu9a","93":"t5m0u61zhkmqp26ongg41g85wpir0o3868lk5gmh1uk9ijp6lfjf7v1k747ivtpelgayprg2ah6ud0zxq7421728apgezzbf7war","94":"bsxlwh2eikokss3yc8t9rpmyf37bt1ojvbsj0d1op3qavu73pekx3k3k0x3xl2prh35f25tk7j5vxoo66f1oq7cyulwp640xxvd0","95":"ur09m5a9sliiz6f8zjhbzl4r97xma384mzd0x2n54pn90s5er7higdxgohqkmi2964bw95gl3n7xijli27j8bn4rntuz9xufjs9g","96":"qdn4wd4qyh6s61gkhk3kas52dynw0hvggo202hsfw7f29k3t0xxse1f52cs8drd7dhxa28szrw42v5eg6f6sw9eiinu6wu9n0rau","97":"g7jnpbm5k3pfbxgft6wzh9onf7temjqamstaf8nuwmmk4zm5os5yru1511oavg2grch5n8p4mzdmxs62qmbeijabazn3ud8tsgfw","98":"yfmvs91kw3brgcm2tido1kmzvh8jx1s28yggdb0k8pv7ehxjsf2ymfphtlzvnknd8b00ano24n94luqd6tkuzlhbseld0wfwdg6z","99":"50bvqawehrwh7tp0brwrkb7bjql7rx1y53r96pv4dg23k2dfcaxytx1uhhcl7cfhucys7epmshumhtr3f5rx8twuf8blgqnsavzv","100":"n4vgzqcbik4a57o99qeydl2i8l47h1fleqalqzbvz1atnzs1kr4taud7uim88fjwnyljz9vajufdomjn83o3xrk7t697ys8rz64l","101":"rshrxc3kemqvxosz9pxi0finflmlubuyfovl02bdj04a4ur960542wovv86ay39y6g9i22yem77uefnl4le3jpsyo0yicpony6ga","102":"y1cdt4miizwzyztn5hrzdkc3qyqe955wbe87l2modj7955vdir34ikk12euhyf5iwkalojo5kq1l0wz2kvpd6vnibsr0fpsym00x","103":"p1a143tn8st9brom7f0o5iuvzn85mn57fgk4f7r7f94mdsxia232zk69cxprm874cqyijqn6vtwbuhtk17zs1ajy2pb8c6pgfl0v","104":"mtksiir2wnkvu0sytgkxn8fk5maso8z44g1s1qwsns7v9s2y5hnaq7k5uen9tvf24ear0j9hg3lnv9xirbn5s9d4cgsy5b0fgsrk","105":"26pk3x1pgyrh876mrm9ur68mwm2g8pupacp75bmzw6s4td6tkkbxp18q74hp5385mb2tls59yjb1cea5xeqpvoyful54peaffn9k","106":"wdc33hhknm0s6xa2ms3c933315fu9tlnbftvqsmlnb7cgj5sozj88cafkggxzcv4a68uw1c2y3h8ju4j2g1tl5y96b78nl5n2dzp","107":"dq76qb4ddn89kbqobm3jdnze3u55yyv3b3ob08hvl9wzq0fter0szmsbi35mhc9b7uy8qqikrn75vmt1m7s4hmfsz7q81s8s1j2f","108":"ku66tgz9onoezgl93yzlnvc2ogsncj6mf5ppm2lpsft8qnnp3hdelpqdbm780mnoyhvgd3a37i0pkfspcqmgr6a4zdqqhc0kn5tz","109":"981llvf8f6sy848kxqvj93ynyewu0t5yqsfzj34avl401dp1cb8ynld2s7l2twa7v811aiwfmjwj1l7xs6bhv3posj03ygk34m7c","110":"pkiaie7k5fj1qbt6p8cxdfloi5n6l8z1x2i9zr1jrz4smorokglakqggt2zbc0wtig1e76vuq3l09rt3g93nobfes5odek9nzv0n","111":"lwdw7f1shu67x92hqo54tmgt7m5jvni7nopejdajnznqdmx2ano6rsiq9q1ojf6z2xkg167x68tqjh15xx0hnb26ai87vri90oap","112":"fuclp0s7g0jlsxligsjfqmvnik7f3dx7r2g1mue6d9vz82djb1520a2smurfziw1xxgeg6pfp3rm9gt3x98kmfm9jfu6k0ep1mot","113":"5oa4993yc7wrvk4bqxs0evalp7iviqtjx8lpdy5wksqi4iw4elytm3dglazy5rz57mykho6n11jovhhsus2sjskaa4y427emej2n","114":"95iupp7zqdyyj1d961pf4kyks478456ubu2elf6wei7ttky3ll4aokxwhn2xq6dusu4kntesdvdth73w5fvf6urvvra6wtc9wxzy","115":"aefa23tkcavwvcrjyir21kcpzg00fvs2mn5nin95gf3zp71k2zlbzp0y11bqfsm6bgc8dya160dg48oiccm2goi1ieb2tduwn0kx","116":"vqpqd9adk380670hj75oga83mz2dcd8682gbemlk4ds1gxwfmeym3ioz7qnyrmp1spf3w6ieshl6vy90aehuw5t1qu2iz2chz6md","117":"yu6we6kux1bh6rgl025lg307uewyhxcl62hvf2di7dgofvgm2h36iloz29tk5vgalpxgivgutqcfpvi868hl8o7azwydx1n8s1ui","118":"gg39a0q2apf08ay0kc7ol72r65d6d0fz8yd4xojqvybdm5jn4cfklwgpq0mzf8yve7kupwa7ibfvyx2u29gaf7u30345mwgx9t4n","119":"vc1wo5p00i0yonwqyy22o025zuwa20xa0z4sdnf7rw3ados9agi28a4nwanyymrofdq150hdxdqvih1ejdyp3hnu87nbp7rp4vs3","120":"wfc40963gjei71n0out369i47nb8txlde4ltkgpp80dj1rdczyjqbxzylg3up27wukyu5kz75hmd9vp77l9wgb7tu9qac9otfr9y","121":"vvthm368h5pj1lry2g6fplllw138p4ttfkhw6jspwvganr29yya2cunhzdgcm5y3w5br78nji97flhdjze17rrcdpi4srkbrewve","122":"6t066ihtfxpby0myjh5lydg29xwei2flyu104frvdng2h0bj7dtztwjuy9iefnmksd2ah1hqh7w9fnohibrtytluw4qgbuhoy737","123":"20vxvfteyc9sj621383iu0mdnxk6wuztxacy9fpwu3fxs8h6p279zoerz5avg4mw00znjpb8zopg242yotiidkbf68ugonqn8v1v","124":"vypofg2ckee2wg1781esdljnby494qd3ppwm31g8vewl2nivswnjdo2p06ztwefbii0cn3avrzfkmlyj2h5o35jzl5hfvl81crnl","125":"254ljzbhdykiqi9lpk75q33my0mbbkfd1l3ampk9ftdmhlj9tbgdn89sohxwavk6ivl5wq2oxevgxq141854zihvqxn5vrxs6lus","126":"zbvlq8z3t55glru9z9msmqo2r3velahmzvkuwlry08js98xuf4ed1bbbt18l1s10mugbxhzyxxcv1qdrk7n52witpco392skvmnj","127":"ira6h7258jnob5mpulirv9tc4qxetqlspx86uq3k2f77g6zid0e1wnmzjn1q5ddavil1hgcloqvmuk71eudlb4e7syd939uxkt77","128":"lyvzv39pxfadodp6jqohpt43oppxrjxiusiwkms4lsbc03y5p1jkg9qk4em9j1g1vp17w4gwaqt13xfqlsbstjl1zdtfi7h92o1b","129":"ob6lxyotqm5l592nu6syzsdbyi0ulcxaxpa7mf4p1fwxoc9oo4duur26xcfvckljj3jjwfzmrn7z9ts2r09o7x7av5ty1adovayk","130":"7qqf40tfz035w3rne9yh71pxais7vxq3k30j75v67ibcxdy4eri3y8ui63knejx5jh7631s9y79yqvp6atmmchayolm5zxaoxl8t","131":"qdwx94xdcs8tjae3gl2ckknwgb6fcquj353tf37zxeosjw2wzhzlllgrbfso7udfvxw409a78f1l9phu3segx6kuyybzuj5lmwzh","132":"eipoy5qe89eumkzj2us7f91it29rpbdzbog676cgk6681wxy64qnzkwl7abwh9flom04ocrqvmte7il6sdxz9t46lanffbp0wz98","133":"auh4qxynzq00kvt82hfyrdf2c3i9tqu1r0e6wqob17039zah0hc8q21u9g9klo4wwi2vxeen4jx1m50vo08j56pndmuphc49mhcr","134":"p2ioa4o4m7y0dv27q87co1a8ziknj8zwta7u7r3kztb3r6ajtcu79l8f81fg43orh4q5ygm4jee93h9bdmjg03b7686ghov87rj3","135":"ryunmeyyfsnczf5lu5hs5hvle33o59oknko7yunfcyif55nsp6mn591ilgro2ka27hmgu33n6fnextreyomlz3cwz3egl30gbhg2","136":"gnlj5uhkg8ihh4e8n1y2xj5ia00w4cyvtcsimc9c7j57it0pql5o2f10yiqxtbt8uvb32r3a22n9uib4m79tczfmlgvlzj1fvorz","137":"i0jgb8b4e183if1el77bt6830qr1nu6hgpfis2c0oc55ktn1i76gsf8bu9ef5n7xvrqwgbr0klp4tq2x8v493f6rddj2mpl99ffg","138":"llaop10i5w2l32rb6x1bg5s6nyk0604idvhbvd9kxpyoubwvx897vsw127wdqnbesnrz96xx2q9xo8zips73om4lg9t4d2o8r4nj","139":"ipsivxvo4za2bdlamkoi1wqyksc9xfhicm4xkfyqv5b0ts1lrr678pbof1sfgodneuno4psl2rllxucoi5def8cgpvoo29hvrpfe","140":"bzqlwpqexie0lvmnpurps5byzt8nl7t1ktvx9piivnfsb4pggaxtzz6qiu4znbwm8d2ruw26b2xvuswjvveo8gg2y7ca29xlhnd5","141":"dysj4859se2zciqa2tru6coy3shpadzhlzdwzdmb9neuflj606z47n0vg6fsie0iwy6ecdz2kon1p05pg5ymf1roc9wrvktyxyhp","142":"14dgb7adghlrfh8j5mqe65d61ezpdnnah1in87eq1itqk9uunbw64xjvtnmcf5k6b7jaewkxl2u40pnjdstcouf2sjvjfrb9n5x8","143":"uyeyp1wrv550kwhzcjgjvse9b10hprmxepe41qdvnd1ijyqz1kz69bbrin84ppwk77r8qgbkwte51bbjj8ohvrgqksm0ziceah17","144":"rxp6nwflgt46peyn17ly1n6istpdi5l8gmxnfmet408ssyumkp7vfmgeymni6560qtq7c5qdtdvsems2llhx5uaijjhp59ip73s8","145":"lscf1gesmh0n7hu33e0qqjwse50ff4t305ia9n74yjjrexmcyndhd61a6acv8xzq2ula47bnjns81gn8uvf2p1upqkg0z03n8qj5","146":"tq3s0jkfqz3102szln1k53oknok6s1o8w4mv3t8lify1kjmiviu4wl7szk1i0t92a07fz4zix8x7am0jiahzcjndiuton0mgk6jr","147":"zdf00kis6um4ul985uf7uiw1qd0whtztt0wdzzwxla6fv0vhej4xg5i9c6sdva99tag40syufjkjbnk6cd9i643iko0q8m98pb3j","148":"3eqw8mdsc57tf0r9jjhb9oec91ek2wc8shqe37650wksrtpcf0zpzpvsjrrus7zknnv1jdo1poj7vwuii6zv0t28qug693ls16ud","149":"xxgn57ac4bbpfzjzhie9ai5d5sxvel3f1k2b4b9p58h2xlxtxm7nuduutlhq2bjchb86wdjwjlkkrmwufi3spmiem6u8duhk06nj","150":"5s05qp7b8g3n32ngw4qje26zn5j0g46fgkthzhv6wbgv98lc79y072emympkrnr8960swcjmdo9htq7ocqq9gmbdci0ss7p0bfk2","151":"hh0stgfz8j7mhefdlksgcbxboy64tbhfn3aa3a9f42h93gynpaobe4zwsfzj77kkg7ipon7t41qrq65rg94mv6j2wtvywi37ts2a","152":"0o6q83bcvj96kzb9g6p52w14ecp36wv99slpwbvw9s0k5x1gi4o4ws17n7gjhcz1p5m1juxcrkvthmj2tnj240xsrxiwk86ir4m2","153":"wx8y710o74ei8q05i2xfx6sv6x1jq6svtife2hlu8atv6g039xv2szmlog6sew9o75pzh6mnxlibhkzwxbqlnvb3vzkikna3gtra","154":"rsu4o5fg5aqb4so3lxhqkz7g2z9jelcxa1v0wixg9afzczfdlsdr4cgeyws4n9jgcb63sw7ki4y0jmb1r199pbmyke0ie7f1mvg0","155":"jk1d0uvbwrxnq233omqsj3yegrpn5slrj0ayecjew23xl615yrghpnw8p1z19rh1cg4jskiohpt85civid9qwsofn1dvpt2z6yfw","156":"mmo7wp1sqxp2vg6qjqk8sump5ho1v8yi484r9jvhdp5qrhviiga99cedt6ghmoc43uot3f9lcudiou1nbk3p54rp9hv4693v64x0","157":"11p5n80zir9tveeqqoxpcs6lk97lwq8pu07tsuk6ztxh6dzbhbidwddlvm6ohik96m3hulmeny1oaz2xcoltysw4ruw7evc3jz0u","158":"norsygqksro5t9soo7o4r6z9aa5q473ohxx6t2twiok01s0ujigk6m0mmctuctp2kzm1il235nmuzyts39j8whqo1ejnknr8nklw","159":"ei4a1t02cxtiwp47372eukapcuaey8axg3fio41rd8dpv0xlpqlnk30ettrpnkbvwsx7q8oixp02lwuiv6kiqk0k31r8ztof7ql3","160":"x5um8vgsffdpv4sk7vinwiqrd8dau2wmo5ln3dfwioet3seq6bmkshwql44x7lz83ov6cjh9bp5e05utglxtg4ww7att12x4ipd0","161":"8iin5zqyxi8njhudrfo5np24jhuywla2ahozjtp4dfesbxjqlsm7689kicibhtq0re0m6w6u1ouetyiaylstnycw6ps1u04omg98","162":"mujm15w253de7yzbh6f1eejyz4w41vjqcu49zum5e9e0y772itvygc0ht3pgll75a0w5qqd2vicvl3uziouy3uh55h2jf6nlv0hh","163":"qq5l9zlz95jzghjp7a9qjf0yvj4e7z0xefl49bgqnlpol1i868t2s0zo6ww2ztptrn9g00puhc06n8pwk8e6k0s4m30bychnmaph","164":"s6xzfglc56drmw1yhcswffcy9ed3k8c20seuvhonjteha2uuw0x21fzxvnn6c2wbwp8hljle1exenj1f0x2h3xodpq90w0kuhsno","165":"c28rfhxgc5jogi72abnfe68jksoj3cfyrubx8y1geldeakv5vs4sgr1ajk1hprwqwxovkdfstxwymntg63uwmo6bbqfuxzwosr7u","166":"bbvr6ogo1n4qbov9c258giqkykki1cbvbgcadq4jbneqlklft39zi9pxo8pbtwpxpgso5vctmj7s5boh2gy5jh276fyi4cme4pyb","167":"tewrnprftjspuwiv0f04u48ltnvrhuwxd282lz4257dojeu9htgnbjq1v5yap9lvo5sno4gxxuehexvukpmks0rtt4hiu2vbbi0h","168":"xg3gdx1tyww9iki7l3i2pd83zk1y4kxxx4qpxr1yaw5c2uen3nep1fzjbgvp42n4d5gepvtdhtkasf2qu92lyidq759pjawzblpk","169":"yxi6s7ho8coddoh31dp364nnmw0oe0kvd4ggvi1jiseptpw0vyt27jrai4z0a26gb61qj69psy3g3290678uou8e6kdzdxcwh713","170":"u5qy4hia9ww2e02m6tpviw0y5ax31u1to42g5qeti5t7jaaopo9x3izvloqjyo2d87im7ios6p098uab6ok3j3eikmfs6lggfhp8","171":"n4dpqer9o7uoqggp2lf9w2ghgcs53s1p83glyn3pd1sc0lj02aj8ljisq6ax5es0pd61uasj0dltf6qgygf9338gp59ipi4i2kby","172":"l37u7vgl3z0im64afsyimnzxojir31wrxp8nxqwfywiw7m3ncikolekmvmly5h718ik89m4oon2b5h1xva8rl5j8hcff8lslvqs6","173":"7hyqq9kopaiy2dg0aozztfx8v0a93w66e95cnzo77tsb4acl8yz8p7dyxgwoy3ijdqp4xtjs34v50amid2tw7cfxbsog7z2r96w0","174":"9mbjoifdaws7z5le50b4j45ompjal8nb631pgykp6gyp1ilu6hk456laa1pe38x1f31hlpmc9nw21s65u9q00yqee74oi30l3qvr","175":"3tw7r5pblv4uzw8wlcxqn4jydyhyw7r5wiurtyjyx9qu7ssf8dozj20mntyhplak35fuzi5nn00le1fzi4mgjg797at2fzwsa7e7","176":"1bknz0fu1um6ef033sc8tvy11f6w7vj1915rk114uhrdslvkrohnxs0cpjzmc7s7cxmxplx9rjnnxkquv7l1p51jayoofn8p8qo5","177":"wblmdn0ifyz2ys7fc3k2cqmsk2exowqb0mthd815fniiztx3gt346hopvoxp3g6x32chv09z1mo9ltdim6v8zwh25fdwy1c2fom3","178":"p0t3b5tzyjz3cdcpv3z1n2r5g3isr3qhmqa6hvj0xc4z4r9aag1kp8908vn5cvmig8ywc3qrxxs69pmkym8xqbm48p3yu7dwzcz7","179":"a5qqa34wuhxc3ruu7n8b4l5qm2wngk3anwucs5gg4bqkv0p8d49up9f6tn0aesao8sbk8l3ga6na4b3vqgrgi1gtqiwz3es6mu69","180":"ogqlfc9tfmilnxlmw393ql8xosgogk8x1w21oooc8s2whfot0phrjcfjn144xflkxmg1rfu01ynbzfbqk3urmeqrlca3swvw2y8n","181":"p6ftcmd57t2oi7gtbibqz3n1kbjvgmu5ns0s4yjcye2kqvvzrwdrrpl7vruvz3rnjeelg3adqg6ug03ak63pq5akspkdo35yeo0t","182":"9y1hz17xd0keubs2k0f0wfoz4cp8kl41obfw3mpg9qq9lqg7dpps3lj9gntwxpf3m0nldfnpoiacsi2y6uptdjw5hxd7mpe5tp4d","183":"6hkatfwu1iwxp8uj1udxaf1nxq9jsqsa0ruoyepd2nvaty4ktu8f0di54idlbnc0zmt33c2gdcukli99ses1a4e8ulj260toyazk","184":"12ey5y7e0xa1sv6mi2oug64xnf6ahhh2y9vs0svb6wr0pzls6gf9z4i0c8lz1ba6lif895zqzt89tugrzxnibxtzvdqq30ny0evm","185":"oozfionyl4jwntkwvvcqqvhe0e8b0alk8c9dnzgemzyj306yjgrh35r0jp15su3lw08s49i2adr0qr1ci7kpokxn7t2xit23y9bj","186":"6b40y9eygp22qjxtpr42tqmdt0576jvuys72yndp34praz4avb9ahhkodgce6mffls9ixsawdx4u1375yorbtlgieia71ffggjeq","187":"73y08uczc9n1thgd7trfo322tvcbb6e4o6kqwib1d1d50xkcegxedsgtrzp22pjulb8144rrreie521fw7ngcn677lb10z4o90mp","188":"gzh0von2y1ktdw6g51rsky9k4rls6wckmnj1152bljzrxmqeqvxxj2gm1dep1zhmd2ca0vea12wyyv1p7005px5bvtwn5nx5thf4","189":"bm0ctfspl5yqm2g92iuwzdwpj0i49r8nlbjyzly59ni7ifg1xwzuwzbgadvhx1p4sue09hikdaggw0b2984kqely37q3tmgurv7m","190":"taox16teqf3fta3xl4s46wdjypd9g3l6yn3rm5ftxlr6tyuh7ncjyvkovxa8vwau5w1rd2z3d88kcst85q1m408xkkr9mw7jnp5x","191":"r4n48ua4bayusx9r17k9st1puvaxer4mcbnmlnw4gjutrh4tn7cgvy6vak6vwnqeo1sux9z3yma8lci5vnnfmjoyhdl9bipof63j","192":"p4p4nq4m9ustmfaf7x7j95uqdv0vgbxwvk0oclqa1p331envxhe3gr0habj3gd20f1ozbsnn4gakkjh8ovgqiq401w9xxtlv8qa7","193":"aszgfxbuk59w273zpe5kjgvc32yc416r1y89i4ak46j4mwnrnz2pyd7mbqcw2pb4h01gxdgpo30j9aykv8rxpneltkk7yjr2r9di","194":"o2pdhzcsv1o29ktad3mbfwna3dig5nk4j0r0vpvyh373875y7rchd4mjpk4mbj6rrraot7h1rsf02jp1qxbylc67cj93upa8dczi","195":"1l1qvchrse3nm8u5a0ds7pkht00y44gct3x1evjh3f1y55pxcs7dywb86iepi4yabhbkvfz5mahrgrg5idk1f3xbidcnc98khw6g","196":"copf2pxwpkv2voaiovk4b4p1ur5kt1pok6kl88declzvm1f7m50gyp560pg3wijp6ufwt5ywkwq4qlq6z09v0qbusdeqytyhivqz","197":"6j6kpqvccc1699j3pdc8empzdbkssfkcsusinpnb7gqg7mhgviul9plmufwi1rotwvezlbd0jgtl8pn1fd0xz7kb9hxiu0wwyx98","198":"vvcfq29cal0l0hvlvspm00cge4vyajaakq8iyt4xspdugh7whcv5tmx71hm1ixkz6isykvugkdb2491sz2j6eu2p646uvfjw4wbq","199":"0avefsocl4t22ufq59kvpj164drpeimmirqwx69hipwsigz0yz00mtjrl6rxsb40b842ewfcqofjmx7gwbond4tqr18yu6ouui90","200":"n0totrxt01rmwfux09juaeandw4awgjuyhzwmm32z31soae21ghngdq7wgas8k4ltm6nxykbm5s833ika73x9xs68n69olso8xeo","201":"zwahegre5ceimwekcetvlpuyy12vjtnvdj8grrm432jk5361gxp7cdiaspxmqo0kkpm74uj48m3crbrsfw3jqu3uhk6ahpzlcib9","202":"t1omoddaixp78pyk8rxdrvdl0umrc50y6nob06c5jyvtjv3zjd6qyuf522fqzfm0p7tygolkbn1zfcgnwjbi5lkp4xsmwl4ha768","203":"nr2watu2zm6snip8azqpqndlcr1985e2sun6rn5t8l4w6lkh2ojkguy3opuowv4sb4lx746jdogjxw1d8v4j5s82mc8n8ed50ymu","204":"1npkepf55p91uetzwmastwx4u1mxb6wilbjjgy7brjfuqb14xdypsit3kfpa9rqq5p4ydyp3osg3bzuegrr2caakww8lgls3s97s","205":"gdnwraa3cc4nu0mledhaa5tldua1fjyes7xb42u0uom7m11funo10pc54szeuy8i9a7rhi7ju5d8xx3yuzwvpq0uo603owmfb6iy","206":"iwcvx6ika5cemtmxuqsakgr0grq40bsr1lrp3b2hzgqlceemyv3zuwcgre5t5b7pis7wurgzxfbjpbcg1eoipf97se378jgm19cy","207":"8uaj61vb6yov0g0r9heq1r5db3wije2fhzqwtjwntw5lrdl7k8y0diwgixe4b01r1ximpjdkwagjzo9stq01x2twofo22qpdllv6","208":"ia27xs3vklx7mqzmc55pq5nake8qwztzfpnq6nqmb6lkbovmlu25oybz94fflpt5ggebsxfhu0qdr5pfpyl9dn4kp1598ymyvc4x","209":"3zstth9y18cfg1golz59q04mujpfbiwmkgqfgp2l9yzwgie3r6ctl2xupee55isl7w8hzhsb1op90ocu4gb73vuhniurczma57kh","210":"tnsk31k8wdtkktb70vt90sdsih8ma2yifta99zndc7ocbz4zk8u37braj17q2gec0kak590fowz2dhqctr374unvl6q81nwob818","211":"6jnlv2jwpr6ntsoy2f6xrslwf0bngcy9sjkfagkut66191y861vv2ixvl3mjx6ec9wcdz0hoqrg2dlizdu1yygjhhtwa8fpxxunb","212":"taresodaf6olvfcd50q6bychop2r98ts8nl8zygdxxa3okuyt0q1arzj66n59relkamnfet4oechuck6j8wj4y8ec50xv0i2ldei","213":"3n7knb90u1k97qa0jdj6mixxatdoqkpxakag4q5qv7zt6ympcorch1zpdatgqgjflsff0u3p805ikp5p3qvh9h9dokpq8wv9ekm0","214":"bkqr4cznpu6vfuleckdmud1bgveu938s89rrvuyvyrciz657lpbsiodeuxcevzhnyttxbqwnitzi0wlvb51dq2apqdoz4y1hm0vv","215":"mpo9lf5alb0mj5jq2tx14uh7kudqryl10deszheu40omerhhoauts0hf84ak2q90q2rxhnoi2o73ytojf8tbivpnv7akuqv4ci7n","216":"io6udvetempw6y0ufn6wbu62qf46wqmzdzeu7zgnfxtmw2eol0yn2jgi3a3i7bx4hqdymaw41l1w3oqd2jk3lyfr7twb6pi2uwah","217":"lmsd7f1zw1usugeo3cylv5yu4cnzm71cchrxnamnt9cnixw5x6l6cwiue6oqzgf0vuntcyaz4d6y14ivt0w96cish7dobgefk7i2","218":"09i4zvl56r63vuoswh5pwttziumkvggol5q3nd57g2zkuv7nfvxkgy0fgjoe0yjm95enw7vxd8z1h5mr7qwgw5lyittopdegdnwc","219":"22kapnn8cztge0mcwywgw918wuns53gwdznhs40cdseq46f8vj5pzbtnv8nzji6ng0x98yowwdzrrn0gxqi6h5pfdl9hwl3j4koq","220":"jg8rdmqbtzrtviejcu8xpfcfzp4ozxe468sw2o626x6vrbrxeawmcfouw5epu73uz94o7wh5q3h7marty8pis89t2qkpmab6fhp3","221":"n5fbgqd0qx7r2yrkzrm5liwf4v7zcksko2hawjfooqd8zh94x0xdgm3m76akqyc5yuahpkpp9gdcfg64dmkiw1sb0ryz1e2xdhbl","222":"kd5rnytjdtuwhrd0ykn89747cis850ko6gs0yuhrfvbms53zzmpvdjzwpi2cq485hzf8xwwktxj8llw1nf093eo62dibllvmpqc8","223":"81eb44k3s9ys1jsrf0g6cm5doptjk8wle7n0akdyqqvy0wlukjrcjb7hn53x660blvouyyhcq977vdjrumfltnf7pnsmlh1hf46a","224":"6fv21cijhs7nzbr1lqq5e7n4h27lmbm642lkkkegwuw68fkm2dhvv6i0oh3vysnie9f7yyqd4yu0pzw011mfqslxmcf2gw5f9qwl","225":"yiijqry8hwytdmus68ovqdqpae5s0xoti1flga81y5bh62togivwuls3u54dxyxh58o8hnjsdug016kvx2jrrzqirjz43mjryp31","226":"u23rt8wc8ngl4qizth3dutel0fok7yzwcxd8ilee6t63xi2wwuuoy1hykffajjsiziispv8lbdd68dfject0dq51junm57u6k0kc","227":"193ywst9pjjiq3hprgh1cu5kg7rrd27m4jrkck690x9q2al5v42qrispfkelpk4dqb06dkplmt2o5u2djlwjw4llme8mc8f7ncx2","228":"hiov41beoaxc173t70bunakplv03osl6p06fbdhju15howp9mce1vo7ormeqsga2ris2r7h8weqgvoywwu1pjo0sk3w2r0ztswzt","229":"6035xk6cebnjdzhai3wyprk7bm38sfnt8jjplzhd5gzt389mybeofq01cjrt15wrewo3gx0rnhm1jujc2xfb5r6r9p2iot4ywiu8","230":"gqc05ebl1devdzh8g3rkdpkta2ox1bkv1h52rco89zr63aajxle3nfrxjocdf5ht8zk7p1qdfv95svh6zof8yh68tbs2pe4why7a","231":"ujtk5d9zbimmde9njin5jenstdvaaxiuxa65if599vluf9q6h0qhig7pwc4pmawwyg2rr9ybnbp1l56nskk7wwvd1vq3j3bcp5uy","232":"4bgbf8h5b1mfqx6mz8vjs1ni8am2tzjibn0j6eaisdkmemy741s0gcmffi2qcydpmthgtwhf4tlvi24abg6rtxjn2i249in0slq4","233":"i4ur9rgl4b7kcfeny68dekqslavyj9lz0ij5ll653keel8a5u0g8hmbrdsob385oksquapt2b04ly2hem5sbzyrcfelhjujriqgu","234":"d8p494ktrrnpvbmnoef44qh31wihfr10s8ohg6ubueqftt82eq3s1591h76xd1odo0o2u1cky8c538fznw3u9dpfrdc0ae9om40m","235":"2gpyxo41echmtzl6h78smeqjgo60rholodqs6yrpaq5n30o1nfpmfasd7bffrbhqkg0jskajd6g8vou3nizy04lccme2d6kjslea","236":"wbqoimou2qw5h4kvrq3e4gdbpbwemtry1va3fcmnd6prem8bjh8ki7x2vllx2tja8ykx92k1ivdijsu1yk562eif9a0bnfhwkqvh","237":"mhed39q7p3tqhvqmvgd8q83x2rrfngg2yt2xhyrt0v1uwqjnm9b64zh622tm3vr06sgoiwwpks6bk9qot5lp4ywhpaxkop3xn57k","238":"w3rrm6gersrxmucc3irmkm0k7jh9oi57xdyyvz3qir5xwktq2grbuvyzgti06q4w9tolclan8wppxrjp7qc6hvzivjbymzv2o30g","239":"mrq0igvclfgn6qvkmlhmt27pj1z9whjq3kpcqoz3rtx64rf89hx5nvyq8kvg5fghzlvysvpjg9t2q8umpht54tbrdq717roayol3","240":"giyo3qlbprv4wawjcekwfqhg9nt797on2c7k8zr87rn58mmru4gweovkg9a97b1dn2muiqjysd8ixaovo5g1p04vqealv40we7ln","241":"zokxszxcq2ur0p1ahunfr4zrp6jazk2grtx6g8r3rhrkfrb9p14kq1ebftldyer0gfc4a5nngli98o68mvzvzmxxwsvcx11lt8qx","242":"6op97cd95c53x9fubn3si501cdg85u5v648p2szlnn741ztykgtbx19gp554k19z2s4pbl2w05a6izsfegtu9vkyi8lp8o05l4s2","243":"xc6ste9w0r2ruyyi6qffmaxk1rc5vx2ia4hqqh6m1yjc4psldtjenfi1g3cnq3xf899gqp6s7i77p6hpv9lvcut97ewl2437l04l","244":"dcsxx79kj5h6rndk5xoz4bh99xjua6sohhlu3v2dzht1qb4zbc5hcj13y0sy6qlpzax69u6ikigk8xur12stdturlniv1srqqrre","245":"n465xaamzkblvoslmnyeptie144s09lqtd92g6j5ze31ujigu9x67zbfhl98masihmezh2dw1oxs0udu4eqnjeuuchvv2u3jm5m6","246":"hscwsotlvb1tw0mcckee1srv0xjfhxmxt8fngsh5muv818bq7zmpy6v0n4wbyqjnj3hprrmxydnrezu5fyvmnc0rhvk2wkr82gss","247":"ddgkdwc5ctn4rge4dv8lwg0hf13o6zxuws4ket4kfq2c2bml8z038kg43kns25gznch9cs4w7m9wms3ge4cstlzu22qnywqi8vfr","248":"ddlqfrcq17z803smwed7seah64qgftvlamw0svpxa47le16xezcus1i2mdixywmsj00y6u8gvagp18pmu6yiv6526v36ea96e6ji","249":"07w0tr2387j6mwf6uzhd93k7gmb8at1eqpydgv6wqnyimlzmb4a7okh2kgyjsx1cv9632w26va8gmeyc6brctxbutvl7xjb8pxnu","250":"ot5nnmawmyv5hrz7nvcaue2ceigumgs3b16pzg5n3tjbgdcgpckqfaxmphg3jmuo6e9znvlg725h5wdkzrur1oipfv14nzlznjyn","251":"fugi6rsba8j43an1nqm5kpewa9yraxfvc7he8a8hv4f3c8ervlx0tyufvkj6upqrbndidmllzpyt8j7okpmpiblvyrve8zyh9jn7","252":"oro5cwpnl39jbbdi637vg82v1o3relixb1fb3riu9xp6ts9nlnq0wab7fee2gpy57po5iheu2z70muu7mhajrdqgt27fihc5eq2m","253":"nt915dq1mrghzcsfbxkdy8yf36grwioatp8ymdyzcql6d0ycaf470ljv6g8hs20sngjrnhbjwd58eye8bfsapo7xf61v5xt3ncjd","254":"x4zpunhsly1oxl7ukzfihopp4y52f8xi74g4lc9z6k038moz4bqdjptemtuso49nbmhiawr00gg945us4lcl0jo359xlqvsguz0i","255":"gqzbdy2v3v0ik8ox9xdzuizkp7c1h1mtww76jrg76pzz0soj5og8zp9dj8928lsca6h0lbgtgfqqvdt97v4w3hwdzzf1f0w0xgy6","256":"ad0b02xcu60txwnycdhaprv7urjpkugsj274vp7doxf8n1x5bplxeuq1deotrptl2tqcabvzi7ghphu0ok0jpg3coxchu4iofr6h","257":"onoqwscn0gp43q036bikgjppf0ze8ehk95p3fwllltwqcqp2shfvf4d12ksykxmsif2n6w3m8n34hlkpuzhw6eoq17zui4su9dj0","258":"sgyi79w1hsirfcir1480a0lkbek3akhx6tm2qbiweym7bmz7m47rzyymvl460nnaytmhjnj2cal0jz7y8bdkxmq2htob776crp20","259":"tu6wi2ad81jdjpjjwmvmw57imidpajnt9kayk0bh540og822mtucvq6t93ienc3kl6p5fu03iq1x6943yxgcehfxdbkymm6r0o04","260":"sl0jxxelpwib759254lsi79g7oae1s4nr0if7uu1yow98ukbw1feidcxfol5pomym1xitjlq6xonnsjbp4lw5m8e6oj6957brb1z","261":"w5s7x93m38pukdcctr00ghztim4341wc6pr5a5qqy675b2ft3qc7o6dlo4kfnu3iqji7tkzs2mtz0swb2fbi9wbmd1ytrkhyxt02","262":"hyved1bwr1iwmkavp3yfx8c1e5oq33j8glxawkzach9nvoz6gyht1g231kh5el8yj9q9ls58433c1v2cxl18gb9k35mri0ehjmc7","263":"74804nxt7ryr1zfnjahoid70knwbbrjtbd5yyiuigo6jhbsg324s1t3xj1vhw2hdi4775hbh43uvx6ld183dzm3dqi9ty8ixozvq","264":"v0ddn48n4algmzvw6rov3nijtnbnvhk7hvlndac838w9xh4mb3v7zavl34273dkksapm4c97vzj7r8rs1akywvgcn9skj4wd5kad","265":"ommt727rc86pniqjyvrcd5384ptimc7wpjw5b7beoxfs964hjtzi5sn0m3nclx9iwj5lamjf3wo3l44ka1vxp7tfo6dygzp06g81","266":"zt1b5vqubh87910bqln6r62mupvlqpvzhy0ox0wt44ru901zkdb0rk1comc2cicws7fc3ip5t4ayvx4xo8ao2a2350jjj4p296ac","267":"la0qxee68yjtlg2z2cq19rbo5s0ih5izknsv96yc6v93o6x0o0h1tvqmlwdkndvt8jct9rc4w078ca0cyncgdtmm85eghj8tp0a8","268":"olygxlhap90cjmq3v3xq2ztvo37z3omsqkdicqi0j9x1bfmob4vjpczh060dm6w21adrzrdu26ido49lpp3htdjpgneyx7mf5v1i","269":"lx9y6munabb3bxzubm0u4za9f5cdwwzmxatl0zdx2go13m91fmd2uwbqb79x8tarz9rez6a11q8f5m1d9irlbr9b5p86jl4gguf3","270":"4hgpu6ql3hnd56qrojuwz19apvo315w0g83d4x71v0bg1fmnga4mx67xhmm0adagdyb09gqgza16fpu97ihc580g52rlie98rt2b","271":"uxhtovmqb73il2xug10plxadcdboxx5iz261dkp2jz8mh0umbr5qj090ggnczoj6n364wg91pdr9wwf9uid1fohx40okmopm8h8v","272":"a9g64lifplg5ew424qgc237y39xumwrmnxg8fn3kip3ifgas1maicryywyqm1l9m1hfx5a8lqp1hug7h5a1u7ic0313zyzdb1m7s","273":"3al95nyd2wnntgwmnnq4507vhpsz5j4jelbqg8ycohx5knwg8mp0rnnvacqwr0tq6pa3t4onq4l8vb6puwcuymc7djhqf3ia69ps","274":"9ji7l0lryah9ej3q2qaodk9dyj5fqj5ojjd1clhcssxg3kr8t76ounwckhgv22em1l82m18p6dmxg4zwh4rjiaz4g9n09lzywsl3","275":"qn7o4athcmquuyxqys7g3g9fxkw9nrm0c0fsal5cd8vtcrv51nr27uiv8zya9dseub33yuyd5m7x2vljtgvb8nxjb008a0t0jxeq","276":"fajei46378uoxmonsusr1ku54makmftzvk7ay8dmwf954wvm36kqz6c4m76q1qs7ivyqkhig2i8lt8aqiuvsrcr884rsfoaumdsm","277":"agysqhzj55s3q2368hqbu1685oaws0d62xxz1aymq96sa1dqm1e84o16jd5nah1yaq8urb969n8ngc2hlt433ai4sgvl0ctylg85","278":"nc1nfr1ugld1hmej55lobh6fqj6oxkhvywgi795g3vj5yaqfk19khbg2b1sdt6dk51qfkc4phrffdsegleyr5k6b9ryk2kht10t6","279":"uf3sxah3fgjnlhyt6mt5zsmzqeplxrzf6fuvl5o52h6a0u8i33cxr8fnl8yfsrwmu9pvqugqddiy6l7yws95atluaxlvnhjxt65a","280":"yk4hhulczm9wnaforud1pck3xgwjfoz5za7ul165kavnxx11sz1bebjxi04h0uxtm1xz69xdssm8t5l7g6vf7s7gkqr7pwgapceo","281":"spnajuhu4kzzefdaql25hy8xuvi81jvl0fslmnqy73o6pbdvzxdzegfmdx4og5clnizw4wjcmim82dpuv9j6kbpyix8470y5cics","282":"9xalr9bbyx4l7dj25uns6ejmxwk4eekl5bbm2a1zr3uhl4bl1bdb3p5y84gng6pori7agtacrsksbydav9ettjvolvhxi2tj3fun","283":"pp098lligamo3l1o00xyrzsdb7f9aorw82ucoy4gmwfzbj2lf8hwqt48cppnidf90r9d44esyky4qxd4xzw52mqkuyk3ktn4hemk","284":"fy8z894cxdaaegsxqkrlwowsbecagv0dc9p5qc76lx793j8u7j1kq1mxosd13scn69j9jxbvb423tsmcznuv7x0gblh73jlt30kj","285":"fsggcf2z0a9g84ntgttte9nq4m1c6urfk1wv4hvh7tq84wthcylp0ooboovorl52rnru02chgrvzfgndn4cb4k1l8cvv9meqltlr","286":"266yxddkwyp4kbb1stqeauo1tot8u1tbxz1j8pe37aliw3895hwrwt2ij5ozq0t4r6mdahtbsa20vpo1kvggapk0303rzbvufmnj","287":"botzitjf3i4hv0x2vkgmo0ogeulrjswl8l6ytypcyotmjhq1pe2ct1ol993w4t3mv4khy6zu7h6y7xflgd3dwki49zuqi1wx0hbp","288":"wku24ugphzwye1czch3pjwtm1ilvuiqlpkrub9hxactxsyofd4me60pvro585hz41s17hp5jsn4muvlky5pp37tjmro10htsnlop","289":"fnpv4mz05a8ollbkx15orh68eeezxrtem2doio6gu61tk1knuzjf4yrc6s7qn49yzxwjttr2ldkwvqfysg19d4y7pp40u85xgu36","290":"douy6rp6ahad9tdukcb9nt1wqfdmy4nxbmo1h1erss3bypgew4fqo87oxoibjckld6ueg0t6ppvbjjgmdy10gicb9n5emlbgmrag","291":"el46w0yojmnjtnxbwkh5rorzo7mkbuw8lg6emzyuki5jtibbjw7ynq13lzjo3d6nz42ik82aiicu2zicef9es31zvd6fqo98vvtr","292":"ko8j1arfpenkhuji2khk0as1u61zyid6kgxx3hyy6irzgizocua81dsomwz5er838tpeamuhajgyrlz0elbz43cahmys071on4ak","293":"a6biniojx4iehr75ujz055z5wrz8wj71sto35r514xsvby6p7hp2xls4r1i4eiq9grei3qrzrix7hm8pxf4tn8zvkomjisihe52g","294":"ha3q43hodotlx24tg7qwk6zp2wef25qrpnl3qc0g6gbnfh537gdmlvt4z2pubbvsv66lycrg1b6vruhd3gu2tplgwfc2av5325lm","295":"y4if54atbult4op25a32dzjufb4gmxuv8yl8jl58iy76nc99o67lzg1i8qi5sburqyiw2qr3124hevf93f8mmrbcmkd0m4w19ku0","296":"awk0gw46j5b34vjuag1w6nrrxuw11el90jabn9hy0wmrhshhz6h52stsxzkxl2rux115ny6j7rrx3hilioq5v7w1d9inaep894js","297":"y3mfc92usxiww6dz94cyxwagtgapxgrvxgd8wgkeu8wql8aujc5q93yfv0wfm2s0zgll8ui1h989djy9ho33qbw3targrhciv25u","298":"ghk62vmgvuktv9izo5d0kd6uqnuqp62jslqdf6uu5wtg2ubd0p8goz40qnpyrxnp3y7mn5nw781797x247yizqnyoth441nm8kkh","299":"5qka0xnwhmneyeiexmy17upf2qtng3x826zggpyb8dszgizqm5jxdxuj9myfikjkrii3ate13f3l7i23ih5jhvhaneordytdih6t","300":"mmkbxr8a14dggftmck9acko7gwvztsircdghwx5fg3c4smpv61pqr1xy5i73jcj54veumjq0t9dwqot4i3kwbdvsr4ccutnus3i5","301":"nkephg57di0viv5jfqr46f3j8s1zhwvfw1j58r9sun8c07780xaq7o0d3narf5poyyu1f29rlahgxmmv3z1lvh9rfg9tsnuk1oc8","302":"8r7zlckmcsuj5ex4zeh5i9ngp4d26gk199niawg7iucvgv0w0wbtx4c5w719i39c6xqt1wx0qm9zhoifoog377416lh23f8bbjfl","303":"j9y39v9dkbxhyqztlxhb2wf91o699xnecnlgfbqt1r0pj4byzdjaa847938ufsciyop46usbxuj9p1n4eh2bvzojpsw2f9st5nhm","304":"nxvt1406oufyd9mkzdz3uurhrdnsqs2oca7s56y0y7gtjx28rhrgnhugm42ywygjo7llxyf7okcbaq2enudvbbajb8f1pplqup86","305":"jxu3xidnabkl9uwemu0ylgx9ofywf5nle09kwee351851uhdjwpyfj3je0h6y95ompxc2rqmkj31k6ijb6w7ovxo35p0s167ichn","306":"ggylyk9o65bag9di0ah9k7ojihhnnzciu9jiobkuq09psxdq5un14xxlqby3kz1o5xhvtkylun7tu64tj0hxxeuqzhzmheco5jim","307":"84cm15fwt79egxyz6xde26xkkoxq43l80gpe6ujii9wbk12wgis5t9lsromlc24mkl89xv4bmm4a760h6llu185g9gsfgrvizdq5","308":"5ow0sj4i42w9hd1mehl0sfcdlw4b5pauuf88tevnfn40ejr04yljmrba2l8ylcvvhdp3xvx9z024alwrtnjssl793eeiomydhly2","309":"a09yqsohounsgtgotguhmb7b4jot2k6y4x36u0fgviiwt5skn3jerywqckzmsnbsv7ppp0avj186sldple5o7cim6ipqyaffhnr4","310":"jd4h6k5skcow05dkwtu5c8cvb8tttkgocrjhntlnw0i36nah08u0lptfrycy45n03aolsk7juu27r5es6c6c8jaj29uqe0rbdh42","311":"7smqsgux61oklg46rc4tkxrfht7r5b4dha08bxs0y2yl08vj90y9nt7iruhdlnh910h9r8a9fsg7kpx917v0op8ofy50m63e9lq7","312":"ipdhzzz2ar4g5eqj1fpkrmdxcr4gh7hr07q8p7furq09jgqpp07odfarq53urjg72113wm6fm4u99g2nlmld6hu4himyb28ftoi2","313":"neqwuvevi30txnlvtga2in1dx0z47xkwabsc6mmp6cjk4iicy83o7648lj6qdjia9nau0ziz14hm1n4xarty7zw7u5mwfo0nezgk","314":"86fc49yx3206f79cfm3tcf6gwzj4qo3fd2offemkmoajxmzkfflli7b8n9hcaragv5p63vqx0fenhqhtlto1ja7xb090munhttn2","315":"w0jhwj11zo7bz8riuu8pldewgpuu572pirllykvoc7eb5171ivrnlk03h9h6a5qk6byy1t5zuth9q44yic0kjrsistvkx3emkfpj","316":"2ct2kf2xn1rgrbmj77erhqk1k4926etcjk2439bxr55bdlyj3r59g6jah4gnvk8e4cb7wwe1kefrscgocfyskyeuphwpw2eyiach","317":"5ko62r2megjctr65ekf9avenfnijl6d7rrb2ktqlw6tshye33u6m5ewuynjrnvkqwqr5iuufzhk9eaefawm9l08feasxxr6wgjgv","318":"j0lfkupk01zhnls0cj0au5zsnbvsx8cjpvh2qhoqytdkq9gc5bo5nt5ooe88n3ykj98rpl5g0si4t9jvujzmu3057ly3xsc9c72g","319":"gaseufrixyspsybukvtao1frsf7ykc5htlzcwp66vs0otsshsqv3c4awco92du6mpn0j7874ng8zxx5hr9ws9lsopszeqsd9z2g9","320":"lgubizy2kyf3g1x0e17u80ytwy3ipajfisd6b4wszu3vlzcsoqpq1ujwoiswmxbo5mnuozfeh5ov17m6hcuv44neodjkzfdwhbsk","321":"ewkowtovxd534to4firdhcq9f2jne5k3a5zft6rm40w77r45ylth1alooa625oags8ledoe01j26ukv5z4nx2xw4o3aotbrw03gn","322":"00q651pf87xfvtrwfiopx0nbj3z2ume22v6dj5xsysjyb9yi6zpkvgohsyn2q4uq4umj2kiu9csfbb0522n03py3gzsmf03o4g78","323":"htss69ku78ovuek1q2bmm6kgax5e3340q0cg0ixjmkru90wi5qqo2wfvd8obmeg0lzp8pvm3ej55bmznj2711jkvqyb3am1518qc","324":"6djvxvip06br4q2kfudl2fh4aq5wfgyi53p9zgz5v7ltkzgi94eytsv7ljxoe69cc5g3pfdt37dv00nwwgdo92wcfukngt8u1kte","325":"0udb9hobwjn0pks13gwz1xrc75nv2v429g8r710bc03ftwfo4nprb9bx0xrrrl4m6afppzj8wyjzfaf7353zh7i1r1y7fw1twaom","326":"x4fu6h4s67jkt0ut27wpl5j91h8v5adupzw5kyd0vnjkkkhzwz8u7udrtnscbhakc9tk7gvdlteq0tzn3o19gb23poagmkgl6kyb","327":"ckypkqgnc5rljrntoh7rlbr8tsoyl3klschd7wbrgvgwfvlhy3tpoefwmjrnkf8ca365kh5jubdsvesvs41pb9ko8sb97d432kxl","328":"bmgd3sudjg7nkvcqbwfp38n7koc90w8s8a16wsd8mq9bk1baig41fz322hwow7yj68usg5ab5d7ds8ghb1jwiogmi0bdtkuhu4iq","329":"4c0qrtl3ninj3wrea6truor16t7jrqzikiv67uza3t877ppkp7q5t4h9k040cv7x5mpr27tx51h2ib0oflgr3q43meepiklfgtzl","330":"1bgyqeghb9o3zgioyd5ypknz7tgk85dc5ir1fshcnyxlhpyd5lt7jomdn7v7s6xno0lj64zazhh51b7h5ubgzs0aj68nqs9jk6ib","331":"e0uokc0jfk77tzvp0cy549mw5f9sd5qwa43e7w042vz9cjn07lluo7awgvjnyi82z50yun5pv9ntv6o4whfwkuuc83zg4ox044ps","332":"5m8v9upuq34n58o73qgh8sfxt5sabptv1jxiowg0nvw8lrzx6kk6haggmyokodx0ivnthrn4mcq7dwkpgrgoxprgmu7se2hcbelc","333":"908resz5htm6m523dxgabprf7dcqwyrbu9t8cd6j4j4nh7cj7035sge3oiw6bjvo9onqgyq5m8ha15ltqr5wswwfmdr6kezlqt9s","334":"niho65zqro82aoszbf5yny6yhfyit4s54wd2pk3p1g4vc8x403za2x5kmqsj0i86gvqwqsnoh36iqoqxyd2hbly7fw4j6chvcj2j","335":"psuo9zsus44q98idts3djlfjfpbfjsnoxtfshq5ltubroyhrc9gvf06ij3jjmfu68ugr7o64smyhg4zvqjd5weaj8kuylyvzcvgp","336":"6yq4va0ywj6j51itaw6eviyd944ug8cmboui2gdzmyzyzgm6p7mtuqb4t74d3tc1r8slc4vtfu94zei9yjvdobv5rh7nbvfazpk9","337":"7biir77e7yjqj2wk9rq0nesew9gbymotgw6x43r2artyd00vonhg0ouqqjl70n1y4yc3sjsmaveruae8o553cb7wfjnmgj2p5nlx","338":"hb6engteflht93o1x6ld6minxwjw7al0l9kvfaz6oxh31vemd5p3hjag12s1imz8oz21zbhz052f7n51ntb4iz4ndlahjcnltjl6","339":"lumepbqo9xfyqerwyv47u754vsc4krvpyg7y5kiihi26wy9x3zlhvn9e4dqvg619oo4x4lyb84lxa71ydb8o12xbq9nglitod4zw","340":"vdzqdm4b74k9mnsi9s7rs2r9g19bi8syabrgxuskyev239882x70tmtz41v6ewf1bg8blui943kkmlc7mzroukt2i5llw5jbovt5","341":"0bjxxztrakvnh9n8z1c6ma7ghynhto4ojzrzpy4h3hn4asfbe7yrzq7svuzoghvw8z4f9kv4cy01epk0c8hltiwyqirelt8wxbz7","342":"w73vi9j5v7pep4cvecjpi3e41gqgzxrpt7l2h3q9qazliqra2cuwy7iejy97jei99bhthljyeklw5vtvx6ohj0f0z41n2q7eipq9","343":"vs4rmiqakaxdnkzq5ffny7kz2eypko7w2ybqj7sg731d3y7w40w3i7c9vv2nekdsnb9rfx84osvohai6jmgvze54qhbeidbze3ae","344":"gkxvudif8im7f18ajn6z6nru8ic3rv9d4wghue9rsfvsn2euirticlhfmp7z6gxrbqpqx9cbw1qsuympzvi4d8kukwmyao5rsll7","345":"90furtlq83xnr87fqc57853hum0n9jbicmsrtwtf6ro07grlgwz9inlnflh9tgep93divvmxiwcrcxthiybyry18qkb80qs08pma","346":"ggymo37wltus38pfovqtosv9uudz37of6keg6sosoh96ahndn122iffpax4beqiesh3qyzpuv7a05uay6udopttby8g4701mkxig","347":"24a8f83l8v3u8ua52lti9y7t3idz7o89t3c0sjjybogip99raq99dpn7uttuhq9drvvuc6le7ug3rer5yanvsdo1ppvvpk141ld2","348":"yg72a89cpnhu6zvz6krniuc00apkd3bile9w3znraop2fykoci8yrz6i2yz40wwwm52a4vuebf78f818ycw4z8w65dwwiu1wxuuw","349":"f0q6we7pkh52ouxhzl7ady6xua4wzsj2ouv3c9igwy97kunhwdn9bxbycpwaam3lfds30n5fwkkfiwojp1yko3q5lep47x7u7y33","350":"4j2i4g8tpwcxp5vwysjr568qmjxdf83zkdvtairfc9h82qh3k024shosckjycfj8gis12oiyeco5axhmwi9sns6q4i65arp08g7u","351":"tn5bsav8brp78rkfbuf40oel7gc4cbb26kg1y4a4suf0j3f0niqbt0wbmgmwsvrdugiaa148m780l859edwmlxh1pyju8715gcv0","352":"4650kxhmis3f7bxit0kdhcwmav50ppjqrw2vlok9i46j98do1460i5474z5jhsudxh3u0i79brs42kau1jpr8a62g15owv1cjazy","353":"mj16maj1z84uw9vak3vl8u6u47j9jwz6j1kqopm5pxoffrzh8fsuhrgqt8pguqsar49aua0kdf07yg1vidkjjrlgye7z2ul7mat7","354":"64h3e8824o3ajdgyo3nfa2i8yhbgdnpyaig64249w1aa6d65qn21ssww3c3pozs7lt7afm3bqm9gbod8frr17wxf0yy0153i2yrv","355":"fheo3nup91flrfo1wlt5f4qssabh40schhswik2lgl8jiprsluqsq4w5vqyh3mh59gzhsva350dg857foqc28h6ot54ufo418xux","356":"upublsqr58ph43ykpfogfalnvhssiib1a2g1v4uf5kmy1t24uok8uf2nwm4zh4gn9kmtk0o6f1fmme7u025pzjr0ytsj1wu0x8vx","357":"udonfzun3extjswqqnmk15ygv4h1sdk9rgyo73aq2bp0bf53u01ah85eztqtx9o9qbekxx7sqz0v049dya6bo8nipu9z5huuy239","358":"6c3aqje1ungh1cn1u943ezcnxcs7absl4imj3czgxo2urbiuhrbnlccskabqcw5t0ssw6mar38obs7abpcxkeflk4z6s9o5zyfwa","359":"41cvdg3blkcoihuo9xim85mkihwbr8lmasssnauq1j6bhel84mnilpobrmj2nfmzffxugry7xk718j3v0xwnc1uwwdlt029be2g9","360":"3orgk0wpkukkgiabte3nhraceszpewklqkbwk4gxvqvz2iicqsherae7aspa0u6jztot0jioc3vhe9e8640kildm0anq8hk6g4tm","361":"rcs47al62pxn09u8k4nyjcijppzbkxswdx3exxegykvwyzm3z58fwqppxnh8z1welwb8m63m1kxq9tnd7blv0j9q3qiyq1et7hxx","362":"574btostqyc93pe3spalqr8i2fmwigf89rk73euwdmtqmdaap987c1df6geuolh2bnwf6ascdb4yjux9v8y7hainuv7c849mub1m","363":"7z5y2hc5olj2wcyhjsbzfuvj9ws0jegf4i7xp63i0nbjamphpyctlsi3x49e7j7g2qbnv45urq4y6sby19e0jevuh2vxd1l5bapz","364":"grw1dc9exe3xjh0bzqtazlkijo7t2q9nbg025u0p9spwnvwf6qbngmjnqioakw7lc1ksoiapciekpviwvgj3815t8egamnvf1htb","365":"qa2jakmsangnl5buglagkx8hdl4vi4n5m6qn9w1gyw4vt3ishdov8wlzcu2gbzsleozqa959o9du54kh6wv95pusqe8tggxa5yxw","366":"vexwze5vxbz5ule4nsyqbbrxdgcye7fv2s0w8ib0bg0wqq2r2r099al3otyeyzzk4u9ynmuszgithbje8je780qctdbzvgeeg5q9","367":"mvu0pwvn8mz8qp3valdmuqh3q7lx6vcv7fwlhicrdamt5c76xjc1xvw9qbff8sypg44aicore7gxuv4q0f6f3tih14e9wfgrujzc","368":"7wbkgw13ikdxrac5gg95rrfiybbk9jyikqh5uam0dpfirr2l5lo0xpd2pipyovj21h11xb7fldik1z1hvpubesa79ho7kz7vd2mr","369":"xz514qvhxqpisouq0of0z6cc37tvib4lhq6xfqfiul7xkfgk8c6yfyne1a6u9pxyr1mmnp42jpr90nsy0fv7htd2l6xyhb4prk8j","370":"928hdgvmpf7f3n964l4ex44vn5a5wlyrhuus6mb03biazftl9fslktd3mughfubbt8ak2zlexiktpqmzs6gszp1jn8al7ns4sq2b","371":"1nai1mmqbp6rmwa0dztin200k4ag601fxpofdpm5kelcwnkfc2qliuxx9fxrixepwk84nagepc9u7o7oor5huyvcn5quor9yh8cb","372":"u71dzywnlfhuvllz9xbshfwl925nk5nok9bhkeyfcladsll92slhu1te67ewrsdagfen6nagk74a503bmkpwwoq6mh0e3poqy6jy","373":"uk7wpfrj92q54q6kzro6ony4fog1j5fn50zbv0brr2vly0pinbbv2bshu5ekpgqks9xe6w5rupzjvfl3yrobo2evcduuehhcs6jw","374":"uweei4v3e7xpmdobxn5a6put2ih6zm6xlyxerupyktwl8rltoyasscpp48p4dteaz67hdpxp40k1u8vxqty6im55o4kamjyyju84","375":"5tiybs5izql4p9n57waj94y57ha1rs39f5koa3ompthxg9eluq2dgjie86xkevyqvxb25ooyhz44muabnqj9mdz9naumxgyn8qiw","376":"hpkxklug93n2afa4wa63d2a6gdtma9c67tdlvrtysqjp69ys5y6rl0wjmxl7p8s67y93pknijaaqx3y7fmookfsu32nmg8xw0xp5","377":"g8mvnk8h607l9jmw2ycme53ryso83ifgxmuoub0ojnb9a1p294cecrjfmd5jtz615p7d6vw27aot7xkohipp9a5vpykchw54j25c","378":"aiit1xn71lyvh578a6gda10uc0slk0p9v5bwqq0olbnnhxbqpiilwafq25srwv98sh9h3pzv9qiqgfvfg1rvesx5rrbt05bx8faz","379":"t0nw3ez8x42tf3jj2fdvywemj8lmhy1ifd5jj6fn9zpots0wioq8xvxae58y9xk4yf6ryiencls839wn1zlhbu029ungyzzrsr8o","380":"hh2d02oeojdiworiizi37m0fqli1p6xquzkss8jzmh0t8zkyf09tzm3uo4jxkjv790ybcwtne8ichjxvx9viwt6s1scdjg24qo23","381":"knerewlittk0je5mms30h22gcgoh2sy40rkqdhlekmh4urs4rm2z99jm9hocvsvclstkmjqs40cxj5dyc0jz8pgdyz4y1g0wjng4","382":"75e113l0j2l0x5rmm8u4u1ruuk83c1h3l4hh645snnp742ub665xy1grbm5m24qt7l743i5mjrbqnzv48wc2obl662vtvtl5c6h8","383":"i98m7q9v08tgi9606459sibb7qscp2ezxvxyg1p8vflopxerjrgqc7d085mnwi6pgxz89xv1wnbkk623kwz3z5iipoee1mwi2x03","384":"mvhs2arp6oznij66n0q5mu5675bolyzmfkzxybjusa9dbgqplrzpv2y3jdd9hzgol2xmfi9h4v6ra62c5cejbpcc7pa17kw07f8o","385":"mevxw2bghrlsnepi2fiqdk8xqi67hj52xvqmq0vce09e23mz13u4gsnabll71gqut4l8a05r9j03augxfz89a6rag6rgnb5d5ebb","386":"ak1rz2jyt03ds0i4a5xl447isccm97sq97ou2b42i8sw48kis6mfbi0qnxz39685vpum6d87ilsfyyjjm7gz1mgcgescwsc78707","387":"pxv6oslksusezhr0yy93jdlgnoe1v9azfa8k6wpi21gatcwo35nuw5m1dwggyuik1srvwj5jt0m2671y5ktbum13apg46okrncem","388":"93kcuwzfvlcefbrrmkxitva4oqx70bcsk1kamthyaxi4ix8crn99gv8hzzb9mzko3l9x8kgivb7syfhdk2p6kk9dah25vx2c4ft2","389":"b723kcyls7kcnqt8zykbofgchsjupvkgng7urh1dsa5csgwrnzm6pxdsvujemf7xugk9g0td4p2y1nz3sxmfi3jcufa57n2ivjel","390":"rv5uyrdv271ne2a6m1efytfpbc17b516wgujtbwvexpt2tcj25tkhmmtco2moz63wk4649ytuf4wmpnyrrl50riq8kqe328jofix","391":"t3ik1zh2sh4v3968wpy8w19o6bcoo6ldfvxsfg8iexicuiaoesbrqgvnsylz083kt4si5k2mu6dv23knpmy7axib9jh8wbaccshx","392":"e92dcc0irnmgnj0ioemsl9yxbp1rzo1jex9clyoys5f6xfh39kvljjqp0ofw4xplx1ske5kftsqve7cziodcsrfvyb4e99i21tkr","393":"fpox5lpifj361nh7goht9895ylqu6fy9l1tlsafjanttewgdx76sm9ul7nsli7cd9s2vtmf4k47mrqk4x1j4ebwfuy2mn6o3cyp0","394":"yuwqna2p58jx8s55ejlv90p4mjw22dhw2waw53z3q3v502j7rf63begcicm1x0od9rzvrwm7rrw7c682dmqutn2evru39u1c1z24","395":"mn2xlqohma3ui82gffymqbewptpw2onwy4h3ylr2tizyh813zq3lxv29fvd3bm2v1z57xkxiknh0sfkdbdw9qyp413jmgzmpbu7j","396":"qcx44m5r7ld7bx1v0rafw9s25ccvn0z92mt67zxh3wa5sar97do7wuhjt156f4evqy4d7pyw9qsrun9ocpjj6cz3nyqpa9vbmbgb","397":"2y19u5o6yhmy29qa2tb00q9yhzef1vc5nk64t3j7zszg2ajjtjqh6ikb7cupd7l2ys740by8wzt5g9x9u98hkmco0kk3x9rwbs9u","398":"ai2a3pcl6iwjyh2560zvyl5jntx5l4ynq6r2f733ory7ag2ov4cxdl6xn2jon6ibdiyhagagi7j8heqjunus6dvvowwc4jl08ch6","399":"idl6ibqfoydgbobeb4g6fa1dzr7j59yaatytd0auneej1xuss5s3n8ytj4rdtj5uknbfffs6ek12n8qr81rc60pd6swnezhiy1l0","400":"6fe08pe0qrvu149taltnzrgc3hubo8ewhqtn2mioib1rv8475ha7kdn7ew4zjshnqwxmichguqzd3xgawlt53mnyxt5sp6s3diod","401":"womx7pdqod21p1z0m00zg1mn0mlsk7toc0poask6cugh53mvmg6hifreyq1g9zuj5jp2pq8iqf1md6iumlk8p30z0rhro6oze109","402":"97dswr6wcpywdxb6t3st72efi1fev0rntjse7wefu3wtttgh75ljh96wonjw3jpeo1wsmtc5g23apa1db7e3kvgauzpgyhjyohyy","403":"s8p0kgm17d3n5vidbqar6cfzhp3r5omsyoev4o4c095xcq38rbrsi99j4bxzj3ha2vrt5ppvj84haaoui963g0lh4069oqr7bpjj","404":"av7zxfko8b6sgod8l20f4mlbbel0fon8it3kcb29yhoovt7bhdiwo6ocv2gyz85wnb09dr1d19vtfufqb5n664xsjigi204td8la","405":"s6mc55lx3bt77s5qd9yfp4cqt7zqvwk371d6rhomsbqijsh8wi4m7c78v93m3m3od6l1ms3u0102wu32w1i38xhwvdrrwt69l8fq","406":"6degdj1hl6ihhdmjuvkvrdnakv3qfnexm2dog32jyy2x7zim3wm4yowk4ahn2g8izly0l9p3c26hpvtitpre1qz0nzma8165ruv1","407":"bqrua5fwc5izezwtjog7da8ol9w667zukxoy1b964h7dfseayjgnefwktikx0ax9q4q0h3qqzxri5kus2y0p2x6w1s4vekbiqfcg","408":"nldfhruadwxz4d6yvwzq58g82c19pq4beb8gsox407rx4jehmw5evbocg0apox1qrzsijm56zwqhwxrigexryz2tm872ssdecede","409":"s31tivw3h43bi75ba0oiy7whhhagzkdfvdzvtiooyrbo5ii6mc9e5oz95oz6igmu57pj2mityjj349x0uu8pb7oi2ajt5ckpcxbl","410":"17wc0pwd2j9ca6oqpc8dsw5mega4sibkcotv6bn8o4ockob9q1vyfirx3iisfhvrou35yidr9l2wsyia3o27n6utfg79eel2dpq4","411":"yafxdx08077vtemttwuskzwrdhvxpfcpxvsv8q4110zl1srzmc0kqb7bvdoplpsuno4sihs707z23pwss45onqd22nrury1l5rev","412":"wutclbo8hphq1al36tf35bdrxs6awaru4colkbd64s4nxyj2ownjary44a9z6hlf8u88h704iiqsk92oxe7yyfvpyawf98phafdo","413":"7jtb3lituqgyyqsyo9tmxfwd8viqj87yqmvpqm9pikbg8r99a75xvwox26zfd0m2kx8g4kybxjq14ycp1nzxy17m6brnf79ms2lg","414":"cl4dssn6dzx2ykjjlm0ybv42gv3s3v6f09rpvt7l557l7dtd64um4xmuzm11qcmobbp7v6n62ehrvslsngzsv6vlwvmiheozglue","415":"89v8s0bi0pds992kn8anezr24scskvhqbtr3ru0mdlujzklflpss7mk1dwtqrwomivxnjyr8602jzdhyzs8bcbffjzo29vxpap18","416":"i3bpc878piadmpk3859bhci0c6q1cuta931zxq3fcxs5jwok6cthokjkqhoq2l8gcs0ix60ix6qmmecgk8m54y1b2smvi2f5h4qc","417":"bhaadegfnhfvt0cvkxy570dpyp9uehxpc1idartnfzab16ctrlap8vceodvifueyrj0inaaji45wsh3cp0k57vrdfbwckveupzz6","418":"5hyk2xdr4jwrq1m8gc7u7qeh7novty16wenv36rwtkpg2jidcz2dnmiro0jc1zbkie3p27u9zo4lc4fentnadjh42il3r47sserc","419":"d09ds0bbonp9e614mxz14wmqvunh5wbg935yvow4ku2xqw02jnabkny09fn1j9bi5i8ozj7kykx0z5c9r180rctu7cf74gh3fnkm","420":"uyjezu2rwo1239e1yddlb1lvac8cn378pfg6t2di7dpeabs91m21742gnkbu0pfqv1jtbjar94m71r3s2zzele7i1v4wfuwabdhv","421":"wj5s32xrqhoq0sng5abhjcmgp1ymmgpgwejg88jl2w2p1wng1ser1lyyxt5torf9g0nmgy8755bltkunjyzin6x1zg6w1thyn733","422":"es9vqlb2z8d6c8c29s0bxvfdmiya14a6p15ar0qpecmn11di1jbr7oqse1ag5saxkoqxbdbofwzx9zkken5kyk8ivetcn22sxwpt","423":"gzgfavessknzcpxr9eieer0ndaahdaqmi900s005dymqcfryf5ih4nkr8sk3hrmvdtytk3kpxmvmj9h9xr0vyr6btmq9s0ojgi9a","424":"5lz9l0sn2t2uep30hjk2utxf9xycyzjpo0xi9kfz2k1efswr1ahh883dumv8q6k09eeagslyq1gfkxwtw1ubybuc76o1wddz2cim","425":"mws2hut7tibotflvwiz68qa3fk7aeyaloyoy6srg67ntd1feuehr5wiu7swsxegssgp7vanzyjqajjvnxl0ugqp4wug9ktuj5bsw","426":"vsmtgf5fip4bsvmkoe66u1qeydnvdfqw8vjviixc9cg8cbl0d522o86s0cp1hdomubj3bxy7ntwvcou56f990gr91t282avuifve","427":"niorezcolbxa8drw3y6p64j9iuzg8e8kzkk6t7ywfditqamoailgshh6bbor39r1vy2oiyw594q04od4jsxe802kh7tyqwathnml","428":"80p3h96htd9tuanfrpmca0m40aa0d61s8ubx9c32khnf8qlk8imhtlslqhbm9xctq3iyihm749zcxfvhxcbuchtsgfb8fsjlyule","429":"bplqzpn3rvd3cob652mw7yp0rpbeb74dx5461ap9elht4ruwlzbbbsk9ll7vu6hd7l3x296gjxhr6g4dxa5nzffhf45idzbgil4o","430":"trgclw6vb9xnea1owh72k9xfz2vdb48qzjyi5ryd663uth7qa2u0ajyub4m9gh5i0mvj4qzv3gq7itqx6ohw4fupnv2dqqefekha","431":"aitz66h50dj52bumpqiy0u0w0x28vyc22ck8ae2gawe1hha0u7hknv1htn5cpnu2y5ylrfxnmgjjue05gki1uwjgdxl8hd9fl9g3","432":"p6kvb1pddhxd3h8b02fws3s1sej19vowj06key26om95jloui5yt38njzghdhnn81ykcym0xawozcr0vx4s6rtx00uxp2p7e1337","433":"2bl4x9p6u740q5aph8u5nv4qiz3ipwfnau6e1lwrr57ndq6b0twccxw9nj8eqr4f5bbkv3kfncv9n3accchqe5vdco2ezb0qxco0","434":"y6ruq7wgyly8tvo3rzub37ywsl3ob2f9wjgmlpdqk9p7s1ypojr8jaox4xjyw1ooqkygkcfwd0lf12mgwzlqcd7lp3ieths59xnc","435":"sfxzveu7tjxuioe2jg15p5l129z22itqbdustlbkw10se807urxjup9ywkvp2y12hpepoz5v55yw95ns97x9thqad61ktlfw0oh4","436":"6olg39m448a6n5k649hw4ebchh6i5dqgqtsff7a9aedt2iqnhon2e0w2rle97frnab0rffijoh97uq57faygy5i8fvjbuq2edjog","437":"wugdz52uqk3opki764jannermfy6q6ouza4evvhkw4e7lvsfus8nnkg4lism26rwy5o0f8m8491sfkyo83szw7zbdsweg0f5fmwa","438":"bx9h3s2uwdsljvzpvxptqbp71ob38oyjhtjnlc1vocf8tthtb13v2xcksfldzsgcddog34qxxmu91wfkwjk0q54sord8hxw1sfxg","439":"ihch5v5q2d12e0t33b8t9u5q6tqbpyq77zxyfz406bgklkcqq9b28w83p3loq62nt8ieesdghaghjfozxuve530jbrn2l24foitk","440":"62tky9o09dvhhw1jfrpdyburspevrj0q227qh3yrk9rlrsutp35qdkza891mk21c358bg7ycon2lewjsgy276nkqj20n8imcab3x","441":"mai54klkjzewlxp4w0yoo8zpebrkwcy3rakpzvtn6lits1zf3mvnvyu7tmgqloztn8oypo0qpaighuyz7q0bfni4ofhj1xrm1nyu","442":"c4lnzl41n2ctoegft4yl3bpbxtm7nk8vwrxj4fi3tcn77uhqre1r3qlze6ygx3m994x2z2qwez6jreqrh6b1li4jfba235lp5dvr","443":"aenhju3o8oc8fcry0qun7iokth1uko3fekz8v7h8gh6t64lpz8gyzchcvvlmqtrjfpmnmzpetny15vr9d8nsunc0qypuea6wr4dp","444":"3oplrkm2wmqzchcl66gp5rnmjsiqgjzh62i044syw9j05jjrzw9zzp00k73ggzw781btsc1jvgct2jcy8eb4l0ivhmoe2jzepx33","445":"hqtr4gmi6j00gb2luu7kq9x2j9ck73iw39w0sqfzmvalupc4mhiz8om1a57rojn12sz2niuoqbacbvxzqufxobrki8f4y6u3opc2","446":"qd0xvcjvxxvr60mu0ezsbrf9u9998ich242zlwt2621qanqarll6cn3v8i9hh8uxdcw9g4e0kgpdzx3ul058uas75j7ho45w3he2","447":"c6cgf79x0qss5hegby8io1jvipobga6rmp2bt731qnkbpd41jduhfzik181txf49n8r57tws9r25zvr64nps7l3i8nimdcu4qauc","448":"rn93xyppbrkb1lcp7ztk97imstt00qluapk9jn2pjbrrwvrfadna6hl6klpodmgggdj1iq7eq06lt155oksbsr2ytfoefg7zehq7","449":"hir70v1xmq5thhg2sapog9a3tvgn1orc1n618zb4eatqkl8xum0t4jmlhtnejlgi8dtcw7kfwpm14oe9cbsmaia0cljxsltzmm28","450":"54h0wd2grbbmi4433yz524ft6c8qo2yu6w2trwlej80omimk38l6yofwbmlkin6cmz7pdzrzrbca92m3qcvlzimftqqe7hj1bgwz","451":"akixve5q5rjjar7mcpem8ng8oshgkzaeaxsg3vgwdugpdh3b3g82vaf2y1t0gg8oy126mfp6pho8k7ic67e55rieshfl57i3qhvs","452":"aibjtqxl7m2baib3wp5bmclfmaoydvbndnqyon48e79ppm39tdcbgtwrga5m7kg34kqwe5h3v6vkhdj3um8jpkw0vbll1i2vq19l","453":"sb9k3qp989ej1q0pc306pwpy6vz7yypkv5b5uj4e9nomdojw0e1tgtb3dpvzywr1m0ej8x5fmegjlqqlcuynq0650fjtlsbfkpoc","454":"trf4hzqsykypes7ipxzl6xrlwre9xn2xesb0juzbu9pnh34264vayonbuql21ortyqeomqjh5z6hzze2eu831asbuut7op3p29k5","455":"973epdsmbtbk3ebdd6vse4znvnsd4u3jp3ti96umnhp5y52e5tt533kvypt4zbii6fqay54kmuhn5q0n66yf6o7zwybs94i2350f","456":"tkm0wc4l73e7v15i84bkxg2t1tjzlahspf91dq0dz42tzb4ymfjozefzb6tz19tash8hog49s6l8rk1vsn2wjjt37rd4kp67dkkg","457":"xibmh6bgqlryyorzoto1ta1v1vtmoog968ykbheqj2h9orgng8ngs9uydi5ms9n7ejzzvzobpei8ek05s2vvzhjxc0vbya3yko5g","458":"6genn2eo919lgv69cm2vcfey11yudsxp57abdycl29o2xb0crfuoz3q49r6zodyn958cjkcghhuyx2qt82q2pjiyygt0ocqy1ob3","459":"2h4ddskqpv4gb96oi0l92y3kkeg2mtgjxw6uoqixd1p4xx3cmo56hph80c0pt4fsqtysntmb1trbu30gz1yq84e1ekresmfnan4k","460":"8tc8wjy68u0pzpth57n69hfbso3bj7u2z1k51cx04kupthuqgdlccu0ppq9rgndlwxyb356ocb3liap3ng4hbz738dsozx6vbtqv","461":"gryhntub2e5o8mt8os6u35y46phkke2up53x9cqug1qzqwco04a1y1v7mjdu1imo8gaebp9f9yul6kv56h4n18gwbz16yss16pfw","462":"q9umil81mkp9xkdhxnulwvalt28b460hbe1rbzimmylq491kskgc02o9k1knn404alnod4hpjzagbrvvsz1ix2kq7ch1ar9n9de9","463":"yye7egsfnczqbb79r223ug8gh40cwwrv1oopeqglp042be261yrc2rgkud4erhtarz2usxzjb0ha15u0h52j00aygf7x5mrjpvvf","464":"86tsvwlujqwz44d351t6qxbv0uxd3x7r3jvaqr67qk6rym5r1zq4lr7jtdz34ehe3khrzszzm20cdfpnm2vyo48miq5951z3kw5r","465":"bfk9xz6ici053ckvlt1w03srkm8gi8k9i0p0jhqz2g280xa4ltlb2tnbfgngsaf0u9jm92wqwyduxbmdrg0vphlbmtxdiw57sdll","466":"m2pjllsibwwsc94rjp73fwrywoq0r338nyowztc4943nakarohrwrx47epzsxwsmrnunsr171xtsss5yx0gp98aurf2p4t9op1gc","467":"1k2zs51jb23o9i9crg0wm46y8603vsyrp1vmyah6dpgvuwk2rtnewmv1teq473mm2nggag65bb3i86sno7ep3uzf3d925ymirq6j","468":"0sahijqerkbqszjkkt394kyewltma8sf3y0rd46mct9h99zzh8tgiyxc98cg1y6n637qrhinaul0ncl6epg1x8ruf45xwwdmmtq4","469":"utwixxmp049np2twi77srciscmd0ir5yrlm4n6v9lnbvq0ei1cdixpide31gidaidglx5gvd50gwnyypt3tfneb7wenqurxmiamr","470":"ylatmsnb6zskkemsqwupm9hhmpxwqtxjn89mobjimjs5z97okm0tvm9j220q390jb2o9o8oo2w3ay1qbvu6qplue0tayokdmcsqf","471":"uexfrh9yu2m6umfhx17n21q2ns15fphlmxjn2z8tg4xul5t6pskao2kwezmhpbkayvtky31hll4ya8ckakkyh0q7j9c5od8ha0kc","472":"koye9ucw4rade2mvoxzt6u6h7kc26y2dzbfotd9u8f263n7mpfafm9h5qsnosgj5z9rodrdy9ha23d1vx7p03vpdp35q6kxg9gf1","473":"gkv1qcvo81skxjwz8t6wpn4un7ebjcaq9zof3b17iimtrp2dtv7ujxvxu65stvis5x1cvioc39y958y42tbp95lvjiu3sl516i4g","474":"9a7g78t79us4matvg93drvqw82mx11dp6tzitlgx5nxac8kvvmv1ekdrfbkrkew5r7kddelmb95zl55o6s2pwgbcsm0hgb73kx0s","475":"mghwmiymsi2r90cjb2nneq4tmxrnamv1w30ccl1s3bt1hs6uxolgvgfe6p1zmuvlnjtwhbi9hx2mrg3joxz32qww86c488d7ybap","476":"fx4g6eze4at1l492o3e2qn6c88bynd4g2lnpeigrehapufsrilso8tx5wvb0fg2d2sp1g4hksj59tv5u6fanmk6ys5u559z8pb3l","477":"rt6o4ns2cu29vvx4yci635xzht4pashu3pjhw081nkvlu40kb13aextm31t8zhg270loopdmtpcls2eu3qksilyo34prx9fweyhz","478":"61b7pqgyv93g977l6icver0ffy9y126gwqpc7rfbm9z7zbdku2ppp1jlt2sylmc3p45b3wlosgmq4mxm5ugg4xr6e8zl1q93qxwj","479":"64yybusbxxba58ewlr69g4vn4opmxplw3d15eawj4bdr5z7k31vtmny3lbaohwunpom6nivpdn9nnuooxvvhgb4vmk0pot81j44u","480":"600qxie3w145qt4pve93irbnpz8zgdlqovhqhwa06uby2yh1pcic4bx0zcctzhnl8tl87l76f95ni08lcq8pg2h9cdmuw7r0yvaw","481":"fej8lfarvbkg4u6xyxyzjkmuvpg0ae4o6md5jpkkmkfp2vpvvoyi6p5svkjata8g1hz3vgc3v2x1uw61oqquubxv7mzst8u6iw9n","482":"vlik553sae88ag75xk62nv9zbf9hmru7jigopin0y21ovoe7t6irckh06uzhb0fbphikiou9pcs935uosttxnjviag9orq7pqij3","483":"uzaztoovytaakc500769jedzh1hf9ron1iqqxmq73jll86mumnwa8c9nb13eumalf4n37k6ya9jftlmekx15fzoec8sn2l92yf8r","484":"s1cq2zg0pk55kq6ls8wfjq7c5o6u87gj60a3fbgtobog6o2c8ne97z3tkkpznsu2kcg2hb8ko77fszgw5notk5j3ymbnse8o62us","485":"np4v4u5i6ghgl83vvh7rnq2ud3azvgtjjqgw5w4a85t6kuzm1x1p4modg4xucixa31cikwtvd0uhkkor2j9cleuqtc3ggyt4lpcx","486":"oc20v7fow0i7t8d9iohxbk6pc9fyf4nin9q1l44xzepf6thqn5p8n8qy5o26zbzqdmdk9k7je871hf06gcqwu7hs6k3ivxai08d2","487":"tqwxacukfxjw9vea89m5tmf47tfy0cpiwz2jjch8ajt5kq50gyyk9udwblpjgnjnyy1vv5rlczc7ysw4lln1l95m05c6cu5mktkm","488":"813js17sms7mlrcxfb9mltwxa9h59uhsgdczb5ds4amdx5gyjlwxhamdxdmkjhwdnvalhjvts5bolgi4k6u0r25ydprtl94bpg9f","489":"dtevbmvx2f75tl1yge10ugnkbwkxgostrme5o9856uwgqo5nxhkujr7jjdhdw76040i5lrbpfqvk54y16semxoufee3djp7jbiwh","490":"9h9en0dni2vxg65s4n1wfuhytbxulnj3zjhc2566cft5td0gykkncr4popci1t8s8swu08msx6xzyhyin06mnsqkqwbz99rm1sju","491":"4i5sbnfkm0b0j4i1pamlndg4dowlmvfeunfddpeurpzinx4mweiiklxhyng803cb151ru9piq0shwrqidcimdt4rnbp6oo0oy7kh","492":"e2l3ws3qvag8skkw9hh1cr1idg5wkffffmfcrk78ce3yv0xlfabuxlszwfviuvyyj7o22t933vxvt07ddnmf5366rr4zmnmwlcoi","493":"w65leyn32wyxjck76ewui1dzku5xovec3r306sgzzazwluxx2346rou4832ja2k6z50zid76yl2haq135q442bewt93hwrdreeht","494":"xl3mfoq7rb4wc4xa8jt01u1atl7nt6d26fjkjop3pb0g1fc9bv28l3idj77315hbx5fb20645aeeo70rnsyti9p4fey3oascpkzg","495":"awfp9om6qdzf2g07k433eb9jvswo1weq56nc0lmzea3es46bccqmab7h3lcvo3aad183s8g7ggcyenpcsfqp7t7l469kq0rvon7t","496":"qf7yonw7boeeqci12cm8j8oh7nx0nz36178plw0n33xeogezc8f0qmrcpkalcto0y2ik620o14bep4twygbcfpgsmc1r0eu6eja6","497":"43zt0dkrpiakrs3gufel8ga22jaucff7h37yrwhtz2z03widgr24oesdhb1ljgr7vy79tfhi0hch88nmjs7t4csc00y0ibe289q2","498":"e0b9n8yxe7tnb9jhhvbfi1znbrb10xp349oj8p40580buut13fe4xsyxrqxu9phpajglvjcn3w1z9435swdf0gwl3gn9aqg2oovr","499":"ufymtqpdfwemfupskqufgclvimibq0hjqgf1uiv528xldskr83ncnpza0zfxzj3pcq7yg1qyqoxst1n4rztf4jo0s06l6j591twb","500":"j9ks7rrhmu288fvsgd5gkddhdch02yrd2o6mxmt9ydhce1ek92ym1njtwb3z5b0sybwob3nucueshoo7di615zxiyv2m5fdmkbos","501":"fe5c41z8t89ozvl2tjk2i8whcfkvls7ncu8msi4ccsevtkdebp2un9xncl3l1zjfv78gwgysgegf6oy3tuv56ilztkg93igndq2a","502":"8mdipv0sd055najbc8o7hippk8lspsey7y935419jyoy9zqxyz8iuxbxeqt1qv0pnfkj4f3zplaefl3svw9yay0kofldmgr7d6y0","503":"tgmr1o2gkeq19l22i1dg8gvn8w6em6stvlghz2rfh49tylq20ikftbqo1so7hug83ppapf2qbfl63m1hk8x9bx7pywwka9t2d4qc","504":"dk6argmtkr5ye2c0qtq3y938ptzv09vur5i3zenlio44txxvlporo141zzc2w4p79ri17gb0da707pc5lbkgw4psdayklaqv602k","505":"8zb2ev4tyo876uljjyafa6sy0i92geiydhsbk1xuzuz6yqvwuw6m2ie04mvofyqj7m8ydnvw5s38tndrd8fo3nyp449ob84u1mos","506":"jppoeghrzgck0qf12k36vbpg9di1fbzri87j9hezt9w1827lruecee5rillthxat62p26zhqz3dzcg0kr8w1uceuh7bl7kb4xo3i","507":"14w2n2jo8r9w84zxopmadzqx3x77jf8a1crdllvn1iyiyce131m2pqopv9ueymrekz7qc7dsp3rf22nuijaphlesu50x0ly4kjiy","508":"bndtxnload9a8a2l6krv8g8dv5zou88arsi3a3pd2ujhvjbujzcifp55i0cr8i61zzdxanbx7p8nzay94lofatvnm1csku2511xm","509":"q7qhm1qcemomt00kfrgy104r9exp8xphdp9g2jdk0t5t20606ovlcxfrgioslbpgnrku7colv9eir1cxvb7kdkd5pqy72ejvx4xx","510":"5rhhrvac3dp1hk4xnqz7yn7y6uk6lp5q2fttswp66izquqmqu9cou1egcttp0piczeel43uu5sak6lf7v87ju89uzijbrv88ko2z","511":"golevrg7x618lnrd95eh7b104j7b07dzpjxr7ce0m6twprdcelhn5zehe0atvi1k0vwfus67ifrboairhjf74il2kbj3fmvsmp5n","512":"muufm657k90zo60sxpdnsos7bvxdn0jqguuq414jv3ohdng7ztn53fd010uqxeiwraercpy5ed4s746m9pp2m5x7l926ugxvqg2o","513":"dohhv0xi3pp4v6zcm6z1ik13bxjosd6tvpqo53mszz4ylzvce26umn4p4de85gr13ry1b49nof51v0qy847f628zclrvamzfzwx6","514":"zon72ug0k30iq317vn0lo2wqeslv367pq3rrmav682gzv7u3734aiq3oq8r8vlnf1mf84tk5322hfnpmsg3pq387blc20r8x2xd4","515":"r3x8kibf7zhi3o5edo0ck664ddzl8xjmthoqbqcd4rhpzgp58vdjk7gg3l0hw31rc3psdm2sqjthuf19gg4e8xgfjbg3h566wlnb","516":"gjhn5lloej02p0jf0a18t505j7xxr0s08tyyke5pr75yyxcldwacdvumczzvqw2m7q620pgxjtwicuo43c4rzprklq4gziqvvrpp","517":"anjwzd51pjg9iwkjs5zrnrpr8msfnn75y4c1b4n6bq52qj58sgjq71i24jnhwaxvvnbd9icvw70832sjra4sd3fs6is9isokqhc5","518":"h9ncqmk3pnaelifyjqlzq3m6g7cvpq0dydhpowj5i3vkoktu6kl0cj1w6wq0nqttaqv6yqysb43ffyfmjkub7fmjd59h3izb2kbn","519":"6ec656xgdsrn8e4eaxlpc6fjx2eebb6st47c7mvqq49w3sukjsn6dynw7cvwi3v44u3099zjdf6yswwt1iaowyymawzywe81ou01","520":"0ydbii1gt52iv7txy667s48c3jh3sqni8bax8g2aux4dl2nefogjms73v9p0v71lgl17p4k9lv523oci67pdr17km3accodrwgr9","521":"rtz9dclarg4x1r6evb1kha3c0klujekv71sjibblj1dzko73e3eue5zexbnhozyuw9k4r6s3fk9ojpdrli5gkszcqu59qkv53xx3","522":"2zyvdbycp53ha5dqq648zcznhornutfa0vkq110mw06bm3p9fwbdiiqtudxqr5cij17meoxjnj9a91wf4j1coo61dmet3bfjhgev","523":"zbwoy607nxpr4da64f3v41u4jcvhnpfhd0u19h82ml3l5fk35f4srxwdp4we34bui1m0ejgxjqc6ib0nuu4g87qpha3gjx3goqh3","524":"0hja42ox9idw2brll15w7324jxyegu9mkndsa7ix874ob2nthg6nksrfrchnval7twpv39tmmurzstva3lscdo66d3c4qayots8i","525":"tultofg7zc5xdj19yc1o5bdnyckkqj2pnxpx8g7qrx9bbcj7cy6x191yu43c3jc28nx6bl4twin5n0at05nvismsw6ifjoom1aar","526":"m8q6o1rilae4xashshef6w6y0r1fj8ctn9rto63cuu8dfezmy9b0m9y6b6azxywgt24y3jnpskdu04cnanx5jcig7x3g31435dr9","527":"cko8zmpgi5ac5pqy6yvb4k34xsnkg55l4r6l1y0cnk8n01mzd1qz3tmeuc6zfc55vqsu6g00ian0mzvfr4psmb4qu61tbacnlu05","528":"x9c1068f62r4uj7fyjiq42e8bc52wc4mixj3bjhja2v078f1mzqvihjtpip3fco1ovmyfe654vqffd6k3aui2iqsnx146zmz7fkk","529":"2v8al31dgezvhvk4so1ws9wjrykljw41imasy3g30jeof6cxw4nuhuly8v0v3qkdzjqirhum4d78wqpijl9xo1wxy6v5kgux4od7","530":"1xx03mbz9rv6tp7uschw9oa5b2vdnrzzz9utf29jsvk5awkd9mi498pqur4k91l6ewhky0r2at4iw80l2v28do6171tlk8oq50ed","531":"5vl1kydxvcl48wmdn8v790kxtt4gwuxs6emuvhojfodvfnodgb3tvqjj32prz4fo6804umzbx3u3okrt9lqctj3lbmuo9ggc6ag1","532":"stii8pmklxzec7vrnx4wmzdyvmwst58z2ft9vdnshx7hd1v0275hr6eermtvusicav662chlgtvxdc8w83uyzu0je555r5jzuip9","533":"h06r7jdvo6fwnlh9sj0m8r8sp19usttfu2z5albizrh9v96fnv2mnh1ishq9x1frtth41nmlt3vgs96vsvhu5p9bdatz0gmasilk","534":"ttmvht215eqfn84ye2jz849rub0zclfgqdxdlioarriddpboiechng2ltln0nsjwdpvuy3550guhaldbwdagqc0k2j89r9ms5nz3","535":"jomunzugrs3yo0182ajvukmhq20g1tjilcsmlao91o3ryrs1ilfmgi927ayedicpodmwn0m36hsjlczjeg45qizrq0gi7azznmds","536":"bmopm5purphawkwzbfhgzob82mg8j48lnt863vq3s4wcdl0ep7v5zrma7oi7dxvveqy4jucc9rs3ba6j1swf706khnp53na8ye2g","537":"919ck84tjbq2na4hzuqva7ko2zybh3j7sz4s0btipoqnt8ovu6cn9ksd78b72bfla7ai86ifonm322nduujzyh5qk54la8qgdtr6","538":"twdhor5s84p7s4pmfx9i9kieaumk7fixmvhfb7l9qy11bf9bo9dwqzb5wxdsjm5kurxl411m5qxovt83lg8jqlngjb7orvdo2zv9","539":"73mol29jn7vvx2059rnel458qo4beyfj2onwl3w20zi1q874gj15lka5a3wqn0ldk6gr3iuj7w9bs3e45uyysplmq6yjdtzuumu2","540":"vrfa4sgx47mblogidi7ai8wwjwrct6w5bfaogdy05rvhk3x11dla4sxjei80uytvqn1c8lrea1mcwx32r38o150arn8anhqh2urn","541":"3arlzgyrlj2rx1a8uzhgr1jqfr45upqo2wwgpt9an2yltsu65efyqpjhryya9n7h1tlcugppja25uhw0ymf75yronlf3nowybr3a","542":"k86g9g4fsjotesaynhexny4r8kfmjw21q13ac4phg942bxs0tjhlcynp5hmmkq0es36o5fg4crrvof0pth8b9ioyiqxbsdx5388d","543":"r96sokibjy8v7ucs0zq0rjayqo0hxt2bs44jpztbqz69ys99agj3qesnypncmzae70c6w7mz3nv9hts1nabxrl7hndpqcv5qjlsh","544":"rb94hi4fw7vvfoufa8yniomtli878vs56muluzzpuqrk2xtt1iqjx5a1me5b6sk2onx25ttt611sr1q65izzuuvbqaaok1ag4j7n","545":"0cqkfiq3t342h8x6mmc0e64qxvdxz92iod0n05omjtekgm6fhk4ldg2wwh1bjyuglsci29bmuizyemuz8df2vdqpn2uljm7gmo28","546":"3v99vb69shqdffe4u2fjqny1lna6yphijlx2et37bmpw3zz25z327rj45mvl5aelbe2f42w5izbs2o1tadw6usqs96jeydve8vdm","547":"7fq6j6ay2kideoz7vqrp6x1hss5prb84txxxeubf0wswlmzkk1iac3maun2n8shfqz8vdzi3u3zbw58w88q2rk0awwhkz4hd6h9o","548":"o1yghmo44orog7yq75oe6om3ebqpw88p2t8amm5mcgtjs9eqj2a2mkd0puasoql2j1uwlqu6klh9v7tsg96r0h0mukb3gqdqur15","549":"329ru3bk83s3hlzgdinup6trqqbbvjsout7v5p2t4ziyhcob4d9fs0x11o09ox040xcbchj4egn3fsuk46tt75atd8nes5p9kika","550":"eo7dri6scqdo0z2agr19m8b358pss71nhoavt57a1o27oko4eqlcc80zqiy7o2q3mt049l0olmex4dto6lxis17khgs8npi8mvuu","551":"arm486cp0s286ekwc0kbpgcug7mfx36sr9f7twq8fy290qpwc3vfaq94x9f2na95pki5zzc907dk7c6skwrlhlc9wp8hjchkvpxo","552":"0z34hfgfjmaf655ptal5cdwmo76ybjrpg90zcvo4pe3qt6vpcegvredqulebxha08z70qec0q2gl6kpquaezj9b06in07iwh8zkc","553":"udll8yf85k9qgakhddvi8u5mjfdj8jo00d4nyxcrgiej9fjyhnhid5fjy0ny7q0t6jrsta92z3maxg2hk8l74jueujobexinpu5o","554":"qac2iqc8vkhthpt1q1r6anst42vtvrtrwmx5lhr7god2jqv8qyz5xcj8qy5dlq4ctr40rrk54hrzmhj4qoei28d2nuteu924jzem","555":"dlke37py65740x2iv774zvxwev4w4ynnz3ycf3ljgpzuvvbzl3zjfbd2j1th6sy5axq792vwqm37o268bd2qij90tzf1p5l2gze3","556":"u2it1zm5al63kq53td9jm5m8pk572ycmi6gv5uj2g1tt0ynla6au4omescro4zfa8ex5buwz8uw9dhofhp09xjo7c4snxay3x128","557":"4rtaaxk9cefpg5jqdahmc2ki9sbkfvnx0qv6yldjg7dwb70ar8u0xeohv5whe25tdnc68odzkqbfi43e93ulett72clmqnbdy4n5","558":"0hicz2mn0qo9may5qjfxc7xi96fbyk38glxb6c0b6ryll9uw62h5edr274o4rq2a8sg6wnk2ssd4oh6xu5429flphjhis3fxi8py","559":"307ucsn6tdy5jg783pedaf6e7j51vrixan824kpx0pgcozuq4hi5k69y466rrb75h1wcl54m8u5cdqxgicbe201i7cy59hpfp1ha","560":"z0s2w84pmr8g51t92h8dcl2vubmd4a0ntt9cwq8jqgz651fd15tne1hf84su3jyvsaxccj95b132v1nqiq6lfu93pgh3qq01288z","561":"8z1gq8wc9oxf4eokursuvttm8cowzvzpwo5xfxol7pguhj8qnhnjzw3z4cukhpo0rhjwldj783dbbo0jbl4c7vbcecwhpnwizx1a","562":"nsawtoguaadabxksyoqry2ux9nywtbahagvqr5uxum1l12ph0fb1dcu5mqq5ukmalmclsgfdsxlkjztnk6z6knxq5gf6scyxjm7i","563":"tfq9b65abptub6gg5jyu43b8wkjp7ca5c580802wuw6x348wvlvp73m5yd8uyqhun0unbhlr332a4bx0qnye5163dq86sirbokfh","564":"ryembkx50dwpzotuxwxexct0rgzhe28s87r2c6ujotdjs3s7119pd6y10n1vfdi631gs74zkt7xw7hksfy396xs0ot2tdrgfqauv","565":"1r2esqcz3gjmonky5nfe6owbilyn3zgbnaiu9q2q89mbr3t0rtc45gv9jtaph4t8dzqu8g1xegz4i8h0u59w5bekbt9qa6bm7mys","566":"3q430c99syavh9gm4lk7qr9j81p5vyfljy75sthkf4dv2g2suxo6ix4g9as6tebclvq5nsv94sf48jryd04owm8t965laafzh4ik","567":"30lrvku7w4c50mvv93ij5s1sp0fckcjfbg1782v3yvjp3fdpym16q005buoh4zqzh70gtif71v5od6q3oo9vibbmy7m6ak6d98lh","568":"es8okisw5v1xukmxdmj7g3pd5j6ozdk8cgsd4wqkawiymc9v6gogmiviaspugbg6z4fu82thsud7tvron0b26lna5y2dbh9sjrfi","569":"cbyhhacg6i70zkjbvg1xl2imp7jtz58d8ogm7k7k4qgqecjguvz259wsgq87dw6aa02d1mi7g5qspanewymuwzk5yh8y7luye23g","570":"bljbw99mzq7nvt8cj84sb1m89rlelstz8h2m0dbtu6iq3yckbkslysewpr4wgpxa8ybzo272loz028mnyazilzd1hjdt2kz9cg3k","571":"i5i6u18f1rm0ht7nzo522d2nbvpnhb9qxb1nvng9og0b01ddiu9hnpb1gfm63tfuvxcp8a05h1m3k3cp4uk2os8vmreowwffjjir","572":"5ik8152m4htuqfjkhxyksmhrtllyz721k39cwnc0uwa7acuu6vwat2wfv39l4smuo3o3plsqcyyhd8hyvszzfro3a6wtauk1debv","573":"ad5wfsx3z3t9yfwzidekb52icrfju5eizkn5moxnroma47zhat9c3b2t9ze9qu5nkoaymxbv9xsgmdwak13k1c4slfk3upl9y0f3","574":"7m80y8rkmc0ezizdyqdh83kixriuk5duawkuifjvarm9xmdmxucl3tu91quaf5ascnj5zt5sgajfwoj5j7bhtxnqsuyrdnyg42yb","575":"g457cbb5nvgclogzvb7zewov5w0o9sx3ceojw7arsj2hcspt1j4kulfp6q46o0tnb9f97yuerik428ndgm5b08dmac10qty2p8qm","576":"vxq9e7uj5g1lqazi0e2wqnmidv031cduz5thacgutvb63ylatz92xktkn6v7t3041cqvi80wpr90dtli3jb3bwzhj3148w0mizki","577":"wdk8axflbh9q6y7hvbw5k2iau4yplz9yqlnofv4p613ah577wa83sccb87kjyydwihvawkcb4t20n2cvhmd11uu5kun0fqa1hzri","578":"nhhgxx10mhl7zbonsshj0gnvcka8ameotlpk97wpjtkehoajookg12c6lov8h838k0ethenn9fgxigy9u3nm5a0vzgl54cnp8ft7","579":"pb8ziyl3s3n2s4160tem2yw7vziwi8u1dt9u9nph5u4aihg9bkge48lh79es4taq9sdzymzq8zphso8u149rpmmdqjleykqtk311","580":"s67h6b2226kffticji2a42wbj0zd57c8gromyg8rid0kc2jqdxlq3ri4uscxsrbi5xmr26h4h8q6xurfqckwd7a2seb42tcj81x2","581":"vuyo5088uh4czhi2y4hnztcspu7rmf4vqpqjrpadoi7pfcguhf32afmmv0pfdo2orgkkkqmiahvip321p1xzxedcq031k2fijngw","582":"9tbm3wi1pb1pbjr9ze31f9l076u7ktrhkofvnt2zfvz4k2sov2ocwzgqrwcd0964fjp3t7rpc29jhez1h8jab9e1l45neol7unzt","583":"kfaxi60xf8v8tjxvjqbm0wvxquub7hppq9z0dwmofed7al1vhckum53aa44hfvj1lz9ufqupcjk6a05tntfnmx5ni8bvid4a7ek2","584":"3j99cevtenbtrrjwhp9jcwn7hky1nfkufoa846dkvyr19r7rgxlo82jp3f32xzmza62egirnvnxv65vtb1nw3k1fivhmmuo683zn","585":"lqkdt05ni89exc9kpvlfsyh556cqlc3om0znx30yyt4b7lr4pyua0couw3uyd2mriwy9esombjhx0p7tplguv56n2k00d2usx9uy","586":"t5ojv54lol8evfz8bmzdunfjiz136hdqj2mg1x6oofu6i2zfm6aneijknd6cixhlmrt3hmasw96s8fa5lupky8lp0ckoc5mecb2d","587":"1ucva79elx45xmskwwfe3gg9bycmdtrb6mvuujzgu6ay6rten5ydvrrohuc8lj35o7tqysrsli7olmi7yg1comd3zidurs7r4qhp","588":"jmvu46j18k1vkx5kak90mhyk7pc2yhtw4vi61vcbz6vhaj053tj5ytu4jg3ltobh515ws6pjgglcshxcyz1ht6apynwo3sehs2tm","589":"df3r1j2e4bru8erj5pijyk9kx7kzvphw8mbkajbuejao8otpc52p5h139rje67wo6qlbyxw8mfd7sua645g1f3c2n6jfyejxgrp4","590":"t7yyzvs0byrtocvj4j9v19ughdnbtm2mh1frsaucwelxld1lexh50l2nrqnayaru4wcppg7g0sn186fwxz9ja40ftijwjct57psr","591":"cjqxdkn0vys64gbp8w0ru2j88m3mr05cxneru8fiqd2cukfusmpft498jn53f9xw6jvcwzjbn9mryvvvlw208vc80048pxj02uz4","592":"d15xzrnedsztz7bdocxsbuc80ffz2qxuoiaol59s4crruis9gvzdffgrldrerqkhppoadb7123kles099ccrdulhokwfvyvdysfz","593":"24fkkx16grl8y71uof0m09064hb577uto6ai3ubnxh4aq6ah8a1rr5s3v9w3fpnfu16aifsi3y7n3re7hrta9m5pb98xvbnobmv3","594":"81g2m74f2wkvmi55wvkfnzjj69c3kukqu6nhmvtigkhtzvumwz6jurdcogdzplizx4ym1isidoykrc6mlj9d2xc4i6gyocajtdow","595":"zkc14oey5zks7xjm16ha2pvvf0ufodgh3ar6n15a3brwz8g61nhivzis1e6kgh9jkjykyjm08ob1reurn4xfmy8ftq7wwlnr8wcr","596":"25uxl49ikla5ne1k1g1ge7zvr4xrev8s716702f72s9vid3m6uyhejsoch8cjm5fg6lom32u8yuq3iyu6u6oq8yngz0xeyj4x242","597":"orcpctxa7vep9dib6chsih4cj45tdt1u0u24mn8kwnd72zxno5c8auttvkpdahewm8c744h3cszd5mc8mljpuxo8rkr1l7wht9f2","598":"3dypzbkaw8g6wczjdd2cqi2tp22kd05etoa7wqpnk6nhj66h9wdp368mi51kyqk9af7xmr0whmaasailajuz5kdjmgaut0cic9dx","599":"yn77zo7fbys1h7ubyyyut5fmtucoiramymp9dwhd29xnilq2azk1c8rfalehsedinaydl9mc1r711pzu8bq4nlds7xsk7ogl0of7","600":"b2lr8j5jwu4z6xex7pgu89vrkrn0b3gn2nvp5t334d4ywte5cxztxrloxdz2kbmyl6mjyjn6jx5wjd9dfbsmi258gklv2if00rjj","601":"640c9b026sksik8sqq56teo5u5b01pv2vt1e9ovg610milc9yya7wo7vi5siqzhyarwf6h4cstev417nf7b4ybb6izo0dserp0k0","602":"9a33guewt05rsq0y2jbcm045f7hfjsd8clykf4wgsf3gamsw22v8dhieg4397ezo3efqziqxvckvi4ejsueao3m4ppc3tlhbwvn6","603":"9se6ym6l57l8x3scl6za8frz3bkwdcp4be4llb5t7u1ab3pnqoggiqor3uwg9g66zo5el1stwhp0f1cd9455zsrcugczzni46jcm","604":"obtwm5kgicg1snerg4ltgqsazv1w147bqey6b08v6b7bc7e0o58tioiaculhoje7efz5qsg7d78y42c30t07rbvo6qan2jfqii2g","605":"hru7be7oc9txcctlpn6h8as315othdv45yviegxpjctfv2c3j91n74bkkfdhab3af1pyf47o9l1hj9rmm4pf19gydv5731ru0ryf","606":"cijw3grcb45k2s2lr4e5a6kslqtb3ayjx3x71p5kfgfmmy4jxny0gvioa8eabsq8e9kk99zqpmeep0svsaxfc317as5nd5lxx0fx","607":"5bme54hwonmslq7nmw3uyj4mvvcx11o486jf3bu6s10i0h3iw7fj4yec0kvv7ndebwqo23d9kz95l5p2gv63z7n1j9vhpk1on0tv","608":"nq4477hrlriuuk82d9r6csb75y91h4yjrfcs99mrkpge43az9f6fzuc2j9shm152fwoo5tn38unu8kmmlox2kav9f38jtfvdunrf","609":"11lfjf85mfu324mzi37kk7fj4obhecoa34fdiki0ygrldni2soojg33rqdwfg7ob0iar6d98llm0tkii9ze18mg7bsgle9ewtqlr","610":"jknv7sxlug03sxfzsx94d55xdvu21kax9vgouyq7jcm7sxmtcdcxskyrm3qryztwc85vdsxjukdfx4ow6plt6v9t2zswgvg0a6nq","611":"uzwcpav2830vejqr3p7pg6w918oxeqo9db7utfkh3daq9xs4x9e0trwiy6m3oucsevtnxgbwu4uwg6kdi2hon57weixmnd50yzhw","612":"n1obb1aqus0gnc9dorkezpuhots0m6tjo8jlr1dxbpnjhqhfk0n2u5ym9akvopm9z9cpxe66icpuhyqp3hwtvtr8i8bitlv6pz6e","613":"n4jz8lleek623cy42aapej5v1vxgxt2t7pdqkmeoiqb8gpuuib53sf9mpfwhjvbzu06kb4z49ioprijka4k86okxpbvpu1mlkfzp","614":"y72drws0dgvefbvxn3lvlmpgdsg0qew08hyracbhiy7edbphf730h633rp9l61tcxowz0r8w1nd9yrpnfoappm7e6sgl4zdcj6ho","615":"0moz3ma80euu79idw0h7si22quu4mulo2snb0wokkphxnbjpmhp4ce77cjxbpo4j0jo5pcou6j8l76qol3gy1i7bz314b1ly5rh7","616":"y9az99n9wdyvyqs15yjl88x0d7m76oqjngbvgzt9h8lau0yk4tbd0ivh1zppy9knxcf8ngnojdgscxvizduuzr1jkl4jtbdjbi38","617":"gjzm1kc55e2scbw3oq73j8a5gts2lhutyzeuty7905lume42xs874ubx0pbabsq6yels1dn6xr0pyn8cisfloxcfeyx6z1pfhzy9","618":"f49dpkx5pux12s82zp2xmxv2jbykt3mx1nc63r98mvxn6pg6pvzvvylim0wpartj76qlj1o1dy9di2zue01sw55i5ru5nop6lj1c","619":"rg81dbxaa6gk38qd52bjcflfkkl54x9q73ghi9v9jtk9siir3oep6jcbthkvhzm61p0ixvnlgegpo6c3oe25ei0p0qrizqix947k","620":"tf6a6swl04aqmz5tfqv39hkw6kji0qn438hfxlyefwufu13gw1a5za3daiuzw1772t0sw6nb3f6qoffyk6u9oozo3ds9393cvno7","621":"2hqr1914t51hzd87ehd93v476lyf7cl45h7x3rs1izrtyj93ipchy7o8uwwcsswet64rsldqyvlczr0ntmvcnht7u9dscya0dnrh","622":"qd03bl06dkloyrixl8deu2fk2d45w0l8d8z66o7fei83n43r3md50obpekhq994zzrd43hq1tnnyqm5ylmep4eexwnsjuhowvzou","623":"vc3fg7a1188z6v3ww5x8dyx21kcfvk9373bs5mtfkmiputxhc7lrn00j3ovf09mzgwpyrhfzh8667328efw8t0d5e82scag1yvtb","624":"2omdag7g3oz2b7lzul10ia74shxmhgoxej21dmb4tttj48cew816ps0djmeffw0fm25b6x2rhghculj45648ikv983dbgiqxm0j3","625":"qob3e11qgd6d4gtvs6fma3oahlcnf3hsdoezdzw9k81ownc7xfq7495ejshblcuostdlkp84xwcorh8y5kayacraypyxnb6kmv5k","626":"wpt95dkbn3ihli6w9bwki3nb6z9jmu5pvao4tdbqzrnt1jmau3y8nxgcae91k2k8h9hi4mfgh6qq7lmebyqx3gv23of0plazai5r","627":"y40ykeb3a2qqkuy419xzpdkseun1mcv9sbj8mkj0aje3xthuhk0rnsa6vbv0wf3nss5zwnv5q66leu7he13vjsf0nz8sk3zwwbky","628":"74akmwshsnhk5qy6h7hzxo4dhksaq6cgsdr9xgsw878g1qrfuftitt46ez7c0260y1v538v0ti1227lqapq2fhzs2bgkwk3hahd5","629":"mmlvr992tngfmyz55nxllr2dz2kjvcae45qd4pbu76hbw64yv6buxphll839u9zzgkbz2i3arn19wjjdvznwobqjz02j0m0calii","630":"e6gv2839oq6xav0msw4uqo31hx2y38ktcnvi9y65jp84mryzua9bmyf0r6p1m3g78wto2h577ckiondzmzog4sk1bzk2jhl4tmec","631":"yikes3lg3skninontl1asq7qgzq7q4x517nkbk8yz3ivwe91naom0s2r2cnu339yjl388l8p12kuexwo123g20lowqv3uvpbtad5","632":"q64iua3ni2xh24kix7qt1c9x755duvaqdzgulxfraxt7dr2a3fyaggr36wj2rygk7j1iv42f7xxw885cx9sgm6fb3lcef0y5blyx","633":"65a05vz2tdczaxm1mxmyodfgvozlso4ds6nel8ix2465k9bindmsck66l8xwnxwwcf9kf8k7oja13rsp02wl1xv9y503cqgmyzqh","634":"7w2by8wnbdv7e4oeru03apyw9xgi0qhfvh8m415e2gpwu5t8ysw46mc8yts8vkde891pmd4wkibb2zzr1d6ghafy9yid6aq0t17a","635":"8juoz63kpyep5dy9krqp3bce9eyclym3of38bjcu7p8yyzgrdlehh2bkau9ccwuyha1frxk6dbihyrza54p426psvz9c9prnmh71","636":"hom0k597kdb599u4hvpbpjguisbkgvtd5poephl3dxjrp4pdk68pegtuk2bpnncapojok4m0r9indnlvh0jn1kde8ngm5l8xyzbe","637":"6u3pu369d82lm5vm2m0788ebsieqtxrji1upm2x2i2rrsq0n8p2wut7ua9uativxo8om8qwkd61xp05fv7tpuilyu7jirgj23mjh","638":"ghpfhklvj2venexyzzfehfl7uv1boafhgtfu9qjhe72xrnwlmc7cucj5nzhqez32buibtriolg0n7ig26veg49lzvwa2i6436see","639":"b5hacyw6yyqq9kci64qhr9j6kh1jpk7gonesrfi1nraibmno58atn2ouy54rpeeevsomxe3vnh5ilqjvtad3pjneyqmfgr3smicp","640":"7uwku55qp2ch4adijaeda4a4t1zvvyx4vmn0x8fet6k5ah54fs1cdxwdwuvwwvfhd342osydstmpuf3lnul1mwqeqbrjsevhpxa5","641":"6xvuqyvbqxhm33vbdp5a2ck4a5z7ta8ge0x1f6k0uvovcj1r27z4fp7v1qy4aypet6uao9d4ahufwst0retuzkoehjzfm9igw114","642":"evvd4s5sk8ercz2bzudfh4lcnbtrc2cx0the6bush35gvool5hii5rlyaxxr3mdg3nq34th7hec6xajwst3m1aquouqcyk306z6y","643":"7plg5jlapotfjbfzrep2oj8w3yu0c6q65lf3up2y2j04lrsquhv9fts35cf6pjc6ulqf6ctfiyokyzwm16tzvj6jlxkqoths4k45","644":"j88rounkmji19e8hq4q6frtajld2j2yjsajdqso9uc8o7r9upztls134fwttki6zy1we9lkdkp6c8nw5p91ic4k743n2flj82yrp","645":"sdlr0p0bhkw8dripr3w0a9u1ir6s9ls2vczvwqdlmxacejvwaf1ii0p9v0o1l6zyt5o5i9wqvy801uqdcrup21wvbkaf9ce8en37","646":"6tw5q9teo4lf8b0y2uxejg3lbcjrhg4xmsqwqx4muv6m7p85z9k6qgj9kdcj1903gh8czf4josaqh5uajfb0f5lil7hcj53v8fzg","647":"xs7c4nnq4ki6u49vs6exsw32467fo72dpkj4cailfp2ij1rkxx14jxjhpxloyh7blszfr0qarlfynbgvxse2pvpv486x1ewl172y","648":"u5h3bobfj5yul6p3nyahufrabsipnzgi6sposv74j6czdo6qogypa792099zq0xeiylpv4zj9evhlu05xvp2r7k8rp037pwfksuq","649":"zifnrv81pvk7axpskzvm25yllxi4f00r4931cog489eodozy9fn4vqo272d552cdrkikq8qw69x5ffc9u8bs7546k4a3w1ad1sa3","650":"vv35ojlm795ur3x9xibcvd46v8elrmk86lcre63nz4w5kgt6wr58bxfcg0vqbxiansb2fgpuetdzekeffp93m27rpbqnkfzuwdky","651":"20ksmslgcfimd36c2rmgodax3xvltbqwf2l4284ph5tlpp1hztp6wkr38202cjwgoqwwpdv0lyxfaztxyp8p2gc4wkdbels19rpy","652":"xj7vv4mctslmtjg9369kjagqxjbezqzhr0uy85h64vil2r5m6n3ymicxicpig2qv414nwnmrz59j5pod98900ltsqhihmeznbpe3","653":"63jw4nb319f0zwlr7jpgf32pyr8qt73ce69xixeo2xs5dercsiywsz2xjwhlqqz93hw8styh6tmoz3gb8lp773xz2zpggrv21xw2","654":"cli61pnr4b3czjz3l8r2xgz7rtwqs1ksu2ypnp8q5198mtl7tye85l6sogutopikha00rdkukf6c7v11v82d6j2exqmt6leudqgc","655":"21xivq5zhzru2cah5trxljy23belj9vs76kqfht2l0025x9tsp7pjb29re09m20x9e2zh8o0gaw4iri0qpv52d0g94yc5a66rgca","656":"07zk08rryj6hwpqutzc4ot7rw3ic3zf46fs4f34lftoufle4tnhga3wkidlibf32yfuoads24o1fxagvf6f5jn9abiztyrf4lnyn","657":"itulckexla6ywtj4gg38yt2gvmiqexsvbvv7cj9l3dyt9fer8pd80kd1kqib1cq5x3t3za4p1syn8mc9hd4j1id0e6ou68hs8p1v","658":"cz6ancnclhgwfn4n02vyqgcb7h93dwvgwooiutl0lw7lg2bb8qzefbylx58wtv34whiiw70hbepxsuv2f808aei84xw2cb8ogekp","659":"lx3woxtpd1bftxcawhwt0f4nndtahpuyjxnlsq5oe9m8lb8c3c8i0g2l0grp3k9i93596px6gzhcyis3pw696w4udu7tsdn7swg7","660":"cn8656350lg8p2xeu7v74rttbqqbd5myb0w9btncb4l2z3hgc3javb7kki03ew06k3vi3dmgaa2pedvx6vg06x5km3z7bjyacefk","661":"nl9uxgp4l5gtw17r2bnyvgl30log0qkn0p73553cpezk86ypw5ke5s3mflxe59d4j8huume32svp3gvd899brc1o2hxz5u2hjir3","662":"q2wtwlcamak3xq5xctql2p54mz6msoaq47bctywvpxlylplw20rv2x551vi2o1or32y8tuf0cj4utu8729asy6lowdu4nm7lera0","663":"kxcnenwdhbhi4jiazki1e539r1d74s5kbd902djd70byofbpmwkufhusg5s0u7kfgd20ifrr8c1yl01f40980gbmysj1v1klwcwl","664":"7v5fpif9affhqrtw5ezx2cxifi01tq11k0at57x3cl7stwwjgepwfnutugevghkhx5j03gb1aa5r0ogbi60a5ynbxhgpdxcfrjmn","665":"fqphuoucyz5ggt5v4fzyd554bbau3qhmm143gdn5zh2c6vncfp0x8id966bqrkbnfz3c2elyysh9wiw078oidvtd5cmjhj7n8wu5","666":"klo5e1qznb14db23nz83f556g5xgohvmudtpm7v8vgoshlkzd7th8vv00yegg922wcgm5y7kqwtjz3jpfp7qv4w3wdqx45ceum7h","667":"h6maqxvw2w6srpf2y6u27555dux3una43c110dxj3nvy91z8k9gz8ks7tvscbep5wo05trsft9gmr5stiyqp4p3t81yohjrrfw82","668":"7tdppws7p835g7y3abh5qfwnbuoctptv0zvx1ym5o1q5mg0b223hn1jiffr10lo9f5z39m1o5sby03ha48uprk0uatp6bi79oagt","669":"hk0lvq7nfdrhqaogmt3vvtvof8vdey0hi0lg6weratm1gdri6ezfsbcdzzj3wppd95fzhrd1pwprwrkew5i6jx4mhzdiqh5oq8yf","670":"x9pyguqp2kse7c6o4oc8kmxd26c3ygz9it13ysg4r1tt5b0kua070j8ctkjtoq70dgcfde8f9hxlgubdtg1zrdc1x5axzlsmnjfb","671":"ik4q3ws2qb6yyqjgkbw62scd0js4ubkbmmtd0z21pe0mz6p9o8bp4ulj7sb08i21hw8l2j374e0dhy4soifayxa8o46p3cwyp450","672":"vhf5nee9lxynz433mhtuegtfalgixz5qa4s8q2n79c0psuelvdhky2f2vghaiyj5amvfy94rhm369n0y6ptomsewz0ay30lgazju","673":"k8963gtpph5gl1cvgxqh0m5hix60x7rsd53ltiomk7mzegqejnyzc79a7tdyrp8exxly5aormccuihws5qbla28m5qiqgxo47eky","674":"40nwz3xaisoomo2n0snmg5i4mu98ci1xbi5rg839f84n3ukxooxnxal17814wxgjy7xaj0xfp8m3rb067hbpuyx9ho8cuu5qyirb","675":"818v1ev6b7veke3ixdp0zxfpl9kvimxxstzrewrbq35601350djois7x1tonus3vu6469u05ml6pmf70g45giuu3g234fl0q2o39","676":"7bszkivkyjga6r7qjcz7tbaxhjnh7fxo2yoqml806dgucm91ff9l0s6i0k7vddv6ht6vvppbmlqum2fzguewt2vvxqazqpafpiis","677":"6jfhllspd3a3zri7t1zm2r8iyrjoz8ju8hv3g12laxh9g3pzih2n86rc4ih6qz0mpda8lvxxal9erzzofkt0szdrz4fq34fdme0d","678":"nq9gqve316v0ezpyxuktyqk6sl2zpa2b37up17x4eox6amko7hhkqb1zj5bey30jj1vs9hw0yvnubit8n7ojr08y8u4ljv5uu2yi","679":"tet10cj353rrtdk7zy1qqad53tx71lzbhzrpuda4zcxihfob6j73z98ymqlrk8u3tum42wucy1n5m0axdznjqa8qryyfvejqmnks","680":"pgu56dupxx6tmc5gwlnyhgonsf1nehsc9crsl9ixbr5s8h3dr0odg0kt5f3aowwgctbjwb2j3fqp34afbp0qlqexgdwwwnkurof6","681":"tpc5q1lxxuejd13ev95atb7iiboyit4cu4sbu6yl4noeesir5vesxmjyahqm4hqrycjx3yqsu0tixtvu8ydpkkkqbqh9eao5lp5a","682":"6al56yyqb3gr7iccd1y7xjuhx3muamp4lp22gw2i7huqevv9c8u3bvaccjtfu2spfva6d6c6lm3e8bcjfi823bsrkyug68u4ce0k","683":"m9fdev4r8r4may8rnwbjl9ucrn1qgecwab14vftegp10o1d3el2p00rxpbem5khjd3lw8r5xkubo2wuyiaksny7qcw2ue8yr1253","684":"2ijtcl21r8e3zhh5gcbee2mc24pid6xi7w2lo4909yk9q26cyoyqn2rn97wi6b4juxd4x3j75ciop5dhoe3x9txp8bx92nbv8iur","685":"orr1kr274gmwuec1p3okjajj2fudvb88xf1ysmfzlgxjzys97v3847i0up4xas9g37y6j4c89wqik1h0ihsr7cf9zdgwr8xamtj0","686":"lftqz0lsyukuayu5fzxhnzgmeqlq6a94zirox4edg3omihbfdpaia2tdud5qiziil9j5t9gpak7phbrindwdpiyt0clxle6y808a","687":"i6xjgr4lr7ijdcdl37li9nseeqxy46x9hmridlnqw3dkaeclvv1daio3qscoj3jci457z3oqd0v3zqes4aydk1bggehzr3u5rsts","688":"wkyfi3vn3aoiuvp5pk1t2xin9ut30bsslg0zw4mk0wih7f7ive2wol4fcud6wm0pv1u7xzbe04elqiw578chzvy3jlsohqoiquho","689":"8o3lomk82f1qjwx106qlfp8maybwb4php9jhp5mnoh6vg8psleznzx82qzxj19lgt967pjvnv6rw96uywwizaw5f82z5dw9d4h84","690":"zf3838f0gnld4uy2h51w4cp0nertf3nb7xc3d2n866i1u2l2h6sm0ew0vy8rjnu6e8jfeyvms6hq2ru8fuvurplt1db4oy6fz02v","691":"fljg19bdlazcookvfky5vptg7m31m1mksr05ugofs6zj9sly6khjs795a656dtc3d121hqoe2r19o2bvpjx983y1qrwwqnssxgkx","692":"b9k493mjc33goq04a134ows84kg82b04ktr65j9rtm9cy7f0m60db9zw1to2v7gie8cvjx8wg0ya7pewv97zhvtrsa06yqm581aj","693":"ml2xwkua0ggftwp9shimv83qetqf5ai2osw78939vri8pxl8in322gjv5xh7v5i0kf6mtipwejk0bhdfdytib9fw6e65fqnq9vbg","694":"qlsk6z32ivocfjsl6zjr83zj3ubsmerywke8bo8s07s821dp26oxh6q0emuupqcmncixe9z4d7va7hpcfeel88bx65451n5b80qw","695":"8rt6hwlvaetuqb1lnm4ddzabs12kmb4j183drm5uijk9upwermuq1o0c3qm5xir99ffml04gmf0dlu034yr0g7hxuko3w110pc70","696":"qr39k52ai3hcyaqe2ydr7ve6hftyawbmzb0b51ettzm7bav6ltjfose73hy24fckuzk32w8ca97maxnf9v6ssc7bdlrfuvw2hxt2","697":"ajcml4mu4r02iohpkjwjegolnxd65m2u48bfprdnsaxq19wtihdb5g97bmxsy6pzl00zirsmo8t8vx2714z1o05j21qmiwa3u5vn","698":"einwoqnzmq8voly97fv27b86dghel57cthejezfiprkt0ygjaxvpw0nbnym59uev8f2bm45pcr7mwc21w3xbd1jfqiconybjk39a","699":"iix7okk51bi74ecp98t0ta5fel0eb9fs4029khyrmwn3ewjncp9u66hcz2dwli22k9aypy5ny96vuys8gf0fli7pz6cul6wew69h","700":"2p3qhjgm05i0tsv5kpqaqmonbp7ab2wu4qidufq7ki6ecz3601sm54thrfoeprcosxa13ui9zd1n2guzdyirxystcg1a91t2gxvo","701":"td4w5ho3gj0uouqsh1v7ezb47eruowr6mhlgce0xakd3tltaj1fq7tkj8b1nvd4d78l2dnkrtmyxchneum4upze0kru37zsgvmth","702":"572fer667u0tkpk1ns7imiiowlugcwrfigjagfyzxvi0v4jqzwslva0qrsx2f2xzl5839s3mq9dmwpt1jht22bij6z3nzwj62p2d","703":"o071og6om1jkgrekzaby07hf0blw9qbp1685ehotp3y1kbw6lnyo6ilzxd01qntys77uuswvzdv3uf79w1p8w0bsqjv7lh07vuw7","704":"9bc96oqo2sxlbyngrj6ve47innipgh9ztjyhccno4iw2utujb3nxrjgte1ik4s43v8316v0jzq8iixd2wgb6dqvg9x04xwxlbm9l","705":"lqkn0w7sx5ku8zyxhyc0naozd7a3dkv1pac4cgeymqtelkh8yunof9pz5qwrsj7e9ttgg2misyns3n5mmx5xadrytrbose590d53","706":"82n2romn7925dx3d7dxjr1vvr544h40x1ve2qoz2wckc2u8fffsqhkwd21fof5y5awbqm5vva2stki8d61d5sjbve6ukqrdyon1h","707":"9jfr8ohhjmw2rdvpeozluoijobwr43sldxgntky0iikeya7uxndnrvqzedzqxbi8ut3ixh6gdq77r0sx61rhdd8r5bga5lo18tyz","708":"fvqnct1c5vfavj3f5orz81hs3e87ets1wucb1zf0gj27oogfw4p296xyjiigeo6v8mntjs7gsomkyf4y5adwb6p4s1top4cbz7u3","709":"9id7jlupso3ideg138vptrlyvszsnw6nrft7ow3mg97ijlg92c5u5bngw5pjy0veuece42nwry2rfz2zjwb0r7rdjoze0dk6egi5","710":"sl9tdzhll9unib9e5bweausz4owtv7heh12n07ni4y4wr5cjgvdybiwe5k2ikciefyzx8iip7icfbg3loxorgxpaga0ztkhbgulp","711":"5qahh9z8pp9n4s76h5x01zx9nul4isvn1sk2r52sjbjrvpawuy8fozqilee8xwxzmcptf6877yh0cjqsd9eh9znthxo59eg1f0pq","712":"8fhnw28uvhmtectlln81kt43w2j8b5lrql8i2koue6ix5drb7w03w94tuc9xkyy1jr6ktmnq0y3p2rzrvtjblsgb6b6ewk33k815","713":"fo0gn72wj7vmib3g2g8cmf28ou9r536k9cw5f04ikappxzt04fge6o5rf6foqw14d2x5vk830m3bofnx653lrc2xcbqit6479mz5","714":"wktdgtye3bx9lj8rfgpqraifjcl4xnlsgqa65p40k3pvwew678sg2rwoxjgdis6eobo3xou4oy63pnzznpcsjfdwmyozj79o0qa9","715":"2zprst4iyv7t417rfjmy2zhwbx4mh3zcn6ynjsxt0wnd6tmmo1rw56zyljx3zqh56sfao7dqs2knndf84jwi4orb8y56p95ccaau","716":"wwjvgvx4ry0f8ohvmuev27joh3xtq3ryvzjsaixrl014v7hj2b8jasp6dqwmlrq25o8rcan94bqm8t4av3yo9famz1j645g2zk71","717":"1n1199dfhbqw9ofqpsatrk5uerq4jao6r6hy9x3i83qi7wujrnycwqv56azr7y6xkziurtr72agc7k1m48ob9204xt26g9cww2i2","718":"6ak9d7yvw41rwryu623y8t5b80l31fvrtzco6oxzo408r5ol94i9zjregnljnqxcly0wer1vbr5fv0jdpbehtmxrqk0vyqxiyvbx","719":"e8z27sqmi66ezeod64qeqv69si233t7z2vu28dv9j2ux8xycegk00qqfmcmxciwhjl2dlovcxb47fv233vhqzoilg931nc6yzljb","720":"4c9xgtdpdmo0mkvfmaqabyobfhz9ie6pduqqifqojvl3f8uzoz8fs0to5i118wrv8zq0nab5w7dkicatn448oqwz8hu3mdagu4yj","721":"2fu84tyue0ekcdv1az2meh0fky4ibgfte9rii44ev22h4g2c6jjp6mekd7gylaqfrm44qh2jgfphwdhebw1bzwuuv7wzl1apbyrd","722":"xz4ojxvsw5s1zccrn4p5zt7phdhc4gvuq30a8gd78ngzkvdrj3ojwe3e7uhw9tkr32r0r7yz2p406phi75j615012oc6h2g7v07u","723":"y2y452898jdrgjv3s9z6a996qmnlkmj24r16uztbcb734h2us9hmlbploeiffxk3xcw79glhwfzrz5963kz091ipfe198d3sq7zp","724":"eldwjpbiikifq5zdjhg3d301fjow1mxhouyr8qg47iq60kwr7j26bycde33d1z5jdu0gf7oyk9ts8gdos9ef3lcin1puiewmfrg0","725":"3fr20gvd1swspit0hx2jx2xbjchoc6xe81ef5x23f7htcu1y4f0get9zh6lgfwotk4m6vov71wb3xdvo0fetslrq40ikqprdtz7l","726":"jshdc40sg1x7bkpho4myaff9sazynfxwqzvtrmkrogenarwtqo8w0goanienqfar6f9nmp8bq3pcxozzv450tqnjmlu2v2lr5324","727":"en4q96v5wckt0refklt8bcinwe9ht5z5c27ltz6hgt9mre5izqhynvtemtp9y3cfq9zkm4m2wx47uwhaixn3v1949xpq1i2clvg0","728":"nq0ztc2nwmj755gricyhl7eyrl3onxne1592awbxqgdhtd0a3v8oirbu9bz4q4zqeuso2adpgji0l2ty9mahngygr5gn2zv9qc14","729":"jc2l4r8kzhqwq0dy8qjkz7yyat9ij3k8cz8y8azacqx9ww9aybb4ykik1o5744qg855zkgih9beairhmtik9dz2x8m9wntzdzinj","730":"suduprkk0bnlzvrzxydhkg51u9ljhynasg1xth0xuhzjty5z42r905lrre4ly3ao6ax9jm1c1txsyjlnmi4arq3h1z1rr3o3wu12","731":"02pr75ptcc2lgdkdrs8hfpsswoqgkrxnhnpobhkm9y6008vabos06xj8meuzqcqyixt43c5oixbe248whs1qxzlcrl4w78xemzs7","732":"p0ixqsz1inoiv4og6nducv38fuya8paasfl15fxcejrpfo1py5sh3n8xearjoqbyo1j9ex28da9s43byd0kw75mnuu1yylr66zfe","733":"y7yo2hi6j19b4bsbgf9340bpu1l442dmuhcgtzhplvh35gwmi5jaihudt0ml1ms5ezympyofe01zqcn0eyppd8hb335iidu1m6r8","734":"q1brgdqikxhe1z0bu389v0qn931mvfs6r0i1s7yh47pyd7vhou2wqyt9r0n0hpaqhsxt1sdvxztrq1ota2ofhlc4g6ymd6q9njmn","735":"bl9u9qjfqepqweum5hwljs1284midemm2h3rvmu40ezi4n1fhw599imc0flvaq8jwc0y5ju33fcuda6dmu0x1kwuo18cm85fhisb","736":"5zzpy6kc0k1u4uwv6mtb363k0m12ylegyabfw7jhzwadjazew1wjkay24mz1p77x41eazizb2y0xafmd0opb3402mplem22ej6zy","737":"ew079okp1lmhkaxo8mytv46xetf56kw8rdyd7023pzls8cv9nilbl0d70e07jm2pxqfha5balgdccsgtj9d0gei7zc30e4oah1lt","738":"3gw1eb90h29j163y940m0aiz0dax0xwksq92h4fyyk5svb08lk4qiqask2wl5qrllldb80ilx9ta4lcg1tib8rowl8xpc0ajtg89","739":"wrdup3e7f1vz07pjt8thq4mmnr6iaeakjy1xwiz6m3lzjy3zq1pc776507pdjk0ddwzfg748am673bwumjsttahkyzvdbt4abfr9","740":"av4a447vht66s7ahti2kgvhqx77csvoqs349kkxoezrc0ejme1jiv2vpwmkl1yxhusvjtl63ya1iz5xk739fp97kid7rztmy6ddp","741":"4qs5zh5pjbjeuykqs3nosvuavofvu8j1y93m0g934k12q7kx10rodhko6lt0y9busxvkvgkpyntea1cc4xk6yugp2v2u3l09uptf","742":"tpoglelmdkvx9o3jel9jd2o64ony4s99t88r97ye1zmheeum9myya7emcwd4p61p5m6c9ravc3cs78ix8wfoz7jyexukj8qzjjrz","743":"h4qxrma6rpqyudg2ei2dqcu8pq10mx2kch37p15dnysyyc78yakapoyal8rp9ud3yn5jn73t5e10yfyuviz0p7l3yy0d6srum87c","744":"6cfg7g1vrhjqdysvizdkm75bb8afczmvthzv2ktkwqjcyyarnegbbm3h98adfefr3yas43387hfsulp8uf8jai0kx4hdiyvao22r","745":"wbj3w5qxo42au5o9egptmdemdgj5m49sd68teqqqyid6zm56sw5m9vwmw8dg3hmyskkiuo0nqozvh330mr43hfi9vyucm6bzykr1","746":"zglr0ekjdbryhnqe4ivprl78lurpbpzoa4ae7k5qgjcyn6d7lfag6o40xpezm71n25fx64bhyr09k62a81ifnwmd02k3x35epwcb","747":"nup0sp9xu411xl5ry5mevjudmwhuj4uua16gpgtekme9ap0u6dqcgn7lffgk9yy97lw02b9nruwaomcd93nwfvb9ws6isbnnywm2","748":"9mbetr8kjstsi6m15bhomkto7ha8f32bg2mc4ijv0a6cp3cls1b393hq8mnfxagqz2rytwax0mgz94dv7k86s3x1o13i2efm0n5e","749":"94801zyzm2tax4g6d3qj7pdta5etz7dikwoka2sapc9jph1ga5i8fsn2m2kkd0n7b86mzqgn7a4vmryppnv08j0i58gw6dp28q29","750":"45a0sx0bi7y1l806bhd3hpkuzoyfzo4vgv3dro1wu450gg8mdjhzdbbuzm58064mska5t3t332eum69behgqqzc6afirf45eebos","751":"8xevp087zxh6zwl46dps3ejv2r881w94hfrib7b1z6camn69hmced5r4sb7dk048g4r25934s9e6mofybq0qtzhsvzhf8d04z6vb","752":"dg1tbh9ltyzaew3yl10s0l0z2sh7nmhiq4z7scsdtsx8221ni3x9bczjseci6rn1jyf2itmoljev4gcbfzt51q0exvzxw6o0fuvq","753":"w9ya3a8lsu3xw69f1ewqqm0nffaag82wzqylaeqgtqp0ft8qjk5hv83ebytasfm7pd1wj42xf2eoh826led3ldjoh1281c7ypslp","754":"euypzvwppcfzokxfw7exi795gb4gqxmir6ev5xzt12icogf0f54p68afto9blgzs6lsjt85sd1c6tkhravy8b9p2pcu23mmilf7g","755":"39pwecodjxkjji2lj9ihq7hh2vgkwbesafgl7nb0nx1ha4ijvvtvsu5mf7wd98mlcziod11kx6qmfxpbhtja94z81wj6orcabz3y","756":"i1l0zx3qi4d9qbibfy9rskd5njuimy2zt0m4mleqbgsa0pjq1ev7qvfti435flao0kbqo9ssn72dhh7iou97xbx161jbmsm2u4r6","757":"7m9w0c4kseqibij1dkiq4jursw1p76ytizl4ywysv8t2a4o1aoc0ezmjbhx8uqn3yu7bwa7ocdu7bvc53xy2m7nh6qy345rv4pdt","758":"incxzifqswgq6x6408e4n32wh6mzpkuqhvrcvc977z23ia6ivj7yb369q8djhi7z5ow7hrvsw5xi48n8uhsue9e76ql168ywzdh3","759":"jt2x9905hkwi0enx8s91yxndec64kiqbelnrcdt1bapdfrfa0no9vzos8d2flv3y5xkz94qr9xv9tsg8vphm9ppg93yvny1rtp91","760":"we1zx8sgaiu8jq50mqgzc00dybamgva798z0bddzhprs2qv7skfz3vkrkf90avzd6vyyff4pm5rgpryaqrhkhu6s1mrpkmslmtx4","761":"nh3p7a9lybgbky7ds9v4xyo4xt1wijyt1sypce877u0fi0wgsezmc31wyzazs5pwruezip3dycoshxo5aiisuc18eoofw4zp6v4p","762":"2376lsobhftzlfob5cdc39f5udq9j1wkf2q5prohacd8648z5rujp3yr2k4xwuc4rpnt1ax1jy96kg3lz78fvdgtm4qjt3g20q7g","763":"jfctaw3vj9g7ej367c4dcfqp5ktm96x9ig2pbedidlkb3o1g4b80x85thqg5kb551j5jywd4c6rsqsktpkhmox6ud1e39b9hefnx","764":"gkc9bokz7w1hzxhkwi4wnyo3owj16odgohhs8eplyefs5ncin47b26y3z7davtosb3mw31a7fk93mrk0b30o9z437ljnmmhbgvn8","765":"xko6ga4sonz5sbd3sf5m2i61jbzauug1fl17ecouudhlf9rr27usty3urqr05500fzad06r1p3iypflemuu0qw6ax4lalcu5ei8z","766":"mg5oas0p9vg2dahdsp10l4oqbs42fhfl6ndhpizyqzttj700cgcjki9ete1ve7rt2nqk4lyrjoaq85uh8bzohbhax0mbdwh0uhm4","767":"hqfi4niis4qutppqxy320qo9037p1uolb4elbeqaqbcsvwbn18np3twnoz4flqeksxz0r673ysim0eit4qneq6vfqsw5ljd4h7xg","768":"7fa75s3vwe7rxr13jf6g4y1eyv27nakwyj9y6xg1joy0342deyoxpp3zl4cg53t6cjpbrxvy5tiey7xrguw1kr4hpxc2gdyf1x37","769":"zk5gl8gqmhos8iiswebvhsrfv9g2vt4omzi1dz92ub72pa3ojg1qa1rvsrzjncq5mmbncwpoas9stvoyjbclgryaqzgsehm0b49e","770":"lml4ugfslh5s3bl3ahsosepz6bwrs7ycxwk5di7ypfbbpy0zfe7q0ppky5luk41y1c1jxs1vwd95e1enjxu5iblyjslbkwy181go","771":"poevtlm4z6y2aavfqvmwdd3sxbhta7ip0qpfpvsra7yx5mtvyqosa348mj6fio8j5d6lo7tea2eokr4e3wfsn3ck8kyb0s0737x1","772":"uyho3h73nubsbaiu031vjuea3xv7yknovd1tbowu8pryruobhm5iylmlf4oyflgx1lvpukvhtcetjfmd7177abycj2x6swkq1ujx","773":"7zdojcimgnisxoshzrvebmqmihgip3vpswnwsey8aux8ujw271k2kcrxqbp0tsifknlr8klh60ok8h4u9efaebwgu5c3ceqbwce9","774":"hbcwjf9rdhwdmtqiainnlb4op8iu77peibhpi793kes3s6pxni9dfso3cb6e3pvm5pflmowi7eqlvqx7dm8yjtsz8uao6f76uo3p","775":"abhlgqz5770kx1yd8p1dk2ger2gfy3u24dute4q9ry65ppf38mkj5n8zevzjnx7bjmnu55p7n2vxtt0ywwkeatepnfkc6fxujvp2","776":"1ng3waserhmss5v826svy8awakvykihyxgspo8d3zhfeyyxymnhf53meme60qf0cnr5xibzs0hkn3armb0aqx3b8opvntff3n9h3","777":"ey9w9r8ycmp1jhd0l8b1dfelufqgtc325cokgmfoedo2z3cmf9odoemgf368uf0na2v1oawo0x0egyst6nnxvzaslkybriwko90d","778":"ghjgtmz1ypiwme8w301qeq1kodp2icbc90cu374p13s0sartndzyadnckud2k3x337kjyqri2lk49n1rvobc65914pfs0xt1ntw4","779":"ler97kh5q3u9hrrbb743xvv8pci76ttk2yfbg7noy6ldwplts6dgkghnirkzj2o0cc1lsftxr9a376f5ydlpsf74aijekihibnvh","780":"y8l7a3n3mn1kwgc4nlp84o2o626wnhpydjjnxxyf2eoumkiou69m2k66hxyen345togb9nrh59osjb8hvrv4zcdsab1dpsjf0n4m","781":"dlaevh8dpgaefh7wbu2l7xj1fpbq6jxg3d3tihlnaxc5jr5p8emvob3ngzac8zt3cq4wp9s5nrni1y2fjyab59zkwbpsruvqaoaa","782":"tgso0ui2pgyvkhvtzkghop3sjrt3bnstpi178h4iuoouirpbmg9fxrx1r6u2ni29z3eu77okurqjxdduotq95r6etjhto5tmnvtl","783":"ccgtcq2fy76v41gzj1fftcp60o8fgcc1wudjbyifu213ksytowc64tclq480mswc93q58zd3extgvl8qru9jbyo8jr4aji8ktdx6","784":"lbdqjwfao49mrbw59vbgv10tjzru5m743qun57wi1dnlft7bj2xkdhbm43wggl1d3dw25ek8iozkpotmbmykto84ihn1vev90ahz","785":"rccn64260sye80kjk9dxdbll6prcps8o7n0ejmyphcqkmd0jb0x4ua238izltsmfn73omjvkujo6f95pl53qo79loka2f2uaxbci","786":"nmm795kc7fj2kmbcgqs7hxdl6g6949d62wb6jbp3jed7dn4auaptqzaymdkcd7jbd6eg9mscwy045ckvg0qww6uogjpvqert51g1","787":"03lt3d61x9aaxvq6l8n3wecj713zr7ezm30eo57qjan72xwpcxu476if17ihxrmlp2xv6irrmxs69thlicpbq0vgvy4px9227tg5","788":"x509dfujlibj4xdrtwwo4hdekj2ddyg4dp3wyrztrpr13omns4fn0fhu2in0fkohpv70b4vmybb51kjn2ybx0dn1j3o3e7w62a0a","789":"n3pm74ncsxket2keqnoeu4vz5zssq9ej8beb3tzhopaaf7ihgnfj5imjtj2bha02dlnlg3i6l7y74o9udjq7p6cadp63sky13gna","790":"a2p4czm7qfgwl2zcv8i4szn2rux136810oruqafkhhr0p32hqbaijewonob16fq097xv7kfky4x2cc13x11mficug42pgaoadvi2","791":"lnm0wovofik3hs9mg2gxkyrqnomy657jw0pb7l7wsx6yl0pjaalylxxktv6d18poo7dmzpvj1zknra7ju2a5iqp4dyk4v0xj8pqc","792":"9qf3wt4tiauxdueepuoc68z3x0lgpr25mve2aj4358uma05khyv6s3w0peflueuybbkabyx9hl6qnox1tvjjbfe6s852805bkvyd","793":"atoy7iz2szkphvoolbwgxytb0eu18e0pd0w46cm04sxtah5ja4dqpbuf28z8wrjhui8gurf18rn0dh5omhezw053q5s429nzpuha","794":"pqw70og1l1oljrm4cdmivvlatkqh2c8bwzt3fx0u5lwn6iyaz46mmwywpdno0fzqeyn7yfwa6dfqtgs8mm0idax87skhi9984b4d","795":"9gy97xr2aj9qwdvfb62ewb7jgfav399k5guy49j6sl1a5mpvri6bre9bhjq4k51hul855v3lonm37bgcsancvju0xg81di8waki5","796":"riarkcly63kafa2bu5zabjxu6n7fwkk0o39rcgqjedmmqlsplsqn80ca508sabpiud5fpstmqpcigouglh3v756ovty31kr2o7rr","797":"n7rplshqivkpo5oyjjvva5kkqxk61dvw0ncj069s6q5utdxv7lry18oks4y4kze7hjxjlmll4dzei47wrl5f8bst96herbal7dt4","798":"i34ep2hghdmizd79uxkt4qkqbnkrb2vgcqggt3b0rv0vwr3trosgv84aeo8dv377zauib86yo2to80ebwo6lcxfoc7asy3qpv64d","799":"gt7hr648jkpcs4j5a9fl9lefh8m6xmuau0xfurg82xilk737eagcky8h30bq311vlctngoybdx1xynqxjbukobmhrtdwmskaina7","800":"r9t4y5f5k8lshp4qimrvl97yvlyfokr6b97mu6ydt4bnpqy31ypefsnoa7zzrh23wybxf99l4dszdbb63svgzctg5a3uu152z1io","801":"pzd8bi78huo5ur0qkomufrxjfch7ojp084lh46uou5z4u9jxa7qvcj8foogf4qsws5xomfx8aq1tggclb8gigoe54bd2huz3osxc","802":"n6d8krmmjwbmn9mtk1lzo8rj6ai76osar3azqgqftvgc87bapgrrom2smfy35o0z85d4bibgiie7awk1wg0u9aovr7pceif3n5mi","803":"esay0d6lx0wocdbrnyjduc4mmkd10n8yyouxpdoq017yw4zc5gvs682bu7mjxz5pug31ulre033843bxnjucrgl2kw9tbzv2kr6i","804":"b62hfsecwvwusec7j3jbgutfg76wfk93fy7fuli992c8d51qqfjpl6me9tvcx6vvisja2434fatjjar7g4aukdjwkls5p4e6m38j","805":"qhlds0yu9ahqypys0exg4nrrm9mu249b3y42rzfly180vazqurubn609ddfnskstuntyaho6yqgmf0bg6z5yntzidrljiwastdso","806":"9442ctiof0v5mfrcw7vz34vhfggguzr2q9xbsg33iwvkp00y9n6zm5nce5n076a3w1t5zssqtcj2eiif13j43x1y1lkzvr5dkcv1","807":"rnpg23r7givgbh84h3b79v9hby8vu77i1atdse7naes5wwbxshvnl0clqygyweqiz9ttapwfv8i7822t3d78mpgdqyx4w92k7hq8","808":"xw1wcit5vkiss4rfl4l9srhc7nfueargrkkq4feabnr6u0qfaei01xpzxbpjvl509mojqwrdlz5i756tja4zdhe1y1e1fp6dz8ev","809":"p6agafvtivwpokwmdxthtqjpuyxnuxn3qbisui6rjttvygzvs4adjhve90ldmrpuuz26l9hzoylv8szu3lvuxt5o0k1uzvki7lha","810":"nxkaxyppo2dwtbxk3ffpx64kh24dmykypssvamn3bm2rujieoiccmpkbljr6omagfkft7s58r6wvg4wcywsdt1s6yksk2egdb6dt","811":"oomuiwy53twkytk4k4pgnyfn705d4yw3oil2d3dwdupo5hosjk99wt06imme258vu3488en755cahdhoz1s40dk11jriv6cdukd5","812":"lnj1sp3b92yb1tkwjh02so2au1xh9s9dnt8wlvx90joec5pd05rkp8drizaf4q4o0ttvrmsdvo7a4k6pyxgkqkliu6fuakpwh9qd","813":"13piog5a9ccdwys95hai0i45zpi4hba4jayluq3nberlgx4efhcup6t5xuam4y4dum4968t06djbq7ac790wr1ru4u6u1yoq9wn8","814":"o929wyn62ph7kuy31swozryb8aquvhihjh87mxqr9gpwnfjrc5ctkddm4vj2lh4ilik4sot5xaz9byzgyfbynlek3pqasgci7gma","815":"46787p8gh0j6rx0j8giaorybayi9ch1mocgsfrssi5hf6dhh7e7ibfsdc5h9ljuaugklnh6okz363rcvl8v7n8sz5etlzfklzz19","816":"2iuahayddlwvytjglu7gvgeclck09hozfigxrtgd7nt9gkpfjrtnj3d3e82a69yn7ccucgkg4xva8s5rvt855pfqcbl653yw0zze","817":"58us0l1y0ffklubr3lfg1yl2v9hfri4pvbqw0d3f7n163duuxljf3ys0rwmvvzbrzfmal3gmc8u5trfob7sy5jqhjlcqxaywq276","818":"17b0o7gx0c62paf33bkcyv23rtwfnbvhr4oghc0mx5s477wb21nhjuxelgfp5ocj7q9t5dsrpjxb2y7a117q3yqan6jzwmlg6fel","819":"uvv1mg1giuuybg6erauat1pp06mw9p4aodf4oyl27lbefgh8qow63e8k49bipod38c3zpb1v895oz333nike3arc22xsxpgo8zmp","820":"zwfj5c71gyxjp7heomjpmbjyqev1ykhfo1x5apdl2s6l835fjc93g507siu4movjdnuwy72rptzuawvjui800ea0drhnssb0428n","821":"svultd74e9mmrxkmd6kgapgo1ffpfc667lrmv3lwyhxy1y1pnlfkvqwoxe1gtqjmoz0s9eqc6j3gbx6729ygiawlyd5s7ix8j6r4","822":"akof52i2c3mg6eafhgmpp4sl3ty43gvkyilxkyynui8wwe074fvckbnpku2z7y2w2jq2h4bw5g7sm7n4ydy8eqxk7x3s6ti3aw1h","823":"eqou1m7nnx48er20xgtnvi7icz58f7i795gc2ia58ejx06sl6gwci2h38gtxx1z4srm4myiklakeg903921hag8n2qij3cjblx7n","824":"wh0gur5jlaoa615e6uz1mxoqjmhlymdhr56w6c5iaoiq985wv4k76kmuoq5pr44bg50affkepd9fuzx37atgj4tqrsfv6mx6ujc6","825":"coeniqnia55guj815rvw700cymx5wws3hsc5rvsg1giq5yydaf3yfkynemlycz7u8dnjhugjrgpc3ws2jg5zy87srre6xju1mer3","826":"vkskl21xwyhw0ssnpknclyn50gcwusl06049quikckn0c9001knol2d5vylrzgxc4iuy9tdj3rq7vez8v0o2x78rhtn1q4ylmo5s","827":"0dzvd005xdqumgbezs9x4as2acqjlt3pvro6sx922iwj25ti7bt6e2q87a5840t2kt31mrz6go341aym9x3y4uqv1rxqat6c1l3z","828":"ob84s8u2u1irfg529caqsi3bqd43nbbwpmae4ha2776telgd3nq4r1qkvvot63sxm4147ym7fu4q6hsuj59aror7t3k6ck72285f","829":"6znboph7a6xfbwdugjygsun679grjt8qs3zn8xrkgosp5g3e96fi47mx9fkftkgx7nc2pvlkjiyu6zp7e1psf9xz5ko59uwxovmm","830":"d114pwxo3zezexm39kdamriiqni42y8zi24uyrgedh6a3nhsva9zbfrb4bzi9vynzm5o22szb3ewgyxfrw0f61apcvlmmx3131di","831":"kiajxo9xsw5hjun3gn4s9ct3e57hvyjrnydlg2x685h0w2wdjp10e92b4xdj355kug6ezjhtxk606dx78k8rf8vpurp9x0cmtxkc","832":"88vvvksw4vl382dwj79zo8dmaotbcdd5i9ygb3e311hmhhkn33om6jnlvmk0q3bfuui1sjk1rf1mqzlklg3od0gn3rlwco8w9ez3","833":"m07y7ij6vvveq5f2ucvlq17990musn8uql07pnkvnkg5v6cpag5dcr95s7l1h1qp91rwjk22m9iatt1lrnsv919i779kbvg03odp","834":"qq4louzyny16ii2vblctw9rkny8mwufwzfg64eb7t0h7a89gfnwxryumq26f9t13xnmpjbkryoba7nhz1fkv2wsk9yj5rxx53n6p","835":"himlzxzr8g6r36c622qdl0x4i54y78sue216vchcw59i270zuaxqfgu1s88izbue3scrrku0kso7kplxf94llgpcgoxcr9m9csn9","836":"n4dsxsunnoevgzevtohu9uf1duabp1bnjwcsmb74e3c08o6lx7jsjg4eibh0mg89zc926uhif1h1znghh0tx5midilgrufvbyo1f","837":"lwplxk6ywe0gip9dy8mdqe8e3gxy8o1uwdzd4ts0k42kwuukle4q9wt1464u80fuaumjmbp06v7usomk16btuzszjzk98iwmlwn5","838":"g1ir5vappwoaczutzzm99t8uc6xzvsd5bjlnc7hnp4xknie1iro1ogtp0n2fhcsq64yr0bd1vdh2alkua0bhb91m3qxi0mdoentw","839":"kunx9jraioxuplpixcwtbuu1kw82kqt68w0s6ic41ujm9p22fxulqx4uguzo7ewecl6wk15ct57nc3wxy2hhfx7oihs383qm39d5","840":"snbbd2kugpjz5bl8r22769k45fqc9yizh2tjbdp3ra0oknn3x2kn8yc9a9b1qxp6t5tylc2msvbhtpgb4glg7pphfqxfwpimnhdv","841":"4ytirj87se6igc9zyn7klfl6k9gsiscke5c2pthus2dilaw6um20qfmbqi7oygydvr4oxrv65goe8gvcv2f04aotx20axmd4aapj","842":"6v5f0lwu247lhfybq6ms12clb98ti1zs4n8f0mz30hes7tubnna14ei9v7ri5bxb06k2u2bgzchwvl0w8ndshp883xfxwans38uy","843":"m9nijsaak8t8m96hnrfxau2851e0l6iqwiavydu6c1uylalqvuz09c315kn3gxde3y2rfiuunrmx0tv11amt5afsqg847hjmv0yw","844":"kwcogkstmcplgwblacm7j7thldipxq89dyis1r81lgxtuipgwpyy6ts85n6x5x8oc1kyxopjmq43u9ekxzhrw4vianbabfd4j14e","845":"ry6j9fwgq63ldpcmj1akbvq55bx9jxjjz3x4weggctrzksn1icc9klo10lzoqmgnzinkrep30ncm32fcf6pr2agf79wmav8j4hqw","846":"pthsvp36sklij1pmifyhetwwu6by6xykrg8dptznmesywy75b11cm52tbzf1c1cikks3m5d1kodzyrn6y5e0pi5q5au47pde02dp","847":"06ahzgfkdv5fz82l9spy87lkwicuc4h7c0m8yi5vqt001ur16v79w2b54u2cxu8dpro1uvk1xq8xzhh82eiwcq7me4sildsks3jn","848":"m4cibzqdvauyrblmb9mkg9pfto4n0s7iobh2vfw6l9anvvtath6gg3sqbw8wbkjfolqrr689ls2voisnpiqstspg7d7rr7znnb3m","849":"gjwr5si0tsyk49wp6km5y725h439tu11c2p7xffk70mutdbv8q45c2v9eok7zs4tvzfhflco84rv9fdykgq8txm1pmb442thtanl","850":"cvx36okhq6xiydnuvgh95rfxa6l4j83lpczbpnefipn7b4vi43y9jlw722gnsgt0unan7a07szylcgvlabvf39rtocgm9w2teesu","851":"me3p7rdd70kgus5r9alkkhgp5dlv3g222yyrt93uyiucgo0j214z8th7wl6qksxr1v2arx5ikpvosi3cn349mknkfr8i7gnglvui","852":"6xnzz7ztk3sgh27r6emq0l5a7gdftzjveez3feu4e566i3alqfl2bdc8c0385k6rms9r7e0tnn1h43drsekh0ttig7syws4xoqcy","853":"3ejael1wi815e31egsmwhgyxg3r2wgjk5j3oadd5s226oby6txctp5tlzpe9ke38t7qyodkpu0ml9fet323cbd4a92zpt6bdzkuz","854":"hvq4svzlqd1fi4fpy4pfbsrhrte9couwt7v37l83np2ppz414zs21401zondm6kqkkvvdsadb1ns1dkvtil7rspxc1jej2jmcq9f","855":"khxxpcrkul9dnxl4xxqq20oii2nq6pmo9tqv3i7eq9ystfgx5962gw010ml36kl7x1w7am4cup2nq79ahqbizhwo1pnbwut8retb","856":"i8vgb3yqieortrf29q9r17rngerjjlyi118x7bilghh728qtmq6cxlobh06j930a6pgjog51kq6we64b56enojj9djhef9fm0flo","857":"ynhxebwj7kcckdj9k6st9obcldn20wtdyx7pvjphjvtqj38w8q7j3b6gd4brzy6tjlr6v98npyzmbamxrguh3mpm4n3r0f0rocmy","858":"6y4iycjj0bo26hesytgd1b1fv91n2y1jeabyurp82fszqpfot564e9wdsg00q9wesk90131fueyre3lxdgf9i2xn5h0ba98t4d1z","859":"dxk17hqwd9ji9owy8whpepyw0sn3f37h1p76g6vk2xzsu2kzj2ndwv5lqr7xb3epbph0jf40tu2jnxe2rzriebzh608dcoueqcd5","860":"sm5579b4y4ud04rmfkanux9sye42pzlk2xq5jfsc9rqjhh8kqgahmaf618trnoqt5dgb8hqxju9d5ttp05xjfy2i4s08uvhpx0gc","861":"v36lao80kluattrof08zsyr65clz6t0dzusxrqvk4oyjj8wmm7ftv432ekjk49itaaoqh0m9plb7isf5rkcrpdebprua774aqvno","862":"uugd25s4aklui7bfg3o49groa0makc25kvahy84h3jfocsw95wnpmjlsoh57r9rudeyilrz8bwv3jsir910xzg6wbk45bv65nnrk","863":"hgn06oi1clcqyps7bnqmr7jrchohzifbro6lazm0loh1qje56po2vt0ub5nmqvoewtx7bu2xs2nyyaatn67tegyq0hb9739njxs1","864":"2jqxcfxw346gyu0uihs799vl4v45joimyvnl1z9gzmslxtz0luhy6hppiydshme837f4kr2sp3yreco3xb987grskvj36z0efnky","865":"whi07uwtr34z934n5r6xnkl6bln4rrdfnv3xoabu8g9x66u81noydr4r0r1rakuwpu59rn6axkro1umb7rs4c1n14w2di7umqhuu","866":"imy4w7pvtkrqe5r3kjggk0t6faunbk4k440nef70ihz8qb8qm28c4banqztnk1n9ce4xvhrnhevix8ekqhyat8b4ryazk7f18ztv","867":"ons4nonjdptwy1deuaj2na5yte9o4zjrh8f0k06uhb5gj0da1i10rm2kdzc0sw42x8itol82vo96smx9sbwx8br07pm8ot5zvtz1","868":"c393ku3gjyo77z0vmuqstftybu3dtrb0fhf1mo7h34dmmpe1n0gjll8czqwpo9o41j4dtgjfnd6bmai5jrgakupwwiqa6b3zw66q","869":"n568l81mzziz7szzznq0xwrpkizwlmuvh7upkr0kb9u7q4b2q1b3ha39mbxkpn47igo8idp0ij0iml1n5bxy0cpmkm8eaut30udl","870":"ogtpoatowrhmy0123j4f5ks9c3f9ofwcl9slxg1uvdwrq216xwmwhlwzmuxjsu3ygs0yer5d0tbw0nkat9047xa0ziqvddxhebhw","871":"ko76qgjb4dvpa12zmaku8859zmm4fdgq8owtmtuc73v2qmqm2u5nd6purwzki9atvd0t1xr6my67h8pnjlczeur99rzwc8rw7jzy","872":"7coyxgd7ip4mk0qt5vfrkpx3lzt9l7gsf00p10zufe4pflgp5qyzruiyek1955j0tiuwstqyb35m44rraqu2wbfmvfzkf7wrma2y","873":"4u45z67gv5g1aypwn3rkrrftx8soq651o4wo7751jcycp65nfc8136ycp0nm44w3lc2rn8ek4u1so7wtthwq21agxbitv238je4m","874":"cpxli7myffi6fnhelh01ie7mxvbe0an3coxtgne40p7tb8jjfpxx1g0s1fwi66yjfx13buhi1pgupu93hn15njmtllxia7ovqwej","875":"09crkdydost7feju6p9spdbaa7viri4xc3aqqu2ewmdhv6o9nhu0m2ubhrissdmnh8rcvick9w6hsj90fg2bdi6qdm1rkxq88yor","876":"kiuqrufiftkoxjn7vm67c6hsrp5rblv58cfhvc6554a4vgped5kvz4r7hh717oy6r7o2d3lnmc0yxq451qsot8b25pukqn4yspb0","877":"vswxed30ybe7zdqs902zd2t1dzzscgbdtzjp40hmdhjav1zrkm12h3n25tfr2yok3mpscc3czur83plwmvevrbygaj1ymypv2hln","878":"xk81x5393cpshr4ecwcqycm8hkox76u8q8hs30l6wc40rc3i4bw6pgvfota77ab9c96ol3pab21v8khgnv1myzhj6tmu3l4871yx","879":"y15236vavpu8hz5wd13yfb503h3rqi2p7ycbcvlo4y3obrfx1iwl3lgpk0ihag4n0murx6nsvqzpbuhfhqxcouu686crhles6tdy","880":"rqcy5yuwnch7tcjgxchefx77rsbuelq48jlrvwenmp5xxgx1cb749757bwj7y0cfhpq6hw87b6eytonysujwc6xda6xsa6hxevd2","881":"j4t9pjpps4w16cr49xla9asklu2hb7l749xiwqkrkfyv8ri8r7msz7o5r1ikgrj0ss2k4i96s876cwjq5xnld2mgwbi6cuctlxd6","882":"19goj0bo8nhnj96k3k3tuo3h7f2z9528u1ksub7g1v2tw1rgwvaly8ks1eoqoc297s6gezsxsdl7fgpmppojz7elpf9s9d5eor2t","883":"rq5o6kk7skv4f7kpdqy9uzet7deep3ww1v13tzz2f4i67bq2ox90rwevmyw3wo9gfwypd6lrz27vs5is9mu851zr3lre789xpqgq","884":"ejqudxfcg0ajg6ww1bi88ggrod3iaf0iz9mxfgny73wltv00t5vvea75r2oj2ngbkablj3tcu89iunxrt2pix8buu21pdo755rxi","885":"o2o3d89whis4vchfa8hy3s8iaoefqxu5hm6lw8i2dl8o6m2dk8elr0rgtaa7m4lwb70fza4fe5k13k5qzvtbzd5s2f0zefqca6p8","886":"1u5twzo6innl8xa0j9ydphy722lll4r3fo22cewmlr40t18c7hvsfgweqa3a31gx5gk6g22ohh1l5ly5nzq1bg3t965luvutiyby","887":"4jw7cjn8yzrjs6iha96m0grk9g62pprjsxn8bcccl6cw5njh8p3tn91iuu8smp99qpccv052wao6wpgd94ezf56uki37bx5y95r9","888":"hyxoclybv9ku4lr54xlmblccsow98428hdxduzbwkm5zxd6231aaltupuzt86g6aokehc5ehdk2imqroa3164otcbs39bmvm4mzw","889":"iggd0tc62pamn9zda4nvo91z3y53mfdr1j2i8wa70gzo6i86a4747lwb69q9oxsp3jphkbyf0mo0txcj9lwi6ekio9709s7kxhjq","890":"ond4cocj1fogu9vggviqs4b1dq6i6fm0qkwjknej5z1vobkck6woufgxei9ikgu3wjgpvw32zm1izkgju5aiguof0jr67l6w2bk0","891":"zbzhjykvjapn0ffs5eipunwljwpkgaqqgq1g01hiaokq0d7i4jv4b6heo0l5v3fynddlkcbdb0kpb95rkti3dr5rd0sk56wuejrb","892":"dg1lfws13pbh9pr2dxk7y1e19twtk2wapohvf1ng2htykouvi1xs4nud2l19nf75d8i79sar0v53ir4kma6diwtnulf43z1q7z4r","893":"78frf5felszq1psvzztahg1cu1nq47uqhdgq6azoh4r89m97ylsh7hzxbrngf0yq4s3ozylioopdtt69oqwgsq1os4vxo75a27vo","894":"yyq9gjcfo6hdaxfhz875flrbqag43yetrntta35c76mw9wf4zn99wnit47qcbvu39h6ofqwf8q98ooyrqa3a0ehgw728ux68950r","895":"b760qrebtyhbnlib77xw8b0ic8q5j4r78t4fytlhoycm4j64becaxvvrx93ofp9bik9vl369sd3j2sv9t2w3aawcoa9rcyhetcei","896":"q00r0ttfhpn229b3k1zwdhefpgwghkoftjlt895z7vfwegdme8ja63bfqcyyxiuioxvi2d38k7duob3thk3guo6v97c0j1i03od7","897":"70g9jpcg5mtu3hsyp5gcxv9ilurdsduwcxmnlv1ro371uujzprk9104qwryuocceuwn1lig90e0azxfdldzi95wi2ks3llp78syj","898":"mu2rfxw20ucev4o4vb0vw3txrabzkr6d9768pz418aggb8gv6j87ztcpncr9rkn5qz1ib81mjkfxac6y3nj9eyap2x0b8gqhazdj","899":"7fkr5rdktjbi2n6xzlg9pplz3gt3lhjw9ivu3txz7p5fm9p7vwkatjbt912kag9zyiab8lng825k8gntvqkh2dsy7vjhssq4fqhg","900":"5zxju06dovr2qijtosykn73h3kk74191hojx71lj2cyfy812pnwjsgoy74148i8n7gp9oyhiezismb2k8rs8qs840maoke4yf8yf","901":"n33p5l101r9e67pzbo06t76c34s53nma1qezbujbq5moum0ek1tarrm9zr6ydbbpkyen4w4ypjciurbic6h82qxedt6nklz4t70v","902":"dlngm279ftcrbrppzjekyiioqhn7qlwhtf28ot766h43jp89o5538r8exaacnxd10xvh8v9vpg93bbl6gnw8ac5nm4aueokeiccg","903":"7nzghcsqra0b9peetec1ruta0bmopm3157bniydmayrs1e2ejp2ks1svdnxza9tqujr4wz85683q9iq1i6vmqftje43tk036v42s","904":"l177h7hv19i2rsemlur2ld2t14er4zx6qbcwjbv0io8q7qnbk37cosw1044h7jz54g23bbx8h8a6ecglcxlfn0rmddzdyw7smrx6","905":"ckjfhe16jpg5tw6ty13f1qso74vresx9yvqilhj5o6r0sb689h2a4qa72b7vqepycbkql3bs23lbugmkr5zstxikln7y6xingj9i","906":"su580jawbbtdova0qeis3k00r427yuuwuso9k35yhxyhhcsrmqxdhoeu86ic6jwfx1d7e5ln0l6194edg0vze8pxegw9ylsngcpx","907":"x54i314lpyn9ysssttivm5lzriodluke5bjflvcrb9ec8q8ikfd5xlmoae2bfnzme1l1vitwhbwjmzv8p9j0iwrb337fwxqbkez3","908":"ql21hpxjdkchh17c0tls3kfn9d4mq3f4oajtb5u7c0uf2qu4y6o2clvni4zf6l1ctgwynhwceldoxoouxd78940sd3dsn6gx82a8","909":"ijz14ql7jh4exjffl4eeel4ujrbecdhi8ainh110v2fn7uhtdzcz5q9azx2w8iein4chrx6c46g2bi5hp476cyx5k32v2yiq4vk6","910":"o6rnx8nkd5bwpntpcprfry26jtxjtuvry41xnyn38nc7wvgfsxwf4sbbqfaobubv3wedz8252kpnn0aq3t21ljn6z6tv3c7rl15z","911":"x8bn2mfu876wssjwigm5ff39fo1cah2uw0rpafj2qh44jc2t1ru79fmteevayaefobnrni1vk0jpit61ooq2c1lknsjf21yhlxmr","912":"oxbqfo1qk0bcye3wls83ghcjwyswiridfnqzipvyl1h9pxutu9i70m4pduvpne61f96uak69ynymb8o92u39efs5978upl406jjp","913":"oqu2nd56p8o0iqnzqlq8u0ejom9esymdpknhbb2r34hi6g1y2686km782n1qabqhkl36p6f4xabtyhts22b4ql1i9vq2hi0foge8","914":"inhjqcuoewbmo580rrr9oyyj8wgh4f8kwelszrv48jyjelzzxz5iqfcw63j5mbr11edr5jb7wtdvtch0t55cmxq1gmr8syypmj45","915":"5ogeqdj02r32tjjiko8e6933lvu4cp01vafghdz3sjo94pa5i7glmroo2my02v49exy069hkkt7pp7e1xp3gzw83fulx6mi09je6","916":"udk662j3l8nvr0k7mzn9cgbrtais8s1ak4xpfu9i4lx88cqxus5b4z5pwsztk4is061cqu8k94p5xv9ux2w98db37zrlsfjvz4lc","917":"qmlovpejm21blsyse54nthlsaif5e4s7lsysydy3ag30i1wtikeat0yjrvdr29am3c6in6uueahd905x3cl72fkddhhh8yfweivb","918":"fl7h0pgq6sox17asr9kn4b5fgx3vlko6pmbd3xiaut69j5tcbuizmbjqe8pl1aza3u3wtnvq2rnrksnzh2wh26q8dzu3hx8q7uow","919":"hwblf64j4id3rxpvv6guzupv8zaz9s8twn7vs6g9rghyn4cilrwk10mvglkt3d7fiau22oc9sj3tmr7nk85vwsd27dgrnx3jh23d","920":"2z2pvqpyo7lkb3xe3p7o0jp6co77h3647iw2yxtrizx5uug3b0ma1nf792pk634sjvb2tgkrp4ks6jyyzgglv5rlyt1i1zfu5088","921":"41j552zlvc27eeogp5yxv5lhbsq34lmx18zsejrz66qn6u5b3ihucj6mhh3jqgctt5z8xcal6yjpgutv3c0l96l1d81lwm6r1mle","922":"z6ijtmfa82gihziwo4eolzdqv7bfgabnujctmzk50a2bnbbvqte5vmjv9m16343lu6289oq27wl530740x30yzaxismdlvf22gxj","923":"jj4e6cda2c1y0z7mg4369sgv1lzhsb028ac38kj8cuekd4frng7kx34paqx8djnl37a2nnf0hny5n6g6lx4wy2sbvcs2q7i1oii7","924":"vzxf4wyhdhh6r72mwupknbc2p62yjor2iumwk6th2hzfn4avyw4w0xjh5pkgltbfvz7ayxi4ukstn7emgqhylhydz0ej6yco9qls","925":"v6li3eb1ys216a9rywkipbzt0u4c07ifsnz94jyzixkthvz24ih4c6tdsgp07epnft8kspz8v4bwlz4pb57rmturbkt4967dd5md","926":"gsvkrsv5rxem2611z7zfxvxo0bqojjcgk133vrv9m2knat67kzs5g6guo0jepxmtatdjazdu6kb954wu3filgxoxa9f3mzz1e8fc","927":"csauue74tlsmv80uu2wcbl1adsmqlde7mguktj9jyqv1s9v70ddmlf8p2vsme2okf6sjach2tb1s8hxzzweq36e2gm7wq8e6s8tb","928":"p4v1pelvojb7z1jfbsbur4x3z8fpsn5mli6m0ozmtkazt7b7s306z0uehu67fr3fugkggiydgsm86s7lkvm9dkdpgvyswo9u24u7","929":"07xoj0u96ba2jz15q2qvkuap7ksxjp2u51fwwsygvrnhiuegq73s7akbv0ziha35qjkrk268kn1okfwqqj6hwlbkc6qyf0s9y140","930":"7v0wi48w6m23qdrpu7jp9a2um4i0pcb6yn96m319iyx4bbhs6as11cvdrezy5mm8o3nndzkjffpg2gtc34uk416s6okuuj5493zd","931":"r7lpzg5lxo5jmk2fm7qgth1q11qg2wh5m4ao5c0ckv76h9vkdfevnlfqvdvksbhof71fj65vp0pmjuti68ppmu7bdqkcj44dx6gh","932":"rp8y89heyuebn40g7avh2q7rzdm6y55vl33z8c4sl4x8sm2h4qfo7kkhdyo19ddoahb32r7v4focibj3qynys98ukahj7p67lfks","933":"zvcicttq9yn2n8n4ht7gpug4mvwgicpg3w1q02hluzmwjc2zflsj3170hroaveg4dcidevsi80jc7cgwt1wz9ojsmw7j2v3zhn39","934":"yh4q8o5dn0gtj2jtqihzj6pfngrv5chth1zulxwqzg7bgz9y6jhlviw936nicegjiugea0op7at3mqlevizs5xzrqc2peysk2tet","935":"aprfep21knj1g8sr39eo5g17gw9skitze16gomoz5z7ioqu0912lhkoll5w6g4o7ilxmsaawduq2vrp8tzadf6i8jdxg7xms0nf7","936":"3wpuwet9f2toouy3d5hyqiaggj4yrje7qw2bbhfwsmc7zqu2u0kpl3m0zdh3u7rfwdmxp4poer1mp42rqjpnbddhlqyr3l1g1gp0","937":"275jycdzfpwi8lwonfgkvt2tujcp7zy6nb49vh7tpriw1hy6v36uuwp1fyxj3fwqlblmt9yzq0gtjojrrht1o6n0izkz9itpbz31","938":"b002li8hz6vcvxhkcpztfvzx8jcech060rv12248zcelijw85rzg1l0osesptwa7xiud2dn26w9wkafxep68mwn8hvfuna5ltivl","939":"1lzdqlfu9vi4ljzsiub0d25jib0la7l4odx905btb1aiool7qn7763ehpzjp9knvdsqzh9ui7n5i0hgotarhb4x2q2ciy6aqmutj","940":"vm2kiq0t2v30mvxznicbmm4soa15j3n2dc9bdcha8nsxf102w460jlprjto4k1e8cmiofmfvoym2h8xfoqf61c4vnfoz9wnxrza7","941":"nwpb1rzf8n4hg78yj5xaz1ahvl7fggs2iq9jh8iz2akkuhawbopybbit3o7gzqau72ulffm9y8dlcdods8fr8ja1v8vf3mzpmowt","942":"j5hwv7oprjqy8hp71l0f7y7eci8jdlitnozh0ihpz8emogilltd8h7qq5g4va2rpd65w4xm9vwcrpafw8erk5w8j46ibwle0umw3","943":"p7c4q2gmpz5qqs095oojezv0gxa0ge5yq70vmrjlzcrtme8e0jasjcdrkl9y1ffhlsqqvmllarw0yq01016b4qgeirls8iplhldl","944":"8vsi8lyjfaylzntpb68sez1403myrb6gvhswxb81i8pyjwx4988pra0gamv8q7133ba0anp9amtw8x76ulfsn8glsv3drzo8w7b2","945":"2mfzcvszl4fokd3q1dm7029kispx41bwaba88xdbdvrzm14qgj3zkpz8vrsl1u5lcjnc416c0rl5mhjsav1infxpu2y30xsfm9td","946":"q72icomj7mavtb32sko7f9651ounxo3dfbu5wl84p0lyeimct2pnf411uiv71n8py2mdjttoj70f3qkxzlaf4ksy4m2a6u1rrz9v","947":"upiknl09vauzl2sy9ppsichcewv4phgd5hn28lcuxsjgv1bt6w9huqu6r5hyn0hnfu8xrgycxy3f8597jjqugusbd7ht1jo3i6ej","948":"wrfvqx7vw4chewy0g7j9bs9lmkq5f1xlpfk2ve4z4onu52lgd32ymlo4bkt8t1pued2zpweuo0maoxplj73m7ezhk82shzg3cvbv","949":"sl4fcuuzu224axaqdyb3ld57f6ljjtpgrhxz0cy49tdf2lxnnfn5jsaj9rlt0vx011rvfarmi6ak00353i5c6am3v8w1d94vojtu","950":"d82ag8xaxriuu5sdgn3o7od1e4qabpssai6yu2u8ucfu3sao3owkm05j0lfwo0fcz2peidt53n949glwe83l8y470oufhyc4hv9n","951":"2k4gnpb1fr7dimp8i502mb9lkwwacq2e6mzdlj6k7wudjyr37sktfiegos5461wu64h7pl1ofr0qcp3himmph6m8qq4wb8k6anp2","952":"2g3jmo4tx6scr0h3vr2kgch1ehwfk9ptz0lvt1fkgl61dsobkajv8qhmtuhuoe2kheg4n6ldqafgphvoq2ss11et5iu8rczu9kqz","953":"1vncmql1q5lx0yo87wda1sgz0c7h2032uuy62ohmb92waqrx2ntc7x0ymkv6y6iws0bsrvq8vuzb35k4xbsp95w4l9ucnlrr7fpp","954":"azbotrm015ip8jsuffcwr2avrije935d3u2qex6dpdqy096w0p3k0e7x4t1oqjexbhac35rgnp79p7ctkcgvuavhjh8ck060i01p","955":"e2jkrd4phyc3tpnkwokjk61xf8gm4u5zhzqecwj6oci7c7dfbhbovr4ssiw2uunjx97mpzx1va67dcss6du7142oibdjb83ct15a","956":"mqev7ssxivca34cmt9tqhai0o614b6v5cnxsbe04gttl6vijxykd0pb2sde2b0y3m4jciery0fkljvtiwgx2vituxcm8pd024zq4","957":"xttkp1m9k1l64prygl22fk1ox16io69xu2fdpm8kzk1t1gecf17pis0t5ahpwfs1r7d49b4o4fsvej2eoz9ncvej8db5zsg2jx0i","958":"3ech5lhgns301tjp9aas35zmn7a45q0ly8hk3rqgcqaq983zc6dhrlc5zuer6auqfkp6krj94t9ois8b3xqop2sm8dz5cu9vgf89","959":"uv2o6pqtivwzy3y0dd5z8kap61b34xkbihsnjwmsku9zfo0ro2gbivh2mc0l38tr0iyjucn5xtzte1lhr7rxva94jexbux7l0k4h","960":"y3n4b5b9twfg8uqzo4l6fa1hmg40w92yedqke6ujl91gwu0xuq447od6c5j060mzqs17s1d2cz9rergdrp6xppgmiu8aj0kvsj8x","961":"uf8z8onb6y1yzz0s3u9db29v712zo4ui01hywiy84518b51gezzzvnjmexau0r82oul85fnujocq412rs6h7fuockah5w9crr6sv","962":"ie5qfm7u7kxmvkueza3utv7djb60pud4slxoa8kpgmnsufdahjx0wnz69byzf6kn17tilhclyt1bfa0lkqn1tor8dvgywcl6b0di","963":"olo3l5fi31xlnbkinewwgajfs4mjxhj34mxqg2t6js6hjlkqkc01riuu1hxid4d3z678fk9liudrh766h4q2ymep84w234mgxl5e","964":"8ikxr2zl1zc0ywnn3zvpe332hxryitdio0m723l2m6tozrbe8lfz5e3pn7lxztudfwtwmjb29dyg8vl1jfzfykpdcn05f45t878n","965":"97laj3w2kakt9fwdvz1vtsqixm4pflyugdhf52v0x3ovwoytyj9buzauaq368413ohegyfhnqii4ydx9yjljrqjex2xpfbb9g9be","966":"z2irnvi0vhb0gz7yz57lmjs3jun4an9laiuj7ybynefwgm81zmq3ao0qzqm7498scz0o7fkj0q8770a3x5gae8i8rqcy07j39ezw","967":"z90t6i9ontiudkjvoj7yap5ur8c3ub01lowgajdm6swdw4dun2vz6z8g6113ieqv6m4z2n51mpdd9qzcgmfccar5nxe74dvylnkv","968":"271r7419vorchfbbik0fnc2xbs0ttx73178hgbg72sc6yqtgln6dpi24qc9tync768780pr2uoedtek0fdxwki82ravh8mppli9k","969":"5vkjq6bfu76i8i9z2yh9007v0eotjqk8h3p07dc19rnkn5qrg6g9ki3ocq9n5bgt91415a44ha23y5qflxn0t86qor1bpubrkovu","970":"wq3hvlwovjulnzmk9ai4gb0qvxq7u54c401fiiacjtj0znpbywxoeejrdch4jo71s9qetbjxtec8abkfqfdm2c4t3nac5tou4uto","971":"5afwhw35f8hkw2xx61op8g0ol2v5jgrn4uwputrgi1y7j4rkuqbppo3hlxgjco8hk1sllrmth54p2nnvfr9o1btj7vxbf5s5z1uz","972":"1psfzeka0945i080m2ribwzu31a4867wu8cgeilabzx8bi0isodjj5i4syi6hdik808u7rvrt3izk2oynhx8s6416ajdtzptzbau","973":"1n7zm8qcb1fstphyc6c9zzok3ic9t8batcp3z3k2qfw5wj2xjmwsj0ihxg1z0y4fk5mewhk09z9x0olcnrv9cmqi813addm6j4l4","974":"5x97linfzhjci7hiu3jzbounj63um2v2rnbc3exbdock3qktfku2a2lu6gutdyz14k26xjmzuoh7b2bu3cg8d52eixus0c5gcmml","975":"hxv7l2b5fezs22lh73glnwo3s319mcyizc0kpipt50l7yu3kmgubr60mrd7zrszbuu875ugm1nv1ia5gs8f1fb7hislv6enu0mms","976":"x4szkh45mwdx0gawtx5jahsyt3punczvedx9tepasygbj66rvqwievl3l7xzbofhyb11t36e15ylny92algz6gd9g6fmrff2b4xv","977":"r6nbo26ftk1devjjat82zcurmyw1cyezut4x5hs68rmcb2atcje3xtmo50d9eki1qy2ckujkmmr02fsi36wxgufmnlquy1nks75u","978":"36zp0t8lxr8tqc482z49zffx7cyvzhja1jcjcuwytf5kulvgr1b0rwjnop412udorwwkqcqhsq9y6j7lrg2jis3x6ndszsvaeye8","979":"nce0461i8vfchan6vmbardg6lcau456hg158adph1194dahnglhmo05zg1ue58ih349qxn3himpkhdsqgo4sxspz2y1gbaiy9sgb","980":"elseuq2fnqvd788wurf08y7sec5meq3bbrjbc7584f2xaqyb1fcar5xwm546x5xejs3pge809zmsnmu3xin3os5dnbrgrna6wb8h","981":"1qk5cbfs4qpzfy6kdj9ss4bq4mq2446a8aav5eybwm9ejew9ibo5ne37bgf8o3tqlv18owad13zkpcv5p88614z83vzyc73xsa1m","982":"4oxuzrr3aftaf0xhhux8gkkr7awhoycb5m837grrb8wszjv11961vbpunwf20b9y97w9up8ifsuk0yxxa9ksgw479m15oda2w2kh","983":"3p342xl8ivnuq62k9rkvpcwom6wcokj8qmtnncrflqfig5x14m7exnixxf5ryv6dv927vsolqleq3blp95azgj656qizrvlpxjgb","984":"5dvr5j9z1o1cvct49uspjzwdlnet2ydp7gauj0ejp3du6z6dq7ljx4rrkvoaeq8810y8g0npur2uakr39tu0o8kvf36xlwzix224","985":"72jxwbmjtp1vfpnok82ayra3ccb6lgqqqvwujn1t2bxhp3i1gzgz8cn1j9n1bp19khyqquj5kl9vdbsranlhiyxgio2a766akxcl","986":"eroaw4hee0jilvb1r1syu4sfodddubmvhdopqf7zn13u9g4v4tles0imrkt7fhcvrfzlo3whulubcvzccm1z4h2h7w8mh7s58n9i","987":"2i78q6hbwtq5agbyl4pxvcr7vgx56yeqerugpltylr9j90uyfy42pufacn4dd85ki3i8sgd6s8t6uff3ptzs61o1kcnlghqcb9x8","988":"gpurhv5mrw0b7we9f2gan12e3evlybka3b2tg02euts86ywq3m883dal84dnggyuqcn9pidfj81aett2g18io8wlh7uxpydsbcyl","989":"o7t43sjw81jid1we8hfjf9v9pc755777wygsh0fy2aacnkxuguipr8sgpt1y67z284wm6m3oej3gc6t53xqwrn36c3i7yipc3tv7","990":"xuob6idhp7mmpkmc382bzn97nd4xu7fz5x82cp1ua267c3xylhbm5pk4wrof0wvv65lfphqmmvkvjvcywwrvarar56lm8inemsrq","991":"2yzr8wfqcnodh6awm8zfqmk4hj8l4v880amt3xuvoep2uf8bdyv9ns59b6k6zrxykv8mse4akodyy5ujnlhqiemgzmppege6rtpr","992":"8bd3u9clr4jr6se539r7ryv8quiejigat9qq0nsa96ejg616zbj25t8fixmv4kv3oowrfw4sgggqroddp5rmlw6s1akzwkyub6xk","993":"pea3ivw39ouvguusnpwcm65b68b92wwvu63w915ojfivgmbtnel49itcgo0dizg9xujvebod8llehkztowi2uu6ntpsvfe0qzh8h","994":"19wk0fkmtneg0cjojqduu8uk3l4e55y4hmpnd4r29o4urhe09rl60v90gcb25h2wrbhihtdvl7vle12igl7i4kpgh64gwczpgnji","995":"s9xttp24pkv7kg0e8tu9kbzwt2k5dylc2or9d3bwnh1nivftaudkpjp28aw46bjy31g7hnv0fblyiqmcos20aoa398s5fbk12t8j","996":"vtzqwkl7ehvn7x8k2wmjgfahjjc2yfiq52wo47ysp802fa8vmr75x7cfnbok0ev8ir0qxttwl4oi7tir49d659o4dvly1wlqvddg","997":"y55bwzpod8snrp683az73wwwt6ecvnra3uqugca4hml7ojdaf32rl7si8b04evxfurerlrnkvheuxuc5d5lhzs7484vwiof8qu8u","998":"90nzvj7u7yvn9xqeiz894mdc1lica5umaf833ia00t74scb6vo3wo1hxz6g6we7hprrix6m9oxmqm2sniyjy58zqykw2zq5u3aqi","999":"vckchrmp6rpyv6gxb4uh6gupkowxutwheafxyadnm6a8takmy6aiswcvzpnwqn1w3pwqhpj4u5ru1cwdgmgs48vmmvy9ry82fvu3","1000":"316byj1gz6u0jxl44sa6w4lxbd7sxps0fl6tf3ndtymv3ukejwejjc205xhjs7wi0kydf33rmfa27vem8s8wjxeeowh82pffqelo","1001":"1vwdfr8bvyd04hgcrn9l85a9s9affknnteb9cdgassxqu2pafxwc6ey9nlgnr4p29z6xoyxob34uzd2umgsbz8ushoverk74tdbj","1002":"2qqoluknn1sekkehllo548e193c7sm67tedpmuj9ckn8t133ysml70kv2f00io61o8wl03bcgy13ohmag1oxw3wzz8oil8irpssd","1003":"2wh70jarqyittq4vdibof4io8uq34sx4nqmfda7mpqvhgdyy2zr9u5r4wtvyi9xwv6uzqqdi6vu1gcn91q8xdu40gmnbik25t618","1004":"af6qnao0exkkg2i8uvq25z25d2wi40apz6yrrganzwvwf27lc5i0dmgabpch2vcx7jq85ol9gto11rdpow7n8auwr7neadnm91y8","1005":"mb9c1vql8ix3fbzd5iztkwta3r43q9j8g9l3n6ldbyartll1107c6yy54p30u4l4rfvdrzc6e7ank7m56vla16d02is5il80pelt","1006":"1vp2vms4h8g52ofwkdrpsrfoikikt2tz3gqk5rg54qswoi8o6its3x448sb7dop3pz4fxdpey4gfxwv0x8bk1fxuyz82glp679di","1007":"kax8ccci6m4318tqd4ail568zuesfejyz5riat3i4wazuyk86utebp2oeif4is2wcx7k7u8nfr9los82f34fp89ob02fgsae67fx","1008":"30rtqi91s7c2cfcywdgcwms73r6ujp1hp84nxoghscniye411e6ctmjw7bku9tssgyvsanuore1kr557jt4q648jr9z39wmy2oj4","1009":"d8nypgeq1i699vb3e9clt1elyuols3cbnbqiy4uc88tdvdyre424qhphysxn8sroy08utdkaoo3tto0rbv00wiphb7lav9lytxa9","1010":"q21maxhgzkoju0ltmlvngci5myt758ayyk0n6hpflkd3p6cbfhvc1gwaypbiby29l6emxwzhl3fextqfpkaygi4ke5xy06nsgprn","1011":"57n4v8bx4xyxb7tulsq5p5cw7j90vu4856fs3cbp640czpx6t1whb9mlj72do85zl7hk1a46ymcwa1wsu90vb8fxnuki9g2tkty6","1012":"42m4mbk8zdn4o2j8fa9w289srb2dvoqd4yfgi3rckqf9mqjtv7837y0yxq7xz724e9m9a2lx2hnp7i3u0q0lf6ayxpv907h4xrk0","1013":"zfjrgz2q90kef7u2is3um75lecidx0nmon113vu6pntbq91rra2xhskacs1mymcwo5bdie8pmtn3cxjlpbiljfv3iooqw0obb1ay","1014":"fwm46ndy0sj3ox2jnyljw9j88s0ghn0kaovetyhhlgnc0ye00knv8gv6dmpc2j77bcbo2goftdik0f0z94zoki7bigs3zqf6qa2n","1015":"fjnjn550gxegqz13lafp5vcdibzh33zx137nhdbyamu4jop2z207vzfltkwwd4oumwjvo3higbr4soyxsf0lo861embklviwopq0","1016":"mloj5g4hetidzvwb0spn20ijzfcn6muqk8cpspirn2eo03faamegv4nnikly93byot8o8454wh9ilwqpiw6elapp3mi0z9g2q4gp","1017":"kdpkoai3yqhhdo9zazh1ek6jl6wcq1eq0c2pfp0r786cajohiod73kxei81lhn6488as37hw8chr9m1l49bhq3v6rmaufys0dhpa","1018":"d0zj8ny7ig7c1hqta30842yz9z94aa8lsqg32ycb0fg1su29l73o9q4cloe03urozdkgkupf7p3c4yqk9iydqucfuv3ypw79wnqv","1019":"meeh7n1ie7a8hrdeuxzjhjpal1vz8nr4v269dtxt445jigfxt1qvxvy9q723b8fdhvlpqq6vwla7fxyaqcepa3nrtdo3q0fgoe3l","1020":"z2agdvx4wkpv93c929fwj2q7tsc2fw973ukcvms77vjc9ecyzzl5xc7d66amq0brw8wytpd4q9w5l77wattoa9p4hcd08hurrt53","1021":"txirvrxbc3l8huui1sk3xvwe3xbwsfqiuiivvuxqm1epf0zl1beusqhjb12cvrxba1d2zcorqw5geaeemmlpdopvsh0u78b2tfv6","1022":"eyqodex4qlozrt5bh4tl4c616hp6e05h8n4bzlrspleacfpa5k93fubgi42ki33po4gf2lfdwqr7fnr4c655faz9infp15l4pqj6","1023":"2iojlz4qcfe1ur8lq0a7mrt9l0d0bf5fubf5102e8ofo8k422vdf4omhsb0elxgif8jor8hwqdekole5ikpfftvoii6uu05472fc","1024":"kcmfk6uz12upomjoq5qmu0wz630mha2z00g3yf6mb3uxu51qh8kr90r8e607jo97y6f8psg688c0fy1jrpdxq0o95jti2bbsheki"}} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json new file mode 100644 index 0000000000..f16c3e3e5c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-10kb.json @@ -0,0 +1 @@ +{"id":"add9df95-3525-412f-b782-a0dbd5d0bada","NonSensitive":"dfb280c7-62a9-4d0b-b0d4-14f311fe0ec8","SensitiveStr":"ecb4222c-d335-4508-a051-654955d9465e","SensitiveInt":1692274068,"SensitiveDict":{"1":"9qze47js1zpiabsm6rvw8opz5uy0g71bvna2nv17yv3d61iegnsyq84oh3alpmvfep8jo2hhjavj8naad5zflzolsc18iweub7v8","2":"5tdm5pa1jl2ez2jvbgm9egtkincfgz5b3xzhxsq6gvwa7d65eyay6zymriq9r2jaosid4fxwilj7fxho58c8q48zhzr4vxuzv26p","3":"f0xmwqestj0d1h1es0g1rg9c8h9owc4sdhk5ul1ycnrmj6ic6owkl7qcgjs4ijgpc5iyujxiy7rd9l7ommu16vfd9nv2xuesgi5e","4":"847h5ttzmrm3dg7ey6bsid5cszjj73x0sno0ewdl30g09e5ebq8y1x91eoy04ua5kurxg7rqpjawbdtbbojgddkadnzyhkef00ed","5":"0xrlp48ceyvtfx9299c7595ae52kokox9qv5ecbjxedu5527kkqk7r0f9jpuw0xxxt109grqlem1xxsdapd1f1z44rvw9v4ggc10","6":"nuun71pabmusdv53vef4vygbypaewtpvfakbzqeep5fayihxjh13nzlhap0cdeyq0a8fi55j499gaecfux5bv2r3en4704lkt9xb","7":"d21yrhbguvwro2jzmon21trvbarj2bc4pwa5cvxs5gurhefvx5x5ufxy2ld4lv2jzgqiyhw6r6jl7620v0zizk954czfczng73c3","8":"lbvelwr350ujsnht9c3f7ckyadbq9mosbe9obfnxu3438yl0glgrg4cosvvl9gag8zkj4tscnx8pg0sssju6jfcbv9r71xpmgof9","9":"lixhbvlktx3s4202rssta5dvgtjaxzuhbrmfrllox39cb5nfau4cje64ag4kdiu6336bq34ci06c9xrt1sear4q7htvd2b17wzh3","10":"mz7umqc743raldw7tink26gqrncj3zai6zqzzy7azjuxmp136cge2riy8crc8bruwqemov2sha4aoa05zg1ahlfei50wfcj248ky","11":"tnrt4pwho6wosdtt7zz72vyoznxykda40atnyr5va1vgxc090t00qho0yy7iu3rgru7osm50ne9qth62v4afhhr31mv4banlbsui","12":"mpaae9i713xwtj0hatveyh3rlxm97emodpckgfffa72s7ufzt2tobh42dcr8wmhv6neojw8a2fyp6s62npb0k5gar43iwrlca0x9","13":"fdneqqcny6ubyirv96rbz1x5zknqh22afqee66vxwvy7dwc5u0hgjauzzfadd563bvmptedj181c6i1qaec6y9fr12xpgx8lqjec","14":"af5o4bdkpgno2oct8iot7fovousiwd19lsct2h03tbntppvw4qirhyjihnazuggtef5vqn46pl3jrviwrddve73co6gbr4qzv1jp","15":"zwe741if01gtebnuseeejzioe7wzi3h3y5hy5tmavagwqtdiic9i75yo0c64ezballqoh313vx7ga1k9xbixr0lgbuhfu8ksrj4p","16":"eahflfvs0x5mkvxia338mh3mm3x59jknsam3gnbas10vutod1dw0f41blupybowil5y2kn3oslsgmoeikeenwx0uap0dv0co3178","17":"rdwythux8nqx42bhbbzsy7bvjcgevelktbt5aw8djdh1vfjyghzjlyq88618txcz3qjogm6j4lfxkdrpv7yrwowsblj6ma5bj5d4","18":"39q2ndc6wpxm5769z7gsknrrkhiu3bvrxngs1qdmo59sv1joso1tq3z5ozzqs3obyuoff2g4b5aztum9eesr2i7oc6m31cbfll5f","19":"k9hzf4kdk8szu0qk0mpbcr6dll8jf1r5xlstooqzs88eejzhr8brhael2l0zgeiv6whpu85e25x642l6ps2e2n8jb0afzwr3b7wx","20":"qp7m3l10r4chn05fwomvd8udambaad20gy445xsiovuv8ewhp4z4rltul3syepqhepqad3katho1kufufeybbzjt5zt85gad17zy","21":"v97uza82ru2jq2urqft4kridnnsruk85n4lndw8otyjn8n7ct9lrewomf5depga2hrptu6yqifuym5oye4gkp0qtrgbhg1k22loa","22":"1v30p1o5a0wtuwp2mftybbzphlg3xpw2jpb60zepv49j5tupvha9o2hletqxmv7vl4l8vyyq1bc7rl0ifjrrbff91e6fcdje7wet","23":"xif7ligwdstp3a3v4egf1cgioyc0xdps9pir8i4tgngzs3xqembjmzq4xrz4odzruddse00hhmxrfuicx55ig87420pouttiiod6","24":"lovq7ainqta1ozmiyqzz01nolwny3hr6f5asr5te8l3eog4uwfqb7adrt4wfssdofci1oshae5cmrnkomvim9r46bs8wkjopoqni","25":"b11srpyex3nt1sn9p30joh4in5owklvgqtdmf9oq2se898k88zegzthkqq7l7qi6acyp6h44pdoqk5bj88l5agt796fjvsxm8169","26":"w27wt6o22x41jg38ha2iqu82oiruw37w1b1u5macmxqi0t3kdh41vfu7bgf13blr4pbd4svsralppddft7k3jdpz7dfzxa49ldmt","27":"j9f9unyawqa8ci0be4nqsbnwi5a776c204k15ons23mhys7bc7ykm7ttecsyxtrixkuntxlk26qpgao9u7s51r857aqntcxt5ktq","28":"1gpsx460e7spiuc30enkygbltmtglur3a64st2ee5d7rdf5bx9xrw0hkfqc3rq293mvob6b52b7e3lk6to7bffiylgstippypzog","29":"3c4alamoof07jbl0gw3an2e4sx0jkxl1di0s32ie1jr5yomt3etdj09ck95r0h58p8vrhqxoqetdw7apxvlttbucx8tkezq8cr9x","30":"j2pcy170lm6luc1nu01ugrlcxsggpkit3sirrp3nf4qvf470lkgv2hp6c2vqsbkmto7s06twff9glywum2y3ivl0n1k333it9du4","31":"cgcli39dzt0hj0ursaxakfkpka9irepzvbsdx07ogoyl9shwhixabk1kahlvkcpidc8k687fwj9g08fnyux72ai54q9zs1w62c23","32":"5byc3br3q6l724ituqdsk7qvvjny9c6yp77n6a45zydlfgi9fbbv17ho0xthkjz5is0bvzqkcqwdf24gd3h6jmm2pubp5rirex9p","33":"q9oo1ccklal9drk1lr4dk85alx4re21xl01p6o5ofisukfscryqklxiok34jtq53owqigimbii718qvofps7wl0ecln7cbf6jlbt","34":"5ckzpge6a1cg8z5cicknjs1x55n8b2lcy17gu8uro97tyovxxi5a130ubjz19j3zwi3l72wv3sen8lq2k87acium45kx1cu2qyof","35":"tz7qg132w5qi569inb54hn70wwy7sbufreopspbtdm01elq1ozl9qa90pq0wbg9aid03yyifr4yk711pjx7z10lotp06lupypwry","36":"73dybhdkumsgmoe95rruec80v1y3qo415zxrwq1ekt6afgch0hofkmbfrkeqqexduv0rgekc1hbtap55nollt1f5sg9clla2xbgv","37":"1aqcjgkm3dxsiqa87kjwdfynlwqoe9mdc29felg1a20pmo784tfr29uv25xoa6icelyxb3forx2dw2aegt816ped3w6z7kl8tupn","38":"t7zubumt7woqcnv920fw8v959m1vy3l8yllp9ywelm7rgfb1h6s554h754bsxpdfjglqq22cn0teuf57plfbkcrtatcn8g2f0sy8","39":"qp4y1bpe55xzscelojvqe6omjus8bevbut3haht9ufl900z5w6ddfudq9c6o264f0f91iamaye01w5u7z749fgoaeaft0s3giv40","40":"hk99cvejeyshfiw5jlmecn9a7d3viwrhlcx58w6xm1v79c1lpu39pz7kzs0ubo04tcgu2rtls6dlcv2xou7zd9zbrlygra3nzu7d","41":"jkh5uwir6epqwj0qnt8jpbj85im6iibokeaqdzpp8585o8seuoje3ioxtn5eizfx408i7p02cogrllqxanw2cz8y39q9011v94c3","42":"dtfp5kw38hfh6te0y1rrgoe8qk94pqp7cnqi5vin7a73254zcui1cjapk8t8dtr5gsklakhouqm4cp1bz25f5u80zs4n5bp8h1d1","43":"0leckg1prka3sbvbhya0dk3rpwqcak8rwejwh6pkbfmc68nh1iwgfnydyrxhz2p7ygszbel2h6atla1uu6gqyze6q1eduuhw80zh","44":"gtiktm0lqhgzz88gvgov6sns4zkmibp1q57gk3vupq6o0t0mx0urmuouoo24xhk06sjelc18eejdb3pwushfw123if9h22ecpoc9","45":"kzptbjo5rkh7p0jyhrcy296hay7t8n6zzgaiorsidh1oomfpb9e5goohvhpxw6ynaqpu6cyb6nhjexgxgxvb6gam3bnui0ii0355","46":"k3q8fkon89ppant3swis0tt9dcvmk3inyzurrdc5rhg13vn66eip0vuhq2to5bfb2j78wthudjcz6px4sdg5lxfh2myjx595biob","47":"vj32urvjr93i45nk4t9m00lkhrbjrrtwqfogn8xc277iu8bbbp7rv4enwbjtg4kg1n8svxy5jejng5tqzjw0z1nnhje1nrxrdu94","48":"0fcgvl3dlrwbjkk44r055j6vmkleqzg8cbe2ne4qxq2plrue93nh3vjd3zh08h5uyw9s95du5b6kvnk8gufzmb5q8s3bjemfy700","49":"c75714w3b5flq5he0uso6wdgybxv3b2t2kwyniajlkae1bvvhsyg1pzcvzi3draxkrwfkjdbzosw506wpwywisqe0tstt8g8ta2s","50":"sg3psireybv4etr63qpur5m4aezzeopo9w9bigh6iqiwndpx2dtij4rjgj2dithrt5cg33qfoj9f7xq16c0jl62jvwvjoarrt8xt","51":"3g9efb5t9b0ty5dwg1wc7xudwvkfngx9a845z2pgx4c5rbdvgovd2k9l77svabimpfbpznqeryii7rdnu0vkeeb1or52jm0lxcij","52":"nvy08acdgmhtiy0iczl1da8o8mrvomjd1tgf4y5eboxfvrrmcf0q60vk6f8tcvxwyoukucq1w4l2ju31yu6nfwog52rjeoizqpbs","53":"4ubre02694gvfb6dqzxut4t1scpuojcswtrn0xue0jw14hxhoh15xa91zk7yhrw4uhzlj4xsrf7m18pg3cf8x7kdx9negsqa2yd9","54":"1ia31ra8p4yumhakor8w4ck02c230s4hacyynpjhhxzrus2fqz9ke16lubyxv9i6kceayfqjblu4diaplh6lja82gs4q4q6w51hh","55":"x48ku1h3h5o4omqtw1kbbi658th9bqrb51u9ud62egu6im7kk8dt8wz1oymgf04pyk8c4qj92346ahrzptovtapmr4efby47nwkc","56":"5uotabm9n5byj9gy3rwsq2hsg2taab7by2698etbj08t9zb9u51yf4i17qiio1ng0t612seq8zcz60w7r9eyeejvucg9vgg2avsy","57":"4dfn56jtup5d3mx7johhhvbz9tefgwlouslw5gfce8xxuqsbkr2x65ykntp19j0fllnxkunk64sqj80k16kznehsx2rli79elba3","58":"dnj3du5zta5vcqsz7plwe0082wk7rn2me04xunjfkmihkemgam6gsirhfadkyvjopwvvxkhe84ianyqkf9mceur97q3mwv2dbzxa","59":"kgoj6765r8xzs0106h8cwz5vu7f1l0ddvofm0gtchldwauqu2yid1hywkeru19yxk5vnjsuv5nfjl78jzqodz92b9geptc1prw12","60":"tcaue8mbhkjsfmftnhy0qg22ccy0w1fmy8d1i4xpzm9s4pwb6ivv5yw2jh8jwj7smk3mtmh9kur1t5vsojr6odhgkr1l91d5cu5f","61":"cechvcezrij2ea41rabsmiz1avp8dvflr3givg6dnqfjs0cr7esi7pb7mft3yc5g08y73795bbsk3jjhr21iqkzh6fkkbwdh2rr0","62":"celnfzj1uipd25kjh1jl74q7npkp6gntvkzkjpn5ipax4q43m062ddf4gg8pvzs6i9g2mbqphpptygltdkc5mj305dh97tdv74tf","63":"8kxk1r9zrqsv3zb3v3bkf7zxclxfydkmvtzeuh51zt0w3c1aop3zxq15vo6w5idrd154ca2umlbdpfq5roj07hozpx2u1qn72g57","64":"ogqtkhmr1df4k29yvs57ayiqght7rodqxui67wtpsmfmva3yw9u8mfloe4xt72pxl1b620jwfresmn1f3bjuayutbcdia0cxs3xd","65":"27phu3a62d1cj9wwqyz524vyo7bxoc1qad1naob2xi0uo3dw5blx8nim5gthmh7tgrng9f3rdmi5fbwoglofum8glbhzmdvsj2th","66":"hynij3cgqcggezuszytkt7cnv0qjq97lhwbirbuv1l9vqemjmrqovkbu74w1060nx359pqwwfwgo3iyy2hl3cay7s1q8darqng8o","67":"kdnnac968jogx8scgufafvwvrj0iyk3evw17zswprn8toukdonl8mh69cx1jbgn3mqm1r3v1ydeebfqg5oni5tubuljbqp13jb35","68":"q7bqn04nirvo0ls2z3tz1v2i6k01ke81w434o708nc6ajd26qi3qhd6r4dogmsw4su1fg0de0qpvw1yck8enw05q5m6zd5gipi2n","69":"rtn7298ijs0k4kt6fwhbfs8njplfx42ugyxnbj4aa5gz9wxrla2jhr6wc2idxqbwx7t4fhtn8ljon30ayarhb46u1505zcpd0s1t","70":"9zntysfwmzq1rvsi2t3sr2zh5udpy1ar3d6bqs3j0d2cgmnt26pdudw66j9rxf1vi1ztm3lb0p4gd71tmcvbdukkzo71vdg4xvif","71":"yklqa8rzjpv4o0246hw41botq7dls78itt2o0f5ihywvdurdj8icp6ag3w21uc9umx1gyicxxb18dkc68fwwp04yfuhzjt20teu1","72":"d4av0chqi949y2dg8nscntr6dgpj2j41ypt9pjctkqjmki23w15mr1inwkup397kx2phq97hjawv0l9sxrbkx6yc3id35l4tze3k","73":"vkne4012phhsdcsn1xegy5eziyxi6qtuse0z49bn7p9dvg37qn91x6okpmbklkwqur7swhmg9b8xomfv9komk8mbkpkz2dbx3wp5","74":"72rfcv52r3bvtaymza4kiv0thyvsxun7ha4yf6f066szwf4ysic410uzf03vp40h21g9vvdgdsgmqmdde0uy5ypnlsl8fhdgrp2j","75":"b1qkpvkqyf5ylylzzoef8ps90sad8zjkbdqb2m3iqjkzn1mxq7627wxwop56q8qvd1cwyh0ltj76aemqipi5hqqggnsd80afursg","76":"d35v3cunt8s3puqys02wft6uqxty8gxo77olhqvvamm8u8bs999r9gwpnqoak7gkiqlgyk0td0uv9yasuxweovng860j5d2fmclh","77":"febksm0249z1z1ebqly3bfwprefs3uxwy95geqhdlncbkudyzoy0pg7iske1w2v6fdlp5il7u9pt06zsgn9te16pc14yi3ezafse","78":"6hiv0nk4emrswr778udshxtjar9y0im36gvm2sontq9qzr718afv1xlypu4libs5nqz5d5k6h7ixdefkoi5jvuou1i7noxzz0cna","79":"eid2wkk3jp4m8j2g2aufm3tnhdesisj5mroqonuxmhovuj9cc6215h1bgxmla2ezf8khis0fpkma9oleyqh56zyppp2a4of2hwye","80":"u09hgdntqkycqckwhe0ftxgxfoqvqmla0ssvarygkfmedjdjbfuyc39ol6g56x4lbsklbm0a0g8bjqbql396xd5cb233vo0edkb2","81":"yob5q3d14pe8k4ix7dsd9x4dw0riiy84jk0694mdics2dgvhpen9x947okfh51l8adibi24q42n49mba7yczlumuadqspqhudp3n","82":"twhpcolc0nvgfkwlmyaedbj49gqzadvtoczodupznap3wgmdbkaaf98326awb6pmb6msoz6q668m6lzqn85hin1i9u743f8g8isu","83":"92v8gj43iy18bpps98ohky9we9yazlp4k4bk1z9a2rh3r7e0oa4igvth88el7bx0sur4tf8dqza4ufy3b2cnazejyj3ndqcobgrs","84":"3uwqalut4diz6xjpryaaw9aetvc1g0ajzxnypchk619zxj6kgrapjhefcsx6qoqr2elhxu418h3yf5sygzi4yh2n0xqzwy7nc9se","85":"tn2kfq5r48e27oh5g3gcnmqhwswip11a1obt5df83kuphgb157ae7qagahemhfngq5rozlwtvkwc5loh96pfrqbv70th63skbdqt","86":"s3p0q74e5f2fjwryhw6vdtm3vke54q58juq7c6bwu5m5siippv9t52nq0y5i31u1wkvmeywop7omvvatestblya5mn0z0lig05vp","87":"zdk0fm2p6xvfwmh1l6mv59om77x0bcpsocfe2hvyh4c6t8w3r44qjamiau20ccbm0jvvhk44hgf568fr9tvkabedefdne33oov9b","88":"p3idf0iajvfvck7az0ddjogayn89hblmaqo8b95rqdxvx5rbsnm4txoeno3lvtb4ckzpyen3k3cs8o6jzxreh1qb3706sr9ly5g0","89":"3fmuct4aupjbuclx22phlz7b33ew82whxntcirgcn5r8sgmc708ibjn80gtih3yfu5l7vyxbb5htk3svps5jrfmakqvurp2vnq8p","90":"25516v2ifoxqvivlpd1x6zrlzfojxyg957g8svamggl6p132sbvjirapnyf2cyekqyotmypattyk34n61072ov6sd94lhr33emg1","91":"xqf0i8yokyw98i3p1ef1ucyv9g96v6vhwx4ztkso6irzcll54t20kho4wrhpt2xont9d0l42kl5m7xuapu1s6aq0mqmq3bl8ubo2","92":"fohdadlq4fyzxxq0vp33uetzxnlxi5bdl3ihfk9grugjykt3netq4hbr0uxw1zfenjfq8b5r8egas7b3q9bdyih1o6bddebp3gys","93":"g9tobhvstwkcat294bbqtzwq0kpb2xuvxgyzow1014oe8b57o63v3l20u36aug1hdy2tqgk5uy94d7i9yv95r5wtulsqzvp97uzg","94":"snqo2jyc5dye8ejyutpw06di5tovsv25chak49ws4vzeudzmf1ux0vwp85lmr0moj5j0190ohjfdpba9exsoiiwjlicd05ak5icf","95":"9egy7ndw8sr49hl9rt8sh0b8qlapngix6pf6ysjcwrs37o0l8n3zlqxfnjzepnaxjgf6n1qu4morw8tmpavd0j1nck58lk8i1hdy","96":"g7vz9wc75rupaeqdev0gcd31nmc7iv0nf2rljrexslbis6dbm7p2tdoq2fufb7aic8mh30917mrzbsst461y0ozeqxmv03f4xz4m","97":"biiprt4jdw10dyhlxdmayxkksz30lwlt3kzw8a5nd6tp5d1v8ljdlfo3gkd8n07oxcfi2mkd58frjiwomnlju2lhgx2r8nsbntpi","98":"n5zd3mpm7bv2y8klzlq3lb5ma03j1hrwsjif51y80f60r2tk8pbk79mycz77frrljt5ddh2nbe093h7fj2ad86y35h1qvltk8lc0","99":"3krxtcd90i41pb66820rodjdopeparzaouoja8g6229gn1m18wco63gq1nxtiv6xt1bv6rs81ogui7y5zukn7eosr9tevdxwvtip","100":"6iorwaxm06cd6l6bstchrg3brbtac7ofaods2rstpya29t0a2nvnjjlnuvarwx2cj76fawfh5ukc3b4ksu5ifnkjspxcb8k68c4t","101":"l5t2x74kpznm9fy8ix1zrbthnudoafu67t9op6r975gws6f022xq1utd69p9vd8k9bh29d09e45vxogtlmff68h72po4ljeanki3","102":"1oxb7e6s0ecx3lpsntxmi5qis6l10a6japco2r82bzi3d6g5p83wcfeog057m496tlz42bd5c73z7opji12kr42a6c0mb6tdaki6"}} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json new file mode 100644 index 0000000000..0a6c195ae8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/sampledata/testdoc-1kb.json @@ -0,0 +1 @@ +{"id":"f5b1a667-0e39-4177-986c-d05d38e1eac8","NonSensitive":"80382481-0cd4-410f-a5dc-44fad3a935ee","SensitiveStr":"8a1506f9-7b15-4f8e-96dd-50033e540260","SensitiveInt":1737683325,"SensitiveDict":{"1":"7oockvrrhrreznj9lgq38e82g96608slltnab3hco7w5alehsmf1yp4r68kojs159ycuqq9syfqvsw3gjmprt0yhbtzoityuro5b","2":"ofoz73i909m7pqhg93o2a05ozoq9x1n1k3817t2vwgym1y3nbzf3d2njgej3iqutlo4xcdyri4fyjfirflpslwg2zfvv159vujex","3":"6371pofp39fuye4mz99vdkc19wo4l1ze7ocgbb12rnwozqgg9g1zqt6fnayhlgjmdjdab4xwgycpjkr8jyn4bp2kr18lo625bznp","4":"hweoq8ktlyi81z1h6j702cted8oxrya1dsh0rm5c5by1b69vmw5czcxqx6aqlmyf20yiurbzgqylns57si31qmo76qyvx2vggynb","5":"tuqwaitw9u230otpniu6txocfi1l3k6mm62f4tkak84wuf5c979qhd2wksx84zieutzblqnpv2n8ub7uivc92kpz7pfk9huulrhd","6":"l8q06w89xgyt23pk0pizu012tjz45ptfpozh8enn3z1ipg00di1n1oot14pjqluh5hlys9s7hkkkzhwv29ob5vvmj2saqptdi4iv","7":"edjikhq40zza6rl4cnn0y92madl59ek7qdiyeszvbkbzcjyqzbjw1dsuni1qspumqj56iza25j839czjt0p0yyl6n7pg7ekfj548","8":"2nqoipmh21552ymx1wvfef4t629tmb33cjom6orbjaa1sb4b7f5xagonllk0q25vqz0murnaseha01s99bktvjcltl0d1892qd8e","9":"gag5o0xxrdhwq7x6m9r0eyp9jg8f6oarcoizpr4j7hlj3ry8n7e5s7yseh8g7n5dweuqcoulbo0gb71hng5c6p73qfddhdfadz1i","10":"wxb3ykip4tehje4uikysdru15n9jvyh5fgcsl1m802lfx6c892vxt1ryyfrlknfp97a0antnfs3vpjref21j0lystuno14t7izg6"}} \ No newline at end of file From 38a198a22128c17980f424672633a4e9934edd03 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 11:11:13 +0200 Subject: [PATCH 04/71] Add non-allocating APIs to encryptors --- .../src/AeadAes256CbcHmac256Algorithm.cs | 19 ++++++++ .../src/CosmosEncryptor.cs | 30 +++++++++++++ .../src/DataEncryptionKey.cs | 22 ++++++++++ .../src/Encryptor.cs | 44 +++++++++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 21 ++++++--- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 30 +++++++++++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 30 +++++++++++++ 8 files changed, 192 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index df416c3efd..73049fd9ca 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -137,6 +137,20 @@ public override byte[] EncryptData(byte[] plainText) return this.EncryptData(plainText, hasAuthenticationTag: true); } + /// + /// Encryption Algorithm + /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + /// + /// Plaintext data to be encrypted + /// Returns the ciphertext corresponding to the plaintext. + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + /// /// Encryption Algorithm /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits @@ -418,5 +432,10 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length); return authenticationTag; } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 462bd56a1f..20dda27ee5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,21 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + /// public override async Task EncryptAsync( byte[] plainText, @@ -67,5 +82,20 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index bcee51d0e1..c505199dd6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -29,6 +29,17 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract byte[] EncryptData(byte[] plainText); + /// + /// Encrypts the plainText with a data encryption key. + /// + /// Plain text value to be encrypted. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Encrypted value. + public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -36,6 +47,17 @@ public abstract class DataEncryptionKey /// Plain text. public abstract byte[] DecryptData(byte[] cipherText); + /// + /// Decrypts the cipherText with a data encryption key. + /// + /// Ciphertext value to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Plain text. + public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 1e7f272ce5..73eeaa0890 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -27,6 +27,28 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Encrypts the plainText using the key and algorithm provided. + /// + /// Plain text. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Cipher text. + public abstract Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -40,5 +62,27 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Decrypts the cipherText using the key and algorithm provided. + /// + /// Ciphertext to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Plain text. + public abstract Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 68d863114e..0377c4257f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -12,6 +12,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey { + private const byte Version = 1; + private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm; private readonly byte[] unwrapKey; @@ -65,7 +67,8 @@ public MdeEncryptionAlgorithm( dekProperties.WrappedDataEncryptionKey); this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( protectedDataEncryptionKey, - encryptionType); + encryptionType, + Version); } else { @@ -80,11 +83,9 @@ public MdeEncryptionAlgorithm( this.RawKey = rawKey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( plaintextDataEncryptionKey, - encryptionType); - + encryptionType, + Version); } - - } /// @@ -125,5 +126,15 @@ public override byte[] DecryptData(byte[] cipherText) { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText); } + + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Encrypt(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ed4cca0596..ef53408b4a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index fd83ef528d..525b6b8d7f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1763,6 +1763,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -1776,6 +1796,16 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 85b7bf3f36..5526551480 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2248,6 +2248,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2261,6 +2281,16 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } } From e0bb8bf39a41454e7ee8b89426c2a433bf4b6073 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 14:24:46 +0200 Subject: [PATCH 05/71] WIP --- .../src/AeadAes256CbcHmac256Algorithm.cs | 26 +++++++++++ .../src/CosmosEncryptor.cs | 10 +++++ .../src/DataEncryptionKey.cs | 14 ++++++ .../src/EncryptionProcessor.cs | 21 ++++++--- .../src/Encryptor.cs | 28 ++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 13 +++++- .../EmulatorTests/LegacyEncryptionTests.cs | 10 +++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 20 +++++++++ .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 +++++++++++++++++++ 9 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index 73049fd9ca..a51e09726b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -26,11 +26,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey /// private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8; + /// + /// Authentication tag size in bytes + /// + private const int AuthenticationTagSizeInBytes = KeySizeInBytes; + /// /// Block size in bytes. AES uses 16 byte blocks. /// private const int BlockSizeInBytes = 16; + /// + /// Size of Initialization Vector in bytes. + /// + private const int IvSizeInBytes = 16; + /// /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) /// @@ -437,5 +447,21 @@ public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cip { throw new NotImplementedException(); } + + public override int GetEncryptByteCount(int plainTextLength) + { + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + throw new NotImplementedException(); + } + + private static int GetCipherTextLength(int inputSize) + { + return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 20dda27ee5..1a0f093176 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -97,5 +97,15 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index c505199dd6..65890ec941 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -40,6 +40,13 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Size of input when encrypted. + public abstract int GetEncryptByteCount(int plainTextLength); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -58,6 +65,13 @@ public abstract class DataEncryptionKey /// Plain text. public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Upper bound size of the input when decrypted. + public abstract int GetDecryptByteCount(int cipherTextLength); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index af10de5e11..bb5beda440 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -99,19 +99,28 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - cipherText = await encryptor.EncryptAsync( + int cipherTextLength = await encryptor.GetEncryptBytesCount( + plainText.Length, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm); + + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + cipherTextWithTypeMarker[0] = (byte)typeMarker; + + int encryptedBytesCount = await encryptor.EncryptAsync( plainText, + plainTextOffset: 0, + plainTextLength: plainText.Length, + cipherTextWithTypeMarker, + outputOffset: 1, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - if (cipherText == null) + if (encryptedBytesCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - - byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1]; - cipherTextWithTypeMarker[0] = (byte)typeMarker; - Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length); + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 73eeaa0890..9362c6f2a0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -49,6 +49,20 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Size of input when encrypted. + public abstract Task GetEncryptBytesCount( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -84,5 +98,19 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Upper bound size of the input when decrypted. + public abstract Task GetDecryptBytesCount( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 0377c4257f..ba606e9412 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -104,7 +104,8 @@ public MdeEncryptionAlgorithm( this.RawKey = rawkey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( dataEncryptionKey, - encryptionType); + encryptionType, + Version); } /// @@ -136,5 +137,15 @@ public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cip { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); } + + public override int GetEncryptByteCount(int plainTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetEncryptByteCount(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetDecryptByteCount(cipherTextLength); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 525b6b8d7f..7b8d85e878 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1806,6 +1806,16 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 5526551480..69924c0a7a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2291,6 +2291,26 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } + + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs new file mode 100644 index 0000000000..329e06e7c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -0,0 +1,43 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System.Linq; + using System.Text; + + [TestClass] + public class AeadAes256CbcHmac256AlgorithmTests + { + private static readonly byte[] RootKey = new byte[32]; + + private static AeadAes256CbcHmac256EncryptionKey key; + private static AeadAes256CbcHmac256Algorithm algorithm; + + [ClassInitialize] + public static void ClassInitialize(TestContext testContext) + { + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); + AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); + } + + [TestMethod] + public void EncryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; + + int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); + byte[] cipherTextBytes = new byte[cipherTextLength]; + + int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); + Assert.Equals(encrypted, cipherTextLength); + + byte[] decrypted = algorithm.DecryptData(cipherTextBytes); + + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); + } + } +} From 9c3c276e389db6bd2971d73bd4ad722e7ab88737 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 15:38:12 +0200 Subject: [PATCH 06/71] Revert solution update --- Microsoft.Azure.Cosmos.sln | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.sln b/Microsoft.Azure.Cosmos.sln index 6fa5e4c3f2..b1d77052bf 100644 --- a/Microsoft.Azure.Cosmos.sln +++ b/Microsoft.Azure.Cosmos.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.12.35209.166 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Azure.Cosmos", "Microsoft.Azure.Cosmos\src\Microsoft.Azure.Cosmos.csproj", "{36F6F6A8-CEC8-4261-9948-903495BC3C25}" EndProject From f33538c684911a9f8d472c800477f0f54d036f9e Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 10:14:19 +0200 Subject: [PATCH 07/71] Add implementation, fix tests --- .../src/CosmosEncryptor.cs | 28 ++++++++++++++++--- .../src/EncryptionProcessor.cs | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 18 +++++++++--- .../Readme.md | 14 +++++----- .../MdeEncryptionProcessorTests.cs | 15 +++++++++- .../TestCommon.cs | 17 ++++++++++- 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 1a0f093176..34c616b227 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -98,14 +98,34 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetEncryptByteCount(plainTextLength); } - public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetDecryptByteCount(cipherTextLength); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index bb5beda440..3d93d0be8a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -120,7 +120,7 @@ public static async Task EncryptAsync( { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 7b8d85e878..d7f8c5a22b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1807,14 +1807,24 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); } - public override Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index bb4836273b..44aae54dd0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-preview.7.24407.12 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | -| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | -| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | -| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | -| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | +| **Encrypt** | **1** | **81.01 μs** | **2.595 μs** | **3.804 μs** | **4.7607** | **1.5869** | **-** | **58.87 KB** | +| Decrypt | 1 | 68.96 μs | 1.627 μs | 2.333 μs | 5.0049 | 1.2207 | - | 61.57 KB | +| **Encrypt** | **10** | **190.05 μs** | **5.716 μs** | **8.556 μs** | **16.1133** | **3.9063** | **-** | **197.66 KB** | +| Decrypt | 10 | 215.60 μs | 4.424 μs | 6.484 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **2,261.05 μs** | **212.952 μs** | **298.529 μs** | **148.4375** | **78.1250** | **74.2188** | **1785.47 KB** | +| Decrypt | 100 | 3,139.31 μs | 316.105 μs | 473.131 μs | 224.6094 | 175.7813 | 130.8594 | 3066.66 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 1396e7e06e..562cc0f0a7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -24,7 +24,7 @@ public class MdeEncryptionProcessorTests private const string dekId = "dekId"; [ClassInitialize] - public static void ClassInitilize(TestContext testContext) + public static void ClassInitialize(TestContext testContext) { _ = testContext; MdeEncryptionProcessorTests.encryptionOptions = new EncryptionOptions() @@ -38,9 +38,22 @@ public static void ClassInitilize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] plainText, int plainTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index 63968723e6..a0ecdab14e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,11 +27,26 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } + internal static int EncryptData(byte[] plainText, int inputOffset, byte[] output, int outputOffset) + { + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, plainText.Length); + + return cipherText.Length; + } + internal static byte[] DecryptData(byte[] cipherText) { return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - + + internal static int DecryptData(byte[] cipherText, int inputOffset, byte[] output, int outputOffset) + { + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset).ToArray()); + Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); + return plainText.Length; + } + internal static Stream ToStream(T input) { string s = JsonConvert.SerializeObject(input); From 6923fd2fdb29a831e6eadd01e26463c2e312d254 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 18:17:17 +0200 Subject: [PATCH 08/71] Switch to randomized encryption for benchmarks --- .../EncryptionBenchmark.cs | 2 +- ...smos.Encryption.Custom.Performance.Tests.csproj | 6 ------ .../Readme.md | 14 +++++++------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index d8e4c7d8a2..3655043648 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -35,7 +35,7 @@ public async Task Setup() Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.encryptionOptions = CreateEncryptionOptions(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index e32a42c36c..4deb4d0edf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -8,12 +8,6 @@ enable - - - - - - diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index bb4836273b..2807f8c809 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-preview.7.24407.12 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **60.05 μs** | **1.537 μs** | **2.300 μs** | **5.0659** | **1.2817** | **-** | **62.65 KB** | -| Decrypt | 1 | 70.76 μs | 0.812 μs | 1.164 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **165.23 μs** | **3.741 μs** | **5.365 μs** | **21.2402** | **3.6621** | **-** | **262.38 KB** | -| Decrypt | 10 | 231.32 μs | 4.627 μs | 6.635 μs | 29.5410 | 3.4180 | - | 363.84 KB | -| **Encrypt** | **100** | **2,572.40 μs** | **242.163 μs** | **362.458 μs** | **201.1719** | **126.9531** | **125.0000** | **2466.27 KB** | -| Decrypt | 100 | 2,952.48 μs | 397.387 μs | 557.081 μs | 255.8594 | 210.9375 | 160.1563 | 3412.88 KB | +| **Encrypt** | **1** | **61.45 μs** | **1.676 μs** | **2.457 μs** | **4.9438** | **1.2207** | **-** | **61.25 KB** | +| Decrypt | 1 | 77.89 μs | 1.959 μs | 2.933 μs | 5.7373 | 1.4648 | - | 71.22 KB | +| **Encrypt** | **10** | **171.64 μs** | **3.341 μs** | **4.791 μs** | **21.2402** | **3.6621** | **-** | **260.97 KB** | +| Decrypt | 10 | 255.57 μs | 7.833 μs | 11.724 μs | 29.2969 | 4.3945 | - | 363.84 KB | +| **Encrypt** | **100** | **2,601.33 μs** | **215.481 μs** | **322.522 μs** | **199.2188** | **125.0000** | **123.0469** | **2464.88 KB** | +| Decrypt | 100 | 3,156.06 μs | 321.419 μs | 481.084 μs | 355.4688 | 300.7813 | 261.7188 | 3413.05 KB | From 03ef682ba696e192985b8ddd0632176882bedcd5 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Tue, 17 Sep 2024 18:05:07 +0200 Subject: [PATCH 09/71] Some more array pooling --- Directory.Build.props | 1 + .../src/ArrayPoolManager.cs | 47 ++++++ .../src/CosmosEncryptor.cs | 8 +- .../src/EncryptionProcessor.cs | 147 +++++++++++++----- .../src/Encryptor.cs | 4 +- .../src/Mirrored/UnixDateTimeConverter.cs | 2 +- .../EmulatorTests/LegacyEncryptionTests.cs | 4 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 4 +- .../EncryptionBenchmark.cs | 2 +- .../Readme.md | 16 +- .../MdeEncryptionProcessorTests.cs | 12 +- .../TestCommon.cs | 10 +- Microsoft.Azure.Cosmos.lutconfig | 6 + 13 files changed, 196 insertions(+), 67 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs create mode 100644 Microsoft.Azure.Cosmos.lutconfig diff --git a/Directory.Build.props b/Directory.Build.props index 04fc0bc918..7b9401e44a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,6 +12,7 @@ 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) $(DefineConstants);PREVIEW;ENCRYPTIONPREVIEW + NU1903 diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs new file mode 100644 index 0000000000..b09a919a5f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -0,0 +1,47 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Buffers; + using System.Collections.Generic; + + internal class ArrayPoolManager : IDisposable + { + private List rentedBuffers = new List(); + private bool disposedValue; + + public byte[] Rent(int minimumLength) + { + byte[] buffer = ArrayPool.Shared.Rent(minimumLength); + this.rentedBuffers.Add(buffer); + return buffer; + } + + protected virtual void Dispose(bool disposing) + { + if (!this.disposedValue) + { + if (disposing) + { + foreach (byte[] buffer in this.rentedBuffers) + { + ArrayPool.Shared.Return(buffer); + } + } + + this.rentedBuffers = null; + this.disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 34c616b227..0288ee79db 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,7 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + /// public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( @@ -83,6 +84,7 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + /// public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( @@ -98,7 +100,8 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + /// + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -113,7 +116,8 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + /// + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3d93d0be8a..e7afe78c2a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; + using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -25,6 +26,9 @@ internal static class EncryptionProcessor // UTF-8 encoding. private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); + private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); private static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings() { @@ -80,6 +84,8 @@ public static async Task EncryptAsync( byte[] cipherText = null; TypeMarker typeMarker; + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + switch (encryptionOptions.EncryptionAlgorithm) { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: @@ -97,20 +103,26 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); + (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } - int cipherTextLength = await encryptor.GetEncryptBytesCount( + int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( plainText.Length, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); + cipherTextWithTypeMarker[0] = (byte)typeMarker; int encryptedBytesCount = await encryptor.EncryptAsync( plainText, plainTextOffset: 0, - plainTextLength: plainText.Length, + plainTextLength, cipherTextWithTypeMarker, outputOffset: 1, encryptionOptions.DataEncryptionKeyId, @@ -121,16 +133,16 @@ public static async Task EncryptAsync( throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - itemJObj[propertyName] = cipherTextWithTypeMarker; + itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); pathsEncrypted.Add(pathToEncrypt); } encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); break; case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: @@ -166,11 +178,11 @@ public static async Task EncryptAsync( } encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 2, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: cipherText, - encryptionOptions.PathsToEncrypt); + encryptionFormatVersion: 2, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: cipherText, + encryptionOptions.PathsToEncrypt); break; default: @@ -278,6 +290,8 @@ private static async Task MdeEncAlgoDecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + JObject plainTextJObj = new JObject(); foreach (string path in encryptionProperties.EncryptedPaths) { @@ -288,25 +302,31 @@ private static async Task MdeEncAlgoDecryptObjectAsync( } byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) { continue; } - byte[] cipherText = new byte[cipherTextWithTypeMarker.Length - 1]; - Buffer.BlockCopy(cipherTextWithTypeMarker, 1, cipherText, 0, cipherTextWithTypeMarker.Length - 1); + int plainTextLength = await encryptor.GetDecryptBytesCountAsync( + cipherTextWithTypeMarker.Length - 1, + encryptionProperties.DataEncryptionKeyId, + encryptionProperties.EncryptionAlgorithm); + + byte[] plainText = arrayPoolManager.Rent(plainTextLength); - byte[] plainText = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + int decryptedCount = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( encryptionProperties, - cipherText, + cipherTextWithTypeMarker, + cipherTextOffset: 1, + cipherTextWithTypeMarker.Length - 1, + plainText, encryptor, diagnosticsContext, cancellationToken); EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText, + plainText.AsSpan(0, decryptedCount), plainTextJObj, propertyName); } @@ -340,9 +360,12 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static async Task MdeEncAlgoDecryptPropertyAsync( + private static async Task MdeEncAlgoDecryptPropertyAsync( EncryptionProperties encryptionProperties, byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] buffer, Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) @@ -352,18 +375,22 @@ private static async Task MdeEncAlgoDecryptPropertyAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - byte[] plainText = await encryptor.DecryptAsync( + int decryptedCount = await encryptor.DecryptAsync( cipherText, + cipherTextOffset, + cipherTextLength, + buffer, + outputOffset: 0, encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - if (plainText == null) + if (decryptedCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); } - return plainText; + return decryptedCount; } private static async Task LegacyEncAlgoDecryptContentAsync( @@ -483,49 +510,71 @@ private static JObject RetrieveEncryptionProperties( return encryptionPropertiesJObj; } - private static (TypeMarker, byte[]) Serialize(JToken propertyValue) + private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) { + byte[] buffer; + int length; switch (propertyValue.Type) { case JTokenType.Undefined: Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null); + return (default, null, -1); case JTokenType.Null: Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null); + return (TypeMarker.Null, null, -1); case JTokenType.Boolean: - return (TypeMarker.Boolean, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlBoolSerializer); + return (TypeMarker.Boolean, buffer, length); case JTokenType.Float: - return (TypeMarker.Double, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlDoubleSerializer); + return (TypeMarker.Double, buffer, length); case JTokenType.Integer: - return (TypeMarker.Long, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeFixed(SqlLongSerializer); + return (TypeMarker.Long, buffer, length); case JTokenType.String: - return (TypeMarker.String, SqlVarCharSerializer.Serialize(propertyValue.ToObject())); + (buffer, length) = SerializeString(propertyValue.ToObject()); + return (TypeMarker.String, buffer, length); case JTokenType.Array: - return (TypeMarker.Array, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Array, buffer, length); case JTokenType.Object: - return (TypeMarker.Object, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Object, buffer, length); default: throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); } + + (byte[] bytes, int length) SerializeFixed(IFixedSizeSerializer serializer) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int bytes = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, bytes); + } + + (byte[] bytes, int length) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int bytes = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, bytes); + } } private static void DeserializeAndAddProperty( TypeMarker typeMarker, - byte[] serializedBytes, + ReadOnlySpan serializedBytes, JObject jObject, string key) { switch (typeMarker) { case TypeMarker.Boolean: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlBoolSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Double: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlDoubleSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Long: - jObject.Add(key, SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes)); + jObject.Add(key, SqlLongSerializer.Deserialize(serializedBytes)); break; case TypeMarker.String: jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); @@ -587,5 +636,27 @@ await EncryptionProcessor.DecryptAsync( // and corresponding decrypted properties are added back in the documents. return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); } + + internal static int GetOriginalBase64Length(string base64string) + { + if (string.IsNullOrEmpty(base64string)) + { + return 0; + } + + int paddingCount = 0; + int characterCount = base64string.Length; + if (base64string[characterCount - 1] == '=') + { + paddingCount++; + } + + if (base64string[characterCount - 2] == '=') + { + paddingCount++; + } + + return (3 * (characterCount / 4)) - paddingCount; + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 9362c6f2a0..469d069313 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -57,7 +57,7 @@ public abstract Task EncryptAsync( /// Identifier for the encryption algorithm. /// Token for cancellation. /// Size of input when encrypted. - public abstract Task GetEncryptBytesCount( + public abstract Task GetEncryptBytesCountAsync( int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, @@ -107,7 +107,7 @@ public abstract Task DecryptAsync( /// Identifier for the encryption algorithm. /// Token for cancellation. /// Upper bound size of the input when decrypted. - public abstract Task GetDecryptBytesCount( + public abstract Task GetDecryptBytesCountAsync( int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs index 9fffa1e3cb..30c23a3a92 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs @@ -59,7 +59,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist try { - totalSeconds = Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); + totalSeconds = System.Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); } catch { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index d7f8c5a22b..9150a739cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1807,7 +1807,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -1817,7 +1817,7 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 69924c0a7a..9f63b3545b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2292,7 +2292,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } - public override async Task GetEncryptBytesCount(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, @@ -2302,7 +2302,7 @@ public override async Task GetEncryptBytesCount(int plainTextLength, string return dek.GetEncryptByteCount(plainTextLength); } - public override async Task GetDecryptBytesCount(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index d8e4c7d8a2..3655043648 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -35,7 +35,7 @@ public async Task Setup() Mock keyProvider = new(); keyProvider .Setup(x => x.FetchDataEncryptionKeyWithoutRawKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Deterministic, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); + .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); this.encryptionOptions = CreateEncryptionOptions(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 44aae54dd0..74a360b0bf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **81.01 μs** | **2.595 μs** | **3.804 μs** | **4.7607** | **1.5869** | **-** | **58.87 KB** | -| Decrypt | 1 | 68.96 μs | 1.627 μs | 2.333 μs | 5.0049 | 1.2207 | - | 61.57 KB | -| **Encrypt** | **10** | **190.05 μs** | **5.716 μs** | **8.556 μs** | **16.1133** | **3.9063** | **-** | **197.66 KB** | -| Decrypt | 10 | 215.60 μs | 4.424 μs | 6.484 μs | 24.6582 | 4.8828 | - | 303.41 KB | -| **Encrypt** | **100** | **2,261.05 μs** | **212.952 μs** | **298.529 μs** | **148.4375** | **78.1250** | **74.2188** | **1785.47 KB** | -| Decrypt | 100 | 3,139.31 μs | 316.105 μs | 473.131 μs | 224.6094 | 175.7813 | 130.8594 | 3066.66 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |-----------:|----------:|----------:|-----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **108.1 μs** | **6.13 μs** | **8.18 μs** | **105.1 μs** | **4.3945** | **1.4648** | **-** | **56.71 KB** | +| Decrypt | 1 | 131.0 μs | 7.17 μs | 10.51 μs | 127.6 μs | 5.3711 | 1.8311 | - | 66.1 KB | +| **Encrypt** | **10** | **277.8 μs** | **13.18 μs** | **18.91 μs** | **282.8 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | +| Decrypt | 10 | 486.4 μs | 57.69 μs | 86.34 μs | 438.9 μs | 23.4375 | 5.8594 | - | 287.62 KB | +| **Encrypt** | **100** | **3,187.7 μs** | **309.58 μs** | **443.99 μs** | **3,045.0 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.54 KB** | +| Decrypt | 100 | 4,828.9 μs | 313.12 μs | 468.67 μs | 4,833.8 μs | 218.7500 | 167.9688 | 125.0000 | 2845.11 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 562cc0f0a7..462bc4122c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -38,22 +38,22 @@ public static void ClassInitialize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] plainText, int plainTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + .ReturnsAsync((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCount(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int _, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index a0ecdab14e..cf043ac178 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,10 +27,10 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } - internal static int EncryptData(byte[] plainText, int inputOffset, byte[] output, int outputOffset) + internal static int EncryptData(byte[] plainText, int inputOffset, int inputLength, byte[] output, int outputOffset) { - byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset).ToArray()); - Buffer.BlockCopy(cipherText, 0, output, outputOffset, plainText.Length); + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, cipherText.Length); return cipherText.Length; } @@ -40,9 +40,9 @@ internal static byte[] DecryptData(byte[] cipherText) return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - internal static int DecryptData(byte[] cipherText, int inputOffset, byte[] output, int outputOffset) + internal static int DecryptData(byte[] cipherText, int inputOffset, int inputLength, byte[] output, int outputOffset) { - byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset).ToArray()); + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset, inputLength).ToArray()); Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); return plainText.Length; } diff --git a/Microsoft.Azure.Cosmos.lutconfig b/Microsoft.Azure.Cosmos.lutconfig new file mode 100644 index 0000000000..596a860301 --- /dev/null +++ b/Microsoft.Azure.Cosmos.lutconfig @@ -0,0 +1,6 @@ + + + true + true + 180000 + \ No newline at end of file From 1044a89aa79e98ba08d4a40dd8fa5c39f47d76e7 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Thu, 19 Sep 2024 16:24:54 +0200 Subject: [PATCH 10/71] Streaming deserialization --- .../src/ArrayPoolManager.cs | 16 +- .../src/EncryptionProcessor.cs | 32 +++- .../src/MemoryTextReader.cs | 161 ++++++++++++++++++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- .../Readme.md | 16 +- .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 ----- .../TestCommon.cs | 36 ++-- 8 files changed, 229 insertions(+), 79 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index b09a919a5f..8cafd07c5d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -8,14 +8,14 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Buffers; using System.Collections.Generic; - internal class ArrayPoolManager : IDisposable + internal class ArrayPoolManager : IDisposable { - private List rentedBuffers = new List(); + private List rentedBuffers = new List(); private bool disposedValue; - public byte[] Rent(int minimumLength) + public T[] Rent(int minimumLength) { - byte[] buffer = ArrayPool.Shared.Rent(minimumLength); + T[] buffer = ArrayPool.Shared.Rent(minimumLength); this.rentedBuffers.Add(buffer); return buffer; } @@ -26,9 +26,9 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - foreach (byte[] buffer in this.rentedBuffers) + foreach (T[] buffer in this.rentedBuffers) { - ArrayPool.Shared.Return(buffer); + ArrayPool.Shared.Return(buffer, clearArray: true); } } @@ -44,4 +44,8 @@ public void Dispose() GC.SuppressFinalize(this); } } + + internal class ArrayPoolManager : ArrayPoolManager + { + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e7afe78c2a..53478d915d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -544,18 +544,18 @@ private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedByt throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); } - (byte[] bytes, int length) SerializeFixed(IFixedSizeSerializer serializer) + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) { byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int bytes = serializer.Serialize(propertyValue.ToObject(), buffer); - return (buffer, bytes); + int length = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, length); } - (byte[] bytes, int length) SerializeString(string value) + (byte[], int) SerializeString(string value) { byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int bytes = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, bytes); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); } } @@ -580,15 +580,31 @@ private static void DeserializeAndAddProperty( jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); break; case TypeMarker.Array: - jObject.Add(key, JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings)); + DeserializeAndAddProperty(serializedBytes); break; case TypeMarker.Object: - jObject.Add(key, JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings)); + DeserializeAndAddProperty(serializedBytes); break; default: Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); break; } + + void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + where T : JToken + { + using ArrayPoolManager manager = new ArrayPoolManager(); + + char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); + + JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); + + using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); + using JsonTextReader reader = new JsonTextReader(memoryTextReader); + + jObject.Add(key, serializer.Deserialize(reader)); + } } private enum TypeMarker : byte diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs new file mode 100644 index 0000000000..b8996ca802 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs @@ -0,0 +1,161 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Diagnostics.Contracts; + using System.IO; + + /// + /// Adjusted implementation of .Net StringReader reading from a Memory instead of a string. + /// + internal class MemoryTextReader : TextReader + { + private Memory chars; + private int length; + private int pos; + private bool closed; + + public MemoryTextReader(Memory chars) + { + this.chars = chars; + this.length = chars.Length; + } + + public override void Close() + { + this.Dispose(true); + } + + protected override void Dispose(bool disposing) + { + this.chars = null; + this.pos = 0; + this.length = 0; + this.closed = true; + base.Dispose(disposing); + } + + [Pure] + public override int Peek() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + if (this.pos == this.length) + { + return -1; + } + + return this.chars.Span[this.pos]; + } + + public override int Read() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + if (this.pos == this.length) + { + return -1; + } + + return this.chars.Span[this.pos++]; + } + + public override int Read(char[] buffer, int index, int count) + { + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + if (index < 0) + { + throw new ArgumentOutOfRangeException(nameof(index)); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + if (buffer.Length - index < count) + { + throw new ArgumentOutOfRangeException(); + } + + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + int n = this.length - this.pos; + if (n > 0) + { + if (n > count) + { + n = count; + } + + this.chars.Span.Slice(this.pos, n).CopyTo(buffer.AsSpan(index, n)); + this.pos += n; + } + + return n; + } + + public override string ReadToEnd() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + this.pos = this.length; + return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); + } + + public override string ReadLine() + { + if (this.closed) + { + throw new InvalidOperationException("Reader is closed"); + } + + int i = this.pos; + while (i < this.length) + { + char ch = this.chars.Span[i]; + if (ch == '\r' || ch == '\n') + { + string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + this.pos = i + 1; + if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') + { + this.pos++; + } + + return result; + } + + i++; + } + + if (i > this.pos) + { + string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + this.pos = i; + return result; + } + + return null; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ef53408b4a..d940744205 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 9f63b3545b..94bcd21c46 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(32, unwrapcount); + Assert.AreEqual(64, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 74a360b0bf..5d231e7843 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |-----------:|----------:|----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **108.1 μs** | **6.13 μs** | **8.18 μs** | **105.1 μs** | **4.3945** | **1.4648** | **-** | **56.71 KB** | -| Decrypt | 1 | 131.0 μs | 7.17 μs | 10.51 μs | 127.6 μs | 5.3711 | 1.8311 | - | 66.1 KB | -| **Encrypt** | **10** | **277.8 μs** | **13.18 μs** | **18.91 μs** | **282.8 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | -| Decrypt | 10 | 486.4 μs | 57.69 μs | 86.34 μs | 438.9 μs | 23.4375 | 5.8594 | - | 287.62 KB | -| **Encrypt** | **100** | **3,187.7 μs** | **309.58 μs** | **443.99 μs** | **3,045.0 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.54 KB** | -| Decrypt | 100 | 4,828.9 μs | 313.12 μs | 468.67 μs | 4,833.8 μs | 218.7500 | 167.9688 | 125.0000 | 2845.11 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |-----------:|----------:|----------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **134.2 μs** | **24.65 μs** | **36.89 μs** | **4.5166** | **1.4648** | **-** | **56.71 KB** | +| Decrypt | 1 | 122.4 μs | 2.57 μs | 3.77 μs | 5.1270 | 1.5869 | - | 64.01 KB | +| **Encrypt** | **10** | **309.7 μs** | **40.78 μs** | **61.04 μs** | **14.6484** | **3.9063** | **-** | **185.33 KB** | +| Decrypt | 10 | 435.5 μs | 28.40 μs | 40.73 μs | 21.4844 | 5.3711 | - | 265.22 KB | +| **Encrypt** | **100** | **3,666.4 μs** | **285.40 μs** | **418.34 μs** | **136.7188** | **70.3125** | **62.5000** | **1670.51 KB** | +| Decrypt | 100 | 4,928.2 μs | 441.88 μs | 661.39 μs | 195.3125 | 121.0938 | 101.5625 | 2617.38 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs deleted file mode 100644 index 329e06e7c5..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ /dev/null @@ -1,43 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Tests -{ - using Microsoft.Azure.Cosmos.Encryption.Custom; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using System.Linq; - using System.Text; - - [TestClass] - public class AeadAes256CbcHmac256AlgorithmTests - { - private static readonly byte[] RootKey = new byte[32]; - - private static AeadAes256CbcHmac256EncryptionKey key; - private static AeadAes256CbcHmac256Algorithm algorithm; - - [ClassInitialize] - public static void ClassInitialize(TestContext testContext) - { - AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); - AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); - } - - [TestMethod] - public void EncryptUsingBufferDecryptsSuccessfully() - { - byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; - - int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); - byte[] cipherTextBytes = new byte[cipherTextLength]; - - int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); - Assert.Equals(encrypted, cipherTextLength); - - byte[] decrypted = algorithm.DecryptData(cipherTextBytes); - - Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); - } - } -} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index cf043ac178..81d8683e57 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -70,7 +70,7 @@ private static JObject ParseStream(Stream stream) internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt" }; + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveArr", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } @@ -83,17 +83,12 @@ internal class TestDoc public int SensitiveInt { get; set; } - public TestDoc() - { - } + public string[] SensitiveArr { get; set; } - public TestDoc(TestDoc other) + public Dictionary SensitiveDict { get; set; } + + public TestDoc() { - this.Id = other.Id; - this.PK = other.PK; - this.NonSensitive = other.NonSensitive; - this.SensitiveStr = other.SensitiveStr; - this.SensitiveInt = other.SensitiveInt; } public override bool Equals(object obj) @@ -103,7 +98,9 @@ public override bool Equals(object obj) && this.PK == doc.PK && this.NonSensitive == doc.NonSensitive && this.SensitiveInt == doc.SensitiveInt - && this.SensitiveStr == this.SensitiveStr; + && this.SensitiveStr == doc.SensitiveStr + && this.SensitiveArr?.Equals(doc.SensitiveArr) == true + && this.SensitiveDict?.Equals(doc.SensitiveDict) == true; } public override int GetHashCode() @@ -114,6 +111,8 @@ public override int GetHashCode() hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveStr); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveInt); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveArr); + hashCode = (hashCode * -1521134295) + EqualityComparer>.Default.GetHashCode(this.SensitiveDict); return hashCode; } @@ -125,7 +124,20 @@ public static TestDoc Create(string partitionKey = null) PK = partitionKey ?? Guid.NewGuid().ToString(), NonSensitive = Guid.NewGuid().ToString(), SensitiveStr = Guid.NewGuid().ToString(), - SensitiveInt = new Random().Next() + SensitiveInt = new Random().Next(), + SensitiveArr = new string[] + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }, + SensitiveDict = new Dictionary + { + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + } }; } From 05bfc508daa7f673c3da34a0887ccbb88f77b8eb Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Fri, 20 Sep 2024 09:12:17 +0200 Subject: [PATCH 11/71] Cleanup --- .../src/Mirrored/UnixDateTimeConverter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs index 30c23a3a92..9fffa1e3cb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Mirrored/UnixDateTimeConverter.cs @@ -59,7 +59,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist try { - totalSeconds = System.Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); + totalSeconds = Convert.ToDouble(reader.Value, CultureInfo.InvariantCulture); } catch { From 5629f74a2a12074ad1fcf8968eb175e35a5a54da Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Fri, 20 Sep 2024 10:06:31 +0200 Subject: [PATCH 12/71] Update MDE and rerun benchmarks --- .../Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../Readme.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index ed4cca0596..ef53408b4a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2807f8c809..2d388b505d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **61.45 μs** | **1.676 μs** | **2.457 μs** | **4.9438** | **1.2207** | **-** | **61.25 KB** | -| Decrypt | 1 | 77.89 μs | 1.959 μs | 2.933 μs | 5.7373 | 1.4648 | - | 71.22 KB | -| **Encrypt** | **10** | **171.64 μs** | **3.341 μs** | **4.791 μs** | **21.2402** | **3.6621** | **-** | **260.97 KB** | -| Decrypt | 10 | 255.57 μs | 7.833 μs | 11.724 μs | 29.2969 | 4.3945 | - | 363.84 KB | -| **Encrypt** | **100** | **2,601.33 μs** | **215.481 μs** | **322.522 μs** | **199.2188** | **125.0000** | **123.0469** | **2464.88 KB** | -| Decrypt | 100 | 3,156.06 μs | 321.419 μs | 481.084 μs | 355.4688 | 300.7813 | 261.7188 | 3413.05 KB | +| **Encrypt** | **1** | **44.46 μs** | **0.661 μs** | **0.969 μs** | **4.2114** | **1.0376** | **-** | **51.96 KB** | +| Decrypt | 1 | 56.00 μs | 1.062 μs | 1.589 μs | 5.0049 | 1.2817 | - | 61.57 KB | +| **Encrypt** | **10** | **131.08 μs** | **0.893 μs** | **1.309 μs** | **16.3574** | **3.1738** | **-** | **200.9 KB** | +| Decrypt | 10 | 174.88 μs | 3.443 μs | 4.938 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **2,052.43 μs** | **230.487 μs** | **344.982 μs** | **160.1563** | **107.4219** | **83.9844** | **1891.44 KB** | +| Decrypt | 100 | 2,791.54 μs | 284.376 μs | 425.641 μs | 234.3750 | 166.0156 | 140.6250 | 3066.91 KB | From 495d2c4d317c18f03c94f31fb6792d1f947fb8d0 Mon Sep 17 00:00:00 2001 From: Juraj Blazek Date: Mon, 16 Sep 2024 11:11:13 +0200 Subject: [PATCH 13/71] Add non-allocating APIs to encryptors --- .../src/AeadAes256CbcHmac256Algorithm.cs | 45 ++++++++++++ .../src/CosmosEncryptor.cs | 64 +++++++++++++++++ .../src/DataEncryptionKey.cs | 36 ++++++++++ .../src/EncryptionProcessor.cs | 19 +++-- .../src/Encryptor.cs | 72 +++++++++++++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 34 +++++++-- .../EmulatorTests/LegacyEncryptionTests.cs | 50 +++++++++++++ .../EmulatorTests/MdeCustomEncryptionTests.cs | 52 +++++++++++++- .../Readme.md | 12 ++-- .../AeadAes256CbcHmac256AlgorithmTests.cs | 43 +++++++++++ .../MdeEncryptionProcessorTests.cs | 19 ++++- .../TestCommon.cs | 58 ++++++++++----- 12 files changed, 465 insertions(+), 39 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index df416c3efd..a51e09726b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -26,11 +26,21 @@ internal class AeadAes256CbcHmac256Algorithm : DataEncryptionKey /// private const int KeySizeInBytes = AeadAes256CbcHmac256EncryptionKey.KeySize / 8; + /// + /// Authentication tag size in bytes + /// + private const int AuthenticationTagSizeInBytes = KeySizeInBytes; + /// /// Block size in bytes. AES uses 16 byte blocks. /// private const int BlockSizeInBytes = 16; + /// + /// Size of Initialization Vector in bytes. + /// + private const int IvSizeInBytes = 16; + /// /// Minimum Length of cipherText without authentication tag. This value is 1 (version byte) + 16 (IV) + 16 (minimum of 1 block of cipher Text) /// @@ -137,6 +147,20 @@ public override byte[] EncryptData(byte[] plainText) return this.EncryptData(plainText, hasAuthenticationTag: true); } + /// + /// Encryption Algorithm + /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits + /// cell_ciphertext = AES-CBC-256(enc_key, cell_iv, cell_data) with PKCS7 padding. + /// cell_tag = HMAC_SHA-2-256(mac_key, versionbyte + cell_iv + cell_ciphertext + versionbyte_length) + /// cell_blob = versionbyte + cell_tag + cell_iv + cell_ciphertext + /// + /// Plaintext data to be encrypted + /// Returns the ciphertext corresponding to the plaintext. + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + /// /// Encryption Algorithm /// cell_iv = HMAC_SHA-2-256(iv_key, cell_data) truncated to 128 bits @@ -418,5 +442,26 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset Buffer.BlockCopy(computedHash, 0, authenticationTag, 0, authenticationTag.Length); return authenticationTag; } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + throw new NotImplementedException(); + } + + public override int GetEncryptByteCount(int plainTextLength) + { + // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. + return sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes + GetCipherTextLength(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + throw new NotImplementedException(); + } + + private static int GetCipherTextLength(int inputSize) + { + return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 462bd56a1f..0288ee79db 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -48,6 +48,22 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + /// + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + /// public override async Task EncryptAsync( byte[] plainText, @@ -67,5 +83,53 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + /// + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + /// + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetEncryptByteCount(plainTextLength); + } + + /// + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.GetDecryptByteCount(cipherTextLength); + } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index bcee51d0e1..65890ec941 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -29,6 +29,24 @@ public abstract class DataEncryptionKey /// Encrypted value. public abstract byte[] EncryptData(byte[] plainText); + /// + /// Encrypts the plainText with a data encryption key. + /// + /// Plain text value to be encrypted. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Encrypted value. + public abstract int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset); + + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Size of input when encrypted. + public abstract int GetEncryptByteCount(int plainTextLength); + /// /// Decrypts the cipherText with a data encryption key. /// @@ -36,6 +54,24 @@ public abstract class DataEncryptionKey /// Plain text. public abstract byte[] DecryptData(byte[] cipherText); + /// + /// Decrypts the cipherText with a data encryption key. + /// + /// Ciphertext value to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Plain text. + public abstract int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Upper bound size of the input when decrypted. + public abstract int GetDecryptByteCount(int cipherTextLength); + /// /// Generates raw data encryption key bytes suitable for use with the provided encryption algorithm. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index af10de5e11..8b4da86a93 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -99,19 +99,28 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - cipherText = await encryptor.EncryptAsync( + int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( + plainText.Length, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm); + + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; + cipherTextWithTypeMarker[0] = (byte)typeMarker; + + int encryptedBytesCount = await encryptor.EncryptAsync( plainText, + plainTextOffset: 0, + plainTextLength: plainText.Length, + cipherTextWithTypeMarker, + outputOffset: 1, encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - if (cipherText == null) + if (encryptedBytesCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - byte[] cipherTextWithTypeMarker = new byte[cipherText.Length + 1]; - cipherTextWithTypeMarker[0] = (byte)typeMarker; - Buffer.BlockCopy(cipherText, 0, cipherTextWithTypeMarker, 1, cipherText.Length); itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 1e7f272ce5..469d069313 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -27,6 +27,42 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Encrypts the plainText using the key and algorithm provided. + /// + /// Plain text. + /// Offset in the plainText array at which to begin using data from. + /// Number of bytes in the plainText array to use as input. + /// Output buffer to write the encrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Cipher text. + public abstract Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + + /// + /// Calculate size of input after encryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Size of input when encrypted. + public abstract Task GetEncryptBytesCountAsync( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + /// /// Decrypts the cipherText using the key and algorithm provided. /// @@ -40,5 +76,41 @@ public abstract Task DecryptAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + + /// + /// Decrypts the cipherText using the key and algorithm provided. + /// + /// Ciphertext to be decrypted. + /// Offset in the cipherText array at which to begin using data from. + /// Number of bytes in the cipherText array to use as input. + /// Output buffer to write the decrypted data to. + /// Offset in the output array at which to begin writing data to. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Plain text. + public abstract Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); + + /// + /// Calculate upper bound size of the input after decryption. + /// + /// Input data size. + /// Identifier of the data encryption key. + /// Identifier for the encryption algorithm. + /// Token for cancellation. + /// Upper bound size of the input when decrypted. + public abstract Task GetDecryptBytesCountAsync( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 68d863114e..ba606e9412 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -12,6 +12,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal sealed class MdeEncryptionAlgorithm : DataEncryptionKey { + private const byte Version = 1; + private readonly AeadAes256CbcHmac256EncryptionAlgorithm mdeAeadAes256CbcHmac256EncryptionAlgorithm; private readonly byte[] unwrapKey; @@ -65,7 +67,8 @@ public MdeEncryptionAlgorithm( dekProperties.WrappedDataEncryptionKey); this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( protectedDataEncryptionKey, - encryptionType); + encryptionType, + Version); } else { @@ -80,11 +83,9 @@ public MdeEncryptionAlgorithm( this.RawKey = rawKey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( plaintextDataEncryptionKey, - encryptionType); - + encryptionType, + Version); } - - } /// @@ -103,7 +104,8 @@ public MdeEncryptionAlgorithm( this.RawKey = rawkey; this.mdeAeadAes256CbcHmac256EncryptionAlgorithm = AeadAes256CbcHmac256EncryptionAlgorithm.GetOrCreate( dataEncryptionKey, - encryptionType); + encryptionType, + Version); } /// @@ -125,5 +127,25 @@ public override byte[] DecryptData(byte[] cipherText) { return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText); } + + public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Encrypt(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.Decrypt(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + + public override int GetEncryptByteCount(int plainTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetEncryptByteCount(plainTextLength); + } + + public override int GetDecryptByteCount(int cipherTextLength) + { + return this.mdeAeadAes256CbcHmac256EncryptionAlgorithm.GetDecryptByteCount(cipherTextLength); + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index fd83ef528d..9150a739cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1763,6 +1763,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -1776,6 +1796,36 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 85b7bf3f36..cb20c71c58 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(32, unwrapcount); + Assert.AreEqual(48, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); @@ -2248,6 +2248,26 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } + public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); + } + + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + if (dek == null) + { + throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); + } + + return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + } + public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2261,6 +2281,36 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2d388b505d..a4921705bc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **44.46 μs** | **0.661 μs** | **0.969 μs** | **4.2114** | **1.0376** | **-** | **51.96 KB** | -| Decrypt | 1 | 56.00 μs | 1.062 μs | 1.589 μs | 5.0049 | 1.2817 | - | 61.57 KB | -| **Encrypt** | **10** | **131.08 μs** | **0.893 μs** | **1.309 μs** | **16.3574** | **3.1738** | **-** | **200.9 KB** | -| Decrypt | 10 | 174.88 μs | 3.443 μs | 4.938 μs | 24.6582 | 4.8828 | - | 303.41 KB | -| **Encrypt** | **100** | **2,052.43 μs** | **230.487 μs** | **344.982 μs** | **160.1563** | **107.4219** | **83.9844** | **1891.44 KB** | -| Decrypt | 100 | 2,791.54 μs | 284.376 μs | 425.641 μs | 234.3750 | 166.0156 | 140.6250 | 3066.91 KB | +| **Encrypt** | **1** | **60.61 μs** | **0.554 μs** | **0.758 μs** | **4.6997** | **1.5869** | **-** | **57.72 KB** | +| Decrypt | 1 | 61.02 μs | 0.984 μs | 1.473 μs | 5.0049 | 1.2817 | - | 61.57 KB | +| **Encrypt** | **10** | **158.70 μs** | **3.114 μs** | **4.565 μs** | **15.8691** | **3.9063** | **-** | **196.51 KB** | +| Decrypt | 10 | 189.60 μs | 1.248 μs | 1.829 μs | 24.6582 | 4.8828 | - | 303.41 KB | +| **Encrypt** | **100** | **1,773.67 μs** | **135.590 μs** | **202.944 μs** | **117.1875** | **50.7813** | **41.0156** | **1784.35 KB** | +| Decrypt | 100 | 2,787.37 μs | 264.329 μs | 395.636 μs | 230.4688 | 158.2031 | 136.7188 | 3067.03 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs new file mode 100644 index 0000000000..329e06e7c5 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -0,0 +1,43 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using System.Linq; + using System.Text; + + [TestClass] + public class AeadAes256CbcHmac256AlgorithmTests + { + private static readonly byte[] RootKey = new byte[32]; + + private static AeadAes256CbcHmac256EncryptionKey key; + private static AeadAes256CbcHmac256Algorithm algorithm; + + [ClassInitialize] + public static void ClassInitialize(TestContext testContext) + { + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); + AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); + } + + [TestMethod] + public void EncryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 } ; + + int cipherTextLength = algorithm.GetEncryptByteCount(plainTextBytes.Length); + byte[] cipherTextBytes = new byte[cipherTextLength]; + + int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); + Assert.Equals(encrypted, cipherTextLength); + + byte[] decrypted = algorithm.DecryptData(cipherTextBytes); + + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 1396e7e06e..95a5b66d39 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -24,7 +24,7 @@ public class MdeEncryptionProcessorTests private const string dekId = "dekId"; [ClassInitialize] - public static void ClassInitilize(TestContext testContext) + public static void ClassInitialize(TestContext testContext) { _ = testContext; MdeEncryptionProcessorTests.encryptionOptions = new EncryptionOptions() @@ -38,9 +38,22 @@ public static void ClassInitilize(TestContext testContext) MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int plainTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? plainTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => + .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => + dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] @@ -108,7 +121,7 @@ await EncryptionProcessor.EncryptAsync( [TestMethod] public async Task EncryptDecryptPropertyWithNullValue() - { + { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs index 63968723e6..1d90625bc9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/TestCommon.cs @@ -27,11 +27,26 @@ internal static byte[] EncryptData(byte[] plainText) return plainText.Select(b => (byte)(b + 1)).ToArray(); } + internal static int EncryptData(byte[] plainText, int inputOffset, int inputLength, byte[] output, int outputOffset) + { + byte[] cipherText = EncryptData(plainText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(cipherText, 0, output, outputOffset, cipherText.Length); + + return cipherText.Length; + } + internal static byte[] DecryptData(byte[] cipherText) { return cipherText.Select(b => (byte)(b - 1)).ToArray(); } - + + internal static int DecryptData(byte[] cipherText, int inputOffset, int inputLength, byte[] output, int outputOffset) + { + byte[] plainText = DecryptData(cipherText.AsSpan(inputOffset, inputLength).ToArray()); + Buffer.BlockCopy(plainText, 0, output, outputOffset, plainText.Length); + return plainText.Length; + } + internal static Stream ToStream(T input) { string s = JsonConvert.SerializeObject(input); @@ -48,14 +63,9 @@ internal static T FromStream(Stream stream) } } - private static JObject ParseStream(Stream stream) - { - return JObject.Load(new JsonTextReader(new StreamReader(stream))); - } - internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt" }; + public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveArr", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } @@ -68,17 +78,12 @@ internal class TestDoc public int SensitiveInt { get; set; } - public TestDoc() - { - } + public string[] SensitiveArr { get; set; } - public TestDoc(TestDoc other) + public Dictionary SensitiveDict { get; set; } + + public TestDoc() { - this.Id = other.Id; - this.PK = other.PK; - this.NonSensitive = other.NonSensitive; - this.SensitiveStr = other.SensitiveStr; - this.SensitiveInt = other.SensitiveInt; } public override bool Equals(object obj) @@ -88,7 +93,9 @@ public override bool Equals(object obj) && this.PK == doc.PK && this.NonSensitive == doc.NonSensitive && this.SensitiveInt == doc.SensitiveInt - && this.SensitiveStr == this.SensitiveStr; + && this.SensitiveStr == doc.SensitiveStr + && this.SensitiveArr?.Equals(doc.SensitiveArr) == true + && this.SensitiveDict?.Equals(doc.SensitiveDict) == true; } public override int GetHashCode() @@ -99,6 +106,8 @@ public override int GetHashCode() hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.NonSensitive); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveStr); hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveInt); + hashCode = (hashCode * -1521134295) + EqualityComparer.Default.GetHashCode(this.SensitiveArr); + hashCode = (hashCode * -1521134295) + EqualityComparer>.Default.GetHashCode(this.SensitiveDict); return hashCode; } @@ -110,7 +119,20 @@ public static TestDoc Create(string partitionKey = null) PK = partitionKey ?? Guid.NewGuid().ToString(), NonSensitive = Guid.NewGuid().ToString(), SensitiveStr = Guid.NewGuid().ToString(), - SensitiveInt = new Random().Next() + SensitiveInt = new Random().Next(), + SensitiveArr = new string[] + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }, + SensitiveDict = new Dictionary + { + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + { Guid.NewGuid().ToString(), Guid.NewGuid().ToString() }, + } }; } From 14bce371afb7208d4381bd0312e01154ca154313 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:16:05 +0200 Subject: [PATCH 14/71] ~ drop repeated DEK calls ~ reduce validations overhead --- .../src/CosmosEncryptor.cs | 104 +++++++++--------- .../src/EncryptionProcessor.cs | 19 ++-- .../src/Encryptor.cs | 10 ++ 3 files changed, 71 insertions(+), 62 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 0288ee79db..2ded90893b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -29,8 +29,7 @@ public CosmosEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) } /// - public override async Task DecryptAsync( - byte[] cipherText, + public override async Task GetEncryptionKeyAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) @@ -45,91 +44,94 @@ public override async Task DecryptAsync( throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); } - return dek.DecryptData(cipherText); + return dek; } /// - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.")] + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + return dek.DecryptData(cipherText); + } + + /// + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.")] + public override async Task DecryptAsync( + byte[] cipherText, + int cipherTextOffset, + int cipherTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); } /// + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.")] public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.EncryptData(plainText); } /// - public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.")] + public override async Task EncryptAsync( + byte[] plainText, + int plainTextOffset, + int plainTextLength, + byte[] output, + int outputOffset, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); } /// - public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + public override async Task GetEncryptBytesCountAsync( + int plainTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetEncryptByteCount(plainTextLength); } /// - public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + public override async Task GetDecryptBytesCountAsync( + int cipherTextLength, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } + DataEncryptionKey dek = await this.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetDecryptByteCount(cipherTextLength); } + } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index a0ea6fa213..a964a04d6d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -55,19 +55,19 @@ public static async Task EncryptAsync( return input; } - if (!encryptionOptions.PathsToEncrypt.Distinct().SequenceEqual(encryptionOptions.PathsToEncrypt)) + if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } foreach (string path in encryptionOptions.PathsToEncrypt) { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf('/') != 0) + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf(',', 1) != -1) { throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); } - if (string.Equals(path.Substring(1), "id")) + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.InvariantCulture)) { throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); } @@ -99,22 +99,19 @@ public static async Task EncryptAsync( (typeMarker, plainText) = EncryptionProcessor.Serialize(propertyValue); - int cipherTextLength = await encryptor.GetEncryptBytesCountAsync( - plainText.Length, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + + int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; cipherTextWithTypeMarker[0] = (byte)typeMarker; - int encryptedBytesCount = await encryptor.EncryptAsync( + int encryptedBytesCount = encryptionKey.EncryptData( plainText, plainTextOffset: 0, plainTextLength: plainText.Length, cipherTextWithTypeMarker, - outputOffset: 1, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm); + outputOffset: 1); if (encryptedBytesCount < 0) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 469d069313..2fd857eeb4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -49,12 +49,22 @@ public abstract Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// + /// Retrieve Data Encryption Key. + /// + /// Identifier of the data encryption key. + /// Identifier of the encryption algorithm. + /// Token for cancellation. + /// Data Encryption Key + public abstract Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); + /// /// Calculate size of input after encryption. /// /// Input data size. /// Identifier of the data encryption key. /// Identifier for the encryption algorithm. + /// Data Encryption Key used. /// Token for cancellation. /// Size of input when encrypted. public abstract Task GetEncryptBytesCountAsync( From 4ff1601869dbc2801c6f57e24573d00a1dde4ab8 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:22:48 +0200 Subject: [PATCH 15/71] ! typo --- Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 2ded90893b..7e705453ce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -121,7 +121,7 @@ public override async Task GetEncryptBytesCountAsync( } /// - [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.")] + [Obsolete("It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.")] public override async Task GetDecryptBytesCountAsync( int cipherTextLength, string dataEncryptionKeyId, From d8a345cc26c744a07ef68f22c69e0668679f8d1c Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 12:24:47 +0200 Subject: [PATCH 16/71] ~ update benchmark --- .../Readme.md | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index f0166dd8b8..dd87b34c36 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,20 +1,19 @@ -``` ini +``` ini BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.108 +.NET SDK=9.0.100-rc.1.24452.12 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **36.44 μs** | **0.518 μs** | **0.759 μs** | **36.40 μs** | **4.0894** | **1.0376** | **-** | **50.81 KB** | -| Decrypt | 1 | 44.51 μs | 0.900 μs | 1.291 μs | 43.83 μs | 4.8828 | 1.2207 | - | 60.1 KB | -| **Encrypt** | **10** | **114.63 μs** | **3.234 μs** | **4.841 μs** | **113.80 μs** | **16.2354** | **3.2959** | **-** | **199.75 KB** | -| Decrypt | 10 | 146.56 μs | 1.205 μs | 1.766 μs | 146.08 μs | 24.4141 | 4.6387 | - | 301.94 KB | -| **Encrypt** | **100** | **1,784.59 μs** | **180.928 μs** | **270.805 μs** | **1,743.86 μs** | **158.2031** | **95.7031** | **82.0313** | **1890.27 KB** | -| Decrypt | 100 | 2,772.68 μs | 261.923 μs | 392.035 μs | 2,648.81 μs | 236.3281 | 158.2031 | 142.5781 | 3064.91 KB | - +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| +| **Encrypt** | **1** | **37.87 μs** | **0.699 μs** | **0.979 μs** | **3.7842** | **0.9766** | **-** | **47.03 KB** | +| Decrypt | 1 | 47.27 μs | 1.148 μs | 1.683 μs | 4.3945 | 1.0986 | - | 54.17 KB | +| **Encrypt** | **10** | **113.49 μs** | **1.380 μs** | **1.934 μs** | **15.1367** | **3.0518** | **-** | **185.81 KB** | +| Decrypt | 10 | 146.93 μs | 1.724 μs | 2.581 μs | 19.5313 | 2.1973 | - | 239.94 KB | +| **Encrypt** | **100** | **1,610.57 μs** | **161.738 μs** | **242.081 μs** | **150.3906** | **107.4219** | **74.2188** | **1773.64 KB** | +| Decrypt | 100 | 2,058.97 μs | 202.464 μs | 303.039 μs | 160.1563 | 107.4219 | 76.1719 | 2042.61 KB | From 03c06e05ce457d3a0fc5c24d09d0d5ec53cc8c79 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 30 Sep 2024 16:22:22 +0200 Subject: [PATCH 17/71] ~ fix tests ~ update api file --- .../src/AeadAes256CbcHmac256Algorithm.cs | 10 +- .../EmulatorTests/LegacyEncryptionTests.cs | 39 ++++- .../EmulatorTests/MdeCustomEncryptionTests.cs | 104 ++++++------- .../AeadAes256CbcHmac256AlgorithmTests.cs | 2 +- .../DotNetSDKEncryptionCustomAPI.json | 139 +++++++++++++++++- .../MdeEncryptionProcessorTests.cs | 13 ++ .../Contracts/ContractEnforcement.cs | 2 +- 7 files changed, 235 insertions(+), 74 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index a51e09726b..903e620207 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -158,7 +158,15 @@ public override byte[] EncryptData(byte[] plainText) /// Returns the ciphertext corresponding to the plaintext. public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) { - throw new NotImplementedException(); + var buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); + + if (buffer.Length > output.Length-outputOffset) + { + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + } + + buffer.CopyTo(output, outputOffset); + return buffer.Length; } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 9150a739cf..6f5b798ed3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -922,7 +922,7 @@ public async Task EncryptionTransactionalBatchWithCustomSerializer() await LegacyEncryptionTests.VerifyItemByReadAsync(LegacyEncryptionTests.itemContainer, doc1ToReplace); } - [TestMethod] + [TestMethod] public async Task VerifyDekOperationWithSystemTextSerializer() { System.Text.Json.JsonSerializerOptions jsonSerializerOptions = new System.Text.Json.JsonSerializerOptions() @@ -1037,6 +1037,23 @@ await LegacyEncryptionTests.IterateDekFeedAsync( isResultOrderExpected: false, "SELECT * from c", itemCountInPage: 3); + + + //clean up + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } } [TestMethod] @@ -1750,7 +1767,7 @@ public override async Task DecryptAsync( throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1770,7 +1787,7 @@ public override async Task DecryptAsync(byte[] cipherText, int cipherTextOf throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1789,7 +1806,7 @@ public override async Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1799,7 +1816,7 @@ public override async Task EncryptAsync( public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1809,7 +1826,7 @@ public override async Task EncryptAsync(byte[] plainText, int plainTextOffs public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); @@ -1819,13 +1836,21 @@ public override async Task GetEncryptBytesCountAsync(int plainTextLength, s public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); return dek.GetDecryptByteCount(cipherTextLength); } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + return await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + } } internal class CustomSerializer : CosmosSerializer diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index cb20c71c58..d5f27926fd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -69,9 +69,9 @@ public static async Task ClassInitialize(TestContext context) MdeCustomEncryptionTests.encryptor = new TestEncryptor(MdeCustomEncryptionTests.dekProvider); MdeCustomEncryptionTests.encryptionContainer = MdeCustomEncryptionTests.itemContainer.WithEncryptor(encryptor); MdeCustomEncryptionTests.encryptionContainerForChangeFeed = MdeCustomEncryptionTests.itemContainerForChangeFeed.WithEncryptor(encryptor); + await MdeCustomEncryptionTests.dekProvider.InitializeAsync(MdeCustomEncryptionTests.database, MdeCustomEncryptionTests.keyContainer.Id); MdeCustomEncryptionTests.dekProperties = await MdeCustomEncryptionTests.CreateDekAsync(MdeCustomEncryptionTests.dekProvider, MdeCustomEncryptionTests.dekId); - } [ClassCleanup] @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(48, unwrapcount); + Assert.AreEqual(32, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); @@ -1153,6 +1153,22 @@ await MdeCustomEncryptionTests.IterateDekFeedAsync( isResultOrderExpected: false, "SELECT * from c", itemCountInPage: 3); + + // cleanup + FeedIterator iterator = containerWithCosmosSystemTextJsonSerializer.GetItemQueryIterator(); + + while (iterator.HasMoreResults) + { + FeedResponse feedResponse = await iterator.ReadNextAsync(); + foreach (TestDocSystemText testDoc in feedResponse) + { + if (testDoc.Id == null) + { + continue; + } + await containerWithCosmosSystemTextJsonSerializer.DeleteItemAsync(testDoc.Id, new PartitionKey(testDoc.PartitionKey)); + } + } } [TestMethod] @@ -2218,54 +2234,36 @@ private class TestEncryptor : Encryptor public DataEncryptionKeyProvider DataEncryptionKeyProvider { get; } public bool FailDecryption { get; set; } + private readonly CosmosEncryptor encryptor; + public TestEncryptor(DataEncryptionKeyProvider dataEncryptionKeyProvider) { - this.DataEncryptionKeyProvider = dataEncryptionKeyProvider; + this.encryptor = new CosmosEncryptor(dataEncryptionKeyProvider); this.FailDecryption = false; } - public override async Task DecryptAsync( - byte[] cipherText, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default) + private void ThrowIfFail(string dataEncryptionKeyId) { if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) { throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); } + } - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText); + public override async Task DecryptAsync( + byte[] cipherText, + string dataEncryptionKeyId, + string encryptionAlgorithm, + CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.DecryptAsync(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task EncryptAsync( @@ -2274,42 +2272,32 @@ public override async Task EncryptAsync( string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.EncryptAsync(plainText, plainTextOffset, plainTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetEncryptByteCount(plainTextLength); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptBytesCountAsync(plainTextLength, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetDecryptBytesCountAsync(cipherTextLength, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); + } - return dek.GetDecryptByteCount(cipherTextLength); + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + this.ThrowIfFail(dataEncryptionKeyId); + return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 329e06e7c5..9c2125f903 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -33,7 +33,7 @@ public void EncryptUsingBufferDecryptsSuccessfully() byte[] cipherTextBytes = new byte[cipherTextLength]; int encrypted = algorithm.EncryptData(plainTextBytes, 0, plainTextBytes.Length, cipherTextBytes, 0); - Assert.Equals(encrypted, cipherTextLength); + Assert.AreEqual(encrypted, cipherTextLength); byte[] decrypted = algorithm.DecryptData(cipherTextBytes); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json index 1912d16407..ccb4b56fff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Contracts/DotNetSDKEncryptionCustomAPI.json @@ -113,20 +113,61 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider get_DataEncryptionKeyProvider();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ "AsyncStateMachineAttribute" ], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -163,6 +204,26 @@ "Attributes": [], "MethodInfo": "Byte[] RawKey;CanRead:True;CanWrite:False;Byte[] get_RawKey();IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "Int32 DecryptData(Byte[], Int32, Int32, Byte[], Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 DecryptData(Byte[], Int32, Int32, Byte[], Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 EncryptData(Byte[], Int32, Int32, Byte[], Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 EncryptData(Byte[], Int32, Int32, Byte[], Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 GetDecryptByteCount(Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 GetDecryptByteCount(Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "Int32 GetEncryptByteCount(Int32)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "Int32 GetEncryptByteCount(Int32);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey Create(Byte[], System.String)": { "Type": "Method", "Attributes": [], @@ -1064,20 +1125,61 @@ ], "MethodInfo": "Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider get_DataEncryptionKeyProvider();IsAbstract:False;IsStatic:False;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { "Type": "Method", "Attributes": [ "AsyncStateMachineAttribute" ], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, - "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]": { + "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { "Type": "Method", "Attributes": [ - "AsyncStateMachineAttribute" + "AsyncStateMachineAttribute", + "ObsoleteAttribute" ], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.DecryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.EncryptData to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetDecryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)[System.Runtime.CompilerServices.AsyncStateMachineAttribute(typeof(Microsoft.Azure.Cosmos.Encryption.Custom.CosmosEncryptor+))]-[System.ObsoleteAttribute(\"It is suggested to use GetEncryptionKeyAsync + key.GetEncryptByteCount to reduce overhead.\")]": { + "Type": "Method", + "Attributes": [ + "AsyncStateMachineAttribute", + "ObsoleteAttribute" + ], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:False;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "Void .ctor(Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKeyProvider)": { "Type": "Constructor", "Attributes": [], @@ -1088,6 +1190,11 @@ } }, "Members": { + "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[Microsoft.Azure.Cosmos.Encryption.Custom.DataEncryptionKey] GetEncryptionKeyAsync(System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, "System.Threading.Tasks.Task`1[System.Byte[]] DecryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken)": { "Type": "Method", "Attributes": [], @@ -1097,6 +1204,26 @@ "Type": "Method", "Attributes": [], "MethodInfo": "System.Threading.Tasks.Task`1[System.Byte[]] EncryptAsync(Byte[], System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] DecryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] EncryptAsync(Byte[], Int32, Int32, Byte[], Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetDecryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" + }, + "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken)": { + "Type": "Method", + "Attributes": [], + "MethodInfo": "System.Threading.Tasks.Task`1[System.Int32] GetEncryptBytesCountAsync(Int32, System.String, System.String, System.Threading.CancellationToken);IsAbstract:True;IsStatic:False;IsVirtual:True;IsGenericMethod:False;IsConstructor:False;IsFinal:False;" } }, "NestedTypes": {} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 95a5b66d39..aec838a413 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -34,7 +34,20 @@ public static void ClassInitialize(TestContext testContext) PathsToEncrypt = TestDoc.PathsToEncrypt }; + var DekMock = new Mock(); + DekMock.Setup(m => m.EncryptData(It.IsAny())) + .Returns((byte[] plainText) => TestCommon.EncryptData(plainText)); + DekMock.Setup(m => m.GetEncryptByteCount(It.IsAny())) + .Returns((int plainTextLength) => plainTextLength); + DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + + MdeEncryptionProcessorTests.mockEncryptor = new Mock(); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string dekId, string algorithm, CancellationToken token) => + dekId == MdeEncryptionProcessorTests.dekId ? DekMock.Object : throw new InvalidOperationException("DEK not found.")); + MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index f0e127495a..c58109d65b 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -200,7 +200,7 @@ public static void ValidateContractContainBreakingChanges( File.WriteAllText($"Contracts/{breakingChangesPath}", localJson); string baselineJson = GetBaselineContract(baselinePath); - ContractEnforcement.ValidateJsonAreSame(localJson, baselineJson); + ContractEnforcement.ValidateJsonAreSame(baselineJson, localJson); } public static void ValidateTelemetryContractContainBreakingChanges( From 3bf77c89a6c571095d6b95b40e8649afee7796b6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:27:54 +0200 Subject: [PATCH 18/71] ~ cleanup --- .../src/AeadAes256CbcHmac256Algorithm.cs | 24 +++++++++++++++---- .../src/EncryptionProcessor.cs | 6 ++--- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index 903e620207..e50c60e8fd 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -158,11 +158,11 @@ public override byte[] EncryptData(byte[] plainText) /// Returns the ciphertext corresponding to the plaintext. public override int EncryptData(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) { - var buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); + byte[] buffer = this.EncryptData(plainText.AsSpan(plainTextOffset, plainTextLength).ToArray()); - if (buffer.Length > output.Length-outputOffset) + if (buffer.Length > output.Length - outputOffset) { - throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes."); } buffer.CopyTo(output, outputOffset); @@ -453,7 +453,15 @@ private byte[] PrepareAuthenticationTag(byte[] iv, byte[] cipherText, int offset public override int DecryptData(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset) { - throw new NotImplementedException(); + byte[] buffer = this.DecryptData(cipherText.AsSpan(cipherTextOffset, cipherTextLength).ToArray(), true); + + if (buffer.Length > output.Length - outputOffset) + { + throw new ArgumentOutOfRangeException($"Output buffer is shorter than required {buffer.Length} bytes"); + } + + buffer.CopyTo(output, outputOffset); + return buffer.Length; } public override int GetEncryptByteCount(int plainTextLength) @@ -464,7 +472,13 @@ public override int GetEncryptByteCount(int plainTextLength) public override int GetDecryptByteCount(int cipherTextLength) { - throw new NotImplementedException(); + int value = cipherTextLength - (sizeof(byte) + AuthenticationTagSizeInBytes + IvSizeInBytes); + if (value < BlockSizeInBytes) + { + throw new ArgumentOutOfRangeException(nameof(cipherTextLength), $"Cipher text length is too short."); + } + + return value; } private static int GetCipherTextLength(int inputSize) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index a964a04d6d..a6ee23b4e5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -55,19 +55,19 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (!encryptionOptions.PathsToEncrypt.Distinct().SequenceEqual(encryptionOptions.PathsToEncrypt)) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } foreach (string path in encryptionOptions.PathsToEncrypt) { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf(',', 1) != -1) + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.LastIndexOf('/') != 0) { throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); } - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.InvariantCulture)) + if (string.Equals(path.Substring(1), "id")) { throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); } From ceaa8b5ee346a3ad779a79521cf95da90176f4d9 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:39:52 +0200 Subject: [PATCH 19/71] + refresh benchmark --- .../Readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index dd87b34c36..64a686244c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -2,7 +2,7 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=9.0.100-rc.1.24452.12 +.NET SDK=8.0.400 [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| -| **Encrypt** | **1** | **37.87 μs** | **0.699 μs** | **0.979 μs** | **3.7842** | **0.9766** | **-** | **47.03 KB** | -| Decrypt | 1 | 47.27 μs | 1.148 μs | 1.683 μs | 4.3945 | 1.0986 | - | 54.17 KB | -| **Encrypt** | **10** | **113.49 μs** | **1.380 μs** | **1.934 μs** | **15.1367** | **3.0518** | **-** | **185.81 KB** | -| Decrypt | 10 | 146.93 μs | 1.724 μs | 2.581 μs | 19.5313 | 2.1973 | - | 239.94 KB | -| **Encrypt** | **100** | **1,610.57 μs** | **161.738 μs** | **242.081 μs** | **150.3906** | **107.4219** | **74.2188** | **1773.64 KB** | -| Decrypt | 100 | 2,058.97 μs | 202.464 μs | 303.039 μs | 160.1563 | 107.4219 | 76.1719 | 2042.61 KB | +| **Encrypt** | **1** | **37.15 μs** | **0.683 μs** | **1.002 μs** | **3.8452** | **0.9766** | **-** | **47.28 KB** | +| Decrypt | 1 | 45.29 μs | 0.757 μs | 1.062 μs | 4.3945 | 1.0986 | - | 54.17 KB | +| **Encrypt** | **10** | **111.83 μs** | **1.252 μs** | **1.874 μs** | **15.1367** | **3.0518** | **-** | **186.07 KB** | +| Decrypt | 10 | 151.46 μs | 2.259 μs | 3.311 μs | 19.5313 | 2.1973 | - | 239.94 KB | +| **Encrypt** | **100** | **1,567.24 μs** | **153.944 μs** | **230.416 μs** | **152.3438** | **109.3750** | **76.1719** | **1773.95 KB** | +| Decrypt | 100 | 2,088.77 μs | 232.084 μs | 347.372 μs | 160.1563 | 113.2813 | 76.1719 | 2042.61 KB | From 611b3ac485adc698df5b3a7fe062cf8c4e42af0e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 11:55:26 +0200 Subject: [PATCH 20/71] + unit test --- .../AeadAes256CbcHmac256AlgorithmTests.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 9c2125f903..900fac6798 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -6,6 +6,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests { using Microsoft.Azure.Cosmos.Encryption.Custom; using Microsoft.VisualStudio.TestTools.UnitTesting; + using System; using System.Linq; using System.Text; @@ -20,6 +21,8 @@ public class AeadAes256CbcHmac256AlgorithmTests [ClassInitialize] public static void ClassInitialize(TestContext testContext) { + _ = testContext; + AeadAes256CbcHmac256AlgorithmTests.key = new AeadAes256CbcHmac256EncryptionKey(RootKey, "AEAes256CbcHmacSha256Randomized"); AeadAes256CbcHmac256AlgorithmTests.algorithm = new AeadAes256CbcHmac256Algorithm(AeadAes256CbcHmac256AlgorithmTests.key, EncryptionType.Randomized, algorithmVersion: 1); } @@ -39,5 +42,20 @@ public void EncryptUsingBufferDecryptsSuccessfully() Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted)); } + + [TestMethod] + public void DecryptUsingBufferDecryptsSuccessfully() + { + byte[] plainTextBytes = new byte[4] { 0, 1, 2, 3 }; + byte[] encrypted = algorithm.EncryptData(plainTextBytes); + + int plainTextMaxLength = algorithm.GetDecryptByteCount(encrypted.Length); + byte[] decrypted = new byte[plainTextMaxLength]; + + int decryptedBytes = algorithm.DecryptData(encrypted, 0, encrypted.Length, decrypted, 0); + + Assert.AreEqual(plainTextBytes.Length, decryptedBytes); + Assert.IsTrue(plainTextBytes.SequenceEqual(decrypted.AsSpan(0, decryptedBytes).ToArray())); + } } } From 8a78fe87cd516919f88782d686e1b2b1e5f85e4b Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 14:35:47 +0200 Subject: [PATCH 21/71] ~ merge fixes and initial cleanup --- .../src/EncryptionProcessor.cs | 50 +++++++++---------- .../src/Encryptor.cs | 36 ------------- .../Readme.md | 16 +++--- 3 files changed, 31 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b657bf40ed..245e99b69d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -49,6 +49,8 @@ public static async Task EncryptAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + EncryptionProcessor.ValidateInputForEncrypt( input, encryptor, @@ -118,7 +120,7 @@ public static async Task EncryptAsync( cipherTextWithTypeMarker[0] = (byte)typeMarker; - int encryptedBytesCount = await encryptionKey.EncryptAsync( + int encryptedBytesCount = encryptionKey.EncryptData( plainText, plainTextOffset: 0, plainTextLength, @@ -287,6 +289,13 @@ private static async Task MdeEncAlgoDecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); JObject plainTextJObj = new JObject(); @@ -304,22 +313,18 @@ private static async Task MdeEncAlgoDecryptObjectAsync( continue; } - int plainTextLength = await encryptor.GetDecryptBytesCountAsync( - cipherTextWithTypeMarker.Length - 1, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = await EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( - encryptionProperties, + int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, - plainText, - encryptor, - diagnosticsContext, - cancellationToken); + plainText); EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], @@ -357,30 +362,19 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static async Task MdeEncAlgoDecryptPropertyAsync( - EncryptionProperties encryptionProperties, + private static int MdeEncAlgoDecryptPropertyAsync( + DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextOffset, int cipherTextLength, - byte[] buffer, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + byte[] buffer) { - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - int decryptedCount = await encryptor.DecryptAsync( + int decryptedCount = encryptionKey.DecryptData( cipherText, cipherTextOffset, cipherTextLength, buffer, - outputOffset: 0, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm, - cancellationToken); + outputOffset: 0); if (decryptedCount < 0) { @@ -397,6 +391,8 @@ private static async Task LegacyEncAlgoDecryptContentAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { + _ = diagnosticsContext; + if (encryptionProperties.EncryptionFormatVersion != 2) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index 51ce14a7cd..2fd857eeb4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -122,41 +122,5 @@ public abstract Task GetDecryptBytesCountAsync( string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default); - - /// - /// Decrypts the cipherText using the key and algorithm provided. - /// - /// Ciphertext to be decrypted. - /// Offset in the cipherText array at which to begin using data from. - /// Number of bytes in the cipherText array to use as input. - /// Output buffer to write the decrypted data to. - /// Offset in the output array at which to begin writing data to. - /// Identifier of the data encryption key. - /// Identifier for the encryption algorithm. - /// Token for cancellation. - /// Plain text. - public abstract Task DecryptAsync( - byte[] cipherText, - int cipherTextOffset, - int cipherTextLength, - byte[] output, - int outputOffset, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default); - - /// - /// Calculate upper bound size of the input after decryption. - /// - /// Input data size. - /// Identifier of the data encryption key. - /// Identifier for the encryption algorithm. - /// Token for cancellation. - /// Upper bound size of the input when decrypted. - public abstract Task GetDecryptBytesCountAsync( - int cipherTextLength, - string dataEncryptionKeyId, - string encryptionAlgorithm, - CancellationToken cancellationToken = default); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 64a686244c..01e1e3a41a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|---------:|--------:|-----------:| -| **Encrypt** | **1** | **37.15 μs** | **0.683 μs** | **1.002 μs** | **3.8452** | **0.9766** | **-** | **47.28 KB** | -| Decrypt | 1 | 45.29 μs | 0.757 μs | 1.062 μs | 4.3945 | 1.0986 | - | 54.17 KB | -| **Encrypt** | **10** | **111.83 μs** | **1.252 μs** | **1.874 μs** | **15.1367** | **3.0518** | **-** | **186.07 KB** | -| Decrypt | 10 | 151.46 μs | 2.259 μs | 3.311 μs | 19.5313 | 2.1973 | - | 239.94 KB | -| **Encrypt** | **100** | **1,567.24 μs** | **153.944 μs** | **230.416 μs** | **152.3438** | **109.3750** | **76.1719** | **1773.95 KB** | -| Decrypt | 100 | 2,088.77 μs | 232.084 μs | 347.372 μs | 160.1563 | 113.2813 | 76.1719 | 2042.61 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **37.10 μs** | **0.527 μs** | **0.788 μs** | **36.82 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | +| Decrypt | 1 | 43.43 μs | 0.686 μs | 1.027 μs | 43.06 μs | 3.9673 | 1.0376 | - | 49.16 KB | +| **Encrypt** | **10** | **115.97 μs** | **2.256 μs** | **3.377 μs** | **117.36 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | +| Decrypt | 10 | 144.93 μs | 2.238 μs | 3.350 μs | 146.64 μs | 15.6250 | 3.1738 | - | 194.3 KB | +| **Encrypt** | **100** | **1,604.78 μs** | **150.864 μs** | **225.806 μs** | **1,557.92 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | +| Decrypt | 100 | 1,943.81 μs | 193.233 μs | 289.223 μs | 1,872.14 μs | 126.9531 | 78.1250 | 42.9688 | 1586.28 KB | From 8ed21357a28d393f6148462dac9ae37ff437e70d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 15:06:56 +0200 Subject: [PATCH 22/71] ~ write directly to output document instead of copying --- .../src/EncryptionProcessor.cs | 21 +++----- .../EmulatorTests/MdeCustomEncryptionTests.cs | 50 ------------------- .../Readme.md | 16 +++--- 3 files changed, 16 insertions(+), 71 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 245e99b69d..cd95f1540b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -298,7 +298,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - JObject plainTextJObj = new JObject(); + List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); @@ -329,15 +329,10 @@ private static async Task MdeEncAlgoDecryptObjectAsync( EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), - plainTextJObj, + document, propertyName); - } - List pathsDecrypted = new List(); - foreach (JProperty property in plainTextJObj.Properties()) - { - document[property.Name] = property.Value; - pathsDecrypted.Add("/" + property.Name); + pathsDecrypted.Add(path); } DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( @@ -557,16 +552,16 @@ private static void DeserializeAndAddProperty( switch (typeMarker) { case TypeMarker.Boolean: - jObject.Add(key, SqlBoolSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); break; case TypeMarker.Double: - jObject.Add(key, SqlDoubleSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); break; case TypeMarker.Long: - jObject.Add(key, SqlLongSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); break; case TypeMarker.String: - jObject.Add(key, SqlVarCharSerializer.Deserialize(serializedBytes)); + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); break; case TypeMarker.Array: DeserializeAndAddProperty(serializedBytes); @@ -592,7 +587,7 @@ void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); using JsonTextReader reader = new JsonTextReader(memoryTextReader); - jObject.Add(key, serializer.Deserialize(reader)); + jObject[key] = serializer.Deserialize(reader); } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 277bacbe89..df51ab0cc5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -2266,26 +2266,6 @@ public override async Task DecryptAsync(byte[] cipherText, int cipherTextOf return await this.encryptor.DecryptAsync(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset, dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); - } - public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, @@ -2319,36 +2299,6 @@ public override async Task GetEncryptionKeyAsync(string dataE this.ThrowIfFail(dataEncryptionKeyId); return await this.encryptor.GetEncryptionKeyAsync(dataEncryptionKeyId, encryptionAlgorithm, cancellationToken); } - - public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); - } - - public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetEncryptByteCount(plainTextLength); - } - - public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - DataEncryptionKey dek = await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - return dek.GetDecryptByteCount(cipherTextLength); - } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 01e1e3a41a..7ffdbdcd60 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **37.10 μs** | **0.527 μs** | **0.788 μs** | **36.82 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | -| Decrypt | 1 | 43.43 μs | 0.686 μs | 1.027 μs | 43.06 μs | 3.9673 | 1.0376 | - | 49.16 KB | -| **Encrypt** | **10** | **115.97 μs** | **2.256 μs** | **3.377 μs** | **117.36 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | -| Decrypt | 10 | 144.93 μs | 2.238 μs | 3.350 μs | 146.64 μs | 15.6250 | 3.1738 | - | 194.3 KB | -| **Encrypt** | **100** | **1,604.78 μs** | **150.864 μs** | **225.806 μs** | **1,557.92 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | -| Decrypt | 100 | 1,943.81 μs | 193.233 μs | 289.223 μs | 1,872.14 μs | 126.9531 | 78.1250 | 42.9688 | 1586.28 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|---------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **37.28 μs** | **0.676 μs** | **0.991 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | +| Decrypt | 1 | 43.12 μs | 1.753 μs | 2.623 μs | 3.6011 | 1.1597 | - | 44.63 KB | +| **Encrypt** | **10** | **115.55 μs** | **1.717 μs** | **2.570 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | +| Decrypt | 10 | 121.81 μs | 2.127 μs | 3.050 μs | 12.9395 | 3.1738 | - | 159.56 KB | +| **Encrypt** | **100** | **1,571.06 μs** | **128.737 μs** | **192.687 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | +| Decrypt | 100 | 1,687.55 μs | 143.998 μs | 215.529 μs | 101.5625 | 62.5000 | 44.9219 | 1253.19 KB | From bbe98458081b72b5a53b8e70d2ca9ba873610b2e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 15:22:54 +0200 Subject: [PATCH 23/71] ! tests --- .../EmulatorTests/LegacyEncryptionTests.cs | 74 ++++++++++++++----- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index c39441fb80..d63f24755c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -103,7 +103,7 @@ public async Task EncryptionCreateDekWithMdeAlgorithmFails() { Assert.AreEqual("For use of 'MdeAeadAes256CbcHmac256Randomized' algorithm, Encryptor or CosmosDataEncryptionKeyProvider needs to be initialized with EncryptionKeyStoreProvider.", ex.Message); } - } + } [TestMethod] public async Task EncryptionRewrapDek() @@ -217,7 +217,7 @@ await LegacyEncryptionTests.IterateDekFeedAsync( QueryDefinition queryDefinition = new QueryDefinition("SELECT * from c where c.id >= @startId and c.id <= @endId ORDER BY c.id ASC") .WithParameter("@startId", "Contoso_v000") .WithParameter("@endId", "Contoso_v999"); - + await LegacyEncryptionTests.IterateDekFeedAsync( dekProvider, new List { contosoV1, contosoV2 }, @@ -405,7 +405,7 @@ public async Task EncryptionChangeFeedDecryptionSuccessful() TestDoc testDoc1 = await LegacyEncryptionTests.CreateItemAsync(LegacyEncryptionTests.encryptionContainer, LegacyEncryptionTests.dekId, TestDoc.PathsToEncrypt); TestDoc testDoc2 = await LegacyEncryptionTests.CreateItemAsync(LegacyEncryptionTests.encryptionContainer, dek2, TestDoc.PathsToEncrypt); - + // change feed iterator await this.ValidateChangeFeedIteratorResponse(LegacyEncryptionTests.encryptionContainer, testDoc1, testDoc2); @@ -597,7 +597,7 @@ public async Task EncryptionRudItemLazyDecryption() { TestDoc testDoc = TestDoc.Create(); // Upsert (item doesn't exist) - ItemResponse > upsertResponse = await LegacyEncryptionTests.encryptionContainer.UpsertItemAsync( + ItemResponse> upsertResponse = await LegacyEncryptionTests.encryptionContainer.UpsertItemAsync( new EncryptableItem(testDoc), new PartitionKey(testDoc.PK), LegacyEncryptionTests.GetRequestOptions(LegacyEncryptionTests.dekId, TestDoc.PathsToEncrypt)); @@ -660,7 +660,7 @@ public async Task EncryptionResourceTokenAuthRestricted() restrictedUserPermission.Token); Database databaseForRestrictedUser = clientForRestrictedUser.GetDatabase(LegacyEncryptionTests.database.Id); - Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(LegacyEncryptionTests.itemContainer.Id); + Container containerForRestrictedUser = databaseForRestrictedUser.GetContainer(LegacyEncryptionTests.itemContainer.Id); Container encryptionContainerForRestrictedUser = containerForRestrictedUser.WithEncryptor(encryptor); await LegacyEncryptionTests.PerformForbiddenOperationAsync(() => @@ -939,15 +939,15 @@ public async Task VerifyDekOperationWithSystemTextSerializer() // get database and container Database databaseWithCosmosSystemTextJsonSerializer = clientWithCosmosSystemTextJsonSerializer.GetDatabase(LegacyEncryptionTests.database.Id); Container containerWithCosmosSystemTextJsonSerializer = databaseWithCosmosSystemTextJsonSerializer.GetContainer(LegacyEncryptionTests.itemContainer.Id); - + // create the Dek container Container dekContainerWithCosmosSystemTextJsonSerializer = await databaseWithCosmosSystemTextJsonSerializer.CreateContainerAsync(Guid.NewGuid().ToString(), "/id", 400); - + CosmosDataEncryptionKeyProvider dekProviderWithCosmosSystemTextJsonSerializer = new CosmosDataEncryptionKeyProvider(new TestKeyWrapProvider()); await dekProviderWithCosmosSystemTextJsonSerializer.InitializeAsync(databaseWithCosmosSystemTextJsonSerializer, dekContainerWithCosmosSystemTextJsonSerializer.Id); - - TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new TestEncryptor(dekProviderWithCosmosSystemTextJsonSerializer); - + + TestEncryptor encryptorWithCosmosSystemTextJsonSerializer = new TestEncryptor(dekProviderWithCosmosSystemTextJsonSerializer); + // enable encryption on container Container encryptionContainerWithCosmosSystemTextJsonSerializer = containerWithCosmosSystemTextJsonSerializer.WithEncryptor(encryptorWithCosmosSystemTextJsonSerializer); @@ -991,7 +991,7 @@ public async Task VerifyDekOperationWithSystemTextSerializer() ItemResponse createTestDoc = await encryptionContainerWithCosmosSystemTextJsonSerializer.CreateItemAsync( testDocSystemText, new PartitionKey(testDocSystemText.PartitionKey), - LegacyEncryptionTests.GetRequestOptions(dekId, new List() { "/status"})); + LegacyEncryptionTests.GetRequestOptions(dekId, new List() { "/status" })); Assert.AreEqual(HttpStatusCode.Created, createTestDoc.StatusCode); @@ -1074,7 +1074,7 @@ public async Task EncryptionTransactionalBatchConflictResponse() Assert.AreEqual(HttpStatusCode.Conflict, batchResponse.StatusCode); Assert.AreEqual(1, batchResponse.Count); } - + private static async Task ValidateSprocResultsAsync(Container container, TestDoc expectedDoc) { string sprocId = Guid.NewGuid().ToString(); @@ -1129,7 +1129,7 @@ private static async Task ValidateQueryResultsAsync( queryResponseIterator = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); queryResponseIteratorForLazyDecryption = container.GetItemQueryIterator(queryDefinition, requestOptions: requestOptions); } - + FeedResponse readDocs = await queryResponseIterator.ReadNextAsync(); Assert.AreEqual(null, readDocs.ContinuationToken); @@ -1373,7 +1373,7 @@ private static async Task IterateDekFeedAsync( : dekProvider.DataEncryptionKeyContainer.GetDataEncryptionKeyQueryIterator( query, requestOptions: requestOptions); - + Assert.IsTrue(dekIterator.HasMoreResults); List readDekIds = new List(); @@ -1588,7 +1588,7 @@ private static async Task CreateDekAsync(CosmosData LegacyEncryptionTests.metadata1); Assert.AreEqual(HttpStatusCode.Created, dekResponse.StatusCode); - + return VerifyDekResponse(dekResponse, dekId); } @@ -1607,7 +1607,7 @@ private static DataEncryptionKeyProperties VerifyDekResponse( Assert.IsNotNull(dekProperties.SelfLink); Assert.IsNotNull(dekProperties.CreatedTime); Assert.IsNotNull(dekProperties.LastModified); - + return dekProperties; } @@ -1737,7 +1737,7 @@ public override Task WrapKeyAsync(byte[] key, Encryptio { this.WrapKeyCallsCount[metadata.Value]++; } - + EncryptionKeyWrapMetadata responseMetadata = new EncryptionKeyWrapMetadata(metadata.Value + LegacyEncryptionTests.metadataUpdateSuffix); int moveBy = metadata.Value == LegacyEncryptionTests.metadata1.Value ? 1 : 2; return Task.FromResult(new EncryptionKeyWrapResult(key.Select(b => (byte)(b + moveBy)).ToArray(), responseMetadata)); @@ -1813,7 +1813,45 @@ public override async Task EncryptAsync( return dek.EncryptData(plainText); } - } + + public override async Task EncryptAsync(byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset); + } + + public override async Task GetEncryptBytesCountAsync(int plainTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetEncryptByteCount(plainTextLength); + } + + public override async Task GetDecryptBytesCountAsync(int cipherTextLength, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + DataEncryptionKey dek = await this.GetEncryptionKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + + return dek.GetDecryptByteCount(cipherTextLength); + } + + public override async Task GetEncryptionKeyAsync(string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) + { + return await this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync( + dataEncryptionKeyId, + encryptionAlgorithm, + cancellationToken); + } + } internal class CustomSerializer : CosmosSerializer { From a107f62b848c4a9eba5c00c94cc6acf05375d1ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 16:02:55 +0200 Subject: [PATCH 24/71] ~ retrieve DataEncryptionKey only once per document --- .../src/EncryptionProcessor.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index cd95f1540b..0370b94771 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -92,6 +92,8 @@ public static async Task EncryptAsync( { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { string propertyName = pathToEncrypt.Substring(1); @@ -112,8 +114,6 @@ public static async Task EncryptAsync( continue; } - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); @@ -298,6 +298,8 @@ private static async Task MdeEncAlgoDecryptObjectAsync( using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { @@ -313,8 +315,6 @@ private static async Task MdeEncAlgoDecryptObjectAsync( continue; } - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); From a1ad02b9095dae60db6e9ccc9df098d0962f9df7 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 1 Oct 2024 16:16:27 +0200 Subject: [PATCH 25/71] ! fix tests ~ bump benchmark --- .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- .../Readme.md | 16 ++++++++-------- .../MdeEncryptionProcessorTests.cs | 5 ++++- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index df51ab0cc5..ec98b39567 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -374,7 +374,7 @@ public async Task ValidateCachingOfProtectedDataEncryptionKey() await MdeCustomEncryptionTests.CreateItemAsync(encryptionContainer, dekId, TestDoc.PathsToEncrypt); testEncryptionKeyStoreProvider.UnWrapKeyCallsCount.TryGetValue(masterKeyUri1.ToString(), out unwrapcount); - Assert.AreEqual(64, unwrapcount); + Assert.AreEqual(4, unwrapcount); // 2 hours default testEncryptionKeyStoreProvider = new TestEncryptionKeyStoreProvider(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 7ffdbdcd60..4df982a29d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,11 +9,11 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|---------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **37.28 μs** | **0.676 μs** | **0.991 μs** | **3.7231** | **0.9766** | **-** | **46.29 KB** | -| Decrypt | 1 | 43.12 μs | 1.753 μs | 2.623 μs | 3.6011 | 1.1597 | - | 44.63 KB | -| **Encrypt** | **10** | **115.55 μs** | **1.717 μs** | **2.570 μs** | **14.1602** | **3.5400** | **-** | **174.92 KB** | -| Decrypt | 10 | 121.81 μs | 2.127 μs | 3.050 μs | 12.9395 | 3.1738 | - | 159.56 KB | -| **Encrypt** | **100** | **1,571.06 μs** | **128.737 μs** | **192.687 μs** | **142.5781** | **95.7031** | **66.4063** | **1660.08 KB** | -| Decrypt | 100 | 1,687.55 μs | 143.998 μs | 215.529 μs | 101.5625 | 62.5000 | 44.9219 | 1253.19 KB | +| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| +| **Encrypt** | **1** | **29.20 μs** | **0.607 μs** | **0.871 μs** | **29.26 μs** | **3.3875** | **2.5940** | **-** | **41.51 KB** | +| Decrypt | 1 | 32.23 μs | 0.528 μs | 0.790 μs | 32.78 μs | 3.2349 | 0.7935 | - | 39.71 KB | +| **Encrypt** | **10** | **107.83 μs** | **1.975 μs** | **2.956 μs** | **107.93 μs** | **13.7939** | **0.6104** | **-** | **170.14 KB** | +| Decrypt | 10 | 116.25 μs | 1.537 μs | 2.301 μs | 117.15 μs | 12.5732 | 1.2207 | - | 154.63 KB | +| **Encrypt** | **100** | **1,493.01 μs** | **314.932 μs** | **471.376 μs** | **1,482.68 μs** | **214.8438** | **173.8281** | **140.6250** | **1655.51 KB** | +| Decrypt | 100 | 1,406.56 μs | 126.286 μs | 189.019 μs | 1,408.25 μs | 138.6719 | 103.5156 | 82.0313 | 1248.58 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index aec838a413..3e626a7453 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -41,7 +41,10 @@ public static void ClassInitialize(TestContext testContext) .Returns((int plainTextLength) => plainTextLength); DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); - + DekMock.Setup(m => m.DecryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.DecryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.GetDecryptByteCount(It.IsAny())) + .Returns((int cipherTextLength) => cipherTextLength); MdeEncryptionProcessorTests.mockEncryptor = new Mock(); MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) From 4f2f072d2c9b3f8025203b425c1801c5302ae538 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 2 Oct 2024 11:40:26 +0200 Subject: [PATCH 26/71] ~ update Aes algorithm to reuse GetEncryptedByteCount --- .../src/AeadAes256CbcHmac256Algorithm.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs index e50c60e8fd..c5ae780a1f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs @@ -201,12 +201,11 @@ protected byte[] EncryptData(byte[] plainText, bool hasAuthenticationTag) // Final blob we return = version + HMAC + iv + cipherText const int hmacStartIndex = 1; - int authenticationTagLen = hasAuthenticationTag ? KeySizeInBytes : 0; + int authenticationTagLen = hasAuthenticationTag ? AuthenticationTagSizeInBytes : 0; int ivStartIndex = hmacStartIndex + authenticationTagLen; int cipherStartIndex = ivStartIndex + BlockSizeInBytes; // this is where hmac starts. - // Output buffer size = size of VersionByte + Authentication Tag + IV + cipher Text blocks. - int outputBufSize = sizeof(byte) + authenticationTagLen + iv.Length + (numBlocks * BlockSizeInBytes); + int outputBufSize = this.GetEncryptByteCount(plainText.Length) - (hasAuthenticationTag ? 0 : authenticationTagLen); byte[] outBuffer = new byte[outputBufSize]; // Store the version and IV rightaway From cbbeee2f664fd63a2db5da70c49044d1e025423f Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 13:54:06 +0200 Subject: [PATCH 27/71] ~ refactor EncryptionProcessor --- .../src/AeAesEncryptionProcessor.cs | 116 +++++ .../src/EncryptionProcessor.cs | 458 ++---------------- .../src/MdeEncryptionProcessor.cs | 124 +++++ .../Transformation/JObjectSqlSerializer.cs | 121 +++++ .../src/Transformation/MdeEncryptor.cs | 55 +++ .../src/Transformation/TypeMarker.cs | 17 + 6 files changed, 472 insertions(+), 419 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs new file mode 100644 index 0000000000..7b7d366295 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs @@ -0,0 +1,116 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal static class AeAesEncryptionProcessor + { + public static async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken cancellationToken) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + EncryptionProperties encryptionProperties = null; + byte[] plainText = null; + byte[] cipherText = null; + + JObject toEncryptJObj = new (); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + toEncryptJObj.Add(propertyName, propertyValue.Value()); + itemJObj.Remove(propertyName); + } + + MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); + Debug.Assert(memoryStream != null); + Debug.Assert(memoryStream.TryGetBuffer(out _)); + plainText = memoryStream.ToArray(); + + cipherText = await encryptor.EncryptAsync( + plainText, + encryptionOptions.DataEncryptionKeyId, + encryptionOptions.EncryptionAlgorithm, + cancellationToken); + + if (cipherText == null) + { + throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); + } + + encryptionProperties = new EncryptionProperties( + encryptionFormatVersion: 2, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: cipherText, + encryptionOptions.PathsToEncrypt); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal static async Task LegacyEncAlgoDecryptContentAsync( + JObject document, + EncryptionProperties encryptionProperties, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 2) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + byte[] plainText = await encryptor.DecryptAsync( + encryptionProperties.EncryptedData, + encryptionProperties.DataEncryptionKeyId, + encryptionProperties.EncryptionAlgorithm, + cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(EncryptionProcessor.DecryptAsync)}."); + JObject plainTextJObj; + using (MemoryStream memoryStream = new (plainText)) + using (StreamReader streamReader = new (memoryStream)) + using (JsonTextReader jsonTextReader = new (streamReader)) + { + jsonTextReader.ArrayPool = JsonArrayPool.Instance; + plainTextJObj = JObject.Load(jsonTextReader); + } + + List pathsDecrypted = new (); + foreach (JProperty property in plainTextJObj.Properties()) + { + document.Add(property.Name, property.Value); + pathsDecrypted.Add("/" + property.Name); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + + return decryptionContext; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 0370b94771..1c2c78fe24 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; - using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -13,7 +12,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Text; using System.Threading; using System.Threading.Tasks; - using Microsoft.Data.Encryption.Cryptography.Serializers; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -22,20 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal static class EncryptionProcessor { - private static readonly SqlSerializerFactory SqlSerializerFactory = new SqlSerializerFactory(); - - // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); - private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); - private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); - - private static readonly JsonSerializerSettings JsonSerializerSettings = new JsonSerializerSettings() + internal static readonly JsonSerializerSettings JsonSerializerSettings = new () { DateParseHandling = DateParseHandling.None, }; - internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new CosmosJsonDotNetSerializer(JsonSerializerSettings); + internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); /// /// If there isn't any PathsToEncrypt, input stream will be returned without any modification. @@ -51,7 +41,7 @@ public static async Task EncryptAsync( { _ = diagnosticsContext; - EncryptionProcessor.ValidateInputForEncrypt( + ValidateInputForEncrypt( input, encryptor, encryptionOptions); @@ -79,118 +69,14 @@ public static async Task EncryptAsync( } } - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); - List pathsEncrypted = new List(); - EncryptionProperties encryptionProperties = null; - byte[] plainText = null; - byte[] cipherText = null; - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - - switch (encryptionOptions.EncryptionAlgorithm) +#pragma warning disable CS0618 // Type or member is obsolete + return encryptionOptions.EncryptionAlgorithm switch { - case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { - string propertyName = pathToEncrypt.Substring(1); - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - if (propertyValue.Type == JTokenType.Null) - { - continue; - } - - (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); - - if (plainText == null) - { - continue; - } - - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); - - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); - - cipherTextWithTypeMarker[0] = (byte)typeMarker; - - int encryptedBytesCount = encryptionKey.EncryptData( - plainText, - plainTextOffset: 0, - plainTextLength, - cipherTextWithTypeMarker, - outputOffset: 1); - - if (encryptedBytesCount < 0) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); - } - - itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); - pathsEncrypted.Add(pathToEncrypt); - } - - encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - break; - - case CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized: - - JObject toEncryptJObj = new JObject(); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { - string propertyName = pathToEncrypt.Substring(1); - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - toEncryptJObj.Add(propertyName, propertyValue.Value()); - itemJObj.Remove(propertyName); - } - - MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); - Debug.Assert(memoryStream != null); - Debug.Assert(memoryStream.TryGetBuffer(out _)); - plainText = memoryStream.ToArray(); - - cipherText = await encryptor.EncryptAsync( - plainText, - encryptionOptions.DataEncryptionKeyId, - encryptionOptions.EncryptionAlgorithm, - cancellationToken); - - if (cipherText == null) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); - } - - encryptionProperties = new EncryptionProperties( - encryptionFormatVersion: 2, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: cipherText, - encryptionOptions.PathsToEncrypt); - break; - - default: - throw new NotSupportedException($"Encryption Algorithm : {encryptionOptions.EncryptionAlgorithm} is not supported."); - } - - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); - input.Dispose(); - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken), + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, cancellationToken), + _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionOptions.EncryptionAlgorithm} is not supported."), + }; +#pragma warning restore CS0618 // Type or member is obsolete } /// @@ -213,8 +99,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = EncryptionProcessor.RetrieveItem(input); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = RetrieveItem(input); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -222,26 +108,9 @@ public static async Task EncryptAsync( return (input, null); } - EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); - DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch - { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( - itemJObj, - encryptor, - encryptionProperties, - diagnosticsContext, - cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( - itemJObj, - encryptionProperties, - encryptor, - diagnosticsContext, - cancellationToken), - _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), - }; - + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); input.Dispose(); - return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext); + return (BaseSerializer.ToStream(itemJObj), decryptionContext); } public static async Task<(JObject, DecryptionContext)> DecryptAsync( @@ -254,175 +123,56 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { return (document, null); } + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionPropertiesJObj, cancellationToken); + + return (document, decryptionContext); + } + + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) + { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); +#pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( - document, + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( - document, + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + itemJObj, encryptionProperties, encryptor, diagnosticsContext, cancellationToken), _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), }; - - return (document, decryptionContext); - } - - private static async Task MdeEncAlgoDecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); - foreach (string path in encryptionProperties.EncryptedPaths) - { - string propertyName = path.Substring(1); - if (!document.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) - { - continue; - } - - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextWithTypeMarker.Length - 1); - - byte[] plainText = arrayPoolManager.Rent(plainTextLength); - - int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( - encryptionKey, - cipherTextWithTypeMarker, - cipherTextOffset: 1, - cipherTextWithTypeMarker.Length - 1, - plainText); - - EncryptionProcessor.DeserializeAndAddProperty( - (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - document, - propertyName); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); +#pragma warning restore CS0618 // Type or member is obsolete return decryptionContext; } - private static DecryptionContext CreateDecryptionContext( + internal static DecryptionContext CreateDecryptionContext( List pathsDecrypted, string dataEncryptionKeyId) { - DecryptionInfo decryptionInfo = new DecryptionInfo( + DecryptionInfo decryptionInfo = new ( pathsDecrypted, dataEncryptionKeyId); - DecryptionContext decryptionContext = new DecryptionContext( + DecryptionContext decryptionContext = new ( new List() { decryptionInfo }); return decryptionContext; } - private static int MdeEncAlgoDecryptPropertyAsync( - DataEncryptionKey encryptionKey, - byte[] cipherText, - int cipherTextOffset, - int cipherTextLength, - byte[] buffer) - { - int decryptedCount = encryptionKey.DecryptData( - cipherText, - cipherTextOffset, - cipherTextLength, - buffer, - outputOffset: 0); - - if (decryptedCount < 0) - { - throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); - } - - return decryptedCount; - } - - private static async Task LegacyEncAlgoDecryptContentAsync( - JObject document, - EncryptionProperties encryptionProperties, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 2) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - byte[] plainText = await encryptor.DecryptAsync( - encryptionProperties.EncryptedData, - encryptionProperties.DataEncryptionKeyId, - encryptionProperties.EncryptionAlgorithm, - cancellationToken) ?? throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); - JObject plainTextJObj; - using (MemoryStream memoryStream = new MemoryStream(plainText)) - using (StreamReader streamReader = new StreamReader(memoryStream)) - using (JsonTextReader jsonTextReader = new JsonTextReader(streamReader)) - { - jsonTextReader.ArrayPool = JsonArrayPool.Instance; - plainTextJObj = JObject.Load(jsonTextReader); - } - - List pathsDecrypted = new List(); - foreach (JProperty property in plainTextJObj.Properties()) - { - document.Add(property.Name, property.Value); - pathsDecrypted.Add("/" + property.Name); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); - - return decryptionContext; - } - private static void ValidateInputForEncrypt( Stream input, Encryptor encryptor, @@ -465,11 +215,11 @@ private static JObject RetrieveItem( Debug.Assert(input != null); JObject itemJObj; - using (StreamReader sr = new StreamReader(input, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true)) - using (JsonTextReader jsonTextReader = new JsonTextReader(sr)) + using (StreamReader sr = new (input, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 1024, leaveOpen: true)) + using (JsonTextReader jsonTextReader = new (sr)) { jsonTextReader.ArrayPool = JsonArrayPool.Instance; - JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings() + JsonSerializerSettings jsonSerializerSettings = new () { DateParseHandling = DateParseHandling.None, MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr @@ -494,129 +244,21 @@ private static JObject RetrieveEncryptionProperties( return encryptionPropertiesJObj; } - private static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) - { - byte[] buffer; - int length; - switch (propertyValue.Type) - { - case JTokenType.Undefined: - Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null, -1); - case JTokenType.Null: - Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null, -1); - case JTokenType.Boolean: - (buffer, length) = SerializeFixed(SqlBoolSerializer); - return (TypeMarker.Boolean, buffer, length); - case JTokenType.Float: - (buffer, length) = SerializeFixed(SqlDoubleSerializer); - return (TypeMarker.Double, buffer, length); - case JTokenType.Integer: - (buffer, length) = SerializeFixed(SqlLongSerializer); - return (TypeMarker.Long, buffer, length); - case JTokenType.String: - (buffer, length) = SerializeString(propertyValue.ToObject()); - return (TypeMarker.String, buffer, length); - case JTokenType.Array: - (buffer, length) = SerializeString(propertyValue.ToString()); - return (TypeMarker.Array, buffer, length); - case JTokenType.Object: - (buffer, length) = SerializeString(propertyValue.ToString()); - return (TypeMarker.Object, buffer, length); - default: - throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); - } - - (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) - { - byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int length = serializer.Serialize(propertyValue.ToObject(), buffer); - return (buffer, length); - } - - (byte[], int) SerializeString(string value) - { - byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int length = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, length); - } - } - - private static void DeserializeAndAddProperty( - TypeMarker typeMarker, - ReadOnlySpan serializedBytes, - JObject jObject, - string key) - { - switch (typeMarker) - { - case TypeMarker.Boolean: - jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Double: - jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Long: - jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.String: - jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); - break; - case TypeMarker.Array: - DeserializeAndAddProperty(serializedBytes); - break; - case TypeMarker.Object: - DeserializeAndAddProperty(serializedBytes); - break; - default: - Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); - break; - } - - void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) - where T : JToken - { - using ArrayPoolManager manager = new ArrayPoolManager(); - - char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); - int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); - - JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); - - using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); - using JsonTextReader reader = new JsonTextReader(memoryTextReader); - - jObject[key] = serializer.Deserialize(reader); - } - } - - private enum TypeMarker : byte - { - Null = 1, // not used - String = 2, - Double = 3, - Long = 4, - Boolean = 5, - Array = 6, - Object = 7, - } - internal static async Task DeserializeAndDecryptResponseAsync( Stream content, Encryptor encryptor, CancellationToken cancellationToken) { - JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); + JObject contentJObj = BaseSerializer.FromStream(content); - if (!(contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is JArray documents)) + if (contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is not JArray documents) { throw new InvalidOperationException("Feed Response body contract was violated. Feed response did not have an array of Documents"); } foreach (JToken value in documents) { - if (!(value is JObject document)) + if (value is not JObject document) { continue; } @@ -624,7 +266,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await EncryptionProcessor.DecryptAsync( + await DecryptAsync( document, encryptor, diagnosticsContext, @@ -634,29 +276,7 @@ await EncryptionProcessor.DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); - } - - internal static int GetOriginalBase64Length(string base64string) - { - if (string.IsNullOrEmpty(base64string)) - { - return 0; - } - - int paddingCount = 0; - int characterCount = base64string.Length; - if (base64string[characterCount - 1] == '=') - { - paddingCount++; - } - - if (base64string[characterCount - 2] == '=') - { - paddingCount++; - } - - return (3 * (characterCount / 4)) - paddingCount; + return BaseSerializer.ToStream(contentJObj); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs new file mode 100644 index 0000000000..e15dbf88f8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Newtonsoft.Json.Linq; + + internal static class MdeEncryptionProcessor + { + public static async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = JObjectSqlSerializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedLength) = MdeEncryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + + itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal static async Task MdeEncAlgoDecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { + string propertyName = path.Substring(1); + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = MdeEncryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + JObjectSqlSerializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs new file mode 100644 index 0000000000..5056b4645c --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs @@ -0,0 +1,121 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using Microsoft.Data.Encryption.Cryptography.Serializers; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal static class JObjectSqlSerializer + { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + + private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; + + internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) + { + byte[] buffer; + int length; + switch (propertyValue.Type) + { + case JTokenType.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null, -1); + case JTokenType.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null, -1); + case JTokenType.Boolean: + (buffer, length) = SerializeFixed(SqlBoolSerializer); + return (TypeMarker.Boolean, buffer, length); + case JTokenType.Float: + (buffer, length) = SerializeFixed(SqlDoubleSerializer); + return (TypeMarker.Double, buffer, length); + case JTokenType.Integer: + (buffer, length) = SerializeFixed(SqlLongSerializer); + return (TypeMarker.Long, buffer, length); + case JTokenType.String: + (buffer, length) = SerializeString(propertyValue.ToObject()); + return (TypeMarker.String, buffer, length); + case JTokenType.Array: + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Array, buffer, length); + case JTokenType.Object: + (buffer, length) = SerializeString(propertyValue.ToString()); + return (TypeMarker.Object, buffer, length); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); + } + + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int length = serializer.Serialize(propertyValue.ToObject(), buffer); + return (buffer, length); + } + + (byte[], int) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); + } + } + + internal static void DeserializeAndAddProperty( + TypeMarker typeMarker, + ReadOnlySpan serializedBytes, + JObject jObject, + string key) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jObject[key] = SqlBoolSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jObject[key] = SqlDoubleSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jObject[key] = SqlLongSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.String: + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + DeserializeAndAddProperty(serializedBytes); + break; + case TypeMarker.Object: + DeserializeAndAddProperty(serializedBytes); + break; + default: + Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + break; + } + + void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + where T : JToken + { + using ArrayPoolManager manager = new (); + + char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); + + JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); + + using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); + using JsonTextReader reader = new (memoryTextReader); + + jObject[key] = serializer.Deserialize(reader); + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs new file mode 100644 index 0000000000..325c1b5f03 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -0,0 +1,55 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + + internal static class MdeEncryptor + { + internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + { + int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; + + byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + + encryptedText[0] = (byte)typeMarker; + + int encryptedLength = encryptionKey.EncryptData( + plainText, + plainTextOffset: 0, + plainTextLength, + encryptedText, + outputOffset: 1); + + if (encryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); + } + + return (encryptedText, encryptedLength + 1); + } + + internal static (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + { + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); + + byte[] plainText = arrayPoolManager.Rent(plainTextLength); + + int decryptedLength = encryptionKey.DecryptData( + cipherText, + cipherTextOffset: 1, + cipherTextLength: cipherText.Length - 1, + plainText, + outputOffset: 0); + + if (decryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null plainText from {nameof(DataEncryptionKey.DecryptData)}."); + } + + return (plainText, decryptedLength); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs new file mode 100644 index 0000000000..f2f31c0927 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/TypeMarker.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + internal enum TypeMarker : byte + { + Null = 1, // not used + String = 2, + Double = 3, + Long = 4, + Boolean = 5, + Array = 6, + Object = 7, + } +} From b6c851cbda5a5fd974e0f7f95d24c6cea683efd7 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 13:59:46 +0200 Subject: [PATCH 28/71] ! names --- .../src/AeAesEncryptionProcessor.cs | 2 +- .../src/EncryptionProcessor.cs | 4 ++-- .../src/MdeEncryptionProcessor.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs index 7b7d366295..0944714aac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs @@ -69,7 +69,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task LegacyEncAlgoDecryptContentAsync( + internal static async Task DecryptContentAsync( JObject document, EncryptionProperties encryptionProperties, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 1c2c78fe24..d274d1d899 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -141,13 +141,13 @@ private static async Task DecryptInternalAsync(Encryptor encr #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await AeAesEncryptionProcessor.DecryptContentAsync( itemJObj, encryptionProperties, encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs index e15dbf88f8..fc601a0916 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -68,7 +68,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task MdeEncAlgoDecryptObjectAsync( + internal static async Task DecryptObjectAsync( JObject document, Encryptor encryptor, EncryptionProperties encryptionProperties, From 72ccae7a87c9fcd49663c8ba82ccb0f1a3be5b7e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 4 Oct 2024 14:30:38 +0200 Subject: [PATCH 29/71] ~ less static --- .../src/EncryptionProcessor.cs | 2 ++ .../src/MdeEncryptionProcessor.cs | 18 +++++++++++------- .../src/Transformation/JObjectSqlSerializer.cs | 16 +++++++++------- .../src/Transformation/MdeEncryptor.cs | 6 +++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index d274d1d899..3a97b051a9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -27,6 +27,8 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); + /// /// If there isn't any PathsToEncrypt, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs index fc601a0916..571c5b4ac7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs @@ -13,9 +13,13 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; - internal static class MdeEncryptionProcessor + internal class MdeEncryptionProcessor { - public static async Task EncryptAsync( + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( Stream input, Encryptor encryptor, EncryptionOptions encryptionOptions, @@ -43,14 +47,14 @@ public static async Task EncryptAsync( } byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = JObjectSqlSerializer.Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); if (plainText == null) { continue; } - (byte[] encryptedBytes, int encryptedLength) = MdeEncryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); pathsEncrypted.Add(pathToEncrypt); @@ -68,7 +72,7 @@ public static async Task EncryptAsync( return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } - internal static async Task DecryptObjectAsync( + internal async Task DecryptObjectAsync( JObject document, Encryptor encryptor, EncryptionProperties encryptionProperties, @@ -102,9 +106,9 @@ internal static async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = MdeEncryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); - JObjectSqlSerializer.DeserializeAndAddProperty( + this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs index 5056b4645c..53ee238481 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using Newtonsoft.Json; using Newtonsoft.Json.Linq; - internal static class JObjectSqlSerializer + internal class JObjectSqlSerializer { private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); @@ -21,7 +21,8 @@ internal static class JObjectSqlSerializer private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; - internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) +#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed + internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JToken propertyValue, ArrayPoolManager arrayPoolManager) { byte[] buffer; int length; @@ -69,8 +70,9 @@ internal static (TypeMarker typeMarker, byte[] serializedBytes, int serializedBy return (buffer, length); } } +#pragma warning restore SA1101 // Prefix local calls with this - internal static void DeserializeAndAddProperty( + internal virtual void DeserializeAndAddProperty( TypeMarker typeMarker, ReadOnlySpan serializedBytes, JObject jObject, @@ -91,17 +93,17 @@ internal static void DeserializeAndAddProperty( jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); break; case TypeMarker.Array: - DeserializeAndAddProperty(serializedBytes); + jObject[key] = Deserialize(serializedBytes); break; case TypeMarker.Object: - DeserializeAndAddProperty(serializedBytes); + jObject[key] = Deserialize(serializedBytes); break; default: Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); break; } - void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) + static T Deserialize(ReadOnlySpan serializedBytes) where T : JToken { using ArrayPoolManager manager = new (); @@ -114,7 +116,7 @@ void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); using JsonTextReader reader = new (memoryTextReader); - jObject[key] = serializer.Deserialize(reader); + return serializer.Deserialize(reader); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index 325c1b5f03..9243c632d6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -6,9 +6,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; - internal static class MdeEncryptor + internal class MdeEncryptor { - internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) { int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; @@ -31,7 +31,7 @@ internal static (byte[] encryptedText, int encryptedLength) Encrypt(DataEncrypti return (encryptedText, encryptedLength + 1); } - internal static (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) { int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); From 5554aa0fa308910b501f07219af1860f2da0f3df Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 13:44:16 +0200 Subject: [PATCH 30/71] ~ merge fixes --- .../src/ArrayPoolManager.cs | 4 +- .../src/CosmosEncryptor.cs | 1 - .../src/EncryptionProcessor.cs | 73 ++++++++++--------- .../src/Encryptor.cs | 1 - .../src/MemoryTextReader.cs | 4 +- .../EmulatorTests/LegacyEncryptionTests.cs | 20 ----- .../MdeEncryptionProcessorTests.cs | 18 ++--- 7 files changed, 53 insertions(+), 68 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index 8cafd07c5d..9679adc1c4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -8,9 +8,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Buffers; using System.Collections.Generic; +#pragma warning disable SA1402 // File may only contain a single type internal class ArrayPoolManager : IDisposable +#pragma warning restore SA1402 // File may only contain a single type { - private List rentedBuffers = new List(); + private List rentedBuffers = new (); private bool disposedValue; public T[] Rent(int minimumLength) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs index 6e6be7822d..454440f5ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptor.cs @@ -64,6 +64,5 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } - } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 7222180f76..b36747b960 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -5,7 +5,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; - using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -25,10 +24,10 @@ internal static class EncryptionProcessor private static readonly SqlSerializerFactory SqlSerializerFactory = new (); // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new SqlVarCharSerializer(size: -1, codePageCharacterEncoding: 65001); - private static readonly SqlBitSerializer SqlBoolSerializer = new SqlBitSerializer(); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new SqlFloatSerializer(); - private static readonly SqlBigIntSerializer SqlLongSerializer = new SqlBigIntSerializer(); + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); private static readonly JsonSerializerSettings JsonSerializerSettings = new () { @@ -79,15 +78,16 @@ public static async Task EncryptAsync( } } - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + JObject itemJObj = BaseSerializer.FromStream(input); List pathsEncrypted = new (encryptionOptions.PathsToEncrypt.Count()); EncryptionProperties encryptionProperties = null; byte[] plainText = null; byte[] cipherText = null; TypeMarker typeMarker; - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + using ArrayPoolManager arrayPoolManager = new (); +#pragma warning disable CS0618 // Type or member is obsolete switch (encryptionOptions.EncryptionAlgorithm) { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: @@ -107,7 +107,7 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = Serialize(propertyValue, arrayPoolManager); if (plainText == null) { @@ -160,7 +160,7 @@ public static async Task EncryptAsync( itemJObj.Remove(propertyName); } - MemoryStream memoryStream = EncryptionProcessor.BaseSerializer.ToStream(toEncryptJObj); + MemoryStream memoryStream = BaseSerializer.ToStream(toEncryptJObj); Debug.Assert(memoryStream != null); Debug.Assert(memoryStream.TryGetBuffer(out _)); plainText = memoryStream.ToArray(); @@ -191,7 +191,7 @@ public static async Task EncryptAsync( itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); input.Dispose(); - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + return BaseSerializer.ToStream(itemJObj); } /// @@ -214,8 +214,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = EncryptionProcessor.RetrieveItem(input); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = RetrieveItem(input); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -227,13 +227,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( itemJObj, encryptionProperties, encryptor, @@ -244,7 +244,7 @@ public static async Task EncryptAsync( #pragma warning restore CS0618 // Type or member is obsolete input.Dispose(); - return (EncryptionProcessor.BaseSerializer.ToStream(itemJObj), decryptionContext); + return (BaseSerializer.ToStream(itemJObj), decryptionContext); } public static async Task<(JObject, DecryptionContext)> DecryptAsync( @@ -257,7 +257,7 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { @@ -268,13 +268,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( document, encryptionProperties, encryptor, @@ -301,11 +301,11 @@ private static async Task MdeEncAlgoDecryptObjectAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - using ArrayPoolManager arrayPoolManager = new ArrayPoolManager(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - List pathsDecrypted = new List(encryptionProperties.EncryptedPaths.Count()); + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); @@ -324,14 +324,14 @@ private static async Task MdeEncAlgoDecryptObjectAsync( byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptPropertyAsync( + int decryptedCount = MdeEncAlgoDecryptProperty( encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, plainText); - EncryptionProcessor.DeserializeAndAddProperty( + DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, @@ -340,7 +340,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( pathsDecrypted.Add(path); } - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + DecryptionContext decryptionContext = CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -362,14 +362,21 @@ private static DecryptionContext CreateDecryptionContext( return decryptionContext; } - private static int MdeEncAlgoDecryptPropertyAsync( + private static int MdeEncAlgoDecryptProperty( DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] buffer) { - if (encryptionProperties.EncryptionFormatVersion != 3) + int decryptedCount = encryptionKey.DecryptData( + cipherText, + cipherTextOffset, + cipherTextLength, + buffer, + outputOffset: 0); + + if (decryptedCount < 0) { throw new InvalidOperationException($"{nameof(Encryptor)} returned null plainText from {nameof(DecryptAsync)}."); } @@ -412,7 +419,7 @@ private static async Task LegacyEncAlgoDecryptContentAsync( pathsDecrypted.Add("/" + property.Name); } - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + DecryptionContext decryptionContext = CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -575,15 +582,15 @@ private static void DeserializeAndAddProperty( void DeserializeAndAddProperty(ReadOnlySpan serializedBytes) where T : JToken { - using ArrayPoolManager manager = new ArrayPoolManager(); + using ArrayPoolManager manager = new (); char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); - using MemoryTextReader memoryTextReader = new MemoryTextReader(new Memory(buffer, 0, length)); - using JsonTextReader reader = new JsonTextReader(memoryTextReader); + using MemoryTextReader memoryTextReader = new (new Memory(buffer, 0, length)); + using JsonTextReader reader = new (memoryTextReader); jObject[key] = serializer.Deserialize(reader); } @@ -605,7 +612,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( Encryptor encryptor, CancellationToken cancellationToken) { - JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); + JObject contentJObj = BaseSerializer.FromStream(content); if (contentJObj.SelectToken(Constants.DocumentsResourcePropertyName) is not JArray documents) { @@ -622,7 +629,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await EncryptionProcessor.DecryptAsync( + await DecryptAsync( document, encryptor, diagnosticsContext, @@ -632,7 +639,7 @@ await EncryptionProcessor.DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); + return BaseSerializer.ToStream(contentJObj); } internal static int GetOriginalBase64Length(string base64string) @@ -657,4 +664,4 @@ internal static int GetOriginalBase64Length(string base64string) return (3 * (characterCount / 4)) - paddingCount; } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs index a517c5fea6..27c152d491 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Encryptor.cs @@ -13,7 +13,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// public abstract class Encryptor { - /// /// Retrieve Data Encryption Key. /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs index b8996ca802..9326fbbf0e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs @@ -135,7 +135,7 @@ public override string ReadLine() char ch = this.chars.Span[i]; if (ch == '\r' || ch == '\n') { - string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); this.pos = i + 1; if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') { @@ -150,7 +150,7 @@ public override string ReadLine() if (i > this.pos) { - string result = new string(this.chars.Slice(this.pos, i - this.pos).ToArray()); + string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); this.pos = i; return result; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 3672f808c6..30f6364a32 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1768,26 +1768,6 @@ public override async Task DecryptAsync( return dek.DecryptData(cipherText); } - public override async Task DecryptAsync(byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dataEncryptionKeyId, string encryptionAlgorithm, CancellationToken cancellationToken = default) - { - if (this.FailDecryption && dataEncryptionKeyId.Equals("failDek")) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned."); - } - - DataEncryptionKey dek = await this.GetEncryptionKeyAsync( - dataEncryptionKeyId, - encryptionAlgorithm, - cancellationToken); - - if (dek == null) - { - throw new InvalidOperationException($"Null {nameof(DataEncryptionKey)} returned from {nameof(this.DataEncryptionKeyProvider.FetchDataEncryptionKeyWithoutRawKeyAsync)}."); - } - - return dek.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset); - } - public override async Task EncryptAsync( byte[] plainText, string dataEncryptionKeyId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 757245a83b..69a9c7afb0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -41,26 +41,24 @@ public static void ClassInitialize(TestContext testContext) .Returns((int plainTextLength) => plainTextLength); DekMock.Setup(m => m.EncryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.EncryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.DecryptData(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((byte[] plainText, int plainTextOffset, int plainTextLength, byte[] output, int outputOffset) => TestCommon.DecryptData(plainText, plainTextOffset, plainTextLength, output, outputOffset)); + DekMock.Setup(m => m.GetDecryptByteCount(It.IsAny())) + .Returns((int cipherTextLength) => cipherTextLength); - MdeEncryptionProcessorTests.mockEncryptor = new Mock(); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor = new Mock(); + mockEncryptor.Setup(m => m.GetEncryptionKeyAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string dekId, string algorithm, CancellationToken token) => dekId == MdeEncryptionProcessorTests.dekId ? DekMock.Object : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor.Setup(m => m.EncryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] plainText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.EncryptData(plainText) : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((byte[] cipherText, string dekId, string algo, CancellationToken t) => dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText) : throw new InvalidOperationException("Null DEK was returned.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.GetDecryptBytesCountAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((int cipherTextLength, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? cipherTextLength : throw new InvalidOperationException("DEK not found.")); - MdeEncryptionProcessorTests.mockEncryptor.Setup(m => m.DecryptAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync((byte[] cipherText, int cipherTextOffset, int cipherTextLength, byte[] output, int outputOffset, string dekId, string algo, CancellationToken t) => - dekId == MdeEncryptionProcessorTests.dekId ? TestCommon.DecryptData(cipherText, cipherTextOffset, cipherTextLength, output, outputOffset) : throw new InvalidOperationException("DEK not found.")); } [TestMethod] From 28620edf2961e70384f29d75476dd90f18e9f592 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:18:17 +0200 Subject: [PATCH 31/71] ~ cleanup --- .../src/EncryptionProcessor.cs | 6 ++---- .../Readme.md | 12 ++++++------ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b36747b960..b36619b11f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -21,8 +21,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom /// internal static class EncryptionProcessor { - private static readonly SqlSerializerFactory SqlSerializerFactory = new (); - // UTF-8 encoding. private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); private static readonly SqlBitSerializer SqlBoolSerializer = new (); @@ -92,7 +90,7 @@ public static async Task EncryptAsync( { case CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized: - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm); + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, cancellationToken); foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { @@ -575,7 +573,7 @@ private static void DeserializeAndAddProperty( DeserializeAndAddProperty(serializedBytes); break; default: - Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + Debug.Fail($"Unexpected type marker {typeMarker}"); break; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 4df982a29d..f8c40c6bce 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **29.20 μs** | **0.607 μs** | **0.871 μs** | **29.26 μs** | **3.3875** | **2.5940** | **-** | **41.51 KB** | -| Decrypt | 1 | 32.23 μs | 0.528 μs | 0.790 μs | 32.78 μs | 3.2349 | 0.7935 | - | 39.71 KB | -| **Encrypt** | **10** | **107.83 μs** | **1.975 μs** | **2.956 μs** | **107.93 μs** | **13.7939** | **0.6104** | **-** | **170.14 KB** | -| Decrypt | 10 | 116.25 μs | 1.537 μs | 2.301 μs | 117.15 μs | 12.5732 | 1.2207 | - | 154.63 KB | -| **Encrypt** | **100** | **1,493.01 μs** | **314.932 μs** | **471.376 μs** | **1,482.68 μs** | **214.8438** | **173.8281** | **140.6250** | **1655.51 KB** | -| Decrypt | 100 | 1,406.56 μs | 126.286 μs | 189.019 μs | 1,408.25 μs | 138.6719 | 103.5156 | 82.0313 | 1248.58 KB | +| **Encrypt** | **1** | **29.20 μs** | **0.559 μs** | **0.836 μs** | **28.66 μs** | **3.3569** | **1.0681** | **-** | **41.24 KB** | +| Decrypt | 1 | 33.10 μs | 0.630 μs | 0.904 μs | 32.57 μs | 3.2349 | 0.7935 | - | 39.7 KB | +| **Encrypt** | **10** | **107.36 μs** | **1.942 μs** | **2.907 μs** | **107.29 μs** | **13.7939** | **0.8545** | **-** | **169.87 KB** | +| Decrypt | 10 | 117.12 μs | 1.939 μs | 2.843 μs | 118.31 μs | 12.5732 | 1.2207 | - | 154.62 KB | +| **Encrypt** | **100** | **1,489.27 μs** | **335.506 μs** | **502.170 μs** | **1,486.77 μs** | **214.8438** | **175.7813** | **140.6250** | **1655.29 KB** | +| Decrypt | 100 | 1,404.43 μs | 149.158 μs | 223.253 μs | 1,408.35 μs | 142.5781 | 107.4219 | 85.9375 | 1248.36 KB | From eb059c84917e280e22d3f7fe441ec83e518ec9c6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:35:29 +0200 Subject: [PATCH 32/71] ~ unwanted changes --- Directory.Build.props | 1 - Microsoft.Azure.Cosmos.lutconfig | 6 ------ 2 files changed, 7 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.lutconfig diff --git a/Directory.Build.props b/Directory.Build.props index 3f8f26ab75..dfbe8b10fc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,7 +12,6 @@ 10.0 $([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../')) $(DefineConstants);PREVIEW;ENCRYPTIONPREVIEW - NU1903 diff --git a/Microsoft.Azure.Cosmos.lutconfig b/Microsoft.Azure.Cosmos.lutconfig deleted file mode 100644 index 596a860301..0000000000 --- a/Microsoft.Azure.Cosmos.lutconfig +++ /dev/null @@ -1,6 +0,0 @@ - - - true - true - 180000 - \ No newline at end of file From cc2eab5cc063a446628155f3123ed80fe731377d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 14:40:22 +0200 Subject: [PATCH 33/71] - unused method --- .../src/EncryptionProcessor.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index b36619b11f..c223f44cae 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -639,27 +639,5 @@ await DecryptAsync( // and corresponding decrypted properties are added back in the documents. return BaseSerializer.ToStream(contentJObj); } - - internal static int GetOriginalBase64Length(string base64string) - { - if (string.IsNullOrEmpty(base64string)) - { - return 0; - } - - int paddingCount = 0; - int characterCount = base64string.Length; - if (base64string[characterCount - 1] == '=') - { - paddingCount++; - } - - if (base64string[characterCount - 2] == '=') - { - paddingCount++; - } - - return (3 * (characterCount / 4)) - paddingCount; - } } } \ No newline at end of file From c9ba3002002dd2f75247b1f9e5212f306fbcad75 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 6 Oct 2024 19:50:25 +0200 Subject: [PATCH 34/71] ~ updates (PR) --- .../src/ArrayPoolManager.cs | 3 +- .../src/EncryptionProcessor.cs | 34 +++++++++---------- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 2 +- .../Readme.md | 12 +++---- .../AeadAes256CbcHmac256AlgorithmTests.cs | 1 - 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs index 9679adc1c4..3546b7155b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/ArrayPoolManager.cs @@ -32,9 +32,10 @@ protected virtual void Dispose(bool disposing) { ArrayPool.Shared.Return(buffer, clearArray: true); } + + this.rentedBuffers = null; } - this.rentedBuffers = null; this.disposedValue = true; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index c223f44cae..1c7f692fe2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -105,16 +105,16 @@ public static async Task EncryptAsync( continue; } - (typeMarker, plainText, int plainTextLength) = Serialize(propertyValue, arrayPoolManager); + (typeMarker, plainText, int plainTextLength) = EncryptionProcessor.Serialize(propertyValue, arrayPoolManager); if (plainText == null) { continue; } - int cipherTextLength = encryptionKey.GetEncryptByteCount(plainText.Length); + int cipherTextLength = encryptionKey.GetEncryptByteCount(plainTextLength); - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(cipherTextLength + 1); + byte[] cipherTextWithTypeMarker = new byte[cipherTextLength + 1]; cipherTextWithTypeMarker[0] = (byte)typeMarker; @@ -130,7 +130,7 @@ public static async Task EncryptAsync( throw new InvalidOperationException($"{nameof(Encryptor)} returned null cipherText from {nameof(EncryptAsync)}."); } - itemJObj[propertyName] = cipherTextWithTypeMarker.AsSpan(0, encryptedBytesCount + 1).ToArray(); + itemJObj[propertyName] = cipherTextWithTypeMarker; pathsEncrypted.Add(pathToEncrypt); } @@ -212,8 +212,8 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); Debug.Assert(diagnosticsContext != null); - JObject itemJObj = RetrieveItem(input); - JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(itemJObj); + JObject itemJObj = EncryptionProcessor.RetrieveItem(input); + JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(itemJObj); if (encryptionPropertiesJObj == null) { @@ -225,13 +225,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( itemJObj, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( itemJObj, encryptionProperties, encryptor, @@ -255,7 +255,7 @@ public static async Task EncryptAsync( Debug.Assert(encryptor != null); - JObject encryptionPropertiesJObj = RetrieveEncryptionProperties(document); + JObject encryptionPropertiesJObj = EncryptionProcessor.RetrieveEncryptionProperties(document); if (encryptionPropertiesJObj == null) { @@ -266,13 +266,13 @@ public static async Task EncryptAsync( #pragma warning disable CS0618 // Type or member is obsolete DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncAlgoDecryptObjectAsync( + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await EncryptionProcessor.MdeEncAlgoDecryptObjectAsync( document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken), - CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await LegacyEncAlgoDecryptContentAsync( + CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized => await EncryptionProcessor.LegacyEncAlgoDecryptContentAsync( document, encryptionProperties, encryptor, @@ -322,14 +322,14 @@ private static async Task MdeEncAlgoDecryptObjectAsync( byte[] plainText = arrayPoolManager.Rent(plainTextLength); - int decryptedCount = MdeEncAlgoDecryptProperty( + int decryptedCount = EncryptionProcessor.MdeEncAlgoDecryptProperty( encryptionKey, cipherTextWithTypeMarker, cipherTextOffset: 1, cipherTextWithTypeMarker.Length - 1, plainText); - DeserializeAndAddProperty( + EncryptionProcessor.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, @@ -338,7 +338,7 @@ private static async Task MdeEncAlgoDecryptObjectAsync( pathsDecrypted.Add(path); } - DecryptionContext decryptionContext = CreateDecryptionContext( + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -417,7 +417,7 @@ private static async Task LegacyEncAlgoDecryptContentAsync( pathsDecrypted.Add("/" + property.Name); } - DecryptionContext decryptionContext = CreateDecryptionContext( + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); @@ -627,7 +627,7 @@ internal static async Task DeserializeAndDecryptResponseAsync( CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(null); using (diagnosticsContext.CreateScope("EncryptionProcessor.DeserializeAndDecryptResponseAsync")) { - await DecryptAsync( + await EncryptionProcessor.DecryptAsync( document, encryptor, diagnosticsContext, @@ -637,7 +637,7 @@ await DecryptAsync( // the contents of contentJObj get decrypted in place for MDE algorithm model, and for legacy model _ei property is removed // and corresponding decrypted properties are added back in the documents. - return BaseSerializer.ToStream(contentJObj); + return EncryptionProcessor.BaseSerializer.ToStream(contentJObj); } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 4c0fae78f1..8769abdb89 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -37,7 +37,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index f8c40c6bce..ced6fc95ed 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,9 +11,9 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **29.20 μs** | **0.559 μs** | **0.836 μs** | **28.66 μs** | **3.3569** | **1.0681** | **-** | **41.24 KB** | -| Decrypt | 1 | 33.10 μs | 0.630 μs | 0.904 μs | 32.57 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **107.36 μs** | **1.942 μs** | **2.907 μs** | **107.29 μs** | **13.7939** | **0.8545** | **-** | **169.87 KB** | -| Decrypt | 10 | 117.12 μs | 1.939 μs | 2.843 μs | 118.31 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,489.27 μs** | **335.506 μs** | **502.170 μs** | **1,486.77 μs** | **214.8438** | **175.7813** | **140.6250** | **1655.29 KB** | -| Decrypt | 100 | 1,404.43 μs | 149.158 μs | 223.253 μs | 1,408.35 μs | 142.5781 | 107.4219 | 85.9375 | 1248.36 KB | +| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | +| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | +| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | +| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | +| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | +| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs index 900fac6798..c2fd0551ff 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/AeadAes256CbcHmac256AlgorithmTests.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Linq; - using System.Text; [TestClass] public class AeadAes256CbcHmac256AlgorithmTests From 9f9cbcafec0b109bffc2bfb471df34da97be3742 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 13:43:29 +0200 Subject: [PATCH 35/71] ~ add stable vs preview release duplicity --- .../{ => AeadAes}/AeAesEncryptionProcessor.cs | 0 .../AeadAes256CbcHmac256Algorithm.cs | 0 .../AeadAes256CbcHmac256EncryptionKey.cs | 0 .../src/{ => AeadAes}/SymmetricKey.cs | 0 .../src/EncryptionContainer.cs | 18 +++ .../src/EncryptionProcessor.cs | 1 + ...soft.Azure.Cosmos.Encryption.Custom.csproj | 7 +- ...zer.cs => JObjectSqlSerializer.Preview.cs} | 4 + .../JObjectSqlSerializer.Stable.cs | 85 ++++++++++++ .../MdeEncryptionProcessor.Preview.cs} | 6 +- .../MdeEncryptionProcessor.Stable.cs | 130 ++++++++++++++++++ .../{ => Transformation}/MemoryTextReader.cs | 0 12 files changed, 247 insertions(+), 4 deletions(-) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeAesEncryptionProcessor.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeadAes256CbcHmac256Algorithm.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/AeadAes256CbcHmac256EncryptionKey.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => AeadAes}/SymmetricKey.cs (100%) rename Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/{JObjectSqlSerializer.cs => JObjectSqlSerializer.Preview.cs} (99%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{MdeEncryptionProcessor.cs => Transformation/MdeEncryptionProcessor.Preview.cs} (97%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/{ => Transformation}/MemoryTextReader.cs (100%) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeAesEncryptionProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256Algorithm.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes256CbcHmac256EncryptionKey.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256EncryptionKey.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/SymmetricKey.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/SymmetricKey.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index b898a77179..d4026cd216 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -1083,5 +1083,23 @@ private async Task> DecryptChangeFeedDocumentsAsync( return decryptItems; } + +#if IS_PREVIEW + public override Task DeleteAllItemsByPartitionKeyStreamAsync(PartitionKey partitionKey, RequestOptions requestOptions = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override Task> GetPartitionKeyRangesAsync(FeedRange feedRange, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override ChangeFeedProcessorBuilder GetChangeFeedProcessorBuilderWithAllVersionsAndDeletes(string processorName, ChangeFeedHandler> onChangesDelegate) + { + throw new NotImplementedException(); + } +#endif + } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3a97b051a9..a2b4d8fb4f 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -12,6 +12,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Text; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json; using Newtonsoft.Json.Linq; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 4c0fae78f1..a192203753 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + IS_PREVIEW @@ -24,20 +25,22 @@ + + - + - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs similarity index 99% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 53ee238481..9e2e934303 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -2,6 +2,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ +#if IS_PREVIEW + namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; @@ -121,3 +123,5 @@ static T Deserialize(ReadOnlySpan serializedBytes) } } } + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs new file mode 100644 index 0000000000..315fd82289 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -0,0 +1,85 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if !IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using Microsoft.Data.Encryption.Cryptography.Serializers; + using Newtonsoft.Json; + using Newtonsoft.Json.Linq; + + internal class JObjectSqlSerializer + { + private static readonly SqlSerializerFactory SqlSerializerFactory = new(); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new(size: -1, codePageCharacterEncoding: 65001); + + private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; + + internal (TypeMarker, byte[]) Serialize(JToken propertyValue) + { + switch (propertyValue.Type) + { + case JTokenType.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null); + case JTokenType.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null); + case JTokenType.Boolean: + return (TypeMarker.Boolean, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.Float: + return (TypeMarker.Double, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.Integer: + return (TypeMarker.Long, SqlSerializerFactory.GetDefaultSerializer().Serialize(propertyValue.ToObject())); + case JTokenType.String: + return (TypeMarker.String, SqlVarCharSerializer.Serialize(propertyValue.ToObject())); + case JTokenType.Array: + return (TypeMarker.Array, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + case JTokenType.Object: + return (TypeMarker.Object, SqlVarCharSerializer.Serialize(propertyValue.ToString())); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); + } + } + + internal virtual void DeserializeAndAddProperty( + TypeMarker typeMarker, + byte[] serializedBytes, + JObject jObject, + string key) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jObject[key] = SqlSerializerFactory.GetDefaultSerializer().Deserialize(serializedBytes); + break; + case TypeMarker.String: + jObject[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + jObject[key] = JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings); + break; + case TypeMarker.Object: + jObject[key] = JsonConvert.DeserializeObject(SqlVarCharSerializer.Deserialize(serializedBytes), JsonSerializerSettings); + break; + default: + Debug.Fail(string.Format("Unexpected type marker {0}", typeMarker)); + break; + } + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs similarity index 97% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 571c5b4ac7..3b34113b6a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -2,7 +2,9 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Encryption.Custom +#if IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Collections.Generic; @@ -10,7 +12,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.Linq; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; using Newtonsoft.Json.Linq; internal class MdeEncryptionProcessor @@ -126,3 +127,4 @@ internal async Task DecryptObjectAsync( } } } +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs new file mode 100644 index 0000000000..0eeb1f1865 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -0,0 +1,130 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if !IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + + internal class MdeEncryptionProcessor + { + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { + string propertyName = pathToEncrypt.Substring(1); + if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText) = this.Serializer.Serialize(propertyValue); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length, arrayPoolManager); + + itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); + return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + } + + internal async Task DecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new(encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { + string propertyName = path.Substring(1); + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount).ToArray(), + document, + propertyName); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs similarity index 100% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs From 64172b8cc616962539765e360e9c2cd14ff99887 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 14:21:37 +0200 Subject: [PATCH 36/71] ~ cleanup and parent branch merge --- .../src/Transformation/JObjectSqlSerializer.Stable.cs | 4 ++-- .../src/Transformation/MdeEncryptionProcessor.Stable.cs | 6 +++--- .../src/Transformation/MemoryTextReader.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs index 315fd82289..f1332baea9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -14,10 +14,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class JObjectSqlSerializer { - private static readonly SqlSerializerFactory SqlSerializerFactory = new(); + private static readonly SqlSerializerFactory SqlSerializerFactory = new (); // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new(size: -1, codePageCharacterEncoding: 65001); + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 0eeb1f1865..fc2b282d86 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -30,7 +30,7 @@ public async Task EncryptAsync( List pathsEncrypted = new (); TypeMarker typeMarker; - using ArrayPoolManager arrayPoolManager = new(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); @@ -87,11 +87,11 @@ internal async Task DecryptObjectAsync( throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } - using ArrayPoolManager arrayPoolManager = new(); + using ArrayPoolManager arrayPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - List pathsDecrypted = new(encryptionProperties.EncryptedPaths.Count()); + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { string propertyName = path.Substring(1); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs index 9326fbbf0e..ec88ccd100 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -namespace Microsoft.Azure.Cosmos.Encryption.Custom +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Diagnostics.Contracts; From 326b1bec05e5848f3b1a11a8e15a0e062d7ac91e Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 19:08:35 +0200 Subject: [PATCH 37/71] ~ master merges --- .../src/Transformation/MdeEncryptionProcessor.Preview.cs | 4 ++-- .../src/Transformation/MdeEncryptionProcessor.Stable.cs | 4 ++-- .../src/Transformation/MdeEncryptor.cs | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 3b34113b6a..a5427b015b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -55,9 +55,9 @@ public async Task EncryptAsync( continue; } - (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); - itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + itemJObj[propertyName] = encryptedBytes; pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index fc2b282d86..a861a05996 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -55,9 +55,9 @@ public async Task EncryptAsync( continue; } - (byte[] encryptedBytes, int encryptedLength) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length, arrayPoolManager); + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainText.Length); - itemJObj[propertyName] = encryptedBytes.AsSpan(0, encryptedLength).ToArray(); + itemJObj[propertyName] = encryptedBytes.ToArray(); pathsEncrypted.Add(pathToEncrypt); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index 9243c632d6..cf5a0ffdf6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -8,11 +8,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class MdeEncryptor { - internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + internal virtual byte[] Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength) { int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; - byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + byte[] encryptedText = new byte[encryptedTextLength]; encryptedText[0] = (byte)typeMarker; @@ -28,7 +28,7 @@ internal virtual (byte[] encryptedText, int encryptedLength) Encrypt(DataEncrypt throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); } - return (encryptedText, encryptedLength + 1); + return encryptedText; } internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) From c520e16a4861aaf2d507da191500b2c436919830 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 19:15:08 +0200 Subject: [PATCH 38/71] - duplicate --- .../src/MemoryTextReader.cs | 161 ------------------ 1 file changed, 161 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs deleted file mode 100644 index 9326fbbf0e..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryTextReader.cs +++ /dev/null @@ -1,161 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System; - using System.Diagnostics.Contracts; - using System.IO; - - /// - /// Adjusted implementation of .Net StringReader reading from a Memory instead of a string. - /// - internal class MemoryTextReader : TextReader - { - private Memory chars; - private int length; - private int pos; - private bool closed; - - public MemoryTextReader(Memory chars) - { - this.chars = chars; - this.length = chars.Length; - } - - public override void Close() - { - this.Dispose(true); - } - - protected override void Dispose(bool disposing) - { - this.chars = null; - this.pos = 0; - this.length = 0; - this.closed = true; - base.Dispose(disposing); - } - - [Pure] - public override int Peek() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - if (this.pos == this.length) - { - return -1; - } - - return this.chars.Span[this.pos]; - } - - public override int Read() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - if (this.pos == this.length) - { - return -1; - } - - return this.chars.Span[this.pos++]; - } - - public override int Read(char[] buffer, int index, int count) - { - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - if (index < 0) - { - throw new ArgumentOutOfRangeException(nameof(index)); - } - - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count)); - } - - if (buffer.Length - index < count) - { - throw new ArgumentOutOfRangeException(); - } - - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - int n = this.length - this.pos; - if (n > 0) - { - if (n > count) - { - n = count; - } - - this.chars.Span.Slice(this.pos, n).CopyTo(buffer.AsSpan(index, n)); - this.pos += n; - } - - return n; - } - - public override string ReadToEnd() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - this.pos = this.length; - return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); - } - - public override string ReadLine() - { - if (this.closed) - { - throw new InvalidOperationException("Reader is closed"); - } - - int i = this.pos; - while (i < this.length) - { - char ch = this.chars.Span[i]; - if (ch == '\r' || ch == '\n') - { - string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); - this.pos = i + 1; - if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') - { - this.pos++; - } - - return result; - } - - i++; - } - - if (i > this.pos) - { - string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); - this.pos = i; - return result; - } - - return null; - } - } -} From 5c408214623b3509ca7d4b87d380f21700e3c909 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 7 Oct 2024 20:02:16 +0200 Subject: [PATCH 39/71] ~ cleanup --- .../Transformation/JObjectSqlSerializer.Preview.cs | 11 +++++------ .../Transformation/MdeEncryptionProcessor.Preview.cs | 4 +++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 9e2e934303..2c706aa361 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -72,13 +72,13 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB return (buffer, length); } } -#pragma warning restore SA1101 // Prefix local calls with this internal virtual void DeserializeAndAddProperty( TypeMarker typeMarker, ReadOnlySpan serializedBytes, JObject jObject, - string key) + string key, + ArrayPoolManager arrayPoolManager) { switch (typeMarker) { @@ -105,12 +105,10 @@ internal virtual void DeserializeAndAddProperty( break; } - static T Deserialize(ReadOnlySpan serializedBytes) + T Deserialize(ReadOnlySpan serializedBytes) where T : JToken { - using ArrayPoolManager manager = new (); - - char[] buffer = manager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); + char[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetDeserializedMaxLength(serializedBytes.Length)); int length = SqlVarCharSerializer.Deserialize(serializedBytes, buffer.AsSpan()); JsonSerializer serializer = JsonSerializer.Create(JsonSerializerSettings); @@ -121,6 +119,7 @@ static T Deserialize(ReadOnlySpan serializedBytes) return serializer.Deserialize(reader); } } +#pragma warning restore SA1101 // Prefix local calls with this } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index a5427b015b..ccb59b0b40 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -88,6 +88,7 @@ internal async Task DecryptObjectAsync( } using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); @@ -113,7 +114,8 @@ internal async Task DecryptObjectAsync( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), document, - propertyName); + propertyName, + charPoolManager); pathsDecrypted.Add(path); } From 99c7a754d96a7f8c42adae0474b66850599982d0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 09:23:27 +0200 Subject: [PATCH 40/71] + Add .NET8.0 target for Cosmos.Encryption.Custom + Address new compiler complaints --- .../src/AeadAes/AeAesEncryptionProcessor.cs | 6 ++++ .../AeadAes/AeadAes256CbcHmac256Algorithm.cs | 4 +++ .../src/Common/CosmosJsonDotNetSerializer.cs | 4 +++ .../src/CosmosDataEncryptionKeyProvider.cs | 4 +++ .../src/DataEncryptionKey.cs | 4 +++ .../src/DataEncryptionKeyContainerCore.cs | 8 ++++++ .../DataEncryptionKeyContainerInlineCore.cs | 4 +++ .../src/DataEncryptionKeyFeedIterator{T}.cs | 6 ++-- .../src/DecryptableFeedResponse.cs | 4 +++ .../src/EncryptionContainer.cs | 28 ++++++++++++------- .../src/EncryptionExceptionFactory.cs | 2 ++ .../src/EncryptionFeedIterator{T}.cs | 5 ++-- .../src/EncryptionKeyWrapMetadata.cs | 13 --------- .../src/EncryptionProcessor.cs | 12 ++++++++ .../src/MdeServices/MdeEncryptionAlgorithm.cs | 5 ++++ .../src/MdeServices/MdeKeyWrapProvider.cs | 8 ++++++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 3 +- .../src/SecurityUtility.cs | 6 ++++ .../JObjectSqlSerializer.Stable.cs | 2 +- .../MdeEncryptionProcessor.Preview.cs | 13 +++++++++ .../MdeEncryptionProcessor.Stable.cs | 12 ++++++++ .../src/Transformation/MemoryTextReader.cs | 19 +++++++++++++ .../EmulatorTests/LegacyEncryptionTests.cs | 14 ++++------ .../EmulatorTests/MdeCustomEncryptionTests.cs | 2 +- ...mos.Encryption.Custom.EmulatorTests.csproj | 2 +- ...Encryption.Custom.Performance.Tests.csproj | 2 +- ...zure.Cosmos.Encryption.Custom.Tests.csproj | 2 +- .../Contracts/ContractEnforcement.cs | 16 +++++++---- 28 files changed, 162 insertions(+), 48 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs index 0944714aac..c7eb3658d4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs @@ -13,6 +13,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using Newtonsoft.Json; using Newtonsoft.Json.Linq; +#pragma warning disable IDE0057 // Use range operator +#pragma warning disable VSTHRD103 // Call async methods when in an async method internal static class AeAesEncryptionProcessor { public static async Task EncryptAsync( @@ -65,6 +67,7 @@ public static async Task EncryptAsync( encryptionOptions.PathsToEncrypt); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + input.Dispose(); return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -113,4 +116,7 @@ internal static async Task DecryptContentAsync( return decryptionContext; } } + +#pragma warning restore IDE0057 // Use range operator +#pragma warning restore VSTHRD103 // Call async methods when in an async method } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs index fab34ec143..3bc0486487 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeadAes256CbcHmac256Algorithm.cs @@ -10,6 +10,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Security.Cryptography; +#pragma warning disable SYSLIB0021 // Type or member is obsolete + /// /// This class implements authenticated encryption algorithm with associated data as described in /// http://tools.ietf.org/html/draft-mcgrew-aead-aes-cbc-hmac-sha2-05 - specifically this implements @@ -484,4 +486,6 @@ private static int GetCipherTextLength(int inputSize) return ((inputSize / BlockSizeInBytes) + 1) * BlockSizeInBytes; } } + +#pragma warning restore SYSLIB0021 // Type or member is obsolete } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs index b00c004476..9ed8056360 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Common/CosmosJsonDotNetSerializer.cs @@ -40,10 +40,14 @@ internal CosmosJsonDotNetSerializer(JsonSerializerSettings jsonSerializerSetting /// The object representing the deserialized stream public T FromStream(Stream stream) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(stream); +#else if (stream == null) { throw new ArgumentNullException(nameof(stream)); } +#endif if (typeof(Stream).IsAssignableFrom(typeof(T))) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs index cd2fe84fbc..c6f1ffb66c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosDataEncryptionKeyProvider.cs @@ -148,10 +148,14 @@ public async Task InitializeAsync( throw new InvalidOperationException($"{nameof(CosmosDataEncryptionKeyProvider)} has already been initialized."); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(database); +#else if (database == null) { throw new ArgumentNullException(nameof(database)); } +#endif ContainerResponse containerResponse = await database.CreateContainerIfNotExistsAsync( containerId, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs index b764debfd5..810227479e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKey.cs @@ -102,10 +102,14 @@ public static DataEncryptionKey Create( byte[] rawKey, string encryptionAlgorithm) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(rawKey); +#else if (rawKey == null) { throw new ArgumentNullException(nameof(rawKey)); } +#endif #pragma warning disable CS0618 // Type or member is obsolete if (!string.Equals(encryptionAlgorithm, CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized)) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs index 90cf907abc..1fcdd8c783 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerCore.cs @@ -67,10 +67,14 @@ public override async Task> CreateData throw new ArgumentException(string.Format("Unsupported Encryption Algorithm {0}", encryptionAlgorithm), nameof(encryptionAlgorithm)); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(encryptionKeyWrapMetadata); +#else if (encryptionKeyWrapMetadata == null) { throw new ArgumentNullException(nameof(encryptionKeyWrapMetadata)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); @@ -155,10 +159,14 @@ public override async Task> RewrapData ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(newWrapMetadata); +#else if (newWrapMetadata == null) { throw new ArgumentNullException(nameof(newWrapMetadata)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs index d918d85d4a..bab5cadd62 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyContainerInlineCore.cs @@ -78,10 +78,14 @@ public override Task> RewrapDataEncryp throw new ArgumentNullException(nameof(id)); } +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(newWrapMetadata); +#else if (newWrapMetadata == null) { throw new ArgumentNullException(nameof(newWrapMetadata)); } +#endif return TaskHelper.RunInlineIfNeededAsync(() => this.dataEncryptionKeyContainerCore.RewrapDataEncryptionKeyAsync(id, newWrapMetadata, encryptionAlgorithm, requestOptions, cancellationToken)); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs index 18e4246fdc..d01422c81b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DataEncryptionKeyFeedIterator{T}.cs @@ -13,7 +13,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal sealed class DataEncryptionKeyFeedIterator : FeedIterator { - private readonly FeedIterator feedIterator; + private readonly DataEncryptionKeyFeedIterator feedIterator; private readonly CosmosResponseFactory responseFactory; public DataEncryptionKeyFeedIterator( @@ -57,7 +57,7 @@ public override async Task> ReadNextAsync(CancellationToken canc if (responseMessage.IsSuccessStatusCode && responseMessage.Content != null) { - dataEncryptionKeyPropertiesList = this.ConvertResponseToDataEncryptionKeyPropertiesList( + dataEncryptionKeyPropertiesList = DataEncryptionKeyFeedIterator.ConvertResponseToDataEncryptionKeyPropertiesList( responseMessage.Content); return (responseMessage, dataEncryptionKeyPropertiesList); @@ -67,7 +67,7 @@ public override async Task> ReadNextAsync(CancellationToken canc } } - private List ConvertResponseToDataEncryptionKeyPropertiesList( + private static List ConvertResponseToDataEncryptionKeyPropertiesList( Stream content) { JObject contentJObj = EncryptionProcessor.BaseSerializer.FromStream(content); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs index c16497ac20..24c07c71e4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/DecryptableFeedResponse.cs @@ -45,10 +45,14 @@ internal static DecryptableFeedResponse CreateResponse( ResponseMessage responseMessage, IReadOnlyCollection resource) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(responseMessage); +#else if (responseMessage == null) { throw new ArgumentNullException(nameof(responseMessage)); } +#endif using (responseMessage) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs index d870f1b835..d13c1c16fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionContainer.cs @@ -122,10 +122,14 @@ public override async Task CreateItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (streamPayload == null) { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("CreateItemStream")) @@ -304,6 +308,10 @@ public override async Task> ReplaceItemAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(item); +#else if (id == null) { throw new ArgumentNullException(nameof(id)); @@ -313,6 +321,7 @@ public override async Task> ReplaceItemAsync( { throw new ArgumentNullException(nameof(item)); } +#endif if (requestOptions is not EncryptionItemRequestOptions encryptionItemRequestOptions || encryptionItemRequestOptions.EncryptionOptions == null) @@ -384,6 +393,10 @@ public override async Task ReplaceItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(id); + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (id == null) { throw new ArgumentNullException(nameof(id)); @@ -393,6 +406,7 @@ public override async Task ReplaceItemStreamAsync( { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("ReplaceItemStream")) @@ -428,11 +442,6 @@ private async Task ReplaceItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, @@ -536,10 +545,14 @@ public override async Task UpsertItemStreamAsync( ItemRequestOptions requestOptions = null, CancellationToken cancellationToken = default) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(streamPayload); +#else if (streamPayload == null) { throw new ArgumentNullException(nameof(streamPayload)); } +#endif CosmosDiagnosticsContext diagnosticsContext = CosmosDiagnosticsContext.Create(requestOptions); using (diagnosticsContext.CreateScope("UpsertItemStream")) @@ -572,11 +585,6 @@ private async Task UpsertItemHelperAsync( cancellationToken); } - if (partitionKey == null) - { - throw new NotSupportedException($"{nameof(partitionKey)} cannot be null for operations using {nameof(EncryptionContainer)}."); - } - streamPayload = await EncryptionProcessor.EncryptAsync( streamPayload, this.Encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs index 55884f1093..3b50114cdb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionExceptionFactory.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal static class EncryptionExceptionFactory { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly internal static ArgumentException InvalidKeySize(string algorithmName, int actualKeylength, int expectedLength) { return new ArgumentException( @@ -28,6 +29,7 @@ internal static ArgumentException InvalidAlgorithmVersion(byte actual, byte expe $"Invalid encryption algorithm version; actual: {actual:X2}, expected: {expected:X2}.", "cipherText"); } +#pragma warning restore CA2208 // Instantiate argument exceptions correctly internal static ArgumentException InvalidAuthenticationTag() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs index ee22f330ac..81f1516e1b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFeedIterator{T}.cs @@ -11,7 +11,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom internal sealed class EncryptionFeedIterator : FeedIterator { - private readonly FeedIterator feedIterator; + private readonly EncryptionFeedIterator feedIterator; private readonly CosmosResponseFactory responseFactory; public EncryptionFeedIterator( @@ -31,8 +31,7 @@ public override async Task> ReadNextAsync(CancellationToken canc if (typeof(T) == typeof(DecryptableItem)) { IReadOnlyCollection resource; - EncryptionFeedIterator encryptionFeedIterator = this.feedIterator as EncryptionFeedIterator; - (responseMessage, resource) = await encryptionFeedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); + (responseMessage, resource) = await this.feedIterator.ReadNextWithoutDecryptionAsync(cancellationToken); return DecryptableFeedResponse.CreateResponse( responseMessage, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs index e3c285531d..5855a011cf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionKeyWrapMetadata.cs @@ -114,18 +114,5 @@ public bool Equals(EncryptionKeyWrapMetadata other) this.Value == other.Value && this.Name == other.Name; } - - internal string GetName(EncryptionKeyWrapMetadata encryptionKeyWrapMetadata) - { - /* A legacy DEK may not have a Name value in meta-data*/ - if (string.IsNullOrWhiteSpace(encryptionKeyWrapMetadata.Name)) - { - return encryptionKeyWrapMetadata.Value; - } - else - { - return encryptionKeyWrapMetadata.Name; - } - } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e1e384d156..53d74390f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -112,7 +112,11 @@ public static async Task EncryptAsync( } DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, itemJObj, encryptionPropertiesJObj, cancellationToken); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return (BaseSerializer.ToStream(itemJObj), decryptionContext); } @@ -181,6 +185,11 @@ private static void ValidateInputForEncrypt( Encryptor encryptor, EncryptionOptions encryptionOptions) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(input); + ArgumentNullException.ThrowIfNull(encryptor); + ArgumentNullException.ThrowIfNull(encryptionOptions); +#else if (input == null) { throw new ArgumentNullException(nameof(input)); @@ -195,7 +204,9 @@ private static void ValidateInputForEncrypt( { throw new ArgumentNullException(nameof(encryptionOptions)); } +#endif +#pragma warning disable CA2208 // Instantiate argument exceptions correctly if (string.IsNullOrWhiteSpace(encryptionOptions.DataEncryptionKeyId)) { throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId)); @@ -210,6 +221,7 @@ private static void ValidateInputForEncrypt( { throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); } +#pragma warning restore CA2208 // Instantiate argument exceptions correctly } private static JObject RetrieveItem( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs index 963ab8705a..ff543dfc0a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeEncryptionAlgorithm.cs @@ -37,6 +37,10 @@ public MdeEncryptionAlgorithm( TimeSpan? cacheTimeToLive, bool withRawKey = false) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(dekProperties); + ArgumentNullException.ThrowIfNull(encryptionKeyStoreProvider); +#else if (dekProperties == null) { throw new ArgumentNullException(nameof(dekProperties)); @@ -46,6 +50,7 @@ public MdeEncryptionAlgorithm( { throw new ArgumentNullException(nameof(encryptionKeyStoreProvider)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( dekProperties.EncryptionKeyWrapMetadata.Name, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs index 6520ecd67e..297a4dac7e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MdeServices/MdeKeyWrapProvider.cs @@ -28,10 +28,14 @@ public override Task UnwrapKeyAsync( EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(metadata); +#else if (metadata == null) { throw new ArgumentNullException(nameof(metadata)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( metadata.Name, @@ -47,10 +51,14 @@ public override Task WrapKeyAsync( EncryptionKeyWrapMetadata metadata, CancellationToken cancellationToken) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(metadata); +#else if (metadata == null) { throw new ArgumentNullException(nameof(metadata)); } +#endif KeyEncryptionKey keyEncryptionKey = KeyEncryptionKey.GetOrCreate( metadata.Name, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index a192203753..d17c1bc48b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -1,6 +1,6 @@ - netstandard2.0 + netstandard2.0;net8.0 Microsoft.Azure.Cosmos.Encryption.Custom Microsoft.Azure.Cosmos.Encryption.Custom $(LangVersion) @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + True IS_PREVIEW diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs index 632f1904d5..9a7ad82bbe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/SecurityUtility.cs @@ -42,7 +42,9 @@ internal static string GetSHA256Hash(byte[] input) using (SHA256 sha256 = SHA256.Create()) { +#pragma warning disable CA1850 // Prefer static 'HashData' method over 'ComputeHash' byte[] hashValue = sha256.ComputeHash(input); +#pragma warning restore CA1850 // Prefer static 'HashData' method over 'ComputeHash' return GetHexString(hashValue); } } @@ -54,8 +56,12 @@ internal static string GetSHA256Hash(byte[] input) internal static void GenerateRandomBytes(byte[] randomBytes) { // Generate random bytes cryptographically. +#if NET8_0_OR_GREATER + RandomNumberGenerator.Fill(randomBytes); +#else using RNGCryptoServiceProvider rngCsp = new (); rngCsp.GetBytes(randomBytes); +#endif } /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs index f1332baea9..209048f907 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Stable.cs @@ -21,7 +21,7 @@ internal class JObjectSqlSerializer private static readonly JsonSerializerSettings JsonSerializerSettings = EncryptionProcessor.JsonSerializerSettings; - internal (TypeMarker, byte[]) Serialize(JToken propertyValue) + internal virtual (TypeMarker, byte[]) Serialize(JToken propertyValue) { switch (propertyValue.Type) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index ccb59b0b40..b0f293a592 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -36,7 +36,11 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -69,7 +73,11 @@ public async Task EncryptAsync( pathsEncrypted); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -95,7 +103,12 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif + if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index a861a05996..d4222b6f1d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -36,7 +36,11 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else string propertyName = pathToEncrypt.Substring(1); +#endif if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) { continue; @@ -69,7 +73,11 @@ public async Task EncryptAsync( pathsEncrypted); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else input.Dispose(); +#endif return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); } @@ -94,7 +102,11 @@ internal async Task DecryptObjectAsync( List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); foreach (string path in encryptionProperties.EncryptedPaths) { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else string propertyName = path.Substring(1); +#endif if (!document.TryGetValue(propertyName, out JToken propertyValue)) { // malformed document, such record shouldn't be there at all diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs index ec88ccd100..893d31b864 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MemoryTextReader.cs @@ -71,6 +71,12 @@ public override int Read() public override int Read(char[] buffer, int index, int count) { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); + ArgumentOutOfRangeException.ThrowIfNegative(index); + ArgumentOutOfRangeException.ThrowIfNegative(count); + ArgumentOutOfRangeException.ThrowIfGreaterThan(count, buffer.Length - index); +#else if (buffer == null) { throw new ArgumentNullException(nameof(buffer)); @@ -90,6 +96,7 @@ public override int Read(char[] buffer, int index, int count) { throw new ArgumentOutOfRangeException(); } +#endif if (this.closed) { @@ -119,7 +126,11 @@ public override string ReadToEnd() } this.pos = this.length; +#if NET8_0_OR_GREATER + return new string(this.chars[this.pos..this.length].Span); +#else return new string(this.chars.Slice(this.pos, this.length - this.pos).ToArray()); +#endif } public override string ReadLine() @@ -135,7 +146,11 @@ public override string ReadLine() char ch = this.chars.Span[i]; if (ch == '\r' || ch == '\n') { +#if NET8_0_OR_GREATER + string result = new (this.chars[this.pos..i].Span); +#else string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); +#endif this.pos = i + 1; if (ch == '\r' && this.pos < this.length && this.chars.Span[this.pos] == '\n') { @@ -150,7 +165,11 @@ public override string ReadLine() if (i > this.pos) { +#if NET8_0_OR_GREATER + string result = new (this.chars[this.pos..i].Span); +#else string result = new (this.chars.Slice(this.pos, i - this.pos).ToArray()); +#endif this.pos = i; return result; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs index 30f6364a32..9ee43b8b07 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/LegacyEncryptionTests.cs @@ -1491,7 +1491,7 @@ private static EncryptionItemRequestOptions GetRequestOptions( }; } - private static TransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( string dekId, List pathsToEncrypt, string ifMatchEtag = null) @@ -1812,13 +1812,11 @@ public override Stream ToStream(T input) MemoryStream streamPayload = new(); using (StreamWriter streamWriter = new(streamPayload, encoding: Encoding.UTF8, bufferSize: 1024, leaveOpen: true)) { - using (JsonWriter writer = new JsonTextWriter(streamWriter)) - { - writer.Formatting = Formatting.None; - this.serializer.Serialize(writer, input); - writer.Flush(); - streamWriter.Flush(); - } + using JsonTextWriter writer = new (streamWriter); + writer.Formatting = Formatting.None; + this.serializer.Serialize(writer, input); + writer.Flush(); + streamWriter.Flush(); } streamPayload.Position = 0; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs index 661b876907..14b1915abb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/MdeCustomEncryptionTests.cs @@ -1822,7 +1822,7 @@ private static EncryptionItemRequestOptions GetRequestOptions( } } - private static TransactionalBatchItemRequestOptions GetBatchItemRequestOptions( + private static EncryptionTransactionalBatchItemRequestOptions GetBatchItemRequestOptions( string dekId, List pathsToEncrypt, string ifMatchEtag = null) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj index c5dfa8db0c..915be6c3f8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/EmulatorTests/Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests.csproj @@ -3,7 +3,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Custom.EmulatorTests diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 4deb4d0edf..821014c014 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -3,7 +3,7 @@ Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests Exe - net6 + net8.0 enable enable diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index eee64aada9..7e1f7d48fe 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -4,7 +4,7 @@ true true AnyCPU - net6.0 + net6.0;net8.0 false false Microsoft.Azure.Cosmos.Encryption.Tests diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs index c58109d65b..84bd8c63c1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/ContractEnforcement.cs @@ -14,7 +14,7 @@ public class ContractEnforcement { - private static readonly InvariantComparer invariantComparer = new InvariantComparer(); + private static readonly InvariantComparer invariantComparer = new (); private static Assembly GetAssemblyLocally(string name) { @@ -91,7 +91,10 @@ private static string GenerateNameWithClassAttributes(Type type) $"{nameof(type.IsValueType)}:{(type.IsValueType ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsNested)}:{(type.IsNested ? bool.TrueString : bool.FalseString)};" + $"{nameof(type.IsGenericType)}:{(type.IsGenericType ? bool.TrueString : bool.FalseString)};" + +#pragma warning disable SYSLIB0050 // 'Type.IsSerializable' is obsolete: 'Formatter-based serialization is obsolete and should not be used. $"{nameof(type.IsSerializable)}:{(type.IsSerializable ? bool.TrueString : bool.FalseString)}"; +#pragma warning restore SYSLIB0050 // 'Type.IsSerializable' is obsolete: 'Formatter-based serialization is obsolete and should not be used. + } private static string GenerateNameWithMethodAttributes(MethodInfo methodInfo) @@ -241,7 +244,7 @@ public static void ValidatePreviewContractContainBreakingChanges( public static string GetCurrentContract(string dllName) { - TypeTree locally = new TypeTree(typeof(object)); + TypeTree locally = new (typeof(object)); Assembly assembly = ContractEnforcement.GetAssemblyLocally(dllName); Type[] exportedTypes = assembly.GetExportedTypes(); ContractEnforcement.BuildTypeTree(locally, exportedTypes); @@ -252,13 +255,13 @@ public static string GetCurrentContract(string dllName) public static string GetCurrentTelemetryContract(string dllName) { - List nonTelemetryModels = new List + List nonTelemetryModels = new() { "AzureVMMetadata", "Compute" }; - TypeTree locally = new TypeTree(typeof(object)); + TypeTree locally = new (typeof(object)); Assembly assembly = ContractEnforcement.GetAssemblyLocally(dllName); Type[] exportedTypes = assembly.GetTypes().Where(t => t!= null && @@ -328,7 +331,10 @@ public static void ValidateJsonAreSame(string baselineJson, string currentJson) private class InvariantComparer : IComparer { - public int Compare(string a, string b) => Comparer.DefaultInvariant.Compare(a, b); + public int Compare(string a, string b) + { + return Comparer.DefaultInvariant.Compare(a, b); + } } } } From 31c20e74721c88cda94b1a58633b382409fde480 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 8 Oct 2024 09:30:13 +0200 Subject: [PATCH 41/71] - remove implicit IsPreview from csproj --- .../src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index d17c1bc48b..83f2758040 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,7 +17,6 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok - True IS_PREVIEW From 90b2cecf0352aa7d1b9ebc15053635bb2f2c9b68 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 11:14:57 +0200 Subject: [PATCH 42/71] + JsonNodeSqlSerializer ! JObjectSqlSerializer remove formatting on serializing inner arrays/objects to string --- .../JObjectSqlSerializer.Preview.cs | 4 +- .../JsonNodeSqlSerializer.Preview.cs | 93 +++++++++++++++++++ .../JsonNodeSqlSerializerTests.cs | 92 ++++++++++++++++++ 3 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs index 2c706aa361..b1ccb5b88c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JObjectSqlSerializer.Preview.cs @@ -49,10 +49,10 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB (buffer, length) = SerializeString(propertyValue.ToObject()); return (TypeMarker.String, buffer, length); case JTokenType.Array: - (buffer, length) = SerializeString(propertyValue.ToString()); + (buffer, length) = SerializeString(propertyValue.ToString(Formatting.None)); return (TypeMarker.Array, buffer, length); case JTokenType.Object: - (buffer, length) = SerializeString(propertyValue.ToString()); + (buffer, length) = SerializeString(propertyValue.ToString(Formatting.None)); return (TypeMarker.Object, buffer, length); default: throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.Type}"); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs new file mode 100644 index 0000000000..e86915700b --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -0,0 +1,93 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if IS_PREVIEW && NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Diagnostics; + using System.Text.Json; + using System.Text.Json.Nodes; + using Microsoft.Data.Encryption.Cryptography.Serializers; + + internal class JsonNodeSqlSerializer + { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + + // UTF-8 encoding. + private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); + +#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed + internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JsonNode propertyValue, ArrayPoolManager arrayPoolManager) + { + byte[] buffer; + int length; + + if (propertyValue == null) + { + return (TypeMarker.Null, null, -1); + } + + switch (propertyValue.GetValueKind()) + { + case JsonValueKind.Undefined: + Debug.Assert(false, "Undefined value cannot be in the JSON"); + return (default, null, -1); + case JsonValueKind.Null: + Debug.Assert(false, "Null type should have been handled by caller"); + return (TypeMarker.Null, null, -1); + case JsonValueKind.True: + (buffer, length) = SerializeFixed(SqlBoolSerializer, true); + return (TypeMarker.Boolean, buffer, length); + case JsonValueKind.False: + (buffer, length) = SerializeFixed(SqlBoolSerializer, false); + return (TypeMarker.Boolean, buffer, length); + case JsonValueKind.Number: + if (long.TryParse(propertyValue.ToJsonString(), out long longValue)) + { + (buffer, length) = SerializeFixed(SqlLongSerializer, longValue); + return (TypeMarker.Long, buffer, length); + } + else if (double.TryParse(propertyValue.ToJsonString(), out double doubleValue)) + { + (buffer, length) = SerializeFixed(SqlDoubleSerializer, doubleValue); + return (TypeMarker.Double, buffer, length); + } + else + { + throw new InvalidOperationException("Unsupported Number type"); + } + + case JsonValueKind.String: + (buffer, length) = SerializeString(propertyValue.GetValue()); + return (TypeMarker.String, buffer, length); + case JsonValueKind.Array: + (buffer, length) = SerializeString(propertyValue.ToJsonString()); + return (TypeMarker.Array, buffer, length); + case JsonValueKind.Object: + (buffer, length) = SerializeString(propertyValue.ToJsonString()); + return (TypeMarker.Object, buffer, length); + default: + throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.GetValueKind()}"); + } + + (byte[], int) SerializeFixed(IFixedSizeSerializer serializer, T value) + { + byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); + int length = serializer.Serialize(value, buffer); + return (buffer, length); + } + + (byte[], int) SerializeString(string value) + { + byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); + int length = SqlVarCharSerializer.Serialize(value, buffer); + return (buffer, length); + } + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs new file mode 100644 index 0000000000..f8f46fea2f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs @@ -0,0 +1,92 @@ +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text.Json.Nodes; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json.Linq; + + [TestClass] + public class JsonNodeSqlSerializerTests + { + private static ArrayPoolManager _poolManager; + + [ClassInitialize] + public static void ClassInitialize(TestContext context) + { + _ = context; + _poolManager = new ArrayPoolManager(); + } + + [TestMethod] + [DynamicData(nameof(SerializationSamples))] + public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[] expectedBytes, int expectedLength) + { + JsonNodeSqlSerializer serializer = new(); + + (TypeMarker serializedType, byte[] serializedBytes, int serializedBytesCount) = serializer.Serialize(testNode, _poolManager); + + Assert.AreEqual((TypeMarker)expectedType, serializedType); + Assert.AreEqual(expectedLength, serializedBytesCount); + if (expectedLength == -1) + { + Assert.IsTrue(serializedBytes == null); + } + else + { + Assert.IsTrue(expectedBytes.SequenceEqual(serializedBytes.AsSpan(0, serializedBytesCount).ToArray())); + } + } + + public static IEnumerable SerializationSamples + { + get + { + List values = new() + { + new object[] {JsonValue.Create((string)null), (byte)TypeMarker.Null, null, -1 }, + new object[] {JsonValue.Create(true), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), 8}, + new object[] {JsonValue.Create(false), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), 8}, + new object[] {JsonValue.Create(192), (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), 8}, + new object[] {JsonValue.Create(192.5), (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), 8}, + new object[] {JsonValue.Create(testString), (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), 11}, + new object[] {JsonValue.Create(testArray), (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), 10}, + new object[] {JsonValue.Create(testClass), (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), 33} + }; + + return values; + } + } + + private static readonly string testString = "Hello world"; + private static readonly int[] testArray = new[] {10, 18, 19}; + private static readonly TestClass testClass = new() { SomeInt = 1, SomeString = "asdf" }; + + private class TestClass + { + public int SomeInt { get; set; } + public string SomeString { get; set; } + } + + private static byte[] GetNewtonsoftValueEquivalent(T value) + { + JObjectSqlSerializer serializer = new (); + JToken token = value switch + { + int[] => new JArray(value), + TestClass => JObject.FromObject(value), + _ => new JValue(value), + }; + (TypeMarker _, byte[] bytes, int lenght) = serializer.Serialize(token, _poolManager); + return bytes.AsSpan(0, lenght).ToArray(); + } + + } +} + +#endif \ No newline at end of file From 6f27d9d6a2c087521dc30ddd6084ff738c303666 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 14:10:54 +0200 Subject: [PATCH 43/71] + initial commit --- .../src/EncryptionOptions.cs | 25 +++ ...soft.Azure.Cosmos.Encryption.Custom.csproj | 1 + .../MdeEncryptionProcessor.Preview.cs | 115 ++----------- .../src/Transformation/MdeEncryptor.cs | 23 +++ .../MdeJObjectEncryptionProcessor.Preview.cs | 160 ++++++++++++++++++ .../MdeJsonNodeEncryptionProcessor.Preview.cs | 115 +++++++++++++ .../SystemTextJson/JsonBytes.cs | 34 ++++ .../SystemTextJson/JsonBytesConverter.cs | 26 +++ .../EncryptionBenchmark.cs | 10 +- ...Encryption.Custom.Performance.Tests.csproj | 1 + .../Transformation/JsonBytesConverterTests.cs | 40 +++++ .../Transformation/JsonBytesTests.cs | 33 ++++ 12 files changed, 475 insertions(+), 108 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index a5dff8f12b..2d9ad7aee6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -6,6 +6,25 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System.Collections.Generic; + /// + /// API for JSON processing + /// + public enum JsonProcessor + { + /// + /// Newtonsoft.Json + /// + Newtonsoft, + +#if NET8_0_OR_GREATER + /// + /// System.Text.Json + /// + /// Available with .NET8.0 package only. + SystemTextJson, +#endif + } + /// /// Options for encryption of data. /// @@ -35,5 +54,11 @@ public sealed class EncryptionOptions /// Example of a path specification: /sensitive /// public IEnumerable PathsToEncrypt { get; set; } + + /// + /// Gets or sets API used for Json processing + /// + /// Setting only applies with Mde encryption is used. + public JsonProcessor JsonProcessor { get; set; } = JsonProcessor.Newtonsoft; } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 83f2758040..d17c1bc48b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -17,6 +17,7 @@ https://github.com/Azure/azure-cosmos-dotnet-v3 http://go.microsoft.com/fwlink/?LinkID=288890 microsoft;azure;cosmos;cosmosdb;documentdb;docdb;nosql;azureofficial;dotnetcore;netcore;netstandard;client;encryption;byok + True IS_PREVIEW diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index b0f293a592..530be77e14 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -7,18 +7,18 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; - using System.Collections.Generic; using System.IO; - using System.Linq; using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; internal class MdeEncryptionProcessor { - internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + internal MdeJObjectEncryptionProcessor JObjectEncryptionProcessor { get; set; } = new MdeJObjectEncryptionProcessor(); - internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); +#if NET8_0_OR_GREATER + internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor(); +#endif public async Task EncryptAsync( Stream input, @@ -26,59 +26,14 @@ public async Task EncryptAsync( EncryptionOptions encryptionOptions, CancellationToken token) { - JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); - List pathsEncrypted = new (); - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new (); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + return encryptionOptions.JsonProcessor switch { + JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #if NET8_0_OR_GREATER - string propertyName = pathToEncrypt[1..]; -#else - string propertyName = pathToEncrypt.Substring(1); + JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #endif - if (!itemJObj.TryGetValue(propertyName, out JToken propertyValue)) - { - continue; - } - - if (propertyValue.Type == JTokenType.Null) - { - continue; - } - - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - - if (plainText == null) - { - continue; - } - - byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); - - itemJObj[propertyName] = encryptedBytes; - pathsEncrypted.Add(pathToEncrypt); - } - - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted); - - itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); -#if NET8_0_OR_GREATER - await input.DisposeAsync(); -#else - input.Dispose(); -#endif - return EncryptionProcessor.BaseSerializer.ToStream(itemJObj); + _ => throw new InvalidOperationException("Unsupported JsonProcessor") + }; } internal async Task DecryptObjectAsync( @@ -88,57 +43,7 @@ internal async Task DecryptObjectAsync( CosmosDiagnosticsContext diagnosticsContext, CancellationToken cancellationToken) { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != 3) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new (); - using ArrayPoolManager charPoolManager = new (); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); - foreach (string path in encryptionProperties.EncryptedPaths) - { -#if NET8_0_OR_GREATER - string propertyName = path[1..]; -#else - string propertyName = path.Substring(1); -#endif - - if (!document.TryGetValue(propertyName, out JToken propertyValue)) - { - // malformed document, such record shouldn't be there at all - continue; - } - - byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); - if (cipherTextWithTypeMarker == null) - { - continue; - } - - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); - - this.Serializer.DeserializeAndAddProperty( - (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - document, - propertyName, - charPoolManager); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - document.Remove(Constants.EncryptedInfo); - return decryptionContext; + return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index cf5a0ffdf6..bb1608a466 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -31,6 +31,29 @@ internal virtual byte[] Encrypt(DataEncryptionKey encryptionKey, TypeMarker type return encryptedText; } + internal virtual (byte[], int) Encrypt(DataEncryptionKey encryptionKey, TypeMarker typeMarker, byte[] plainText, int plainTextLength, ArrayPoolManager arrayPoolManager) + { + int encryptedTextLength = encryptionKey.GetEncryptByteCount(plainTextLength) + 1; + + byte[] encryptedText = arrayPoolManager.Rent(encryptedTextLength); + + encryptedText[0] = (byte)typeMarker; + + int encryptedLength = encryptionKey.EncryptData( + plainText, + plainTextOffset: 0, + plainTextLength, + encryptedText, + outputOffset: 1); + + if (encryptedLength < 0) + { + throw new InvalidOperationException($"{nameof(DataEncryptionKey)} returned null cipherText from {nameof(DataEncryptionKey.EncryptData)}."); + } + + return (encryptedText, encryptedTextLength); + } + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) { int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs new file mode 100644 index 0000000000..d606e2fc28 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -0,0 +1,160 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if IS_PREVIEW + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Newtonsoft.Json.Linq; + + internal class MdeJObjectEncryptionProcessor + { + internal JObjectSqlSerializer Serializer { get; set; } = new JObjectSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JObject itemJObj = EncryptionProcessor.BaseSerializer.FromStream(input); + + Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); + +#if NET8_0_OR_GREATER + await input.DisposeAsync(); +#else + input.Dispose(); +#endif + + return result; + } + + public async Task EncryptAsync( + JObject input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else + string propertyName = pathToEncrypt.Substring(1); +#endif + if (!input.TryGetValue(propertyName, out JToken propertyValue)) + { + continue; + } + + if (propertyValue.Type == JTokenType.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); + + input[propertyName] = encryptedBytes; + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + input.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); + + return EncryptionProcessor.BaseSerializer.ToStream(input); + } + + internal async Task DecryptObjectAsync( + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + foreach (string path in encryptionProperties.EncryptedPaths) + { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else + string propertyName = path.Substring(1); +#endif + + if (!document.TryGetValue(propertyName, out JToken propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + byte[] cipherTextWithTypeMarker = propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName, + charPoolManager); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs new file mode 100644 index 0000000000..45a886d3ba --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -0,0 +1,115 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if IS_PREVIEW && NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System.Collections.Generic; + using System.IO; + using System.Text.Json; + using System.Text.Json.Nodes; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + + internal class MdeJsonNodeEncryptionProcessor + { + internal JsonNodeSqlSerializer Serializer { get; set; } = new JsonNodeSqlSerializer(); + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + internal JsonSerializerOptions JsonSerializerOptions { get; set; } + + private JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; + + public MdeJsonNodeEncryptionProcessor() + { + this.JsonSerializerOptions = new JsonSerializerOptions(); + this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); + } + + public async Task EncryptAsync( + Stream input, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + JsonNode itemJObj = JsonNode.Parse(input); + + Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); + + await input.DisposeAsync(); + return result; + } + + public async Task EncryptAsync( + JsonNode document, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken token) + { + List pathsEncrypted = new (); + TypeMarker typeMarker; + + using ArrayPoolManager arrayPoolManager = new (); + + JsonObject itemObj = document.AsObject(); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) + { +#if NET8_0_OR_GREATER + string propertyName = pathToEncrypt[1..]; +#else + string propertyName = pathToEncrypt.Substring(1); +#endif + if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) + { + continue; + } + + if (propertyValue == null || propertyValue.GetValueKind() == JsonValueKind.Null) + { + continue; + } + + byte[] plainText = null; + (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + + if (plainText == null) + { + continue; + } + + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); + + itemObj[propertyName] = JsonValue.Create(new JsonBytes(encryptedBytes, 0, encryptedBytesCount)); + pathsEncrypted.Add(pathToEncrypt); + } + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted); + + JsonNode propertiesNode = JsonSerializer.SerializeToNode(encryptionProperties); + + itemObj.Add(Constants.EncryptedInfo, propertiesNode); + + MemoryStream ms = new (); + Utf8JsonWriter writer = new (ms, this.jsonWriterOptions); + + JsonSerializer.Serialize(writer, document, this.JsonSerializerOptions); + + ms.Position = 0; + return ms; + } + } +} + +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs new file mode 100644 index 0000000000..a8bc77f75b --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson +{ + using System; + + internal class JsonBytes + { + internal byte[] Bytes { get; private set; } + + internal int Offset { get; private set; } + + internal int Length { get; private set; } + + public JsonBytes(byte[] bytes, int offset, int length) + { + ArgumentNullException.ThrowIfNull(bytes); + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegative(length); + if (bytes.Length < offset + length) + { + throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); + } + + this.Bytes = bytes; + this.Offset = offset; + this.Length = length; + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs new file mode 100644 index 0000000000..d9048375c0 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs @@ -0,0 +1,26 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson +{ + using System; + using System.Text.Json; + using System.Text.Json.Serialization; + + internal class JsonBytesConverter : JsonConverter + { + public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + throw new NotImplementedException(); + } + + public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) + { + writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 638222785f..1e84efccc7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -25,6 +25,9 @@ public partial class EncryptionBenchmark [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } + [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson)] + public JsonProcessor JsonProcessor { get; set; } + [GlobalSetup] public async Task Setup() { @@ -38,7 +41,7 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.encryptionOptions = CreateEncryptionOptions(); + this.encryptionOptions = this.CreateEncryptionOptions(); this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( @@ -74,13 +77,14 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private static EncryptionOptions CreateEncryptionOptions() + private EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() { DataEncryptionKeyId = "dekId", EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = this.JsonProcessor, }; return options; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 821014c014..a36482571b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -6,6 +6,7 @@ net8.0 enable enable + true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs new file mode 100644 index 0000000000..fbef47e872 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs @@ -0,0 +1,40 @@ +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ + using System; + using System.IO; + using System.Text.Json; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class JsonBytesConverterTests + { + [TestMethod] + public void Write_Results_IdenticalToNewtonsoft() + { + byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; + + JsonBytes jsonBytes = new (bytes, 5, 5); + + using MemoryStream ms = new (); + using Utf8JsonWriter writer = new (ms); + + JsonBytesConverter jsonConverter = new (); + jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); + + writer.Flush(); + ms.Flush(); + ms.Position = 0; + StreamReader sr = new(ms); + string systemTextResult = sr.ReadToEnd(); + + byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); + string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); + + Assert.AreEqual(systemTextResult, newtonsoftResult); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs new file mode 100644 index 0000000000..a29f378ff2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs @@ -0,0 +1,33 @@ +#if NET8_0_OR_GREATER + +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ + using System; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class JsonBytesTests + { + [TestMethod] + public void Ctor_ThrowsForInvalidInputs() + { + Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); + Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); + Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); + Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); + } + + [TestMethod] + public void Properties_AreSetCorrectly() + { + byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; + JsonBytes jsonBytes = new (bytes, 1, 5); + + Assert.AreEqual(1, jsonBytes.Offset); + Assert.AreEqual(5, jsonBytes.Length); + Assert.AreSame(bytes, jsonBytes.Bytes); + } + } +} +#endif From d7c06d656315b2974af1826f7f9184656b6ad0ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 14:24:30 +0200 Subject: [PATCH 44/71] ! EncryptioProperties System.Text.Json annotations + tests for MdeEncryptionProperties --- .../src/EncryptionProperties.cs | 6 +++ .../MdeEncryptionProcessorTests.cs | 45 +++++++++++++------ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs index 8e0ccaf84a..5108cfa157 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs @@ -5,23 +5,29 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System.Collections.Generic; + using System.Text.Json.Serialization; using Newtonsoft.Json; internal class EncryptionProperties { [JsonProperty(PropertyName = Constants.EncryptionFormatVersion)] + [JsonPropertyName(Constants.EncryptionFormatVersion)] public int EncryptionFormatVersion { get; } [JsonProperty(PropertyName = Constants.EncryptionDekId)] + [JsonPropertyName(Constants.EncryptionDekId)] public string DataEncryptionKeyId { get; } [JsonProperty(PropertyName = Constants.EncryptionAlgorithm)] + [JsonPropertyName(Constants.EncryptionAlgorithm)] public string EncryptionAlgorithm { get; } [JsonProperty(PropertyName = Constants.EncryptedData)] + [JsonPropertyName(Constants.EncryptedData)] public byte[] EncryptedData { get; } [JsonProperty(PropertyName = Constants.EncryptedPaths)] + [JsonPropertyName(Constants.EncryptedPaths)] public IEnumerable EncryptedPaths { get; } public EncryptionProperties( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 69a9c7afb0..35722f40eb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -20,19 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests public class MdeEncryptionProcessorTests { private static Mock mockEncryptor; - private static EncryptionOptions encryptionOptions; private const string dekId = "dekId"; [ClassInitialize] public static void ClassInitialize(TestContext testContext) { _ = testContext; - encryptionOptions = new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt - }; Mock DekMock = new(); DekMock.Setup(m => m.EncryptData(It.IsAny())) @@ -125,12 +118,13 @@ await EncryptionProcessor.EncryptAsync( } [TestMethod] - public async Task EncryptDecryptPropertyWithNullValue() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -146,11 +140,12 @@ public async Task EncryptDecryptPropertyWithNullValue() } [TestMethod] - public async Task ValidateEncryptDecryptDocument() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -166,7 +161,8 @@ public async Task ValidateEncryptDecryptDocument() } [TestMethod] - public async Task ValidateDecryptStream() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); @@ -209,7 +205,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() Assert.IsNull(decryptionContext); } - private static async Task VerifyEncryptionSucceeded(TestDoc testDoc) + private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions) { Stream encryptedStream = await EncryptionProcessor.EncryptAsync( testDoc.ToStream(), @@ -287,5 +283,26 @@ private static void VerifyDecryptionSucceeded( } } } + + public static IEnumerable EncryptionOptionsCombinations => new[] { + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft + } + }, +#if NET8_0_OR_GREATER + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.SystemTextJson + } + }, +#endif + }; } -} +} \ No newline at end of file From cb9d1665dd563edd6f5b633a6983899bdd9069ad Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 14:51:57 +0200 Subject: [PATCH 45/71] + bump benchmark --- .../Readme.md | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index ced6fc95ed..8be5532ca4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -3,17 +3,23 @@ BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores .NET SDK=8.0.400 - [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 + [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | -| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | -| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | -| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | +| Method | DocumentSizeInKb | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **Newtonsoft** | **22.40 μs** | **0.342 μs** | **0.501 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | +| Decrypt | 1 | Newtonsoft | 25.81 μs | 0.305 μs | 0.427 μs | 0.1526 | 0.0305 | - | 39.19 KB | +| **Encrypt** | **1** | **SystemTextJson** | **14.72 μs** | **0.275 μs** | **0.411 μs** | **0.0916** | **0.0153** | **-** | **22.55 KB** | +| Decrypt | 1 | SystemTextJson | 25.69 μs | 0.290 μs | 0.396 μs | 0.1526 | 0.0305 | - | 39.19 KB | +| **Encrypt** | **10** | **Newtonsoft** | **83.98 μs** | **0.437 μs** | **0.613 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | +| Decrypt | 10 | Newtonsoft | 99.39 μs | 0.553 μs | 0.827 μs | 0.6104 | 0.1221 | - | 152.45 KB | +| **Encrypt** | **10** | **SystemTextJson** | **41.92 μs** | **0.212 μs** | **0.304 μs** | **0.4272** | **0.0610** | **-** | **103.06 KB** | +| Decrypt | 10 | SystemTextJson | 99.43 μs | 0.558 μs | 0.835 μs | 0.6104 | 0.1221 | - | 152.45 KB | +| **Encrypt** | **100** | **Newtonsoft** | **1,074.93 μs** | **11.946 μs** | **17.510 μs** | **25.3906** | **23.4375** | **21.4844** | **1638.32 KB** | +| Decrypt | 100 | Newtonsoft | 1,133.11 μs | 20.544 μs | 29.463 μs | 17.5781 | 15.6250 | 15.6250 | 1229.43 KB | +| **Encrypt** | **100** | **SystemTextJson** | **797.64 μs** | **15.574 μs** | **22.828 μs** | **26.3672** | **26.3672** | **26.3672** | **942.81 KB** | +| Decrypt | 100 | SystemTextJson | 1,120.97 μs | 14.956 μs | 22.386 μs | 19.5313 | 17.5781 | 17.5781 | 1229.45 KB | From 5fc51650fd36a42d64553ba626b4825b2fba46eb Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 9 Oct 2024 16:07:35 +0200 Subject: [PATCH 46/71] ~ wip --- .../JsonNodeSqlSerializer.Preview.cs | 34 +++++++++ .../MdeJObjectEncryptionProcessor.Preview.cs | 10 +-- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 69 +++++++++++++++++++ 3 files changed, 108 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs index e86915700b..283e91bc3b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -88,6 +88,40 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB return (buffer, length); } } + + internal virtual void DeserializeAndAddProperty( + TypeMarker typeMarker, + ReadOnlySpan serializedBytes, + JsonNode jsonNode, + string key, + ArrayPoolManager arrayPoolManager) + { + switch (typeMarker) + { + case TypeMarker.Boolean: + jsonNode[key] = SqlBoolSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Double: + jsonNode[key] = SqlDoubleSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Long: + jsonNode[key] = SqlLongSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.String: + jsonNode[key] = SqlVarCharSerializer.Deserialize(serializedBytes); + break; + case TypeMarker.Array: + jsonNode[key] = JsonNode.Parse(serializedBytes); + break; + case TypeMarker.Object: + jsonNode[key] = JsonNode.Parse(serializedBytes); + break; + default: + Debug.Fail($"Unexpected type marker {typeMarker}"); + break; + } + } + } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index d606e2fc28..189bd34e42 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -96,11 +96,11 @@ public async Task EncryptAsync( } internal async Task DecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) { _ = diagnosticsContext; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 45a886d3ba..197e044e3b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -6,13 +6,16 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { + using System; using System.Collections.Generic; using System.IO; + using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; + using Newtonsoft.Json.Linq; internal class MdeJsonNodeEncryptionProcessor { @@ -109,6 +112,72 @@ public async Task EncryptAsync( ms.Position = 0; return ms; } + + internal async Task DecryptObjectAsync( + JsonNode document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (encryptionProperties.EncryptionFormatVersion != 3) + { + throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + using ArrayPoolManager charPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + + JsonObject itemObj = document.AsObject(); + + foreach (string path in encryptionProperties.EncryptedPaths) + { +#if NET8_0_OR_GREATER + string propertyName = path[1..]; +#else + string propertyName = path.Substring(1); +#endif + + if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) + { + // malformed document, such record shouldn't be there at all + continue; + } + + //TODO: figure out if we can get char span out from JsonNode + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(propertyValue.) + + byte[] cipherTextWithTypeMarker = Convert.FromBase64String.TryFromBase64StringBase64Encoder propertyValue.propertyValue.ToObject(); + if (cipherTextWithTypeMarker == null) + { + continue; + } + + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + + this.Serializer.DeserializeAndAddProperty( + (TypeMarker)cipherTextWithTypeMarker[0], + plainText.AsSpan(0, decryptedCount), + document, + propertyName, + charPoolManager); + + pathsDecrypted.Add(path); + } + + DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( + pathsDecrypted, + encryptionProperties.DataEncryptionKeyId); + + document.Remove(Constants.EncryptedInfo); + return decryptionContext; + } } } From c9e7f306cb0ec39026add9d19871860b9fc09c6a Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 00:34:58 +0200 Subject: [PATCH 47/71] + initial for JsonNode decryption - drop unnecessary classes --- .../src/EncryptionProcessor.cs | 102 +++++++++++++++++- .../MdeEncryptionProcessor.Preview.cs | 15 +++ .../MdeJsonNodeEncryptionProcessor.Preview.cs | 31 ++---- .../SystemTextJson/JsonBytes.cs | 34 ------ .../SystemTextJson/JsonBytesConverter.cs | 26 ----- .../EncryptionBenchmark.cs | 1 + .../Readme.md | 24 ++--- .../Transformation/JsonBytesConverterTests.cs | 40 ------- .../Transformation/JsonBytesTests.cs | 33 ------ 9 files changed, 135 insertions(+), 171 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 53d74390f8..c2c6abd498 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,6 +10,10 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; + using System.Text.Json; +#if NET8_0_OR_GREATER + using System.Text.Json.Nodes; +#endif using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; @@ -27,6 +31,7 @@ internal static class EncryptionProcessor }; internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -120,6 +125,54 @@ public static async Task EncryptAsync( return (BaseSerializer.ToStream(itemJObj), decryptionContext); } + public static async Task<(Stream, DecryptionContext)> DecryptAsync( + Stream input, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + JsonProcessor jsonProcessor, + CancellationToken cancellationToken) + { + return jsonProcessor switch + { + JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), +#if NET8_0_OR_GREATER + JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), +#endif + _ => throw new InvalidOperationException("Unsupported Json Processor") + }; + } + +#if NET8_0_OR_GREATER + public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( + Stream input, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (input == null) + { + return (input, null); + } + + Debug.Assert(input.CanSeek); + Debug.Assert(encryptor != null); + Debug.Assert(diagnosticsContext != null); + + JsonNode document = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken); + + (JsonNode decryptedDocument, DecryptionContext context) = await DecryptAsync(document, encryptor, diagnosticsContext, cancellationToken); + await input.DisposeAsync(); + + MemoryStream ms = new (); + Utf8JsonWriter writer = new (ms, EncryptionProcessor.JsonWriterOptions); + + System.Text.Json.JsonSerializer.Serialize(writer, decryptedDocument); + + ms.Position = 0; + return (ms, context); + } +#endif + public static async Task<(JObject, DecryptionContext)> DecryptAsync( JObject document, Encryptor encryptor, @@ -142,6 +195,53 @@ public static async Task EncryptAsync( return (document, decryptionContext); } +#if NET8_0_OR_GREATER + public static async Task<(JsonNode, DecryptionContext)> DecryptAsync( + JsonNode document, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + Debug.Assert(document != null); + + Debug.Assert(encryptor != null); + + if (!document.AsObject().TryGetPropertyValue(Constants.EncryptedInfo, out JsonNode encryptionPropertiesNode)) + { + throw new InvalidOperationException("Encryption properties deserialization failed."); + } + + EncryptionProperties encryptionProperties; + try + { + encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(encryptionPropertiesNode); + } + catch (Exception) + { + return (document, null); + } + + DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionProperties, cancellationToken); + + return (document, decryptionContext); + } + + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JsonNode itemNode, EncryptionProperties encryptionProperties, CancellationToken cancellationToken) + { + DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch + { + CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( + itemNode, + encryptor, + encryptionProperties, + diagnosticsContext, + cancellationToken), + _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), + }; + return decryptionContext; + } +#endif + private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); @@ -240,7 +340,7 @@ private static JObject RetrieveItem( MaxDepth = 64, // https://github.com/advisories/GHSA-5crp-9r3c-p9vr }; - itemJObj = JsonSerializer.Create(jsonSerializerSettings).Deserialize(jsonTextReader); + itemJObj = Newtonsoft.Json.JsonSerializer.Create(jsonSerializerSettings).Deserialize(jsonTextReader); } return itemJObj; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 530be77e14..ecca230b6a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -8,6 +8,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.IO; +#if NET8_0_OR_GREATER + using System.Text.Json.Nodes; +#endif using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json.Linq; @@ -45,6 +48,18 @@ internal async Task DecryptObjectAsync( { return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); } + +#if NET8_0_OR_GREATER + internal async Task DecryptObjectAsync( + JsonNode document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + return await this.JsonNodeEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); + } +#endif } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 197e044e3b..188bcbff21 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -10,12 +10,11 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Collections.Generic; using System.IO; using System.Linq; + using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Newtonsoft.Json.Linq; internal class MdeJsonNodeEncryptionProcessor { @@ -23,16 +22,8 @@ internal class MdeJsonNodeEncryptionProcessor internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - internal JsonSerializerOptions JsonSerializerOptions { get; set; } - private JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; - public MdeJsonNodeEncryptionProcessor() - { - this.JsonSerializerOptions = new JsonSerializerOptions(); - this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); - } - public async Task EncryptAsync( Stream input, Encryptor encryptor, @@ -64,11 +55,7 @@ public async Task EncryptAsync( foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { -#if NET8_0_OR_GREATER string propertyName = pathToEncrypt[1..]; -#else - string propertyName = pathToEncrypt.Substring(1); -#endif if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) { continue; @@ -89,7 +76,7 @@ public async Task EncryptAsync( (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); - itemObj[propertyName] = JsonValue.Create(new JsonBytes(encryptedBytes, 0, encryptedBytesCount)); + itemObj[propertyName] = JsonValue.Create(new Memory(encryptedBytes, 0, encryptedBytesCount)); pathsEncrypted.Add(pathToEncrypt); } @@ -107,7 +94,7 @@ public async Task EncryptAsync( MemoryStream ms = new (); Utf8JsonWriter writer = new (ms, this.jsonWriterOptions); - JsonSerializer.Serialize(writer, document, this.JsonSerializerOptions); + JsonSerializer.Serialize(writer, document); ms.Position = 0; return ms; @@ -138,11 +125,7 @@ internal async Task DecryptObjectAsync( foreach (string path in encryptionProperties.EncryptedPaths) { -#if NET8_0_OR_GREATER string propertyName = path[1..]; -#else - string propertyName = path.Substring(1); -#endif if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) { @@ -150,10 +133,8 @@ internal async Task DecryptObjectAsync( continue; } - //TODO: figure out if we can get char span out from JsonNode - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(propertyValue.) - - byte[] cipherTextWithTypeMarker = Convert.FromBase64String.TryFromBase64StringBase64Encoder propertyValue.propertyValue.ToObject(); + // can we get to internal JsonNode buffers to avoid string allocation here? + byte[] cipherTextWithTypeMarker = Convert.FromBase64String(propertyValue.GetValue()); if (cipherTextWithTypeMarker == null) { continue; @@ -175,7 +156,7 @@ internal async Task DecryptObjectAsync( pathsDecrypted, encryptionProperties.DataEncryptionKeyId); - document.Remove(Constants.EncryptedInfo); + itemObj.Remove(Constants.EncryptedInfo); return decryptionContext; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs deleted file mode 100644 index a8bc77f75b..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - - internal class JsonBytes - { - internal byte[] Bytes { get; private set; } - - internal int Offset { get; private set; } - - internal int Length { get; private set; } - - public JsonBytes(byte[] bytes, int offset, int length) - { - ArgumentNullException.ThrowIfNull(bytes); - ArgumentOutOfRangeException.ThrowIfNegative(offset); - ArgumentOutOfRangeException.ThrowIfNegative(length); - if (bytes.Length < offset + length) - { - throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); - } - - this.Bytes = bytes; - this.Offset = offset; - this.Length = length; - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs deleted file mode 100644 index d9048375c0..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - using System.Text.Json; - using System.Text.Json.Serialization; - - internal class JsonBytesConverter : JsonConverter - { - public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) - { - writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 1e84efccc7..0d28361369 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -74,6 +74,7 @@ await EncryptionProcessor.DecryptAsync( new MemoryStream(this.encryptedData!), this.encryptor, new CosmosDiagnosticsContext(), + this.JsonProcessor, CancellationToken.None); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 8be5532ca4..b2bcc93f17 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -11,15 +11,15 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **Newtonsoft** | **22.40 μs** | **0.342 μs** | **0.501 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | -| Decrypt | 1 | Newtonsoft | 25.81 μs | 0.305 μs | 0.427 μs | 0.1526 | 0.0305 | - | 39.19 KB | -| **Encrypt** | **1** | **SystemTextJson** | **14.72 μs** | **0.275 μs** | **0.411 μs** | **0.0916** | **0.0153** | **-** | **22.55 KB** | -| Decrypt | 1 | SystemTextJson | 25.69 μs | 0.290 μs | 0.396 μs | 0.1526 | 0.0305 | - | 39.19 KB | -| **Encrypt** | **10** | **Newtonsoft** | **83.98 μs** | **0.437 μs** | **0.613 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | -| Decrypt | 10 | Newtonsoft | 99.39 μs | 0.553 μs | 0.827 μs | 0.6104 | 0.1221 | - | 152.45 KB | -| **Encrypt** | **10** | **SystemTextJson** | **41.92 μs** | **0.212 μs** | **0.304 μs** | **0.4272** | **0.0610** | **-** | **103.06 KB** | -| Decrypt | 10 | SystemTextJson | 99.43 μs | 0.558 μs | 0.835 μs | 0.6104 | 0.1221 | - | 152.45 KB | -| **Encrypt** | **100** | **Newtonsoft** | **1,074.93 μs** | **11.946 μs** | **17.510 μs** | **25.3906** | **23.4375** | **21.4844** | **1638.32 KB** | -| Decrypt | 100 | Newtonsoft | 1,133.11 μs | 20.544 μs | 29.463 μs | 17.5781 | 15.6250 | 15.6250 | 1229.43 KB | -| **Encrypt** | **100** | **SystemTextJson** | **797.64 μs** | **15.574 μs** | **22.828 μs** | **26.3672** | **26.3672** | **26.3672** | **942.81 KB** | -| Decrypt | 100 | SystemTextJson | 1,120.97 μs | 14.956 μs | 22.386 μs | 19.5313 | 17.5781 | 17.5781 | 1229.45 KB | +| **Encrypt** | **1** | **Newtonsoft** | **23.24 μs** | **0.606 μs** | **0.888 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | +| Decrypt | 1 | Newtonsoft | 24.95 μs | 0.230 μs | 0.344 μs | 0.1526 | 0.0305 | - | 39.27 KB | +| **Encrypt** | **1** | **SystemTextJson** | **14.46 μs** | **0.165 μs** | **0.242 μs** | **0.0916** | **0.0153** | **-** | **22.48 KB** | +| Decrypt | 1 | SystemTextJson | 16.05 μs | 0.284 μs | 0.416 μs | 0.0916 | 0.0305 | - | 22.1 KB | +| **Encrypt** | **10** | **Newtonsoft** | **84.60 μs** | **0.908 μs** | **1.359 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | +| Decrypt | 10 | Newtonsoft | 98.39 μs | 0.856 μs | 1.255 μs | 0.6104 | 0.1221 | - | 152.53 KB | +| **Encrypt** | **10** | **SystemTextJson** | **41.72 μs** | **0.341 μs** | **0.501 μs** | **0.4272** | **0.0610** | **-** | **102.99 KB** | +| Decrypt | 10 | SystemTextJson | 46.91 μs | 0.430 μs | 0.630 μs | 0.4272 | 0.0610 | - | 105.06 KB | +| **Encrypt** | **100** | **Newtonsoft** | **1,072.91 μs** | **13.802 μs** | **20.231 μs** | **25.3906** | **21.4844** | **21.4844** | **1638.34 KB** | +| Decrypt | 100 | Newtonsoft | 1,107.93 μs | 12.955 μs | 18.990 μs | 17.5781 | 15.6250 | 15.6250 | 1229.52 KB | +| **Encrypt** | **100** | **SystemTextJson** | **794.00 μs** | **25.274 μs** | **37.047 μs** | **24.4141** | **24.4141** | **24.4141** | **942.73 KB** | +| Decrypt | 100 | SystemTextJson | 819.31 μs | 14.159 μs | 20.754 μs | 22.4609 | 22.4609 | 22.4609 | 1037.04 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs deleted file mode 100644 index fbef47e872..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.IO; - using System.Text.Json; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesConverterTests - { - [TestMethod] - public void Write_Results_IdenticalToNewtonsoft() - { - byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - - JsonBytes jsonBytes = new (bytes, 5, 5); - - using MemoryStream ms = new (); - using Utf8JsonWriter writer = new (ms); - - JsonBytesConverter jsonConverter = new (); - jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); - - writer.Flush(); - ms.Flush(); - ms.Position = 0; - StreamReader sr = new(ms); - string systemTextResult = sr.ReadToEnd(); - - byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); - string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); - - Assert.AreEqual(systemTextResult, newtonsoftResult); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs deleted file mode 100644 index a29f378ff2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesTests - { - [TestMethod] - public void Ctor_ThrowsForInvalidInputs() - { - Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); - } - - [TestMethod] - public void Properties_AreSetCorrectly() - { - byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - JsonBytes jsonBytes = new (bytes, 1, 5); - - Assert.AreEqual(1, jsonBytes.Offset); - Assert.AreEqual(5, jsonBytes.Length); - Assert.AreSame(bytes, jsonBytes.Bytes); - } - } -} -#endif From fed6a3a21d8af9f958147cc45bc5264152d9c89c Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 13:35:32 +0200 Subject: [PATCH 48/71] ~ polishing and benchmark refresh --- .../JsonNodeSqlSerializer.Preview.cs | 25 ++++++----------- .../src/Transformation/MdeEncryptor.cs | 6 ++-- .../MdeJObjectEncryptionProcessor.Preview.cs | 2 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 11 ++++---- .../Readme.md | 28 +++++++++---------- 5 files changed, 31 insertions(+), 41 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs index 283e91bc3b..3fff2166de 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -89,39 +89,30 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB } } - internal virtual void DeserializeAndAddProperty( + internal virtual JsonNode Deserialize( TypeMarker typeMarker, ReadOnlySpan serializedBytes, - JsonNode jsonNode, - string key, ArrayPoolManager arrayPoolManager) { switch (typeMarker) { case TypeMarker.Boolean: - jsonNode[key] = SqlBoolSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlBoolSerializer.Deserialize(serializedBytes)); case TypeMarker.Double: - jsonNode[key] = SqlDoubleSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlDoubleSerializer.Deserialize(serializedBytes)); case TypeMarker.Long: - jsonNode[key] = SqlLongSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlLongSerializer.Deserialize(serializedBytes)); case TypeMarker.String: - jsonNode[key] = SqlVarCharSerializer.Deserialize(serializedBytes); - break; + return JsonValue.Create(SqlVarCharSerializer.Deserialize(serializedBytes)); case TypeMarker.Array: - jsonNode[key] = JsonNode.Parse(serializedBytes); - break; + return JsonNode.Parse(serializedBytes); case TypeMarker.Object: - jsonNode[key] = JsonNode.Parse(serializedBytes); - break; + return JsonNode.Parse(serializedBytes); default: Debug.Fail($"Unexpected type marker {typeMarker}"); - break; + return null; } } - } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs index bb1608a466..3b7054611e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptor.cs @@ -54,16 +54,16 @@ internal virtual (byte[], int) Encrypt(DataEncryptionKey encryptionKey, TypeMark return (encryptedText, encryptedTextLength); } - internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, ArrayPoolManager arrayPoolManager) + internal virtual (byte[] plainText, int plainTextLength) Decrypt(DataEncryptionKey encryptionKey, byte[] cipherText, int cipherTextLength, ArrayPoolManager arrayPoolManager) { - int plainTextLength = encryptionKey.GetDecryptByteCount(cipherText.Length - 1); + int plainTextLength = encryptionKey.GetDecryptByteCount(cipherTextLength - 1); byte[] plainText = arrayPoolManager.Rent(plainTextLength); int decryptedLength = encryptionKey.DecryptData( cipherText, cipherTextOffset: 1, - cipherTextLength: cipherText.Length - 1, + cipherTextLength: cipherTextLength - 1, plainText, outputOffset: 0); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 189bd34e42..15260aa4dc 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -135,7 +135,7 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 188bcbff21..02fc91466e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -134,19 +134,18 @@ internal async Task DecryptObjectAsync( } // can we get to internal JsonNode buffers to avoid string allocation here? - byte[] cipherTextWithTypeMarker = Convert.FromBase64String(propertyValue.GetValue()); - if (cipherTextWithTypeMarker == null) + string base64String = propertyValue.GetValue(); + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent((base64String.Length * sizeof(char) * 3 / 4) + 4); + if (!Convert.TryFromBase64Chars(base64String, cipherTextWithTypeMarker, out int cipherTextLength)) { continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); - this.Serializer.DeserializeAndAddProperty( + document[propertyName] = this.Serializer.Deserialize( (TypeMarker)cipherTextWithTypeMarker[0], plainText.AsSpan(0, decryptedCount), - document, - propertyName, charPoolManager); pathsDecrypted.Add(path); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index b2bcc93f17..e59c09ed0a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,8 +1,8 @@ ``` ini -BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.26100.2033) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.400 +.NET SDK=8.0.403 [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 @@ -11,15 +11,15 @@ LaunchCount=2 WarmupCount=10 ``` | Method | DocumentSizeInKb | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |-------- |----------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **Newtonsoft** | **23.24 μs** | **0.606 μs** | **0.888 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | -| Decrypt | 1 | Newtonsoft | 24.95 μs | 0.230 μs | 0.344 μs | 0.1526 | 0.0305 | - | 39.27 KB | -| **Encrypt** | **1** | **SystemTextJson** | **14.46 μs** | **0.165 μs** | **0.242 μs** | **0.0916** | **0.0153** | **-** | **22.48 KB** | -| Decrypt | 1 | SystemTextJson | 16.05 μs | 0.284 μs | 0.416 μs | 0.0916 | 0.0305 | - | 22.1 KB | -| **Encrypt** | **10** | **Newtonsoft** | **84.60 μs** | **0.908 μs** | **1.359 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | -| Decrypt | 10 | Newtonsoft | 98.39 μs | 0.856 μs | 1.255 μs | 0.6104 | 0.1221 | - | 152.53 KB | -| **Encrypt** | **10** | **SystemTextJson** | **41.72 μs** | **0.341 μs** | **0.501 μs** | **0.4272** | **0.0610** | **-** | **102.99 KB** | -| Decrypt | 10 | SystemTextJson | 46.91 μs | 0.430 μs | 0.630 μs | 0.4272 | 0.0610 | - | 105.06 KB | -| **Encrypt** | **100** | **Newtonsoft** | **1,072.91 μs** | **13.802 μs** | **20.231 μs** | **25.3906** | **21.4844** | **21.4844** | **1638.34 KB** | -| Decrypt | 100 | Newtonsoft | 1,107.93 μs | 12.955 μs | 18.990 μs | 17.5781 | 15.6250 | 15.6250 | 1229.52 KB | -| **Encrypt** | **100** | **SystemTextJson** | **794.00 μs** | **25.274 μs** | **37.047 μs** | **24.4141** | **24.4141** | **24.4141** | **942.73 KB** | -| Decrypt | 100 | SystemTextJson | 819.31 μs | 14.159 μs | 20.754 μs | 22.4609 | 22.4609 | 22.4609 | 1037.04 KB | +| **Encrypt** | **1** | **Newtonsoft** | **22.23 μs** | **0.392 μs** | **0.562 μs** | **0.1526** | **0.0305** | **-** | **36.44 KB** | +| Decrypt | 1 | Newtonsoft | 25.65 μs | 0.482 μs | 0.691 μs | 0.1221 | - | - | 39.27 KB | +| **Encrypt** | **1** | **SystemTextJson** | **15.61 μs** | **0.886 μs** | **1.242 μs** | **0.0916** | **0.0305** | **-** | **22.48 KB** | +| Decrypt | 1 | SystemTextJson | 14.68 μs | 0.334 μs | 0.479 μs | 0.0763 | 0.0153 | - | 20.83 KB | +| **Encrypt** | **10** | **Newtonsoft** | **83.23 μs** | **1.608 μs** | **2.147 μs** | **0.6104** | **0.1221** | **-** | **166.64 KB** | +| Decrypt | 10 | Newtonsoft | 101.62 μs | 1.638 μs | 2.349 μs | 0.6104 | 0.1221 | - | 152.53 KB | +| **Encrypt** | **10** | **SystemTextJson** | **41.49 μs** | **0.317 μs** | **0.464 μs** | **0.4272** | **0.0610** | **-** | **102.99 KB** | +| Decrypt | 10 | SystemTextJson | 41.53 μs | 0.505 μs | 0.725 μs | 0.3662 | 0.0610 | - | 94.09 KB | +| **Encrypt** | **100** | **Newtonsoft** | **1,081.23 μs** | **13.538 μs** | **18.978 μs** | **25.3906** | **23.4375** | **21.4844** | **1638.32 KB** | +| Decrypt | 100 | Newtonsoft | 1,135.00 μs | 32.719 μs | 45.867 μs | 17.5781 | 15.6250 | 15.6250 | 1229.52 KB | +| **Encrypt** | **100** | **SystemTextJson** | **819.27 μs** | **22.564 μs** | **33.074 μs** | **25.3906** | **25.3906** | **25.3906** | **942.76 KB** | +| Decrypt | 100 | SystemTextJson | 698.30 μs | 20.402 μs | 29.905 μs | 21.4844 | 21.4844 | 21.4844 | 927.92 KB | From 38cc928b7bde9298688a532f2ac231be2e751e06 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 17:03:27 +0200 Subject: [PATCH 49/71] - remove explicit System.Text.Json 8.0.5 --- .../src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index da7c2053e9..cca4d719bf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -38,11 +38,11 @@ - + - + @@ -57,10 +57,6 @@ - - - - From 2af9ddb0e3e4bbd7930efd199d4e6ac819feaced Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 19:37:29 +0200 Subject: [PATCH 50/71] ~ propagate changes from master --- .../src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs | 2 +- .../Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index d606e2fc28..110cf35eaa 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if IS_PREVIEW +#if ENCRYPTION_CUSTOM_PREVIEW namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 45a886d3ba..365fb7de9e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if IS_PREVIEW && NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { From 54b0ad9b1b07c085ff5cd5dd90b398359969e9b0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 10 Oct 2024 19:48:53 +0200 Subject: [PATCH 51/71] ~ fixes for preview x non-preview branching ~ cleanup ~ merge master followup fixes --- .../src/EncryptionProcessor.cs | 11 +++++++---- .../Transformation/MdeEncryptionProcessor.Stable.cs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index c2c6abd498..843eb5b3d7 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,8 +10,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; - using System.Text.Json; #if NET8_0_OR_GREATER + using System.Text.Json; using System.Text.Json.Nodes; #endif using System.Threading; @@ -31,7 +31,10 @@ internal static class EncryptionProcessor }; internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); + +#if NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; +#endif private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -135,14 +138,14 @@ public static async Task EncryptAsync( return jsonProcessor switch { JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") }; } -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( Stream input, Encryptor encryptor, @@ -195,7 +198,7 @@ public static async Task EncryptAsync( return (document, decryptionContext); } -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(JsonNode, DecryptionContext)> DecryptAsync( JsonNode document, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 1f9b628617..659ca74c91 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -119,7 +119,7 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], From ad15ceb6894977b3e6ab9f86b7725b1d304a7acd Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 11 Oct 2024 11:00:52 +0200 Subject: [PATCH 52/71] + tests ! no encryption info scenario behaves same on all paths ~ cleanup --- .../src/EncryptionProcessor.cs | 12 +- .../JsonNodeSqlSerializer.Preview.cs | 3 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 9 +- .../MdeEncryptionProcessorTests.cs | 255 +++++++++++++++++- .../JsonNodeSqlSerializerTests.cs | 79 ++++++ 5 files changed, 336 insertions(+), 22 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 843eb5b3d7..fd52424331 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,7 +10,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using System.Text.Json; using System.Text.Json.Nodes; #endif @@ -32,7 +32,7 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; #endif @@ -164,6 +164,12 @@ public static async Task EncryptAsync( JsonNode document = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken); (JsonNode decryptedDocument, DecryptionContext context) = await DecryptAsync(document, encryptor, diagnosticsContext, cancellationToken); + if (context == null) + { + input.Position = 0; + return (input, null); + } + await input.DisposeAsync(); MemoryStream ms = new (); @@ -211,7 +217,7 @@ public static async Task EncryptAsync( if (!document.AsObject().TryGetPropertyValue(Constants.EncryptedInfo, out JsonNode encryptionPropertiesNode)) { - throw new InvalidOperationException("Encryption properties deserialization failed."); + return (document, null); } EncryptionProperties encryptionProperties; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs index 0b98d254b8..c96b76a731 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs @@ -91,8 +91,7 @@ internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedB internal virtual JsonNode Deserialize( TypeMarker typeMarker, - ReadOnlySpan serializedBytes, - ArrayPoolManager arrayPoolManager) + ReadOnlySpan serializedBytes) { switch (typeMarker) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index e46f7c05e7..f90d9938f9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -10,7 +10,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Collections.Generic; using System.IO; using System.Linq; - using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; @@ -18,12 +17,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class MdeJsonNodeEncryptionProcessor { + private readonly JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; + internal JsonNodeSqlSerializer Serializer { get; set; } = new JsonNodeSqlSerializer(); internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - private JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; - public async Task EncryptAsync( Stream input, Encryptor encryptor, @@ -115,7 +114,6 @@ internal async Task DecryptObjectAsync( } using ArrayPoolManager arrayPoolManager = new (); - using ArrayPoolManager charPoolManager = new (); DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); @@ -145,8 +143,7 @@ internal async Task DecryptObjectAsync( document[propertyName] = this.Serializer.Deserialize( (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), - charPoolManager); + plainText.AsSpan(0, decryptedCount)); pathsDecrypted.Add(path); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 35722f40eb..9e20bd249c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -8,6 +8,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using System.Collections.Generic; using System.IO; using System.Linq; +#if NET8_0_OR_GREATER + using System.Text.Json.Nodes; +#endif using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom; @@ -55,14 +58,19 @@ public static void ClassInitialize(TestContext testContext) } [TestMethod] - public async Task InvalidPathToEncrypt() + [DataRow(JsonProcessor.Newtonsoft)] +#if NET8_0_OR_GREATER + [DataRow(JsonProcessor.SystemTextJson)] +#endif + public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) { TestDoc testDoc = TestDoc.Create(); EncryptionOptions encryptionOptionsWithInvalidPathToEncrypt = new() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = new List() { "/SensitiveStr", "/Invalid" } + PathsToEncrypt = new List() { "/SensitiveStr", "/Invalid" }, + JsonProcessor = jsonProcessor, }; Stream encryptedStream = await EncryptionProcessor.EncryptAsync( @@ -90,14 +98,19 @@ public async Task InvalidPathToEncrypt() } [TestMethod] - public async Task DuplicatePathToEncrypt() + [DataRow(JsonProcessor.Newtonsoft)] +#if NET8_0_OR_GREATER + [DataRow(JsonProcessor.SystemTextJson)] +#endif + public async Task DuplicatePathToEncrypt(JsonProcessor jsonProcessor) { TestDoc testDoc = TestDoc.Create(); EncryptionOptions encryptionOptionsWithDuplicatePathToEncrypt = new() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = new List() { "/SensitiveStr", "/SensitiveStr" } + PathsToEncrypt = new List() { "/SensitiveStr", "/SensitiveStr" }, + JsonProcessor = jsonProcessor, }; try @@ -119,12 +132,12 @@ await EncryptionProcessor.EncryptAsync( [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions) + public async Task EncryptDecryptPropertyWithNullValue_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); + JObject encryptedDoc = await VerifyEncryptionSucceededNewtonsoft(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -139,13 +152,37 @@ public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encrypti decryptionContext); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [TestMethod] + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task EncryptDecryptPropertyWithNullValue_VerifyBySystemText(EncryptionOptions encryptionOptions) + { + TestDoc testDoc = TestDoc.Create(); + testDoc.SensitiveStr = null; + + JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); + + (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedDoc, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } +#endif + [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions) + public async Task ValidateEncryptDecryptDocument_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); + JObject encryptedDoc = await VerifyEncryptionSucceededNewtonsoft(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -160,9 +197,60 @@ public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOpt decryptionContext); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [TestMethod] + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateEncryptDecryptDocument_VerifyBySystemText(EncryptionOptions encryptionOptions) + { + TestDoc testDoc = TestDoc.Create(); + + JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); + + (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedDoc, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } +#endif + [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) + public async Task ValidateDecryptByNewtonsoftStream_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) + { + TestDoc testDoc = TestDoc.Create(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + testDoc.ToStream(), + mockEncryptor.Object, + encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + (Stream decryptedStream, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedStream, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + JsonProcessor.Newtonsoft, + CancellationToken.None); + + JObject decryptedDoc = EncryptionProcessor.BaseSerializer.FromStream(decryptedStream); + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } + + [TestMethod] + [DynamicData(nameof(EncryptionOptionsStreamTestCombinations))] + public async Task ValidateDecryptBySystemTextStream_VerifyByNewtonsoft(EncryptionOptions encryptionOptions, JsonProcessor decryptionJsonProcessor) { TestDoc testDoc = TestDoc.Create(); @@ -177,6 +265,7 @@ public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) encryptedStream, mockEncryptor.Object, new CosmosDiagnosticsContext(), + decryptionJsonProcessor, CancellationToken.None); JObject decryptedDoc = EncryptionProcessor.BaseSerializer.FromStream(decryptedStream); @@ -187,8 +276,42 @@ public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) decryptionContext); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [TestMethod] + [DynamicData(nameof(EncryptionOptionsStreamTestCombinations))] + public async Task ValidateDecryptBySystemTextStream_VerifyBySystemText(EncryptionOptions encryptionOptions, JsonProcessor decryptionJsonProcessor) + { + TestDoc testDoc = TestDoc.Create(); + + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + testDoc.ToStream(), + mockEncryptor.Object, + encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + (Stream decryptedStream, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( + encryptedStream, + mockEncryptor.Object, + new CosmosDiagnosticsContext(), + decryptionJsonProcessor, + CancellationToken.None); + + JsonNode decryptedDoc = JsonNode.Parse(decryptedStream); + VerifyDecryptionSucceeded( + decryptedDoc, + testDoc, + TestDoc.PathsToEncrypt.Count, + decryptionContext); + } +#endif + [TestMethod] - public async Task DecryptStreamWithoutEncryptedProperty() + [DataRow(JsonProcessor.Newtonsoft)] +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + [DataRow(JsonProcessor.SystemTextJson)] +#endif + public async Task DecryptStreamWithoutEncryptedProperty(JsonProcessor processor) { TestDoc testDoc = TestDoc.Create(); Stream docStream = testDoc.ToStream(); @@ -197,6 +320,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() docStream, mockEncryptor.Object, new CosmosDiagnosticsContext(), + processor, CancellationToken.None); Assert.IsTrue(decryptedStream.CanSeek); @@ -205,7 +329,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() Assert.IsNull(decryptionContext); } - private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions) + private static async Task VerifyEncryptionSucceededNewtonsoft(TestDoc testDoc, EncryptionOptions encryptionOptions) { Stream encryptedStream = await EncryptionProcessor.EncryptAsync( testDoc.ToStream(), @@ -249,6 +373,51 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, En return encryptedDoc; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + private static async Task VerifyEncryptionSucceededSystemText(TestDoc testDoc, EncryptionOptions encryptionOptions) + { + Stream encryptedStream = await EncryptionProcessor.EncryptAsync( + testDoc.ToStream(), + mockEncryptor.Object, + encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + JsonNode encryptedDoc = JsonNode.Parse(encryptedStream, documentOptions: new System.Text.Json.JsonDocumentOptions() { }); + + Assert.AreEqual(testDoc.Id, encryptedDoc["id"].GetValue()); + Assert.AreEqual(testDoc.PK, encryptedDoc[nameof(TestDoc.PK)].GetValue()); + Assert.AreEqual(testDoc.NonSensitive, encryptedDoc[nameof(TestDoc.NonSensitive)].GetValue()); + Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); + Assert.AreNotEqual(testDoc.SensitiveInt, encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); // not equal since value is encrypted + + JsonNode eiJProp = encryptedDoc[Constants.EncryptedInfo]; + Assert.IsNotNull(eiJProp); + Assert.IsNotNull(eiJProp.AsObject()); + EncryptionProperties encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(eiJProp); + + Assert.IsNotNull(encryptionProperties); + Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); + Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion); + Assert.IsNull(encryptionProperties.EncryptedData); + Assert.IsNotNull(encryptionProperties.EncryptedPaths); + + if (testDoc.SensitiveStr == null) + { + AssertNullableValueKind(null, encryptedDoc, nameof(TestDoc.SensitiveStr)); // since null value is not encrypted + Assert.AreEqual(TestDoc.PathsToEncrypt.Count - 1, encryptionProperties.EncryptedPaths.Count()); + } + else + { + Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); + Assert.AreNotEqual(testDoc.SensitiveStr, encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); // not equal since value is encrypted + Assert.AreEqual(TestDoc.PathsToEncrypt.Count, encryptionProperties.EncryptedPaths.Count()); + } + + return encryptedDoc; + } +#endif + private static void VerifyDecryptionSucceeded( JObject decryptedDoc, TestDoc expectedDoc, @@ -284,6 +453,56 @@ private static void VerifyDecryptionSucceeded( } } +#if NET8_0_OR_GREATER + private static void VerifyDecryptionSucceeded( + JsonNode decryptedDoc, + TestDoc expectedDoc, + int pathCount, + DecryptionContext decryptionContext, + bool invalidPathsConfigured = false) + { + AssertNullableValueKind(expectedDoc.SensitiveStr, decryptedDoc, nameof(TestDoc.SensitiveStr)); + Assert.AreEqual(expectedDoc.SensitiveInt, decryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); + Assert.IsNull(decryptedDoc[Constants.EncryptedInfo]); + + Assert.IsNotNull(decryptionContext); + Assert.IsNotNull(decryptionContext.DecryptionInfoList); + DecryptionInfo decryptionInfo = decryptionContext.DecryptionInfoList[0]; + Assert.AreEqual(dekId, decryptionInfo.DataEncryptionKeyId); + if (expectedDoc.SensitiveStr == null) + { + Assert.AreEqual(pathCount - 1, decryptionInfo.PathsDecrypted.Count); + Assert.IsTrue(TestDoc.PathsToEncrypt.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + else + { + Assert.AreEqual(pathCount, decryptionInfo.PathsDecrypted.Count); + + if (!invalidPathsConfigured) + { + Assert.IsFalse(TestDoc.PathsToEncrypt.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + else + { + Assert.IsTrue(TestDoc.PathsToEncrypt.Exists(path => !decryptionInfo.PathsDecrypted.Contains(path))); + } + } + } + + private static void AssertNullableValueKind(T expectedValue, JsonNode node, string propertyName) where T : class + { + if (expectedValue == null) + { + Assert.IsTrue(node.AsObject().ContainsKey(propertyName)); + Assert.AreEqual(null, node[propertyName]); + } + else + { + Assert.AreEqual(expectedValue, node[propertyName].GetValue()); + } + } +#endif + public static IEnumerable EncryptionOptionsCombinations => new[] { new object[] { new EncryptionOptions() { @@ -304,5 +523,19 @@ private static void VerifyDecryptionSucceeded( }, #endif }; + + public static IEnumerable EncryptionOptionsStreamTestCombinations + { + get + { + foreach (object[] encryptionOptions in EncryptionOptionsCombinations) + { + yield return new object[] { encryptionOptions[0], JsonProcessor.Newtonsoft }; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + yield return new object[] { encryptionOptions[0], JsonProcessor.SystemTextJson }; +#endif + } + } + } } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs index deb6bb7b35..e28bd409a1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs @@ -5,6 +5,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation using System; using System.Collections.Generic; using System.Linq; + using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Azure.Cosmos.Encryption.Custom; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; @@ -43,6 +44,79 @@ public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[ } } + [TestMethod] + [DynamicData(nameof(DeserializationSamples))] + public void Deserialize_SupportedValue(byte typeMarkerByte, byte[] serializedBytes, JsonNode expectedNode) + { + JsonNodeSqlSerializer serializer = new(); + TypeMarker typeMarker = (TypeMarker)typeMarkerByte; + JsonNode deserializedNode = serializer.Deserialize(typeMarker, serializedBytes); + + if ((expectedNode as JsonValue) != null) + { + AssertValueNodeEquality(expectedNode, deserializedNode); + return; + } + + if ((expectedNode as JsonArray) != null) + { + Assert.IsNotNull(deserializedNode as JsonArray); + + JsonArray expectedArray = expectedNode.AsArray(); + JsonArray deserializedArray = deserializedNode.AsArray(); + + Assert.AreEqual(expectedArray.Count, deserializedArray.Count); + + for (int i = 0; i < deserializedNode.AsArray().Count; i++) + { + AssertValueNodeEquality(expectedArray[i], deserializedArray[i]); + } + return; + } + + if ((expectedNode as JsonObject) != null) + { + Assert.IsNotNull(deserializedNode as JsonObject); + + JsonObject expectedObject = expectedNode.AsObject(); + JsonObject deserializedObject = deserializedNode.AsObject(); + + Assert.AreEqual(expectedObject.Count, deserializedObject.Count); + + foreach (KeyValuePair expected in expectedObject) + { + Assert.IsTrue(deserializedObject.ContainsKey(expected.Key)); + AssertValueNodeEquality(expected.Value, deserializedObject[expected.Key]); + } + return; + } + + Assert.Fail("Attempt to validate unsupported JsonNode type"); + } + + private static void AssertValueNodeEquality(JsonNode expectedNode, JsonNode actualNode) + { + JsonValue expectedValueNode = expectedNode.AsValue(); + JsonValue actualValueNode = actualNode.AsValue(); + + Assert.AreEqual(expectedValueNode.GetValueKind(), actualValueNode.GetValueKind()); + Assert.AreEqual(expectedValueNode.ToString(), actualValueNode.ToString()); + } + + public static IEnumerable DeserializationSamples + { + get + { + yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), JsonValue.Create(true) }; + yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), JsonValue.Create(false) }; + yield return new object[] { (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), JsonValue.Create(192) }; + yield return new object[] { (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), JsonValue.Create(192.5) }; + yield return new object[] { (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), JsonValue.Create(testString) }; + yield return new object[] { (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), JsonNode.Parse("[10,18,19]") }; + yield return new object[] { (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), JsonNode.Parse(testClass.ToJson()) }; + } + } + public static IEnumerable SerializationSamples { get @@ -71,6 +145,11 @@ private class TestClass { public int SomeInt { get; set; } public string SomeString { get; set; } + + public string ToJson() + { + return JsonSerializer.Serialize(this); + } } private static byte[] GetNewtonsoftValueEquivalent(T value) From d9c315d8f76ccb528a9ced24857d7adf3ef31da6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Sun, 13 Oct 2024 17:27:35 +0200 Subject: [PATCH 53/71] ~ complete merge from master & update benchmark --- .../MdeJObjectEncryptionProcessor.Preview.cs | 82 +++++++++++++++---- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 33 ++++++-- .../Readme.md | 34 ++++---- .../MdeEncryptionProcessorTests.cs | 40 ++++++--- 4 files changed, 144 insertions(+), 45 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 110cf35eaa..4d70ec8a2c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -52,6 +52,14 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + +#if NET8_0_OR_GREATER + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; +#endif + Dictionary compressedPaths = new (); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { #if NET8_0_OR_GREATER @@ -69,26 +77,41 @@ public async Task EncryptAsync( continue; } - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + byte[] processedBytes = null; + (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - if (plainText == null) + if (processedBytes == null) { continue; } - byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); +#if NET8_0_OR_GREATER + if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); + processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytes = compressedBytes; + } +#endif + + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength); input[propertyName] = encryptedBytes; + pathsEncrypted.Add(pathToEncrypt); } +#if NET8_0_OR_GREATER + compressor?.Dispose(); +#endif EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, + encryptionFormatVersion: compressionEnabled ? 4 : 3, encryptionOptions.EncryptionAlgorithm, encryptionOptions.DataEncryptionKeyId, encryptedData: null, - pathsEncrypted); + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); input.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); @@ -96,15 +119,15 @@ public async Task EncryptAsync( } internal async Task DecryptObjectAsync( - JObject document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) + JObject document, + Encryptor encryptor, + EncryptionProperties encryptionProperties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3) + if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -115,6 +138,24 @@ internal async Task DecryptObjectAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + +#if NET8_0_OR_GREATER + BrotliCompressor decompressor = null; + if (encryptionProperties.EncryptionFormatVersion == 4) + { + bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; + if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) + { + throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}"); + } + + if (containsCompressed) + { + decompressor = new (); + } + } +#endif + foreach (string path in encryptionProperties.EncryptedPaths) { #if NET8_0_OR_GREATER @@ -135,11 +176,24 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + +#if NET8_0_OR_GREATER + if (decompressor != null) + { + if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) + { + byte[] buffer = arrayPoolManager.Rent(decompressedSize); + processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); + + bytes = buffer; + } + } +#endif this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), + bytes.AsSpan(0, processedBytes), document, propertyName, charPoolManager); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index 365fb7de9e..63b2428832 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -59,6 +59,14 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + +#if NET8_0_OR_GREATER + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; +#endif + Dictionary compressedPaths = new (); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { #if NET8_0_OR_GREATER @@ -76,26 +84,39 @@ public async Task EncryptAsync( continue; } - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + byte[] processedBytes = null; + (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - if (plainText == null) + if (processedBytes == null) { continue; } - (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength, arrayPoolManager); +#if NET8_0_OR_GREATER + if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); + processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytes = compressedBytes; + } +#endif + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); itemObj[propertyName] = JsonValue.Create(new JsonBytes(encryptedBytes, 0, encryptedBytesCount)); pathsEncrypted.Add(pathToEncrypt); } +#if NET8_0_OR_GREATER + compressor?.Dispose(); +#endif EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, + encryptionFormatVersion: compressionEnabled ? 4 : 3, encryptionOptions.EncryptionAlgorithm, encryptionOptions.DataEncryptionKeyId, encryptedData: null, - pathsEncrypted); + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); JsonNode propertiesNode = JsonSerializer.SerializeToNode(encryptionProperties); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 2913528571..b7110fd6eb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,17 +9,23 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **None** | **22.45 μs** | **0.447 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **40.94 KB** | -| Decrypt | 1 | None | 26.65 μs | 0.165 μs | 0.247 μs | 0.1526 | 0.0305 | - | 40.32 KB | -| **Encrypt** | **1** | **Brotli** | **27.92 μs** | **0.212 μs** | **0.317 μs** | **0.1526** | **0.0305** | **-** | **37.3 KB** | -| Decrypt | 1 | Brotli | 33.45 μs | 0.718 μs | 1.075 μs | 0.1221 | - | - | 39.95 KB | -| **Encrypt** | **10** | **None** | **83.56 μs** | **0.358 μs** | **0.502 μs** | **0.6104** | **0.1221** | **-** | **167.12 KB** | -| Decrypt | 10 | None | 100.88 μs | 0.437 μs | 0.627 μs | 0.6104 | 0.1221 | - | 153.59 KB | -| **Encrypt** | **10** | **Brotli** | **111.94 μs** | **0.456 μs** | **0.669 μs** | **0.6104** | **0.1221** | **-** | **164.26 KB** | -| Decrypt | 10 | Brotli | 121.06 μs | 2.794 μs | 4.182 μs | 0.4883 | - | - | 141.31 KB | -| **Encrypt** | **100** | **None** | **1,194.10 μs** | **37.744 μs** | **56.494 μs** | **23.4375** | **23.4375** | **21.4844** | **1638.78 KB** | -| Decrypt | 100 | None | 1,247.89 μs | 32.037 μs | 47.952 μs | 17.5781 | 15.6250 | 15.6250 | 1230.56 KB | -| **Encrypt** | **100** | **Brotli** | **1,199.53 μs** | **30.018 μs** | **44.930 μs** | **13.6719** | **11.7188** | **9.7656** | **1347 KB** | -| Decrypt | 100 | Brotli | 1,196.64 μs | 22.702 μs | 33.979 μs | 11.7188 | 9.7656 | 9.7656 | 1097.75 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.40 μs** | **0.387 μs** | **0.580 μs** | **0.1526** | **0.0305** | **-** | **41.08 KB** | +| Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.215 μs | 0.322 μs | 0.1526 | 0.0305 | - | 40.39 KB | +| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.84 μs** | **0.245 μs** | **0.358 μs** | **0.0916** | **0.0305** | **-** | **22.75 KB** | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **27.89 μs** | **0.220 μs** | **0.329 μs** | **0.1526** | **0.0305** | **-** | **37.45 KB** | +| Decrypt | 1 | Brotli | Newtonsoft | 33.39 μs | 0.612 μs | 0.878 μs | 0.1221 | - | - | 40.02 KB | +| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **21.43 μs** | **0.168 μs** | **0.251 μs** | **0.0916** | **0.0305** | **-** | **21.82 KB** | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **82.79 μs** | **0.439 μs** | **0.643 μs** | **0.6104** | **0.1221** | **-** | **167.26 KB** | +| Decrypt | 10 | None | Newtonsoft | 98.98 μs | 0.499 μs | 0.747 μs | 0.6104 | 0.1221 | - | 153.66 KB | +| **Encrypt** | **10** | **None** | **SystemTextJson** | **40.74 μs** | **0.214 μs** | **0.321 μs** | **0.4272** | **0.0610** | **-** | **103.26 KB** | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **112.08 μs** | **1.172 μs** | **1.681 μs** | **0.6104** | **0.1221** | **-** | **164.4 KB** | +| Decrypt | 10 | Brotli | Newtonsoft | 117.21 μs | 0.920 μs | 1.349 μs | 0.4883 | - | - | 141.38 KB | +| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **69.51 μs** | **0.491 μs** | **0.719 μs** | **0.2441** | **-** | **-** | **84.58 KB** | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,165.87 μs** | **41.512 μs** | **62.133 μs** | **23.4375** | **21.4844** | **19.5313** | **1638.94 KB** | +| Decrypt | 100 | None | Newtonsoft | 1,166.04 μs | 32.206 μs | 48.204 μs | 17.5781 | 15.6250 | 15.6250 | 1230.62 KB | +| **Encrypt** | **100** | **None** | **SystemTextJson** | **854.64 μs** | **35.123 μs** | **51.482 μs** | **21.4844** | **21.4844** | **21.4844** | **942.96 KB** | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,121.21 μs** | **24.814 μs** | **37.141 μs** | **13.6719** | **11.7188** | **9.7656** | **1347.12 KB** | +| Decrypt | 100 | Brotli | Newtonsoft | 1,135.33 μs | 11.013 μs | 16.483 μs | 11.7188 | 9.7656 | 9.7656 | 1097.84 KB | +| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **986.73 μs** | **19.142 μs** | **28.058 μs** | **21.4844** | **21.4844** | **21.4844** | **749.06 KB** | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 59a5a215f1..4a28e93aac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -295,6 +295,7 @@ private static void VerifyDecryptionSucceeded( DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft, CompressionOptions = new CompressionOptions() { Algorithm = CompressionOptions.CompressionAlgorithm.None @@ -307,6 +308,19 @@ private static void VerifyDecryptionSucceeded( DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.SystemTextJson, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.None + } + } + }, + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft, CompressionOptions = new CompressionOptions() { Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, @@ -319,36 +333,40 @@ private static void VerifyDecryptionSucceeded( DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = JsonProcessor.Newtonsoft, CompressionOptions = new CompressionOptions() { Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, } } - } -#endif - }; - } - } - - public static IEnumerable EncryptionOptionsCombinations => new[] { + }, new object[] { new EncryptionOptions() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft + JsonProcessor = JsonProcessor.SystemTextJson, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.Fastest + } } }, -#if NET8_0_OR_GREATER new object[] { new EncryptionOptions() { DataEncryptionKeyId = dekId, EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson + JsonProcessor = JsonProcessor.SystemTextJson, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, + } } - }, + } #endif }; } From 996252bc9e9ca7a1a431e496e498cefc59034b2d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 11:20:10 +0200 Subject: [PATCH 54/71] + initial --- .../src/EncryptionOptions.cs | 6 + .../src/EncryptionProcessor.cs | 38 +++ .../src/EncryptionPropertiesWrapper.cs | 19 ++ .../MdeEncryptionProcessor.Preview.cs | 1 + .../src/Transformation/StreamProcessor.cs | 236 ++++++++++++++++++ .../EncryptionBenchmark.cs | 5 +- 6 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index 0559e988c3..aadb2dbbd3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -22,6 +22,12 @@ public enum JsonProcessor /// /// Available with .NET8.0 package only. SystemTextJson, + + /// + /// Ut8JsonReader/Writer + /// + /// Available with .NET8.0 package only. + Stream, #endif } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 6f9d87f064..8922432a34 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -34,6 +34,7 @@ internal static class EncryptionProcessor #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; + private static readonly StreamProcessor StreamProcessor = new (); #endif private static readonly MdeEncryptionProcessor MdeEncryptionProcessor = new (); @@ -140,6 +141,7 @@ public static async Task EncryptAsync( JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), + JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") }; @@ -182,6 +184,42 @@ public static async Task EncryptAsync( } #endif +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( + Stream input, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (input == null) + { + return (input, null); + } + + Debug.Assert(input.CanSeek); + Debug.Assert(encryptor != null); + Debug.Assert(diagnosticsContext != null); + input.Position = 0; + + EncryptionPropertiesWrapper properties = System.Text.Json.JsonSerializer.Deserialize(input); + input.Position = 0; + if (properties?.EncryptionProperties == null) + { + return (input, null); + } + + (Stream decryptedDocument, DecryptionContext context) = await StreamProcessor.DecryptStreamAsync(input, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + if (context == null) + { + input.Position = 0; + return (input, null); + } + + await input.DisposeAsync(); + return (decryptedDocument, context); + } +#endif + public static async Task<(JObject, DecryptionContext)> DecryptAsync( JObject document, Encryptor encryptor, diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs new file mode 100644 index 0000000000..4c75c16032 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.Text.Json.Serialization; + + internal class EncryptionPropertiesWrapper + { + [JsonPropertyName(Constants.EncryptedInfo)] + public EncryptionProperties EncryptionProperties { get; } + + public EncryptionPropertiesWrapper(EncryptionProperties properties) + { + this.EncryptionProperties = properties; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 635901552d..8649b36079 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -34,6 +34,7 @@ public async Task EncryptAsync( JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #if NET8_0_OR_GREATER JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), + JsonProcessor.Stream => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), #endif _ => throw new InvalidOperationException("Unsupported JsonProcessor") }; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs new file mode 100644 index 0000000000..c2ae7040c2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -0,0 +1,236 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class StreamProcessor + { + private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); + + internal async Task<(Stream, DecryptionContext)> DecryptStreamAsync( + Stream inputStream, + Encryptor encryptor, + EncryptionProperties properties, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + if (properties.EncryptionFormatVersion != 3 && properties.EncryptionFormatVersion != 4) + { + throw new NotSupportedException($"Unknown encryption format version: {properties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); + } + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(properties.DataEncryptionKeyId, properties.EncryptionAlgorithm, cancellationToken); + + List pathsDecrypted = new (properties.EncryptedPaths.Count()); + + MemoryStream outputStream = new (); + Utf8JsonWriter writer = new (outputStream); + + // we determine initial buffer size based on max uncompressed path length, it still might need scale out in case there is large non-encrypted object, but likehood is rather low + int bufferSize = 16384; // Math.Max(16384, properties.CompressedEncryptedPaths.Values.Max()); + byte[] buffer = arrayPoolManager.Rent(bufferSize); + + JsonReaderState state = new (this.jsonReaderOptions); + + int leftOver = 0; + + bool isFinalBlock = false; + + while (!isFinalBlock) + { + int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + int dataSize = dataLength + leftOver; + isFinalBlock = dataSize == 0; + long bytesConsumed = 0; + + // processing itself here + bytesConsumed = this.TransformReadBuffer( + buffer.AsSpan(0, dataSize), + isFinalBlock, + writer, + ref state, + pathsDecrypted, + properties, + arrayPoolManager, + encryptionKey); + + leftOver = dataSize - (int)bytesConsumed; + + // we need to scale out buffer + if (leftOver == dataSize) + { + bufferSize *= 2; + byte[] newBuffer = arrayPoolManager.Rent(bufferSize); + buffer.AsSpan(0, leftOver).CopyTo(newBuffer); + buffer = newBuffer; + } + else if (leftOver != 0) + { + buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + } + } + + writer.Flush(); + inputStream.Position = 0; + outputStream.Position = 0; + + return ( + outputStream, + EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId)); + } + + /* + private static Dictionary GetUtf8DecryptionList(EncryptionProperties properties) + { + Dictionary output = new (properties.EncryptedPaths.Count()); + foreach (KeyValuePair compressedPath in properties.CompressedEncryptedPaths) + { + byte[] utf8String = Encoding.UTF8.GetBytes(compressedPath.Key, 1, compressedPath.Key.Length - 1); + output[utf8String] = compressedPath.Value; + } + + foreach (string encryptedPath in properties.EncryptedPaths) + { + byte[] utf8String = Encoding.UTF8.GetBytes(encryptedPath, 1, encryptedPath.Length - 1); + output.TryAdd(utf8String, -1); + } + + return output; + }*/ + + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + { + Utf8JsonReader json = new (buffer, isFinalBlock, state); + + string decryptPropertyName = null; + + while (json.Read()) + { + JsonTokenType tokenType = json.TokenType; + + switch (tokenType) + { + case JsonTokenType.String: + if (decryptPropertyName == null) + { + writer.WriteRawValue(json.ValueSpan); + } + else + { + this.TransformDecryptProperty( + json.GetBytesFromBase64(), + writer, + decryptPropertyName, + properties, + encryptionKey, + arrayPoolManager); + + pathsDecrypted.Add("/" + decryptPropertyName); + } + + decryptPropertyName = null; + break; + case JsonTokenType.Number: + decryptPropertyName = null; + writer.WriteRawValue(json.ValueSpan); + break; + case JsonTokenType.None: + decryptPropertyName = null; + break; + case JsonTokenType.StartObject: + decryptPropertyName = null; + writer.WriteStartObject(); + break; + case JsonTokenType.EndObject: + decryptPropertyName = null; + writer.WriteEndObject(); + break; + case JsonTokenType.StartArray: + decryptPropertyName = null; + writer.WriteStartArray(); + break; + case JsonTokenType.EndArray: + decryptPropertyName = null; + writer.WriteEndArray(); + break; + case JsonTokenType.PropertyName: + string propertyName = json.GetString(); + if (properties.EncryptedPaths.Contains("/" + propertyName)) + { + decryptPropertyName = propertyName; + } + + writer.WritePropertyName(json.ValueSpan); + break; + case JsonTokenType.Comment: + break; + case JsonTokenType.True: + decryptPropertyName = null; + writer.WriteBooleanValue(true); + break; + case JsonTokenType.False: + decryptPropertyName = null; + writer.WriteBooleanValue(false); + break; + case JsonTokenType.Null: + decryptPropertyName = null; + writer.WriteNullValue(); + break; + } + } + + state = json.CurrentState; + return json.BytesConsumed; + } + + private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) + { + BrotliCompressor decompressor = null; + if (properties.EncryptionFormatVersion == 4) + { + bool containsCompressed = properties.CompressedEncryptedPaths?.Any() == true; + if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) + { + throw new NotSupportedException($"Unknown compression algorithm {properties.CompressionAlgorithm}"); + } + + if (containsCompressed) + { + decompressor = new (); + } + } + + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); + + if (decompressor != null) + { + if (properties.CompressedEncryptedPaths?.TryGetValue(decryptPropertyName, out int decompressedSize) == true) + { + byte[] buffer = arrayPoolManager.Rent(decompressedSize); + processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); + + bytes = buffer; + } + } + + writer.WriteRawValue(bytes.AsSpan(0, processedBytes)); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 4bdcdc2941..051b87d69d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -28,7 +28,7 @@ public partial class EncryptionBenchmark [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } - [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson)] + [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson,*/ JsonProcessor.Stream)] public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -59,6 +59,7 @@ public async Task Setup() this.encryptedData = memoryStream.ToArray(); } + /* [Benchmark] public async Task Encrypt() { @@ -68,7 +69,7 @@ await EncryptionProcessor.EncryptAsync( this.encryptionOptions, new CosmosDiagnosticsContext(), CancellationToken.None); - } + }*/ [Benchmark] public async Task Decrypt() From 758e98d900bc3e005533c5e89f1e427355ce1c44 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 16:21:40 +0200 Subject: [PATCH 55/71] + tests, fixes and benchmark --- .../src/EncryptionProcessor.cs | 48 +++++++++- .../src/EncryptionPropertiesWrapper.cs | 4 +- .../src/Transformation/StreamProcessor.cs | 96 +++++++++++++++---- .../EncryptionBenchmark.cs | 17 +++- .../Readme.md | 40 +++----- .../MdeEncryptionProcessorTests.cs | 4 + 6 files changed, 157 insertions(+), 52 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 8922432a34..59a43f2206 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -147,6 +147,45 @@ public static async Task EncryptAsync( }; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public static async Task DecryptAsync( + Stream input, + Stream output, + Encryptor encryptor, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + if (input == null) + { + return null; + } + + Debug.Assert(input.CanSeek); + Debug.Assert(output.CanWrite); + Debug.Assert(output.CanSeek); + Debug.Assert(encryptor != null); + Debug.Assert(diagnosticsContext != null); + input.Position = 0; + + EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); + input.Position = 0; + if (properties?.EncryptionProperties == null) + { + return null; + } + + DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + if (context == null) + { + input.Position = 0; + return null; + } + + await input.DisposeAsync(); + return context; + } +#endif + #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( Stream input, @@ -201,14 +240,16 @@ public static async Task EncryptAsync( Debug.Assert(diagnosticsContext != null); input.Position = 0; - EncryptionPropertiesWrapper properties = System.Text.Json.JsonSerializer.Deserialize(input); + EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); input.Position = 0; if (properties?.EncryptionProperties == null) { return (input, null); } - (Stream decryptedDocument, DecryptionContext context) = await StreamProcessor.DecryptStreamAsync(input, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + MemoryStream ms = new MemoryStream(); + + DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, ms, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) { input.Position = 0; @@ -216,8 +257,9 @@ public static async Task EncryptAsync( } await input.DisposeAsync(); - return (decryptedDocument, context); + return (ms, context); } + #endif public static async Task<(JObject, DecryptionContext)> DecryptAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs index 4c75c16032..8c343a12de 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionPropertiesWrapper.cs @@ -11,9 +11,9 @@ internal class EncryptionPropertiesWrapper [JsonPropertyName(Constants.EncryptedInfo)] public EncryptionProperties EncryptionProperties { get; } - public EncryptionPropertiesWrapper(EncryptionProperties properties) + public EncryptionPropertiesWrapper(EncryptionProperties encryptionProperties) { - this.EncryptionProperties = properties; + this.EncryptionProperties = encryptionProperties; } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs index c2ae7040c2..b24b8d9d89 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -6,21 +6,30 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; + using System.Buffers; + using System.Buffers.Text; using System.Collections.Generic; using System.IO; using System.Linq; + using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; + using Microsoft.Data.Encryption.Cryptography.Serializers; internal class StreamProcessor { + private static readonly SqlBitSerializer SqlBoolSerializer = new (); + private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); + private static readonly SqlBigIntSerializer SqlLongSerializer = new (); + private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - internal async Task<(Stream, DecryptionContext)> DecryptStreamAsync( + internal async Task DecryptStreamAsync( Stream inputStream, + Stream outputStream, Encryptor encryptor, EncryptionProperties properties, CosmosDiagnosticsContext diagnosticsContext, @@ -39,7 +48,6 @@ internal class StreamProcessor List pathsDecrypted = new (properties.EncryptedPaths.Count()); - MemoryStream outputStream = new (); Utf8JsonWriter writer = new (outputStream); // we determine initial buffer size based on max uncompressed path length, it still might need scale out in case there is large non-encrypted object, but likehood is rather low @@ -51,6 +59,7 @@ internal class StreamProcessor int leftOver = 0; bool isFinalBlock = false; + bool isIgnoredBlock = false; while (!isFinalBlock) { @@ -65,6 +74,7 @@ internal class StreamProcessor isFinalBlock, writer, ref state, + ref isIgnoredBlock, pathsDecrypted, properties, arrayPoolManager, @@ -90,9 +100,7 @@ internal class StreamProcessor inputStream.Position = 0; outputStream.Position = 0; - return ( - outputStream, - EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId)); + return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } /* @@ -114,27 +122,37 @@ private static Dictionary GetUtf8DecryptionList(EncryptionPropertie return output; }*/ - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { - Utf8JsonReader json = new (buffer, isFinalBlock, state); + Utf8JsonReader reader = new (buffer, isFinalBlock, state); string decryptPropertyName = null; - while (json.Read()) + while (reader.Read()) { - JsonTokenType tokenType = json.TokenType; + JsonTokenType tokenType = reader.TokenType; + + if (isIgnoredBlock && reader.CurrentDepth == 1 && tokenType == JsonTokenType.EndObject) + { + isIgnoredBlock = false; + continue; + } + else if (isIgnoredBlock) + { + continue; + } switch (tokenType) { case JsonTokenType.String: if (decryptPropertyName == null) { - writer.WriteRawValue(json.ValueSpan); + writer.WriteStringValue(reader.ValueSpan); } else { this.TransformDecryptProperty( - json.GetBytesFromBase64(), + ref reader, writer, decryptPropertyName, properties, @@ -148,7 +166,7 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW break; case JsonTokenType.Number: decryptPropertyName = null; - writer.WriteRawValue(json.ValueSpan); + writer.WriteRawValue(reader.ValueSpan); break; case JsonTokenType.None: decryptPropertyName = null; @@ -170,13 +188,19 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW writer.WriteEndArray(); break; case JsonTokenType.PropertyName: - string propertyName = json.GetString(); + string propertyName = reader.GetString(); if (properties.EncryptedPaths.Contains("/" + propertyName)) { decryptPropertyName = propertyName; } - writer.WritePropertyName(json.ValueSpan); + if (propertyName == Constants.EncryptedInfo) + { + isIgnoredBlock = true; + break; + } + + writer.WritePropertyName(reader.ValueSpan); break; case JsonTokenType.Comment: break; @@ -195,11 +219,11 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW } } - state = json.CurrentState; - return json.BytesConsumed; + state = reader.CurrentState; + return reader.BytesConsumed; } - private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) + private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) { BrotliCompressor decompressor = null; if (properties.EncryptionFormatVersion == 4) @@ -216,11 +240,22 @@ private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonW } } - (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextWithTypeMarker.Length, arrayPoolManager); + byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(reader.ValueSpan.Length); + + // necessary for proper un-escaping + int initialLength = reader.CopyString(cipherTextWithTypeMarker); + + OperationStatus status = Base64.DecodeFromUtf8InPlace(cipherTextWithTypeMarker.AsSpan(0, initialLength), out int cipherTextLength); + if (status != OperationStatus.Done) + { + throw new InvalidOperationException($"Base64 decoding failed: {status}"); + } + + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); if (decompressor != null) { - if (properties.CompressedEncryptedPaths?.TryGetValue(decryptPropertyName, out int decompressedSize) == true) + if (properties.CompressedEncryptedPaths?.TryGetValue("/" + decryptPropertyName, out int decompressedSize) == true) { byte[] buffer = arrayPoolManager.Rent(decompressedSize); processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); @@ -229,7 +264,28 @@ private void TransformDecryptProperty(byte[] cipherTextWithTypeMarker, Utf8JsonW } } - writer.WriteRawValue(bytes.AsSpan(0, processedBytes)); + Span bytesToWrite = bytes.AsSpan(0, processedBytes); + switch ((TypeMarker)cipherTextWithTypeMarker[0]) + { + case TypeMarker.String: + writer.WriteStringValue(bytesToWrite); + break; + case TypeMarker.Long: + writer.WriteNumberValue(SqlLongSerializer.Deserialize(bytesToWrite)); + break; + case TypeMarker.Double: + writer.WriteNumberValue(SqlDoubleSerializer.Deserialize(bytesToWrite)); + break; + case TypeMarker.Boolean: + writer.WriteBooleanValue(SqlBoolSerializer.Deserialize(bytesToWrite)); + break; + case TypeMarker.Null: + writer.WriteNullValue(); + break; + default: + writer.WriteRawValue(bytes.AsSpan(0, processedBytes), true); + break; + } } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 051b87d69d..395eed045d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -22,13 +22,15 @@ public partial class EncryptionBenchmark private byte[]? encryptedData; private byte[]? plaintext; + private static readonly MemoryStream recycledStream = new (); + [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } - [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson,*/ JsonProcessor.Stream)] + [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, */JsonProcessor.Stream)] public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -82,6 +84,19 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } + [Benchmark] + public async Task DecryptToProvidedStream() + { + await EncryptionProcessor.DecryptAsync( + new MemoryStream(this.encryptedData!), + EncryptionBenchmark.recycledStream, + this.encryptor, + new CosmosDiagnosticsContext(), + CancellationToken.None); + + EncryptionBenchmark.recycledStream.Position = 0; + } + private EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 523b62413f..d872d7f1a1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,29 +9,17 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.84 μs** | **0.676 μs** | **1.013 μs** | **0.1526** | **0.0305** | **-** | **41.08 KB** | -| Decrypt | 1 | None | Newtonsoft | 26.85 μs | 0.273 μs | 0.409 μs | 0.1526 | 0.0305 | - | 40.47 KB | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **15.41 μs** | **0.224 μs** | **0.335 μs** | **0.0916** | **0.0305** | **-** | **22.64 KB** | -| Decrypt | 1 | None | SystemTextJson | 14.41 μs | 0.121 μs | 0.174 μs | 0.0763 | 0.0153 | - | 20.95 KB | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **28.44 μs** | **0.246 μs** | **0.369 μs** | **0.1526** | **0.0305** | **-** | **37.45 KB** | -| Decrypt | 1 | Brotli | Newtonsoft | 34.21 μs | 0.795 μs | 1.189 μs | 0.1221 | - | - | 40.1 KB | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **21.68 μs** | **0.264 μs** | **0.378 μs** | **0.0610** | **-** | **-** | **21.71 KB** | -| Decrypt | 1 | Brotli | SystemTextJson | 20.41 μs | 0.167 μs | 0.249 μs | 0.0610 | 0.0305 | - | 20.01 KB | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **82.84 μs** | **0.495 μs** | **0.725 μs** | **0.6104** | **0.1221** | **-** | **167.26 KB** | -| Decrypt | 10 | None | Newtonsoft | 100.04 μs | 0.733 μs | 1.096 μs | 0.6104 | 0.1221 | - | 153.74 KB | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.34 μs** | **0.245 μs** | **0.351 μs** | **0.4272** | **0.0610** | **-** | **103.15 KB** | -| Decrypt | 10 | None | SystemTextJson | 41.09 μs | 0.264 μs | 0.395 μs | 0.3662 | 0.0610 | - | 94.2 KB | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **112.09 μs** | **0.821 μs** | **1.203 μs** | **0.6104** | **0.1221** | **-** | **164.4 KB** | -| Decrypt | 10 | Brotli | Newtonsoft | 119.50 μs | 1.371 μs | 1.966 μs | 0.4883 | - | - | 141.45 KB | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **70.75 μs** | **0.423 μs** | **0.620 μs** | **0.2441** | **-** | **-** | **84.47 KB** | -| Decrypt | 10 | Brotli | SystemTextJson | 64.51 μs | 1.042 μs | 1.560 μs | 0.2441 | - | - | 80.27 KB | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,142.95 μs** | **36.247 μs** | **54.253 μs** | **23.4375** | **21.4844** | **19.5313** | **1638.94 KB** | -| Decrypt | 100 | None | Newtonsoft | 1,160.91 μs | 26.561 μs | 39.755 μs | 17.5781 | 15.6250 | 15.6250 | 1230.71 KB | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **835.31 μs** | **25.982 μs** | **38.084 μs** | **26.3672** | **26.3672** | **26.3672** | **942.9 KB** | -| Decrypt | 100 | None | SystemTextJson | 731.05 μs | 23.379 μs | 33.530 μs | 18.5547 | 18.5547 | 18.5547 | 928 KB | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,138.53 μs** | **21.347 μs** | **31.952 μs** | **13.6719** | **11.7188** | **9.7656** | **1347.1 KB** | -| Decrypt | 100 | Brotli | Newtonsoft | 1,150.43 μs | 15.475 μs | 22.684 μs | 11.7188 | 9.7656 | 9.7656 | 1097.91 KB | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **994.72 μs** | **26.940 μs** | **39.489 μs** | **19.5313** | **19.5313** | **19.5313** | **748.94 KB** | -| Decrypt | 100 | Brotli | SystemTextJson | 886.36 μs | 14.437 μs | 21.162 μs | 17.5781 | 17.5781 | 17.5781 | 782.67 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |----------:|---------:|---------:|-------:|-------:|-------:|----------:| +| **Decrypt** | **1** | **None** | **Stream** | **12.63 μs** | **0.146 μs** | **0.213 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | +| DecryptToProvidedStream | 1 | None | Stream | 12.58 μs | 0.082 μs | 0.115 μs | 0.0458 | 0.0153 | - | 10.9 KB | +| **Decrypt** | **1** | **Brotli** | **Stream** | **19.30 μs** | **0.744 μs** | **1.066 μs** | **0.0305** | **-** | **-** | **12.99 KB** | +| DecryptToProvidedStream | 1 | Brotli | Stream | 18.39 μs | 0.320 μs | 0.449 μs | 0.0305 | - | - | 11.58 KB | +| **Decrypt** | **10** | **None** | **Stream** | **26.64 μs** | **0.150 μs** | **0.220 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | +| DecryptToProvidedStream | 10 | None | Stream | 25.71 μs | 0.138 μs | 0.203 μs | 0.0610 | 0.0305 | - | 17.65 KB | +| **Decrypt** | **10** | **Brotli** | **Stream** | **53.89 μs** | **0.631 μs** | **0.945 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | +| DecryptToProvidedStream | 10 | Brotli | Stream | 54.60 μs | 0.605 μs | 0.887 μs | 0.0610 | - | - | 18.33 KB | +| **Decrypt** | **100** | **None** | **Stream** | **450.58 μs** | **6.572 μs** | **9.837 μs** | **8.3008** | **8.3008** | **8.3008** | **320.02 KB** | +| DecryptToProvidedStream | 100 | None | Stream | 379.11 μs | 4.473 μs | 6.415 μs | 4.3945 | 4.3945 | 4.3945 | 163 KB | +| **Decrypt** | **100** | **Brotli** | **Stream** | **303.48 μs** | **5.305 μs** | **7.940 μs** | **5.8594** | **5.8594** | **5.8594** | **215.95 KB** | +| DecryptToProvidedStream | 100 | Brotli | Stream | 253.71 μs | 3.448 μs | 5.054 μs | 2.9297 | 2.9297 | 2.9297 | 111.63 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 271ab6671a..95a0658821 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -61,6 +61,7 @@ public static void ClassInitialize(TestContext testContext) [DataRow(JsonProcessor.Newtonsoft)] #if NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] + [DataRow(JsonProcessor.Stream)] #endif public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) { @@ -101,6 +102,7 @@ public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) [DataRow(JsonProcessor.Newtonsoft)] #if NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] + [DataRow(JsonProcessor.Stream)] #endif public async Task DuplicatePathToEncrypt(JsonProcessor jsonProcessor) { @@ -310,6 +312,7 @@ public async Task ValidateDecryptBySystemTextStream_VerifyBySystemText(Encryptio [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] + [DataRow(JsonProcessor.Stream)] #endif public async Task DecryptStreamWithoutEncryptedProperty(JsonProcessor processor) { @@ -600,6 +603,7 @@ public static IEnumerable EncryptionOptionsStreamTestCombinations yield return new object[] { encryptionOptions[0], JsonProcessor.Newtonsoft }; #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER yield return new object[] { encryptionOptions[0], JsonProcessor.SystemTextJson }; + yield return new object[] { encryptionOptions[0], JsonProcessor.Stream }; #endif } } From 420a35d8f8d45f7d1816bf1e6670c7fffca85da0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 18:28:42 +0200 Subject: [PATCH 56/71] ! bugfix --- .../src/EncryptionProcessor.cs | 2 +- .../src/Transformation/StreamProcessor.cs | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 59a43f2206..776fe9d304 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -247,7 +247,7 @@ public static async Task DecryptAsync( return (input, null); } - MemoryStream ms = new MemoryStream(); + MemoryStream ms = new (); DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, ms, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs index b24b8d9d89..cba10ff0bb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -50,9 +50,9 @@ internal async Task DecryptStreamAsync( Utf8JsonWriter writer = new (outputStream); - // we determine initial buffer size based on max uncompressed path length, it still might need scale out in case there is large non-encrypted object, but likehood is rather low - int bufferSize = 16384; // Math.Max(16384, properties.CompressedEncryptedPaths.Values.Max()); + int bufferSize = 16384; byte[] buffer = arrayPoolManager.Rent(bufferSize); + bufferSize = buffer.Length; JsonReaderState state = new (this.jsonReaderOptions); @@ -61,6 +61,8 @@ internal async Task DecryptStreamAsync( bool isFinalBlock = false; bool isIgnoredBlock = false; + string decryptPropertyName = null; + while (!isFinalBlock) { int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); @@ -75,6 +77,7 @@ internal async Task DecryptStreamAsync( writer, ref state, ref isIgnoredBlock, + ref decryptPropertyName, pathsDecrypted, properties, arrayPoolManager, @@ -87,7 +90,7 @@ internal async Task DecryptStreamAsync( { bufferSize *= 2; byte[] newBuffer = arrayPoolManager.Rent(bufferSize); - buffer.AsSpan(0, leftOver).CopyTo(newBuffer); + buffer.AsSpan().CopyTo(newBuffer); buffer = newBuffer; } else if (leftOver != 0) @@ -122,12 +125,10 @@ private static Dictionary GetUtf8DecryptionList(EncryptionPropertie return output; }*/ - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); - string decryptPropertyName = null; - while (reader.Read()) { JsonTokenType tokenType = reader.TokenType; From d08be05dcfcfb3e3c25f18daf9fc8b31d86b2a69 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 14 Oct 2024 18:45:56 +0200 Subject: [PATCH 57/71] ~ benchmark update --- .../Readme.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index d872d7f1a1..aaf21846e5 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,17 +9,17 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |-------------- |----------:|---------:|---------:|-------:|-------:|-------:|----------:| -| **Decrypt** | **1** | **None** | **Stream** | **12.63 μs** | **0.146 μs** | **0.213 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | -| DecryptToProvidedStream | 1 | None | Stream | 12.58 μs | 0.082 μs | 0.115 μs | 0.0458 | 0.0153 | - | 10.9 KB | -| **Decrypt** | **1** | **Brotli** | **Stream** | **19.30 μs** | **0.744 μs** | **1.066 μs** | **0.0305** | **-** | **-** | **12.99 KB** | -| DecryptToProvidedStream | 1 | Brotli | Stream | 18.39 μs | 0.320 μs | 0.449 μs | 0.0305 | - | - | 11.58 KB | -| **Decrypt** | **10** | **None** | **Stream** | **26.64 μs** | **0.150 μs** | **0.220 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | -| DecryptToProvidedStream | 10 | None | Stream | 25.71 μs | 0.138 μs | 0.203 μs | 0.0610 | 0.0305 | - | 17.65 KB | -| **Decrypt** | **10** | **Brotli** | **Stream** | **53.89 μs** | **0.631 μs** | **0.945 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | -| DecryptToProvidedStream | 10 | Brotli | Stream | 54.60 μs | 0.605 μs | 0.887 μs | 0.0610 | - | - | 18.33 KB | -| **Decrypt** | **100** | **None** | **Stream** | **450.58 μs** | **6.572 μs** | **9.837 μs** | **8.3008** | **8.3008** | **8.3008** | **320.02 KB** | -| DecryptToProvidedStream | 100 | None | Stream | 379.11 μs | 4.473 μs | 6.415 μs | 4.3945 | 4.3945 | 4.3945 | 163 KB | -| **Decrypt** | **100** | **Brotli** | **Stream** | **303.48 μs** | **5.305 μs** | **7.940 μs** | **5.8594** | **5.8594** | **5.8594** | **215.95 KB** | -| DecryptToProvidedStream | 100 | Brotli | Stream | 253.71 μs | 3.448 μs | 5.054 μs | 2.9297 | 2.9297 | 2.9297 | 111.63 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |----------:|----------:|----------:|-------:|-------:|-------:|----------:| +| **Decrypt** | **1** | **None** | **Stream** | **12.80 μs** | **0.161 μs** | **0.237 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | +| DecryptToProvidedStream | 1 | None | Stream | 12.82 μs | 0.075 μs | 0.108 μs | 0.0458 | 0.0153 | - | 10.9 KB | +| **Decrypt** | **1** | **Brotli** | **Stream** | **19.41 μs** | **0.613 μs** | **0.899 μs** | **0.0305** | **-** | **-** | **12.99 KB** | +| DecryptToProvidedStream | 1 | Brotli | Stream | 18.52 μs | 0.206 μs | 0.288 μs | 0.0305 | - | - | 11.58 KB | +| **Decrypt** | **10** | **None** | **Stream** | **26.96 μs** | **0.103 μs** | **0.148 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | +| DecryptToProvidedStream | 10 | None | Stream | 25.94 μs | 0.104 μs | 0.149 μs | 0.0610 | 0.0305 | - | 17.65 KB | +| **Decrypt** | **10** | **Brotli** | **Stream** | **53.24 μs** | **0.602 μs** | **0.882 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | +| DecryptToProvidedStream | 10 | Brotli | Stream | 54.38 μs | 0.471 μs | 0.691 μs | 0.0610 | - | - | 18.33 KB | +| **Decrypt** | **100** | **None** | **Stream** | **336.31 μs** | **7.637 μs** | **11.194 μs** | **5.8594** | **5.8594** | **5.8594** | **225.27 KB** | +| DecryptToProvidedStream | 100 | None | Stream | 283.21 μs | 2.668 μs | 3.993 μs | 2.9297 | 2.9297 | 2.9297 | 115.98 KB | +| **Decrypt** | **100** | **Brotli** | **Stream** | **487.48 μs** | **7.638 μs** | **11.433 μs** | **6.8359** | **6.8359** | **6.8359** | **225.84 KB** | +| DecryptToProvidedStream | 100 | Brotli | Stream | 457.04 μs | 10.030 μs | 14.384 μs | 3.4180 | 3.4180 | 3.4180 | 116.52 KB | From 0bf29b82eb21c499f5cd19ea11e59c43aa6cc7cc Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 08:56:50 +0200 Subject: [PATCH 58/71] + cleanup --- .../src/Transformation/StreamProcessor.cs | 54 +++++++------------ .../MdeEncryptionProcessorTests.cs | 7 +++ 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs index cba10ff0bb..2f95a8ce33 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs @@ -19,12 +19,15 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation internal class StreamProcessor { + private const string EncryptionPropertiesPath = "/" + Constants.EncryptedInfo; private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); private static readonly SqlBigIntSerializer SqlLongSerializer = new (); private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + internal static int InitialBufferSize { get; set; } = 16384; + internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); internal async Task DecryptStreamAsync( @@ -48,11 +51,9 @@ internal async Task DecryptStreamAsync( List pathsDecrypted = new (properties.EncryptedPaths.Count()); - Utf8JsonWriter writer = new (outputStream); + using Utf8JsonWriter writer = new (outputStream); - int bufferSize = 16384; - byte[] buffer = arrayPoolManager.Rent(bufferSize); - bufferSize = buffer.Length; + byte[] buffer = arrayPoolManager.Rent(InitialBufferSize); JsonReaderState state = new (this.jsonReaderOptions); @@ -63,6 +64,8 @@ internal async Task DecryptStreamAsync( string decryptPropertyName = null; + bool containsCompressed = properties.CompressedEncryptedPaths?.Count > 0; + while (!isFinalBlock) { int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); @@ -80,6 +83,7 @@ internal async Task DecryptStreamAsync( ref decryptPropertyName, pathsDecrypted, properties, + containsCompressed, arrayPoolManager, encryptionKey); @@ -88,8 +92,7 @@ internal async Task DecryptStreamAsync( // we need to scale out buffer if (leftOver == dataSize) { - bufferSize *= 2; - byte[] newBuffer = arrayPoolManager.Rent(bufferSize); + byte[] newBuffer = arrayPoolManager.Rent(buffer.Length * 2); buffer.AsSpan().CopyTo(newBuffer); buffer = newBuffer; } @@ -100,32 +103,12 @@ internal async Task DecryptStreamAsync( } writer.Flush(); - inputStream.Position = 0; outputStream.Position = 0; return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } - /* - private static Dictionary GetUtf8DecryptionList(EncryptionProperties properties) - { - Dictionary output = new (properties.EncryptedPaths.Count()); - foreach (KeyValuePair compressedPath in properties.CompressedEncryptedPaths) - { - byte[] utf8String = Encoding.UTF8.GetBytes(compressedPath.Key, 1, compressedPath.Key.Length - 1); - output[utf8String] = compressedPath.Value; - } - - foreach (string encryptedPath in properties.EncryptedPaths) - { - byte[] utf8String = Encoding.UTF8.GetBytes(encryptedPath, 1, encryptedPath.Length - 1); - output.TryAdd(utf8String, -1); - } - - return output; - }*/ - - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); @@ -158,9 +141,10 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW decryptPropertyName, properties, encryptionKey, + containsCompressed, arrayPoolManager); - pathsDecrypted.Add("/" + decryptPropertyName); + pathsDecrypted.Add(decryptPropertyName); } decryptPropertyName = null; @@ -189,13 +173,12 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW writer.WriteEndArray(); break; case JsonTokenType.PropertyName: - string propertyName = reader.GetString(); - if (properties.EncryptedPaths.Contains("/" + propertyName)) + string propertyName = "/" + reader.GetString(); + if (properties.EncryptedPaths.Contains(propertyName)) { decryptPropertyName = propertyName; } - - if (propertyName == Constants.EncryptedInfo) + else if (propertyName == StreamProcessor.EncryptionPropertiesPath) { isIgnoredBlock = true; break; @@ -224,12 +207,11 @@ private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonW return reader.BytesConsumed; } - private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, ArrayPoolManager arrayPoolManager) + private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, bool containsCompressed, ArrayPoolManager arrayPoolManager) { BrotliCompressor decompressor = null; if (properties.EncryptionFormatVersion == 4) { - bool containsCompressed = properties.CompressedEncryptedPaths?.Any() == true; if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) { throw new NotSupportedException($"Unknown compression algorithm {properties.CompressionAlgorithm}"); @@ -254,9 +236,9 @@ private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); - if (decompressor != null) + if (containsCompressed) { - if (properties.CompressedEncryptedPaths?.TryGetValue("/" + decryptPropertyName, out int decompressedSize) == true) + if (properties.CompressedEncryptedPaths?.TryGetValue(decryptPropertyName, out int decompressedSize) == true) { byte[] buffer = arrayPoolManager.Rent(decompressedSize); processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 95a0658821..404bdc9928 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -14,6 +14,9 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom; +#if NET8_0_OR_GREATER + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; +#endif using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json.Linq; @@ -30,6 +33,10 @@ public static void ClassInitialize(TestContext testContext) { _ = testContext; +#if NET8_0_OR_GREATER + StreamProcessor.InitialBufferSize = 16; //we force smallest possible initial buffer to make sure both secondary reads and resize paths are executed +#endif + Mock DekMock = new(); DekMock.Setup(m => m.EncryptData(It.IsAny())) .Returns((byte[] plainText) => TestCommon.EncryptData(plainText)); From 2b2209f165c18ab3fdaa3f30f2808973f061209d Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 15:56:02 +0200 Subject: [PATCH 59/71] + stream serializer + tests + Stream provided benchmarks via RecyclableMemoryStream --- .../src/EncryptionProcessor.cs | 67 ++++ .../src/RentArrayBufferWriter.cs | 203 +++++++++++ .../MdeEncryptionProcessor.Preview.cs | 26 +- ...cessor.cs => StreamProcessor.Decryptor.cs} | 11 +- .../StreamProcessor.Encryptor.cs | 344 ++++++++++++++++++ .../EncryptionBenchmark.cs | 30 +- ...Encryption.Custom.Performance.Tests.csproj | 1 + .../Readme.md | 88 ++++- .../MdeEncryptionProcessorTests.cs | 100 ++--- 9 files changed, 762 insertions(+), 108 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs rename Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/{StreamProcessor.cs => StreamProcessor.Decryptor.cs} (94%) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 776fe9d304..10bc00a0f0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -91,6 +91,60 @@ public static async Task EncryptAsync( #pragma warning restore CS0618 // Type or member is obsolete } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER + public static async Task EncryptAsync( + Stream input, + Stream output, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CosmosDiagnosticsContext diagnosticsContext, + CancellationToken cancellationToken) + { + _ = diagnosticsContext; + + ValidateInputForEncrypt( + input, + encryptor, + encryptionOptions); + + if (!encryptionOptions.PathsToEncrypt.Any()) + { + await input.CopyToAsync(output, cancellationToken); + return; + } + + if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + { + throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); + } + + foreach (string path in encryptionOptions.PathsToEncrypt) + { + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) + { + throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); + } + + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) + { + throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); + } + } + + if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + { + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); + } + + if (encryptionOptions.JsonProcessor != JsonProcessor.Stream) + { + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); + } + + await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); + } +#endif + /// /// If there isn't any data that needs to be decrypted, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. @@ -153,6 +207,7 @@ public static async Task DecryptAsync( Stream output, Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, + JsonProcessor jsonProcessor, CancellationToken cancellationToken) { if (input == null) @@ -160,6 +215,11 @@ public static async Task DecryptAsync( return null; } + if (jsonProcessor != JsonProcessor.Stream) + { + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); + } + Debug.Assert(input.CanSeek); Debug.Assert(output.CanWrite); Debug.Assert(output.CanSeek); @@ -171,9 +231,16 @@ public static async Task DecryptAsync( input.Position = 0; if (properties?.EncryptionProperties == null) { + await input.CopyToAsync(output, cancellationToken: cancellationToken); return null; } + if (properties.EncryptionProperties.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + { + input.Position = 0; + throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); + } + DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs new file mode 100644 index 0000000000..cfd342d12a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs @@ -0,0 +1,203 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom; + +#if NET8_0_OR_GREATER + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +/// +/// https://gist.github.com/ahsonkhan/c76a1cc4dc7107537c3fdc0079a68b35 +/// Standard ArrayBufferWriter is not using pooled memory +/// +internal class RentArrayBufferWriter : IBufferWriter, IDisposable +{ + private const int MinimumBufferSize = 256; + + private byte[] rentedBuffer; + private int written; + private long committed; + + public RentArrayBufferWriter(int initialCapacity = MinimumBufferSize) + { + if (initialCapacity <= 0) + { + throw new ArgumentException(null, nameof(initialCapacity)); + } + + this.rentedBuffer = ArrayPool.Shared.Rent(initialCapacity); + this.written = 0; + this.committed = 0; + } + + public (byte[], int) WrittenBuffer => (this.rentedBuffer, this.written); + + public Memory WrittenMemory + { + get + { + this.CheckIfDisposed(); + + return this.rentedBuffer.AsMemory(0, this.written); + } + } + + public Span WrittenSpan + { + get + { + this.CheckIfDisposed(); + + return this.rentedBuffer.AsSpan(0, this.written); + } + } + + public int BytesWritten + { + get + { + this.CheckIfDisposed(); + + return this.written; + } + } + + public long BytesCommitted + { + get + { + this.CheckIfDisposed(); + + return this.committed; + } + } + + public void Clear() + { + this.CheckIfDisposed(); + + this.ClearHelper(); + } + + private void ClearHelper() + { + this.rentedBuffer.AsSpan(0, this.written).Clear(); + this.written = 0; + } + + public async Task CopyToAsync(Stream stream, CancellationToken cancellationToken = default) + { + this.CheckIfDisposed(); + + ArgumentNullException.ThrowIfNull(stream); + + await stream.WriteAsync(new Memory(this.rentedBuffer, 0, this.written), cancellationToken).ConfigureAwait(false); + this.committed += this.written; + + this.ClearHelper(); + } + + public void CopyTo(Stream stream) + { + this.CheckIfDisposed(); + + ArgumentNullException.ThrowIfNull(stream); + + stream.Write(this.rentedBuffer, 0, this.written); + this.committed += this.written; + + this.ClearHelper(); + } + + public void Advance(int count) + { + this.CheckIfDisposed(); + + ArgumentOutOfRangeException.ThrowIfLessThan(count, 0); + + if (this.written > this.rentedBuffer.Length - count) + { + throw new InvalidOperationException("Cannot advance past the end of the buffer."); + } + + this.written += count; + } + + // Returns the rented buffer back to the pool + public void Dispose() + { + if (this.rentedBuffer == null) + { + return; + } + + ArrayPool.Shared.Return(this.rentedBuffer, clearArray: true); + this.rentedBuffer = null; + this.written = 0; + } + + private void CheckIfDisposed() + { + ObjectDisposedException.ThrowIf(this.rentedBuffer == null, this); + } + + public Memory GetMemory(int sizeHint = 0) + { + this.CheckIfDisposed(); + + ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); + + this.CheckAndResizeBuffer(sizeHint); + return this.rentedBuffer.AsMemory(this.written); + } + + public Span GetSpan(int sizeHint = 0) + { + this.CheckIfDisposed(); + + ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); + + this.CheckAndResizeBuffer(sizeHint); + return this.rentedBuffer.AsSpan(this.written); + } + + private void CheckAndResizeBuffer(int sizeHint) + { + Debug.Assert(sizeHint >= 0); + + if (sizeHint == 0) + { + sizeHint = MinimumBufferSize; + } + + int availableSpace = this.rentedBuffer.Length - this.written; + + if (sizeHint > availableSpace) + { + int growBy = sizeHint > this.rentedBuffer.Length ? sizeHint : this.rentedBuffer.Length; + + int newSize = checked(this.rentedBuffer.Length + growBy); + + byte[] oldBuffer = this.rentedBuffer; + + this.rentedBuffer = ArrayPool.Shared.Rent(newSize); + + Debug.Assert(oldBuffer.Length >= this.written); + Debug.Assert(this.rentedBuffer.Length >= this.written); + + oldBuffer.AsSpan(0, this.written).CopyTo(this.rentedBuffer); + ArrayPool.Shared.Return(oldBuffer, clearArray: true); + } + + Debug.Assert(this.rentedBuffer.Length - this.written > 0); + Debug.Assert(this.rentedBuffer.Length - this.written >= sizeHint); + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 8649b36079..2dc3eb7e10 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -21,6 +21,8 @@ internal class MdeEncryptionProcessor #if NET8_0_OR_GREATER internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor(); + + internal StreamProcessor StreamProcessor { get; set; } = new StreamProcessor(); #endif public async Task EncryptAsync( @@ -29,15 +31,29 @@ public async Task EncryptAsync( EncryptionOptions encryptionOptions, CancellationToken token) { +#if NET8_0_OR_GREATER + switch (encryptionOptions.JsonProcessor) + { + case JsonProcessor.Newtonsoft: + return await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); + + case JsonProcessor.SystemTextJson: + return await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); + case JsonProcessor.Stream: + MemoryStream ms = new (); + await this.StreamProcessor.EncryptStreamAsync(input, ms, encryptor, encryptionOptions, token); + return ms; + + default: + throw new InvalidOperationException("Unsupported JsonProcessor"); + } +#else return encryptionOptions.JsonProcessor switch { JsonProcessor.Newtonsoft => await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), -#if NET8_0_OR_GREATER - JsonProcessor.SystemTextJson => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), - JsonProcessor.Stream => await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token), -#endif - _ => throw new InvalidOperationException("Unsupported JsonProcessor") + _ => throw new InvalidOperationException("Unsupported JsonProcessor"), }; +#endif } internal async Task DecryptObjectAsync( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs similarity index 94% rename from Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs rename to Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 2f95a8ce33..6489a1f3e3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -11,20 +11,19 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Collections.Generic; using System.IO; using System.Linq; - using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.Encryption.Cryptography.Serializers; - internal class StreamProcessor + internal partial class StreamProcessor { private const string EncryptionPropertiesPath = "/" + Constants.EncryptedInfo; private static readonly SqlBitSerializer SqlBoolSerializer = new (); private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); private static readonly SqlBigIntSerializer SqlLongSerializer = new (); - private readonly JsonReaderOptions jsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + private static readonly JsonReaderOptions JsonReaderOptions = new () { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; internal static int InitialBufferSize { get; set; } = 16384; @@ -55,7 +54,7 @@ internal async Task DecryptStreamAsync( byte[] buffer = arrayPoolManager.Rent(InitialBufferSize); - JsonReaderState state = new (this.jsonReaderOptions); + JsonReaderState state = new (StreamProcessor.JsonReaderOptions); int leftOver = 0; @@ -74,7 +73,7 @@ internal async Task DecryptStreamAsync( long bytesConsumed = 0; // processing itself here - bytesConsumed = this.TransformReadBuffer( + bytesConsumed = this.TransformDecryptBuffer( buffer.AsSpan(0, dataSize), isFinalBlock, writer, @@ -108,7 +107,7 @@ internal async Task DecryptStreamAsync( return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } - private long TransformReadBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformDecryptBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs new file mode 100644 index 0000000000..9f521cf9ef --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -0,0 +1,344 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal partial class StreamProcessor + { + private readonly byte[] encryptionPropertiesNameBytes = Encoding.UTF8.GetBytes(Constants.EncryptedInfo); + + internal async Task EncryptStreamAsync( + Stream inputStream, + Stream outputStream, + Encryptor encryptor, + EncryptionOptions encryptionOptions, + CancellationToken cancellationToken) + { + List pathsEncrypted = new (); + + using ArrayPoolManager arrayPoolManager = new (); + + DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, cancellationToken); + + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; + + Dictionary compressedPaths = new (); + + using Utf8JsonWriter writer = new (outputStream); + + byte[] buffer = arrayPoolManager.Rent(InitialBufferSize); + + JsonReaderState state = new (StreamProcessor.JsonReaderOptions); + + int leftOver = 0; + + bool isFinalBlock = false; + + Utf8JsonWriter encryptionPayloadWriter = null; + string encryptPropertyName = null; + RentArrayBufferWriter bufferWriter = null; + + while (!isFinalBlock) + { + int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); + int dataSize = dataLength + leftOver; + isFinalBlock = dataSize == 0; + long bytesConsumed = 0; + + bytesConsumed = this.TransformEncryptBuffer( + buffer.AsSpan(0, dataSize), + isFinalBlock, + writer, + ref encryptionPayloadWriter, + ref bufferWriter, + ref state, + ref encryptPropertyName, + pathsEncrypted, + compressedPaths, + compressor, + arrayPoolManager, + encryptionKey, + encryptionOptions); + + leftOver = dataSize - (int)bytesConsumed; + + // we need to scale out buffer + if (leftOver == dataSize) + { + byte[] newBuffer = arrayPoolManager.Rent(buffer.Length * 2); + buffer.AsSpan().CopyTo(newBuffer); + buffer = newBuffer; + } + else if (leftOver != 0) + { + buffer.AsSpan(dataSize - leftOver, leftOver).CopyTo(buffer); + } + } + + await inputStream.DisposeAsync(); + + EncryptionProperties encryptionProperties = new ( + encryptionFormatVersion: compressionEnabled ? 4 : 3, + encryptionOptions.EncryptionAlgorithm, + encryptionOptions.DataEncryptionKeyId, + encryptedData: null, + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); + + writer.WritePropertyName(this.encryptionPropertiesNameBytes); + JsonSerializer.Serialize(writer, encryptionProperties); + writer.WriteEndObject(); + + writer.Flush(); + outputStream.Position = 0; + } + + private long TransformEncryptBuffer( + Span buffer, + bool isFinalBlock, + Utf8JsonWriter writer, + ref Utf8JsonWriter encryptionPayloadWriter, + ref RentArrayBufferWriter bufferWriter, + ref JsonReaderState state, + ref string encryptPropertyName, + List pathsEncrypted, + Dictionary compressedPaths, + BrotliCompressor compressor, + ArrayPoolManager arrayPoolManager, + DataEncryptionKey encryptionKey, + EncryptionOptions encryptionOptions) + { + Utf8JsonReader reader = new (buffer, isFinalBlock, state); + + while (reader.Read()) + { + Utf8JsonWriter currentWriter = encryptionPayloadWriter ?? writer; + + JsonTokenType tokenType = reader.TokenType; + + switch (tokenType) + { + case JsonTokenType.None: + break; + case JsonTokenType.StartObject: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartObject(); + } + else + { + currentWriter.WriteStartObject(); + } + + break; + case JsonTokenType.EndObject: + if (reader.CurrentDepth == 0) + { + continue; + } + + currentWriter.WriteEndObject(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; + encryptionPayloadWriter.Dispose(); + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.StartArray: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartArray(); + } + else + { + currentWriter.WriteStartArray(); + } + + break; + case JsonTokenType.EndArray: + currentWriter.WriteEndArray(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; + encryptionPayloadWriter.Dispose(); + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.PropertyName: + string propertyName = "/" + reader.GetString(); + if (encryptionOptions.PathsToEncrypt.Contains(propertyName, StringComparer.Ordinal)) + { + encryptPropertyName = propertyName; + } + + currentWriter.WritePropertyName(reader.ValueSpan); + break; + case JsonTokenType.Comment: + currentWriter.WriteCommentValue(reader.ValueSpan); + break; + case JsonTokenType.String: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); + int length = reader.CopyString(bytes); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteStringValue(reader.ValueSpan); + } + + break; + case JsonTokenType.Number: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteRawValue(reader.ValueSpan); + } + + break; + case JsonTokenType.True: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(true, arrayPoolManager); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(true); + } + + break; + case JsonTokenType.False: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(false, arrayPoolManager); + Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(false); + } + + break; + case JsonTokenType.Null: + currentWriter.WriteNullValue(); + break; + } + } + + state = reader.CurrentState; + return reader.BytesConsumed; + } + + private static (byte[] buffer, int length) Serialize(bool value, ArrayPoolManager arrayPoolManager) + { + int byteCount = StreamProcessor.SqlBoolSerializer.GetSerializedMaxByteCount(); + byte[] buffer = arrayPoolManager.Rent(byteCount); + int length = StreamProcessor.SqlBoolSerializer.Serialize(value, buffer); + + return (buffer, length); + } + + private static (TypeMarker typeMarker, byte[] buffer, int length) SerializeNumber(ReadOnlySpan utf8bytes, ArrayPoolManager arrayPoolManager) + { + if (long.TryParse(utf8bytes, out long longValue)) + { + return Serialize(longValue, arrayPoolManager); + } + else if (double.TryParse(utf8bytes, out double doubleValue)) + { + return Serialize(doubleValue, arrayPoolManager); + } + else + { + throw new InvalidOperationException("Unsupported Number type"); + } + } + + private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(long value, ArrayPoolManager arrayPoolManager) + { + int byteCount = StreamProcessor.SqlLongSerializer.GetSerializedMaxByteCount(); + byte[] buffer = arrayPoolManager.Rent(byteCount); + int length = StreamProcessor.SqlLongSerializer.Serialize(value, buffer); + + return (TypeMarker.Long, buffer, length); + } + + private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(double value, ArrayPoolManager arrayPoolManager) + { + int byteCount = StreamProcessor.SqlDoubleSerializer.GetSerializedMaxByteCount(); + byte[] buffer = arrayPoolManager.Rent(byteCount); + int length = StreamProcessor.SqlDoubleSerializer.Serialize(value, buffer); + + return (TypeMarker.Double, buffer, length); + } + + private Span TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) + { + byte[] processedBytes = payload; + int processedBytesLength = payloadSize; + + if (compressor != null && payloadSize >= options.CompressionOptions.MinimalCompressedLength) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(payloadSize)); + processedBytesLength = compressor.Compress(pathsCompressed, encryptName, processedBytes, payloadSize, compressedBytes); + processedBytes = compressedBytes; + } + + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); + + pathsEncrypted.Add(encryptName); + return encryptedBytes.AsSpan(0, encryptedBytesCount); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 395eed045d..ad8931c305 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -3,6 +3,7 @@ using System.IO; using BenchmarkDotNet.Attributes; using Microsoft.Data.Encryption.Cryptography; + using Microsoft.IO; using Moq; [RPlotExporter] @@ -16,21 +17,21 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); + private readonly RecyclableMemoryStreamManager recyclableMemoryStreamManager = new (); + private CosmosEncryptor? encryptor; private EncryptionOptions? encryptionOptions; private byte[]? encryptedData; private byte[]? plaintext; - private static readonly MemoryStream recycledStream = new (); - [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } - [Params(/*JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, */JsonProcessor.Stream)] + [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, JsonProcessor.Stream)] public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -61,7 +62,7 @@ public async Task Setup() this.encryptedData = memoryStream.ToArray(); } - /* + [Benchmark] public async Task Encrypt() { @@ -71,7 +72,20 @@ await EncryptionProcessor.EncryptAsync( this.encryptionOptions, new CosmosDiagnosticsContext(), CancellationToken.None); - }*/ + } + + [Benchmark] + public async Task EncryptToProvidedStream() + { + using RecyclableMemoryStream rms = new (this.recyclableMemoryStreamManager); + await EncryptionProcessor.EncryptAsync( + new MemoryStream(this.plaintext!), + rms, + this.encryptor, + this.encryptionOptions, + new CosmosDiagnosticsContext(), + CancellationToken.None); + } [Benchmark] public async Task Decrypt() @@ -87,14 +101,14 @@ await EncryptionProcessor.DecryptAsync( [Benchmark] public async Task DecryptToProvidedStream() { + using RecyclableMemoryStream rms = new(this.recyclableMemoryStreamManager); await EncryptionProcessor.DecryptAsync( new MemoryStream(this.encryptedData!), - EncryptionBenchmark.recycledStream, + rms, this.encryptor, new CosmosDiagnosticsContext(), + this.JsonProcessor, CancellationToken.None); - - EncryptionBenchmark.recycledStream.Position = 0; } private EncryptionOptions CreateEncryptionOptions() diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index edcdac998b..2adf26ad02 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -17,6 +17,7 @@ + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index aaf21846e5..6d65279c26 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,17 +9,77 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |-------------- |----------:|----------:|----------:|-------:|-------:|-------:|----------:| -| **Decrypt** | **1** | **None** | **Stream** | **12.80 μs** | **0.161 μs** | **0.237 μs** | **0.0458** | **0.0153** | **-** | **12.31 KB** | -| DecryptToProvidedStream | 1 | None | Stream | 12.82 μs | 0.075 μs | 0.108 μs | 0.0458 | 0.0153 | - | 10.9 KB | -| **Decrypt** | **1** | **Brotli** | **Stream** | **19.41 μs** | **0.613 μs** | **0.899 μs** | **0.0305** | **-** | **-** | **12.99 KB** | -| DecryptToProvidedStream | 1 | Brotli | Stream | 18.52 μs | 0.206 μs | 0.288 μs | 0.0305 | - | - | 11.58 KB | -| **Decrypt** | **10** | **None** | **Stream** | **26.96 μs** | **0.103 μs** | **0.148 μs** | **0.1221** | **0.0305** | **-** | **28.77 KB** | -| DecryptToProvidedStream | 10 | None | Stream | 25.94 μs | 0.104 μs | 0.149 μs | 0.0610 | 0.0305 | - | 17.65 KB | -| **Decrypt** | **10** | **Brotli** | **Stream** | **53.24 μs** | **0.602 μs** | **0.882 μs** | **0.1221** | **0.0610** | **-** | **29.45 KB** | -| DecryptToProvidedStream | 10 | Brotli | Stream | 54.38 μs | 0.471 μs | 0.691 μs | 0.0610 | - | - | 18.33 KB | -| **Decrypt** | **100** | **None** | **Stream** | **336.31 μs** | **7.637 μs** | **11.194 μs** | **5.8594** | **5.8594** | **5.8594** | **225.27 KB** | -| DecryptToProvidedStream | 100 | None | Stream | 283.21 μs | 2.668 μs | 3.993 μs | 2.9297 | 2.9297 | 2.9297 | 115.98 KB | -| **Decrypt** | **100** | **Brotli** | **Stream** | **487.48 μs** | **7.638 μs** | **11.433 μs** | **6.8359** | **6.8359** | **6.8359** | **225.84 KB** | -| DecryptToProvidedStream | 100 | Brotli | Stream | 457.04 μs | 10.030 μs | 14.384 μs | 3.4180 | 3.4180 | 3.4180 | 116.52 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **23.89 μs** | **0.489 μs** | **0.702 μs** | **0.1526** | **0.0305** | **-** | **42064 B** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | Newtonsoft | 28.65 μs | 0.340 μs | 0.488 μs | 0.1221 | - | - | 41440 B | +| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **SystemTextJson** | **15.67 μs** | **0.206 μs** | **0.309 μs** | **0.0916** | **0.0305** | **-** | **23184 B** | +| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | SystemTextJson | 15.46 μs | 0.124 μs | 0.186 μs | 0.0610 | 0.0305 | - | 21448 B | +| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **Stream** | **14.29 μs** | **0.159 μs** | **0.237 μs** | **0.0610** | **0.0153** | **-** | **18408 B** | +| EncryptToProvidedStream | 1 | None | Stream | 14.03 μs | 0.225 μs | 0.323 μs | 0.0458 | 0.0153 | - | 12272 B | +| Decrypt | 1 | None | Stream | 13.75 μs | 0.258 μs | 0.370 μs | 0.0305 | - | - | 12456 B | +| DecryptToProvidedStream | 1 | None | Stream | 14.10 μs | 0.056 μs | 0.082 μs | 0.0458 | 0.0153 | - | 11288 B | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **29.79 μs** | **0.287 μs** | **0.429 μs** | **0.1526** | **0.0305** | **-** | **38344 B** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | Newtonsoft | 36.13 μs | 0.684 μs | 1.003 μs | 0.1221 | - | - | 41064 B | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **22.27 μs** | **0.206 μs** | **0.309 μs** | **0.0610** | **-** | **-** | **22232 B** | +| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | SystemTextJson | 21.63 μs | 0.161 μs | 0.240 μs | 0.0610 | 0.0305 | - | 20488 B | +| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **Stream** | **22.12 μs** | **0.287 μs** | **0.420 μs** | **0.0610** | **0.0305** | **-** | **17464 B** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 22.00 μs | 0.212 μs | 0.305 μs | 0.0305 | - | - | 12552 B | +| Decrypt | 1 | Brotli | Stream | 19.76 μs | 0.131 μs | 0.196 μs | 0.0305 | - | - | 13000 B | +| DecryptToProvidedStream | 1 | Brotli | Stream | 20.27 μs | 0.194 μs | 0.290 μs | 0.0305 | - | - | 11832 B | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **90.20 μs** | **1.743 μs** | **2.609 μs** | **0.6104** | **0.1221** | **-** | **171273 B** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | Newtonsoft | 105.28 μs | 1.740 μs | 2.551 μs | 0.6104 | 0.1221 | - | 157425 B | +| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **SystemTextJson** | **42.25 μs** | **0.164 μs** | **0.245 μs** | **0.4272** | **0.0610** | **-** | **105625 B** | +| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | SystemTextJson | 42.52 μs | 0.270 μs | 0.395 μs | 0.3662 | 0.0610 | - | 96464 B | +| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **Stream** | **43.70 μs** | **0.568 μs** | **0.850 μs** | **0.3052** | **0.0610** | **-** | **87488 B** | +| EncryptToProvidedStream | 10 | None | Stream | 40.54 μs | 0.283 μs | 0.424 μs | 0.1221 | - | - | 41608 B | +| Decrypt | 10 | None | Stream | 28.14 μs | 0.105 μs | 0.150 μs | 0.0916 | 0.0305 | - | 29304 B | +| DecryptToProvidedStream | 10 | None | Stream | 27.86 μs | 0.111 μs | 0.163 μs | 0.0610 | 0.0305 | - | 18200 B | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.64 μs** | **0.974 μs** | **1.397 μs** | **0.6104** | **0.1221** | **-** | **168345 B** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | Newtonsoft | 124.43 μs | 0.985 μs | 1.475 μs | 0.4883 | - | - | 144849 B | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **72.28 μs** | **0.366 μs** | **0.514 μs** | **0.2441** | **-** | **-** | **86497 B** | +| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | SystemTextJson | 66.36 μs | 0.830 μs | 1.242 μs | 0.2441 | - | - | 82201 B | +| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **Stream** | **91.08 μs** | **3.161 μs** | **4.731 μs** | **0.2441** | **-** | **-** | **68369 B** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 97.09 μs | 1.725 μs | 2.581 μs | 0.1221 | - | - | 37025 B | +| Decrypt | 10 | Brotli | Stream | 57.83 μs | 0.657 μs | 0.963 μs | 0.1221 | 0.0610 | - | 29848 B | +| DecryptToProvidedStream | 10 | Brotli | Stream | 57.69 μs | 0.554 μs | 0.830 μs | 0.0610 | - | - | 18744 B | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,167.44 μs** | **45.257 μs** | **67.739 μs** | **25.3906** | **23.4375** | **21.4844** | **1678336 B** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | Newtonsoft | 1,182.35 μs | 23.743 μs | 34.803 μs | 17.5781 | 15.6250 | 15.6250 | 1260244 B | +| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **SystemTextJson** | **830.10 μs** | **27.127 μs** | **40.603 μs** | **25.3906** | **25.3906** | **25.3906** | **965525 B** | +| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | SystemTextJson | 749.29 μs | 19.292 μs | 28.279 μs | 18.5547 | 18.5547 | 18.5547 | 950282 B | +| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **Stream** | **637.45 μs** | **34.205 μs** | **51.196 μs** | **14.6484** | **14.6484** | **14.6484** | **719521 B** | +| EncryptToProvidedStream | 100 | None | Stream | 385.08 μs | 5.025 μs | 7.521 μs | 4.8828 | 4.3945 | 4.3945 | 271565 B | +| Decrypt | 100 | None | Stream | 380.02 μs | 11.443 μs | 17.128 μs | 6.3477 | 6.3477 | 6.3477 | 230536 B | +| DecryptToProvidedStream | 100 | None | Stream | 304.54 μs | 8.678 μs | 12.989 μs | 2.9297 | 2.9297 | 2.9297 | 118897 B | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,172.02 μs** | **19.488 μs** | **29.169 μs** | **13.6719** | **11.7188** | **9.7656** | **1379452 B** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | Newtonsoft | 1,153.94 μs | 12.008 μs | 17.602 μs | 11.7188 | 9.7656 | 9.7656 | 1124251 B | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **995.73 μs** | **27.736 μs** | **40.655 μs** | **21.4844** | **21.4844** | **21.4844** | **766965 B** | +| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | SystemTextJson | 892.93 μs | 16.896 μs | 25.288 μs | 17.5781 | 17.5781 | 17.5781 | 801458 B | +| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **Stream** | **770.06 μs** | **11.317 μs** | **16.939 μs** | **10.7422** | **10.7422** | **10.7422** | **520929 B** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 606.21 μs | 9.107 μs | 13.631 μs | 2.9297 | 2.9297 | 2.9297 | 222081 B | +| Decrypt | 100 | Brotli | Stream | 537.74 μs | 8.192 μs | 12.007 μs | 6.3477 | 6.3477 | 6.3477 | 230938 B | +| DecryptToProvidedStream | 100 | Brotli | Stream | 464.80 μs | 4.408 μs | 6.461 μs | 3.4180 | 3.4180 | 3.4180 | 119300 B | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 404bdc9928..0f19a8ebc6 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests using System; using System.Collections.Generic; using System.IO; + using System.IO.Compression; using System.Linq; #if NET8_0_OR_GREATER using System.Text.Json.Nodes; @@ -520,84 +521,33 @@ private static void AssertNullableValueKind(T expectedValue, JsonNode node, s } #endif - public static IEnumerable EncryptionOptionsCombinations => new[] { - new object[] { new EncryptionOptions() + private static EncryptionOptions CreateEncryptionOptions(JsonProcessor processor, CompressionOptions.CompressionAlgorithm compressionAlgorithm, CompressionLevel compressionLevel) + { + return new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + JsonProcessor = processor, + CompressionOptions = new CompressionOptions() { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.None - } + Algorithm = compressionAlgorithm, + CompressionLevel = compressionLevel } - }, + }; + } + + public static IEnumerable EncryptionOptionsCombinations => new[] { + new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, #if NET8_0_OR_GREATER - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.None - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.Fastest - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.Newtonsoft, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.Fastest - } - } - }, - new object[] { new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt, - JsonProcessor = JsonProcessor.SystemTextJson, - CompressionOptions = new CompressionOptions() - { - Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, - CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, - } - } - } + new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, + new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, + new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, #endif }; From 42462d5670d78c9a5b563d8748122a2f4e900bbe Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 19:01:55 +0200 Subject: [PATCH 60/71] ~ preview branching fixes --- .../src/EncryptionOptions.cs | 2 +- .../src/Transformation/StreamProcessor.Decryptor.cs | 2 +- .../src/Transformation/StreamProcessor.Encryptor.cs | 2 +- .../EncryptionBenchmark.cs | 12 ++++++++++++ ...Cosmos.Encryption.Custom.Performance.Tests.csproj | 5 ++++- .../MdeEncryptionProcessorTests.cs | 10 +++++----- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index aadb2dbbd3..af8ab293e9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -16,7 +16,7 @@ public enum JsonProcessor /// Newtonsoft, -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER /// /// System.Text.Json /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 6489a1f3e3..6459dd235d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index 9f521cf9ef..e19b802800 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // ------------------------------------------------------------ -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index ad8931c305..c851750c53 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -3,7 +3,9 @@ using System.IO; using BenchmarkDotNet.Attributes; using Microsoft.Data.Encryption.Cryptography; +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER using Microsoft.IO; +#endif using Moq; [RPlotExporter] @@ -17,7 +19,9 @@ public partial class EncryptionBenchmark new EncryptionKeyWrapMetadata("name", "value"), DateTime.UtcNow); private static readonly Mock StoreProvider = new(); +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private readonly RecyclableMemoryStreamManager recyclableMemoryStreamManager = new (); +#endif private CosmosEncryptor? encryptor; @@ -31,7 +35,11 @@ public partial class EncryptionBenchmark [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, JsonProcessor.Stream)] +#else + [Params(JsonProcessor.Newtonsoft)] +#endif public JsonProcessor JsonProcessor { get; set; } [GlobalSetup] @@ -74,6 +82,7 @@ await EncryptionProcessor.EncryptAsync( CancellationToken.None); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [Benchmark] public async Task EncryptToProvidedStream() { @@ -86,6 +95,7 @@ await EncryptionProcessor.EncryptAsync( new CosmosDiagnosticsContext(), CancellationToken.None); } +#endif [Benchmark] public async Task Decrypt() @@ -98,6 +108,7 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [Benchmark] public async Task DecryptToProvidedStream() { @@ -110,6 +121,7 @@ await EncryptionProcessor.DecryptAsync( this.JsonProcessor, CancellationToken.None); } +#endif private EncryptionOptions CreateEncryptionOptions() { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 2adf26ad02..b04c78e2a8 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -17,10 +17,13 @@ - + + + + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 0f19a8ebc6..c7f3595fc4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -34,7 +34,7 @@ public static void ClassInitialize(TestContext testContext) { _ = testContext; -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER StreamProcessor.InitialBufferSize = 16; //we force smallest possible initial buffer to make sure both secondary reads and resize paths are executed #endif @@ -67,7 +67,7 @@ public static void ClassInitialize(TestContext testContext) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif @@ -108,7 +108,7 @@ public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif @@ -471,7 +471,7 @@ private static void VerifyDecryptionSucceeded( } } -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER private static void VerifyDecryptionSucceeded( JsonNode decryptedDoc, TestDoc expectedDoc, @@ -539,7 +539,7 @@ private static EncryptionOptions CreateEncryptionOptions(JsonProcessor processor public static IEnumerable EncryptionOptionsCombinations => new[] { new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, -#if NET8_0_OR_GREATER +#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, From 703830f91877375ad01f84c86d208cee9759d27f Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Tue, 15 Oct 2024 19:15:56 +0200 Subject: [PATCH 61/71] + add support for Stream deserialization of obsoleted encryption algorithm --- .../src/EncryptionProcessor.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 10bc00a0f0..e373fc9e67 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -235,13 +235,25 @@ public static async Task DecryptAsync( return null; } - if (properties.EncryptionProperties.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + DecryptionContext context; +#pragma warning disable CS0618 // Type or member is obsolete + if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) + { + context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); + } + else if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized) + { + (Stream stream, context) = await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken); + await stream.CopyToAsync(output, cancellationToken); + output.Position = 0; + } + else { input.Position = 0; - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); + throw new NotSupportedException($"Streaming mode is not supported for encryption algorithm {properties.EncryptionProperties.EncryptionAlgorithm}"); } +#pragma warning restore CS0618 // Type or member is obsolete - DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); if (context == null) { input.Position = 0; From 431952402d0c7f429abc494da75c9f82b5fadd28 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Wed, 16 Oct 2024 12:10:12 +0200 Subject: [PATCH 62/71] ~ final touches --- .../src/EncryptionProcessor.cs | 4 +- .../src/RentArrayBufferWriter.cs | 10 +- .../StreamProcessor.Decryptor.cs | 17 +- .../StreamProcessor.Encryptor.cs | 25 +-- ...Encryption.Custom.Performance.Tests.csproj | 3 +- .../Readme.md | 148 +++++++++--------- .../TestDoc.cs | 2 +- 7 files changed, 114 insertions(+), 95 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index e373fc9e67..93209d2450 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -63,7 +63,7 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } @@ -113,7 +113,7 @@ public static async Task EncryptAsync( return; } - if (encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) + if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) { throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs index cfd342d12a..5b37ded4fb 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs @@ -37,7 +37,15 @@ public RentArrayBufferWriter(int initialCapacity = MinimumBufferSize) this.committed = 0; } - public (byte[], int) WrittenBuffer => (this.rentedBuffer, this.written); + public (byte[], int) WrittenBuffer + { + get + { + this.CheckIfDisposed(); + + return (this.rentedBuffer, this.written); + } + } public Memory WrittenMemory { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 6459dd235d..442800d6e0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -56,6 +56,8 @@ internal async Task DecryptStreamAsync( JsonReaderState state = new (StreamProcessor.JsonReaderOptions); + HashSet encryptedPaths = properties.EncryptedPaths as HashSet ?? new (properties.EncryptedPaths, StringComparer.Ordinal); + int leftOver = 0; bool isFinalBlock = false; @@ -80,6 +82,7 @@ internal async Task DecryptStreamAsync( ref state, ref isIgnoredBlock, ref decryptPropertyName, + encryptedPaths, pathsDecrypted, properties, containsCompressed, @@ -107,7 +110,7 @@ internal async Task DecryptStreamAsync( return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); } - private long TransformDecryptBuffer(Span buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) + private long TransformDecryptBuffer(ReadOnlySpan buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, HashSet encryptedPaths, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) { Utf8JsonReader reader = new (buffer, isFinalBlock, state); @@ -173,13 +176,17 @@ private long TransformDecryptBuffer(Span buffer, bool isFinalBlock, Utf8Js break; case JsonTokenType.PropertyName: string propertyName = "/" + reader.GetString(); - if (properties.EncryptedPaths.Contains(propertyName)) + if (encryptedPaths.Contains(propertyName)) { decryptPropertyName = propertyName; } else if (propertyName == StreamProcessor.EncryptionPropertiesPath) { - isIgnoredBlock = true; + if (!reader.TrySkip()) + { + isIgnoredBlock = true; + } + break; } @@ -246,7 +253,7 @@ private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter } } - Span bytesToWrite = bytes.AsSpan(0, processedBytes); + ReadOnlySpan bytesToWrite = bytes.AsSpan(0, processedBytes); switch ((TypeMarker)cipherTextWithTypeMarker[0]) { case TypeMarker.String: @@ -265,7 +272,7 @@ private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer.WriteNullValue(); break; default: - writer.WriteRawValue(bytes.AsSpan(0, processedBytes), true); + writer.WriteRawValue(bytesToWrite, true); break; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index e19b802800..3fd4b0e4f9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -8,7 +8,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System; using System.Collections.Generic; using System.IO; - using System.Linq; using System.Text; using System.Text.Json; using System.Threading; @@ -36,6 +35,8 @@ internal async Task EncryptStreamAsync( BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; + HashSet pathsToEncrypt = encryptionOptions.PathsToEncrypt as HashSet ?? new (encryptionOptions.PathsToEncrypt, StringComparer.Ordinal); + Dictionary compressedPaths = new (); using Utf8JsonWriter writer = new (outputStream); @@ -67,6 +68,7 @@ internal async Task EncryptStreamAsync( ref bufferWriter, ref state, ref encryptPropertyName, + pathsToEncrypt, pathsEncrypted, compressedPaths, compressor, @@ -109,13 +111,14 @@ internal async Task EncryptStreamAsync( } private long TransformEncryptBuffer( - Span buffer, + ReadOnlySpan buffer, bool isFinalBlock, Utf8JsonWriter writer, ref Utf8JsonWriter encryptionPayloadWriter, ref RentArrayBufferWriter bufferWriter, ref JsonReaderState state, ref string encryptPropertyName, + HashSet pathsToEncrypt, List pathsEncrypted, Dictionary compressedPaths, BrotliCompressor compressor, @@ -159,7 +162,7 @@ private long TransformEncryptBuffer( { currentWriter.Flush(); (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; @@ -189,7 +192,7 @@ private long TransformEncryptBuffer( { currentWriter.Flush(); (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; @@ -202,7 +205,7 @@ private long TransformEncryptBuffer( break; case JsonTokenType.PropertyName: string propertyName = "/" + reader.GetString(); - if (encryptionOptions.PathsToEncrypt.Contains(propertyName, StringComparer.Ordinal)) + if (pathsToEncrypt.Contains(propertyName)) { encryptPropertyName = propertyName; } @@ -217,7 +220,7 @@ private long TransformEncryptBuffer( { byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); int length = reader.CopyString(bytes); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } @@ -231,13 +234,13 @@ private long TransformEncryptBuffer( if (encryptPropertyName != null && encryptionPayloadWriter == null) { (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } else { - currentWriter.WriteRawValue(reader.ValueSpan); + currentWriter.WriteRawValue(reader.ValueSpan, true); } break; @@ -245,7 +248,7 @@ private long TransformEncryptBuffer( if (encryptPropertyName != null && encryptionPayloadWriter == null) { (byte[] bytes, int length) = Serialize(true, arrayPoolManager); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } @@ -259,7 +262,7 @@ private long TransformEncryptBuffer( if (encryptPropertyName != null && encryptionPayloadWriter == null) { (byte[] bytes, int length) = Serialize(false, arrayPoolManager); - Span encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); + ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); currentWriter.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; } @@ -322,7 +325,7 @@ private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(doub return (TypeMarker.Double, buffer, length); } - private Span TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) + private ReadOnlySpan TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) { byte[] processedBytes = payload; int processedBytesLength = payloadSize; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index b04c78e2a8..5c154e0a9a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -7,6 +7,7 @@ enable enable true + $(DefineConstants);ENCRYPTION_CUSTOM_PREVIEW @@ -20,7 +21,7 @@ - + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 6d65279c26..22cb620269 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,77 +9,77 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|--------:|--------:|--------:|----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **23.89 μs** | **0.489 μs** | **0.702 μs** | **0.1526** | **0.0305** | **-** | **42064 B** | -| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | Newtonsoft | 28.65 μs | 0.340 μs | 0.488 μs | 0.1221 | - | - | 41440 B | -| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **15.67 μs** | **0.206 μs** | **0.309 μs** | **0.0916** | **0.0305** | **-** | **23184 B** | -| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | SystemTextJson | 15.46 μs | 0.124 μs | 0.186 μs | 0.0610 | 0.0305 | - | 21448 B | -| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **Stream** | **14.29 μs** | **0.159 μs** | **0.237 μs** | **0.0610** | **0.0153** | **-** | **18408 B** | -| EncryptToProvidedStream | 1 | None | Stream | 14.03 μs | 0.225 μs | 0.323 μs | 0.0458 | 0.0153 | - | 12272 B | -| Decrypt | 1 | None | Stream | 13.75 μs | 0.258 μs | 0.370 μs | 0.0305 | - | - | 12456 B | -| DecryptToProvidedStream | 1 | None | Stream | 14.10 μs | 0.056 μs | 0.082 μs | 0.0458 | 0.0153 | - | 11288 B | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **29.79 μs** | **0.287 μs** | **0.429 μs** | **0.1526** | **0.0305** | **-** | **38344 B** | -| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | Newtonsoft | 36.13 μs | 0.684 μs | 1.003 μs | 0.1221 | - | - | 41064 B | -| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **22.27 μs** | **0.206 μs** | **0.309 μs** | **0.0610** | **-** | **-** | **22232 B** | -| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | SystemTextJson | 21.63 μs | 0.161 μs | 0.240 μs | 0.0610 | 0.0305 | - | 20488 B | -| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **Stream** | **22.12 μs** | **0.287 μs** | **0.420 μs** | **0.0610** | **0.0305** | **-** | **17464 B** | -| EncryptToProvidedStream | 1 | Brotli | Stream | 22.00 μs | 0.212 μs | 0.305 μs | 0.0305 | - | - | 12552 B | -| Decrypt | 1 | Brotli | Stream | 19.76 μs | 0.131 μs | 0.196 μs | 0.0305 | - | - | 13000 B | -| DecryptToProvidedStream | 1 | Brotli | Stream | 20.27 μs | 0.194 μs | 0.290 μs | 0.0305 | - | - | 11832 B | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **90.20 μs** | **1.743 μs** | **2.609 μs** | **0.6104** | **0.1221** | **-** | **171273 B** | -| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | Newtonsoft | 105.28 μs | 1.740 μs | 2.551 μs | 0.6104 | 0.1221 | - | 157425 B | -| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **42.25 μs** | **0.164 μs** | **0.245 μs** | **0.4272** | **0.0610** | **-** | **105625 B** | -| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | SystemTextJson | 42.52 μs | 0.270 μs | 0.395 μs | 0.3662 | 0.0610 | - | 96464 B | -| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **Stream** | **43.70 μs** | **0.568 μs** | **0.850 μs** | **0.3052** | **0.0610** | **-** | **87488 B** | -| EncryptToProvidedStream | 10 | None | Stream | 40.54 μs | 0.283 μs | 0.424 μs | 0.1221 | - | - | 41608 B | -| Decrypt | 10 | None | Stream | 28.14 μs | 0.105 μs | 0.150 μs | 0.0916 | 0.0305 | - | 29304 B | -| DecryptToProvidedStream | 10 | None | Stream | 27.86 μs | 0.111 μs | 0.163 μs | 0.0610 | 0.0305 | - | 18200 B | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.64 μs** | **0.974 μs** | **1.397 μs** | **0.6104** | **0.1221** | **-** | **168345 B** | -| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | Newtonsoft | 124.43 μs | 0.985 μs | 1.475 μs | 0.4883 | - | - | 144849 B | -| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **72.28 μs** | **0.366 μs** | **0.514 μs** | **0.2441** | **-** | **-** | **86497 B** | -| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | SystemTextJson | 66.36 μs | 0.830 μs | 1.242 μs | 0.2441 | - | - | 82201 B | -| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **Stream** | **91.08 μs** | **3.161 μs** | **4.731 μs** | **0.2441** | **-** | **-** | **68369 B** | -| EncryptToProvidedStream | 10 | Brotli | Stream | 97.09 μs | 1.725 μs | 2.581 μs | 0.1221 | - | - | 37025 B | -| Decrypt | 10 | Brotli | Stream | 57.83 μs | 0.657 μs | 0.963 μs | 0.1221 | 0.0610 | - | 29848 B | -| DecryptToProvidedStream | 10 | Brotli | Stream | 57.69 μs | 0.554 μs | 0.830 μs | 0.0610 | - | - | 18744 B | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,167.44 μs** | **45.257 μs** | **67.739 μs** | **25.3906** | **23.4375** | **21.4844** | **1678336 B** | -| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | Newtonsoft | 1,182.35 μs | 23.743 μs | 34.803 μs | 17.5781 | 15.6250 | 15.6250 | 1260244 B | -| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **830.10 μs** | **27.127 μs** | **40.603 μs** | **25.3906** | **25.3906** | **25.3906** | **965525 B** | -| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | SystemTextJson | 749.29 μs | 19.292 μs | 28.279 μs | 18.5547 | 18.5547 | 18.5547 | 950282 B | -| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **Stream** | **637.45 μs** | **34.205 μs** | **51.196 μs** | **14.6484** | **14.6484** | **14.6484** | **719521 B** | -| EncryptToProvidedStream | 100 | None | Stream | 385.08 μs | 5.025 μs | 7.521 μs | 4.8828 | 4.3945 | 4.3945 | 271565 B | -| Decrypt | 100 | None | Stream | 380.02 μs | 11.443 μs | 17.128 μs | 6.3477 | 6.3477 | 6.3477 | 230536 B | -| DecryptToProvidedStream | 100 | None | Stream | 304.54 μs | 8.678 μs | 12.989 μs | 2.9297 | 2.9297 | 2.9297 | 118897 B | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,172.02 μs** | **19.488 μs** | **29.169 μs** | **13.6719** | **11.7188** | **9.7656** | **1379452 B** | -| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | Newtonsoft | 1,153.94 μs | 12.008 μs | 17.602 μs | 11.7188 | 9.7656 | 9.7656 | 1124251 B | -| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **995.73 μs** | **27.736 μs** | **40.655 μs** | **21.4844** | **21.4844** | **21.4844** | **766965 B** | -| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | SystemTextJson | 892.93 μs | 16.896 μs | 25.288 μs | 17.5781 | 17.5781 | 17.5781 | 801458 B | -| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **Stream** | **770.06 μs** | **11.317 μs** | **16.939 μs** | **10.7422** | **10.7422** | **10.7422** | **520929 B** | -| EncryptToProvidedStream | 100 | Brotli | Stream | 606.21 μs | 9.107 μs | 13.631 μs | 2.9297 | 2.9297 | 2.9297 | 222081 B | -| Decrypt | 100 | Brotli | Stream | 537.74 μs | 8.192 μs | 12.007 μs | 6.3477 | 6.3477 | 6.3477 | 230938 B | -| DecryptToProvidedStream | 100 | Brotli | Stream | 464.80 μs | 4.408 μs | 6.461 μs | 3.4180 | 3.4180 | 3.4180 | 119300 B | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.53 μs** | **0.511 μs** | **0.733 μs** | **22.29 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.224 μs | 0.322 μs | 26.23 μs | 0.1526 | 0.0305 | - | 41440 B | +| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.33 μs** | **0.137 μs** | **0.201 μs** | **14.32 μs** | **0.0916** | **0.0153** | **-** | **22904 B** | +| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | SystemTextJson | 14.54 μs | 0.124 μs | 0.186 μs | 14.52 μs | 0.0610 | 0.0305 | - | 21448 B | +| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **Stream** | **12.85 μs** | **0.095 μs** | **0.143 μs** | **12.84 μs** | **0.0610** | **0.0153** | **-** | **17528 B** | +| EncryptToProvidedStream | 1 | None | Stream | 13.00 μs | 0.096 μs | 0.141 μs | 12.98 μs | 0.0458 | 0.0153 | - | 11392 B | +| Decrypt | 1 | None | Stream | 13.01 μs | 0.152 μs | 0.228 μs | 13.05 μs | 0.0458 | 0.0153 | - | 12672 B | +| DecryptToProvidedStream | 1 | None | Stream | 13.48 μs | 0.132 μs | 0.197 μs | 13.45 μs | 0.0458 | 0.0153 | - | 11504 B | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **27.94 μs** | **0.226 μs** | **0.338 μs** | **27.96 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | Newtonsoft | 33.49 μs | 0.910 μs | 1.335 μs | 33.99 μs | 0.1221 | - | - | 41064 B | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **20.92 μs** | **0.136 μs** | **0.199 μs** | **20.95 μs** | **0.0610** | **-** | **-** | **21952 B** | +| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | SystemTextJson | 20.53 μs | 0.136 μs | 0.200 μs | 20.52 μs | 0.0610 | 0.0305 | - | 20488 B | +| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **Stream** | **21.15 μs** | **1.037 μs** | **1.521 μs** | **20.52 μs** | **0.0610** | **0.0305** | **-** | **16584 B** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 20.57 μs | 0.213 μs | 0.292 μs | 20.57 μs | 0.0305 | - | - | 11672 B | +| Decrypt | 1 | Brotli | Stream | 21.14 μs | 2.212 μs | 3.311 μs | 19.46 μs | 0.0305 | - | - | 13216 B | +| DecryptToProvidedStream | 1 | Brotli | Stream | 19.60 μs | 0.439 μs | 0.600 μs | 19.52 μs | 0.0305 | - | - | 12048 B | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **84.82 μs** | **3.002 μs** | **4.208 μs** | **83.32 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | Newtonsoft | 112.98 μs | 15.294 μs | 21.934 μs | 100.38 μs | 0.6104 | 0.1221 | - | 157425 B | +| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.85 μs** | **0.868 μs** | **1.272 μs** | **41.40 μs** | **0.4272** | **0.0610** | **-** | **105345 B** | +| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | SystemTextJson | 41.79 μs | 0.501 μs | 0.718 μs | 41.64 μs | 0.3662 | 0.0610 | - | 96464 B | +| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **Stream** | **39.63 μs** | **0.658 μs** | **0.923 μs** | **39.41 μs** | **0.3052** | **0.0610** | **-** | **82928 B** | +| EncryptToProvidedStream | 10 | None | Stream | 36.59 μs | 0.272 μs | 0.399 μs | 36.57 μs | 0.1221 | - | - | 37048 B | +| Decrypt | 10 | None | Stream | 28.64 μs | 0.378 μs | 0.517 μs | 28.59 μs | 0.1221 | 0.0305 | - | 29520 B | +| DecryptToProvidedStream | 10 | None | Stream | 27.61 μs | 0.237 μs | 0.332 μs | 27.64 μs | 0.0610 | 0.0305 | - | 18416 B | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **115.28 μs** | **3.336 μs** | **4.677 μs** | **113.71 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | Newtonsoft | 118.98 μs | 1.530 μs | 2.195 μs | 118.76 μs | 0.4883 | - | - | 144849 B | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **71.40 μs** | **0.799 μs** | **1.145 μs** | **71.23 μs** | **0.2441** | **-** | **-** | **86217 B** | +| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | SystemTextJson | 73.37 μs | 7.283 μs | 10.676 μs | 67.12 μs | 0.2441 | - | - | 82201 B | +| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **Stream** | **90.10 μs** | **3.136 μs** | **4.693 μs** | **88.92 μs** | **0.2441** | **-** | **-** | **63809 B** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 97.27 μs | 1.885 μs | 2.703 μs | 97.35 μs | 0.1221 | - | - | 32465 B | +| Decrypt | 10 | Brotli | Stream | 58.48 μs | 0.956 μs | 1.372 μs | 58.59 μs | 0.1221 | 0.0610 | - | 30064 B | +| DecryptToProvidedStream | 10 | Brotli | Stream | 59.12 μs | 1.160 μs | 1.664 μs | 59.14 μs | 0.0610 | - | - | 18960 B | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,199.74 μs** | **42.805 μs** | **64.069 μs** | **1,206.48 μs** | **23.4375** | **21.4844** | **21.4844** | **1677978 B** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | Newtonsoft | 1,177.48 μs | 25.746 μs | 38.535 μs | 1,172.04 μs | 17.5781 | 15.6250 | 15.6250 | 1260228 B | +| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **SystemTextJson** | **824.48 μs** | **31.605 μs** | **47.305 μs** | **812.80 μs** | **25.3906** | **25.3906** | **25.3906** | **965259 B** | +| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | SystemTextJson | 814.40 μs | 50.865 μs | 76.132 μs | 811.34 μs | 21.4844 | 21.4844 | 21.4844 | 950333 B | +| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **Stream** | **636.72 μs** | **31.468 μs** | **47.099 μs** | **630.15 μs** | **16.6016** | **16.6016** | **16.6016** | **678066 B** | +| EncryptToProvidedStream | 100 | None | Stream | 383.33 μs | 7.441 μs | 10.671 μs | 384.69 μs | 4.3945 | 4.3945 | 4.3945 | 230133 B | +| Decrypt | 100 | None | Stream | 384.93 μs | 12.519 μs | 18.738 μs | 383.59 μs | 5.8594 | 5.8594 | 5.8594 | 230753 B | +| DecryptToProvidedStream | 100 | None | Stream | 295.19 μs | 7.094 μs | 10.618 μs | 296.11 μs | 3.4180 | 3.4180 | 3.4180 | 119116 B | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,178.06 μs** | **63.246 μs** | **94.664 μs** | **1,152.03 μs** | **13.6719** | **11.7188** | **9.7656** | **1379183 B** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | Newtonsoft | 1,175.01 μs | 41.917 μs | 61.441 μs | 1,156.01 μs | 11.7188 | 9.7656 | 9.7656 | 1124274 B | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **1,050.27 μs** | **31.128 μs** | **46.591 μs** | **1,052.70 μs** | **17.5781** | **17.5781** | **17.5781** | **766642 B** | +| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | SystemTextJson | 926.80 μs | 28.605 μs | 41.025 μs | 925.73 μs | 18.5547 | 18.5547 | 18.5547 | 801460 B | +| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **Stream** | **757.11 μs** | **19.549 μs** | **29.260 μs** | **754.55 μs** | **10.7422** | **10.7422** | **10.7422** | **479493 B** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 563.46 μs | 9.960 μs | 14.284 μs | 561.60 μs | 2.9297 | 2.9297 | 2.9297 | 180637 B | +| Decrypt | 100 | Brotli | Stream | 542.34 μs | 14.514 μs | 21.724 μs | 542.04 μs | 6.8359 | 6.8359 | 6.8359 | 231162 B | +| DecryptToProvidedStream | 100 | Brotli | Stream | 463.69 μs | 9.130 μs | 12.800 μs | 460.71 μs | 3.4180 | 3.4180 | 3.4180 | 119506 B | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs index 4c4d42a6ee..9d6713e5f0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/TestDoc.cs @@ -7,7 +7,7 @@ public partial class EncryptionBenchmark { internal class TestDoc { - public static List PathsToEncrypt { get; } = new List() { "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; + public static HashSet PathsToEncrypt { get; } = new HashSet{ "/SensitiveStr", "/SensitiveInt", "/SensitiveDict" }; [JsonProperty("id")] public string Id { get; set; } = default!; From fa0020c66bcf8d92379604d271a0b635277d81dd Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 18 Oct 2024 20:28:55 +0200 Subject: [PATCH 63/71] ~ PR based changes --- .../src/AeadAes/AeAesEncryptionProcessor.cs | 2 +- .../src/EncryptionFormatVersion.cs | 13 + .../src/EncryptionOptionsExtensions.cs | 20 + .../src/EncryptionProcessor.cs | 38 +- .../MdeEncryptionProcessor.Stable.cs | 2 +- .../MdeJObjectEncryptionProcessor.Preview.cs | 4 +- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 4 +- .../StreamProcessor.Decryptor.cs | 202 +++++----- .../StreamProcessor.Encryptor.cs | 372 ++++++++---------- 9 files changed, 309 insertions(+), 348 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs index c7eb3658d4..0822ffc0f9 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/AeadAes/AeAesEncryptionProcessor.cs @@ -81,7 +81,7 @@ internal static async Task DecryptContentAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 2) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.AeAes) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs new file mode 100644 index 0000000000..035df18c61 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionFormatVersion.cs @@ -0,0 +1,13 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + internal static class EncryptionFormatVersion + { + public const int AeAes = 2; + public const int Mde = 3; + public const int MdeWithCompression = 4; + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs index 1297652a25..09f02d71d0 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs @@ -5,6 +5,8 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom { using System; + using System.Collections.Generic; + using System.Linq; internal static class EncryptionOptionsExtensions { @@ -31,6 +33,24 @@ internal static void Validate(this EncryptionOptions options) #pragma warning restore CA2208 // Instantiate argument exceptions correctly } + if (options.PathsToEncrypt is not HashSet && options.PathsToEncrypt.Distinct().Count() != options.PathsToEncrypt.Count()) + { + throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); + } + + foreach (string path in options.PathsToEncrypt) + { + if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) + { + throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(options.PathsToEncrypt)}"); + } + + if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) + { + throw new InvalidOperationException($"{nameof(options.PathsToEncrypt)} includes a invalid path: '{path}'."); + } + } + options.CompressionOptions?.Validate(); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 93209d2450..8612eb3958 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -63,24 +63,6 @@ public static async Task EncryptAsync( return input; } - if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) - { - throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); - } - - foreach (string path in encryptionOptions.PathsToEncrypt) - { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) - { - throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); - } - - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) - { - throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); - } - } - #pragma warning disable CS0618 // Type or member is obsolete return encryptionOptions.EncryptionAlgorithm switch { @@ -113,24 +95,6 @@ public static async Task EncryptAsync( return; } - if (encryptionOptions.PathsToEncrypt is not HashSet && encryptionOptions.PathsToEncrypt.Distinct().Count() != encryptionOptions.PathsToEncrypt.Count()) - { - throw new InvalidOperationException("Duplicate paths in PathsToEncrypt passed via EncryptionOptions."); - } - - foreach (string path in encryptionOptions.PathsToEncrypt) - { - if (string.IsNullOrWhiteSpace(path) || path[0] != '/' || path.IndexOf('/', 1) != -1) - { - throw new InvalidOperationException($"Invalid path {path ?? string.Empty}, {nameof(encryptionOptions.PathsToEncrypt)}"); - } - - if (path.AsSpan(1).Equals("id".AsSpan(), StringComparison.Ordinal)) - { - throw new InvalidOperationException($"{nameof(encryptionOptions.PathsToEncrypt)} includes a invalid path: '{path}'."); - } - } - if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) { throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); @@ -250,7 +214,7 @@ public static async Task DecryptAsync( else { input.Position = 0; - throw new NotSupportedException($"Streaming mode is not supported for encryption algorithm {properties.EncryptionProperties.EncryptionAlgorithm}"); + throw new NotSupportedException($"Encryption Algorithm: {properties.EncryptionProperties.EncryptionAlgorithm} is not supported."); } #pragma warning restore CS0618 // Type or member is obsolete diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs index 659ca74c91..fc8a20f718 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Stable.cs @@ -90,7 +90,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs index 6d7b49ac04..a3b60dee57 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJObjectEncryptionProcessor.Preview.cs @@ -127,7 +127,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -141,7 +141,7 @@ internal async Task DecryptObjectAsync( #if NET8_0_OR_GREATER BrotliCompressor decompressor = null; - if (encryptionProperties.EncryptionFormatVersion == 4) + if (encryptionProperties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) { bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs index abee2721fb..b7fcda6b43 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs @@ -142,7 +142,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) + if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -157,7 +157,7 @@ internal async Task DecryptObjectAsync( #if NET8_0_OR_GREATER BrotliCompressor decompressor = null; - if (encryptionProperties.EncryptionFormatVersion == 4) + if (encryptionProperties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) { bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 442800d6e0..342fe3bebf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -39,7 +39,7 @@ internal async Task DecryptStreamAsync( { _ = diagnosticsContext; - if (properties.EncryptionFormatVersion != 3 && properties.EncryptionFormatVersion != 4) + if (properties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && properties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) { throw new NotSupportedException($"Unknown encryption format version: {properties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -75,19 +75,7 @@ internal async Task DecryptStreamAsync( long bytesConsumed = 0; // processing itself here - bytesConsumed = this.TransformDecryptBuffer( - buffer.AsSpan(0, dataSize), - isFinalBlock, - writer, - ref state, - ref isIgnoredBlock, - ref decryptPropertyName, - encryptedPaths, - pathsDecrypted, - properties, - containsCompressed, - arrayPoolManager, - encryptionKey); + bytesConsumed = TransformDecryptBuffer(buffer.AsSpan(0, dataSize)); leftOver = dataSize - (int)bytesConsumed; @@ -108,115 +96,115 @@ internal async Task DecryptStreamAsync( outputStream.Position = 0; return EncryptionProcessor.CreateDecryptionContext(pathsDecrypted, properties.DataEncryptionKeyId); - } - - private long TransformDecryptBuffer(ReadOnlySpan buffer, bool isFinalBlock, Utf8JsonWriter writer, ref JsonReaderState state, ref bool isIgnoredBlock, ref string decryptPropertyName, HashSet encryptedPaths, List pathsDecrypted, EncryptionProperties properties, bool containsCompressed, ArrayPoolManager arrayPoolManager, DataEncryptionKey encryptionKey) - { - Utf8JsonReader reader = new (buffer, isFinalBlock, state); - while (reader.Read()) + long TransformDecryptBuffer(ReadOnlySpan buffer) { - JsonTokenType tokenType = reader.TokenType; + Utf8JsonReader reader = new (buffer, isFinalBlock, state); - if (isIgnoredBlock && reader.CurrentDepth == 1 && tokenType == JsonTokenType.EndObject) + while (reader.Read()) { - isIgnoredBlock = false; - continue; - } - else if (isIgnoredBlock) - { - continue; - } + JsonTokenType tokenType = reader.TokenType; + + if (isIgnoredBlock && reader.CurrentDepth == 1 && tokenType == JsonTokenType.EndObject) + { + isIgnoredBlock = false; + continue; + } + else if (isIgnoredBlock) + { + continue; + } + + switch (tokenType) + { + case JsonTokenType.String: + if (decryptPropertyName == null) + { + writer.WriteStringValue(reader.ValueSpan); + } + else + { + this.TransformDecryptProperty( + ref reader, + writer, + decryptPropertyName, + properties, + encryptionKey, + containsCompressed, + arrayPoolManager); + + pathsDecrypted.Add(decryptPropertyName); + } - switch (tokenType) - { - case JsonTokenType.String: - if (decryptPropertyName == null) - { - writer.WriteStringValue(reader.ValueSpan); - } - else - { - this.TransformDecryptProperty( - ref reader, - writer, - decryptPropertyName, - properties, - encryptionKey, - containsCompressed, - arrayPoolManager); - - pathsDecrypted.Add(decryptPropertyName); - } - - decryptPropertyName = null; - break; - case JsonTokenType.Number: - decryptPropertyName = null; - writer.WriteRawValue(reader.ValueSpan); - break; - case JsonTokenType.None: - decryptPropertyName = null; - break; - case JsonTokenType.StartObject: - decryptPropertyName = null; - writer.WriteStartObject(); - break; - case JsonTokenType.EndObject: - decryptPropertyName = null; - writer.WriteEndObject(); - break; - case JsonTokenType.StartArray: - decryptPropertyName = null; - writer.WriteStartArray(); - break; - case JsonTokenType.EndArray: - decryptPropertyName = null; - writer.WriteEndArray(); - break; - case JsonTokenType.PropertyName: - string propertyName = "/" + reader.GetString(); - if (encryptedPaths.Contains(propertyName)) - { - decryptPropertyName = propertyName; - } - else if (propertyName == StreamProcessor.EncryptionPropertiesPath) - { - if (!reader.TrySkip()) + decryptPropertyName = null; + break; + case JsonTokenType.Number: + decryptPropertyName = null; + writer.WriteRawValue(reader.ValueSpan); + break; + case JsonTokenType.None: + decryptPropertyName = null; + break; + case JsonTokenType.StartObject: + decryptPropertyName = null; + writer.WriteStartObject(); + break; + case JsonTokenType.EndObject: + decryptPropertyName = null; + writer.WriteEndObject(); + break; + case JsonTokenType.StartArray: + decryptPropertyName = null; + writer.WriteStartArray(); + break; + case JsonTokenType.EndArray: + decryptPropertyName = null; + writer.WriteEndArray(); + break; + case JsonTokenType.PropertyName: + string propertyName = "/" + reader.GetString(); + if (encryptedPaths.Contains(propertyName)) { - isIgnoredBlock = true; + decryptPropertyName = propertyName; + } + else if (propertyName == StreamProcessor.EncryptionPropertiesPath) + { + if (!reader.TrySkip()) + { + isIgnoredBlock = true; + } + + break; } + writer.WritePropertyName(reader.ValueSpan); break; - } - - writer.WritePropertyName(reader.ValueSpan); - break; - case JsonTokenType.Comment: - break; - case JsonTokenType.True: - decryptPropertyName = null; - writer.WriteBooleanValue(true); - break; - case JsonTokenType.False: - decryptPropertyName = null; - writer.WriteBooleanValue(false); - break; - case JsonTokenType.Null: - decryptPropertyName = null; - writer.WriteNullValue(); - break; + case JsonTokenType.Comment: + break; + case JsonTokenType.True: + decryptPropertyName = null; + writer.WriteBooleanValue(true); + break; + case JsonTokenType.False: + decryptPropertyName = null; + writer.WriteBooleanValue(false); + break; + case JsonTokenType.Null: + decryptPropertyName = null; + writer.WriteNullValue(); + break; + } } - } - state = reader.CurrentState; - return reader.BytesConsumed; + state = reader.CurrentState; + return reader.BytesConsumed; + } } private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, bool containsCompressed, ArrayPoolManager arrayPoolManager) { BrotliCompressor decompressor = null; - if (properties.EncryptionFormatVersion == 4) + if (properties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) { if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) { diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index 3fd4b0e4f9..a3980ae56a 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -60,21 +60,7 @@ internal async Task EncryptStreamAsync( isFinalBlock = dataSize == 0; long bytesConsumed = 0; - bytesConsumed = this.TransformEncryptBuffer( - buffer.AsSpan(0, dataSize), - isFinalBlock, - writer, - ref encryptionPayloadWriter, - ref bufferWriter, - ref state, - ref encryptPropertyName, - pathsToEncrypt, - pathsEncrypted, - compressedPaths, - compressor, - arrayPoolManager, - encryptionKey, - encryptionOptions); + bytesConsumed = TransformEncryptBuffer(buffer.AsSpan(0, dataSize)); leftOver = dataSize - (int)bytesConsumed; @@ -108,178 +94,186 @@ internal async Task EncryptStreamAsync( writer.Flush(); outputStream.Position = 0; - } - - private long TransformEncryptBuffer( - ReadOnlySpan buffer, - bool isFinalBlock, - Utf8JsonWriter writer, - ref Utf8JsonWriter encryptionPayloadWriter, - ref RentArrayBufferWriter bufferWriter, - ref JsonReaderState state, - ref string encryptPropertyName, - HashSet pathsToEncrypt, - List pathsEncrypted, - Dictionary compressedPaths, - BrotliCompressor compressor, - ArrayPoolManager arrayPoolManager, - DataEncryptionKey encryptionKey, - EncryptionOptions encryptionOptions) - { - Utf8JsonReader reader = new (buffer, isFinalBlock, state); - while (reader.Read()) + long TransformEncryptBuffer(ReadOnlySpan buffer) { - Utf8JsonWriter currentWriter = encryptionPayloadWriter ?? writer; + Utf8JsonReader reader = new (buffer, isFinalBlock, state); - JsonTokenType tokenType = reader.TokenType; - - switch (tokenType) + while (reader.Read()) { - case JsonTokenType.None: - break; - case JsonTokenType.StartObject: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); - encryptionPayloadWriter.WriteStartObject(); - } - else - { - currentWriter.WriteStartObject(); - } - - break; - case JsonTokenType.EndObject: - if (reader.CurrentDepth == 0) - { - continue; - } - - currentWriter.WriteEndObject(); - if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) - { - currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Object, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - writer.WriteBase64StringValue(encryptedBytes); - - encryptPropertyName = null; - encryptionPayloadWriter.Dispose(); - encryptionPayloadWriter = null; - bufferWriter.Dispose(); - bufferWriter = null; - } - - break; - case JsonTokenType.StartArray: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); - encryptionPayloadWriter.WriteStartArray(); - } - else - { - currentWriter.WriteStartArray(); - } - - break; - case JsonTokenType.EndArray: - currentWriter.WriteEndArray(); - if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) - { - currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Array, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - writer.WriteBase64StringValue(encryptedBytes); - - encryptPropertyName = null; - encryptionPayloadWriter.Dispose(); - encryptionPayloadWriter = null; - bufferWriter.Dispose(); - bufferWriter = null; - } - - break; - case JsonTokenType.PropertyName: - string propertyName = "/" + reader.GetString(); - if (pathsToEncrypt.Contains(propertyName)) - { - encryptPropertyName = propertyName; - } - - currentWriter.WritePropertyName(reader.ValueSpan); - break; - case JsonTokenType.Comment: - currentWriter.WriteCommentValue(reader.ValueSpan); - break; - case JsonTokenType.String: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); - int length = reader.CopyString(bytes); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.String, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteStringValue(reader.ValueSpan); - } - - break; - case JsonTokenType.Number: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, typeMarker, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteRawValue(reader.ValueSpan, true); - } - - break; - case JsonTokenType.True: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - (byte[] bytes, int length) = Serialize(true, arrayPoolManager); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteBooleanValue(true); - } - - break; - case JsonTokenType.False: - if (encryptPropertyName != null && encryptionPayloadWriter == null) - { - (byte[] bytes, int length) = Serialize(false, arrayPoolManager); - ReadOnlySpan encryptedBytes = this.TransformEncryptPayload(bytes, length, TypeMarker.Boolean, encryptPropertyName, encryptionOptions, encryptionKey, pathsEncrypted, compressedPaths, compressor, arrayPoolManager); - currentWriter.WriteBase64StringValue(encryptedBytes); - encryptPropertyName = null; - } - else - { - currentWriter.WriteBooleanValue(false); - } - - break; - case JsonTokenType.Null: - currentWriter.WriteNullValue(); - break; + Utf8JsonWriter currentWriter = encryptionPayloadWriter ?? writer; + + JsonTokenType tokenType = reader.TokenType; + + switch (tokenType) + { + case JsonTokenType.None: + break; + case JsonTokenType.StartObject: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartObject(); + } + else + { + currentWriter.WriteStartObject(); + } + + break; + case JsonTokenType.EndObject: + if (reader.CurrentDepth == 0) + { + continue; + } + + currentWriter.WriteEndObject(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Object); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; +#pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct + encryptionPayloadWriter.Dispose(); +#pragma warning restore VSTHRD103 // Call async methods when in an async method + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.StartArray: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + bufferWriter = new RentArrayBufferWriter(); + encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + encryptionPayloadWriter.WriteStartArray(); + } + else + { + currentWriter.WriteStartArray(); + } + + break; + case JsonTokenType.EndArray: + currentWriter.WriteEndArray(); + if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) + { + currentWriter.Flush(); + (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Array); + writer.WriteBase64StringValue(encryptedBytes); + + encryptPropertyName = null; +#pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct + encryptionPayloadWriter.Dispose(); +#pragma warning restore VSTHRD103 // Call async methods when in an async method + encryptionPayloadWriter = null; + bufferWriter.Dispose(); + bufferWriter = null; + } + + break; + case JsonTokenType.PropertyName: + string propertyName = "/" + reader.GetString(); + if (pathsToEncrypt.Contains(propertyName)) + { + encryptPropertyName = propertyName; + } + + currentWriter.WritePropertyName(reader.ValueSpan); + break; + case JsonTokenType.Comment: + currentWriter.WriteCommentValue(reader.ValueSpan); + break; + case JsonTokenType.String: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + byte[] bytes = arrayPoolManager.Rent(reader.ValueSpan.Length); + int length = reader.CopyString(bytes); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.String); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteStringValue(reader.ValueSpan); + } + + break; + case JsonTokenType.Number: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (TypeMarker typeMarker, byte[] bytes, int length) = SerializeNumber(reader.ValueSpan, arrayPoolManager); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, typeMarker); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteRawValue(reader.ValueSpan, true); + } + + break; + case JsonTokenType.True: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(true, arrayPoolManager); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Boolean); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(true); + } + + break; + case JsonTokenType.False: + if (encryptPropertyName != null && encryptionPayloadWriter == null) + { + (byte[] bytes, int length) = Serialize(false, arrayPoolManager); + ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Boolean); + currentWriter.WriteBase64StringValue(encryptedBytes); + encryptPropertyName = null; + } + else + { + currentWriter.WriteBooleanValue(false); + } + + break; + case JsonTokenType.Null: + currentWriter.WriteNullValue(); + break; + } } + + state = reader.CurrentState; + return reader.BytesConsumed; } - state = reader.CurrentState; - return reader.BytesConsumed; + ReadOnlySpan TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker) + { + byte[] processedBytes = payload; + int processedBytesLength = payloadSize; + + if (compressor != null && payloadSize >= encryptionOptions.CompressionOptions.MinimalCompressedLength) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(payloadSize)); + processedBytesLength = compressor.Compress(compressedPaths, encryptPropertyName, processedBytes, payloadSize, compressedBytes); + processedBytes = compressedBytes; + } + + (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); + + pathsEncrypted.Add(encryptPropertyName); + return encryptedBytes.AsSpan(0, encryptedBytesCount); + } } private static (byte[] buffer, int length) Serialize(bool value, ArrayPoolManager arrayPoolManager) @@ -324,24 +318,6 @@ private static (TypeMarker typeMarker, byte[] buffer, int length) Serialize(doub return (TypeMarker.Double, buffer, length); } - - private ReadOnlySpan TransformEncryptPayload(byte[] payload, int payloadSize, TypeMarker typeMarker, string encryptName, EncryptionOptions options, DataEncryptionKey encryptionKey, List pathsEncrypted, Dictionary pathsCompressed, BrotliCompressor compressor, ArrayPoolManager arrayPoolManager) - { - byte[] processedBytes = payload; - int processedBytesLength = payloadSize; - - if (compressor != null && payloadSize >= options.CompressionOptions.MinimalCompressedLength) - { - byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(payloadSize)); - processedBytesLength = compressor.Compress(pathsCompressed, encryptName, processedBytes, payloadSize, compressedBytes); - processedBytes = compressedBytes; - } - - (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); - - pathsEncrypted.Add(encryptName); - return encryptedBytes.AsSpan(0, encryptedBytesCount); - } } } #endif \ No newline at end of file From c98f2bf7a5082f1863d91dff61b8082d054c0739 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 14:48:27 +0200 Subject: [PATCH 64/71] - drop JsonNode serializer/deserializer --- .../src/EncryptionOptions.cs | 6 - .../src/EncryptionProcessor.cs | 90 ------- .../JsonNodeSqlSerializer.Preview.cs | 117 --------- .../MdeEncryptionProcessor.Preview.cs | 17 -- .../MdeJsonNodeEncryptionProcessor.Preview.cs | 224 ------------------ .../SystemTextJson/JsonBytes.cs | 34 --- .../SystemTextJson/JsonBytesConverter.cs | 26 -- .../EncryptionBenchmark.cs | 2 +- .../MdeEncryptionProcessorTests.cs | 101 -------- .../Transformation/JsonBytesConverterTests.cs | 40 ---- .../Transformation/JsonBytesTests.cs | 33 --- .../JsonNodeSqlSerializerTests.cs | 171 ------------- 12 files changed, 1 insertion(+), 860 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index af8ab293e9..525a8182c3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -17,12 +17,6 @@ public enum JsonProcessor Newtonsoft, #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - /// - /// System.Text.Json - /// - /// Available with .NET8.0 package only. - SystemTextJson, - /// /// Ut8JsonReader/Writer /// diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 8612eb3958..7bf05fb930 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -10,10 +10,6 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom using System.IO; using System.Linq; using System.Text; -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - using System.Text.Json; - using System.Text.Json.Nodes; -#endif using System.Threading; using System.Threading.Tasks; using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; @@ -33,7 +29,6 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); #endif @@ -158,7 +153,6 @@ public static async Task EncryptAsync( { JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") @@ -229,43 +223,6 @@ public static async Task DecryptAsync( } #endif -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( - Stream input, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - if (input == null) - { - return (input, null); - } - - Debug.Assert(input.CanSeek); - Debug.Assert(encryptor != null); - Debug.Assert(diagnosticsContext != null); - - JsonNode document = await JsonNode.ParseAsync(input, cancellationToken: cancellationToken); - - (JsonNode decryptedDocument, DecryptionContext context) = await DecryptAsync(document, encryptor, diagnosticsContext, cancellationToken); - if (context == null) - { - input.Position = 0; - return (input, null); - } - - await input.DisposeAsync(); - - MemoryStream ms = new (); - Utf8JsonWriter writer = new (ms, EncryptionProcessor.JsonWriterOptions); - - System.Text.Json.JsonSerializer.Serialize(writer, decryptedDocument); - - ms.Position = 0; - return (ms, context); - } -#endif - #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( Stream input, @@ -327,53 +284,6 @@ public static async Task DecryptAsync( return (document, decryptionContext); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(JsonNode, DecryptionContext)> DecryptAsync( - JsonNode document, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - Debug.Assert(document != null); - - Debug.Assert(encryptor != null); - - if (!document.AsObject().TryGetPropertyValue(Constants.EncryptedInfo, out JsonNode encryptionPropertiesNode)) - { - return (document, null); - } - - EncryptionProperties encryptionProperties; - try - { - encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(encryptionPropertiesNode); - } - catch (Exception) - { - return (document, null); - } - - DecryptionContext decryptionContext = await DecryptInternalAsync(encryptor, diagnosticsContext, document, encryptionProperties, cancellationToken); - - return (document, decryptionContext); - } - - private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JsonNode itemNode, EncryptionProperties encryptionProperties, CancellationToken cancellationToken) - { - DecryptionContext decryptionContext = encryptionProperties.EncryptionAlgorithm switch - { - CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized => await MdeEncryptionProcessor.DecryptObjectAsync( - itemNode, - encryptor, - encryptionProperties, - diagnosticsContext, - cancellationToken), - _ => throw new NotSupportedException($"Encryption Algorithm : {encryptionProperties.EncryptionAlgorithm} is not supported."), - }; - return decryptionContext; - } -#endif - private static async Task DecryptInternalAsync(Encryptor encryptor, CosmosDiagnosticsContext diagnosticsContext, JObject itemJObj, JObject encryptionPropertiesJObj, CancellationToken cancellationToken) { EncryptionProperties encryptionProperties = encryptionPropertiesJObj.ToObject(); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs deleted file mode 100644 index c96b76a731..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/JsonNodeSqlSerializer.Preview.cs +++ /dev/null @@ -1,117 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation -{ - using System; - using System.Diagnostics; - using System.Text.Json; - using System.Text.Json.Nodes; - using Microsoft.Data.Encryption.Cryptography.Serializers; - - internal class JsonNodeSqlSerializer - { - private static readonly SqlBitSerializer SqlBoolSerializer = new (); - private static readonly SqlFloatSerializer SqlDoubleSerializer = new (); - private static readonly SqlBigIntSerializer SqlLongSerializer = new (); - - // UTF-8 encoding. - private static readonly SqlVarCharSerializer SqlVarCharSerializer = new (size: -1, codePageCharacterEncoding: 65001); - -#pragma warning disable SA1101 // Prefix local calls with this - false positive on SerializeFixed - internal virtual (TypeMarker typeMarker, byte[] serializedBytes, int serializedBytesCount) Serialize(JsonNode propertyValue, ArrayPoolManager arrayPoolManager) - { - byte[] buffer; - int length; - - if (propertyValue == null) - { - return (TypeMarker.Null, null, -1); - } - - switch (propertyValue.GetValueKind()) - { - case JsonValueKind.Undefined: - Debug.Assert(false, "Undefined value cannot be in the JSON"); - return (default, null, -1); - case JsonValueKind.Null: - Debug.Assert(false, "Null type should have been handled by caller"); - return (TypeMarker.Null, null, -1); - case JsonValueKind.True: - (buffer, length) = SerializeFixed(SqlBoolSerializer, true); - return (TypeMarker.Boolean, buffer, length); - case JsonValueKind.False: - (buffer, length) = SerializeFixed(SqlBoolSerializer, false); - return (TypeMarker.Boolean, buffer, length); - case JsonValueKind.Number: - if (long.TryParse(propertyValue.ToJsonString(), out long longValue)) - { - (buffer, length) = SerializeFixed(SqlLongSerializer, longValue); - return (TypeMarker.Long, buffer, length); - } - else if (double.TryParse(propertyValue.ToJsonString(), out double doubleValue)) - { - (buffer, length) = SerializeFixed(SqlDoubleSerializer, doubleValue); - return (TypeMarker.Double, buffer, length); - } - else - { - throw new InvalidOperationException("Unsupported Number type"); - } - - case JsonValueKind.String: - (buffer, length) = SerializeString(propertyValue.GetValue()); - return (TypeMarker.String, buffer, length); - case JsonValueKind.Array: - (buffer, length) = SerializeString(propertyValue.ToJsonString()); - return (TypeMarker.Array, buffer, length); - case JsonValueKind.Object: - (buffer, length) = SerializeString(propertyValue.ToJsonString()); - return (TypeMarker.Object, buffer, length); - default: - throw new InvalidOperationException($" Invalid or Unsupported Data Type Passed : {propertyValue.GetValueKind()}"); - } - - (byte[], int) SerializeFixed(IFixedSizeSerializer serializer, T value) - { - byte[] buffer = arrayPoolManager.Rent(serializer.GetSerializedMaxByteCount()); - int length = serializer.Serialize(value, buffer); - return (buffer, length); - } - - (byte[], int) SerializeString(string value) - { - byte[] buffer = arrayPoolManager.Rent(SqlVarCharSerializer.GetSerializedMaxByteCount(value.Length)); - int length = SqlVarCharSerializer.Serialize(value, buffer); - return (buffer, length); - } - } - - internal virtual JsonNode Deserialize( - TypeMarker typeMarker, - ReadOnlySpan serializedBytes) - { - switch (typeMarker) - { - case TypeMarker.Boolean: - return JsonValue.Create(SqlBoolSerializer.Deserialize(serializedBytes)); - case TypeMarker.Double: - return JsonValue.Create(SqlDoubleSerializer.Deserialize(serializedBytes)); - case TypeMarker.Long: - return JsonValue.Create(SqlLongSerializer.Deserialize(serializedBytes)); - case TypeMarker.String: - return JsonValue.Create(SqlVarCharSerializer.Deserialize(serializedBytes)); - case TypeMarker.Array: - return JsonNode.Parse(serializedBytes); - case TypeMarker.Object: - return JsonNode.Parse(serializedBytes); - default: - Debug.Fail($"Unexpected type marker {typeMarker}"); - return null; - } - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index 2dc3eb7e10..389a709207 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -20,8 +20,6 @@ internal class MdeEncryptionProcessor internal MdeJObjectEncryptionProcessor JObjectEncryptionProcessor { get; set; } = new MdeJObjectEncryptionProcessor(); #if NET8_0_OR_GREATER - internal MdeJsonNodeEncryptionProcessor JsonNodeEncryptionProcessor { get; set; } = new MdeJsonNodeEncryptionProcessor(); - internal StreamProcessor StreamProcessor { get; set; } = new StreamProcessor(); #endif @@ -36,9 +34,6 @@ public async Task EncryptAsync( { case JsonProcessor.Newtonsoft: return await this.JObjectEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); - - case JsonProcessor.SystemTextJson: - return await this.JsonNodeEncryptionProcessor.EncryptAsync(input, encryptor, encryptionOptions, token); case JsonProcessor.Stream: MemoryStream ms = new (); await this.StreamProcessor.EncryptStreamAsync(input, ms, encryptor, encryptionOptions, token); @@ -65,18 +60,6 @@ internal async Task DecryptObjectAsync( { return await this.JObjectEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); } - -#if NET8_0_OR_GREATER - internal async Task DecryptObjectAsync( - JsonNode document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - return await this.JsonNodeEncryptionProcessor.DecryptObjectAsync(document, encryptor, encryptionProperties, diagnosticsContext, cancellationToken); - } -#endif } } #endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs deleted file mode 100644 index b7fcda6b43..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeJsonNodeEncryptionProcessor.Preview.cs +++ /dev/null @@ -1,224 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation -{ - using System; - using System.Collections.Generic; - using System.IO; - using System.Linq; - using System.Text.Json; - using System.Text.Json.Nodes; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - - internal class MdeJsonNodeEncryptionProcessor - { - private readonly JsonWriterOptions jsonWriterOptions = new () { SkipValidation = true }; - - internal JsonNodeSqlSerializer Serializer { get; set; } = new JsonNodeSqlSerializer(); - - internal MdeEncryptor Encryptor { get; set; } = new MdeEncryptor(); - - internal JsonSerializerOptions JsonSerializerOptions { get; set; } - - public MdeJsonNodeEncryptionProcessor() - { - this.JsonSerializerOptions = new JsonSerializerOptions(); - this.JsonSerializerOptions.Converters.Add(new JsonBytesConverter()); - } - - public async Task EncryptAsync( - Stream input, - Encryptor encryptor, - EncryptionOptions encryptionOptions, - CancellationToken token) - { - JsonNode itemJObj = JsonNode.Parse(input); - - Stream result = await this.EncryptAsync(itemJObj, encryptor, encryptionOptions, token); - - await input.DisposeAsync(); - return result; - } - - public async Task EncryptAsync( - JsonNode document, - Encryptor encryptor, - EncryptionOptions encryptionOptions, - CancellationToken token) - { - List pathsEncrypted = new (); - TypeMarker typeMarker; - - using ArrayPoolManager arrayPoolManager = new (); - - JsonObject itemObj = document.AsObject(); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); - - bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; - -#if NET8_0_OR_GREATER - BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli - ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; -#endif - Dictionary compressedPaths = new (); - - foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) - { -#if NET8_0_OR_GREATER - string propertyName = pathToEncrypt[1..]; -#else - string propertyName = pathToEncrypt.Substring(1); -#endif - if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) - { - continue; - } - - if (propertyValue == null || propertyValue.GetValueKind() == JsonValueKind.Null) - { - continue; - } - - byte[] processedBytes = null; - (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - - if (processedBytes == null) - { - continue; - } - -#if NET8_0_OR_GREATER - if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) - { - byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); - processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); - processedBytes = compressedBytes; - } -#endif - (byte[] encryptedBytes, int encryptedBytesCount) = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength, arrayPoolManager); - - itemObj[propertyName] = JsonValue.Create(new Memory(encryptedBytes, 0, encryptedBytesCount)); - pathsEncrypted.Add(pathToEncrypt); - } - -#if NET8_0_OR_GREATER - compressor?.Dispose(); -#endif - EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: compressionEnabled ? 4 : 3, - encryptionOptions.EncryptionAlgorithm, - encryptionOptions.DataEncryptionKeyId, - encryptedData: null, - pathsEncrypted, - encryptionOptions.CompressionOptions.Algorithm, - compressedPaths); - - JsonNode propertiesNode = JsonSerializer.SerializeToNode(encryptionProperties); - - itemObj.Add(Constants.EncryptedInfo, propertiesNode); - - MemoryStream ms = new (); - Utf8JsonWriter writer = new (ms, this.jsonWriterOptions); - - JsonSerializer.Serialize(writer, document); - - ms.Position = 0; - return ms; - } - - internal async Task DecryptObjectAsync( - JsonNode document, - Encryptor encryptor, - EncryptionProperties encryptionProperties, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - _ = diagnosticsContext; - - if (encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.Mde && encryptionProperties.EncryptionFormatVersion != EncryptionFormatVersion.MdeWithCompression) - { - throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); - } - - using ArrayPoolManager arrayPoolManager = new (); - - DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); - - List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); - - JsonObject itemObj = document.AsObject(); - -#if NET8_0_OR_GREATER - BrotliCompressor decompressor = null; - if (encryptionProperties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) - { - bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; - if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) - { - throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}"); - } - - if (containsCompressed) - { - decompressor = new (); - } - } -#endif - - foreach (string path in encryptionProperties.EncryptedPaths) - { - string propertyName = path[1..]; - - if (!itemObj.TryGetPropertyValue(propertyName, out JsonNode propertyValue)) - { - // malformed document, such record shouldn't be there at all - continue; - } - - // can we get to internal JsonNode buffers to avoid string allocation here? - string base64String = propertyValue.GetValue(); - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent((base64String.Length * sizeof(char) * 3 / 4) + 4); - if (!Convert.TryFromBase64Chars(base64String, cipherTextWithTypeMarker, out int cipherTextLength)) - { - continue; - } - - (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, cipherTextLength, arrayPoolManager); - -#if NET8_0_OR_GREATER - if (decompressor != null) - { - if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) - { - byte[] buffer = arrayPoolManager.Rent(decompressedSize); - processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); - - bytes = buffer; - } - } -#endif - document[propertyName] = this.Serializer.Deserialize( - (TypeMarker)cipherTextWithTypeMarker[0], - bytes.AsSpan(0, processedBytes)); - - pathsDecrypted.Add(path); - } - - DecryptionContext decryptionContext = EncryptionProcessor.CreateDecryptionContext( - pathsDecrypted, - encryptionProperties.DataEncryptionKeyId); - - itemObj.Remove(Constants.EncryptedInfo); - return decryptionContext; - } - } -} - -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs deleted file mode 100644 index a8bc77f75b..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytes.cs +++ /dev/null @@ -1,34 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - - internal class JsonBytes - { - internal byte[] Bytes { get; private set; } - - internal int Offset { get; private set; } - - internal int Length { get; private set; } - - public JsonBytes(byte[] bytes, int offset, int length) - { - ArgumentNullException.ThrowIfNull(bytes); - ArgumentOutOfRangeException.ThrowIfNegative(offset); - ArgumentOutOfRangeException.ThrowIfNegative(length); - if (bytes.Length < offset + length) - { - throw new ArgumentOutOfRangeException(null, "Offset + Length > bytes.Length"); - } - - this.Bytes = bytes; - this.Offset = offset; - this.Length = length; - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs deleted file mode 100644 index d9048375c0..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/SystemTextJson/JsonBytesConverter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson -{ - using System; - using System.Text.Json; - using System.Text.Json.Serialization; - - internal class JsonBytesConverter : JsonConverter - { - public override JsonBytes Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException(); - } - - public override void Write(Utf8JsonWriter writer, JsonBytes value, JsonSerializerOptions options) - { - writer.WriteBase64StringValue(value.Bytes.AsSpan(value.Offset, value.Length)); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index c851750c53..b436edd547 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -36,7 +36,7 @@ public partial class EncryptionBenchmark public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [Params(JsonProcessor.Newtonsoft, JsonProcessor.SystemTextJson, JsonProcessor.Stream)] + [Params(JsonProcessor.Newtonsoft, JsonProcessor.Stream)] #else [Params(JsonProcessor.Newtonsoft)] #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 9e05872546..46fde1350c 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -68,7 +68,6 @@ public static void ClassInitialize(TestContext testContext) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) @@ -109,7 +108,6 @@ public async Task InvalidPathToEncrypt(JsonProcessor jsonProcessor) [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif public async Task DuplicatePathToEncrypt(JsonProcessor jsonProcessor) @@ -162,30 +160,6 @@ public async Task EncryptDecryptPropertyWithNullValue_VerifyByNewtonsoft(Encrypt decryptionContext); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [TestMethod] - [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task EncryptDecryptPropertyWithNullValue_VerifyBySystemText(EncryptionOptions encryptionOptions) - { - TestDoc testDoc = TestDoc.Create(); - testDoc.SensitiveStr = null; - - JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); - - (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( - encryptedDoc, - mockEncryptor.Object, - new CosmosDiagnosticsContext(), - CancellationToken.None); - - VerifyDecryptionSucceeded( - decryptedDoc, - testDoc, - TestDoc.PathsToEncrypt.Count, - decryptionContext); - } -#endif - [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] public async Task ValidateEncryptDecryptDocument_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) @@ -207,29 +181,6 @@ public async Task ValidateEncryptDecryptDocument_VerifyByNewtonsoft(EncryptionOp decryptionContext); } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [TestMethod] - [DynamicData(nameof(EncryptionOptionsCombinations))] - public async Task ValidateEncryptDecryptDocument_VerifyBySystemText(EncryptionOptions encryptionOptions) - { - TestDoc testDoc = TestDoc.Create(); - - JsonNode encryptedDoc = await VerifyEncryptionSucceededSystemText(testDoc, encryptionOptions); - - (JsonNode decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( - encryptedDoc, - mockEncryptor.Object, - new CosmosDiagnosticsContext(), - CancellationToken.None); - - VerifyDecryptionSucceeded( - decryptedDoc, - testDoc, - TestDoc.PathsToEncrypt.Count, - decryptionContext); - } -#endif - [TestMethod] [DynamicData(nameof(EncryptionOptionsCombinations))] public async Task ValidateDecryptByNewtonsoftStream_VerifyByNewtonsoft(EncryptionOptions encryptionOptions) @@ -319,7 +270,6 @@ public async Task ValidateDecryptBySystemTextStream_VerifyBySystemText(Encryptio [TestMethod] [DataRow(JsonProcessor.Newtonsoft)] #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - [DataRow(JsonProcessor.SystemTextJson)] [DataRow(JsonProcessor.Stream)] #endif public async Task DecryptStreamWithoutEncryptedProperty(JsonProcessor processor) @@ -389,53 +339,6 @@ private static async Task VerifyEncryptionSucceededNewtonsoft(TestDoc t return encryptedDoc; } -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static async Task VerifyEncryptionSucceededSystemText(TestDoc testDoc, EncryptionOptions encryptionOptions) - { - Stream encryptedStream = await EncryptionProcessor.EncryptAsync( - testDoc.ToStream(), - mockEncryptor.Object, - encryptionOptions, - new CosmosDiagnosticsContext(), - CancellationToken.None); - - JsonNode encryptedDoc = JsonNode.Parse(encryptedStream, documentOptions: new System.Text.Json.JsonDocumentOptions() { }); - - Assert.AreEqual(testDoc.Id, encryptedDoc["id"].GetValue()); - Assert.AreEqual(testDoc.PK, encryptedDoc[nameof(TestDoc.PK)].GetValue()); - Assert.AreEqual(testDoc.NonSensitive, encryptedDoc[nameof(TestDoc.NonSensitive)].GetValue()); - Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); - Assert.AreNotEqual(testDoc.SensitiveInt, encryptedDoc[nameof(TestDoc.SensitiveInt)].GetValue()); // not equal since value is encrypted - - JsonNode eiJProp = encryptedDoc[Constants.EncryptedInfo]; - Assert.IsNotNull(eiJProp); - Assert.IsNotNull(eiJProp.AsObject()); - EncryptionProperties encryptionProperties = System.Text.Json.JsonSerializer.Deserialize(eiJProp); - - Assert.IsNotNull(encryptionProperties); - Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); - - int expectedVersion = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None ? 4 : 3; - Assert.AreEqual(expectedVersion, encryptionProperties.EncryptionFormatVersion); - Assert.IsNull(encryptionProperties.EncryptedData); - Assert.IsNotNull(encryptionProperties.EncryptedPaths); - - if (testDoc.SensitiveStr == null) - { - AssertNullableValueKind(null, encryptedDoc, nameof(TestDoc.SensitiveStr)); // since null value is not encrypted - Assert.AreEqual(TestDoc.PathsToEncrypt.Count - 1, encryptionProperties.EncryptedPaths.Count()); - } - else - { - Assert.IsNotNull(encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); - Assert.AreNotEqual(testDoc.SensitiveStr, encryptedDoc[nameof(TestDoc.SensitiveStr)].GetValue()); // not equal since value is encrypted - Assert.AreEqual(TestDoc.PathsToEncrypt.Count, encryptionProperties.EncryptedPaths.Count()); - } - - return encryptedDoc; - } -#endif - private static void VerifyDecryptionSucceeded( JObject decryptedDoc, TestDoc expectedDoc, @@ -540,13 +443,10 @@ private static EncryptionOptions CreateEncryptionOptions(JsonProcessor processor public static IEnumerable EncryptionOptionsCombinations => new[] { new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.None, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, - new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.Fastest) }, new object[] { CreateEncryptionOptions(JsonProcessor.Newtonsoft, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, - new object[] { CreateEncryptionOptions(JsonProcessor.SystemTextJson, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, new object[] { CreateEncryptionOptions(JsonProcessor.Stream, CompressionOptions.CompressionAlgorithm.Brotli, CompressionLevel.NoCompression) }, #endif }; @@ -559,7 +459,6 @@ public static IEnumerable EncryptionOptionsStreamTestCombinations { yield return new object[] { encryptionOptions[0], JsonProcessor.Newtonsoft }; #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - yield return new object[] { encryptionOptions[0], JsonProcessor.SystemTextJson }; yield return new object[] { encryptionOptions[0], JsonProcessor.Stream }; #endif } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs deleted file mode 100644 index fbef47e872..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesConverterTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.IO; - using System.Text.Json; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesConverterTests - { - [TestMethod] - public void Write_Results_IdenticalToNewtonsoft() - { - byte[] bytes = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - - JsonBytes jsonBytes = new (bytes, 5, 5); - - using MemoryStream ms = new (); - using Utf8JsonWriter writer = new (ms); - - JsonBytesConverter jsonConverter = new (); - jsonConverter.Write(writer, jsonBytes, JsonSerializerOptions.Default); - - writer.Flush(); - ms.Flush(); - ms.Position = 0; - StreamReader sr = new(ms); - string systemTextResult = sr.ReadToEnd(); - - byte[] newtonsoftBytes = bytes.AsSpan(5, 5).ToArray(); - string newtonsoftResult = Newtonsoft.Json.JsonConvert.SerializeObject(newtonsoftBytes); - - Assert.AreEqual(systemTextResult, newtonsoftResult); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs deleted file mode 100644 index a29f378ff2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonBytesTests.cs +++ /dev/null @@ -1,33 +0,0 @@ -#if NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation.SystemTextJson; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - [TestClass] - public class JsonBytesTests - { - [TestMethod] - public void Ctor_ThrowsForInvalidInputs() - { - Assert.ThrowsException(() => new JsonBytes(null, 1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], -1, 1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 0, -1)); - Assert.ThrowsException(() => new JsonBytes(new byte[10], 8, 8)); - } - - [TestMethod] - public void Properties_AreSetCorrectly() - { - byte[] bytes = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 }; - JsonBytes jsonBytes = new (bytes, 1, 5); - - Assert.AreEqual(1, jsonBytes.Offset); - Assert.AreEqual(5, jsonBytes.Length); - Assert.AreSame(bytes, jsonBytes.Bytes); - } - } -} -#endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs deleted file mode 100644 index e28bd409a1..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/JsonNodeSqlSerializerTests.cs +++ /dev/null @@ -1,171 +0,0 @@ -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - -namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Text.Json; - using System.Text.Json.Nodes; - using Microsoft.Azure.Cosmos.Encryption.Custom; - using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; - using Microsoft.VisualStudio.TestTools.UnitTesting; - using Newtonsoft.Json.Linq; - - [TestClass] - public class JsonNodeSqlSerializerTests - { - private static ArrayPoolManager _poolManager; - - [ClassInitialize] - public static void ClassInitialize(TestContext context) - { - _ = context; - _poolManager = new ArrayPoolManager(); - } - - [TestMethod] - [DynamicData(nameof(SerializationSamples))] - public void Serialize_SupportedValue(JsonNode testNode, byte expectedType, byte[] expectedBytes, int expectedLength) - { - JsonNodeSqlSerializer serializer = new(); - - (TypeMarker serializedType, byte[] serializedBytes, int serializedBytesCount) = serializer.Serialize(testNode, _poolManager); - - Assert.AreEqual((TypeMarker)expectedType, serializedType); - Assert.AreEqual(expectedLength, serializedBytesCount); - if (expectedLength == -1) - { - Assert.IsTrue(serializedBytes == null); - } - else - { - Assert.IsTrue(expectedBytes.SequenceEqual(serializedBytes.AsSpan(0, serializedBytesCount).ToArray())); - } - } - - [TestMethod] - [DynamicData(nameof(DeserializationSamples))] - public void Deserialize_SupportedValue(byte typeMarkerByte, byte[] serializedBytes, JsonNode expectedNode) - { - JsonNodeSqlSerializer serializer = new(); - TypeMarker typeMarker = (TypeMarker)typeMarkerByte; - JsonNode deserializedNode = serializer.Deserialize(typeMarker, serializedBytes); - - if ((expectedNode as JsonValue) != null) - { - AssertValueNodeEquality(expectedNode, deserializedNode); - return; - } - - if ((expectedNode as JsonArray) != null) - { - Assert.IsNotNull(deserializedNode as JsonArray); - - JsonArray expectedArray = expectedNode.AsArray(); - JsonArray deserializedArray = deserializedNode.AsArray(); - - Assert.AreEqual(expectedArray.Count, deserializedArray.Count); - - for (int i = 0; i < deserializedNode.AsArray().Count; i++) - { - AssertValueNodeEquality(expectedArray[i], deserializedArray[i]); - } - return; - } - - if ((expectedNode as JsonObject) != null) - { - Assert.IsNotNull(deserializedNode as JsonObject); - - JsonObject expectedObject = expectedNode.AsObject(); - JsonObject deserializedObject = deserializedNode.AsObject(); - - Assert.AreEqual(expectedObject.Count, deserializedObject.Count); - - foreach (KeyValuePair expected in expectedObject) - { - Assert.IsTrue(deserializedObject.ContainsKey(expected.Key)); - AssertValueNodeEquality(expected.Value, deserializedObject[expected.Key]); - } - return; - } - - Assert.Fail("Attempt to validate unsupported JsonNode type"); - } - - private static void AssertValueNodeEquality(JsonNode expectedNode, JsonNode actualNode) - { - JsonValue expectedValueNode = expectedNode.AsValue(); - JsonValue actualValueNode = actualNode.AsValue(); - - Assert.AreEqual(expectedValueNode.GetValueKind(), actualValueNode.GetValueKind()); - Assert.AreEqual(expectedValueNode.ToString(), actualValueNode.ToString()); - } - - public static IEnumerable DeserializationSamples - { - get - { - yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), JsonValue.Create(true) }; - yield return new object[] { (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), JsonValue.Create(false) }; - yield return new object[] { (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), JsonValue.Create(192) }; - yield return new object[] { (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), JsonValue.Create(192.5) }; - yield return new object[] { (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), JsonValue.Create(testString) }; - yield return new object[] { (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), JsonNode.Parse("[10,18,19]") }; - yield return new object[] { (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), JsonNode.Parse(testClass.ToJson()) }; - } - } - - public static IEnumerable SerializationSamples - { - get - { - List values = new() - { - new object[] {JsonValue.Create((string)null), (byte)TypeMarker.Null, null, -1 }, - new object[] {JsonValue.Create(true), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(true), 8}, - new object[] {JsonValue.Create(false), (byte)TypeMarker.Boolean, GetNewtonsoftValueEquivalent(false), 8}, - new object[] {JsonValue.Create(192), (byte)TypeMarker.Long, GetNewtonsoftValueEquivalent(192), 8}, - new object[] {JsonValue.Create(192.5), (byte)TypeMarker.Double, GetNewtonsoftValueEquivalent(192.5), 8}, - new object[] {JsonValue.Create(testString), (byte)TypeMarker.String, GetNewtonsoftValueEquivalent(testString), 11}, - new object[] {JsonValue.Create(testArray), (byte)TypeMarker.Array, GetNewtonsoftValueEquivalent(testArray), 10}, - new object[] {JsonValue.Create(testClass), (byte)TypeMarker.Object, GetNewtonsoftValueEquivalent(testClass), 33} - }; - - return values; - } - } - - private static readonly string testString = "Hello world"; - private static readonly int[] testArray = new[] {10, 18, 19}; - private static readonly TestClass testClass = new() { SomeInt = 1, SomeString = "asdf" }; - - private class TestClass - { - public int SomeInt { get; set; } - public string SomeString { get; set; } - - public string ToJson() - { - return JsonSerializer.Serialize(this); - } - } - - private static byte[] GetNewtonsoftValueEquivalent(T value) - { - JObjectSqlSerializer serializer = new (); - JToken token = value switch - { - int[] => new JArray(value), - TestClass => JObject.FromObject(value), - _ => new JValue(value), - }; - (TypeMarker _, byte[] bytes, int lenght) = serializer.Serialize(token, _poolManager); - return bytes.AsSpan(0, lenght).ToArray(); - } - - } -} - -#endif \ No newline at end of file From 3be6843a30260e8fe980171f27cd9abab17681b0 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 14:57:53 +0200 Subject: [PATCH 65/71] + rms --- .../RecyclableMemoryStreamMirror/README.md | 3 + .../RecyclableMemoryStream.cs | 1585 +++++++++++++++++ ...RecyclableMemoryStreamManager.EventArgs.cs | 456 +++++ .../RecyclableMemoryStreamManager.Events.cs | 288 +++ .../RecyclableMemoryStreamManager.cs | 989 ++++++++++ 5 files changed, 3321 insertions(+) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md new file mode 100644 index 0000000000..eed315c783 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md @@ -0,0 +1,3 @@ +# Microsoft.IO.RecyclableMemoryStream 3.0.1 + +Mirrored from https://github.com/microsoft/Microsoft.IO.RecyclableMemoryStream/tree/master/src diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs new file mode 100644 index 0000000000..92b003d71e --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs @@ -0,0 +1,1585 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// The MIT License (MIT) +// +// Copyright (c) 2015-2016 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Buffers; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Runtime.CompilerServices; + using System.Threading; + using System.Threading.Tasks; + + /// + /// MemoryStream implementation that deals with pooling and managing memory streams which use potentially large + /// buffers. + /// + /// + /// This class works in tandem with the to supply MemoryStream-derived + /// objects to callers, while avoiding these specific problems: + /// + /// + /// LOH allocations + /// Since all large buffers are pooled, they will never incur a Gen2 GC + /// + /// + /// Memory wasteA standard memory stream doubles its size when it runs out of room. This + /// leads to continual memory growth as each stream approaches the maximum allowed size. + /// + /// + /// Memory copying + /// Each time a MemoryStream grows, all the bytes are copied into new buffers. + /// This implementation only copies the bytes when is called. + /// + /// + /// Memory fragmentation + /// By using homogeneous buffer sizes, it ensures that blocks of memory + /// can be easily reused. + /// + /// + /// + /// + /// The stream is implemented on top of a series of uniformly-sized blocks. As the stream's length grows, + /// additional blocks are retrieved from the memory manager. It is these blocks that are pooled, not the stream + /// object itself. + /// + /// + /// The biggest wrinkle in this implementation is when is called. This requires a single + /// contiguous buffer. If only a single block is in use, then that block is returned. If multiple blocks + /// are in use, we retrieve a larger buffer from the memory manager. These large buffers are also pooled, + /// split by size--they are multiples/exponentials of a chunk size (1 MB by default). + /// + /// + /// Once a large buffer is assigned to the stream the small blocks are NEVER again used for this stream. All operations take place on the + /// large buffer. The large buffer can be replaced by a larger buffer from the pool as needed. All blocks and large buffers + /// are maintained in the stream until the stream is disposed (unless AggressiveBufferReturn is enabled in the stream manager). + /// + /// + /// A further wrinkle is what happens when the stream is longer than the maximum allowable array length under .NET. This is allowed + /// when only blocks are in use, and only the Read/Write APIs are used. Once a stream grows to this size, any attempt to convert it + /// to a single buffer will result in an exception. Similarly, if a stream is already converted to use a single larger buffer, then + /// it cannot grow beyond the limits of the maximum allowable array size. + /// + /// + /// Any method that modifies the stream has the potential to throw an OutOfMemoryException, either because + /// the stream is beyond the limits set in RecyclableStreamManager, or it would result in a buffer larger than + /// the maximum array size supported by .NET. + /// + /// + public sealed class RecyclableMemoryStream : MemoryStream, IBufferWriter + { + /// + /// All of these blocks must be the same size. + /// + private readonly List blocks; + + private readonly Guid id; + + private readonly RecyclableMemoryStreamManager memoryManager; + + private readonly string tag; + + private readonly long creationTimestamp; + + /// + /// This list is used to store buffers once they're replaced by something larger. + /// This is for the cases where you have users of this class that may hold onto the buffers longer + /// than they should and you want to prevent race conditions which could corrupt the data. + /// + private List dirtyBuffers; + + private bool disposed; + + /// + /// This is only set by GetBuffer() if the necessary buffer is larger than a single block size, or on + /// construction if the caller immediately requests a single large buffer. + /// + /// If this field is non-null, it contains the concatenation of the bytes found in the individual + /// blocks. Once it is created, this (or a larger) largeBuffer will be used for the life of the stream. + /// + private byte[] largeBuffer; + + /// + /// Gets unique identifier for this stream across its entire lifetime. + /// + /// Object has been disposed. + internal Guid Id + { + get + { + this.CheckDisposed(); + return this.id; + } + } + + /// + /// Gets a temporary identifier for the current usage of this stream. + /// + /// Object has been disposed. + internal string Tag + { + get + { + this.CheckDisposed(); + return this.tag; + } + } + + /// + /// Gets the memory manager being used by this stream. + /// + /// Object has been disposed. + internal RecyclableMemoryStreamManager MemoryManager + { + get + { + this.CheckDisposed(); + return this.memoryManager; + } + } + + /// + /// Gets call stack of the constructor. It is only set if is true, + /// which should only be in debugging situations. + /// + internal string AllocationStack { get; } + + /// + /// Gets call stack of the call. It is only set if is true, + /// which should only be in debugging situations. + /// + internal string DisposeStack { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager) + : this(memoryManager, Guid.NewGuid(), null, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id) + : this(memoryManager, id, null, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A string identifying this stream for logging and debugging purposes. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag) + : this(memoryManager, Guid.NewGuid(), tag, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag) + : this(memoryManager, id, tag, 0, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag, long requestedSize) + : this(memoryManager, Guid.NewGuid(), tag, requestedSize, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize) + : this(memoryManager, id, tag, requestedSize, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The memory manager. + /// A unique identifier which can be used to trace usages of the stream. + /// A string identifying this stream for logging and debugging purposes. + /// The initial requested size to prevent future allocations. + /// An initial buffer to use. This buffer will be owned by the stream and returned to the memory manager upon Dispose. + internal RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize, byte[] initialLargeBuffer) + : base(Array.Empty()) + { + this.memoryManager = memoryManager; + this.id = id; + this.tag = tag; + this.blocks = new List(); + this.creationTimestamp = Stopwatch.GetTimestamp(); + + long actualRequestedSize = Math.Max(requestedSize, this.memoryManager.OptionsValue.BlockSize); + + if (initialLargeBuffer == null) + { + this.EnsureCapacity(actualRequestedSize); + } + else + { + this.largeBuffer = initialLargeBuffer; + } + + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + this.AllocationStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamCreated(this.id, this.tag, requestedSize, actualRequestedSize); + this.memoryManager.ReportUsageReport(); + } + + /// + /// Finalizes an instance of the class. + /// + /// Failing to dispose indicates a bug in the code using streams. Care should be taken to properly account for stream lifetime. + ~RecyclableMemoryStream() + { + this.Dispose(false); + } + + /// + /// Returns the memory used by this stream back to the pool. + /// + /// Whether we're disposing (true), or being called by the finalizer (false). + protected override void Dispose(bool disposing) + { + if (this.disposed) + { + string doubleDisposeStack = null; + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + doubleDisposeStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamDoubleDisposed(this.id, this.tag, this.AllocationStack, this.DisposeStack, doubleDisposeStack); + return; + } + + this.disposed = true; + TimeSpan lifetime = TimeSpan.FromTicks((Stopwatch.GetTimestamp() - this.creationTimestamp) * TimeSpan.TicksPerSecond / Stopwatch.Frequency); + + if (this.memoryManager.OptionsValue.GenerateCallStacks) + { + this.DisposeStack = Environment.StackTrace; + } + + this.memoryManager.ReportStreamDisposed(this.id, this.tag, lifetime, this.AllocationStack, this.DisposeStack); + + if (disposing) + { + GC.SuppressFinalize(this); + } + else + { + // We're being finalized. + this.memoryManager.ReportStreamFinalized(this.id, this.tag, this.AllocationStack); + + if (AppDomain.CurrentDomain.IsFinalizingForUnload()) + { + // If we're being finalized because of a shutdown, don't go any further. + // We have no idea what's already been cleaned up. Triggering events may cause + // a crash. + base.Dispose(disposing); + return; + } + } + + this.memoryManager.ReportStreamLength(this.length); + + if (this.largeBuffer != null) + { + this.memoryManager.ReturnLargeBuffer(this.largeBuffer, this.id, this.tag); + } + + if (this.dirtyBuffers != null) + { + foreach (byte[] buffer in this.dirtyBuffers) + { + this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); + } + } + + this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); + this.memoryManager.ReportUsageReport(); + this.blocks.Clear(); + + base.Dispose(disposing); + } + + /// + /// Equivalent to Dispose. + /// + public override void Close() + { + this.Dispose(true); + } + + /// + /// Gets or sets the capacity. + /// + /// + /// + /// Capacity is always in multiples of the memory manager's block size, unless + /// the large buffer is in use. Capacity never decreases during a stream's lifetime. + /// Explicitly setting the capacity to a lower value than the current value will have no effect. + /// This is because the buffers are all pooled by chunks and there's little reason to + /// allow stream truncation. + /// + /// + /// Writing past the current capacity will cause to automatically increase, until MaximumStreamCapacity is reached. + /// + /// + /// If the capacity is larger than int.MaxValue, then InvalidOperationException will be thrown. If you anticipate using + /// larger streams, use the property instead. + /// + /// + /// Object has been disposed. + /// Capacity is larger than int.MaxValue. + public override int Capacity + { + get + { + this.CheckDisposed(); + if (this.largeBuffer != null) + { + return this.largeBuffer.Length; + } + + long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; + if (size > int.MaxValue) + { + throw new InvalidOperationException($"{nameof(this.Capacity)} is larger than int.MaxValue. Use {nameof(this.Capacity64)} instead."); + } + + return (int)size; + } + + set => this.Capacity64 = value; + } + + /// + /// Gets or sets returns a 64-bit version of capacity, for streams larger than int.MaxValue in length. + /// + public long Capacity64 + { + get + { + this.CheckDisposed(); + if (this.largeBuffer != null) + { + return this.largeBuffer.Length; + } + + long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; + return size; + } + + set + { + this.CheckDisposed(); + this.EnsureCapacity(value); + } + } + + private long length; + + /// + /// Gets the number of bytes written to this stream. + /// + /// Object has been disposed. + /// If the buffer has already been converted to a large buffer, then the maximum length is limited by the maximum allowed array length in .NET. + public override long Length + { + get + { + this.CheckDisposed(); + return this.length; + } + } + + private long position; + + /// + /// Gets or sets the current position in the stream. + /// + /// Object has been disposed. + /// A negative value was passed. + /// Stream is in large-buffer mode, but an attempt was made to set the position past the maximum allowed array length. + /// If the buffer has already been converted to a large buffer, then the maximum length (and thus position) is limited by the maximum allowed array length in .NET. + public override long Position + { + get + { + this.CheckDisposed(); + return this.position; + } + + set + { + this.CheckDisposed(); + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); + } + + if (this.largeBuffer != null && value > RecyclableMemoryStreamManager.MaxArrayLength) + { + throw new InvalidOperationException($"Once the stream is converted to a single large buffer, position cannot be set past {RecyclableMemoryStreamManager.MaxArrayLength}."); + } + + this.position = value; + } + } + + /// + /// Gets a value indicating whether whether the stream can currently read. + /// + public override bool CanRead => !this.disposed; + + /// + /// Gets a value indicating whether whether the stream can currently seek. + /// + public override bool CanSeek => !this.disposed; + + /// + /// Gets a value indicating whether the steram can timeout. + /// + /// Always false + public override bool CanTimeout => false; + + /// + /// Gets a value indicating whether whether the stream can currently write. + /// + public override bool CanWrite => !this.disposed; + + /// + /// Returns a single buffer containing the contents of the stream. + /// The buffer may be longer than the stream length. + /// + /// A byte[] buffer. + /// IMPORTANT: Doing a after calling GetBuffer invalidates the buffer. The old buffer is held onto + /// until is called, but the next time GetBuffer is called, a new buffer from the pool will be required. + /// Object has been disposed. + /// stream is too large for a contiguous buffer. + public override byte[] GetBuffer() + { + this.CheckDisposed(); + + if (this.largeBuffer != null) + { + return this.largeBuffer; + } + + if (this.blocks.Count == 1) + { + return this.blocks[0]; + } + + // Buffer needs to reflect the capacity, not the length, because + // it's possible that people will manipulate the buffer directly + // and set the length afterward. Capacity sets the expectation + // for the size of the buffer. + byte[] newBuffer = this.memoryManager.GetLargeBuffer(this.Capacity64, this.id, this.tag); + + // InternalRead will check for existence of largeBuffer, so make sure we + // don't set it until after we've copied the data. + this.AssertLengthIsSmall(); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + this.largeBuffer = newBuffer; + + if (this.blocks.Count > 0 && this.memoryManager.OptionsValue.AggressiveBufferReturn) + { + this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); + this.blocks.Clear(); + } + + return this.largeBuffer; + } + +#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + /// + public override void CopyTo(Stream destination, int bufferSize) + { + this.WriteTo(destination, this.position, this.length - this.position); + } +#endif + + /// Asynchronously reads all the bytes from the current position in this stream and writes them to another stream. + /// The stream to which the contents of the current stream will be copied. + /// This parameter is ignored. + /// The token to monitor for cancellation requests. + /// A task that represents the asynchronous copy operation. + /// + /// is . + /// Either the current stream or the destination stream is disposed. + /// The current stream does not support reading, or the destination stream does not support writing. + /// Similarly to MemoryStream's behavior, CopyToAsync will adjust the source stream's position by the number of bytes written to the destination stream, as a Read would do. + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(destination); +#else + if (destination == null) + { + throw new ArgumentNullException(nameof(destination)); + } +#endif + + this.CheckDisposed(); + + if (this.length == 0) + { + return Task.CompletedTask; + } + + long startPos = this.position; + long count = this.length - startPos; + this.position += count; + + if (destination is MemoryStream destinationRMS) + { + this.WriteTo(destinationRMS, startPos, count); + return Task.CompletedTask; + } + else + { + if (this.largeBuffer == null) + { + if (this.blocks.Count == 1) + { + this.AssertLengthIsSmall(); + return destination.WriteAsync(this.blocks[0], (int)startPos, (int)count, cancellationToken); + } + else + { + return CopyToAsyncImpl(destination, this.GetBlockAndRelativeOffset(startPos), count, this.blocks, cancellationToken); + } + } + else + { + this.AssertLengthIsSmall(); + return destination.WriteAsync(this.largeBuffer, (int)startPos, (int)count, cancellationToken); + } + } + + static async Task CopyToAsyncImpl(Stream destination, BlockAndOffset blockAndOffset, long count, List blocks, CancellationToken cancellationToken) + { + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + while (bytesRemaining > 0) + { + byte[] block = blocks[currentBlock]; + int amountToCopy = (int)Math.Min(block.Length - currentOffset, bytesRemaining); +#if NET8_0_OR_GREATER + await destination.WriteAsync(block.AsMemory(currentOffset, amountToCopy), cancellationToken); +#else + await destination.WriteAsync(block, currentOffset, amountToCopy, cancellationToken); +#endif + bytesRemaining -= amountToCopy; + ++currentBlock; + currentOffset = 0; + } + } + } + + private byte[] bufferWriterTempBuffer; + + /// + /// Notifies the stream that bytes were written to the buffer returned by or . + /// Seeks forward by bytes. + /// + /// + /// You must request a new buffer after calling Advance to continue writing more data and cannot write to a previously acquired buffer. + /// + /// How many bytes to advance. + /// Object has been disposed. + /// is negative. + /// is larger than the size of the previously requested buffer. + public void Advance(int count) + { + this.CheckDisposed(); + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} must be non-negative."); + } + + byte[] buffer = this.bufferWriterTempBuffer; + if (buffer != null) + { + if (count > buffer.Length) + { + throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {buffer.Length}."); + } + + this.Write(buffer, 0, count); + this.ReturnTempBuffer(buffer); + this.bufferWriterTempBuffer = null; + } + else + { + long bufferSize = this.largeBuffer == null + ? this.memoryManager.OptionsValue.BlockSize - this.GetBlockAndRelativeOffset(this.position).Offset + : this.largeBuffer.Length - this.position; + + if (count > bufferSize) + { + throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {bufferSize}."); + } + + this.position += count; + this.length = Math.Max(this.position, this.length); + } + } + + private void ReturnTempBuffer(byte[] buffer) + { + if (buffer.Length == this.memoryManager.OptionsValue.BlockSize) + { + this.memoryManager.ReturnBlock(buffer, this.id, this.tag); + } + else + { + this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); + } + } + + /// + /// + /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), + /// or setting Position after calling GetMemory() invalidates the memory. + /// + public Memory GetMemory(int sizeHint = 0) + { + return this.GetWritableBuffer(sizeHint); + } + + /// + /// + /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), + /// or setting Position after calling GetSpan() invalidates the span. + /// + public Span GetSpan(int sizeHint = 0) + { + return this.GetWritableBuffer(sizeHint); + } + + /// + /// When callers to GetSpan() or GetMemory() request a buffer that is larger than the remaining size of the current block + /// this method return a temp buffer. When Advance() is called, that temp buffer is then copied into the stream. + /// + private ArraySegment GetWritableBuffer(int sizeHint) + { + this.CheckDisposed(); + if (sizeHint < 0) + { + throw new ArgumentOutOfRangeException(nameof(sizeHint), $"{nameof(sizeHint)} must be non-negative."); + } + + int minimumBufferSize = Math.Max(sizeHint, 1); + + this.EnsureCapacity(this.position + minimumBufferSize); + if (this.bufferWriterTempBuffer != null) + { + this.ReturnTempBuffer(this.bufferWriterTempBuffer); + this.bufferWriterTempBuffer = null; + } + + if (this.largeBuffer != null) + { + return new ArraySegment(this.largeBuffer, (int)this.position, this.largeBuffer.Length - (int)this.position); + } + + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + int remainingBytesInBlock = this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset; + if (remainingBytesInBlock >= minimumBufferSize) + { + return new ArraySegment(this.blocks[blockAndOffset.Block], blockAndOffset.Offset, this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset); + } + + this.bufferWriterTempBuffer = minimumBufferSize > this.memoryManager.OptionsValue.BlockSize ? + this.memoryManager.GetLargeBuffer(minimumBufferSize, this.id, this.tag) : + this.memoryManager.GetBlock(); + + return new ArraySegment(this.bufferWriterTempBuffer); + } + + /// + /// Returns a sequence containing the contents of the stream. + /// + /// A ReadOnlySequence of bytes. + /// IMPORTANT: Calling Write(), GetMemory(), GetSpan(), Dispose(), or Close() after calling GetReadOnlySequence() invalidates the sequence. + /// Object has been disposed. + public ReadOnlySequence GetReadOnlySequence() + { + this.CheckDisposed(); + + if (this.largeBuffer != null) + { + this.AssertLengthIsSmall(); + return new ReadOnlySequence(this.largeBuffer, 0, (int)this.length); + } + + if (this.blocks.Count == 1) + { + this.AssertLengthIsSmall(); + return new ReadOnlySequence(this.blocks[0], 0, (int)this.length); + } + + BlockSegment first = new (this.blocks[0]); + BlockSegment last = first; + + for (int blockIdx = 1; last.RunningIndex + last.Memory.Length < this.length; blockIdx++) + { + last = last.Append(this.blocks[blockIdx]); + } + + return new ReadOnlySequence(first, 0, last, (int)(this.length - last.RunningIndex)); + } + + private sealed class BlockSegment : ReadOnlySequenceSegment + { + public BlockSegment(Memory memory) + { + this.Memory = memory; + } + + public BlockSegment Append(Memory memory) + { + BlockSegment nextSegment = new (memory) { RunningIndex = this.RunningIndex + this.Memory.Length }; + this.Next = nextSegment; + return nextSegment; + } + } + + /// + /// Returns an ArraySegment that wraps a single buffer containing the contents of the stream. + /// + /// An ArraySegment containing a reference to the underlying bytes. + /// Returns if a buffer can be returned; otherwise, . + public override bool TryGetBuffer(out ArraySegment buffer) + { + this.CheckDisposed(); + + try + { + if (this.length <= RecyclableMemoryStreamManager.MaxArrayLength) + { + buffer = new ArraySegment(this.GetBuffer(), 0, (int)this.Length); + return true; + } + } + catch (OutOfMemoryException) + { + } + +#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER + buffer = ArraySegment.Empty; +#else + buffer = default; +#endif + return false; + } + + /// + /// Returns a new array with a copy of the buffer's contents. You should almost certainly be using combined with the to + /// access the bytes in this stream. Calling ToArray will destroy the benefits of pooled buffers, but it is included + /// for the sake of completeness. + /// + /// Object has been disposed. + /// The current object disallows ToArray calls. + /// The length of the stream is too long for a contiguous array. + /// Array of bytes +#pragma warning disable CS0809 + [Obsolete("This method has degraded performance vs. GetBuffer and should be avoided.")] + public override byte[] ToArray() + { + this.CheckDisposed(); + + string stack = this.memoryManager.OptionsValue.GenerateCallStacks ? Environment.StackTrace : null; + this.memoryManager.ReportStreamToArray(this.id, this.tag, stack, this.length); + + if (this.memoryManager.OptionsValue.ThrowExceptionOnToArray) + { + throw new NotSupportedException("The underlying RecyclableMemoryStreamManager is configured to not allow calls to ToArray."); + } + + byte[] newBuffer = new byte[this.Length]; + + Debug.Assert(this.length <= int.MaxValue); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + + return newBuffer; + } +#pragma warning restore CS0809 + + /// + /// Reads from the current position into the provided buffer. + /// + /// Destination buffer. + /// Offset into buffer at which to start placing the read bytes. + /// Number of bytes to read. + /// The number of bytes read. + /// buffer is null. + /// offset or count is less than 0. + /// offset subtracted from the buffer length is less than count. + /// Object has been disposed. + public override int Read(byte[] buffer, int offset, int count) + { + return this.SafeRead(buffer, offset, count, ref this.position); + } + + /// + /// Reads from the specified position into the provided buffer. + /// + /// Destination buffer. + /// Offset into buffer at which to start placing the read bytes. + /// Number of bytes to read. + /// Position in the stream to start reading from. + /// The number of bytes read. + /// is null. + /// or is less than 0. + /// subtracted from the buffer length is less than . + /// Object has been disposed. + public int SafeRead(byte[] buffer, int offset, int count, ref long streamPosition) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0) + { + throw new ArgumentOutOfRangeException(nameof(offset), $"{nameof(offset)} cannot be negative."); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} cannot be negative."); + } + + if (offset + count > buffer.Length) + { + throw new ArgumentException($"{nameof(buffer)} length must be at least {nameof(offset)} + {nameof(count)}."); + } + + int amountRead = this.InternalRead(buffer, offset, count, streamPosition); + streamPosition += amountRead; + return amountRead; + } + + /// + /// Reads from the current position into the provided buffer. + /// + /// Destination buffer. + /// The number of bytes read. + /// Object has been disposed. +#if NETSTANDARD2_0 + public int Read(Span buffer) +#else + public override int Read(Span buffer) +#endif + { + return this.SafeRead(buffer, ref this.position); + } + + /// + /// Reads from the specified position into the provided buffer. + /// + /// Destination buffer. + /// Position in the stream to start reading from. + /// The number of bytes read. + /// Object has been disposed. + public int SafeRead(Span buffer, ref long streamPosition) + { + this.CheckDisposed(); + + int amountRead = this.InternalRead(buffer, streamPosition); + streamPosition += amountRead; + return amountRead; + } + + /// + /// Writes the buffer to the stream. + /// + /// Source buffer. + /// Start position. + /// Number of bytes to write. + /// buffer is null. + /// offset or count is negative. + /// buffer.Length - offset is not less than count. + /// Object has been disposed. + public override void Write(byte[] buffer, int offset, int count) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0) + { + throw new ArgumentOutOfRangeException( + nameof(offset), + offset, + $"{nameof(offset)} must be in the range of 0 - {nameof(buffer)}.{nameof(buffer.Length)}-1."); + } + + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count), count, $"{nameof(count)} must be non-negative."); + } + + if (count + offset > buffer.Length) + { + throw new ArgumentException($"{nameof(count)} must be greater than {nameof(buffer)}.{nameof(buffer.Length)} - {nameof(offset)}."); + } + + int blockSize = this.memoryManager.OptionsValue.BlockSize; + long end = this.position + count; + + this.EnsureCapacity(end); + + if (this.largeBuffer == null) + { + int bytesRemaining = count; + int bytesWritten = 0; + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + + while (bytesRemaining > 0) + { + byte[] currentBlock = this.blocks[blockAndOffset.Block]; + int remainingInBlock = blockSize - blockAndOffset.Offset; + int amountToWriteInBlock = Math.Min(remainingInBlock, bytesRemaining); + + Buffer.BlockCopy( + buffer, + offset + bytesWritten, + currentBlock, + blockAndOffset.Offset, + amountToWriteInBlock); + + bytesRemaining -= amountToWriteInBlock; + bytesWritten += amountToWriteInBlock; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + } + else + { + Buffer.BlockCopy(buffer, offset, this.largeBuffer, (int)this.position, count); + } + + this.position = end; + this.length = Math.Max(this.position, this.length); + } + + /// + /// Writes the buffer to the stream. + /// + /// Source buffer. + /// buffer is null. + /// Object has been disposed. +#if NETSTANDARD2_0 + public void Write(ReadOnlySpan source) +#else + public override void Write(ReadOnlySpan source) +#endif + { + this.CheckDisposed(); + + int blockSize = this.memoryManager.OptionsValue.BlockSize; + long end = this.position + source.Length; + + this.EnsureCapacity(end); + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); + + while (source.Length > 0) + { + byte[] currentBlock = this.blocks[blockAndOffset.Block]; + int remainingInBlock = blockSize - blockAndOffset.Offset; + int amountToWriteInBlock = Math.Min(remainingInBlock, source.Length); +#if NET8_0_OR_GREATER + source[..amountToWriteInBlock] + .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); + + source = source[amountToWriteInBlock..]; +#else + source.Slice(0, amountToWriteInBlock) + .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); + + source = source.Slice(amountToWriteInBlock); +#endif + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + } + else + { + source.CopyTo(this.largeBuffer.AsSpan((int)this.position)); + } + + this.position = end; + this.length = Math.Max(this.position, this.length); + } + + /// + /// Returns a useful string for debugging. This should not normally be called in actual production code. + /// + /// String with debug data. + public override string ToString() + { + if (!this.disposed) + { + return $"Id = {this.Id}, Tag = {this.Tag}, Length = {this.Length:N0} bytes"; + } + else + { + // Avoid properties because of the dispose check, but the fields themselves are not cleared. + return $"Disposed: Id = {this.id}, Tag = {this.tag}, Final Length: {this.length:N0} bytes"; + } + } + + /// + /// Writes a single byte to the current position in the stream. + /// + /// byte value to write. + /// Object has been disposed. + public override void WriteByte(byte value) + { + this.CheckDisposed(); + + long end = this.position + 1; + + if (this.largeBuffer == null) + { + int blockSize = this.memoryManager.OptionsValue.BlockSize; + + int block = (int)Math.DivRem(this.position, blockSize, out long index); + + if (block >= this.blocks.Count) + { + this.EnsureCapacity(end); + } + + this.blocks[block][index] = value; + } + else + { + if (this.position >= this.largeBuffer.Length) + { + this.EnsureCapacity(end); + } + + this.largeBuffer[this.position] = value; + } + + this.position = end; + + if (this.position > this.length) + { + this.length = this.position; + } + } + + /// + /// Reads a single byte from the current position in the stream. + /// + /// The byte at the current position, or -1 if the position is at the end of the stream. + /// Object has been disposed. + public override int ReadByte() + { + return this.SafeReadByte(ref this.position); + } + + /// + /// Reads a single byte from the specified position in the stream. + /// + /// The position in the stream to read from. + /// The byte at the current position, or -1 if the position is at the end of the stream. + /// Object has been disposed. + public int SafeReadByte(ref long streamPosition) + { + this.CheckDisposed(); + if (streamPosition == this.length) + { + return -1; + } + + byte value; + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(streamPosition); + value = this.blocks[blockAndOffset.Block][blockAndOffset.Offset]; + } + else + { + value = this.largeBuffer[streamPosition]; + } + + streamPosition++; + return value; + } + + /// + /// Sets the length of the stream. + /// + /// length of the stream + /// value is negative or larger than . + /// Object has been disposed. + public override void SetLength(long value) + { + this.CheckDisposed(); + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); + } + + this.EnsureCapacity(value); + + this.length = value; + if (this.position > value) + { + this.position = value; + } + } + + /// + /// Sets the position to the offset from the seek location. + /// + /// How many bytes to move. + /// From where. + /// The new position. + /// Object has been disposed. + /// is larger than . + /// Invalid seek origin. + /// Attempt to set negative position. + public override long Seek(long offset, SeekOrigin loc) + { + this.CheckDisposed(); + long newPosition = loc switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => offset + this.position, + SeekOrigin.End => offset + this.length, + _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), + }; + if (newPosition < 0) + { + throw new IOException("Seek before beginning."); + } + + this.position = newPosition; + return this.position; + } + + /// + /// Synchronously writes this stream's bytes to the argument stream. + /// + /// Destination stream. + /// Important: This does a synchronous write, which may not be desired in some situations. + /// is null. + /// Object has been disposed. + public override void WriteTo(Stream stream) + { + this.WriteTo(stream, 0, this.length); + } + + /// + /// Synchronously writes this stream's bytes, starting at offset, for count bytes, to the argument stream. + /// + /// Destination stream. + /// Offset in source. + /// Number of bytes to write. + /// is null. + /// + /// is less than 0, or + is beyond this 's length. + /// + /// Object has been disposed. + public void WriteTo(Stream stream, long offset, long count) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(stream); +#else + if (stream == null) + { + throw new ArgumentNullException(nameof(stream)); + } +#endif + + if (offset < 0 || offset + count > this.length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the {nameof(stream)}.", + innerException: null); + } + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[currentBlock]; + int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); + stream.Write(block, currentOffset, amountToCopy); + + bytesRemaining -= amountToCopy; + + ++currentBlock; + currentOffset = 0; + } + } + else + { + stream.Write(this.largeBuffer, (int)offset, (int)count); + } + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// The entire stream is written to the target array. + /// > is null. + /// Object has been disposed. + public void WriteTo(byte[] buffer) + { + this.WriteTo(buffer, 0, this.Length); + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// Offset in the source stream, from which to start. + /// Number of bytes to write. + /// > is null. + /// + /// is less than 0, or + is beyond this stream's length. + /// + /// Object has been disposed. + public void WriteTo(byte[] buffer, long offset, long count) + { + this.WriteTo(buffer, offset, count, 0); + } + + /// + /// Writes bytes from the current stream to a destination byte array. + /// + /// Target buffer. + /// Offset in the source stream, from which to start. + /// Number of bytes to write. + /// Offset in the target byte array to start writing + /// buffer is null + /// + /// is less than 0, or + is beyond this stream's length. + /// + /// + /// is less than 0, or + is beyond the target 's length. + /// + /// Object has been disposed. + public void WriteTo(byte[] buffer, long offset, long count, int targetOffset) + { + this.CheckDisposed(); +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (offset < 0 || offset + count > this.length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the stream.", + innerException: null); + } + + if (targetOffset < 0 || count + targetOffset > buffer.Length) + { + throw new ArgumentOutOfRangeException( + message: $"{nameof(targetOffset)} must not be negative and {nameof(targetOffset)} + {nameof(count)} must not exceed the length of the target {nameof(buffer)}.", + innerException: null); + } + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); + long bytesRemaining = count; + int currentBlock = blockAndOffset.Block; + int currentOffset = blockAndOffset.Offset; + int currentTargetOffset = targetOffset; + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[currentBlock]; + int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); + Buffer.BlockCopy(block, currentOffset, buffer, currentTargetOffset, amountToCopy); + + bytesRemaining -= amountToCopy; + + ++currentBlock; + currentOffset = 0; + currentTargetOffset += amountToCopy; + } + } + else + { + this.AssertLengthIsSmall(); + Buffer.BlockCopy(this.largeBuffer, (int)offset, buffer, targetOffset, (int)count); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void CheckDisposed() + { + if (this.disposed) + { + this.ThrowDisposedException(); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void ThrowDisposedException() + { + throw new ObjectDisposedException($"The stream with Id {this.id} and Tag {this.tag} is disposed."); + } + + private int InternalRead(byte[] buffer, int offset, int count, long fromPosition) + { + if (this.length - fromPosition <= 0) + { + return 0; + } + + int amountToCopy; + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); + int bytesWritten = 0; + int bytesRemaining = (int)Math.Min(count, this.length - fromPosition); + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[blockAndOffset.Block]; + amountToCopy = Math.Min( + block.Length - blockAndOffset.Offset, + bytesRemaining); + Buffer.BlockCopy( + block, + blockAndOffset.Offset, + buffer, + bytesWritten + offset, + amountToCopy); + + bytesWritten += amountToCopy; + bytesRemaining -= amountToCopy; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + + return bytesWritten; + } + + amountToCopy = (int)Math.Min(count, this.length - fromPosition); + Buffer.BlockCopy(this.largeBuffer, (int)fromPosition, buffer, offset, amountToCopy); + return amountToCopy; + } + + private int InternalRead(Span buffer, long fromPosition) + { + if (this.length - fromPosition <= 0) + { + return 0; + } + + int amountToCopy; + + if (this.largeBuffer == null) + { + BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); + int bytesWritten = 0; + int bytesRemaining = (int)Math.Min(buffer.Length, this.length - fromPosition); + + while (bytesRemaining > 0) + { + byte[] block = this.blocks[blockAndOffset.Block]; + amountToCopy = Math.Min( + block.Length - blockAndOffset.Offset, + bytesRemaining); +#if NET8_0_OR_GREATER + block.AsSpan(blockAndOffset.Offset, amountToCopy) + .CopyTo(buffer[bytesWritten..]); +#else + block.AsSpan(blockAndOffset.Offset, amountToCopy) + .CopyTo(buffer.Slice(bytesWritten)); +#endif + + bytesWritten += amountToCopy; + bytesRemaining -= amountToCopy; + + ++blockAndOffset.Block; + blockAndOffset.Offset = 0; + } + + return bytesWritten; + } + + amountToCopy = (int)Math.Min(buffer.Length, this.length - fromPosition); + this.largeBuffer.AsSpan((int)fromPosition, amountToCopy).CopyTo(buffer); + return amountToCopy; + } + + private struct BlockAndOffset + { + public int Block; + public int Offset; + + public BlockAndOffset(int block, int offset) + { + this.Block = block; + this.Offset = offset; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private BlockAndOffset GetBlockAndRelativeOffset(long offset) + { + int blockSize = this.memoryManager.OptionsValue.BlockSize; + int blockIndex = (int)Math.DivRem(offset, blockSize, out long offsetIndex); + return new BlockAndOffset(blockIndex, (int)offsetIndex); + } + + private void EnsureCapacity(long newCapacity) + { + if (newCapacity > this.memoryManager.OptionsValue.MaximumStreamCapacity && this.memoryManager.OptionsValue.MaximumStreamCapacity > 0) + { + this.memoryManager.ReportStreamOverCapacity(this.id, this.tag, newCapacity, this.AllocationStack); + + throw new OutOfMemoryException($"Requested capacity is too large: {newCapacity}. Limit is {this.memoryManager.OptionsValue.MaximumStreamCapacity}."); + } + + if (this.largeBuffer != null) + { + if (newCapacity > this.largeBuffer.Length) + { + byte[] newBuffer = this.memoryManager.GetLargeBuffer(newCapacity, this.id, this.tag); + Debug.Assert(this.length <= int.MaxValue); + this.InternalRead(newBuffer, 0, (int)this.length, 0); + this.ReleaseLargeBuffer(); + this.largeBuffer = newBuffer; + } + } + else + { + // Let's save some re-allocation of the blocks list + long blocksRequired = (newCapacity / this.memoryManager.OptionsValue.BlockSize) + 1; + if (this.blocks.Capacity < blocksRequired) + { + this.blocks.Capacity = (int)blocksRequired; + } + + while (this.Capacity64 < newCapacity) + { + this.blocks.Add(this.memoryManager.GetBlock()); + } + } + } + + /// + /// Release the large buffer (either stores it for eventual release or returns it immediately). + /// + private void ReleaseLargeBuffer() + { + Debug.Assert(this.largeBuffer != null); + + if (this.memoryManager.OptionsValue.AggressiveBufferReturn) + { + this.memoryManager.ReturnLargeBuffer(this.largeBuffer!, this.id, this.tag); + } + else + { + // We most likely will only ever need space for one + this.dirtyBuffers ??= new List(1); + this.dirtyBuffers.Add(this.largeBuffer!); + } + + this.largeBuffer = null; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void AssertLengthIsSmall() + { + Debug.Assert(this.length <= int.MaxValue, "this.length was assumed to be <= Int32.MaxValue, but was larger."); + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs new file mode 100644 index 0000000000..7c450a418a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs @@ -0,0 +1,456 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + + /// + /// Wrapper for EventArgs + /// + public sealed partial class RecyclableMemoryStreamManager + { + /// + /// Arguments for the event. + /// + public sealed class StreamCreatedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets requested stream size. + /// + public long RequestedSize { get; } + + /// + /// Gets actual stream size. + /// + public long ActualSize { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// The requested stream size. + /// The actual stream size. + public StreamCreatedEventArgs(Guid guid, string tag, long requestedSize, long actualSize) + { + this.Id = guid; + this.Tag = tag; + this.RequestedSize = requestedSize; + this.ActualSize = actualSize; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamDisposedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Gets stack where stream was disposed. + /// + public string DisposeStack { get; } + + /// + /// Gets lifetime of the stream. + /// + public TimeSpan Lifetime { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Lifetime of the stream + /// Stack of original allocation. + /// Dispose stack. + public StreamDisposedEventArgs(Guid guid, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) + { + this.Id = guid; + this.Tag = tag; + this.Lifetime = lifetime; + this.AllocationStack = allocationStack; + this.DisposeStack = disposeStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamDoubleDisposedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Gets first dispose stack. + /// + public string DisposeStack1 { get; } + + /// + /// Gets second dispose stack. + /// + public string DisposeStack2 { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of original allocation. + /// First dispose stack. + /// Second dispose stack. + public StreamDoubleDisposedEventArgs(Guid guid, string tag, string allocationStack, string disposeStack1, string disposeStack2) + { + this.Id = guid; + this.Tag = tag; + this.AllocationStack = allocationStack; + this.DisposeStack1 = disposeStack1; + this.DisposeStack2 = disposeStack2; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamFinalizedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where the stream was allocated. + /// + public string AllocationStack { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of original allocation. + public StreamFinalizedEventArgs(Guid guid, string tag, string allocationStack) + { + this.Id = guid; + this.Tag = tag; + this.AllocationStack = allocationStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamConvertedToArrayEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets stack where ToArray was called. + /// + public string Stack { get; } + + /// + /// Gets length of stack. + /// + public long Length { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Stack of ToArray call. + /// Length of stream. + public StreamConvertedToArrayEventArgs(Guid guid, string tag, string stack, long length) + { + this.Id = guid; + this.Tag = tag; + this.Stack = stack; + this.Length = length; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamOverCapacityEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets original allocation stack. + /// + public string AllocationStack { get; } + + /// + /// Gets requested capacity. + /// + public long RequestedCapacity { get; } + + /// + /// Gets maximum capacity. + /// + public long MaximumCapacity { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Requested capacity. + /// Maximum stream capacity of the manager. + /// Original allocation stack. + internal StreamOverCapacityEventArgs(Guid guid, string tag, long requestedCapacity, long maximumCapacity, string allocationStack) + { + this.Id = guid; + this.Tag = tag; + this.RequestedCapacity = requestedCapacity; + this.MaximumCapacity = maximumCapacity; + this.AllocationStack = allocationStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class BlockCreatedEventArgs : EventArgs + { + /// + /// Gets how many bytes are currently in use from the small pool. + /// + public long SmallPoolInUse { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Number of bytes currently in use from the small pool. + internal BlockCreatedEventArgs(long smallPoolInUse) + { + this.SmallPoolInUse = smallPoolInUse; + } + } + + /// + /// Arguments for the events. + /// + public sealed class LargeBufferCreatedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets a value indicating whether whether the buffer was satisfied from the pool or not. + /// + public bool Pooled { get; } + + /// + /// Gets required buffer size. + /// + public long RequiredSize { get; } + + /// + /// Gets how many bytes are in use from the large pool. + /// + public long LargePoolInUse { get; } + + /// + /// Gets if the buffer was not satisfied from the pool, and is turned on, then. + /// this will contain the call stack of the allocation request. + /// + public string CallStack { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Required size of the new buffer. + /// How many bytes from the large pool are currently in use. + /// Whether the buffer was satisfied from the pool or not. + /// Call stack of the allocation, if it wasn't pooled. + internal LargeBufferCreatedEventArgs(Guid guid, string tag, long requiredSize, long largePoolInUse, bool pooled, string callStack) + { + this.RequiredSize = requiredSize; + this.LargePoolInUse = largePoolInUse; + this.Pooled = pooled; + this.Id = guid; + this.Tag = tag; + this.CallStack = callStack; + } + } + + /// + /// Arguments for the event. + /// + public sealed class BufferDiscardedEventArgs : EventArgs + { + /// + /// Gets unique ID for the stream. + /// + public Guid Id { get; } + + /// + /// Gets optional Tag for the event. + /// + public string Tag { get; } + + /// + /// Gets type of the buffer. + /// + public Events.MemoryStreamBufferType BufferType { get; } + + /// + /// Gets the reason this buffer was discarded. + /// + public Events.MemoryStreamDiscardReason Reason { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Unique ID of the stream. + /// Tag of the stream. + /// Type of buffer being discarded. + /// The reason for the discard. + internal BufferDiscardedEventArgs(Guid guid, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) + { + this.Id = guid; + this.Tag = tag; + this.BufferType = bufferType; + this.Reason = reason; + } + } + + /// + /// Arguments for the event. + /// + public sealed class StreamLengthEventArgs : EventArgs + { + /// + /// Gets length of the stream. + /// + public long Length { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Length of the strength. + public StreamLengthEventArgs(long length) + { + this.Length = length; + } + } + + /// + /// Arguments for the event. + /// + public sealed class UsageReportEventArgs : EventArgs + { + /// + /// Gets bytes from the small pool currently in use. + /// + public long SmallPoolInUseBytes { get; } + + /// + /// Gets bytes from the small pool currently available. + /// + public long SmallPoolFreeBytes { get; } + + /// + /// Gets bytes from the large pool currently in use. + /// + public long LargePoolInUseBytes { get; } + + /// + /// Gets bytes from the large pool currently available. + /// + public long LargePoolFreeBytes { get; } + + /// + /// Initializes a new instance of the class. + /// + /// Bytes from the small pool currently in use. + /// Bytes from the small pool currently available. + /// Bytes from the large pool currently in use. + /// Bytes from the large pool currently available. + public UsageReportEventArgs( + long smallPoolInUseBytes, + long smallPoolFreeBytes, + long largePoolInUseBytes, + long largePoolFreeBytes) + { + this.SmallPoolInUseBytes = smallPoolInUseBytes; + this.SmallPoolFreeBytes = smallPoolFreeBytes; + this.LargePoolInUseBytes = largePoolInUseBytes; + this.LargePoolFreeBytes = largePoolFreeBytes; + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs new file mode 100644 index 0000000000..81a24f9de8 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs @@ -0,0 +1,288 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// --------------------------------------------------------------------- +// Copyright (c) 2015 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// --------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Diagnostics.Tracing; + + /// + /// Holder for Events + /// + public sealed partial class RecyclableMemoryStreamManager + { + /// + /// ETW events for RecyclableMemoryStream. + /// + [EventSource(Name = "Microsoft-IO-RecyclableMemoryStream", Guid = "{B80CD4E4-890E-468D-9CBA-90EB7C82DFC7}")] + public sealed class Events : EventSource + { + /// + /// Static log object, through which all events are written. + /// +#pragma warning disable SA1401 // Fields should be private +#pragma warning disable CA2211 // Non-constant fields should not be visible + public static Events Writer = new (); +#pragma warning restore CA2211 // Non-constant fields should not be visible +#pragma warning restore SA1401 // Fields should be private + + /// + /// Type of buffer. + /// + public enum MemoryStreamBufferType + { + /// + /// Small block buffer. + /// + Small, + + /// + /// Large pool buffer. + /// + Large, + } + + /// + /// The possible reasons for discarding a buffer. + /// + public enum MemoryStreamDiscardReason + { + /// + /// Buffer was too large to be re-pooled. + /// + TooLarge, + + /// + /// There are enough free bytes in the pool. + /// + EnoughFree, + } + + /// + /// Logged when a stream object is created. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Requested size of the stream. + /// Actual size given to the stream from the pool. + [Event(1, Level = EventLevel.Verbose, Version = 2)] + public void MemoryStreamCreated(Guid guid, string tag, long requestedSize, long actualSize) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(1, guid, tag ?? string.Empty, requestedSize, actualSize); + } + } + + /// + /// Logged when the stream is disposed. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Lifetime in milliseconds of the stream + /// Call stack of initial allocation. + /// Call stack of the dispose. + [Event(2, Level = EventLevel.Verbose, Version = 3)] + public void MemoryStreamDisposed(Guid guid, string tag, long lifetimeMs, string allocationStack, string disposeStack) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(2, guid, tag ?? string.Empty, lifetimeMs, allocationStack ?? string.Empty, disposeStack ?? string.Empty); + } + } + + /// + /// Logged when the stream is disposed for the second time. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of initial allocation. + /// Call stack of the first dispose. + /// Call stack of the second dispose. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(3, Level = EventLevel.Critical)] + public void MemoryStreamDoubleDispose( + Guid guid, + string tag, + string allocationStack, + string disposeStack1, + string disposeStack2) + { + if (this.IsEnabled()) + { + this.WriteEvent( + 3, + guid, + tag ?? string.Empty, + allocationStack ?? string.Empty, + disposeStack1 ?? string.Empty, + disposeStack2 ?? string.Empty); + } + } + + /// + /// Logged when a stream is finalized. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of initial allocation. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(4, Level = EventLevel.Error)] + public void MemoryStreamFinalized(Guid guid, string tag, string allocationStack) + { + if (this.IsEnabled()) + { + this.WriteEvent(4, guid, tag ?? string.Empty, allocationStack ?? string.Empty); + } + } + + /// + /// Logged when ToArray is called on a stream. + /// + /// A unique ID for this stream. + /// A temporary ID for this stream, usually indicates current usage. + /// Call stack of the ToArray call. + /// Length of stream. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(5, Level = EventLevel.Verbose, Version = 2)] + public void MemoryStreamToArray(Guid guid, string tag, string stack, long size) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(5, guid, tag ?? string.Empty, stack ?? string.Empty, size); + } + } + + /// + /// Logged when the RecyclableMemoryStreamManager is initialized. + /// + /// Size of blocks, in bytes. + /// Size of the large buffer multiple, in bytes. + /// Maximum buffer size, in bytes. + [Event(6, Level = EventLevel.Informational)] + public void MemoryStreamManagerInitialized(int blockSize, int largeBufferMultiple, int maximumBufferSize) + { + if (this.IsEnabled()) + { + this.WriteEvent(6, blockSize, largeBufferMultiple, maximumBufferSize); + } + } + + /// + /// Logged when a new block is created. + /// + /// Number of bytes in the small pool currently in use. + [Event(7, Level = EventLevel.Warning, Version = 2)] + public void MemoryStreamNewBlockCreated(long smallPoolInUseBytes) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(7, smallPoolInUseBytes); + } + } + + /// + /// Logged when a new large buffer is created. + /// + /// Requested size. + /// Number of bytes in the large pool in use. + [Event(8, Level = EventLevel.Warning, Version = 3)] + public void MemoryStreamNewLargeBufferCreated(long requiredSize, long largePoolInUseBytes) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(8, requiredSize, largePoolInUseBytes); + } + } + + /// + /// Logged when a buffer is created that is too large to pool. + /// + /// Unique stream ID. + /// A temporary ID for this stream, usually indicates current usage. + /// Size requested by the caller. + /// Call stack of the requested stream. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(9, Level = EventLevel.Verbose, Version = 3)] + public void MemoryStreamNonPooledLargeBufferCreated(Guid guid, string tag, long requiredSize, string allocationStack) + { + if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) + { + this.WriteEvent(9, guid, tag ?? string.Empty, requiredSize, allocationStack ?? string.Empty); + } + } + + /// + /// Logged when a buffer is discarded (not put back in the pool, but given to GC to clean up). + /// + /// Unique stream ID. + /// A temporary ID for this stream, usually indicates current usage. + /// Type of the buffer being discarded. + /// Reason for the discard. + /// Number of free small pool blocks. + /// Bytes free in the small pool. + /// Bytes in use from the small pool. + /// Number of free large pool blocks. + /// Bytes free in the large pool. + /// Bytes in use from the large pool. + [Event(10, Level = EventLevel.Warning, Version = 2)] + public void MemoryStreamDiscardBuffer( + Guid guid, + string tag, + MemoryStreamBufferType bufferType, + MemoryStreamDiscardReason reason, + long smallBlocksFree, + long smallPoolBytesFree, + long smallPoolBytesInUse, + long largeBlocksFree, + long largePoolBytesFree, + long largePoolBytesInUse) + { + if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) + { + this.WriteEvent(10, guid, tag ?? string.Empty, bufferType, reason, smallBlocksFree, smallPoolBytesFree, smallPoolBytesInUse, largeBlocksFree, largePoolBytesFree, largePoolBytesInUse); + } + } + + /// + /// Logged when a stream grows beyond the maximum capacity. + /// + /// Unique stream ID + /// A temporary ID for this stream, usually indicates current usage. + /// The requested capacity. + /// Maximum capacity, as configured by RecyclableMemoryStreamManager. + /// Call stack for the capacity request. + /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. + [Event(11, Level = EventLevel.Error, Version = 3)] + public void MemoryStreamOverCapacity(Guid guid, string tag, long requestedCapacity, long maxCapacity, string allocationStack) + { + if (this.IsEnabled()) + { + this.WriteEvent(11, guid, tag ?? string.Empty, requestedCapacity, maxCapacity, allocationStack ?? string.Empty); + } + } + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs new file mode 100644 index 0000000000..8348ccc809 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs @@ -0,0 +1,989 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +// --------------------------------------------------------------------- +// Copyright (c) 2015-2016 Microsoft +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +// --------------------------------------------------------------------- +namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Runtime.CompilerServices; + using System.Threading; + using Microsoft.Extensions.Logging; + + /// + /// Manages pools of objects. + /// + /// + /// + /// There are two pools managed in here. The small pool contains same-sized buffers that are handed to streams + /// as they write more data. + /// + /// + /// For scenarios that need to call , the large pool contains buffers of various sizes, all + /// multiples/exponentials of (1 MB by default). They are split by size to avoid overly-wasteful buffer + /// usage. There should be far fewer 8 MB buffers than 1 MB buffers, for example. + /// + /// + public partial class RecyclableMemoryStreamManager + { + /// + /// Maximum length of a single array. + /// + /// See documentation at https://docs.microsoft.com/dotnet/api/system.array?view=netcore-3.1 + /// + internal const int MaxArrayLength = 0X7FFFFFC7; + + /// + /// Default block size, in bytes. + /// + public const int DefaultBlockSize = 128 * 1024; + + /// + /// Default large buffer multiple, in bytes. + /// + public const int DefaultLargeBufferMultiple = 1024 * 1024; + + /// + /// Default maximum buffer size, in bytes. + /// + public const int DefaultMaximumBufferSize = 128 * 1024 * 1024; + + // 0 to indicate unbounded + private const long DefaultMaxSmallPoolFreeBytes = 0L; + private const long DefaultMaxLargePoolFreeBytes = 0L; + + private readonly long[] largeBufferFreeSize; + private readonly long[] largeBufferInUseSize; + + private readonly ConcurrentStack[] largePools; + + private readonly ConcurrentStack smallPool; + +#pragma warning disable SA1401 // Fields should be private - performance reasons + internal readonly Options OptionsValue; +#pragma warning restore SA1401 // Fields should be private + + private long smallPoolFreeSize; + private long smallPoolInUseSize; + + /// + /// Gets settings for controlling the behavior of RecyclableMemoryStream + /// + public Options Settings => this.OptionsValue; + + /// + /// Gets number of bytes in small pool not currently in use. + /// + public long SmallPoolFreeSize => this.smallPoolFreeSize; + + /// + /// Gets number of bytes currently in use by stream from the small pool. + /// + public long SmallPoolInUseSize => this.smallPoolInUseSize; + + /// + /// Gets number of bytes in large pool not currently in use. + /// + public long LargePoolFreeSize + { + get + { + long sum = 0; + foreach (long freeSize in this.largeBufferFreeSize) + { + sum += freeSize; + } + + return sum; + } + } + + /// + /// Gets number of bytes currently in use by streams from the large pool. + /// + public long LargePoolInUseSize + { + get + { + long sum = 0; + foreach (long inUseSize in this.largeBufferInUseSize) + { + sum += inUseSize; + } + + return sum; + } + } + + /// + /// Gets how many blocks are in the small pool. + /// + public long SmallBlocksFree => this.smallPool.Count; + + /// + /// Gets how many buffers are in the large pool. + /// + public long LargeBuffersFree + { + get + { + long free = 0; + foreach (ConcurrentStack pool in this.largePools) + { + free += pool.Count; + } + + return free; + } + } + + /// + /// Parameters for customizing the behavior of + /// + public class Options + { + /// + /// Gets or sets the size of the pooled blocks. This must be greater than 0. + /// + /// The default size 131,072 (128KB) + public int BlockSize { get; set; } = DefaultBlockSize; + + /// + /// Gets or sets each large buffer will be a multiple exponential of this value + /// + /// The default value is 1,048,576 (1MB) + public int LargeBufferMultiple { get; set; } = DefaultLargeBufferMultiple; + + /// + /// Gets or sets buffer beyond this length are not pooled. + /// + /// The default value is 134,217,728 (128MB) + public int MaximumBufferSize { get; set; } = DefaultMaximumBufferSize; + + /// + /// Gets or sets maximum number of bytes to keep available in the small pool. + /// + /// + /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. + /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. + /// + public long MaximumSmallPoolFreeBytes { get; set; } + + /// + /// Gets or sets maximum number of bytes to keep available in the large pools. + /// + /// + /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. + /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. + /// + public long MaximumLargePoolFreeBytes { get; set; } + + /// + /// Gets or sets a value indicating whether whether to use the exponential allocation strategy (see documentation). + /// + /// The default value is false. + public bool UseExponentialLargeBuffer { get; set; } = false; + + /// + /// Gets or sets maximum stream capacity in bytes. Attempts to set a larger capacity will + /// result in an exception. + /// + /// The default value of 0 indicates no limit. + public long MaximumStreamCapacity { get; set; } = 0; + + /// + /// Gets or sets a value indicating whether whether to save call stacks for stream allocations. This can help in debugging. + /// It should NEVER be turned on generally in production. + /// + public bool GenerateCallStacks { get; set; } = false; + + /// + /// Gets or sets a value indicating whether whether dirty buffers can be immediately returned to the buffer pool. + /// + /// + /// + /// When is called on a stream and creates a single large buffer, if this setting is enabled, the other blocks will be returned + /// to the buffer pool immediately. + /// + /// + /// Note when enabling this setting that the user is responsible for ensuring that any buffer previously + /// retrieved from a stream which is subsequently modified is not used after modification (as it may no longer + /// be valid). + /// + /// + public bool AggressiveBufferReturn { get; set; } = false; + + /// + /// Gets or sets a value indicating whether causes an exception to be thrown if is ever called. + /// + /// Calling defeats the purpose of a pooled buffer. Use this property to discover code that is calling . If this is + /// set and is called, a NotSupportedException will be thrown. + public bool ThrowExceptionOnToArray { get; set; } = false; + + /// + /// Gets or sets a value indicating whether zero out buffers on allocation and before returning them to the pool. + /// + /// Setting this to true causes a performance hit and should only be set if one wants to avoid accidental data leaks. + public bool ZeroOutBuffer { get; set; } = false; + + /// + /// Creates a new object. + /// + public Options() + { + } + + /// + /// Creates a new object with the most common options. + /// + /// Size of the blocks in the small pool. + /// Size of the large buffer multiple + /// Maximum poolable buffer size. + /// Maximum bytes to hold in the small pool. + /// Maximum bytes to hold in each of the large pools. + public Options(int blockSize, int largeBufferMultiple, int maximumBufferSize, long maximumSmallPoolFreeBytes, long maximumLargePoolFreeBytes) + { + this.BlockSize = blockSize; + this.LargeBufferMultiple = largeBufferMultiple; + this.MaximumBufferSize = maximumBufferSize; + this.MaximumSmallPoolFreeBytes = maximumSmallPoolFreeBytes; + this.MaximumLargePoolFreeBytes = maximumLargePoolFreeBytes; + } + } + + /// + /// Initializes the memory manager with the default block/buffer specifications. This pool may have unbounded growth unless you modify . + /// + public RecyclableMemoryStreamManager() + : this(new Options()) + { + } + + /// + /// Initializes the memory manager with the given block requiredSize. + /// + /// Object specifying options for stream behavior. + /// + /// is not a positive number, + /// or is not a positive number, + /// or is less than options.BlockSize, + /// or is negative, + /// or is negative, + /// or is not a multiple/exponential of . + /// + public RecyclableMemoryStreamManager(Options options) + { + if (options.BlockSize <= 0) + { + throw new InvalidOperationException($"{nameof(options.BlockSize)} must be a positive number"); + } + + if (options.LargeBufferMultiple <= 0) + { + throw new InvalidOperationException($"{nameof(options.LargeBufferMultiple)} must be a positive number"); + } + + if (options.MaximumBufferSize < options.BlockSize) + { + throw new InvalidOperationException($"{nameof(options.MaximumBufferSize)} must be at least {nameof(options.BlockSize)}"); + } + + if (options.MaximumSmallPoolFreeBytes < 0) + { + throw new InvalidOperationException($"{nameof(options.MaximumSmallPoolFreeBytes)} must be non-negative"); + } + + if (options.MaximumLargePoolFreeBytes < 0) + { + throw new InvalidOperationException($"{nameof(options.MaximumLargePoolFreeBytes)} must be non-negative"); + } + + this.OptionsValue = options; + + if (!this.IsLargeBufferSize(options.MaximumBufferSize)) + { + throw new InvalidOperationException( + $"{nameof(options.MaximumBufferSize)} is not {(options.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {nameof(options.LargeBufferMultiple)}."); + } + + this.smallPool = new ConcurrentStack(); + int numLargePools = options.UseExponentialLargeBuffer + ? (int)Math.Log(options.MaximumBufferSize / options.LargeBufferMultiple, 2) + 1 + : options.MaximumBufferSize / options.LargeBufferMultiple; + + // +1 to store size of bytes in use that are too large to be pooled + this.largeBufferInUseSize = new long[numLargePools + 1]; + this.largeBufferFreeSize = new long[numLargePools]; + + this.largePools = new ConcurrentStack[numLargePools]; + + for (int i = 0; i < this.largePools.Length; ++i) + { + this.largePools[i] = new ConcurrentStack(); + } + + Events.Writer.MemoryStreamManagerInitialized(options.BlockSize, options.LargeBufferMultiple, options.MaximumBufferSize); + } + + /// + /// Removes and returns a single block from the pool. + /// + /// A byte[] array. + internal byte[] GetBlock() + { + Interlocked.Add(ref this.smallPoolInUseSize, this.OptionsValue.BlockSize); + + if (!this.smallPool.TryPop(out byte[] block)) + { + // We'll add this back to the pool when the stream is disposed + // (unless our free pool is too large) +#if NET6_0_OR_GREATER + block = this.OptionsValue.ZeroOutBuffer ? GC.AllocateArray(this.OptionsValue.BlockSize) : GC.AllocateUninitializedArray(this.OptionsValue.BlockSize); +#else + block = new byte[this.OptionsValue.BlockSize]; +#endif + this.ReportBlockCreated(); + } + else + { + Interlocked.Add(ref this.smallPoolFreeSize, -this.OptionsValue.BlockSize); + } + + return block; + } + + /// + /// Returns a buffer of arbitrary size from the large buffer pool. This buffer + /// will be at least the requiredSize and always be a multiple/exponential of largeBufferMultiple. + /// + /// The minimum length of the buffer. + /// Unique ID for the stream. + /// The tag of the stream returning this buffer, for logging if necessary. + /// A buffer of at least the required size. + /// Requested array size is larger than the maximum allowed. + internal byte[] GetLargeBuffer(long requiredSize, Guid id, string tag) + { + requiredSize = this.RoundToLargeBufferSize(requiredSize); + + if (requiredSize > MaxArrayLength) + { + throw new OutOfMemoryException($"Required buffer size exceeds maximum array length of {MaxArrayLength}."); + } + + int poolIndex = this.GetPoolIndex(requiredSize); + + bool createdNew = false; + bool pooled = true; + string callStack = null; + + byte[] buffer; + if (poolIndex < this.largePools.Length) + { + if (!this.largePools[poolIndex].TryPop(out buffer)) + { + buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); + createdNew = true; + } + else + { + Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], -buffer.Length); + } + } + else + { + // Buffer is too large to pool. They get a new buffer. + + // We still want to track the size, though, and we've reserved a slot + // in the end of the in-use array for non-pooled bytes in use. + poolIndex = this.largeBufferInUseSize.Length - 1; + + // We still want to round up to reduce heap fragmentation. + buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); + if (this.OptionsValue.GenerateCallStacks) + { + // Grab the stack -- we want to know who requires such large buffers + callStack = Environment.StackTrace; + } + + createdNew = true; + pooled = false; + } + + Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], buffer.Length); + if (createdNew) + { + this.ReportLargeBufferCreated(id, tag, requiredSize, pooled: pooled, callStack); + } + + return buffer; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static byte[] AllocateArray(long requiredSize, bool zeroInitializeArray) => +#if NET6_0_OR_GREATER + zeroInitializeArray ? GC.AllocateArray((int)requiredSize) : GC.AllocateUninitializedArray((int)requiredSize); +#else + new byte[requiredSize]; +#endif + } + + private long RoundToLargeBufferSize(long requiredSize) + { + if (this.OptionsValue.UseExponentialLargeBuffer) + { + long pow = 1; + while (this.OptionsValue.LargeBufferMultiple * pow < requiredSize) + { + pow <<= 1; + } + + return this.OptionsValue.LargeBufferMultiple * pow; + } + else + { + return (requiredSize + this.OptionsValue.LargeBufferMultiple - 1) / this.OptionsValue.LargeBufferMultiple * this.OptionsValue.LargeBufferMultiple; + } + } + + private bool IsLargeBufferSize(int value) + { + return value != 0 && (this.OptionsValue.UseExponentialLargeBuffer + ? value == this.RoundToLargeBufferSize(value) + : value % this.OptionsValue.LargeBufferMultiple == 0); + } + + private int GetPoolIndex(long length) + { + if (this.OptionsValue.UseExponentialLargeBuffer) + { + int index = 0; + while (this.OptionsValue.LargeBufferMultiple << index < length) + { + ++index; + } + + return index; + } + else + { + return (int)((length / this.OptionsValue.LargeBufferMultiple) - 1); + } + } + + /// + /// Returns the buffer to the large pool. + /// + /// The buffer to return. + /// Unique stream ID. + /// The tag of the stream returning this buffer, for logging if necessary. + /// is null. + /// buffer.Length is not a multiple/exponential of (it did not originate from this pool). + internal void ReturnLargeBuffer(byte[] buffer, Guid id, string tag) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(buffer); +#else + if (buffer == null) + { + throw new ArgumentNullException(nameof(buffer)); + } +#endif + + if (!this.IsLargeBufferSize(buffer.Length)) + { + throw new ArgumentException($"{nameof(buffer)} did not originate from this memory manager. The size is not " + + $"{(this.OptionsValue.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {this.OptionsValue.LargeBufferMultiple}."); + } + + this.ZeroOutMemoryIfEnabled(buffer); + int poolIndex = this.GetPoolIndex(buffer.Length); + if (poolIndex < this.largePools.Length) + { + if ((this.largePools[poolIndex].Count + 1) * buffer.Length <= this.OptionsValue.MaximumLargePoolFreeBytes || + this.OptionsValue.MaximumLargePoolFreeBytes == 0) + { + this.largePools[poolIndex].Push(buffer); + Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], buffer.Length); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.EnoughFree); + } + } + else + { + // This is a non-poolable buffer, but we still want to track its size for in-use + // analysis. We have space in the InUse array for this. + poolIndex = this.largeBufferInUseSize.Length - 1; + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.TooLarge); + } + + Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], -buffer.Length); + } + + /// + /// Returns the blocks to the pool. + /// + /// Collection of blocks to return to the pool. + /// Unique Stream ID. + /// The tag of the stream returning these blocks, for logging if necessary. + /// is null. + /// contains buffers that are the wrong size (or null) for this memory manager. + internal void ReturnBlocks(List blocks, Guid id, string tag) + { +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(blocks); +#else + if (blocks == null) + { + throw new ArgumentNullException(nameof(blocks)); + } +#endif + + long bytesToReturn = blocks.Count * (long)this.OptionsValue.BlockSize; + Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); + + foreach (byte[] block in blocks) + { + if (block == null || block.Length != this.OptionsValue.BlockSize) + { + throw new ArgumentException($"{nameof(blocks)} contains buffers that are not {nameof(this.OptionsValue.BlockSize)} in length.", nameof(blocks)); + } + } + + foreach (byte[] block in blocks) + { + this.ZeroOutMemoryIfEnabled(block); + if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) + { + Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); + this.smallPool.Push(block); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); + break; + } + } + } + + /// + /// Returns a block to the pool. + /// + /// Block to return to the pool. + /// Unique Stream ID. + /// The tag of the stream returning this, for logging if necessary. + /// is null. + /// is the wrong size for this memory manager. + internal void ReturnBlock(byte[] block, Guid id, string tag) + { + int bytesToReturn = this.OptionsValue.BlockSize; + Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); + +#if NET8_0_OR_GREATER + ArgumentNullException.ThrowIfNull(block); +#else + if (block == null) + { + throw new ArgumentNullException(nameof(block)); + } +#endif + + if (block.Length != this.OptionsValue.BlockSize) + { + throw new ArgumentException($"{nameof(block)} is not not {nameof(this.OptionsValue.BlockSize)} in length."); + } + + this.ZeroOutMemoryIfEnabled(block); + if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) + { + Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); + this.smallPool.Push(block); + } + else + { + this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ZeroOutMemoryIfEnabled(byte[] buffer) + { + if (this.OptionsValue.ZeroOutBuffer) + { +#if NET6_0_OR_GREATER + Array.Clear(buffer); +#else + Array.Clear(buffer, 0, buffer.Length); +#endif + } + } + + internal void ReportBlockCreated() + { + Events.Writer.MemoryStreamNewBlockCreated(this.smallPoolInUseSize); + this.BlockCreated?.Invoke(this, new BlockCreatedEventArgs(this.smallPoolInUseSize)); + } + + internal void ReportLargeBufferCreated(Guid id, string tag, long requiredSize, bool pooled, string callStack) + { + if (pooled) + { + Events.Writer.MemoryStreamNewLargeBufferCreated(requiredSize, this.LargePoolInUseSize); + } + else + { + Events.Writer.MemoryStreamNonPooledLargeBufferCreated(id, tag, requiredSize, callStack); + } + + this.LargeBufferCreated?.Invoke(this, new LargeBufferCreatedEventArgs(id, tag, requiredSize, this.LargePoolInUseSize, pooled, callStack)); + } + + internal void ReportBufferDiscarded(Guid id, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) + { + Events.Writer.MemoryStreamDiscardBuffer( + id, + tag, + bufferType, + reason, + this.SmallBlocksFree, + this.smallPoolFreeSize, + this.smallPoolInUseSize, + this.LargeBuffersFree, + this.LargePoolFreeSize, + this.LargePoolInUseSize); + this.BufferDiscarded?.Invoke(this, new BufferDiscardedEventArgs(id, tag, bufferType, reason)); + } + + internal void ReportStreamCreated(Guid id, string tag, long requestedSize, long actualSize) + { + Events.Writer.MemoryStreamCreated(id, tag, requestedSize, actualSize); + this.StreamCreated?.Invoke(this, new StreamCreatedEventArgs(id, tag, requestedSize, actualSize)); + } + + internal void ReportStreamDisposed(Guid id, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) + { + Events.Writer.MemoryStreamDisposed(id, tag, (long)lifetime.TotalMilliseconds, allocationStack, disposeStack); + this.StreamDisposed?.Invoke(this, new StreamDisposedEventArgs(id, tag, lifetime, allocationStack, disposeStack)); + } + + internal void ReportStreamDoubleDisposed(Guid id, string tag, string allocationStack, string disposeStack1, string disposeStack2) + { + Events.Writer.MemoryStreamDoubleDispose(id, tag, allocationStack, disposeStack1, disposeStack2); + this.StreamDoubleDisposed?.Invoke(this, new StreamDoubleDisposedEventArgs(id, tag, allocationStack, disposeStack1, disposeStack2)); + } + + internal void ReportStreamFinalized(Guid id, string tag, string allocationStack) + { + Events.Writer.MemoryStreamFinalized(id, tag, allocationStack); + this.StreamFinalized?.Invoke(this, new StreamFinalizedEventArgs(id, tag, allocationStack)); + } + + internal void ReportStreamLength(long bytes) + { + this.StreamLength?.Invoke(this, new StreamLengthEventArgs(bytes)); + } + + internal void ReportStreamToArray(Guid id, string tag, string stack, long length) + { + Events.Writer.MemoryStreamToArray(id, tag, stack, length); + this.StreamConvertedToArray?.Invoke(this, new StreamConvertedToArrayEventArgs(id, tag, stack, length)); + } + + internal void ReportStreamOverCapacity(Guid id, string tag, long requestedCapacity, string allocationStack) + { + Events.Writer.MemoryStreamOverCapacity(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack); + this.StreamOverCapacity?.Invoke(this, new StreamOverCapacityEventArgs(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack)); + } + + internal void ReportUsageReport() + { + this.UsageReport?.Invoke(this, new UsageReportEventArgs(this.smallPoolInUseSize, this.smallPoolFreeSize, this.LargePoolInUseSize, this.LargePoolFreeSize)); + } + + /// + /// Retrieve a new object with no tag and a default initial capacity. + /// + /// A . + public RecyclableMemoryStream GetStream() + { + return new RecyclableMemoryStream(this); + } + + /// + /// Retrieve a new object with no tag and a default initial capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id) + { + return new RecyclableMemoryStream(this, id); + } + + /// + /// Retrieve a new object with the given tag and a default initial capacity. + /// + /// A tag which can be used to track the source of the stream. + /// A . + public RecyclableMemoryStream GetStream(string tag) + { + return new RecyclableMemoryStream(this, tag); + } + + /// + /// Retrieve a new object with the given tag and a default initial capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag) + { + return new RecyclableMemoryStream(this, id, tag); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity. + /// + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// A . + public RecyclableMemoryStream GetStream(string tag, long requiredSize) + { + return new RecyclableMemoryStream(this, tag, requiredSize); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity. + /// + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize) + { + return new RecyclableMemoryStream(this, id, tag, requiredSize); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity, possibly using + /// a single contiguous underlying buffer. + /// + /// Retrieving a which provides a single contiguous buffer can be useful in situations + /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying + /// buffers to a single large one. This is most helpful when you know that you will always call + /// on the underlying stream. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// Whether to attempt to use a single contiguous buffer. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize, bool asContiguousBuffer) + { + if (!asContiguousBuffer || requiredSize <= this.OptionsValue.BlockSize) + { + return this.GetStream(id, tag, requiredSize); + } + + return new RecyclableMemoryStream(this, id, tag, requiredSize, this.GetLargeBuffer(requiredSize, id, tag)); + } + + /// + /// Retrieve a new object with the given tag and at least the given capacity, possibly using + /// a single contiguous underlying buffer. + /// + /// Retrieving a which provides a single contiguous buffer can be useful in situations + /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying + /// buffers to a single large one. This is most helpful when you know that you will always call + /// on the underlying stream. + /// A tag which can be used to track the source of the stream. + /// The minimum desired capacity for the stream. + /// Whether to attempt to use a single contiguous buffer. + /// A . + public RecyclableMemoryStream GetStream(string tag, long requiredSize, bool asContiguousBuffer) + { + return this.GetStream(Guid.NewGuid(), tag, requiredSize, asContiguousBuffer); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// The offset from the start of the buffer to copy from. + /// The number of bytes to copy from the buffer. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, byte[] buffer, int offset, int count) + { + RecyclableMemoryStream stream = null; + try + { + stream = new RecyclableMemoryStream(this, id, tag, count); + stream.Write(buffer, offset, count); + stream.Position = 0; + return stream; + } + catch + { + stream?.Dispose(); + throw; + } + } + + /// + /// Retrieve a new object with the contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(byte[] buffer) + { + return this.GetStream(null, buffer, 0, buffer.Length); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// The offset from the start of the buffer to copy from. + /// The number of bytes to copy from the buffer. + /// A . + public RecyclableMemoryStream GetStream(string tag, byte[] buffer, int offset, int count) + { + return this.GetStream(Guid.NewGuid(), tag, buffer, offset, count); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A unique identifier which can be used to trace usages of the stream. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(Guid id, string tag, ReadOnlySpan buffer) + { + RecyclableMemoryStream stream = null; + try + { + stream = new RecyclableMemoryStream(this, id, tag, buffer.Length); + stream.Write(buffer); + stream.Position = 0; + return stream; + } + catch + { + stream?.Dispose(); + throw; + } + } + + /// + /// Retrieve a new object with the contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(ReadOnlySpan buffer) + { + return this.GetStream(null, buffer); + } + + /// + /// Retrieve a new object with the given tag and with contents copied from the provided + /// buffer. The provided buffer is not wrapped or used after construction. + /// + /// The new stream's position is set to the beginning of the stream when returned. + /// A tag which can be used to track the source of the stream. + /// The byte buffer to copy data from. + /// A . + public RecyclableMemoryStream GetStream(string tag, ReadOnlySpan buffer) + { + return this.GetStream(Guid.NewGuid(), tag, buffer); + } + + /// + /// Triggered when a new block is created. + /// + public event EventHandler BlockCreated; + + /// + /// Triggered when a new large buffer is created. + /// + public event EventHandler LargeBufferCreated; + + /// + /// Triggered when a new stream is created. + /// + public event EventHandler StreamCreated; + + /// + /// Triggered when a stream is disposed. + /// + public event EventHandler StreamDisposed; + + /// + /// Triggered when a stream is disposed of twice (an error). + /// + public event EventHandler StreamDoubleDisposed; + + /// + /// Triggered when a stream is finalized. + /// + public event EventHandler StreamFinalized; + + /// + /// Triggered when a stream is disposed to report the stream's length. + /// + public event EventHandler StreamLength; + + /// + /// Triggered when a user converts a stream to array. + /// + public event EventHandler StreamConvertedToArray; + + /// + /// Triggered when a stream is requested to expand beyond the maximum length specified by the responsible RecyclableMemoryStreamManager. + /// + public event EventHandler StreamOverCapacity; + + /// + /// Triggered when a buffer of either type is discarded, along with the reason for the discard. + /// + public event EventHandler BufferDiscarded; + + /// + /// Periodically triggered to report usage statistics. + /// + public event EventHandler UsageReport; + } +} \ No newline at end of file From de555a83f2af260ca200be8a44e3aff9f096addb Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Mon, 21 Oct 2024 16:03:37 +0200 Subject: [PATCH 66/71] - drop rentArrayBufferWriter --- .../src/MemoryStreamManager.cs | 41 ++++ .../src/RentArrayBufferWriter.cs | 211 ------------------ .../src/StreamManager.cs | 31 +++ .../StreamProcessor.Encryptor.cs | 24 +- .../Readme.md | 124 +++++----- 5 files changed, 137 insertions(+), 294 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs create mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs new file mode 100644 index 0000000000..0edbdf52b2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs @@ -0,0 +1,41 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; + + /// + /// Memory Stream manager + /// + /// Placeholder + internal class MemoryStreamManager : StreamManager + { + private readonly RecyclableMemoryStreamManager streamManager = new (); + + /// + /// Create stream + /// + /// Desired minimal capacity of stream. + /// Instance of stream. + public override Stream CreateStream(int hintSize = 0) + { + return new RecyclableMemoryStream(this.streamManager, null, hintSize); + } + + /// + /// Dispose of used Stream (return to pool) + /// + /// Stream to dispose. + /// ValueTask.CompletedTask + public async override ValueTask ReturnStreamAsync(Stream stream) + { + await stream.DisposeAsync(); + } + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs deleted file mode 100644 index 5b37ded4fb..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RentArrayBufferWriter.cs +++ /dev/null @@ -1,211 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Custom; - -#if NET8_0_OR_GREATER - -using System; -using System.Buffers; -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -/// -/// https://gist.github.com/ahsonkhan/c76a1cc4dc7107537c3fdc0079a68b35 -/// Standard ArrayBufferWriter is not using pooled memory -/// -internal class RentArrayBufferWriter : IBufferWriter, IDisposable -{ - private const int MinimumBufferSize = 256; - - private byte[] rentedBuffer; - private int written; - private long committed; - - public RentArrayBufferWriter(int initialCapacity = MinimumBufferSize) - { - if (initialCapacity <= 0) - { - throw new ArgumentException(null, nameof(initialCapacity)); - } - - this.rentedBuffer = ArrayPool.Shared.Rent(initialCapacity); - this.written = 0; - this.committed = 0; - } - - public (byte[], int) WrittenBuffer - { - get - { - this.CheckIfDisposed(); - - return (this.rentedBuffer, this.written); - } - } - - public Memory WrittenMemory - { - get - { - this.CheckIfDisposed(); - - return this.rentedBuffer.AsMemory(0, this.written); - } - } - - public Span WrittenSpan - { - get - { - this.CheckIfDisposed(); - - return this.rentedBuffer.AsSpan(0, this.written); - } - } - - public int BytesWritten - { - get - { - this.CheckIfDisposed(); - - return this.written; - } - } - - public long BytesCommitted - { - get - { - this.CheckIfDisposed(); - - return this.committed; - } - } - - public void Clear() - { - this.CheckIfDisposed(); - - this.ClearHelper(); - } - - private void ClearHelper() - { - this.rentedBuffer.AsSpan(0, this.written).Clear(); - this.written = 0; - } - - public async Task CopyToAsync(Stream stream, CancellationToken cancellationToken = default) - { - this.CheckIfDisposed(); - - ArgumentNullException.ThrowIfNull(stream); - - await stream.WriteAsync(new Memory(this.rentedBuffer, 0, this.written), cancellationToken).ConfigureAwait(false); - this.committed += this.written; - - this.ClearHelper(); - } - - public void CopyTo(Stream stream) - { - this.CheckIfDisposed(); - - ArgumentNullException.ThrowIfNull(stream); - - stream.Write(this.rentedBuffer, 0, this.written); - this.committed += this.written; - - this.ClearHelper(); - } - - public void Advance(int count) - { - this.CheckIfDisposed(); - - ArgumentOutOfRangeException.ThrowIfLessThan(count, 0); - - if (this.written > this.rentedBuffer.Length - count) - { - throw new InvalidOperationException("Cannot advance past the end of the buffer."); - } - - this.written += count; - } - - // Returns the rented buffer back to the pool - public void Dispose() - { - if (this.rentedBuffer == null) - { - return; - } - - ArrayPool.Shared.Return(this.rentedBuffer, clearArray: true); - this.rentedBuffer = null; - this.written = 0; - } - - private void CheckIfDisposed() - { - ObjectDisposedException.ThrowIf(this.rentedBuffer == null, this); - } - - public Memory GetMemory(int sizeHint = 0) - { - this.CheckIfDisposed(); - - ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); - - this.CheckAndResizeBuffer(sizeHint); - return this.rentedBuffer.AsMemory(this.written); - } - - public Span GetSpan(int sizeHint = 0) - { - this.CheckIfDisposed(); - - ArgumentOutOfRangeException.ThrowIfLessThan(sizeHint, 0); - - this.CheckAndResizeBuffer(sizeHint); - return this.rentedBuffer.AsSpan(this.written); - } - - private void CheckAndResizeBuffer(int sizeHint) - { - Debug.Assert(sizeHint >= 0); - - if (sizeHint == 0) - { - sizeHint = MinimumBufferSize; - } - - int availableSpace = this.rentedBuffer.Length - this.written; - - if (sizeHint > availableSpace) - { - int growBy = sizeHint > this.rentedBuffer.Length ? sizeHint : this.rentedBuffer.Length; - - int newSize = checked(this.rentedBuffer.Length + growBy); - - byte[] oldBuffer = this.rentedBuffer; - - this.rentedBuffer = ArrayPool.Shared.Rent(newSize); - - Debug.Assert(oldBuffer.Length >= this.written); - Debug.Assert(this.rentedBuffer.Length >= this.written); - - oldBuffer.AsSpan(0, this.written).CopyTo(this.rentedBuffer); - ArrayPool.Shared.Return(oldBuffer, clearArray: true); - } - - Debug.Assert(this.rentedBuffer.Length - this.written > 0); - Debug.Assert(this.rentedBuffer.Length - this.written >= sizeHint); - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs new file mode 100644 index 0000000000..ed7831b783 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +#if NET8_0_OR_GREATER +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO; + using System.Threading.Tasks; + + /// + /// Abstraction for pooling streams + /// + public abstract class StreamManager + { + /// + /// Create stream + /// + /// Desired minimal size of stream. + /// Instance of stream. + public abstract Stream CreateStream(int hintSize = 0); + + /// + /// Dispose of used Stream (return to pool) + /// + /// Stream to dispose. + /// ValueTask.CompletedTask + public abstract ValueTask ReturnStreamAsync(Stream stream); + } +} +#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index a3980ae56a..905d640dd4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -6,17 +6,21 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; + using System.Buffers; using System.Collections.Generic; using System.IO; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; + using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; internal partial class StreamProcessor { private readonly byte[] encryptionPropertiesNameBytes = Encoding.UTF8.GetBytes(Constants.EncryptedInfo); + private readonly RecyclableMemoryStreamManager streamManager = new (); + internal async Task EncryptStreamAsync( Stream inputStream, Stream outputStream, @@ -51,7 +55,7 @@ internal async Task EncryptStreamAsync( Utf8JsonWriter encryptionPayloadWriter = null; string encryptPropertyName = null; - RentArrayBufferWriter bufferWriter = null; + RecyclableMemoryStream bufferWriter = null; while (!isFinalBlock) { @@ -112,8 +116,8 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) case JsonTokenType.StartObject: if (encryptPropertyName != null && encryptionPayloadWriter == null) { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + bufferWriter = new RecyclableMemoryStream(this.streamManager); + encryptionPayloadWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); encryptionPayloadWriter.WriteStartObject(); } else @@ -132,16 +136,17 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) { currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + byte[] bytes = bufferWriter.GetBuffer(); + int length = (int)bufferWriter.Length; ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Object); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; #pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct encryptionPayloadWriter.Dispose(); + bufferWriter.Dispose(); #pragma warning restore VSTHRD103 // Call async methods when in an async method encryptionPayloadWriter = null; - bufferWriter.Dispose(); bufferWriter = null; } @@ -149,8 +154,8 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) case JsonTokenType.StartArray: if (encryptPropertyName != null && encryptionPayloadWriter == null) { - bufferWriter = new RentArrayBufferWriter(); - encryptionPayloadWriter = new Utf8JsonWriter(bufferWriter); + bufferWriter = new RecyclableMemoryStream(this.streamManager); + encryptionPayloadWriter = new Utf8JsonWriter((IBufferWriter)bufferWriter); encryptionPayloadWriter.WriteStartArray(); } else @@ -164,16 +169,17 @@ long TransformEncryptBuffer(ReadOnlySpan buffer) if (reader.CurrentDepth == 1 && encryptionPayloadWriter != null) { currentWriter.Flush(); - (byte[] bytes, int length) = bufferWriter.WrittenBuffer; + byte[] bytes = bufferWriter.GetBuffer(); + int length = (int)bufferWriter.Length; ReadOnlySpan encryptedBytes = TransformEncryptPayload(bytes, length, TypeMarker.Array); writer.WriteBase64StringValue(encryptedBytes); encryptPropertyName = null; #pragma warning disable VSTHRD103 // Call async methods when in an async method - this method cannot be async, Utf8JsonReader is ref struct encryptionPayloadWriter.Dispose(); + bufferWriter.Dispose(); #pragma warning restore VSTHRD103 // Call async methods when in an async method encryptionPayloadWriter = null; - bufferWriter.Dispose(); bufferWriter = null; } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 22cb620269..31a77807b2 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -9,77 +9,53 @@ Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|------------------------ |----------------- |--------------------- |--------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| -| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.53 μs** | **0.511 μs** | **0.733 μs** | **22.29 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | -| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.224 μs | 0.322 μs | 26.23 μs | 0.1526 | 0.0305 | - | 41440 B | -| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.33 μs** | **0.137 μs** | **0.201 μs** | **14.32 μs** | **0.0916** | **0.0153** | **-** | **22904 B** | -| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | SystemTextJson | 14.54 μs | 0.124 μs | 0.186 μs | 14.52 μs | 0.0610 | 0.0305 | - | 21448 B | -| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **Stream** | **12.85 μs** | **0.095 μs** | **0.143 μs** | **12.84 μs** | **0.0610** | **0.0153** | **-** | **17528 B** | -| EncryptToProvidedStream | 1 | None | Stream | 13.00 μs | 0.096 μs | 0.141 μs | 12.98 μs | 0.0458 | 0.0153 | - | 11392 B | -| Decrypt | 1 | None | Stream | 13.01 μs | 0.152 μs | 0.228 μs | 13.05 μs | 0.0458 | 0.0153 | - | 12672 B | -| DecryptToProvidedStream | 1 | None | Stream | 13.48 μs | 0.132 μs | 0.197 μs | 13.45 μs | 0.0458 | 0.0153 | - | 11504 B | -| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **27.94 μs** | **0.226 μs** | **0.338 μs** | **27.96 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | -| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | Newtonsoft | 33.49 μs | 0.910 μs | 1.335 μs | 33.99 μs | 0.1221 | - | - | 41064 B | -| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **20.92 μs** | **0.136 μs** | **0.199 μs** | **20.95 μs** | **0.0610** | **-** | **-** | **21952 B** | -| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | SystemTextJson | 20.53 μs | 0.136 μs | 0.200 μs | 20.52 μs | 0.0610 | 0.0305 | - | 20488 B | -| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **Stream** | **21.15 μs** | **1.037 μs** | **1.521 μs** | **20.52 μs** | **0.0610** | **0.0305** | **-** | **16584 B** | -| EncryptToProvidedStream | 1 | Brotli | Stream | 20.57 μs | 0.213 μs | 0.292 μs | 20.57 μs | 0.0305 | - | - | 11672 B | -| Decrypt | 1 | Brotli | Stream | 21.14 μs | 2.212 μs | 3.311 μs | 19.46 μs | 0.0305 | - | - | 13216 B | -| DecryptToProvidedStream | 1 | Brotli | Stream | 19.60 μs | 0.439 μs | 0.600 μs | 19.52 μs | 0.0305 | - | - | 12048 B | -| **Encrypt** | **10** | **None** | **Newtonsoft** | **84.82 μs** | **3.002 μs** | **4.208 μs** | **83.32 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | -| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | Newtonsoft | 112.98 μs | 15.294 μs | 21.934 μs | 100.38 μs | 0.6104 | 0.1221 | - | 157425 B | -| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.85 μs** | **0.868 μs** | **1.272 μs** | **41.40 μs** | **0.4272** | **0.0610** | **-** | **105345 B** | -| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | SystemTextJson | 41.79 μs | 0.501 μs | 0.718 μs | 41.64 μs | 0.3662 | 0.0610 | - | 96464 B | -| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **Stream** | **39.63 μs** | **0.658 μs** | **0.923 μs** | **39.41 μs** | **0.3052** | **0.0610** | **-** | **82928 B** | -| EncryptToProvidedStream | 10 | None | Stream | 36.59 μs | 0.272 μs | 0.399 μs | 36.57 μs | 0.1221 | - | - | 37048 B | -| Decrypt | 10 | None | Stream | 28.64 μs | 0.378 μs | 0.517 μs | 28.59 μs | 0.1221 | 0.0305 | - | 29520 B | -| DecryptToProvidedStream | 10 | None | Stream | 27.61 μs | 0.237 μs | 0.332 μs | 27.64 μs | 0.0610 | 0.0305 | - | 18416 B | -| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **115.28 μs** | **3.336 μs** | **4.677 μs** | **113.71 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | -| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | Newtonsoft | 118.98 μs | 1.530 μs | 2.195 μs | 118.76 μs | 0.4883 | - | - | 144849 B | -| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **71.40 μs** | **0.799 μs** | **1.145 μs** | **71.23 μs** | **0.2441** | **-** | **-** | **86217 B** | -| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | SystemTextJson | 73.37 μs | 7.283 μs | 10.676 μs | 67.12 μs | 0.2441 | - | - | 82201 B | -| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **Stream** | **90.10 μs** | **3.136 μs** | **4.693 μs** | **88.92 μs** | **0.2441** | **-** | **-** | **63809 B** | -| EncryptToProvidedStream | 10 | Brotli | Stream | 97.27 μs | 1.885 μs | 2.703 μs | 97.35 μs | 0.1221 | - | - | 32465 B | -| Decrypt | 10 | Brotli | Stream | 58.48 μs | 0.956 μs | 1.372 μs | 58.59 μs | 0.1221 | 0.0610 | - | 30064 B | -| DecryptToProvidedStream | 10 | Brotli | Stream | 59.12 μs | 1.160 μs | 1.664 μs | 59.14 μs | 0.0610 | - | - | 18960 B | -| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,199.74 μs** | **42.805 μs** | **64.069 μs** | **1,206.48 μs** | **23.4375** | **21.4844** | **21.4844** | **1677978 B** | -| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | Newtonsoft | 1,177.48 μs | 25.746 μs | 38.535 μs | 1,172.04 μs | 17.5781 | 15.6250 | 15.6250 | 1260228 B | -| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **824.48 μs** | **31.605 μs** | **47.305 μs** | **812.80 μs** | **25.3906** | **25.3906** | **25.3906** | **965259 B** | -| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | SystemTextJson | 814.40 μs | 50.865 μs | 76.132 μs | 811.34 μs | 21.4844 | 21.4844 | 21.4844 | 950333 B | -| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **Stream** | **636.72 μs** | **31.468 μs** | **47.099 μs** | **630.15 μs** | **16.6016** | **16.6016** | **16.6016** | **678066 B** | -| EncryptToProvidedStream | 100 | None | Stream | 383.33 μs | 7.441 μs | 10.671 μs | 384.69 μs | 4.3945 | 4.3945 | 4.3945 | 230133 B | -| Decrypt | 100 | None | Stream | 384.93 μs | 12.519 μs | 18.738 μs | 383.59 μs | 5.8594 | 5.8594 | 5.8594 | 230753 B | -| DecryptToProvidedStream | 100 | None | Stream | 295.19 μs | 7.094 μs | 10.618 μs | 296.11 μs | 3.4180 | 3.4180 | 3.4180 | 119116 B | -| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,178.06 μs** | **63.246 μs** | **94.664 μs** | **1,152.03 μs** | **13.6719** | **11.7188** | **9.7656** | **1379183 B** | -| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | Newtonsoft | 1,175.01 μs | 41.917 μs | 61.441 μs | 1,156.01 μs | 11.7188 | 9.7656 | 9.7656 | 1124274 B | -| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **1,050.27 μs** | **31.128 μs** | **46.591 μs** | **1,052.70 μs** | **17.5781** | **17.5781** | **17.5781** | **766642 B** | -| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | SystemTextJson | 926.80 μs | 28.605 μs | 41.025 μs | 925.73 μs | 18.5547 | 18.5547 | 18.5547 | 801460 B | -| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **Stream** | **757.11 μs** | **19.549 μs** | **29.260 μs** | **754.55 μs** | **10.7422** | **10.7422** | **10.7422** | **479493 B** | -| EncryptToProvidedStream | 100 | Brotli | Stream | 563.46 μs | 9.960 μs | 14.284 μs | 561.60 μs | 2.9297 | 2.9297 | 2.9297 | 180637 B | -| Decrypt | 100 | Brotli | Stream | 542.34 μs | 14.514 μs | 21.724 μs | 542.04 μs | 6.8359 | 6.8359 | 6.8359 | 231162 B | -| DecryptToProvidedStream | 100 | Brotli | Stream | 463.69 μs | 9.130 μs | 12.800 μs | 460.71 μs | 3.4180 | 3.4180 | 3.4180 | 119506 B | +| Method | DocumentSizeInKb | CompressionAlgorithm | JsonProcessor | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | +|------------------------ |----------------- |--------------------- |-------------- |------------:|----------:|----------:|------------:|--------:|--------:|--------:|----------:| +| **Encrypt** | **1** | **None** | **Newtonsoft** | **22.51 μs** | **0.393 μs** | **0.576 μs** | **22.63 μs** | **0.1526** | **0.0305** | **-** | **41784 B** | +| EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | None | Newtonsoft | 27.10 μs | 0.124 μs | 0.174 μs | 27.07 μs | 0.1526 | 0.0305 | - | 41440 B | +| DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **None** | **Stream** | **12.82 μs** | **0.063 μs** | **0.091 μs** | **12.78 μs** | **0.0610** | **0.0153** | **-** | **17768 B** | +| EncryptToProvidedStream | 1 | None | Stream | 12.86 μs | 0.127 μs | 0.190 μs | 12.86 μs | 0.0458 | 0.0153 | - | 11632 B | +| Decrypt | 1 | None | Stream | 12.90 μs | 0.169 μs | 0.253 μs | 12.89 μs | 0.0458 | 0.0153 | - | 12672 B | +| DecryptToProvidedStream | 1 | None | Stream | 13.60 μs | 0.189 μs | 0.271 μs | 13.58 μs | 0.0458 | 0.0153 | - | 11504 B | +| **Encrypt** | **1** | **Brotli** | **Newtonsoft** | **28.87 μs** | **0.346 μs** | **0.474 μs** | **28.74 μs** | **0.1526** | **0.0305** | **-** | **38064 B** | +| EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 1 | Brotli | Newtonsoft | 35.28 μs | 0.905 μs | 1.269 μs | 35.40 μs | 0.1221 | - | - | 41064 B | +| DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **1** | **Brotli** | **Stream** | **21.52 μs** | **0.750 μs** | **1.026 μs** | **21.21 μs** | **0.0610** | **0.0305** | **-** | **16824 B** | +| EncryptToProvidedStream | 1 | Brotli | Stream | 20.80 μs | 0.228 μs | 0.312 μs | 20.76 μs | 0.0305 | - | - | 11912 B | +| Decrypt | 1 | Brotli | Stream | 19.55 μs | 0.443 μs | 0.636 μs | 19.33 μs | 0.0305 | - | - | 13216 B | +| DecryptToProvidedStream | 1 | Brotli | Stream | 19.86 μs | 0.192 μs | 0.270 μs | 19.82 μs | 0.0305 | - | - | 12048 B | +| **Encrypt** | **10** | **None** | **Newtonsoft** | **96.62 μs** | **10.278 μs** | **15.384 μs** | **86.34 μs** | **0.6104** | **0.1221** | **-** | **170993 B** | +| EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | None | Newtonsoft | 106.98 μs | 3.407 μs | 5.100 μs | 104.40 μs | 0.6104 | 0.1221 | - | 157425 B | +| DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **None** | **Stream** | **39.15 μs** | **0.200 μs** | **0.281 μs** | **39.16 μs** | **0.3052** | **0.0610** | **-** | **83168 B** | +| EncryptToProvidedStream | 10 | None | Stream | 39.27 μs | 2.127 μs | 2.982 μs | 39.00 μs | 0.1221 | - | - | 37288 B | +| Decrypt | 10 | None | Stream | 28.94 μs | 0.369 μs | 0.518 μs | 28.91 μs | 0.0916 | 0.0305 | - | 29520 B | +| DecryptToProvidedStream | 10 | None | Stream | 27.56 μs | 0.167 μs | 0.235 μs | 27.54 μs | 0.0610 | 0.0305 | - | 18416 B | +| **Encrypt** | **10** | **Brotli** | **Newtonsoft** | **116.87 μs** | **0.707 μs** | **0.991 μs** | **116.89 μs** | **0.6104** | **0.1221** | **-** | **168065 B** | +| EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 10 | Brotli | Newtonsoft | 144.59 μs | 14.519 μs | 21.282 μs | 139.95 μs | 0.4883 | - | - | 144849 B | +| DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **10** | **Brotli** | **Stream** | **91.28 μs** | **3.359 μs** | **5.027 μs** | **89.89 μs** | **0.2441** | **-** | **-** | **64049 B** | +| EncryptToProvidedStream | 10 | Brotli | Stream | 98.61 μs | 1.831 μs | 2.741 μs | 99.15 μs | 0.1221 | - | - | 32705 B | +| Decrypt | 10 | Brotli | Stream | 60.11 μs | 1.366 μs | 2.044 μs | 59.71 μs | 0.1221 | 0.0610 | - | 30064 B | +| DecryptToProvidedStream | 10 | Brotli | Stream | 58.15 μs | 1.689 μs | 2.422 μs | 58.25 μs | - | - | - | 18960 B | +| **Encrypt** | **100** | **None** | **Newtonsoft** | **1,087.44 μs** | **15.865 μs** | **23.254 μs** | **1,085.47 μs** | **21.4844** | **19.5313** | **19.5313** | **1677999 B** | +| EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | None | Newtonsoft | 1,124.12 μs | 15.278 μs | 22.395 μs | 1,123.48 μs | 17.5781 | 15.6250 | 15.6250 | 1260236 B | +| DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **None** | **Stream** | **517.26 μs** | **7.106 μs** | **10.636 μs** | **520.35 μs** | **14.6484** | **14.6484** | **14.6484** | **678303 B** | +| EncryptToProvidedStream | 100 | None | Stream | 339.83 μs | 5.149 μs | 7.706 μs | 337.59 μs | 4.3945 | 4.3945 | 4.3945 | 230367 B | +| Decrypt | 100 | None | Stream | 346.38 μs | 10.316 μs | 15.440 μs | 343.34 μs | 6.3477 | 6.3477 | 6.3477 | 230757 B | +| DecryptToProvidedStream | 100 | None | Stream | 280.22 μs | 4.289 μs | 6.420 μs | 278.61 μs | 3.4180 | 3.4180 | 3.4180 | 119111 B | +| **Encrypt** | **100** | **Brotli** | **Newtonsoft** | **1,113.95 μs** | **15.209 μs** | **22.764 μs** | **1,103.81 μs** | **13.6719** | **9.7656** | **9.7656** | **1379180 B** | +| EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| Decrypt | 100 | Brotli | Newtonsoft | 1,138.03 μs | 8.340 μs | 12.224 μs | 1,137.53 μs | 11.7188 | 9.7656 | 9.7656 | 1124260 B | +| DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | +| **Encrypt** | **100** | **Brotli** | **Stream** | **723.60 μs** | **10.132 μs** | **15.165 μs** | **719.90 μs** | **11.7188** | **11.7188** | **11.7188** | **479748 B** | +| EncryptToProvidedStream | 100 | Brotli | Stream | 551.93 μs | 7.420 μs | 10.641 μs | 550.24 μs | 2.9297 | 2.9297 | 2.9297 | 180882 B | +| Decrypt | 100 | Brotli | Stream | 540.31 μs | 12.842 μs | 19.222 μs | 542.34 μs | 6.8359 | 6.8359 | 6.8359 | 231164 B | +| DecryptToProvidedStream | 100 | Brotli | Stream | 452.60 μs | 3.476 μs | 5.203 μs | 452.38 μs | 3.4180 | 3.4180 | 3.4180 | 119509 B | From 8ce362205c636927889ab36f13fb95b9d3468dc6 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:05:20 +0200 Subject: [PATCH 67/71] ~ fix the mess after merge --- .../src/EncryptionProcessor.cs | 110 ------------------ 1 file changed, 110 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 68ed36b4ee..3c031fb7b3 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -29,7 +29,6 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static readonly StreamProcessor StreamProcessor = new (); private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); #endif @@ -114,10 +113,6 @@ public static async Task EncryptAsync( } #endif - await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); - } -#endif - /// /// If there isn't any data that needs to be decrypted, input stream will be returned without any modification. /// Else input stream will be disposed, and a new stream is returned. @@ -168,8 +163,6 @@ public static async Task EncryptAsync( JsonProcessor.Newtonsoft => await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken), #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), - JsonProcessor.SystemTextJson => await DecryptJsonNodeAsync(input, encryptor, diagnosticsContext, cancellationToken), - JsonProcessor.Stream => await DecryptStreamAsync(input, encryptor, diagnosticsContext, cancellationToken), #endif _ => throw new InvalidOperationException("Unsupported Json Processor") }; @@ -239,109 +232,6 @@ public static async Task DecryptAsync( } #endif -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(Stream, DecryptionContext)> DecryptJsonNodeAsync( - Stream input, - Stream output, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - JsonProcessor jsonProcessor, - CancellationToken cancellationToken) - { - if (input == null) - { - return null; - } - - if (jsonProcessor != JsonProcessor.Stream) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); - } - - Debug.Assert(input.CanSeek); - Debug.Assert(output.CanWrite); - Debug.Assert(output.CanSeek); - Debug.Assert(encryptor != null); - Debug.Assert(diagnosticsContext != null); - input.Position = 0; - - EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); - input.Position = 0; - if (properties?.EncryptionProperties == null) - { - await input.CopyToAsync(output, cancellationToken: cancellationToken); - return null; - } - - DecryptionContext context; -#pragma warning disable CS0618 // Type or member is obsolete - if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) - { - context = await StreamProcessor.DecryptStreamAsync(input, output, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); - } - else if (properties.EncryptionProperties.EncryptionAlgorithm == CosmosEncryptionAlgorithm.AEAes256CbcHmacSha256Randomized) - { - (Stream stream, context) = await DecryptAsync(input, encryptor, diagnosticsContext, cancellationToken); - await stream.CopyToAsync(output, cancellationToken); - output.Position = 0; - } - else - { - input.Position = 0; - throw new NotSupportedException($"Encryption Algorithm: {properties.EncryptionProperties.EncryptionAlgorithm} is not supported."); - } -#pragma warning restore CS0618 // Type or member is obsolete - - if (context == null) - { - input.Position = 0; - return null; - } - - await input.DisposeAsync(); - return context; - } -#endif - -#if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( - Stream input, - Encryptor encryptor, - CosmosDiagnosticsContext diagnosticsContext, - CancellationToken cancellationToken) - { - if (input == null) - { - return (input, null); - } - - Debug.Assert(input.CanSeek); - Debug.Assert(encryptor != null); - Debug.Assert(diagnosticsContext != null); - input.Position = 0; - - EncryptionPropertiesWrapper properties = await System.Text.Json.JsonSerializer.DeserializeAsync(input, cancellationToken: cancellationToken); - input.Position = 0; - if (properties?.EncryptionProperties == null) - { - return (input, null); - } - - MemoryStream ms = new (); - - DecryptionContext context = await StreamProcessor.DecryptStreamAsync(input, ms, encryptor, properties.EncryptionProperties, diagnosticsContext, cancellationToken); - if (context == null) - { - input.Position = 0; - return (input, null); - } - - await input.DisposeAsync(); - return (ms, context); - } - -#endif - #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER public static async Task<(Stream, DecryptionContext)> DecryptStreamAsync( Stream input, From 10ad1d4dc9036e2d2fee8d6979445dfd76243611 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:08:08 +0200 Subject: [PATCH 68/71] ~ fix merge mess --- .../src/EncryptionProcessor.cs | 9 --------- .../StreamProcessor.Decryptor.cs | 18 ------------------ ....Encryption.Custom.Performance.Tests.csproj | 4 ---- 3 files changed, 31 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 3c031fb7b3..7bf05fb930 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -29,7 +29,6 @@ internal static class EncryptionProcessor internal static readonly CosmosJsonDotNetSerializer BaseSerializer = new (JsonSerializerSettings); #if ENCRYPTION_CUSTOM_PREVIEW && NET8_0_OR_GREATER - private static readonly JsonWriterOptions JsonWriterOptions = new () { SkipValidation = true }; private static readonly StreamProcessor StreamProcessor = new (); #endif @@ -91,10 +90,6 @@ public static async Task EncryptAsync( return; } - if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); - } if (encryptionOptions.EncryptionAlgorithm != CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized) { throw new NotSupportedException($"Streaming mode is only allowed for {nameof(CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized)}"); @@ -105,10 +100,6 @@ public static async Task EncryptAsync( throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); } - if (encryptionOptions.JsonProcessor != JsonProcessor.Stream) - { - throw new NotSupportedException($"Streaming mode is only allowed for {nameof(JsonProcessor.Stream)}"); - } await EncryptionProcessor.StreamProcessor.EncryptStreamAsync(input, output, encryptor, encryptionOptions, cancellationToken); } #endif diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs index 00d7f7f224..d4ca1ccfdf 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Decryptor.cs @@ -72,8 +72,6 @@ internal async Task DecryptStreamAsync( string decryptPropertyName = null; - bool containsCompressed = properties.CompressedEncryptedPaths?.Count > 0; - while (!isFinalBlock) { int dataLength = await inputStream.ReadAsync(buffer.AsMemory(leftOver, buffer.Length - leftOver), cancellationToken); @@ -199,24 +197,9 @@ long TransformDecryptBuffer(ReadOnlySpan buffer) state = reader.CurrentState; return reader.BytesConsumed; } - } - - private void TransformDecryptProperty(ref Utf8JsonReader reader, Utf8JsonWriter writer, string decryptPropertyName, EncryptionProperties properties, DataEncryptionKey encryptionKey, bool containsCompressed, ArrayPoolManager arrayPoolManager) - { - BrotliCompressor decompressor = null; - if (properties.EncryptionFormatVersion == EncryptionFormatVersion.MdeWithCompression) - { - if (properties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) - { - throw new NotSupportedException($"Unknown compression algorithm {properties.CompressionAlgorithm}"); - } void TransformDecryptProperty(ref Utf8JsonReader reader) { - decompressor = new (); - } - } - byte[] cipherTextWithTypeMarker = arrayPoolManager.Rent(reader.ValueSpan.Length); // necessary for proper un-escaping @@ -238,7 +221,6 @@ void TransformDecryptProperty(ref Utf8JsonReader reader) bytes = buffer; } - } ReadOnlySpan bytesToWrite = bytes.AsSpan(0, processedBytes); switch ((TypeMarker)cipherTextWithTypeMarker[0]) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index c8669fd9a8..23f9b23530 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -26,10 +26,6 @@ - - - - From 81394b299e2468dbecd771b1df80c416e55f3524 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:18:46 +0200 Subject: [PATCH 69/71] - drop JsonNode based benchmarks --- .../Readme.md | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index 22cb620269..691dbf0a5d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -15,10 +15,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 1 | None | Newtonsoft | 26.31 μs | 0.224 μs | 0.322 μs | 26.23 μs | 0.1526 | 0.0305 | - | 41440 B | | DecryptToProvidedStream | 1 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **None** | **SystemTextJson** | **14.33 μs** | **0.137 μs** | **0.201 μs** | **14.32 μs** | **0.0916** | **0.0153** | **-** | **22904 B** | -| EncryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | None | SystemTextJson | 14.54 μs | 0.124 μs | 0.186 μs | 14.52 μs | 0.0610 | 0.0305 | - | 21448 B | -| DecryptToProvidedStream | 1 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **1** | **None** | **Stream** | **12.85 μs** | **0.095 μs** | **0.143 μs** | **12.84 μs** | **0.0610** | **0.0153** | **-** | **17528 B** | | EncryptToProvidedStream | 1 | None | Stream | 13.00 μs | 0.096 μs | 0.141 μs | 12.98 μs | 0.0458 | 0.0153 | - | 11392 B | | Decrypt | 1 | None | Stream | 13.01 μs | 0.152 μs | 0.228 μs | 13.05 μs | 0.0458 | 0.0153 | - | 12672 B | @@ -27,10 +23,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 1 | Brotli | Newtonsoft | 33.49 μs | 0.910 μs | 1.335 μs | 33.99 μs | 0.1221 | - | - | 41064 B | | DecryptToProvidedStream | 1 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **1** | **Brotli** | **SystemTextJson** | **20.92 μs** | **0.136 μs** | **0.199 μs** | **20.95 μs** | **0.0610** | **-** | **-** | **21952 B** | -| EncryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 1 | Brotli | SystemTextJson | 20.53 μs | 0.136 μs | 0.200 μs | 20.52 μs | 0.0610 | 0.0305 | - | 20488 B | -| DecryptToProvidedStream | 1 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **1** | **Brotli** | **Stream** | **21.15 μs** | **1.037 μs** | **1.521 μs** | **20.52 μs** | **0.0610** | **0.0305** | **-** | **16584 B** | | EncryptToProvidedStream | 1 | Brotli | Stream | 20.57 μs | 0.213 μs | 0.292 μs | 20.57 μs | 0.0305 | - | - | 11672 B | | Decrypt | 1 | Brotli | Stream | 21.14 μs | 2.212 μs | 3.311 μs | 19.46 μs | 0.0305 | - | - | 13216 B | @@ -39,10 +31,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 10 | None | Newtonsoft | 112.98 μs | 15.294 μs | 21.934 μs | 100.38 μs | 0.6104 | 0.1221 | - | 157425 B | | DecryptToProvidedStream | 10 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **None** | **SystemTextJson** | **41.85 μs** | **0.868 μs** | **1.272 μs** | **41.40 μs** | **0.4272** | **0.0610** | **-** | **105345 B** | -| EncryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | None | SystemTextJson | 41.79 μs | 0.501 μs | 0.718 μs | 41.64 μs | 0.3662 | 0.0610 | - | 96464 B | -| DecryptToProvidedStream | 10 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **10** | **None** | **Stream** | **39.63 μs** | **0.658 μs** | **0.923 μs** | **39.41 μs** | **0.3052** | **0.0610** | **-** | **82928 B** | | EncryptToProvidedStream | 10 | None | Stream | 36.59 μs | 0.272 μs | 0.399 μs | 36.57 μs | 0.1221 | - | - | 37048 B | | Decrypt | 10 | None | Stream | 28.64 μs | 0.378 μs | 0.517 μs | 28.59 μs | 0.1221 | 0.0305 | - | 29520 B | @@ -51,10 +39,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 10 | Brotli | Newtonsoft | 118.98 μs | 1.530 μs | 2.195 μs | 118.76 μs | 0.4883 | - | - | 144849 B | | DecryptToProvidedStream | 10 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **10** | **Brotli** | **SystemTextJson** | **71.40 μs** | **0.799 μs** | **1.145 μs** | **71.23 μs** | **0.2441** | **-** | **-** | **86217 B** | -| EncryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 10 | Brotli | SystemTextJson | 73.37 μs | 7.283 μs | 10.676 μs | 67.12 μs | 0.2441 | - | - | 82201 B | -| DecryptToProvidedStream | 10 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **10** | **Brotli** | **Stream** | **90.10 μs** | **3.136 μs** | **4.693 μs** | **88.92 μs** | **0.2441** | **-** | **-** | **63809 B** | | EncryptToProvidedStream | 10 | Brotli | Stream | 97.27 μs | 1.885 μs | 2.703 μs | 97.35 μs | 0.1221 | - | - | 32465 B | | Decrypt | 10 | Brotli | Stream | 58.48 μs | 0.956 μs | 1.372 μs | 58.59 μs | 0.1221 | 0.0610 | - | 30064 B | @@ -63,10 +47,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 100 | None | Newtonsoft | 1,177.48 μs | 25.746 μs | 38.535 μs | 1,172.04 μs | 17.5781 | 15.6250 | 15.6250 | 1260228 B | | DecryptToProvidedStream | 100 | None | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **None** | **SystemTextJson** | **824.48 μs** | **31.605 μs** | **47.305 μs** | **812.80 μs** | **25.3906** | **25.3906** | **25.3906** | **965259 B** | -| EncryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | None | SystemTextJson | 814.40 μs | 50.865 μs | 76.132 μs | 811.34 μs | 21.4844 | 21.4844 | 21.4844 | 950333 B | -| DecryptToProvidedStream | 100 | None | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **100** | **None** | **Stream** | **636.72 μs** | **31.468 μs** | **47.099 μs** | **630.15 μs** | **16.6016** | **16.6016** | **16.6016** | **678066 B** | | EncryptToProvidedStream | 100 | None | Stream | 383.33 μs | 7.441 μs | 10.671 μs | 384.69 μs | 4.3945 | 4.3945 | 4.3945 | 230133 B | | Decrypt | 100 | None | Stream | 384.93 μs | 12.519 μs | 18.738 μs | 383.59 μs | 5.8594 | 5.8594 | 5.8594 | 230753 B | @@ -75,10 +55,6 @@ LaunchCount=2 WarmupCount=10 | EncryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | | Decrypt | 100 | Brotli | Newtonsoft | 1,175.01 μs | 41.917 μs | 61.441 μs | 1,156.01 μs | 11.7188 | 9.7656 | 9.7656 | 1124274 B | | DecryptToProvidedStream | 100 | Brotli | Newtonsoft | NA | NA | NA | NA | - | - | - | - | -| **Encrypt** | **100** | **Brotli** | **SystemTextJson** | **1,050.27 μs** | **31.128 μs** | **46.591 μs** | **1,052.70 μs** | **17.5781** | **17.5781** | **17.5781** | **766642 B** | -| EncryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | -| Decrypt | 100 | Brotli | SystemTextJson | 926.80 μs | 28.605 μs | 41.025 μs | 925.73 μs | 18.5547 | 18.5547 | 18.5547 | 801460 B | -| DecryptToProvidedStream | 100 | Brotli | SystemTextJson | NA | NA | NA | NA | - | - | - | - | | **Encrypt** | **100** | **Brotli** | **Stream** | **757.11 μs** | **19.549 μs** | **29.260 μs** | **754.55 μs** | **10.7422** | **10.7422** | **10.7422** | **479493 B** | | EncryptToProvidedStream | 100 | Brotli | Stream | 563.46 μs | 9.960 μs | 14.284 μs | 561.60 μs | 2.9297 | 2.9297 | 2.9297 | 180637 B | | Decrypt | 100 | Brotli | Stream | 542.34 μs | 14.514 μs | 21.724 μs | 542.04 μs | 6.8359 | 6.8359 | 6.8359 | 231162 B | From b4da46cfebae637c4150f8f6dbe039c3b926a204 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Thu, 24 Oct 2024 08:33:53 +0200 Subject: [PATCH 70/71] - drop files we don't need yet --- .../src/MemoryStreamManager.cs | 41 ------------------- .../src/StreamManager.cs | 31 -------------- 2 files changed, 72 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs deleted file mode 100644 index 0edbdf52b2..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/MemoryStreamManager.cs +++ /dev/null @@ -1,41 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System.IO; - using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; - - /// - /// Memory Stream manager - /// - /// Placeholder - internal class MemoryStreamManager : StreamManager - { - private readonly RecyclableMemoryStreamManager streamManager = new (); - - /// - /// Create stream - /// - /// Desired minimal capacity of stream. - /// Instance of stream. - public override Stream CreateStream(int hintSize = 0) - { - return new RecyclableMemoryStream(this.streamManager, null, hintSize); - } - - /// - /// Dispose of used Stream (return to pool) - /// - /// Stream to dispose. - /// ValueTask.CompletedTask - public async override ValueTask ReturnStreamAsync(Stream stream) - { - await stream.DisposeAsync(); - } - } -} -#endif \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs deleted file mode 100644 index ed7831b783..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/StreamManager.cs +++ /dev/null @@ -1,31 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -#if NET8_0_OR_GREATER -namespace Microsoft.Azure.Cosmos.Encryption.Custom -{ - using System.IO; - using System.Threading.Tasks; - - /// - /// Abstraction for pooling streams - /// - public abstract class StreamManager - { - /// - /// Create stream - /// - /// Desired minimal size of stream. - /// Instance of stream. - public abstract Stream CreateStream(int hintSize = 0); - - /// - /// Dispose of used Stream (return to pool) - /// - /// Stream to dispose. - /// ValueTask.CompletedTask - public abstract ValueTask ReturnStreamAsync(Stream stream); - } -} -#endif \ No newline at end of file From ff5a8442cee6b5f8eb0ea8706225a48ac08fe971 Mon Sep 17 00:00:00 2001 From: Jan Hyka Date: Fri, 25 Oct 2024 22:52:56 +0200 Subject: [PATCH 71/71] ~ reference RecyclableMemoryStream 3.0.1 package instead --- ...soft.Azure.Cosmos.Encryption.Custom.csproj | 1 + .../RecyclableMemoryStreamMirror/README.md | 3 - .../RecyclableMemoryStream.cs | 1585 ----------------- ...RecyclableMemoryStreamManager.EventArgs.cs | 456 ----- .../RecyclableMemoryStreamManager.Events.cs | 288 --- .../RecyclableMemoryStreamManager.cs | 989 ---------- .../StreamProcessor.Encryptor.cs | 2 +- 7 files changed, 2 insertions(+), 3322 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs delete mode 100644 Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj index 3b94db54c2..f5a6c610ac 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj @@ -43,6 +43,7 @@ + diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md deleted file mode 100644 index eed315c783..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Microsoft.IO.RecyclableMemoryStream 3.0.1 - -Mirrored from https://github.com/microsoft/Microsoft.IO.RecyclableMemoryStream/tree/master/src diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs deleted file mode 100644 index 92b003d71e..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStream.cs +++ /dev/null @@ -1,1585 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -// The MIT License (MIT) -// -// Copyright (c) 2015-2016 Microsoft -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. -namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror -{ - using System; - using System.Buffers; - using System.Collections.Generic; - using System.Diagnostics; - using System.IO; - using System.Runtime.CompilerServices; - using System.Threading; - using System.Threading.Tasks; - - /// - /// MemoryStream implementation that deals with pooling and managing memory streams which use potentially large - /// buffers. - /// - /// - /// This class works in tandem with the to supply MemoryStream-derived - /// objects to callers, while avoiding these specific problems: - /// - /// - /// LOH allocations - /// Since all large buffers are pooled, they will never incur a Gen2 GC - /// - /// - /// Memory wasteA standard memory stream doubles its size when it runs out of room. This - /// leads to continual memory growth as each stream approaches the maximum allowed size. - /// - /// - /// Memory copying - /// Each time a MemoryStream grows, all the bytes are copied into new buffers. - /// This implementation only copies the bytes when is called. - /// - /// - /// Memory fragmentation - /// By using homogeneous buffer sizes, it ensures that blocks of memory - /// can be easily reused. - /// - /// - /// - /// - /// The stream is implemented on top of a series of uniformly-sized blocks. As the stream's length grows, - /// additional blocks are retrieved from the memory manager. It is these blocks that are pooled, not the stream - /// object itself. - /// - /// - /// The biggest wrinkle in this implementation is when is called. This requires a single - /// contiguous buffer. If only a single block is in use, then that block is returned. If multiple blocks - /// are in use, we retrieve a larger buffer from the memory manager. These large buffers are also pooled, - /// split by size--they are multiples/exponentials of a chunk size (1 MB by default). - /// - /// - /// Once a large buffer is assigned to the stream the small blocks are NEVER again used for this stream. All operations take place on the - /// large buffer. The large buffer can be replaced by a larger buffer from the pool as needed. All blocks and large buffers - /// are maintained in the stream until the stream is disposed (unless AggressiveBufferReturn is enabled in the stream manager). - /// - /// - /// A further wrinkle is what happens when the stream is longer than the maximum allowable array length under .NET. This is allowed - /// when only blocks are in use, and only the Read/Write APIs are used. Once a stream grows to this size, any attempt to convert it - /// to a single buffer will result in an exception. Similarly, if a stream is already converted to use a single larger buffer, then - /// it cannot grow beyond the limits of the maximum allowable array size. - /// - /// - /// Any method that modifies the stream has the potential to throw an OutOfMemoryException, either because - /// the stream is beyond the limits set in RecyclableStreamManager, or it would result in a buffer larger than - /// the maximum array size supported by .NET. - /// - /// - public sealed class RecyclableMemoryStream : MemoryStream, IBufferWriter - { - /// - /// All of these blocks must be the same size. - /// - private readonly List blocks; - - private readonly Guid id; - - private readonly RecyclableMemoryStreamManager memoryManager; - - private readonly string tag; - - private readonly long creationTimestamp; - - /// - /// This list is used to store buffers once they're replaced by something larger. - /// This is for the cases where you have users of this class that may hold onto the buffers longer - /// than they should and you want to prevent race conditions which could corrupt the data. - /// - private List dirtyBuffers; - - private bool disposed; - - /// - /// This is only set by GetBuffer() if the necessary buffer is larger than a single block size, or on - /// construction if the caller immediately requests a single large buffer. - /// - /// If this field is non-null, it contains the concatenation of the bytes found in the individual - /// blocks. Once it is created, this (or a larger) largeBuffer will be used for the life of the stream. - /// - private byte[] largeBuffer; - - /// - /// Gets unique identifier for this stream across its entire lifetime. - /// - /// Object has been disposed. - internal Guid Id - { - get - { - this.CheckDisposed(); - return this.id; - } - } - - /// - /// Gets a temporary identifier for the current usage of this stream. - /// - /// Object has been disposed. - internal string Tag - { - get - { - this.CheckDisposed(); - return this.tag; - } - } - - /// - /// Gets the memory manager being used by this stream. - /// - /// Object has been disposed. - internal RecyclableMemoryStreamManager MemoryManager - { - get - { - this.CheckDisposed(); - return this.memoryManager; - } - } - - /// - /// Gets call stack of the constructor. It is only set if is true, - /// which should only be in debugging situations. - /// - internal string AllocationStack { get; } - - /// - /// Gets call stack of the call. It is only set if is true, - /// which should only be in debugging situations. - /// - internal string DisposeStack { get; private set; } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager. - public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager) - : this(memoryManager, Guid.NewGuid(), null, 0, null) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager. - /// A unique identifier which can be used to trace usages of the stream. - public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id) - : this(memoryManager, id, null, 0, null) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager. - /// A string identifying this stream for logging and debugging purposes. - public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag) - : this(memoryManager, Guid.NewGuid(), tag, 0, null) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager. - /// A unique identifier which can be used to trace usages of the stream. - /// A string identifying this stream for logging and debugging purposes. - public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag) - : this(memoryManager, id, tag, 0, null) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager. - /// A string identifying this stream for logging and debugging purposes. - /// The initial requested size to prevent future allocations. - public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, string tag, long requestedSize) - : this(memoryManager, Guid.NewGuid(), tag, requestedSize, null) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager - /// A unique identifier which can be used to trace usages of the stream. - /// A string identifying this stream for logging and debugging purposes. - /// The initial requested size to prevent future allocations. - public RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize) - : this(memoryManager, id, tag, requestedSize, null) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// The memory manager. - /// A unique identifier which can be used to trace usages of the stream. - /// A string identifying this stream for logging and debugging purposes. - /// The initial requested size to prevent future allocations. - /// An initial buffer to use. This buffer will be owned by the stream and returned to the memory manager upon Dispose. - internal RecyclableMemoryStream(RecyclableMemoryStreamManager memoryManager, Guid id, string tag, long requestedSize, byte[] initialLargeBuffer) - : base(Array.Empty()) - { - this.memoryManager = memoryManager; - this.id = id; - this.tag = tag; - this.blocks = new List(); - this.creationTimestamp = Stopwatch.GetTimestamp(); - - long actualRequestedSize = Math.Max(requestedSize, this.memoryManager.OptionsValue.BlockSize); - - if (initialLargeBuffer == null) - { - this.EnsureCapacity(actualRequestedSize); - } - else - { - this.largeBuffer = initialLargeBuffer; - } - - if (this.memoryManager.OptionsValue.GenerateCallStacks) - { - this.AllocationStack = Environment.StackTrace; - } - - this.memoryManager.ReportStreamCreated(this.id, this.tag, requestedSize, actualRequestedSize); - this.memoryManager.ReportUsageReport(); - } - - /// - /// Finalizes an instance of the class. - /// - /// Failing to dispose indicates a bug in the code using streams. Care should be taken to properly account for stream lifetime. - ~RecyclableMemoryStream() - { - this.Dispose(false); - } - - /// - /// Returns the memory used by this stream back to the pool. - /// - /// Whether we're disposing (true), or being called by the finalizer (false). - protected override void Dispose(bool disposing) - { - if (this.disposed) - { - string doubleDisposeStack = null; - if (this.memoryManager.OptionsValue.GenerateCallStacks) - { - doubleDisposeStack = Environment.StackTrace; - } - - this.memoryManager.ReportStreamDoubleDisposed(this.id, this.tag, this.AllocationStack, this.DisposeStack, doubleDisposeStack); - return; - } - - this.disposed = true; - TimeSpan lifetime = TimeSpan.FromTicks((Stopwatch.GetTimestamp() - this.creationTimestamp) * TimeSpan.TicksPerSecond / Stopwatch.Frequency); - - if (this.memoryManager.OptionsValue.GenerateCallStacks) - { - this.DisposeStack = Environment.StackTrace; - } - - this.memoryManager.ReportStreamDisposed(this.id, this.tag, lifetime, this.AllocationStack, this.DisposeStack); - - if (disposing) - { - GC.SuppressFinalize(this); - } - else - { - // We're being finalized. - this.memoryManager.ReportStreamFinalized(this.id, this.tag, this.AllocationStack); - - if (AppDomain.CurrentDomain.IsFinalizingForUnload()) - { - // If we're being finalized because of a shutdown, don't go any further. - // We have no idea what's already been cleaned up. Triggering events may cause - // a crash. - base.Dispose(disposing); - return; - } - } - - this.memoryManager.ReportStreamLength(this.length); - - if (this.largeBuffer != null) - { - this.memoryManager.ReturnLargeBuffer(this.largeBuffer, this.id, this.tag); - } - - if (this.dirtyBuffers != null) - { - foreach (byte[] buffer in this.dirtyBuffers) - { - this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); - } - } - - this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); - this.memoryManager.ReportUsageReport(); - this.blocks.Clear(); - - base.Dispose(disposing); - } - - /// - /// Equivalent to Dispose. - /// - public override void Close() - { - this.Dispose(true); - } - - /// - /// Gets or sets the capacity. - /// - /// - /// - /// Capacity is always in multiples of the memory manager's block size, unless - /// the large buffer is in use. Capacity never decreases during a stream's lifetime. - /// Explicitly setting the capacity to a lower value than the current value will have no effect. - /// This is because the buffers are all pooled by chunks and there's little reason to - /// allow stream truncation. - /// - /// - /// Writing past the current capacity will cause to automatically increase, until MaximumStreamCapacity is reached. - /// - /// - /// If the capacity is larger than int.MaxValue, then InvalidOperationException will be thrown. If you anticipate using - /// larger streams, use the property instead. - /// - /// - /// Object has been disposed. - /// Capacity is larger than int.MaxValue. - public override int Capacity - { - get - { - this.CheckDisposed(); - if (this.largeBuffer != null) - { - return this.largeBuffer.Length; - } - - long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; - if (size > int.MaxValue) - { - throw new InvalidOperationException($"{nameof(this.Capacity)} is larger than int.MaxValue. Use {nameof(this.Capacity64)} instead."); - } - - return (int)size; - } - - set => this.Capacity64 = value; - } - - /// - /// Gets or sets returns a 64-bit version of capacity, for streams larger than int.MaxValue in length. - /// - public long Capacity64 - { - get - { - this.CheckDisposed(); - if (this.largeBuffer != null) - { - return this.largeBuffer.Length; - } - - long size = (long)this.blocks.Count * this.memoryManager.OptionsValue.BlockSize; - return size; - } - - set - { - this.CheckDisposed(); - this.EnsureCapacity(value); - } - } - - private long length; - - /// - /// Gets the number of bytes written to this stream. - /// - /// Object has been disposed. - /// If the buffer has already been converted to a large buffer, then the maximum length is limited by the maximum allowed array length in .NET. - public override long Length - { - get - { - this.CheckDisposed(); - return this.length; - } - } - - private long position; - - /// - /// Gets or sets the current position in the stream. - /// - /// Object has been disposed. - /// A negative value was passed. - /// Stream is in large-buffer mode, but an attempt was made to set the position past the maximum allowed array length. - /// If the buffer has already been converted to a large buffer, then the maximum length (and thus position) is limited by the maximum allowed array length in .NET. - public override long Position - { - get - { - this.CheckDisposed(); - return this.position; - } - - set - { - this.CheckDisposed(); - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); - } - - if (this.largeBuffer != null && value > RecyclableMemoryStreamManager.MaxArrayLength) - { - throw new InvalidOperationException($"Once the stream is converted to a single large buffer, position cannot be set past {RecyclableMemoryStreamManager.MaxArrayLength}."); - } - - this.position = value; - } - } - - /// - /// Gets a value indicating whether whether the stream can currently read. - /// - public override bool CanRead => !this.disposed; - - /// - /// Gets a value indicating whether whether the stream can currently seek. - /// - public override bool CanSeek => !this.disposed; - - /// - /// Gets a value indicating whether the steram can timeout. - /// - /// Always false - public override bool CanTimeout => false; - - /// - /// Gets a value indicating whether whether the stream can currently write. - /// - public override bool CanWrite => !this.disposed; - - /// - /// Returns a single buffer containing the contents of the stream. - /// The buffer may be longer than the stream length. - /// - /// A byte[] buffer. - /// IMPORTANT: Doing a after calling GetBuffer invalidates the buffer. The old buffer is held onto - /// until is called, but the next time GetBuffer is called, a new buffer from the pool will be required. - /// Object has been disposed. - /// stream is too large for a contiguous buffer. - public override byte[] GetBuffer() - { - this.CheckDisposed(); - - if (this.largeBuffer != null) - { - return this.largeBuffer; - } - - if (this.blocks.Count == 1) - { - return this.blocks[0]; - } - - // Buffer needs to reflect the capacity, not the length, because - // it's possible that people will manipulate the buffer directly - // and set the length afterward. Capacity sets the expectation - // for the size of the buffer. - byte[] newBuffer = this.memoryManager.GetLargeBuffer(this.Capacity64, this.id, this.tag); - - // InternalRead will check for existence of largeBuffer, so make sure we - // don't set it until after we've copied the data. - this.AssertLengthIsSmall(); - this.InternalRead(newBuffer, 0, (int)this.length, 0); - this.largeBuffer = newBuffer; - - if (this.blocks.Count > 0 && this.memoryManager.OptionsValue.AggressiveBufferReturn) - { - this.memoryManager.ReturnBlocks(this.blocks, this.id, this.tag); - this.blocks.Clear(); - } - - return this.largeBuffer; - } - -#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER - /// - public override void CopyTo(Stream destination, int bufferSize) - { - this.WriteTo(destination, this.position, this.length - this.position); - } -#endif - - /// Asynchronously reads all the bytes from the current position in this stream and writes them to another stream. - /// The stream to which the contents of the current stream will be copied. - /// This parameter is ignored. - /// The token to monitor for cancellation requests. - /// A task that represents the asynchronous copy operation. - /// - /// is . - /// Either the current stream or the destination stream is disposed. - /// The current stream does not support reading, or the destination stream does not support writing. - /// Similarly to MemoryStream's behavior, CopyToAsync will adjust the source stream's position by the number of bytes written to the destination stream, as a Read would do. - public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(destination); -#else - if (destination == null) - { - throw new ArgumentNullException(nameof(destination)); - } -#endif - - this.CheckDisposed(); - - if (this.length == 0) - { - return Task.CompletedTask; - } - - long startPos = this.position; - long count = this.length - startPos; - this.position += count; - - if (destination is MemoryStream destinationRMS) - { - this.WriteTo(destinationRMS, startPos, count); - return Task.CompletedTask; - } - else - { - if (this.largeBuffer == null) - { - if (this.blocks.Count == 1) - { - this.AssertLengthIsSmall(); - return destination.WriteAsync(this.blocks[0], (int)startPos, (int)count, cancellationToken); - } - else - { - return CopyToAsyncImpl(destination, this.GetBlockAndRelativeOffset(startPos), count, this.blocks, cancellationToken); - } - } - else - { - this.AssertLengthIsSmall(); - return destination.WriteAsync(this.largeBuffer, (int)startPos, (int)count, cancellationToken); - } - } - - static async Task CopyToAsyncImpl(Stream destination, BlockAndOffset blockAndOffset, long count, List blocks, CancellationToken cancellationToken) - { - long bytesRemaining = count; - int currentBlock = blockAndOffset.Block; - int currentOffset = blockAndOffset.Offset; - while (bytesRemaining > 0) - { - byte[] block = blocks[currentBlock]; - int amountToCopy = (int)Math.Min(block.Length - currentOffset, bytesRemaining); -#if NET8_0_OR_GREATER - await destination.WriteAsync(block.AsMemory(currentOffset, amountToCopy), cancellationToken); -#else - await destination.WriteAsync(block, currentOffset, amountToCopy, cancellationToken); -#endif - bytesRemaining -= amountToCopy; - ++currentBlock; - currentOffset = 0; - } - } - } - - private byte[] bufferWriterTempBuffer; - - /// - /// Notifies the stream that bytes were written to the buffer returned by or . - /// Seeks forward by bytes. - /// - /// - /// You must request a new buffer after calling Advance to continue writing more data and cannot write to a previously acquired buffer. - /// - /// How many bytes to advance. - /// Object has been disposed. - /// is negative. - /// is larger than the size of the previously requested buffer. - public void Advance(int count) - { - this.CheckDisposed(); - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} must be non-negative."); - } - - byte[] buffer = this.bufferWriterTempBuffer; - if (buffer != null) - { - if (count > buffer.Length) - { - throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {buffer.Length}."); - } - - this.Write(buffer, 0, count); - this.ReturnTempBuffer(buffer); - this.bufferWriterTempBuffer = null; - } - else - { - long bufferSize = this.largeBuffer == null - ? this.memoryManager.OptionsValue.BlockSize - this.GetBlockAndRelativeOffset(this.position).Offset - : this.largeBuffer.Length - this.position; - - if (count > bufferSize) - { - throw new InvalidOperationException($"Cannot advance past the end of the buffer, which has a size of {bufferSize}."); - } - - this.position += count; - this.length = Math.Max(this.position, this.length); - } - } - - private void ReturnTempBuffer(byte[] buffer) - { - if (buffer.Length == this.memoryManager.OptionsValue.BlockSize) - { - this.memoryManager.ReturnBlock(buffer, this.id, this.tag); - } - else - { - this.memoryManager.ReturnLargeBuffer(buffer, this.id, this.tag); - } - } - - /// - /// - /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), - /// or setting Position after calling GetMemory() invalidates the memory. - /// - public Memory GetMemory(int sizeHint = 0) - { - return this.GetWritableBuffer(sizeHint); - } - - /// - /// - /// IMPORTANT: Calling Write(), GetBuffer(), TryGetBuffer(), Seek(), GetLength(), Advance(), - /// or setting Position after calling GetSpan() invalidates the span. - /// - public Span GetSpan(int sizeHint = 0) - { - return this.GetWritableBuffer(sizeHint); - } - - /// - /// When callers to GetSpan() or GetMemory() request a buffer that is larger than the remaining size of the current block - /// this method return a temp buffer. When Advance() is called, that temp buffer is then copied into the stream. - /// - private ArraySegment GetWritableBuffer(int sizeHint) - { - this.CheckDisposed(); - if (sizeHint < 0) - { - throw new ArgumentOutOfRangeException(nameof(sizeHint), $"{nameof(sizeHint)} must be non-negative."); - } - - int minimumBufferSize = Math.Max(sizeHint, 1); - - this.EnsureCapacity(this.position + minimumBufferSize); - if (this.bufferWriterTempBuffer != null) - { - this.ReturnTempBuffer(this.bufferWriterTempBuffer); - this.bufferWriterTempBuffer = null; - } - - if (this.largeBuffer != null) - { - return new ArraySegment(this.largeBuffer, (int)this.position, this.largeBuffer.Length - (int)this.position); - } - - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); - int remainingBytesInBlock = this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset; - if (remainingBytesInBlock >= minimumBufferSize) - { - return new ArraySegment(this.blocks[blockAndOffset.Block], blockAndOffset.Offset, this.MemoryManager.OptionsValue.BlockSize - blockAndOffset.Offset); - } - - this.bufferWriterTempBuffer = minimumBufferSize > this.memoryManager.OptionsValue.BlockSize ? - this.memoryManager.GetLargeBuffer(minimumBufferSize, this.id, this.tag) : - this.memoryManager.GetBlock(); - - return new ArraySegment(this.bufferWriterTempBuffer); - } - - /// - /// Returns a sequence containing the contents of the stream. - /// - /// A ReadOnlySequence of bytes. - /// IMPORTANT: Calling Write(), GetMemory(), GetSpan(), Dispose(), or Close() after calling GetReadOnlySequence() invalidates the sequence. - /// Object has been disposed. - public ReadOnlySequence GetReadOnlySequence() - { - this.CheckDisposed(); - - if (this.largeBuffer != null) - { - this.AssertLengthIsSmall(); - return new ReadOnlySequence(this.largeBuffer, 0, (int)this.length); - } - - if (this.blocks.Count == 1) - { - this.AssertLengthIsSmall(); - return new ReadOnlySequence(this.blocks[0], 0, (int)this.length); - } - - BlockSegment first = new (this.blocks[0]); - BlockSegment last = first; - - for (int blockIdx = 1; last.RunningIndex + last.Memory.Length < this.length; blockIdx++) - { - last = last.Append(this.blocks[blockIdx]); - } - - return new ReadOnlySequence(first, 0, last, (int)(this.length - last.RunningIndex)); - } - - private sealed class BlockSegment : ReadOnlySequenceSegment - { - public BlockSegment(Memory memory) - { - this.Memory = memory; - } - - public BlockSegment Append(Memory memory) - { - BlockSegment nextSegment = new (memory) { RunningIndex = this.RunningIndex + this.Memory.Length }; - this.Next = nextSegment; - return nextSegment; - } - } - - /// - /// Returns an ArraySegment that wraps a single buffer containing the contents of the stream. - /// - /// An ArraySegment containing a reference to the underlying bytes. - /// Returns if a buffer can be returned; otherwise, . - public override bool TryGetBuffer(out ArraySegment buffer) - { - this.CheckDisposed(); - - try - { - if (this.length <= RecyclableMemoryStreamManager.MaxArrayLength) - { - buffer = new ArraySegment(this.GetBuffer(), 0, (int)this.Length); - return true; - } - } - catch (OutOfMemoryException) - { - } - -#if NET6_0_OR_GREATER || NETSTANDARD2_1_OR_GREATER - buffer = ArraySegment.Empty; -#else - buffer = default; -#endif - return false; - } - - /// - /// Returns a new array with a copy of the buffer's contents. You should almost certainly be using combined with the to - /// access the bytes in this stream. Calling ToArray will destroy the benefits of pooled buffers, but it is included - /// for the sake of completeness. - /// - /// Object has been disposed. - /// The current object disallows ToArray calls. - /// The length of the stream is too long for a contiguous array. - /// Array of bytes -#pragma warning disable CS0809 - [Obsolete("This method has degraded performance vs. GetBuffer and should be avoided.")] - public override byte[] ToArray() - { - this.CheckDisposed(); - - string stack = this.memoryManager.OptionsValue.GenerateCallStacks ? Environment.StackTrace : null; - this.memoryManager.ReportStreamToArray(this.id, this.tag, stack, this.length); - - if (this.memoryManager.OptionsValue.ThrowExceptionOnToArray) - { - throw new NotSupportedException("The underlying RecyclableMemoryStreamManager is configured to not allow calls to ToArray."); - } - - byte[] newBuffer = new byte[this.Length]; - - Debug.Assert(this.length <= int.MaxValue); - this.InternalRead(newBuffer, 0, (int)this.length, 0); - - return newBuffer; - } -#pragma warning restore CS0809 - - /// - /// Reads from the current position into the provided buffer. - /// - /// Destination buffer. - /// Offset into buffer at which to start placing the read bytes. - /// Number of bytes to read. - /// The number of bytes read. - /// buffer is null. - /// offset or count is less than 0. - /// offset subtracted from the buffer length is less than count. - /// Object has been disposed. - public override int Read(byte[] buffer, int offset, int count) - { - return this.SafeRead(buffer, offset, count, ref this.position); - } - - /// - /// Reads from the specified position into the provided buffer. - /// - /// Destination buffer. - /// Offset into buffer at which to start placing the read bytes. - /// Number of bytes to read. - /// Position in the stream to start reading from. - /// The number of bytes read. - /// is null. - /// or is less than 0. - /// subtracted from the buffer length is less than . - /// Object has been disposed. - public int SafeRead(byte[] buffer, int offset, int count, ref long streamPosition) - { - this.CheckDisposed(); -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(buffer); -#else - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } -#endif - - if (offset < 0) - { - throw new ArgumentOutOfRangeException(nameof(offset), $"{nameof(offset)} cannot be negative."); - } - - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count), $"{nameof(count)} cannot be negative."); - } - - if (offset + count > buffer.Length) - { - throw new ArgumentException($"{nameof(buffer)} length must be at least {nameof(offset)} + {nameof(count)}."); - } - - int amountRead = this.InternalRead(buffer, offset, count, streamPosition); - streamPosition += amountRead; - return amountRead; - } - - /// - /// Reads from the current position into the provided buffer. - /// - /// Destination buffer. - /// The number of bytes read. - /// Object has been disposed. -#if NETSTANDARD2_0 - public int Read(Span buffer) -#else - public override int Read(Span buffer) -#endif - { - return this.SafeRead(buffer, ref this.position); - } - - /// - /// Reads from the specified position into the provided buffer. - /// - /// Destination buffer. - /// Position in the stream to start reading from. - /// The number of bytes read. - /// Object has been disposed. - public int SafeRead(Span buffer, ref long streamPosition) - { - this.CheckDisposed(); - - int amountRead = this.InternalRead(buffer, streamPosition); - streamPosition += amountRead; - return amountRead; - } - - /// - /// Writes the buffer to the stream. - /// - /// Source buffer. - /// Start position. - /// Number of bytes to write. - /// buffer is null. - /// offset or count is negative. - /// buffer.Length - offset is not less than count. - /// Object has been disposed. - public override void Write(byte[] buffer, int offset, int count) - { - this.CheckDisposed(); -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(buffer); -#else - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } -#endif - - if (offset < 0) - { - throw new ArgumentOutOfRangeException( - nameof(offset), - offset, - $"{nameof(offset)} must be in the range of 0 - {nameof(buffer)}.{nameof(buffer.Length)}-1."); - } - - if (count < 0) - { - throw new ArgumentOutOfRangeException(nameof(count), count, $"{nameof(count)} must be non-negative."); - } - - if (count + offset > buffer.Length) - { - throw new ArgumentException($"{nameof(count)} must be greater than {nameof(buffer)}.{nameof(buffer.Length)} - {nameof(offset)}."); - } - - int blockSize = this.memoryManager.OptionsValue.BlockSize; - long end = this.position + count; - - this.EnsureCapacity(end); - - if (this.largeBuffer == null) - { - int bytesRemaining = count; - int bytesWritten = 0; - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); - - while (bytesRemaining > 0) - { - byte[] currentBlock = this.blocks[blockAndOffset.Block]; - int remainingInBlock = blockSize - blockAndOffset.Offset; - int amountToWriteInBlock = Math.Min(remainingInBlock, bytesRemaining); - - Buffer.BlockCopy( - buffer, - offset + bytesWritten, - currentBlock, - blockAndOffset.Offset, - amountToWriteInBlock); - - bytesRemaining -= amountToWriteInBlock; - bytesWritten += amountToWriteInBlock; - - ++blockAndOffset.Block; - blockAndOffset.Offset = 0; - } - } - else - { - Buffer.BlockCopy(buffer, offset, this.largeBuffer, (int)this.position, count); - } - - this.position = end; - this.length = Math.Max(this.position, this.length); - } - - /// - /// Writes the buffer to the stream. - /// - /// Source buffer. - /// buffer is null. - /// Object has been disposed. -#if NETSTANDARD2_0 - public void Write(ReadOnlySpan source) -#else - public override void Write(ReadOnlySpan source) -#endif - { - this.CheckDisposed(); - - int blockSize = this.memoryManager.OptionsValue.BlockSize; - long end = this.position + source.Length; - - this.EnsureCapacity(end); - - if (this.largeBuffer == null) - { - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(this.position); - - while (source.Length > 0) - { - byte[] currentBlock = this.blocks[blockAndOffset.Block]; - int remainingInBlock = blockSize - blockAndOffset.Offset; - int amountToWriteInBlock = Math.Min(remainingInBlock, source.Length); -#if NET8_0_OR_GREATER - source[..amountToWriteInBlock] - .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); - - source = source[amountToWriteInBlock..]; -#else - source.Slice(0, amountToWriteInBlock) - .CopyTo(currentBlock.AsSpan(blockAndOffset.Offset)); - - source = source.Slice(amountToWriteInBlock); -#endif - - ++blockAndOffset.Block; - blockAndOffset.Offset = 0; - } - } - else - { - source.CopyTo(this.largeBuffer.AsSpan((int)this.position)); - } - - this.position = end; - this.length = Math.Max(this.position, this.length); - } - - /// - /// Returns a useful string for debugging. This should not normally be called in actual production code. - /// - /// String with debug data. - public override string ToString() - { - if (!this.disposed) - { - return $"Id = {this.Id}, Tag = {this.Tag}, Length = {this.Length:N0} bytes"; - } - else - { - // Avoid properties because of the dispose check, but the fields themselves are not cleared. - return $"Disposed: Id = {this.id}, Tag = {this.tag}, Final Length: {this.length:N0} bytes"; - } - } - - /// - /// Writes a single byte to the current position in the stream. - /// - /// byte value to write. - /// Object has been disposed. - public override void WriteByte(byte value) - { - this.CheckDisposed(); - - long end = this.position + 1; - - if (this.largeBuffer == null) - { - int blockSize = this.memoryManager.OptionsValue.BlockSize; - - int block = (int)Math.DivRem(this.position, blockSize, out long index); - - if (block >= this.blocks.Count) - { - this.EnsureCapacity(end); - } - - this.blocks[block][index] = value; - } - else - { - if (this.position >= this.largeBuffer.Length) - { - this.EnsureCapacity(end); - } - - this.largeBuffer[this.position] = value; - } - - this.position = end; - - if (this.position > this.length) - { - this.length = this.position; - } - } - - /// - /// Reads a single byte from the current position in the stream. - /// - /// The byte at the current position, or -1 if the position is at the end of the stream. - /// Object has been disposed. - public override int ReadByte() - { - return this.SafeReadByte(ref this.position); - } - - /// - /// Reads a single byte from the specified position in the stream. - /// - /// The position in the stream to read from. - /// The byte at the current position, or -1 if the position is at the end of the stream. - /// Object has been disposed. - public int SafeReadByte(ref long streamPosition) - { - this.CheckDisposed(); - if (streamPosition == this.length) - { - return -1; - } - - byte value; - if (this.largeBuffer == null) - { - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(streamPosition); - value = this.blocks[blockAndOffset.Block][blockAndOffset.Offset]; - } - else - { - value = this.largeBuffer[streamPosition]; - } - - streamPosition++; - return value; - } - - /// - /// Sets the length of the stream. - /// - /// length of the stream - /// value is negative or larger than . - /// Object has been disposed. - public override void SetLength(long value) - { - this.CheckDisposed(); - if (value < 0) - { - throw new ArgumentOutOfRangeException(nameof(value), $"{nameof(value)} must be non-negative."); - } - - this.EnsureCapacity(value); - - this.length = value; - if (this.position > value) - { - this.position = value; - } - } - - /// - /// Sets the position to the offset from the seek location. - /// - /// How many bytes to move. - /// From where. - /// The new position. - /// Object has been disposed. - /// is larger than . - /// Invalid seek origin. - /// Attempt to set negative position. - public override long Seek(long offset, SeekOrigin loc) - { - this.CheckDisposed(); - long newPosition = loc switch - { - SeekOrigin.Begin => offset, - SeekOrigin.Current => offset + this.position, - SeekOrigin.End => offset + this.length, - _ => throw new ArgumentException("Invalid seek origin.", nameof(loc)), - }; - if (newPosition < 0) - { - throw new IOException("Seek before beginning."); - } - - this.position = newPosition; - return this.position; - } - - /// - /// Synchronously writes this stream's bytes to the argument stream. - /// - /// Destination stream. - /// Important: This does a synchronous write, which may not be desired in some situations. - /// is null. - /// Object has been disposed. - public override void WriteTo(Stream stream) - { - this.WriteTo(stream, 0, this.length); - } - - /// - /// Synchronously writes this stream's bytes, starting at offset, for count bytes, to the argument stream. - /// - /// Destination stream. - /// Offset in source. - /// Number of bytes to write. - /// is null. - /// - /// is less than 0, or + is beyond this 's length. - /// - /// Object has been disposed. - public void WriteTo(Stream stream, long offset, long count) - { - this.CheckDisposed(); -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(stream); -#else - if (stream == null) - { - throw new ArgumentNullException(nameof(stream)); - } -#endif - - if (offset < 0 || offset + count > this.length) - { - throw new ArgumentOutOfRangeException( - message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the {nameof(stream)}.", - innerException: null); - } - - if (this.largeBuffer == null) - { - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); - long bytesRemaining = count; - int currentBlock = blockAndOffset.Block; - int currentOffset = blockAndOffset.Offset; - - while (bytesRemaining > 0) - { - byte[] block = this.blocks[currentBlock]; - int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); - stream.Write(block, currentOffset, amountToCopy); - - bytesRemaining -= amountToCopy; - - ++currentBlock; - currentOffset = 0; - } - } - else - { - stream.Write(this.largeBuffer, (int)offset, (int)count); - } - } - - /// - /// Writes bytes from the current stream to a destination byte array. - /// - /// Target buffer. - /// The entire stream is written to the target array. - /// > is null. - /// Object has been disposed. - public void WriteTo(byte[] buffer) - { - this.WriteTo(buffer, 0, this.Length); - } - - /// - /// Writes bytes from the current stream to a destination byte array. - /// - /// Target buffer. - /// Offset in the source stream, from which to start. - /// Number of bytes to write. - /// > is null. - /// - /// is less than 0, or + is beyond this stream's length. - /// - /// Object has been disposed. - public void WriteTo(byte[] buffer, long offset, long count) - { - this.WriteTo(buffer, offset, count, 0); - } - - /// - /// Writes bytes from the current stream to a destination byte array. - /// - /// Target buffer. - /// Offset in the source stream, from which to start. - /// Number of bytes to write. - /// Offset in the target byte array to start writing - /// buffer is null - /// - /// is less than 0, or + is beyond this stream's length. - /// - /// - /// is less than 0, or + is beyond the target 's length. - /// - /// Object has been disposed. - public void WriteTo(byte[] buffer, long offset, long count, int targetOffset) - { - this.CheckDisposed(); -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(buffer); -#else - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } -#endif - - if (offset < 0 || offset + count > this.length) - { - throw new ArgumentOutOfRangeException( - message: $"{nameof(offset)} must not be negative and {nameof(offset)} + {nameof(count)} must not exceed the length of the stream.", - innerException: null); - } - - if (targetOffset < 0 || count + targetOffset > buffer.Length) - { - throw new ArgumentOutOfRangeException( - message: $"{nameof(targetOffset)} must not be negative and {nameof(targetOffset)} + {nameof(count)} must not exceed the length of the target {nameof(buffer)}.", - innerException: null); - } - - if (this.largeBuffer == null) - { - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(offset); - long bytesRemaining = count; - int currentBlock = blockAndOffset.Block; - int currentOffset = blockAndOffset.Offset; - int currentTargetOffset = targetOffset; - - while (bytesRemaining > 0) - { - byte[] block = this.blocks[currentBlock]; - int amountToCopy = (int)Math.Min((long)block.Length - currentOffset, bytesRemaining); - Buffer.BlockCopy(block, currentOffset, buffer, currentTargetOffset, amountToCopy); - - bytesRemaining -= amountToCopy; - - ++currentBlock; - currentOffset = 0; - currentTargetOffset += amountToCopy; - } - } - else - { - this.AssertLengthIsSmall(); - Buffer.BlockCopy(this.largeBuffer, (int)offset, buffer, targetOffset, (int)count); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void CheckDisposed() - { - if (this.disposed) - { - this.ThrowDisposedException(); - } - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private void ThrowDisposedException() - { - throw new ObjectDisposedException($"The stream with Id {this.id} and Tag {this.tag} is disposed."); - } - - private int InternalRead(byte[] buffer, int offset, int count, long fromPosition) - { - if (this.length - fromPosition <= 0) - { - return 0; - } - - int amountToCopy; - - if (this.largeBuffer == null) - { - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); - int bytesWritten = 0; - int bytesRemaining = (int)Math.Min(count, this.length - fromPosition); - - while (bytesRemaining > 0) - { - byte[] block = this.blocks[blockAndOffset.Block]; - amountToCopy = Math.Min( - block.Length - blockAndOffset.Offset, - bytesRemaining); - Buffer.BlockCopy( - block, - blockAndOffset.Offset, - buffer, - bytesWritten + offset, - amountToCopy); - - bytesWritten += amountToCopy; - bytesRemaining -= amountToCopy; - - ++blockAndOffset.Block; - blockAndOffset.Offset = 0; - } - - return bytesWritten; - } - - amountToCopy = (int)Math.Min(count, this.length - fromPosition); - Buffer.BlockCopy(this.largeBuffer, (int)fromPosition, buffer, offset, amountToCopy); - return amountToCopy; - } - - private int InternalRead(Span buffer, long fromPosition) - { - if (this.length - fromPosition <= 0) - { - return 0; - } - - int amountToCopy; - - if (this.largeBuffer == null) - { - BlockAndOffset blockAndOffset = this.GetBlockAndRelativeOffset(fromPosition); - int bytesWritten = 0; - int bytesRemaining = (int)Math.Min(buffer.Length, this.length - fromPosition); - - while (bytesRemaining > 0) - { - byte[] block = this.blocks[blockAndOffset.Block]; - amountToCopy = Math.Min( - block.Length - blockAndOffset.Offset, - bytesRemaining); -#if NET8_0_OR_GREATER - block.AsSpan(blockAndOffset.Offset, amountToCopy) - .CopyTo(buffer[bytesWritten..]); -#else - block.AsSpan(blockAndOffset.Offset, amountToCopy) - .CopyTo(buffer.Slice(bytesWritten)); -#endif - - bytesWritten += amountToCopy; - bytesRemaining -= amountToCopy; - - ++blockAndOffset.Block; - blockAndOffset.Offset = 0; - } - - return bytesWritten; - } - - amountToCopy = (int)Math.Min(buffer.Length, this.length - fromPosition); - this.largeBuffer.AsSpan((int)fromPosition, amountToCopy).CopyTo(buffer); - return amountToCopy; - } - - private struct BlockAndOffset - { - public int Block; - public int Offset; - - public BlockAndOffset(int block, int offset) - { - this.Block = block; - this.Offset = offset; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private BlockAndOffset GetBlockAndRelativeOffset(long offset) - { - int blockSize = this.memoryManager.OptionsValue.BlockSize; - int blockIndex = (int)Math.DivRem(offset, blockSize, out long offsetIndex); - return new BlockAndOffset(blockIndex, (int)offsetIndex); - } - - private void EnsureCapacity(long newCapacity) - { - if (newCapacity > this.memoryManager.OptionsValue.MaximumStreamCapacity && this.memoryManager.OptionsValue.MaximumStreamCapacity > 0) - { - this.memoryManager.ReportStreamOverCapacity(this.id, this.tag, newCapacity, this.AllocationStack); - - throw new OutOfMemoryException($"Requested capacity is too large: {newCapacity}. Limit is {this.memoryManager.OptionsValue.MaximumStreamCapacity}."); - } - - if (this.largeBuffer != null) - { - if (newCapacity > this.largeBuffer.Length) - { - byte[] newBuffer = this.memoryManager.GetLargeBuffer(newCapacity, this.id, this.tag); - Debug.Assert(this.length <= int.MaxValue); - this.InternalRead(newBuffer, 0, (int)this.length, 0); - this.ReleaseLargeBuffer(); - this.largeBuffer = newBuffer; - } - } - else - { - // Let's save some re-allocation of the blocks list - long blocksRequired = (newCapacity / this.memoryManager.OptionsValue.BlockSize) + 1; - if (this.blocks.Capacity < blocksRequired) - { - this.blocks.Capacity = (int)blocksRequired; - } - - while (this.Capacity64 < newCapacity) - { - this.blocks.Add(this.memoryManager.GetBlock()); - } - } - } - - /// - /// Release the large buffer (either stores it for eventual release or returns it immediately). - /// - private void ReleaseLargeBuffer() - { - Debug.Assert(this.largeBuffer != null); - - if (this.memoryManager.OptionsValue.AggressiveBufferReturn) - { - this.memoryManager.ReturnLargeBuffer(this.largeBuffer!, this.id, this.tag); - } - else - { - // We most likely will only ever need space for one - this.dirtyBuffers ??= new List(1); - this.dirtyBuffers.Add(this.largeBuffer!); - } - - this.largeBuffer = null; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void AssertLengthIsSmall() - { - Debug.Assert(this.length <= int.MaxValue, "this.length was assumed to be <= Int32.MaxValue, but was larger."); - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs deleted file mode 100644 index 7c450a418a..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.EventArgs.cs +++ /dev/null @@ -1,456 +0,0 @@ -// ------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -// ------------------------------------------------------------ - -namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror -{ - using System; - - /// - /// Wrapper for EventArgs - /// - public sealed partial class RecyclableMemoryStreamManager - { - /// - /// Arguments for the event. - /// - public sealed class StreamCreatedEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets requested stream size. - /// - public long RequestedSize { get; } - - /// - /// Gets actual stream size. - /// - public long ActualSize { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// The requested stream size. - /// The actual stream size. - public StreamCreatedEventArgs(Guid guid, string tag, long requestedSize, long actualSize) - { - this.Id = guid; - this.Tag = tag; - this.RequestedSize = requestedSize; - this.ActualSize = actualSize; - } - } - - /// - /// Arguments for the event. - /// - public sealed class StreamDisposedEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets stack where the stream was allocated. - /// - public string AllocationStack { get; } - - /// - /// Gets stack where stream was disposed. - /// - public string DisposeStack { get; } - - /// - /// Gets lifetime of the stream. - /// - public TimeSpan Lifetime { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Lifetime of the stream - /// Stack of original allocation. - /// Dispose stack. - public StreamDisposedEventArgs(Guid guid, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) - { - this.Id = guid; - this.Tag = tag; - this.Lifetime = lifetime; - this.AllocationStack = allocationStack; - this.DisposeStack = disposeStack; - } - } - - /// - /// Arguments for the event. - /// - public sealed class StreamDoubleDisposedEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets stack where the stream was allocated. - /// - public string AllocationStack { get; } - - /// - /// Gets first dispose stack. - /// - public string DisposeStack1 { get; } - - /// - /// Gets second dispose stack. - /// - public string DisposeStack2 { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Stack of original allocation. - /// First dispose stack. - /// Second dispose stack. - public StreamDoubleDisposedEventArgs(Guid guid, string tag, string allocationStack, string disposeStack1, string disposeStack2) - { - this.Id = guid; - this.Tag = tag; - this.AllocationStack = allocationStack; - this.DisposeStack1 = disposeStack1; - this.DisposeStack2 = disposeStack2; - } - } - - /// - /// Arguments for the event. - /// - public sealed class StreamFinalizedEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets stack where the stream was allocated. - /// - public string AllocationStack { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Stack of original allocation. - public StreamFinalizedEventArgs(Guid guid, string tag, string allocationStack) - { - this.Id = guid; - this.Tag = tag; - this.AllocationStack = allocationStack; - } - } - - /// - /// Arguments for the event. - /// - public sealed class StreamConvertedToArrayEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets stack where ToArray was called. - /// - public string Stack { get; } - - /// - /// Gets length of stack. - /// - public long Length { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Stack of ToArray call. - /// Length of stream. - public StreamConvertedToArrayEventArgs(Guid guid, string tag, string stack, long length) - { - this.Id = guid; - this.Tag = tag; - this.Stack = stack; - this.Length = length; - } - } - - /// - /// Arguments for the event. - /// - public sealed class StreamOverCapacityEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets original allocation stack. - /// - public string AllocationStack { get; } - - /// - /// Gets requested capacity. - /// - public long RequestedCapacity { get; } - - /// - /// Gets maximum capacity. - /// - public long MaximumCapacity { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Requested capacity. - /// Maximum stream capacity of the manager. - /// Original allocation stack. - internal StreamOverCapacityEventArgs(Guid guid, string tag, long requestedCapacity, long maximumCapacity, string allocationStack) - { - this.Id = guid; - this.Tag = tag; - this.RequestedCapacity = requestedCapacity; - this.MaximumCapacity = maximumCapacity; - this.AllocationStack = allocationStack; - } - } - - /// - /// Arguments for the event. - /// - public sealed class BlockCreatedEventArgs : EventArgs - { - /// - /// Gets how many bytes are currently in use from the small pool. - /// - public long SmallPoolInUse { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Number of bytes currently in use from the small pool. - internal BlockCreatedEventArgs(long smallPoolInUse) - { - this.SmallPoolInUse = smallPoolInUse; - } - } - - /// - /// Arguments for the events. - /// - public sealed class LargeBufferCreatedEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets a value indicating whether whether the buffer was satisfied from the pool or not. - /// - public bool Pooled { get; } - - /// - /// Gets required buffer size. - /// - public long RequiredSize { get; } - - /// - /// Gets how many bytes are in use from the large pool. - /// - public long LargePoolInUse { get; } - - /// - /// Gets if the buffer was not satisfied from the pool, and is turned on, then. - /// this will contain the call stack of the allocation request. - /// - public string CallStack { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Required size of the new buffer. - /// How many bytes from the large pool are currently in use. - /// Whether the buffer was satisfied from the pool or not. - /// Call stack of the allocation, if it wasn't pooled. - internal LargeBufferCreatedEventArgs(Guid guid, string tag, long requiredSize, long largePoolInUse, bool pooled, string callStack) - { - this.RequiredSize = requiredSize; - this.LargePoolInUse = largePoolInUse; - this.Pooled = pooled; - this.Id = guid; - this.Tag = tag; - this.CallStack = callStack; - } - } - - /// - /// Arguments for the event. - /// - public sealed class BufferDiscardedEventArgs : EventArgs - { - /// - /// Gets unique ID for the stream. - /// - public Guid Id { get; } - - /// - /// Gets optional Tag for the event. - /// - public string Tag { get; } - - /// - /// Gets type of the buffer. - /// - public Events.MemoryStreamBufferType BufferType { get; } - - /// - /// Gets the reason this buffer was discarded. - /// - public Events.MemoryStreamDiscardReason Reason { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Unique ID of the stream. - /// Tag of the stream. - /// Type of buffer being discarded. - /// The reason for the discard. - internal BufferDiscardedEventArgs(Guid guid, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) - { - this.Id = guid; - this.Tag = tag; - this.BufferType = bufferType; - this.Reason = reason; - } - } - - /// - /// Arguments for the event. - /// - public sealed class StreamLengthEventArgs : EventArgs - { - /// - /// Gets length of the stream. - /// - public long Length { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Length of the strength. - public StreamLengthEventArgs(long length) - { - this.Length = length; - } - } - - /// - /// Arguments for the event. - /// - public sealed class UsageReportEventArgs : EventArgs - { - /// - /// Gets bytes from the small pool currently in use. - /// - public long SmallPoolInUseBytes { get; } - - /// - /// Gets bytes from the small pool currently available. - /// - public long SmallPoolFreeBytes { get; } - - /// - /// Gets bytes from the large pool currently in use. - /// - public long LargePoolInUseBytes { get; } - - /// - /// Gets bytes from the large pool currently available. - /// - public long LargePoolFreeBytes { get; } - - /// - /// Initializes a new instance of the class. - /// - /// Bytes from the small pool currently in use. - /// Bytes from the small pool currently available. - /// Bytes from the large pool currently in use. - /// Bytes from the large pool currently available. - public UsageReportEventArgs( - long smallPoolInUseBytes, - long smallPoolFreeBytes, - long largePoolInUseBytes, - long largePoolFreeBytes) - { - this.SmallPoolInUseBytes = smallPoolInUseBytes; - this.SmallPoolFreeBytes = smallPoolFreeBytes; - this.LargePoolInUseBytes = largePoolInUseBytes; - this.LargePoolFreeBytes = largePoolFreeBytes; - } - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs deleted file mode 100644 index 81a24f9de8..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.Events.cs +++ /dev/null @@ -1,288 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -// --------------------------------------------------------------------- -// Copyright (c) 2015 Microsoft -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// --------------------------------------------------------------------- -namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror -{ - using System; - using System.Diagnostics.Tracing; - - /// - /// Holder for Events - /// - public sealed partial class RecyclableMemoryStreamManager - { - /// - /// ETW events for RecyclableMemoryStream. - /// - [EventSource(Name = "Microsoft-IO-RecyclableMemoryStream", Guid = "{B80CD4E4-890E-468D-9CBA-90EB7C82DFC7}")] - public sealed class Events : EventSource - { - /// - /// Static log object, through which all events are written. - /// -#pragma warning disable SA1401 // Fields should be private -#pragma warning disable CA2211 // Non-constant fields should not be visible - public static Events Writer = new (); -#pragma warning restore CA2211 // Non-constant fields should not be visible -#pragma warning restore SA1401 // Fields should be private - - /// - /// Type of buffer. - /// - public enum MemoryStreamBufferType - { - /// - /// Small block buffer. - /// - Small, - - /// - /// Large pool buffer. - /// - Large, - } - - /// - /// The possible reasons for discarding a buffer. - /// - public enum MemoryStreamDiscardReason - { - /// - /// Buffer was too large to be re-pooled. - /// - TooLarge, - - /// - /// There are enough free bytes in the pool. - /// - EnoughFree, - } - - /// - /// Logged when a stream object is created. - /// - /// A unique ID for this stream. - /// A temporary ID for this stream, usually indicates current usage. - /// Requested size of the stream. - /// Actual size given to the stream from the pool. - [Event(1, Level = EventLevel.Verbose, Version = 2)] - public void MemoryStreamCreated(Guid guid, string tag, long requestedSize, long actualSize) - { - if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) - { - this.WriteEvent(1, guid, tag ?? string.Empty, requestedSize, actualSize); - } - } - - /// - /// Logged when the stream is disposed. - /// - /// A unique ID for this stream. - /// A temporary ID for this stream, usually indicates current usage. - /// Lifetime in milliseconds of the stream - /// Call stack of initial allocation. - /// Call stack of the dispose. - [Event(2, Level = EventLevel.Verbose, Version = 3)] - public void MemoryStreamDisposed(Guid guid, string tag, long lifetimeMs, string allocationStack, string disposeStack) - { - if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) - { - this.WriteEvent(2, guid, tag ?? string.Empty, lifetimeMs, allocationStack ?? string.Empty, disposeStack ?? string.Empty); - } - } - - /// - /// Logged when the stream is disposed for the second time. - /// - /// A unique ID for this stream. - /// A temporary ID for this stream, usually indicates current usage. - /// Call stack of initial allocation. - /// Call stack of the first dispose. - /// Call stack of the second dispose. - /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. - [Event(3, Level = EventLevel.Critical)] - public void MemoryStreamDoubleDispose( - Guid guid, - string tag, - string allocationStack, - string disposeStack1, - string disposeStack2) - { - if (this.IsEnabled()) - { - this.WriteEvent( - 3, - guid, - tag ?? string.Empty, - allocationStack ?? string.Empty, - disposeStack1 ?? string.Empty, - disposeStack2 ?? string.Empty); - } - } - - /// - /// Logged when a stream is finalized. - /// - /// A unique ID for this stream. - /// A temporary ID for this stream, usually indicates current usage. - /// Call stack of initial allocation. - /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. - [Event(4, Level = EventLevel.Error)] - public void MemoryStreamFinalized(Guid guid, string tag, string allocationStack) - { - if (this.IsEnabled()) - { - this.WriteEvent(4, guid, tag ?? string.Empty, allocationStack ?? string.Empty); - } - } - - /// - /// Logged when ToArray is called on a stream. - /// - /// A unique ID for this stream. - /// A temporary ID for this stream, usually indicates current usage. - /// Call stack of the ToArray call. - /// Length of stream. - /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. - [Event(5, Level = EventLevel.Verbose, Version = 2)] - public void MemoryStreamToArray(Guid guid, string tag, string stack, long size) - { - if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) - { - this.WriteEvent(5, guid, tag ?? string.Empty, stack ?? string.Empty, size); - } - } - - /// - /// Logged when the RecyclableMemoryStreamManager is initialized. - /// - /// Size of blocks, in bytes. - /// Size of the large buffer multiple, in bytes. - /// Maximum buffer size, in bytes. - [Event(6, Level = EventLevel.Informational)] - public void MemoryStreamManagerInitialized(int blockSize, int largeBufferMultiple, int maximumBufferSize) - { - if (this.IsEnabled()) - { - this.WriteEvent(6, blockSize, largeBufferMultiple, maximumBufferSize); - } - } - - /// - /// Logged when a new block is created. - /// - /// Number of bytes in the small pool currently in use. - [Event(7, Level = EventLevel.Warning, Version = 2)] - public void MemoryStreamNewBlockCreated(long smallPoolInUseBytes) - { - if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) - { - this.WriteEvent(7, smallPoolInUseBytes); - } - } - - /// - /// Logged when a new large buffer is created. - /// - /// Requested size. - /// Number of bytes in the large pool in use. - [Event(8, Level = EventLevel.Warning, Version = 3)] - public void MemoryStreamNewLargeBufferCreated(long requiredSize, long largePoolInUseBytes) - { - if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) - { - this.WriteEvent(8, requiredSize, largePoolInUseBytes); - } - } - - /// - /// Logged when a buffer is created that is too large to pool. - /// - /// Unique stream ID. - /// A temporary ID for this stream, usually indicates current usage. - /// Size requested by the caller. - /// Call stack of the requested stream. - /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. - [Event(9, Level = EventLevel.Verbose, Version = 3)] - public void MemoryStreamNonPooledLargeBufferCreated(Guid guid, string tag, long requiredSize, string allocationStack) - { - if (this.IsEnabled(EventLevel.Verbose, EventKeywords.None)) - { - this.WriteEvent(9, guid, tag ?? string.Empty, requiredSize, allocationStack ?? string.Empty); - } - } - - /// - /// Logged when a buffer is discarded (not put back in the pool, but given to GC to clean up). - /// - /// Unique stream ID. - /// A temporary ID for this stream, usually indicates current usage. - /// Type of the buffer being discarded. - /// Reason for the discard. - /// Number of free small pool blocks. - /// Bytes free in the small pool. - /// Bytes in use from the small pool. - /// Number of free large pool blocks. - /// Bytes free in the large pool. - /// Bytes in use from the large pool. - [Event(10, Level = EventLevel.Warning, Version = 2)] - public void MemoryStreamDiscardBuffer( - Guid guid, - string tag, - MemoryStreamBufferType bufferType, - MemoryStreamDiscardReason reason, - long smallBlocksFree, - long smallPoolBytesFree, - long smallPoolBytesInUse, - long largeBlocksFree, - long largePoolBytesFree, - long largePoolBytesInUse) - { - if (this.IsEnabled(EventLevel.Warning, EventKeywords.None)) - { - this.WriteEvent(10, guid, tag ?? string.Empty, bufferType, reason, smallBlocksFree, smallPoolBytesFree, smallPoolBytesInUse, largeBlocksFree, largePoolBytesFree, largePoolBytesInUse); - } - } - - /// - /// Logged when a stream grows beyond the maximum capacity. - /// - /// Unique stream ID - /// A temporary ID for this stream, usually indicates current usage. - /// The requested capacity. - /// Maximum capacity, as configured by RecyclableMemoryStreamManager. - /// Call stack for the capacity request. - /// Note: Stacks will only be populated if RecyclableMemoryStreamManager.GenerateCallStacks is true. - [Event(11, Level = EventLevel.Error, Version = 3)] - public void MemoryStreamOverCapacity(Guid guid, string tag, long requestedCapacity, long maxCapacity, string allocationStack) - { - if (this.IsEnabled()) - { - this.WriteEvent(11, guid, tag ?? string.Empty, requestedCapacity, maxCapacity, allocationStack ?? string.Empty); - } - } - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs deleted file mode 100644 index 8348ccc809..0000000000 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/RecyclableMemoryStreamMirror/RecyclableMemoryStreamManager.cs +++ /dev/null @@ -1,989 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -// --------------------------------------------------------------------- -// Copyright (c) 2015-2016 Microsoft -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -// --------------------------------------------------------------------- -namespace Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror -{ - using System; - using System.Collections.Concurrent; - using System.Collections.Generic; - using System.Runtime.CompilerServices; - using System.Threading; - using Microsoft.Extensions.Logging; - - /// - /// Manages pools of objects. - /// - /// - /// - /// There are two pools managed in here. The small pool contains same-sized buffers that are handed to streams - /// as they write more data. - /// - /// - /// For scenarios that need to call , the large pool contains buffers of various sizes, all - /// multiples/exponentials of (1 MB by default). They are split by size to avoid overly-wasteful buffer - /// usage. There should be far fewer 8 MB buffers than 1 MB buffers, for example. - /// - /// - public partial class RecyclableMemoryStreamManager - { - /// - /// Maximum length of a single array. - /// - /// See documentation at https://docs.microsoft.com/dotnet/api/system.array?view=netcore-3.1 - /// - internal const int MaxArrayLength = 0X7FFFFFC7; - - /// - /// Default block size, in bytes. - /// - public const int DefaultBlockSize = 128 * 1024; - - /// - /// Default large buffer multiple, in bytes. - /// - public const int DefaultLargeBufferMultiple = 1024 * 1024; - - /// - /// Default maximum buffer size, in bytes. - /// - public const int DefaultMaximumBufferSize = 128 * 1024 * 1024; - - // 0 to indicate unbounded - private const long DefaultMaxSmallPoolFreeBytes = 0L; - private const long DefaultMaxLargePoolFreeBytes = 0L; - - private readonly long[] largeBufferFreeSize; - private readonly long[] largeBufferInUseSize; - - private readonly ConcurrentStack[] largePools; - - private readonly ConcurrentStack smallPool; - -#pragma warning disable SA1401 // Fields should be private - performance reasons - internal readonly Options OptionsValue; -#pragma warning restore SA1401 // Fields should be private - - private long smallPoolFreeSize; - private long smallPoolInUseSize; - - /// - /// Gets settings for controlling the behavior of RecyclableMemoryStream - /// - public Options Settings => this.OptionsValue; - - /// - /// Gets number of bytes in small pool not currently in use. - /// - public long SmallPoolFreeSize => this.smallPoolFreeSize; - - /// - /// Gets number of bytes currently in use by stream from the small pool. - /// - public long SmallPoolInUseSize => this.smallPoolInUseSize; - - /// - /// Gets number of bytes in large pool not currently in use. - /// - public long LargePoolFreeSize - { - get - { - long sum = 0; - foreach (long freeSize in this.largeBufferFreeSize) - { - sum += freeSize; - } - - return sum; - } - } - - /// - /// Gets number of bytes currently in use by streams from the large pool. - /// - public long LargePoolInUseSize - { - get - { - long sum = 0; - foreach (long inUseSize in this.largeBufferInUseSize) - { - sum += inUseSize; - } - - return sum; - } - } - - /// - /// Gets how many blocks are in the small pool. - /// - public long SmallBlocksFree => this.smallPool.Count; - - /// - /// Gets how many buffers are in the large pool. - /// - public long LargeBuffersFree - { - get - { - long free = 0; - foreach (ConcurrentStack pool in this.largePools) - { - free += pool.Count; - } - - return free; - } - } - - /// - /// Parameters for customizing the behavior of - /// - public class Options - { - /// - /// Gets or sets the size of the pooled blocks. This must be greater than 0. - /// - /// The default size 131,072 (128KB) - public int BlockSize { get; set; } = DefaultBlockSize; - - /// - /// Gets or sets each large buffer will be a multiple exponential of this value - /// - /// The default value is 1,048,576 (1MB) - public int LargeBufferMultiple { get; set; } = DefaultLargeBufferMultiple; - - /// - /// Gets or sets buffer beyond this length are not pooled. - /// - /// The default value is 134,217,728 (128MB) - public int MaximumBufferSize { get; set; } = DefaultMaximumBufferSize; - - /// - /// Gets or sets maximum number of bytes to keep available in the small pool. - /// - /// - /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. - /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. - /// - public long MaximumSmallPoolFreeBytes { get; set; } - - /// - /// Gets or sets maximum number of bytes to keep available in the large pools. - /// - /// - /// Trying to return buffers to the pool beyond this limit will result in them being garbage collected. - /// The default value is 0, but all users should set a reasonable value depending on your application's memory requirements. - /// - public long MaximumLargePoolFreeBytes { get; set; } - - /// - /// Gets or sets a value indicating whether whether to use the exponential allocation strategy (see documentation). - /// - /// The default value is false. - public bool UseExponentialLargeBuffer { get; set; } = false; - - /// - /// Gets or sets maximum stream capacity in bytes. Attempts to set a larger capacity will - /// result in an exception. - /// - /// The default value of 0 indicates no limit. - public long MaximumStreamCapacity { get; set; } = 0; - - /// - /// Gets or sets a value indicating whether whether to save call stacks for stream allocations. This can help in debugging. - /// It should NEVER be turned on generally in production. - /// - public bool GenerateCallStacks { get; set; } = false; - - /// - /// Gets or sets a value indicating whether whether dirty buffers can be immediately returned to the buffer pool. - /// - /// - /// - /// When is called on a stream and creates a single large buffer, if this setting is enabled, the other blocks will be returned - /// to the buffer pool immediately. - /// - /// - /// Note when enabling this setting that the user is responsible for ensuring that any buffer previously - /// retrieved from a stream which is subsequently modified is not used after modification (as it may no longer - /// be valid). - /// - /// - public bool AggressiveBufferReturn { get; set; } = false; - - /// - /// Gets or sets a value indicating whether causes an exception to be thrown if is ever called. - /// - /// Calling defeats the purpose of a pooled buffer. Use this property to discover code that is calling . If this is - /// set and is called, a NotSupportedException will be thrown. - public bool ThrowExceptionOnToArray { get; set; } = false; - - /// - /// Gets or sets a value indicating whether zero out buffers on allocation and before returning them to the pool. - /// - /// Setting this to true causes a performance hit and should only be set if one wants to avoid accidental data leaks. - public bool ZeroOutBuffer { get; set; } = false; - - /// - /// Creates a new object. - /// - public Options() - { - } - - /// - /// Creates a new object with the most common options. - /// - /// Size of the blocks in the small pool. - /// Size of the large buffer multiple - /// Maximum poolable buffer size. - /// Maximum bytes to hold in the small pool. - /// Maximum bytes to hold in each of the large pools. - public Options(int blockSize, int largeBufferMultiple, int maximumBufferSize, long maximumSmallPoolFreeBytes, long maximumLargePoolFreeBytes) - { - this.BlockSize = blockSize; - this.LargeBufferMultiple = largeBufferMultiple; - this.MaximumBufferSize = maximumBufferSize; - this.MaximumSmallPoolFreeBytes = maximumSmallPoolFreeBytes; - this.MaximumLargePoolFreeBytes = maximumLargePoolFreeBytes; - } - } - - /// - /// Initializes the memory manager with the default block/buffer specifications. This pool may have unbounded growth unless you modify . - /// - public RecyclableMemoryStreamManager() - : this(new Options()) - { - } - - /// - /// Initializes the memory manager with the given block requiredSize. - /// - /// Object specifying options for stream behavior. - /// - /// is not a positive number, - /// or is not a positive number, - /// or is less than options.BlockSize, - /// or is negative, - /// or is negative, - /// or is not a multiple/exponential of . - /// - public RecyclableMemoryStreamManager(Options options) - { - if (options.BlockSize <= 0) - { - throw new InvalidOperationException($"{nameof(options.BlockSize)} must be a positive number"); - } - - if (options.LargeBufferMultiple <= 0) - { - throw new InvalidOperationException($"{nameof(options.LargeBufferMultiple)} must be a positive number"); - } - - if (options.MaximumBufferSize < options.BlockSize) - { - throw new InvalidOperationException($"{nameof(options.MaximumBufferSize)} must be at least {nameof(options.BlockSize)}"); - } - - if (options.MaximumSmallPoolFreeBytes < 0) - { - throw new InvalidOperationException($"{nameof(options.MaximumSmallPoolFreeBytes)} must be non-negative"); - } - - if (options.MaximumLargePoolFreeBytes < 0) - { - throw new InvalidOperationException($"{nameof(options.MaximumLargePoolFreeBytes)} must be non-negative"); - } - - this.OptionsValue = options; - - if (!this.IsLargeBufferSize(options.MaximumBufferSize)) - { - throw new InvalidOperationException( - $"{nameof(options.MaximumBufferSize)} is not {(options.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {nameof(options.LargeBufferMultiple)}."); - } - - this.smallPool = new ConcurrentStack(); - int numLargePools = options.UseExponentialLargeBuffer - ? (int)Math.Log(options.MaximumBufferSize / options.LargeBufferMultiple, 2) + 1 - : options.MaximumBufferSize / options.LargeBufferMultiple; - - // +1 to store size of bytes in use that are too large to be pooled - this.largeBufferInUseSize = new long[numLargePools + 1]; - this.largeBufferFreeSize = new long[numLargePools]; - - this.largePools = new ConcurrentStack[numLargePools]; - - for (int i = 0; i < this.largePools.Length; ++i) - { - this.largePools[i] = new ConcurrentStack(); - } - - Events.Writer.MemoryStreamManagerInitialized(options.BlockSize, options.LargeBufferMultiple, options.MaximumBufferSize); - } - - /// - /// Removes and returns a single block from the pool. - /// - /// A byte[] array. - internal byte[] GetBlock() - { - Interlocked.Add(ref this.smallPoolInUseSize, this.OptionsValue.BlockSize); - - if (!this.smallPool.TryPop(out byte[] block)) - { - // We'll add this back to the pool when the stream is disposed - // (unless our free pool is too large) -#if NET6_0_OR_GREATER - block = this.OptionsValue.ZeroOutBuffer ? GC.AllocateArray(this.OptionsValue.BlockSize) : GC.AllocateUninitializedArray(this.OptionsValue.BlockSize); -#else - block = new byte[this.OptionsValue.BlockSize]; -#endif - this.ReportBlockCreated(); - } - else - { - Interlocked.Add(ref this.smallPoolFreeSize, -this.OptionsValue.BlockSize); - } - - return block; - } - - /// - /// Returns a buffer of arbitrary size from the large buffer pool. This buffer - /// will be at least the requiredSize and always be a multiple/exponential of largeBufferMultiple. - /// - /// The minimum length of the buffer. - /// Unique ID for the stream. - /// The tag of the stream returning this buffer, for logging if necessary. - /// A buffer of at least the required size. - /// Requested array size is larger than the maximum allowed. - internal byte[] GetLargeBuffer(long requiredSize, Guid id, string tag) - { - requiredSize = this.RoundToLargeBufferSize(requiredSize); - - if (requiredSize > MaxArrayLength) - { - throw new OutOfMemoryException($"Required buffer size exceeds maximum array length of {MaxArrayLength}."); - } - - int poolIndex = this.GetPoolIndex(requiredSize); - - bool createdNew = false; - bool pooled = true; - string callStack = null; - - byte[] buffer; - if (poolIndex < this.largePools.Length) - { - if (!this.largePools[poolIndex].TryPop(out buffer)) - { - buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); - createdNew = true; - } - else - { - Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], -buffer.Length); - } - } - else - { - // Buffer is too large to pool. They get a new buffer. - - // We still want to track the size, though, and we've reserved a slot - // in the end of the in-use array for non-pooled bytes in use. - poolIndex = this.largeBufferInUseSize.Length - 1; - - // We still want to round up to reduce heap fragmentation. - buffer = AllocateArray(requiredSize, this.OptionsValue.ZeroOutBuffer); - if (this.OptionsValue.GenerateCallStacks) - { - // Grab the stack -- we want to know who requires such large buffers - callStack = Environment.StackTrace; - } - - createdNew = true; - pooled = false; - } - - Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], buffer.Length); - if (createdNew) - { - this.ReportLargeBufferCreated(id, tag, requiredSize, pooled: pooled, callStack); - } - - return buffer; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static byte[] AllocateArray(long requiredSize, bool zeroInitializeArray) => -#if NET6_0_OR_GREATER - zeroInitializeArray ? GC.AllocateArray((int)requiredSize) : GC.AllocateUninitializedArray((int)requiredSize); -#else - new byte[requiredSize]; -#endif - } - - private long RoundToLargeBufferSize(long requiredSize) - { - if (this.OptionsValue.UseExponentialLargeBuffer) - { - long pow = 1; - while (this.OptionsValue.LargeBufferMultiple * pow < requiredSize) - { - pow <<= 1; - } - - return this.OptionsValue.LargeBufferMultiple * pow; - } - else - { - return (requiredSize + this.OptionsValue.LargeBufferMultiple - 1) / this.OptionsValue.LargeBufferMultiple * this.OptionsValue.LargeBufferMultiple; - } - } - - private bool IsLargeBufferSize(int value) - { - return value != 0 && (this.OptionsValue.UseExponentialLargeBuffer - ? value == this.RoundToLargeBufferSize(value) - : value % this.OptionsValue.LargeBufferMultiple == 0); - } - - private int GetPoolIndex(long length) - { - if (this.OptionsValue.UseExponentialLargeBuffer) - { - int index = 0; - while (this.OptionsValue.LargeBufferMultiple << index < length) - { - ++index; - } - - return index; - } - else - { - return (int)((length / this.OptionsValue.LargeBufferMultiple) - 1); - } - } - - /// - /// Returns the buffer to the large pool. - /// - /// The buffer to return. - /// Unique stream ID. - /// The tag of the stream returning this buffer, for logging if necessary. - /// is null. - /// buffer.Length is not a multiple/exponential of (it did not originate from this pool). - internal void ReturnLargeBuffer(byte[] buffer, Guid id, string tag) - { -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(buffer); -#else - if (buffer == null) - { - throw new ArgumentNullException(nameof(buffer)); - } -#endif - - if (!this.IsLargeBufferSize(buffer.Length)) - { - throw new ArgumentException($"{nameof(buffer)} did not originate from this memory manager. The size is not " + - $"{(this.OptionsValue.UseExponentialLargeBuffer ? "an exponential" : "a multiple")} of {this.OptionsValue.LargeBufferMultiple}."); - } - - this.ZeroOutMemoryIfEnabled(buffer); - int poolIndex = this.GetPoolIndex(buffer.Length); - if (poolIndex < this.largePools.Length) - { - if ((this.largePools[poolIndex].Count + 1) * buffer.Length <= this.OptionsValue.MaximumLargePoolFreeBytes || - this.OptionsValue.MaximumLargePoolFreeBytes == 0) - { - this.largePools[poolIndex].Push(buffer); - Interlocked.Add(ref this.largeBufferFreeSize[poolIndex], buffer.Length); - } - else - { - this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.EnoughFree); - } - } - else - { - // This is a non-poolable buffer, but we still want to track its size for in-use - // analysis. We have space in the InUse array for this. - poolIndex = this.largeBufferInUseSize.Length - 1; - this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Large, Events.MemoryStreamDiscardReason.TooLarge); - } - - Interlocked.Add(ref this.largeBufferInUseSize[poolIndex], -buffer.Length); - } - - /// - /// Returns the blocks to the pool. - /// - /// Collection of blocks to return to the pool. - /// Unique Stream ID. - /// The tag of the stream returning these blocks, for logging if necessary. - /// is null. - /// contains buffers that are the wrong size (or null) for this memory manager. - internal void ReturnBlocks(List blocks, Guid id, string tag) - { -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(blocks); -#else - if (blocks == null) - { - throw new ArgumentNullException(nameof(blocks)); - } -#endif - - long bytesToReturn = blocks.Count * (long)this.OptionsValue.BlockSize; - Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); - - foreach (byte[] block in blocks) - { - if (block == null || block.Length != this.OptionsValue.BlockSize) - { - throw new ArgumentException($"{nameof(blocks)} contains buffers that are not {nameof(this.OptionsValue.BlockSize)} in length.", nameof(blocks)); - } - } - - foreach (byte[] block in blocks) - { - this.ZeroOutMemoryIfEnabled(block); - if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) - { - Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); - this.smallPool.Push(block); - } - else - { - this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); - break; - } - } - } - - /// - /// Returns a block to the pool. - /// - /// Block to return to the pool. - /// Unique Stream ID. - /// The tag of the stream returning this, for logging if necessary. - /// is null. - /// is the wrong size for this memory manager. - internal void ReturnBlock(byte[] block, Guid id, string tag) - { - int bytesToReturn = this.OptionsValue.BlockSize; - Interlocked.Add(ref this.smallPoolInUseSize, -bytesToReturn); - -#if NET8_0_OR_GREATER - ArgumentNullException.ThrowIfNull(block); -#else - if (block == null) - { - throw new ArgumentNullException(nameof(block)); - } -#endif - - if (block.Length != this.OptionsValue.BlockSize) - { - throw new ArgumentException($"{nameof(block)} is not not {nameof(this.OptionsValue.BlockSize)} in length."); - } - - this.ZeroOutMemoryIfEnabled(block); - if (this.OptionsValue.MaximumSmallPoolFreeBytes == 0 || this.SmallPoolFreeSize < this.OptionsValue.MaximumSmallPoolFreeBytes) - { - Interlocked.Add(ref this.smallPoolFreeSize, this.OptionsValue.BlockSize); - this.smallPool.Push(block); - } - else - { - this.ReportBufferDiscarded(id, tag, Events.MemoryStreamBufferType.Small, Events.MemoryStreamDiscardReason.EnoughFree); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ZeroOutMemoryIfEnabled(byte[] buffer) - { - if (this.OptionsValue.ZeroOutBuffer) - { -#if NET6_0_OR_GREATER - Array.Clear(buffer); -#else - Array.Clear(buffer, 0, buffer.Length); -#endif - } - } - - internal void ReportBlockCreated() - { - Events.Writer.MemoryStreamNewBlockCreated(this.smallPoolInUseSize); - this.BlockCreated?.Invoke(this, new BlockCreatedEventArgs(this.smallPoolInUseSize)); - } - - internal void ReportLargeBufferCreated(Guid id, string tag, long requiredSize, bool pooled, string callStack) - { - if (pooled) - { - Events.Writer.MemoryStreamNewLargeBufferCreated(requiredSize, this.LargePoolInUseSize); - } - else - { - Events.Writer.MemoryStreamNonPooledLargeBufferCreated(id, tag, requiredSize, callStack); - } - - this.LargeBufferCreated?.Invoke(this, new LargeBufferCreatedEventArgs(id, tag, requiredSize, this.LargePoolInUseSize, pooled, callStack)); - } - - internal void ReportBufferDiscarded(Guid id, string tag, Events.MemoryStreamBufferType bufferType, Events.MemoryStreamDiscardReason reason) - { - Events.Writer.MemoryStreamDiscardBuffer( - id, - tag, - bufferType, - reason, - this.SmallBlocksFree, - this.smallPoolFreeSize, - this.smallPoolInUseSize, - this.LargeBuffersFree, - this.LargePoolFreeSize, - this.LargePoolInUseSize); - this.BufferDiscarded?.Invoke(this, new BufferDiscardedEventArgs(id, tag, bufferType, reason)); - } - - internal void ReportStreamCreated(Guid id, string tag, long requestedSize, long actualSize) - { - Events.Writer.MemoryStreamCreated(id, tag, requestedSize, actualSize); - this.StreamCreated?.Invoke(this, new StreamCreatedEventArgs(id, tag, requestedSize, actualSize)); - } - - internal void ReportStreamDisposed(Guid id, string tag, TimeSpan lifetime, string allocationStack, string disposeStack) - { - Events.Writer.MemoryStreamDisposed(id, tag, (long)lifetime.TotalMilliseconds, allocationStack, disposeStack); - this.StreamDisposed?.Invoke(this, new StreamDisposedEventArgs(id, tag, lifetime, allocationStack, disposeStack)); - } - - internal void ReportStreamDoubleDisposed(Guid id, string tag, string allocationStack, string disposeStack1, string disposeStack2) - { - Events.Writer.MemoryStreamDoubleDispose(id, tag, allocationStack, disposeStack1, disposeStack2); - this.StreamDoubleDisposed?.Invoke(this, new StreamDoubleDisposedEventArgs(id, tag, allocationStack, disposeStack1, disposeStack2)); - } - - internal void ReportStreamFinalized(Guid id, string tag, string allocationStack) - { - Events.Writer.MemoryStreamFinalized(id, tag, allocationStack); - this.StreamFinalized?.Invoke(this, new StreamFinalizedEventArgs(id, tag, allocationStack)); - } - - internal void ReportStreamLength(long bytes) - { - this.StreamLength?.Invoke(this, new StreamLengthEventArgs(bytes)); - } - - internal void ReportStreamToArray(Guid id, string tag, string stack, long length) - { - Events.Writer.MemoryStreamToArray(id, tag, stack, length); - this.StreamConvertedToArray?.Invoke(this, new StreamConvertedToArrayEventArgs(id, tag, stack, length)); - } - - internal void ReportStreamOverCapacity(Guid id, string tag, long requestedCapacity, string allocationStack) - { - Events.Writer.MemoryStreamOverCapacity(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack); - this.StreamOverCapacity?.Invoke(this, new StreamOverCapacityEventArgs(id, tag, requestedCapacity, this.OptionsValue.MaximumStreamCapacity, allocationStack)); - } - - internal void ReportUsageReport() - { - this.UsageReport?.Invoke(this, new UsageReportEventArgs(this.smallPoolInUseSize, this.smallPoolFreeSize, this.LargePoolInUseSize, this.LargePoolFreeSize)); - } - - /// - /// Retrieve a new object with no tag and a default initial capacity. - /// - /// A . - public RecyclableMemoryStream GetStream() - { - return new RecyclableMemoryStream(this); - } - - /// - /// Retrieve a new object with no tag and a default initial capacity. - /// - /// A unique identifier which can be used to trace usages of the stream. - /// A . - public RecyclableMemoryStream GetStream(Guid id) - { - return new RecyclableMemoryStream(this, id); - } - - /// - /// Retrieve a new object with the given tag and a default initial capacity. - /// - /// A tag which can be used to track the source of the stream. - /// A . - public RecyclableMemoryStream GetStream(string tag) - { - return new RecyclableMemoryStream(this, tag); - } - - /// - /// Retrieve a new object with the given tag and a default initial capacity. - /// - /// A unique identifier which can be used to trace usages of the stream. - /// A tag which can be used to track the source of the stream. - /// A . - public RecyclableMemoryStream GetStream(Guid id, string tag) - { - return new RecyclableMemoryStream(this, id, tag); - } - - /// - /// Retrieve a new object with the given tag and at least the given capacity. - /// - /// A tag which can be used to track the source of the stream. - /// The minimum desired capacity for the stream. - /// A . - public RecyclableMemoryStream GetStream(string tag, long requiredSize) - { - return new RecyclableMemoryStream(this, tag, requiredSize); - } - - /// - /// Retrieve a new object with the given tag and at least the given capacity. - /// - /// A unique identifier which can be used to trace usages of the stream. - /// A tag which can be used to track the source of the stream. - /// The minimum desired capacity for the stream. - /// A . - public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize) - { - return new RecyclableMemoryStream(this, id, tag, requiredSize); - } - - /// - /// Retrieve a new object with the given tag and at least the given capacity, possibly using - /// a single contiguous underlying buffer. - /// - /// Retrieving a which provides a single contiguous buffer can be useful in situations - /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying - /// buffers to a single large one. This is most helpful when you know that you will always call - /// on the underlying stream. - /// A unique identifier which can be used to trace usages of the stream. - /// A tag which can be used to track the source of the stream. - /// The minimum desired capacity for the stream. - /// Whether to attempt to use a single contiguous buffer. - /// A . - public RecyclableMemoryStream GetStream(Guid id, string tag, long requiredSize, bool asContiguousBuffer) - { - if (!asContiguousBuffer || requiredSize <= this.OptionsValue.BlockSize) - { - return this.GetStream(id, tag, requiredSize); - } - - return new RecyclableMemoryStream(this, id, tag, requiredSize, this.GetLargeBuffer(requiredSize, id, tag)); - } - - /// - /// Retrieve a new object with the given tag and at least the given capacity, possibly using - /// a single contiguous underlying buffer. - /// - /// Retrieving a which provides a single contiguous buffer can be useful in situations - /// where the initial size is known and it is desirable to avoid copying data between the smaller underlying - /// buffers to a single large one. This is most helpful when you know that you will always call - /// on the underlying stream. - /// A tag which can be used to track the source of the stream. - /// The minimum desired capacity for the stream. - /// Whether to attempt to use a single contiguous buffer. - /// A . - public RecyclableMemoryStream GetStream(string tag, long requiredSize, bool asContiguousBuffer) - { - return this.GetStream(Guid.NewGuid(), tag, requiredSize, asContiguousBuffer); - } - - /// - /// Retrieve a new object with the given tag and with contents copied from the provided - /// buffer. The provided buffer is not wrapped or used after construction. - /// - /// The new stream's position is set to the beginning of the stream when returned. - /// A unique identifier which can be used to trace usages of the stream. - /// A tag which can be used to track the source of the stream. - /// The byte buffer to copy data from. - /// The offset from the start of the buffer to copy from. - /// The number of bytes to copy from the buffer. - /// A . - public RecyclableMemoryStream GetStream(Guid id, string tag, byte[] buffer, int offset, int count) - { - RecyclableMemoryStream stream = null; - try - { - stream = new RecyclableMemoryStream(this, id, tag, count); - stream.Write(buffer, offset, count); - stream.Position = 0; - return stream; - } - catch - { - stream?.Dispose(); - throw; - } - } - - /// - /// Retrieve a new object with the contents copied from the provided - /// buffer. The provided buffer is not wrapped or used after construction. - /// - /// The new stream's position is set to the beginning of the stream when returned. - /// The byte buffer to copy data from. - /// A . - public RecyclableMemoryStream GetStream(byte[] buffer) - { - return this.GetStream(null, buffer, 0, buffer.Length); - } - - /// - /// Retrieve a new object with the given tag and with contents copied from the provided - /// buffer. The provided buffer is not wrapped or used after construction. - /// - /// The new stream's position is set to the beginning of the stream when returned. - /// A tag which can be used to track the source of the stream. - /// The byte buffer to copy data from. - /// The offset from the start of the buffer to copy from. - /// The number of bytes to copy from the buffer. - /// A . - public RecyclableMemoryStream GetStream(string tag, byte[] buffer, int offset, int count) - { - return this.GetStream(Guid.NewGuid(), tag, buffer, offset, count); - } - - /// - /// Retrieve a new object with the given tag and with contents copied from the provided - /// buffer. The provided buffer is not wrapped or used after construction. - /// - /// The new stream's position is set to the beginning of the stream when returned. - /// A unique identifier which can be used to trace usages of the stream. - /// A tag which can be used to track the source of the stream. - /// The byte buffer to copy data from. - /// A . - public RecyclableMemoryStream GetStream(Guid id, string tag, ReadOnlySpan buffer) - { - RecyclableMemoryStream stream = null; - try - { - stream = new RecyclableMemoryStream(this, id, tag, buffer.Length); - stream.Write(buffer); - stream.Position = 0; - return stream; - } - catch - { - stream?.Dispose(); - throw; - } - } - - /// - /// Retrieve a new object with the contents copied from the provided - /// buffer. The provided buffer is not wrapped or used after construction. - /// - /// The new stream's position is set to the beginning of the stream when returned. - /// The byte buffer to copy data from. - /// A . - public RecyclableMemoryStream GetStream(ReadOnlySpan buffer) - { - return this.GetStream(null, buffer); - } - - /// - /// Retrieve a new object with the given tag and with contents copied from the provided - /// buffer. The provided buffer is not wrapped or used after construction. - /// - /// The new stream's position is set to the beginning of the stream when returned. - /// A tag which can be used to track the source of the stream. - /// The byte buffer to copy data from. - /// A . - public RecyclableMemoryStream GetStream(string tag, ReadOnlySpan buffer) - { - return this.GetStream(Guid.NewGuid(), tag, buffer); - } - - /// - /// Triggered when a new block is created. - /// - public event EventHandler BlockCreated; - - /// - /// Triggered when a new large buffer is created. - /// - public event EventHandler LargeBufferCreated; - - /// - /// Triggered when a new stream is created. - /// - public event EventHandler StreamCreated; - - /// - /// Triggered when a stream is disposed. - /// - public event EventHandler StreamDisposed; - - /// - /// Triggered when a stream is disposed of twice (an error). - /// - public event EventHandler StreamDoubleDisposed; - - /// - /// Triggered when a stream is finalized. - /// - public event EventHandler StreamFinalized; - - /// - /// Triggered when a stream is disposed to report the stream's length. - /// - public event EventHandler StreamLength; - - /// - /// Triggered when a user converts a stream to array. - /// - public event EventHandler StreamConvertedToArray; - - /// - /// Triggered when a stream is requested to expand beyond the maximum length specified by the responsible RecyclableMemoryStreamManager. - /// - public event EventHandler StreamOverCapacity; - - /// - /// Triggered when a buffer of either type is discarded, along with the reason for the discard. - /// - public event EventHandler BufferDiscarded; - - /// - /// Periodically triggered to report usage statistics. - /// - public event EventHandler UsageReport; - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs index 905d640dd4..3679a71908 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/StreamProcessor.Encryptor.cs @@ -13,7 +13,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation using System.Text.Json; using System.Threading; using System.Threading.Tasks; - using Microsoft.Azure.Cosmos.Encryption.Custom.RecyclableMemoryStreamMirror; + using Microsoft.IO; internal partial class StreamProcessor {