From c794fdd504365563ceeca43b1f029787db2c8f15 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Fri, 12 Apr 2024 15:06:47 -0700 Subject: [PATCH 01/38] 2.25.0 Release notes (#1307) --- Release Notes/Release Notes v2.25.0.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 Release Notes/Release Notes v2.25.0.md diff --git a/Release Notes/Release Notes v2.25.0.md b/Release Notes/Release Notes v2.25.0.md new file mode 100644 index 00000000000..ddc3b28ef61 --- /dev/null +++ b/Release Notes/Release Notes v2.25.0.md @@ -0,0 +1,20 @@ +# .NET Driver Version 2.25.0 Release Notes + +This is the general availability release for the 2.25.0 version of the driver. + +NOTICE: MongoDB 3.6 reached end-of-life in April 2021. The .NET/C# Driver will be removing support for MongoDB 3.6 in an upcoming release. + +The main new features in 2.25.0 include: ++ Support of MONGODB-OIDC Authentication mechanism - [CSHARP-4448](https://jira.mongodb.org/browse/CSHARP-4448) ++ MONGODB-OIDC: Automatic token acquisition for Azure Identity Provider - [CSHARP-4474](https://jira.mongodb.org/browse/CSHARP-4474) ++ Improved error message when no matching constructor found - [CSHARP-5007](https://jira.mongodb.org/browse/CSHARP-5007) ++ Driver Container and Kubernetes Awareness - [CSHARP-4718](https://jira.mongodb.org/browse/CSHARP-4718) ++ Logging of executed MQL for a LINQ query - [CSHARP-4684](https://jira.mongodb.org/browse/CSHARP-4684) ++ Allow custom service names with srvServiceName URI option - [CSHARP-3745](https://jira.mongodb.org/browse/CSHARP-3745) ++ BulkWrite enumerates requests argument only once - [CSHARP-1378](https://jira.mongodb.org/browse/CSHARP-1378) ++ Support of Range Explicit Encryption - [CSHARP-5009](https://jira.mongodb.org/browse/CSHARP-5009) ++ Multiple bug fixes and improvements. + +The full list of issues resolved in this release is available at [CSHARP JIRA project](https://jira.mongodb.org/issues/?jql=project%20%3D%20CSHARP%20AND%20fixVersion%20%3D%202.25.0%20ORDER%20BY%20key%20ASC). + +Documentation on the .NET driver can be found [here](https://www.mongodb.com/docs/drivers/csharp/v2.25.0}/). From b2b672550cf7a523c234b9657ab5e9694c31726b Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:25:52 -0400 Subject: [PATCH 02/38] CSHARP-5021: Replace use of DisposableEnvironmentVariable with EnvironmentVariableProvider (#1305) * CSHARP-5021: Use EnvironmentVariableProvider where relevant * fix build issues for net472 * better variable naming * use simpler code * missed environment change --- .../Core/Connections/ClientDocumentHelper.cs | 18 ++-- .../Core/Servers/ServerMonitor.cs | 24 +++-- .../DisposableEnvironmentVariable.cs | 20 ----- .../Connections/ClientDocumentHelperTests.cs | 90 ++++++++----------- .../Core/Servers/ServerMonitorTests.cs | 52 ++++++----- .../ClientDocumentHelperTests.cs | 29 ++---- 6 files changed, 98 insertions(+), 135 deletions(-) delete mode 100644 tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs diff --git a/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs b/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs index 62775af3361..970434c7bca 100644 --- a/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs +++ b/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs @@ -142,24 +142,24 @@ string GetName() { string result = null; - if ((Environment.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || - Environment.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null) + if ((__environmentVariableProvider.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || + __environmentVariableProvider.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null) { result = awsLambdaName; } - if (Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null) + if (__environmentVariableProvider.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null) { if (result != null) return null; result = azureFuncName; } - if (Environment.GetEnvironmentVariable("K_SERVICE") != null || Environment.GetEnvironmentVariable("FUNCTION_NAME") != null) + if (__environmentVariableProvider.GetEnvironmentVariable("K_SERVICE") != null || __environmentVariableProvider.GetEnvironmentVariable("FUNCTION_NAME") != null) { if (result != null) return null; result = gcpFuncName; } - if (Environment.GetEnvironmentVariable("VERCEL") != null) + if (__environmentVariableProvider.GetEnvironmentVariable("VERCEL") != null) { if (result != null && result != awsLambdaName) return null; @@ -172,9 +172,9 @@ string GetName() string GetRegion(string name) => name switch { - awsLambdaName => Environment.GetEnvironmentVariable("AWS_REGION"), - gcpFuncName => Environment.GetEnvironmentVariable("FUNCTION_REGION"), - vercelName => Environment.GetEnvironmentVariable("VERCEL_REGION"), + awsLambdaName => __environmentVariableProvider.GetEnvironmentVariable("AWS_REGION"), + gcpFuncName => __environmentVariableProvider.GetEnvironmentVariable("FUNCTION_REGION"), + vercelName => __environmentVariableProvider.GetEnvironmentVariable("VERCEL_REGION"), _ => null }; @@ -211,7 +211,7 @@ BsonDocument GetContainerDocument() } int? GetIntValue(string environmentVariable) => - int.TryParse(Environment.GetEnvironmentVariable(environmentVariable), out var value) ? value : null; + int.TryParse(__environmentVariableProvider.GetEnvironmentVariable(environmentVariable), out var value) ? value : null; } internal static BsonDocument CreateOSDocument() diff --git a/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs b/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs index 5316288590c..52e68c40018 100644 --- a/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs +++ b/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs @@ -38,6 +38,7 @@ internal sealed class ServerMonitor : IServerMonitor private HeartbeatDelay _heartbeatDelay; private readonly bool _isStreamingEnabled; private readonly object _lock = new object(); + private readonly IEnvironmentVariableProvider _environmentVariableProvider; private readonly EventLogger _eventLoggerSdam; private readonly ILogger _logger; private readonly CancellationToken _monitorCancellationToken; // used to cancel the entire monitor @@ -59,7 +60,8 @@ public ServerMonitor( ServerMonitorSettings serverMonitorSettings, IEventSubscriber eventSubscriber, ServerApi serverApi, - ILoggerFactory loggerFactory) + ILoggerFactory loggerFactory, + IEnvironmentVariableProvider environmentVariableProvider = null) : this( serverId, endPoint, @@ -74,7 +76,8 @@ public ServerMonitor( serverApi, loggerFactory?.CreateLogger()), serverApi, - loggerFactory) + loggerFactory, + environmentVariableProvider) { } @@ -86,7 +89,8 @@ public ServerMonitor( IEventSubscriber eventSubscriber, IRoundTripTimeMonitor roundTripTimeMonitor, ServerApi serverApi, - ILoggerFactory loggerFactory) + ILoggerFactory loggerFactory, + IEnvironmentVariableProvider environmentVariableProvider = null) { _monitorCancellationTokenSource = new CancellationTokenSource(); _serverId = Ensure.IsNotNull(serverId, nameof(serverId)); @@ -107,6 +111,8 @@ public ServerMonitor( _logger = loggerFactory?.CreateLogger(); _eventLoggerSdam = loggerFactory.CreateEventLogger(eventSubscriber); + _environmentVariableProvider = environmentVariableProvider ?? EnvironmentVariableProvider.Instance; + _isStreamingEnabled = serverMonitorSettings.ServerMonitoringMode switch { ServerMonitoringMode.Stream => true, @@ -243,12 +249,12 @@ private bool IsRunningInFaaS() * gcp.func: K_SERVICE, FUNCTION_NAME * vercel: VERCEL */ - return (Environment.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || - Environment.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null || - Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null || - Environment.GetEnvironmentVariable("K_SERVICE") != null || - Environment.GetEnvironmentVariable("FUNCTION_NAME") != null || - Environment.GetEnvironmentVariable("VERCEL") != null; + return (_environmentVariableProvider.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || + _environmentVariableProvider.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null || + _environmentVariableProvider.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null || + _environmentVariableProvider.GetEnvironmentVariable("K_SERVICE") != null || + _environmentVariableProvider.GetEnvironmentVariable("FUNCTION_NAME") != null || + _environmentVariableProvider.GetEnvironmentVariable("VERCEL") != null; } private bool IsUsingStreamingProtocol(HelloResult helloResult) => _isStreamingEnabled && helloResult?.TopologyVersion != null; diff --git a/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs b/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs deleted file mode 100644 index 53ae610b5be..00000000000 --- a/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace MongoDB.Driver.Core.TestHelpers; - -public sealed class DisposableEnvironmentVariable : IDisposable -{ - private readonly string _initialValue; - private readonly string _name; - - public DisposableEnvironmentVariable(string variable) - { - var parts = variable.Split(new [] {'='}, StringSplitOptions.RemoveEmptyEntries); - _name = parts[0]; - var value = parts.Length > 1 ? parts[1] : "dummy"; - _initialValue = Environment.GetEnvironmentVariable(_name); - Environment.SetEnvironmentVariable(_name, value); - } - - public void Dispose() => Environment.SetEnvironmentVariable(_name, _initialValue); -} diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs index 938d71c3a06..07a1fd81c3f 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs @@ -144,32 +144,16 @@ public void CreateEnvDocument_should_return_expected_result(bool isDockerToBeDet var fileSystemProviderMock = new Mock(); var environmentVariableProviderMock = new Mock(); - if (isKubernetesToBeDetected) - { - environmentVariableProviderMock.Setup(p => p.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns("dummy"); - } - else - { - environmentVariableProviderMock.Setup(p => p.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns(null as string); - } + environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns(isKubernetesToBeDetected ? "dummy" : null); + environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable("VERCEL")).Returns("dummy"); - if (isDockerToBeDetected) - { - fileSystemProviderMock.Setup(p => p.File.Exists("/.dockerenv")).Returns(true); - } - else - { - fileSystemProviderMock.Setup(p => p.File.Exists("/.dockerenv")).Returns(false); - - } + fileSystemProviderMock.Setup(fileSystem => fileSystem.File.Exists("/.dockerenv")).Returns(isDockerToBeDetected); ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); ClientDocumentHelper.SetFileSystemProvider(fileSystemProviderMock.Object); - using (new DisposableEnvironmentVariable("VERCEL")) - { - var result = ClientDocumentHelper.CreateEnvDocument(); - result.Should().Be(expected); - } + + var result = ClientDocumentHelper.CreateEnvDocument(); + result.Should().Be(expected); } [Fact] @@ -253,39 +237,39 @@ public void Prefer_vercel_over_aws_env_name_when_both_specified( [Values(awsEnv, azureEnv, gcpEnv, vercelEnv)] string left, [Values(awsEnv, azureEnv, gcpEnv, vercelEnv)] string right) { - RequireEnvironment - .Check() - .EnvironmentVariable("AWS_EXECUTION_ENV", isDefined: false) - .EnvironmentVariable("AWS_LAMBDA_RUNTIME_API", isDefined: false) - .EnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", isDefined: false) - .EnvironmentVariable("K_SERVICE", isDefined: false) - .EnvironmentVariable("FUNCTION_NAME", isDefined: false) - .EnvironmentVariable("VERCEL", isDefined: false); - - using (new DisposableEnvironmentVariable(left)) - using (new DisposableEnvironmentVariable(right)) + var variableLeftParts = left.Split('='); + var variableRightParts = right.Split('='); + + var environmentVariableProviderMock = new Mock(); + + environmentVariableProviderMock + .Setup(env => env.GetEnvironmentVariable(variableLeftParts[0])).Returns(variableLeftParts.Length > 1 ? variableLeftParts[1] : "dummy"); + + environmentVariableProviderMock + .Setup(env => env.GetEnvironmentVariable(variableRightParts[0])).Returns(variableRightParts.Length > 1 ? variableRightParts[1] : "dummy"); + + ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); + + var clientEnvDocument = ClientDocumentHelper.CreateEnvDocument(); + if (left == right) { - var clientEnvDocument = ClientDocumentHelper.CreateEnvDocument(); - if (left == right) - { - var expectedName = left switch - { - awsEnv => awsLambdaName, - azureEnv => azureFuncName, - gcpEnv => gcpFuncName, - vercelEnv => vercelName, - _ => throw new Exception($"Unexpected env {left}."), - }; - clientEnvDocument["name"].Should().Be(BsonValue.Create(expectedName)); - } - else if ((left == awsEnv && right == vercelEnv) || (left == vercelEnv && right == awsEnv)) // exception - { - clientEnvDocument["name"].Should().Be(BsonValue.Create(vercelName)); - } - else + var expectedName = left switch { - clientEnvDocument.Should().BeNull(); - } + awsEnv => awsLambdaName, + azureEnv => azureFuncName, + gcpEnv => gcpFuncName, + vercelEnv => vercelName, + _ => throw new Exception($"Unexpected env {left}."), + }; + clientEnvDocument["name"].Should().Be(BsonValue.Create(expectedName)); + } + else if ((left == awsEnv && right == vercelEnv) || (left == vercelEnv && right == awsEnv)) // exception + { + clientEnvDocument["name"].Should().Be(BsonValue.Create(vercelName)); + } + else + { + clientEnvDocument.Should().BeNull(); } } diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs index 710a197116a..e73ad213e7e 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs @@ -461,30 +461,34 @@ public void ServerMonitor_without_serverApi_but_with_loadBalancedConnection_shou [InlineData("VERCEL")] public void Should_use_polling_protocol_if_running_in_FaaS_platform(string environmentVariable) { - using (new DisposableEnvironmentVariable(environmentVariable)) - { - var capturedEvents = new EventCapturer() - .Capture() - .Capture(); + var environmentVariableParts = environmentVariable.Split('='); - var serverMonitorSettings = new ServerMonitorSettings(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); - var subject = CreateSubject(out var mockConnection, out _, out _, capturedEvents, serverMonitorSettings: serverMonitorSettings); + var environmentVariableProviderMock = new Mock(); + environmentVariableProviderMock + .Setup(env => env.GetEnvironmentVariable(environmentVariableParts[0])) + .Returns(environmentVariableParts.Length > 1 ? environmentVariableParts[1] : "dummy"); - SetupHeartbeatConnection(mockConnection, isStreamable: false, autoFillStreamingResponses: false); - mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); - mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + var capturedEvents = new EventCapturer() + .Capture() + .Capture(); - subject.Initialize(); - SpinWait.SpinUntil(() => capturedEvents.Count >= 6, TimeSpan.FromSeconds(5)).Should().BeTrue(); - subject.Dispose(); - - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - } + var serverMonitorSettings = new ServerMonitorSettings(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); + var subject = CreateSubject(out var mockConnection, out _, out _, capturedEvents, serverMonitorSettings: serverMonitorSettings, environmentVariableProviderMock: environmentVariableProviderMock); + + SetupHeartbeatConnection(mockConnection, isStreamable: true, autoFillStreamingResponses: false); + mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + + subject.Initialize(); + SpinWait.SpinUntil(() => capturedEvents.Count >= 6, TimeSpan.FromSeconds(5)).Should().BeTrue(); + subject.Dispose(); + + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); } // private methods @@ -496,7 +500,8 @@ private ServerMonitor CreateSubject( bool captureConnectionEvents = false, ServerApi serverApi = null, bool loadBalanced = false, - ServerMonitorSettings serverMonitorSettings = null) + ServerMonitorSettings serverMonitorSettings = null, + Mock environmentVariableProviderMock = null) { mockRoundTripTimeMonitor = new Mock(); @@ -526,7 +531,8 @@ private ServerMonitor CreateSubject( eventCapturer ?? new EventCapturer(), mockRoundTripTimeMonitor.Object, serverApi, - LoggerFactory); + LoggerFactory, + environmentVariableProviderMock?.Object); } private void SetupHeartbeatConnection(MockConnection connection, bool isStreamable = false, bool autoFillStreamingResponses = true) diff --git a/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs b/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs index 0c800c69c13..aa2dd101984 100644 --- a/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs +++ b/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs @@ -26,6 +26,7 @@ using MongoDB.Driver.Core.TestHelpers; using MongoDB.Driver.Core.TestHelpers.XunitExtensions; using MongoDB.TestHelpers.XunitExtensions; +using Moq; using Xunit; namespace MongoDB.Driver.Tests @@ -65,15 +66,17 @@ public async Task Handhake_should_handle_faas_env_variables( var descriptionElements = environmentVariableDescriptionDocument .Elements .ToList(); - descriptionElements - // Ensure a test is not launched on actual env - .ForEach(e => RequireEnvironment.Check().EnvironmentVariable(e.Name, isDefined: false)); - var bundleElements = descriptionElements.Select(e => new DisposableEnvironmentVariable(e.Name, GetValue(e.Value.ToString()))).ToList(); + + var environmentVariableProviderMock = new Mock(); + + descriptionElements.ForEach(e => + environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable(e.Name)).Returns(GetValue(e.Value.ToString()))); + + ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); var eventCapturer = new EventCapturer() .Capture(e => e.CommandName == OppressiveLanguageConstants.LegacyHelloCommandName || e.CommandName == "hello"); - using (var bundle = new DisposableBundle(bundleElements)) using (var client = DriverTestConfiguration.CreateDisposableClient(eventCapturer)) { var database = client.GetDatabase("db"); @@ -101,22 +104,6 @@ public async Task Handhake_should_handle_faas_env_variables( string GetValue(string input) => input == "#longA#" ? new string('a', 512) : input; } - - // nested type - private class DisposableEnvironmentVariable : IDisposable - { - private readonly string _initialValue; - private readonly string _name; - - public DisposableEnvironmentVariable(string name, string value) - { - _name = name; - _initialValue = Environment.GetEnvironmentVariable(name); - Environment.SetEnvironmentVariable(name, value); - } - - public void Dispose() => Environment.SetEnvironmentVariable(_name, _initialValue); - } } internal static class ClientDocumentHelperReflector From 51e83ec1f3e2a7dfe1fb747c41517f45774fb49c Mon Sep 17 00:00:00 2001 From: Damien Guard Date: Thu, 18 Apr 2024 14:35:09 +0100 Subject: [PATCH 03/38] =?UTF-8?q?Revert=20"CSHARP-5021:=20Replace=20use=20?= =?UTF-8?q?of=20DisposableEnvironmentVariable=20with=20Enviro=E2=80=A6"=20?= =?UTF-8?q?(#1310)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit b2b672550cf7a523c234b9657ab5e9694c31726b. --- .../Core/Connections/ClientDocumentHelper.cs | 18 ++-- .../Core/Servers/ServerMonitor.cs | 24 ++--- .../DisposableEnvironmentVariable.cs | 20 +++++ .../Connections/ClientDocumentHelperTests.cs | 90 +++++++++++-------- .../Core/Servers/ServerMonitorTests.cs | 52 +++++------ .../ClientDocumentHelperTests.cs | 29 ++++-- 6 files changed, 135 insertions(+), 98 deletions(-) create mode 100644 tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs diff --git a/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs b/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs index 970434c7bca..62775af3361 100644 --- a/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs +++ b/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs @@ -142,24 +142,24 @@ string GetName() { string result = null; - if ((__environmentVariableProvider.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || - __environmentVariableProvider.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null) + if ((Environment.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || + Environment.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null) { result = awsLambdaName; } - if (__environmentVariableProvider.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null) + if (Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null) { if (result != null) return null; result = azureFuncName; } - if (__environmentVariableProvider.GetEnvironmentVariable("K_SERVICE") != null || __environmentVariableProvider.GetEnvironmentVariable("FUNCTION_NAME") != null) + if (Environment.GetEnvironmentVariable("K_SERVICE") != null || Environment.GetEnvironmentVariable("FUNCTION_NAME") != null) { if (result != null) return null; result = gcpFuncName; } - if (__environmentVariableProvider.GetEnvironmentVariable("VERCEL") != null) + if (Environment.GetEnvironmentVariable("VERCEL") != null) { if (result != null && result != awsLambdaName) return null; @@ -172,9 +172,9 @@ string GetName() string GetRegion(string name) => name switch { - awsLambdaName => __environmentVariableProvider.GetEnvironmentVariable("AWS_REGION"), - gcpFuncName => __environmentVariableProvider.GetEnvironmentVariable("FUNCTION_REGION"), - vercelName => __environmentVariableProvider.GetEnvironmentVariable("VERCEL_REGION"), + awsLambdaName => Environment.GetEnvironmentVariable("AWS_REGION"), + gcpFuncName => Environment.GetEnvironmentVariable("FUNCTION_REGION"), + vercelName => Environment.GetEnvironmentVariable("VERCEL_REGION"), _ => null }; @@ -211,7 +211,7 @@ BsonDocument GetContainerDocument() } int? GetIntValue(string environmentVariable) => - int.TryParse(__environmentVariableProvider.GetEnvironmentVariable(environmentVariable), out var value) ? value : null; + int.TryParse(Environment.GetEnvironmentVariable(environmentVariable), out var value) ? value : null; } internal static BsonDocument CreateOSDocument() diff --git a/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs b/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs index 52e68c40018..5316288590c 100644 --- a/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs +++ b/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs @@ -38,7 +38,6 @@ internal sealed class ServerMonitor : IServerMonitor private HeartbeatDelay _heartbeatDelay; private readonly bool _isStreamingEnabled; private readonly object _lock = new object(); - private readonly IEnvironmentVariableProvider _environmentVariableProvider; private readonly EventLogger _eventLoggerSdam; private readonly ILogger _logger; private readonly CancellationToken _monitorCancellationToken; // used to cancel the entire monitor @@ -60,8 +59,7 @@ public ServerMonitor( ServerMonitorSettings serverMonitorSettings, IEventSubscriber eventSubscriber, ServerApi serverApi, - ILoggerFactory loggerFactory, - IEnvironmentVariableProvider environmentVariableProvider = null) + ILoggerFactory loggerFactory) : this( serverId, endPoint, @@ -76,8 +74,7 @@ public ServerMonitor( serverApi, loggerFactory?.CreateLogger()), serverApi, - loggerFactory, - environmentVariableProvider) + loggerFactory) { } @@ -89,8 +86,7 @@ public ServerMonitor( IEventSubscriber eventSubscriber, IRoundTripTimeMonitor roundTripTimeMonitor, ServerApi serverApi, - ILoggerFactory loggerFactory, - IEnvironmentVariableProvider environmentVariableProvider = null) + ILoggerFactory loggerFactory) { _monitorCancellationTokenSource = new CancellationTokenSource(); _serverId = Ensure.IsNotNull(serverId, nameof(serverId)); @@ -111,8 +107,6 @@ public ServerMonitor( _logger = loggerFactory?.CreateLogger(); _eventLoggerSdam = loggerFactory.CreateEventLogger(eventSubscriber); - _environmentVariableProvider = environmentVariableProvider ?? EnvironmentVariableProvider.Instance; - _isStreamingEnabled = serverMonitorSettings.ServerMonitoringMode switch { ServerMonitoringMode.Stream => true, @@ -249,12 +243,12 @@ private bool IsRunningInFaaS() * gcp.func: K_SERVICE, FUNCTION_NAME * vercel: VERCEL */ - return (_environmentVariableProvider.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || - _environmentVariableProvider.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null || - _environmentVariableProvider.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null || - _environmentVariableProvider.GetEnvironmentVariable("K_SERVICE") != null || - _environmentVariableProvider.GetEnvironmentVariable("FUNCTION_NAME") != null || - _environmentVariableProvider.GetEnvironmentVariable("VERCEL") != null; + return (Environment.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || + Environment.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null || + Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null || + Environment.GetEnvironmentVariable("K_SERVICE") != null || + Environment.GetEnvironmentVariable("FUNCTION_NAME") != null || + Environment.GetEnvironmentVariable("VERCEL") != null; } private bool IsUsingStreamingProtocol(HelloResult helloResult) => _isStreamingEnabled && helloResult?.TopologyVersion != null; diff --git a/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs b/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs new file mode 100644 index 00000000000..53ae610b5be --- /dev/null +++ b/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs @@ -0,0 +1,20 @@ +using System; + +namespace MongoDB.Driver.Core.TestHelpers; + +public sealed class DisposableEnvironmentVariable : IDisposable +{ + private readonly string _initialValue; + private readonly string _name; + + public DisposableEnvironmentVariable(string variable) + { + var parts = variable.Split(new [] {'='}, StringSplitOptions.RemoveEmptyEntries); + _name = parts[0]; + var value = parts.Length > 1 ? parts[1] : "dummy"; + _initialValue = Environment.GetEnvironmentVariable(_name); + Environment.SetEnvironmentVariable(_name, value); + } + + public void Dispose() => Environment.SetEnvironmentVariable(_name, _initialValue); +} diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs index 07a1fd81c3f..938d71c3a06 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs @@ -144,16 +144,32 @@ public void CreateEnvDocument_should_return_expected_result(bool isDockerToBeDet var fileSystemProviderMock = new Mock(); var environmentVariableProviderMock = new Mock(); - environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns(isKubernetesToBeDetected ? "dummy" : null); - environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable("VERCEL")).Returns("dummy"); + if (isKubernetesToBeDetected) + { + environmentVariableProviderMock.Setup(p => p.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns("dummy"); + } + else + { + environmentVariableProviderMock.Setup(p => p.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns(null as string); + } - fileSystemProviderMock.Setup(fileSystem => fileSystem.File.Exists("/.dockerenv")).Returns(isDockerToBeDetected); + if (isDockerToBeDetected) + { + fileSystemProviderMock.Setup(p => p.File.Exists("/.dockerenv")).Returns(true); + } + else + { + fileSystemProviderMock.Setup(p => p.File.Exists("/.dockerenv")).Returns(false); + + } ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); ClientDocumentHelper.SetFileSystemProvider(fileSystemProviderMock.Object); - - var result = ClientDocumentHelper.CreateEnvDocument(); - result.Should().Be(expected); + using (new DisposableEnvironmentVariable("VERCEL")) + { + var result = ClientDocumentHelper.CreateEnvDocument(); + result.Should().Be(expected); + } } [Fact] @@ -237,39 +253,39 @@ public void Prefer_vercel_over_aws_env_name_when_both_specified( [Values(awsEnv, azureEnv, gcpEnv, vercelEnv)] string left, [Values(awsEnv, azureEnv, gcpEnv, vercelEnv)] string right) { - var variableLeftParts = left.Split('='); - var variableRightParts = right.Split('='); - - var environmentVariableProviderMock = new Mock(); - - environmentVariableProviderMock - .Setup(env => env.GetEnvironmentVariable(variableLeftParts[0])).Returns(variableLeftParts.Length > 1 ? variableLeftParts[1] : "dummy"); - - environmentVariableProviderMock - .Setup(env => env.GetEnvironmentVariable(variableRightParts[0])).Returns(variableRightParts.Length > 1 ? variableRightParts[1] : "dummy"); - - ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); - - var clientEnvDocument = ClientDocumentHelper.CreateEnvDocument(); - if (left == right) + RequireEnvironment + .Check() + .EnvironmentVariable("AWS_EXECUTION_ENV", isDefined: false) + .EnvironmentVariable("AWS_LAMBDA_RUNTIME_API", isDefined: false) + .EnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", isDefined: false) + .EnvironmentVariable("K_SERVICE", isDefined: false) + .EnvironmentVariable("FUNCTION_NAME", isDefined: false) + .EnvironmentVariable("VERCEL", isDefined: false); + + using (new DisposableEnvironmentVariable(left)) + using (new DisposableEnvironmentVariable(right)) { - var expectedName = left switch + var clientEnvDocument = ClientDocumentHelper.CreateEnvDocument(); + if (left == right) { - awsEnv => awsLambdaName, - azureEnv => azureFuncName, - gcpEnv => gcpFuncName, - vercelEnv => vercelName, - _ => throw new Exception($"Unexpected env {left}."), - }; - clientEnvDocument["name"].Should().Be(BsonValue.Create(expectedName)); - } - else if ((left == awsEnv && right == vercelEnv) || (left == vercelEnv && right == awsEnv)) // exception - { - clientEnvDocument["name"].Should().Be(BsonValue.Create(vercelName)); - } - else - { - clientEnvDocument.Should().BeNull(); + var expectedName = left switch + { + awsEnv => awsLambdaName, + azureEnv => azureFuncName, + gcpEnv => gcpFuncName, + vercelEnv => vercelName, + _ => throw new Exception($"Unexpected env {left}."), + }; + clientEnvDocument["name"].Should().Be(BsonValue.Create(expectedName)); + } + else if ((left == awsEnv && right == vercelEnv) || (left == vercelEnv && right == awsEnv)) // exception + { + clientEnvDocument["name"].Should().Be(BsonValue.Create(vercelName)); + } + else + { + clientEnvDocument.Should().BeNull(); + } } } diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs index e73ad213e7e..710a197116a 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs @@ -461,34 +461,30 @@ public void ServerMonitor_without_serverApi_but_with_loadBalancedConnection_shou [InlineData("VERCEL")] public void Should_use_polling_protocol_if_running_in_FaaS_platform(string environmentVariable) { - var environmentVariableParts = environmentVariable.Split('='); - - var environmentVariableProviderMock = new Mock(); - environmentVariableProviderMock - .Setup(env => env.GetEnvironmentVariable(environmentVariableParts[0])) - .Returns(environmentVariableParts.Length > 1 ? environmentVariableParts[1] : "dummy"); - - var capturedEvents = new EventCapturer() - .Capture() - .Capture(); - - var serverMonitorSettings = new ServerMonitorSettings(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); - var subject = CreateSubject(out var mockConnection, out _, out _, capturedEvents, serverMonitorSettings: serverMonitorSettings, environmentVariableProviderMock: environmentVariableProviderMock); + using (new DisposableEnvironmentVariable(environmentVariable)) + { + var capturedEvents = new EventCapturer() + .Capture() + .Capture(); - SetupHeartbeatConnection(mockConnection, isStreamable: true, autoFillStreamingResponses: false); - mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); - mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + var serverMonitorSettings = new ServerMonitorSettings(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); + var subject = CreateSubject(out var mockConnection, out _, out _, capturedEvents, serverMonitorSettings: serverMonitorSettings); - subject.Initialize(); - SpinWait.SpinUntil(() => capturedEvents.Count >= 6, TimeSpan.FromSeconds(5)).Should().BeTrue(); - subject.Dispose(); + SetupHeartbeatConnection(mockConnection, isStreamable: false, autoFillStreamingResponses: false); + mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + subject.Initialize(); + SpinWait.SpinUntil(() => capturedEvents.Count >= 6, TimeSpan.FromSeconds(5)).Should().BeTrue(); + subject.Dispose(); + + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + } } // private methods @@ -500,8 +496,7 @@ private ServerMonitor CreateSubject( bool captureConnectionEvents = false, ServerApi serverApi = null, bool loadBalanced = false, - ServerMonitorSettings serverMonitorSettings = null, - Mock environmentVariableProviderMock = null) + ServerMonitorSettings serverMonitorSettings = null) { mockRoundTripTimeMonitor = new Mock(); @@ -531,8 +526,7 @@ private ServerMonitor CreateSubject( eventCapturer ?? new EventCapturer(), mockRoundTripTimeMonitor.Object, serverApi, - LoggerFactory, - environmentVariableProviderMock?.Object); + LoggerFactory); } private void SetupHeartbeatConnection(MockConnection connection, bool isStreamable = false, bool autoFillStreamingResponses = true) diff --git a/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs b/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs index aa2dd101984..0c800c69c13 100644 --- a/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs +++ b/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs @@ -26,7 +26,6 @@ using MongoDB.Driver.Core.TestHelpers; using MongoDB.Driver.Core.TestHelpers.XunitExtensions; using MongoDB.TestHelpers.XunitExtensions; -using Moq; using Xunit; namespace MongoDB.Driver.Tests @@ -66,17 +65,15 @@ public async Task Handhake_should_handle_faas_env_variables( var descriptionElements = environmentVariableDescriptionDocument .Elements .ToList(); - - var environmentVariableProviderMock = new Mock(); - - descriptionElements.ForEach(e => - environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable(e.Name)).Returns(GetValue(e.Value.ToString()))); - - ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); + descriptionElements + // Ensure a test is not launched on actual env + .ForEach(e => RequireEnvironment.Check().EnvironmentVariable(e.Name, isDefined: false)); + var bundleElements = descriptionElements.Select(e => new DisposableEnvironmentVariable(e.Name, GetValue(e.Value.ToString()))).ToList(); var eventCapturer = new EventCapturer() .Capture(e => e.CommandName == OppressiveLanguageConstants.LegacyHelloCommandName || e.CommandName == "hello"); + using (var bundle = new DisposableBundle(bundleElements)) using (var client = DriverTestConfiguration.CreateDisposableClient(eventCapturer)) { var database = client.GetDatabase("db"); @@ -104,6 +101,22 @@ public async Task Handhake_should_handle_faas_env_variables( string GetValue(string input) => input == "#longA#" ? new string('a', 512) : input; } + + // nested type + private class DisposableEnvironmentVariable : IDisposable + { + private readonly string _initialValue; + private readonly string _name; + + public DisposableEnvironmentVariable(string name, string value) + { + _name = name; + _initialValue = Environment.GetEnvironmentVariable(name); + Environment.SetEnvironmentVariable(name, value); + } + + public void Dispose() => Environment.SetEnvironmentVariable(_name, _initialValue); + } } internal static class ClientDocumentHelperReflector From 17d8e0699c69d5bea81b5d9c1eff478952b4ccc0 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Fri, 19 Apr 2024 00:25:53 -0400 Subject: [PATCH 04/38] CSHARP-5021: Use EnvironmentVariableProvider where relevant (#1311) --- .../Core/Connections/ClientDocumentHelper.cs | 20 ++-- .../Core/Servers/ServerMonitor.cs | 24 +++-- .../DisposableEnvironmentVariable.cs | 20 ---- .../Connections/ClientDocumentHelperTests.cs | 93 ++++++++----------- .../Core/Servers/ServerMonitorTests.cs | 52 ++++++----- .../ClientDocumentHelperTests.cs | 29 ++---- 6 files changed, 102 insertions(+), 136 deletions(-) delete mode 100644 tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs diff --git a/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs b/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs index 62775af3361..36168d69840 100644 --- a/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs +++ b/src/MongoDB.Driver.Core/Core/Connections/ClientDocumentHelper.cs @@ -34,7 +34,7 @@ internal class ClientDocumentHelper private static IEnvironmentVariableProvider __environmentVariableProvider; private static IFileSystemProvider __fileSystemProvider; - private static void Initialize() + internal static void Initialize() { __driverDocument = new Lazy(CreateDriverDocument); __envDocument = new Lazy(CreateEnvDocument); @@ -142,24 +142,24 @@ string GetName() { string result = null; - if ((Environment.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || - Environment.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null) + if ((__environmentVariableProvider.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || + __environmentVariableProvider.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null) { result = awsLambdaName; } - if (Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null) + if (__environmentVariableProvider.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null) { if (result != null) return null; result = azureFuncName; } - if (Environment.GetEnvironmentVariable("K_SERVICE") != null || Environment.GetEnvironmentVariable("FUNCTION_NAME") != null) + if (__environmentVariableProvider.GetEnvironmentVariable("K_SERVICE") != null || __environmentVariableProvider.GetEnvironmentVariable("FUNCTION_NAME") != null) { if (result != null) return null; result = gcpFuncName; } - if (Environment.GetEnvironmentVariable("VERCEL") != null) + if (__environmentVariableProvider.GetEnvironmentVariable("VERCEL") != null) { if (result != null && result != awsLambdaName) return null; @@ -172,9 +172,9 @@ string GetName() string GetRegion(string name) => name switch { - awsLambdaName => Environment.GetEnvironmentVariable("AWS_REGION"), - gcpFuncName => Environment.GetEnvironmentVariable("FUNCTION_REGION"), - vercelName => Environment.GetEnvironmentVariable("VERCEL_REGION"), + awsLambdaName => __environmentVariableProvider.GetEnvironmentVariable("AWS_REGION"), + gcpFuncName => __environmentVariableProvider.GetEnvironmentVariable("FUNCTION_REGION"), + vercelName => __environmentVariableProvider.GetEnvironmentVariable("VERCEL_REGION"), _ => null }; @@ -211,7 +211,7 @@ BsonDocument GetContainerDocument() } int? GetIntValue(string environmentVariable) => - int.TryParse(Environment.GetEnvironmentVariable(environmentVariable), out var value) ? value : null; + int.TryParse(__environmentVariableProvider.GetEnvironmentVariable(environmentVariable), out var value) ? value : null; } internal static BsonDocument CreateOSDocument() diff --git a/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs b/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs index 5316288590c..52e68c40018 100644 --- a/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs +++ b/src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs @@ -38,6 +38,7 @@ internal sealed class ServerMonitor : IServerMonitor private HeartbeatDelay _heartbeatDelay; private readonly bool _isStreamingEnabled; private readonly object _lock = new object(); + private readonly IEnvironmentVariableProvider _environmentVariableProvider; private readonly EventLogger _eventLoggerSdam; private readonly ILogger _logger; private readonly CancellationToken _monitorCancellationToken; // used to cancel the entire monitor @@ -59,7 +60,8 @@ public ServerMonitor( ServerMonitorSettings serverMonitorSettings, IEventSubscriber eventSubscriber, ServerApi serverApi, - ILoggerFactory loggerFactory) + ILoggerFactory loggerFactory, + IEnvironmentVariableProvider environmentVariableProvider = null) : this( serverId, endPoint, @@ -74,7 +76,8 @@ public ServerMonitor( serverApi, loggerFactory?.CreateLogger()), serverApi, - loggerFactory) + loggerFactory, + environmentVariableProvider) { } @@ -86,7 +89,8 @@ public ServerMonitor( IEventSubscriber eventSubscriber, IRoundTripTimeMonitor roundTripTimeMonitor, ServerApi serverApi, - ILoggerFactory loggerFactory) + ILoggerFactory loggerFactory, + IEnvironmentVariableProvider environmentVariableProvider = null) { _monitorCancellationTokenSource = new CancellationTokenSource(); _serverId = Ensure.IsNotNull(serverId, nameof(serverId)); @@ -107,6 +111,8 @@ public ServerMonitor( _logger = loggerFactory?.CreateLogger(); _eventLoggerSdam = loggerFactory.CreateEventLogger(eventSubscriber); + _environmentVariableProvider = environmentVariableProvider ?? EnvironmentVariableProvider.Instance; + _isStreamingEnabled = serverMonitorSettings.ServerMonitoringMode switch { ServerMonitoringMode.Stream => true, @@ -243,12 +249,12 @@ private bool IsRunningInFaaS() * gcp.func: K_SERVICE, FUNCTION_NAME * vercel: VERCEL */ - return (Environment.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || - Environment.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null || - Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null || - Environment.GetEnvironmentVariable("K_SERVICE") != null || - Environment.GetEnvironmentVariable("FUNCTION_NAME") != null || - Environment.GetEnvironmentVariable("VERCEL") != null; + return (_environmentVariableProvider.GetEnvironmentVariable("AWS_EXECUTION_ENV")?.StartsWith("AWS_Lambda_") ?? false) || + _environmentVariableProvider.GetEnvironmentVariable("AWS_LAMBDA_RUNTIME_API") != null || + _environmentVariableProvider.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null || + _environmentVariableProvider.GetEnvironmentVariable("K_SERVICE") != null || + _environmentVariableProvider.GetEnvironmentVariable("FUNCTION_NAME") != null || + _environmentVariableProvider.GetEnvironmentVariable("VERCEL") != null; } private bool IsUsingStreamingProtocol(HelloResult helloResult) => _isStreamingEnabled && helloResult?.TopologyVersion != null; diff --git a/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs b/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs deleted file mode 100644 index 53ae610b5be..00000000000 --- a/tests/MongoDB.Driver.Core.TestHelpers/DisposableEnvironmentVariable.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace MongoDB.Driver.Core.TestHelpers; - -public sealed class DisposableEnvironmentVariable : IDisposable -{ - private readonly string _initialValue; - private readonly string _name; - - public DisposableEnvironmentVariable(string variable) - { - var parts = variable.Split(new [] {'='}, StringSplitOptions.RemoveEmptyEntries); - _name = parts[0]; - var value = parts.Length > 1 ? parts[1] : "dummy"; - _initialValue = Environment.GetEnvironmentVariable(_name); - Environment.SetEnvironmentVariable(_name, value); - } - - public void Dispose() => Environment.SetEnvironmentVariable(_name, _initialValue); -} diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs index 938d71c3a06..8b645eb3def 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Connections/ClientDocumentHelperTests.cs @@ -31,6 +31,7 @@ public class ClientDocumentHelperTests [Fact] public void CreateClientDocument_should_return_expected_result() { + ClientDocumentHelper.Initialize(); var result = ClientDocumentHelper.CreateClientDocument(null, null); var names = result.Names.ToList(); @@ -100,6 +101,8 @@ public void CreateClientDocument_with_library_info_should_return_expected_result var expectedDriverName = libName == null ? "mongo-csharp-driver" : $"mongo-csharp-driver|{libName}"; var libraryInfo = libName != null ? new LibraryInfo(libName, libVersion) : null; + + ClientDocumentHelper.Initialize(); var driverDocument = ClientDocumentHelper.CreateClientDocument(null, libraryInfo)["driver"]; driverDocument["name"].AsString.Should().Be(expectedDriverName); @@ -144,32 +147,16 @@ public void CreateEnvDocument_should_return_expected_result(bool isDockerToBeDet var fileSystemProviderMock = new Mock(); var environmentVariableProviderMock = new Mock(); - if (isKubernetesToBeDetected) - { - environmentVariableProviderMock.Setup(p => p.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns("dummy"); - } - else - { - environmentVariableProviderMock.Setup(p => p.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns(null as string); - } + environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST")).Returns(isKubernetesToBeDetected ? "dummy" : null); + environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable("VERCEL")).Returns("dummy"); - if (isDockerToBeDetected) - { - fileSystemProviderMock.Setup(p => p.File.Exists("/.dockerenv")).Returns(true); - } - else - { - fileSystemProviderMock.Setup(p => p.File.Exists("/.dockerenv")).Returns(false); - - } + fileSystemProviderMock.Setup(fileSystem => fileSystem.File.Exists("/.dockerenv")).Returns(isDockerToBeDetected); ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); ClientDocumentHelper.SetFileSystemProvider(fileSystemProviderMock.Object); - using (new DisposableEnvironmentVariable("VERCEL")) - { - var result = ClientDocumentHelper.CreateEnvDocument(); - result.Should().Be(expected); - } + + var result = ClientDocumentHelper.CreateEnvDocument(); + result.Should().Be(expected); } [Fact] @@ -253,39 +240,39 @@ public void Prefer_vercel_over_aws_env_name_when_both_specified( [Values(awsEnv, azureEnv, gcpEnv, vercelEnv)] string left, [Values(awsEnv, azureEnv, gcpEnv, vercelEnv)] string right) { - RequireEnvironment - .Check() - .EnvironmentVariable("AWS_EXECUTION_ENV", isDefined: false) - .EnvironmentVariable("AWS_LAMBDA_RUNTIME_API", isDefined: false) - .EnvironmentVariable("FUNCTIONS_WORKER_RUNTIME", isDefined: false) - .EnvironmentVariable("K_SERVICE", isDefined: false) - .EnvironmentVariable("FUNCTION_NAME", isDefined: false) - .EnvironmentVariable("VERCEL", isDefined: false); - - using (new DisposableEnvironmentVariable(left)) - using (new DisposableEnvironmentVariable(right)) + var variableLeftParts = left.Split('='); + var variableRightParts = right.Split('='); + + var environmentVariableProviderMock = new Mock(); + + environmentVariableProviderMock + .Setup(env => env.GetEnvironmentVariable(variableLeftParts[0])).Returns(variableLeftParts.Length > 1 ? variableLeftParts[1] : "dummy"); + + environmentVariableProviderMock + .Setup(env => env.GetEnvironmentVariable(variableRightParts[0])).Returns(variableRightParts.Length > 1 ? variableRightParts[1] : "dummy"); + + ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); + + var clientEnvDocument = ClientDocumentHelper.CreateEnvDocument(); + if (left == right) { - var clientEnvDocument = ClientDocumentHelper.CreateEnvDocument(); - if (left == right) + var expectedName = left switch { - var expectedName = left switch - { - awsEnv => awsLambdaName, - azureEnv => azureFuncName, - gcpEnv => gcpFuncName, - vercelEnv => vercelName, - _ => throw new Exception($"Unexpected env {left}."), - }; - clientEnvDocument["name"].Should().Be(BsonValue.Create(expectedName)); - } - else if ((left == awsEnv && right == vercelEnv) || (left == vercelEnv && right == awsEnv)) // exception - { - clientEnvDocument["name"].Should().Be(BsonValue.Create(vercelName)); - } - else - { - clientEnvDocument.Should().BeNull(); - } + awsEnv => awsLambdaName, + azureEnv => azureFuncName, + gcpEnv => gcpFuncName, + vercelEnv => vercelName, + _ => throw new Exception($"Unexpected env {left}."), + }; + clientEnvDocument["name"].Should().Be(BsonValue.Create(expectedName)); + } + else if ((left == awsEnv && right == vercelEnv) || (left == vercelEnv && right == awsEnv)) // exception + { + clientEnvDocument["name"].Should().Be(BsonValue.Create(vercelName)); + } + else + { + clientEnvDocument.Should().BeNull(); } } diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs index 710a197116a..e73ad213e7e 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Servers/ServerMonitorTests.cs @@ -461,30 +461,34 @@ public void ServerMonitor_without_serverApi_but_with_loadBalancedConnection_shou [InlineData("VERCEL")] public void Should_use_polling_protocol_if_running_in_FaaS_platform(string environmentVariable) { - using (new DisposableEnvironmentVariable(environmentVariable)) - { - var capturedEvents = new EventCapturer() - .Capture() - .Capture(); + var environmentVariableParts = environmentVariable.Split('='); - var serverMonitorSettings = new ServerMonitorSettings(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); - var subject = CreateSubject(out var mockConnection, out _, out _, capturedEvents, serverMonitorSettings: serverMonitorSettings); + var environmentVariableProviderMock = new Mock(); + environmentVariableProviderMock + .Setup(env => env.GetEnvironmentVariable(environmentVariableParts[0])) + .Returns(environmentVariableParts.Length > 1 ? environmentVariableParts[1] : "dummy"); - SetupHeartbeatConnection(mockConnection, isStreamable: false, autoFillStreamingResponses: false); - mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); - mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + var capturedEvents = new EventCapturer() + .Capture() + .Capture(); - subject.Initialize(); - SpinWait.SpinUntil(() => capturedEvents.Count >= 6, TimeSpan.FromSeconds(5)).Should().BeTrue(); - subject.Dispose(); - - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); - } + var serverMonitorSettings = new ServerMonitorSettings(TimeSpan.FromSeconds(5), TimeSpan.FromMilliseconds(10)); + var subject = CreateSubject(out var mockConnection, out _, out _, capturedEvents, serverMonitorSettings: serverMonitorSettings, environmentVariableProviderMock: environmentVariableProviderMock); + + SetupHeartbeatConnection(mockConnection, isStreamable: true, autoFillStreamingResponses: false); + mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + mockConnection.EnqueueCommandResponseMessage(CreateHeartbeatCommandResponseMessage()); + + subject.Initialize(); + SpinWait.SpinUntil(() => capturedEvents.Count >= 6, TimeSpan.FromSeconds(5)).Should().BeTrue(); + subject.Dispose(); + + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); + capturedEvents.Next().Should().BeOfType().Subject.Awaited.Should().Be(false); } // private methods @@ -496,7 +500,8 @@ private ServerMonitor CreateSubject( bool captureConnectionEvents = false, ServerApi serverApi = null, bool loadBalanced = false, - ServerMonitorSettings serverMonitorSettings = null) + ServerMonitorSettings serverMonitorSettings = null, + Mock environmentVariableProviderMock = null) { mockRoundTripTimeMonitor = new Mock(); @@ -526,7 +531,8 @@ private ServerMonitor CreateSubject( eventCapturer ?? new EventCapturer(), mockRoundTripTimeMonitor.Object, serverApi, - LoggerFactory); + LoggerFactory, + environmentVariableProviderMock?.Object); } private void SetupHeartbeatConnection(MockConnection connection, bool isStreamable = false, bool autoFillStreamingResponses = true) diff --git a/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs b/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs index 0c800c69c13..aa2dd101984 100644 --- a/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs +++ b/tests/MongoDB.Driver.Tests/ClientDocumentHelperTests.cs @@ -26,6 +26,7 @@ using MongoDB.Driver.Core.TestHelpers; using MongoDB.Driver.Core.TestHelpers.XunitExtensions; using MongoDB.TestHelpers.XunitExtensions; +using Moq; using Xunit; namespace MongoDB.Driver.Tests @@ -65,15 +66,17 @@ public async Task Handhake_should_handle_faas_env_variables( var descriptionElements = environmentVariableDescriptionDocument .Elements .ToList(); - descriptionElements - // Ensure a test is not launched on actual env - .ForEach(e => RequireEnvironment.Check().EnvironmentVariable(e.Name, isDefined: false)); - var bundleElements = descriptionElements.Select(e => new DisposableEnvironmentVariable(e.Name, GetValue(e.Value.ToString()))).ToList(); + + var environmentVariableProviderMock = new Mock(); + + descriptionElements.ForEach(e => + environmentVariableProviderMock.Setup(env => env.GetEnvironmentVariable(e.Name)).Returns(GetValue(e.Value.ToString()))); + + ClientDocumentHelper.SetEnvironmentVariableProvider(environmentVariableProviderMock.Object); var eventCapturer = new EventCapturer() .Capture(e => e.CommandName == OppressiveLanguageConstants.LegacyHelloCommandName || e.CommandName == "hello"); - using (var bundle = new DisposableBundle(bundleElements)) using (var client = DriverTestConfiguration.CreateDisposableClient(eventCapturer)) { var database = client.GetDatabase("db"); @@ -101,22 +104,6 @@ public async Task Handhake_should_handle_faas_env_variables( string GetValue(string input) => input == "#longA#" ? new string('a', 512) : input; } - - // nested type - private class DisposableEnvironmentVariable : IDisposable - { - private readonly string _initialValue; - private readonly string _name; - - public DisposableEnvironmentVariable(string name, string value) - { - _name = name; - _initialValue = Environment.GetEnvironmentVariable(name); - Environment.SetEnvironmentVariable(name, value); - } - - public void Dispose() => Environment.SetEnvironmentVariable(_name, _initialValue); - } } internal static class ClientDocumentHelperReflector From bc66d5928c0b5d86d6c376b21137bfc472c3682b Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Mon, 22 Apr 2024 11:29:07 -0700 Subject: [PATCH 05/38] Update performance tests to run on net8.0 (#1309) --- .../MongoDB.Driver.Benchmarks.csproj | 2 +- evergreen/evergreen.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/MongoDB.Driver.Benchmarks/MongoDB.Driver.Benchmarks.csproj b/benchmarks/MongoDB.Driver.Benchmarks/MongoDB.Driver.Benchmarks.csproj index c34974be4ed..c51d7740066 100644 --- a/benchmarks/MongoDB.Driver.Benchmarks/MongoDB.Driver.Benchmarks.csproj +++ b/benchmarks/MongoDB.Driver.Benchmarks/MongoDB.Driver.Benchmarks.csproj @@ -6,7 +6,7 @@ - net6.0 + net8.0 Exe diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index cf4f521212a..6c75ee348cc 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -1486,7 +1486,7 @@ tasks: vars: FRAMEWORK: net80 - - name: performance-tests-net60-server-v6.0 + - name: performance-tests-net80-server-v6.0 commands: - func: bootstrap-mongo-orchestration vars: @@ -2405,7 +2405,7 @@ buildvariants: run_on: - rhel90-dbx-perf-large tasks: - - name: performance-tests-net60-server-v6.0 + - name: performance-tests-net80-server-v6.0 # AWS Lambda tests - name: aws-lambda-tests From 8af9316816ca338a83d62d2812a3b7c06cb3c736 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:01:58 -0400 Subject: [PATCH 06/38] CSHARP-4828: Handling errors in WithTransaction (#1312) --- src/MongoDB.Driver/IClientSession.cs | 18 ++++++++++++++++++ .../WithTransactionExample1.cs | 12 ++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/MongoDB.Driver/IClientSession.cs b/src/MongoDB.Driver/IClientSession.cs index 7dfb7fd5614..a0809fcae3c 100644 --- a/src/MongoDB.Driver/IClientSession.cs +++ b/src/MongoDB.Driver/IClientSession.cs @@ -139,6 +139,15 @@ public interface IClientSession : IDisposable /// /// Executes a callback within a transaction, with retries if needed. /// + /// + /// If a command inside the callback fails, it may cause the transaction on the server to be + /// aborted. This situation is normally handled transparently by the driver. However, if the + /// application does not return that error from the callback, the driver will not be able to + /// determine whether the transaction was aborted or not. The driver will then retry the + /// callback indefinitely. To avoid this situation, the application MUST NOT silently handle + /// errors within the callback. If the application needs to handle errors within the + /// callback, it MUST return them after doing so. + /// /// The user defined callback. /// The transaction options. /// The cancellation token. @@ -149,6 +158,15 @@ public interface IClientSession : IDisposable /// /// Executes a callback within a transaction, with retries if needed. /// + /// + /// If a command inside the callback fails, it may cause the transaction on the server to be + /// aborted. This situation is normally handled transparently by the driver. However, if the + /// application does not return that error from the callback, the driver will not be able to + /// determine whether the transaction was aborted or not. The driver will then retry the + /// callback indefinitely. To avoid this situation, the application MUST NOT silently handle + /// errors within the callback. If the application needs to handle errors within the + /// callback, it MUST return them after doing so. + /// /// The user defined callback. /// The transaction options. /// The cancellation token. diff --git a/tests/MongoDB.Driver.Examples/TransactionExamplesForDocs/WithTransactionExample1.cs b/tests/MongoDB.Driver.Examples/TransactionExamplesForDocs/WithTransactionExample1.cs index 7240e534e9f..9251a5da4e7 100644 --- a/tests/MongoDB.Driver.Examples/TransactionExamplesForDocs/WithTransactionExample1.cs +++ b/tests/MongoDB.Driver.Examples/TransactionExamplesForDocs/WithTransactionExample1.cs @@ -65,8 +65,16 @@ public void Example1() result = session.WithTransaction( (s, ct) => { - collection1.InsertOne(s, new BsonDocument("abc", 1), cancellationToken: ct); - collection2.InsertOne(s, new BsonDocument("xyz", 999), cancellationToken: ct); + try + { + collection1.InsertOne(s, new BsonDocument("abc", 1), cancellationToken: ct); + collection2.InsertOne(s, new BsonDocument("xyz", 999), cancellationToken: ct); + } + catch (MongoWriteException) + { + // Do something in response to the exception + throw; // NOTE: You must rethrow the exception otherwise an infinite loop can occur. + } return "Inserted into collections in different databases"; }, transactionOptions, From 4f1d0ddda72b815e87362dea58cbcd803ec6e25f Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:18:49 -0700 Subject: [PATCH 07/38] CSHARP-5035: Improve serverless tests to obtain secrets from vault (#1314) --- evergreen/evergreen.yml | 99 +++++++++++++------------------ evergreen/run-serverless-tests.sh | 5 +- 2 files changed, 43 insertions(+), 61 deletions(-) diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 6c75ee348cc..00705a88752 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -738,9 +738,6 @@ functions: - "FLE_AZURE_CLIENT_SECRET" - "FLE_GCP_EMAIL" - "FLE_GCP_PRIVATE_KEY" - - "SERVERLESS_ATLAS_USER" - - "SERVERLESS_ATLAS_PASSWORD" - - "SERVERLESS_URI" script: | ${PREPARE_SHELL} AUTH=${AUTH} \ @@ -769,47 +766,6 @@ functions: FRAMEWORK=${FRAMEWORK} \ evergreen/run-tests.sh - create-serverless-instance: - - command: shell.exec - params: - shell: bash - include_expansions_in_env: - - "SERVERLESS_API_PUBLIC_KEY" - - "SERVERLESS_API_PRIVATE_KEY" - script: | - ${PREPARE_SHELL} - if [ "Terminating" = "${SERVERLESS_PROXY_TYPE}" ]; then - SERVERLESS_GROUP="${TERMINATING_PROXY_SERVERLESS_DRIVERS_GROUP}" - else - SERVERLESS_GROUP="${SERVERLESS_DRIVERS_GROUP}" - fi - SERVERLESS_DRIVERS_GROUP="$SERVERLESS_GROUP" \ - LOADBALANCED=ON \ - bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh - - command: expansions.update - params: - file: serverless-expansion.yml - - delete-serverless-instance-if-configured: - - command: shell.exec - params: - shell: bash - include_expansions_in_env: - - "SERVERLESS_API_PUBLIC_KEY" - - "SERVERLESS_API_PRIVATE_KEY" - script: | - if [ "" != "${SERVERLESS}" ]; then - ${PREPARE_SHELL} - if [ "Terminating" = "${SERVERLESS_PROXY_TYPE}" ]; then - SERVERLESS_GROUP="${TERMINATING_PROXY_SERVERLESS_DRIVERS_GROUP}" - else - SERVERLESS_GROUP="${SERVERLESS_DRIVERS_GROUP}" - fi - SERVERLESS_DRIVERS_GROUP="$SERVERLESS_GROUP" \ - SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ - bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - fi - start-kms-mock-servers: - command: shell.exec params: @@ -1059,9 +1015,6 @@ pre: - func: make-files-executable post: - # Removed, causing timeouts - # - func: upload-working-dir - - func: delete-serverless-instance-if-configured - func: upload-mo-artifacts - func: upload-test-results - func: cleanup @@ -1279,7 +1232,6 @@ tasks: - name: test-serverless exec_timeout_secs: 2700 # 45 minutes: 15 for setup + 30 for tests commands: - - func: create-serverless-instance - func: run-serverless-tests - name: test-ocsp-rsa-valid-cert-server-staples-ca-responder @@ -1949,17 +1901,17 @@ axes: variables: FRAMEWORK: netstandard21 - - id: serverless_proxy_type - display_name: Serverless Proxy Type + - id: serverless + display_name: Serverless values: - id: "Passthrough" display_name: "Serverless Passthrough Proxy" variables: - SERVERLESS_PROXY_TYPE: Passthrough + VAULT_NAME: "serverless" - id: "Terminating" display_name: "Serverless Terminating Proxy" variables: - SERVERLESS_PROXY_TYPE: Terminating + VAULT_NAME: "serverless_next" - id: build-target display_name: CI build target @@ -2194,6 +2146,35 @@ task_groups: tasks: - test-oidc-azure + - name: serverless-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch-source + - func: prepare-resources + - func: fix-absolute-paths + - func: make-files-executable + - func: assume-ec2-role + - command: subprocess.exec + params: + binary: bash + env: + VAULT_NAME: ${VAULT_NAME} + args: + - ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh + - command: expansions.update + params: + file: serverless-expansion.yml + teardown_group: + - func: upload-test-results + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh + tasks: + - test-serverless + buildvariants: - matrix_name: stable-api-tests matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "standalone", auth: "auth", ssl: "nossl", os: "windows-64" } @@ -2388,16 +2369,18 @@ buildvariants: # Serverless tests - matrix_name: serverless-tests-windows - matrix_spec: { auth: "auth", ssl: "ssl", compressor: "zlib", os: "windows-64", target_framework: "*", serverless_proxy_type: "*" } - display_name: "${serverless_proxy_type} ${compressor} ${auth} ${ssl} ${os} ${target_framework}" + matrix_spec: { auth: "auth", ssl: "ssl", compressor: "zlib", os: "windows-64", target_framework: ["net472", "netstandard20", "netstandard21"], serverless: "*" } + display_name: "${serverless} ${compressor} ${auth} ${ssl} ${os} ${target_framework}" + batchtime: 10080 # 7 days tasks: - - name: test-serverless + - name: serverless-task-group - matrix_name: serverless-tests-ubuntu - matrix_spec: { auth: "auth", ssl: "ssl", compressor: "zlib", os: "ubuntu-2004", target_framework: ["netstandard20", "netstandard21"], serverless_proxy_type: "*" } - display_name: "${serverless_proxy_type} ${compressor} ${auth} ${ssl} ${os} ${target_framework}" + matrix_spec: { auth: "auth", ssl: "ssl", compressor: "zlib", os: "ubuntu-2004", target_framework: ["netstandard20", "netstandard21"], serverless: "*" } + display_name: "${serverless} ${compressor} ${auth} ${ssl} ${os} ${target_framework}" + batchtime: 10080 # 7 days tasks: - - name: test-serverless + - name: serverless-task-group # Performance tests - name: driver-performance-tests diff --git a/evergreen/run-serverless-tests.sh b/evergreen/run-serverless-tests.sh index 101dd3452d5..ccbc770befe 100644 --- a/evergreen/run-serverless-tests.sh +++ b/evergreen/run-serverless-tests.sh @@ -7,9 +7,6 @@ set -o errexit # Exit the script with error if any of the commands fail # AUTH Authentication flag, must be "auth" # FRAMEWORK Used in build.cake "TestServerless" task, must be set # OS Operating system, must be set -# SERVERLESS_ATLAS_USER Authentication user, must be set -# SERVERLESS_ATLAS_PASSWORD Authentiction password, must be set -# SERVERLESS_URI Single atlas proxy serverless uri, must be set # SSL TLS connection flag, must be "ssl" # CRYPT_SHARED_LIB_PATH The path to crypt_shared library # Modified/exported environment variables: @@ -49,6 +46,8 @@ else done fi +source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh + # Assume "mongodb+srv" protocol export MONGODB_URI="mongodb+srv://${SERVERLESS_ATLAS_USER}:${SERVERLESS_ATLAS_PASSWORD}@${SERVERLESS_URI:14}" export SERVERLESS="true" From 7c4216e73ddcd72611b95fb2455b39f8c5c8596a Mon Sep 17 00:00:00 2001 From: rstam Date: Thu, 25 Apr 2024 18:44:32 -0700 Subject: [PATCH 08/38] CSHARP-5067: Filters with boolean fields or properties should use correct representation of true. --- .../ExpressionToFilterTranslator.cs | 4 +- .../MemberExpressionToFilterTranslator.cs | 6 +- .../ParameterExpressionToFilterTranslator.cs | 4 +- .../Jira/CSharp5067Tests.cs | 490 ++++++++++++++++++ 4 files changed, 500 insertions(+), 4 deletions(-) create mode 100644 tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5067Tests.cs diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionToFilterTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionToFilterTranslator.cs index 3477edb39ca..2a3a3c3a363 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionToFilterTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionToFilterTranslator.cs @@ -18,6 +18,7 @@ using System.Linq.Expressions; using MongoDB.Bson.Serialization; using MongoDB.Driver.Linq.Linq3Implementation.Ast.Filters; +using MongoDB.Driver.Linq.Linq3Implementation.Misc; using MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToAggregationExpressionTranslators; using MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToFilterTranslators.ExpressionTranslators; using MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToFilterTranslators.MethodTranslators; @@ -118,7 +119,8 @@ private static AstFilter TranslateUsingQueryOperators(TranslationContext context if (expression.Type == typeof(bool)) { var field = ExpressionToFilterFieldTranslator.Translate(context, expression); - return AstFilter.Eq(field, true); + var serializedTrue = SerializationHelper.SerializeValue(field.Serializer, true); + return AstFilter.Eq(field, serializedTrue); } throw new ExpressionNotSupportedException(expression); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/MemberExpressionToFilterTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/MemberExpressionToFilterTranslator.cs index eb9910eda3f..1cc6e257060 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/MemberExpressionToFilterTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/MemberExpressionToFilterTranslator.cs @@ -34,7 +34,8 @@ public static AstFilter Translate(TranslationContext context, MemberExpression e if (fieldInfo.FieldType == typeof(bool)) { var field = ExpressionToFilterFieldTranslator.Translate(context, expression); - return AstFilter.Eq(field, true); + var serializedTrue = SerializationHelper.SerializeValue(field.Serializer, true); + return AstFilter.Eq(field, serializedTrue); } } @@ -50,7 +51,8 @@ public static AstFilter Translate(TranslationContext context, MemberExpression e if (propertyInfo.PropertyType == typeof(bool)) { var field = ExpressionToFilterFieldTranslator.Translate(context, expression); - return AstFilter.Eq(field, true); + var serializedTrue = SerializationHelper.SerializeValue(field.Serializer, true); + return AstFilter.Eq(field, serializedTrue); } } diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/ParameterExpressionToFilterTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/ParameterExpressionToFilterTranslator.cs index b8e4e2ee45b..04d9cb78a94 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/ParameterExpressionToFilterTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/ExpressionTranslators/ParameterExpressionToFilterTranslator.cs @@ -15,6 +15,7 @@ using System.Linq.Expressions; using MongoDB.Driver.Linq.Linq3Implementation.Ast.Filters; +using MongoDB.Driver.Linq.Linq3Implementation.Misc; namespace MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToFilterTranslators.ExpressionTranslators { @@ -28,7 +29,8 @@ public static AstFilter Translate(TranslationContext context, ParameterExpressio { var serializer = context.KnownSerializersRegistry.GetSerializer(expression); var field = AstFilter.Field(symbol.Name, serializer); - return AstFilter.Eq(field, true); + var serializedTrue = SerializationHelper.SerializeValue(field.Serializer, true); + return AstFilter.Eq(field, serializedTrue); } } diff --git a/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5067Tests.cs b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5067Tests.cs new file mode 100644 index 00000000000..b3ba622f780 --- /dev/null +++ b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5067Tests.cs @@ -0,0 +1,490 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Linq; +using FluentAssertions; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver.Linq; +using MongoDB.TestHelpers.XunitExtensions; +using Xunit; + +namespace MongoDB.Driver.Tests.Linq.Linq3Implementation.Jira +{ + public class CSharp5067Tests : Linq3IntegrationTest + { + [Theory] + [ParameterAttributeData] + public void Where_with_bool_field_represented_as_boolean_should_work( + [Values(false, true)] bool justField, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = justField ? + collection.AsQueryable().Where(x => x.BoolFieldRepresentedAsBoolean) : + collection.AsQueryable().Where(x => x.BoolFieldRepresentedAsBoolean == true); + + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $match : { BoolFieldRepresentedAsBoolean : true } }"); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(2); + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_field_represented_as_int32_should_work( + [Values(false, true)] bool justField, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = justField ? + collection.AsQueryable().Where(x => x.BoolFieldRepresentedAsInt32) : + collection.AsQueryable().Where(x => x.BoolFieldRepresentedAsInt32 == true); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2 && justField) + { + AssertStages(stages, "{ $match : { BoolFieldRepresentedAsInt32 : true } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolFieldRepresentedAsInt32 : 1 } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_field_represented_as_string_should_work( + [Values(false, true)] bool justField, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = justField ? + collection.AsQueryable().Where(x => x.BoolFieldRepresentedAsString) : + collection.AsQueryable().Where(x => x.BoolFieldRepresentedAsString == true); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2 && justField) + { + AssertStages(stages, "{ $match : { BoolFieldRepresentedAsString : true } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolFieldRepresentedAsString : 'true' } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_array_field_represented_as_booleans_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable().Where(x => x.BoolArrayFieldRepresentedAsBoolean.Any(p => p)); + + var stages = Translate(collection, queryable); + if (linqProvider == LinqProvider.V2) + { + AssertStages(stages, "{ $match : { BoolArrayFieldRepresentedAsBoolean : { $elemMatch : { $eq : true } } } }"); // LINQ2 query is different but OK + } + else + { + AssertStages(stages, "{ $match : { BoolArrayFieldRepresentedAsBoolean : true } }"); + } + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(2); + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_array_field_represented_as_int32s_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable().Where(x => x.BoolArrayFieldRepresentedAsInt32.Any(p => p)); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2) + { + AssertStages(stages, "{ $match : { BoolArrayFieldRepresentedAsInt32 : { $elemMatch : { $eq : true } } } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolArrayFieldRepresentedAsInt32 : 1 } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_array_field_represented_as_strings_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable().Where(x => x.BoolArrayFieldRepresentedAsString.Any(p => p)); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2) + { + AssertStages(stages, "{ $match : { BoolArrayFieldRepresentedAsString : { $elemMatch : { $eq : true } } } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolArrayFieldRepresentedAsString : 'true' } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_property_represented_as_boolean_should_work( + [Values(false, true)] bool justProperty, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = justProperty ? + collection.AsQueryable().Where(x => x.BoolPropertyRepresentedAsBoolean) : + collection.AsQueryable().Where(x => x.BoolPropertyRepresentedAsBoolean == true); + + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $match : { BoolPropertyRepresentedAsBoolean : true } }"); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(2); + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_property_represented_as_int32_should_work( + [Values(false, true)] bool justProperty, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = justProperty ? + collection.AsQueryable().Where(x => x.BoolPropertyRepresentedAsInt32) : + collection.AsQueryable().Where(x => x.BoolPropertyRepresentedAsInt32 == true); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2 && justProperty) + { + AssertStages(stages, "{ $match : { BoolPropertyRepresentedAsInt32 : true } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolPropertyRepresentedAsInt32 : 1 } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_property_represented_as_string_should_work( + [Values(false, true)] bool justProperty, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = justProperty ? + collection.AsQueryable().Where(x => x.BoolPropertyRepresentedAsString) : + collection.AsQueryable().Where(x => x.BoolPropertyRepresentedAsString == true); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2 && justProperty) + { + AssertStages(stages, "{ $match : { BoolPropertyRepresentedAsString : true } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolPropertyRepresentedAsString : 'true' } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_array_property_represented_as_booleans_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsBoolean.Any(p => p)); + + var stages = Translate(collection, queryable); + if (linqProvider == LinqProvider.V2) + { + AssertStages(stages, "{ $match : { BoolArrayPropertyRepresentedAsBoolean : { $elemMatch : { $eq : true } } } }"); // LINQ2 query is different but OK + } + else + { + AssertStages(stages, "{ $match : { BoolArrayPropertyRepresentedAsBoolean : true } }"); + } + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(2); + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_array_property_represented_as_int32s_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsInt32.Any(p => p)); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2) + { + AssertStages(stages, "{ $match : { BoolArrayPropertyRepresentedAsInt32 : { $elemMatch : { $eq : true } } } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolArrayPropertyRepresentedAsInt32 : 1 } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_with_bool_array_property_represented_as_strings_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsString.Any(p => p)); + + var stages = Translate(collection, queryable); + var results = queryable.ToList(); + + if (linqProvider == LinqProvider.V2) + { + AssertStages(stages, "{ $match : { BoolArrayPropertyRepresentedAsString : { $elemMatch : { $eq : true } } } }"); // LINQ2 query is wrong + results.Should().BeEmpty(); // LINQ2 results are wrong + } + else + { + AssertStages(stages, "{ $match : { BoolArrayPropertyRepresentedAsString : 'true' } }"); + results.Select(x => x.Id).Should().Equal(2); + } + } + + [Theory] + [InlineData("property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : true } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : true } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("property[0] == false", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : false } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("property[0] == false", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : false } }", new[] { 1 }, LinqProvider.V3)] + [InlineData("property[0] == true", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : true } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("property[0] == true", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : true } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("!property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : { $ne : true } } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("!property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : { $ne : true } } }", new[] { 1 }, LinqProvider.V3)] + [InlineData("!(property[0] == false)", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : { $ne : false } } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("!(property[0] == false)", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : { $ne : false } } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("!(property[0] == true)", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : { $ne : true } } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("!(property[0] == true)", "{ $match : { 'BoolArrayPropertyRepresentedAsBoolean.0' : { $ne : true } } }", new[] { 1 }, LinqProvider.V3)] + public void Where_with_expression_using_bool_array_property_represented_as_booleans_should_work( + string expression, + string expectedStage, + int[] expectedResults, + LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = expression switch + { + "property[0]" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsBoolean[0]), + "property[0] == false" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsBoolean[0] == false), + "property[0] == true" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsBoolean[0] == true), + "!property[0]" => collection.AsQueryable().Where(x => !x.BoolArrayPropertyRepresentedAsBoolean[0]), + "!(property[0] == false)" => collection.AsQueryable().Where(x => !(x.BoolArrayPropertyRepresentedAsBoolean[0] == false)), + "!(property[0] == true)" => collection.AsQueryable().Where(x => !(x.BoolArrayPropertyRepresentedAsBoolean[0] == true)), + _ => throw new Exception() + }; + + var stages = Translate(collection, queryable); + AssertStages(stages, expectedStage); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(expectedResults); + } + + [Theory] + [InlineData("property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : true } }", new int[0], LinqProvider.V2)] // LINQ2 query and results are wrong + [InlineData("property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : 1 } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("property[0] == false", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : 0 } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("property[0] == false", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : { $ne : 1 } } }", new[] { 1 }, LinqProvider.V3)] + [InlineData("property[0] == true", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : 1 } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("property[0] == true", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : 1 } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("!property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : { $ne : true } } }", new[] { 1, 2 }, LinqProvider.V2)] // LINQ2 query and results are wrong + [InlineData("!property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : { $ne : 1 } } }", new[] { 1 }, LinqProvider.V3)] + [InlineData("!(property[0] == false)", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : { $ne : 0 } } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("!(property[0] == false)", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : 1 } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("!(property[0] == true)", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : { $ne : 1 } } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("!(property[0] == true)", "{ $match : { 'BoolArrayPropertyRepresentedAsInt32.0' : { $ne : 1 } } }", new[] { 1 }, LinqProvider.V3)] + public void Where_with_expression_using_bool_array_property_represented_as_int32s_should_work( + string expression, + string expectedStage, + int[] expectedResults, + LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = expression switch + { + "property[0]" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsInt32[0]), + "property[0] == false" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsInt32[0] == false), + "property[0] == true" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsInt32[0] == true), + "!property[0]" => collection.AsQueryable().Where(x => !x.BoolArrayPropertyRepresentedAsInt32[0]), + "!(property[0] == false)" => collection.AsQueryable().Where(x => !(x.BoolArrayPropertyRepresentedAsInt32[0] == false)), + "!(property[0] == true)" => collection.AsQueryable().Where(x => !(x.BoolArrayPropertyRepresentedAsInt32[0] == true)), + _ => throw new Exception() + }; + + var stages = Translate(collection, queryable); + AssertStages(stages, expectedStage); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(expectedResults); + } + + [Theory] + [InlineData("property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : true } }", new int[0], LinqProvider.V2)] // LINQ2 query and results are wrong + [InlineData("property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : 'true' } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("property[0] == false", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : 'false' } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("property[0] == false", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : { $ne : 'true' } } }", new[] { 1 }, LinqProvider.V3)] + [InlineData("property[0] == true", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : 'true' } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("property[0] == true", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : 'true' } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("!property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : { $ne : true } } }", new[] { 1, 2 }, LinqProvider.V2)] // LINQ2 query and results are wrong + [InlineData("!property[0]", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : { $ne : 'true' } } }", new[] { 1 }, LinqProvider.V3)] + [InlineData("!(property[0] == false)", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : { $ne : 'false' } } }", new[] { 2 }, LinqProvider.V2)] + [InlineData("!(property[0] == false)", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : 'true' } }", new[] { 2 }, LinqProvider.V3)] + [InlineData("!(property[0] == true)", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : { $ne : 'true' } } }", new[] { 1 }, LinqProvider.V2)] + [InlineData("!(property[0] == true)", "{ $match : { 'BoolArrayPropertyRepresentedAsString.0' : { $ne : 'true' } } }", new[] { 1 }, LinqProvider.V3)] + public void Where_with_expression_using_bool_array_property_represented_as_strings_should_work( + string expression, + string expectedStage, + int[] expectedResults, + LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = expression switch + { + "property[0]" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsString[0]), + "property[0] == false" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsString[0] == false), + "property[0] == true" => collection.AsQueryable().Where(x => x.BoolArrayPropertyRepresentedAsString[0] == true), + "!property[0]" => collection.AsQueryable().Where(x => !x.BoolArrayPropertyRepresentedAsString[0]), + "!(property[0] == false)" => collection.AsQueryable().Where(x => !(x.BoolArrayPropertyRepresentedAsString[0] == false)), + "!(property[0] == true)" => collection.AsQueryable().Where(x => !(x.BoolArrayPropertyRepresentedAsString[0] == true)), + _ => throw new Exception() + }; + + var stages = Translate(collection, queryable); + AssertStages(stages, expectedStage); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(expectedResults); + } + + private IMongoCollection GetCollection(LinqProvider linqProvider) + { + var collection = GetCollection("test", linqProvider); + CreateCollection( + collection, + new C + { + Id = 1, + BoolArrayFieldRepresentedAsBoolean = new[] { false }, + BoolArrayFieldRepresentedAsInt32 = new[] { false }, + BoolArrayFieldRepresentedAsString = new[] { false }, + BoolArrayPropertyRepresentedAsBoolean = new[] { false }, + BoolArrayPropertyRepresentedAsInt32 = new[] { false }, + BoolArrayPropertyRepresentedAsString = new[] { false }, + BoolFieldRepresentedAsBoolean = false, + BoolFieldRepresentedAsInt32 = false, + BoolFieldRepresentedAsString = false, + BoolPropertyRepresentedAsBoolean = false, + BoolPropertyRepresentedAsInt32 = false, + BoolPropertyRepresentedAsString = false + }, + new C { + Id = 2, + BoolArrayFieldRepresentedAsBoolean = new[] { true }, + BoolArrayFieldRepresentedAsInt32 = new[] { true }, + BoolArrayFieldRepresentedAsString = new[] { true }, + BoolArrayPropertyRepresentedAsBoolean = new[] { true }, + BoolArrayPropertyRepresentedAsInt32 = new[] { true }, + BoolArrayPropertyRepresentedAsString = new[] { true }, + BoolFieldRepresentedAsBoolean = true, + BoolFieldRepresentedAsInt32 = true, + BoolFieldRepresentedAsString = true, + BoolPropertyRepresentedAsBoolean = true, + BoolPropertyRepresentedAsInt32 = true, + BoolPropertyRepresentedAsString = true + }); + return collection; + } + + private class C + { + public int Id { get; set; } + public bool[] BoolArrayFieldRepresentedAsBoolean; + [BsonRepresentation(BsonType.Int32)] public bool[] BoolArrayFieldRepresentedAsInt32; + [BsonRepresentation(BsonType.String)] public bool[] BoolArrayFieldRepresentedAsString; + public bool[] BoolArrayPropertyRepresentedAsBoolean; + [BsonRepresentation(BsonType.Int32)] public bool[] BoolArrayPropertyRepresentedAsInt32; + [BsonRepresentation(BsonType.String)] public bool[] BoolArrayPropertyRepresentedAsString; + public bool BoolFieldRepresentedAsBoolean; + [BsonRepresentation(BsonType.Int32)] public bool BoolFieldRepresentedAsInt32; + [BsonRepresentation(BsonType.String)] public bool BoolFieldRepresentedAsString; + public bool BoolPropertyRepresentedAsBoolean { get; set; } + [BsonRepresentation(BsonType.Int32)] public bool BoolPropertyRepresentedAsInt32 { get; set; } + [BsonRepresentation(BsonType.String)] public bool BoolPropertyRepresentedAsString { get; set; } + } + } +} From 757a2381898ea1d667f5007e5c6904ac0315b960 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Tue, 7 May 2024 13:41:01 -0400 Subject: [PATCH 09/38] CSHARP-5069: Properly dispose of classes implementing IDisposable (#1316) --- .../AutoEncryptionLibMongoController.cs | 20 +++++++++++-------- .../Encryption/LibMongoCryptControllerBase.cs | 13 +++++++----- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/MongoDB.Driver/Encryption/AutoEncryptionLibMongoController.cs b/src/MongoDB.Driver/Encryption/AutoEncryptionLibMongoController.cs index e1adb9950d9..f0ab7195324 100644 --- a/src/MongoDB.Driver/Encryption/AutoEncryptionLibMongoController.cs +++ b/src/MongoDB.Driver/Encryption/AutoEncryptionLibMongoController.cs @@ -188,8 +188,9 @@ private void ProcessNeedCollectionInfoState(CryptContext context, string databas } var database = _metadataClient.GetDatabase(databaseName); - var filterBytes = context.GetOperation().ToArray(); - var filterDocument = new RawBsonDocument(filterBytes); + using var binary = context.GetOperation(); + var filterBytes = binary.ToArray(); + using var filterDocument = new RawBsonDocument(filterBytes); var filter = new BsonDocumentFilterDefinition(filterDocument); var options = new ListCollectionsOptions { Filter = filter }; var cursor = database.ListCollections(options, cancellationToken); @@ -206,8 +207,9 @@ private async Task ProcessNeedCollectionInfoStateAsync(CryptContext context, str } var database = _metadataClient.GetDatabase(databaseName); - var filterBytes = context.GetOperation().ToArray(); - var filterDocument = new RawBsonDocument(filterBytes); + using var binary = context.GetOperation(); + var filterBytes = binary.ToArray(); + using var filterDocument = new RawBsonDocument(filterBytes); var filter = new BsonDocumentFilterDefinition(filterDocument); var options = new ListCollectionsOptions { Filter = filter }; var cursor = await database.ListCollectionsAsync(options, cancellationToken).ConfigureAwait(false); @@ -218,8 +220,9 @@ private async Task ProcessNeedCollectionInfoStateAsync(CryptContext context, str private void ProcessNeedMongoMarkingsState(CryptContext context, string databaseName, CancellationToken cancellationToken) { var database = _mongocryptdClient.Value.GetDatabase(databaseName); - var commandBytes = context.GetOperation().ToArray(); - var commandDocument = new RawBsonDocument(commandBytes); + using var binary = context.GetOperation(); + var commandBytes = binary.ToArray(); + using var commandDocument = new RawBsonDocument(commandBytes); var command = new BsonDocumentCommand(commandDocument); BsonDocument response = null; @@ -242,8 +245,9 @@ private void ProcessNeedMongoMarkingsState(CryptContext context, string database private async Task ProcessNeedMongoMarkingsStateAsync(CryptContext context, string databaseName, CancellationToken cancellationToken) { var database = _mongocryptdClient.Value.GetDatabase(databaseName); - var commandBytes = context.GetOperation().ToArray(); - var commandDocument = new RawBsonDocument(commandBytes); + using var binary = context.GetOperation(); + var commandBytes = binary.ToArray(); + using var commandDocument = new RawBsonDocument(commandBytes); var command = new BsonDocumentCommand(commandDocument); BsonDocument response = null; diff --git a/src/MongoDB.Driver/Encryption/LibMongoCryptControllerBase.cs b/src/MongoDB.Driver/Encryption/LibMongoCryptControllerBase.cs index 949d259302e..59e7f754650 100644 --- a/src/MongoDB.Driver/Encryption/LibMongoCryptControllerBase.cs +++ b/src/MongoDB.Driver/Encryption/LibMongoCryptControllerBase.cs @@ -252,8 +252,9 @@ private async Task ProcessNeedKmsStateAsync(CryptContext context, CancellationTo private void ProcessNeedMongoKeysState(CryptContext context, CancellationToken cancellationToken) { - var filterBytes = context.GetOperation().ToArray(); - var filterDocument = new RawBsonDocument(filterBytes); + using var binary = context.GetOperation(); + var filterBytes = binary.ToArray(); + using var filterDocument = new RawBsonDocument(filterBytes); var filter = new BsonDocumentFilterDefinition(filterDocument); var cursor = _keyVaultCollection.Value.FindSync(filter, cancellationToken: cancellationToken); var results = cursor.ToList(cancellationToken); @@ -288,8 +289,9 @@ private async Task ProcessNeedKmsCredentialsAsync(CryptContext context, Cancella private async Task ProcessNeedMongoKeysStateAsync(CryptContext context, CancellationToken cancellationToken) { - var filterBytes = context.GetOperation().ToArray(); - var filterDocument = new RawBsonDocument(filterBytes); + using var binary = context.GetOperation(); + var filterBytes = binary.ToArray(); + using var filterDocument = new RawBsonDocument(filterBytes); var filter = new BsonDocumentFilterDefinition(filterDocument); var cursor = await _keyVaultCollection.Value.FindAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); var results = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); @@ -298,7 +300,8 @@ private async Task ProcessNeedMongoKeysStateAsync(CryptContext context, Cancella private byte[] ProcessReadyState(CryptContext context) { - return context.FinalizeForEncryption().ToArray(); + using var binary = context.FinalizeForEncryption(); + return binary.ToArray(); } private void SendKmsRequest(KmsRequest request, CancellationToken cancellation) From 93923108039739b6a39242c3b8f413374ba82419 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Tue, 7 May 2024 13:09:27 -0700 Subject: [PATCH 10/38] CSHARP-4610: OIDC: Automatic token acquisition for GCP Identity Provider (#1315) --- evergreen/evergreen.yml | 60 +++++++++++++- evergreen/run-mongodb-oidc-azure-tests.sh | 17 ---- evergreen/run-mongodb-oidc-env-tests.sh | 34 ++++++++ .../auth/tests/legacy/connection-string.json | 64 ++++++++++++++- .../auth/tests/legacy/connection-string.yml | 46 ++++++++++- .../Authentication/Oidc/AzureOidcCallback.cs | 47 ++--------- .../Authentication/Oidc/GcpOidcCallback.cs | 43 ++++++++++ .../Oidc/HttpRequestOidcCallback.cs | 81 +++++++++++++++++++ .../Oidc/OidcCallbackAdapterFactory.cs | 3 +- .../Authentication/Oidc/OidcConfiguration.cs | 11 +-- .../Core/Configuration/ConnectionString.cs | 23 ++++-- .../Oidc/OidcConfigurationTests.cs | 10 +++ .../UnifiedTestOperations/UnifiedEntityMap.cs | 1 + 13 files changed, 364 insertions(+), 76 deletions(-) delete mode 100644 evergreen/run-mongodb-oidc-azure-tests.sh create mode 100644 evergreen/run-mongodb-oidc-env-tests.sh create mode 100644 src/MongoDB.Driver.Core/Core/Authentication/Oidc/GcpOidcCallback.cs create mode 100644 src/MongoDB.Driver.Core/Core/Authentication/Oidc/HttpRequestOidcCallback.cs diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 00705a88752..67bfa43f2af 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -1222,13 +1222,33 @@ tasks: set -o errexit ${PREPARE_SHELL} - dotnet build ./tests/MongoDB.Driver.Tests/MongoDB.Driver.Tests.csproj - tar czf /tmp/mongo-csharp-driver.tgz ./tests/MongoDB.Driver.Tests/bin/Debug/net6.0 ./evergreen/run-mongodb-oidc-azure-tests.sh + dotnet build + tar czf /tmp/mongo-csharp-driver.tgz tests/*.Tests/bin/Debug/net6.0/ ./evergreen/run-mongodb-oidc-env-tests.sh export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-csharp-driver.tgz - export AZUREOIDC_TEST_CMD="./evergreen/run-mongodb-oidc-azure-tests.sh" + export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./evergreen/run-mongodb-oidc-env-tests.sh" bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh + - name: test-oidc-gcp + commands: + - command: shell.exec + params: + shell: bash + working_dir: mongo-csharp-driver + script: |- + set -o errexit + ${PREPARE_SHELL} + + # Copy secrets-export.sh created by '${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh' script to read secrets inside the vm + cp $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/secrets-export.sh ./secrets-export.sh + + dotnet build + tar czf /tmp/mongo-csharp-driver.tgz tests/*.Tests/bin/Debug/net6.0/ ./evergreen/run-mongodb-oidc-env-tests.sh ./secrets-export.sh + + export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-csharp-driver.tgz + export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./evergreen/run-mongodb-oidc-env-tests.sh" + bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh + - name: test-serverless exec_timeout_secs: 2700 # 45 minutes: 15 for setup + 30 for tests commands: @@ -2146,6 +2166,31 @@ task_groups: tasks: - test-oidc-azure + - name: oidc-auth-gcp-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 # 30 minutes + setup_group: + - func: fetch-source + - func: prepare-resources + - func: fix-absolute-paths + - func: make-files-executable + - func: install-dotnet + - command: subprocess.exec + params: + binary: bash + env: + GCPOIDC_VMNAME_PREFIX: "CSHARP_DRIVER" + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/setup.sh + teardown_group: + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh + tasks: + - test-oidc-gcp + - name: serverless-task-group setup_group_can_fail_task: true setup_group_timeout_secs: 1800 # 30 minutes @@ -2293,7 +2338,7 @@ buildvariants: - name: plain-auth-tests - matrix_name: mongodb-oidc-test-tests - matrix_spec: { os: [ "ubuntu-2004", "macos-1100" ] } + matrix_spec: { os: [ "ubuntu-2004" ] } display_name: "MongoDB-OIDC Auth (test) - ${os}" batchtime: 20160 # 14 days tasks: @@ -2306,6 +2351,13 @@ buildvariants: tasks: - name: oidc-auth-azure-task-group +- matrix_name: mongodb-oidc-gcp-tests + matrix_spec: { os: [ "ubuntu-2004" ] } + display_name: "MongoDB-OIDC Auth (gcp) - ${os}" + batchtime: 20160 # 14 days + tasks: + - name: oidc-auth-gcp-task-group + - matrix_name: "ocsp-tests" matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "rapid", "latest"], auth: "noauth", ssl: "ssl", topology: "standalone", os: "windows-64" } display_name: "OCSP ${version} ${os}" diff --git a/evergreen/run-mongodb-oidc-azure-tests.sh b/evergreen/run-mongodb-oidc-azure-tests.sh deleted file mode 100644 index e68409824a5..00000000000 --- a/evergreen/run-mongodb-oidc-azure-tests.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Don't trace since the URI contains a password that shouldn't show up in the logs -set -o errexit # Exit the script with error if any of the commands fail - -DOTNET_SDK_PATH="$(pwd)/.dotnet" - -echo "Downloading .NET SDK installer into $DOTNET_SDK_PATH folder..." -curl -Lfo ./dotnet-install.sh https://dot.net/v1/dotnet-install.sh -echo "Installing .NET LTS SDK..." -bash ./dotnet-install.sh --channel 6.0 --install-dir "$DOTNET_SDK_PATH" --no-path -export PATH=$PATH:$DOTNET_SDK_PATH - -source ./env.sh -MONGODB_URI="mongodb://${OIDC_ADMIN_USER}:${OIDC_ADMIN_PWD}@${MONGODB_URI:10}?authSource=admin" - -dotnet test --no-build --framework net6.0 --filter Category=MongoDbOidc -e OIDC_ENV=azure -e TOKEN_RESOURCE="${AZUREOIDC_RESOURCE}" -e MONGODB_URI="${MONGODB_URI}" --results-directory ./build/test-results --logger "console;verbosity=detailed" ./tests/MongoDB.Driver.Tests/bin/Debug/net6.0/MongoDB.Driver.Tests.dll diff --git a/evergreen/run-mongodb-oidc-env-tests.sh b/evergreen/run-mongodb-oidc-env-tests.sh new file mode 100644 index 00000000000..94f63e31eed --- /dev/null +++ b/evergreen/run-mongodb-oidc-env-tests.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# Don't trace since the URI contains a password that shouldn't show up in the logs +set -o errexit # Exit the script with error if any of the commands fail + +DOTNET_SDK_PATH="$(pwd)/.dotnet" + +echo "Downloading .NET SDK installer into $DOTNET_SDK_PATH folder..." +curl -Lfo ./dotnet-install.sh https://dot.net/v1/dotnet-install.sh +echo "Installing .NET LTS SDK..." +bash ./dotnet-install.sh --channel 6.0 --install-dir "$DOTNET_SDK_PATH" --no-path +export PATH=$DOTNET_SDK_PATH:$PATH + +if [ "$OIDC_ENV" == "azure" ]; then + source ./env.sh + TOKEN_RESOURCE="$AZUREOIDC_RESOURCE" +elif [ "$OIDC_ENV" == "gcp" ]; then + source ./secrets-export.sh + TOKEN_RESOURCE="$GCPOIDC_AUDIENCE" +else + echo "Unrecognized OIDC_ENV $OIDC_ENV" + exit 1 +fi + +if [[ "$MONGODB_URI" =~ ^mongodb:.* ]]; then + MONGODB_URI="mongodb://${OIDC_ADMIN_USER}:${OIDC_ADMIN_PWD}@${MONGODB_URI:10}?authSource=admin" +elif [[ "$MONGODB_URI" =~ ^mongodb\+srv:.* ]]; then + MONGODB_URI="mongodb+srv://${OIDC_ADMIN_USER}:${OIDC_ADMIN_PWD}@${MONGODB_URI:14}?authSource=admin" +else + echo "Unexpected MONGODB_URI format: $MONGODB_URI" + exit 1 +fi + +dotnet test --no-build --framework net6.0 --filter Category=MongoDbOidc -e OIDC_ENV="$OIDC_ENV" -e TOKEN_RESOURCE="$TOKEN_RESOURCE" -e MONGODB_URI="$MONGODB_URI" --results-directory ./build/test-results --logger "console;verbosity=detailed" ./tests/**/*.Tests.dll diff --git a/specifications/auth/tests/legacy/connection-string.json b/specifications/auth/tests/legacy/connection-string.json index 3c7a6b1d148..f4c7f8c88ea 100644 --- a/specifications/auth/tests/legacy/connection-string.json +++ b/specifications/auth/tests/legacy/connection-string.json @@ -482,7 +482,7 @@ } }, { - "description": "should recognise the mechanism with test integration (MONGODB-OIDC)", + "description": "should recognise the mechanism with test environment (MONGODB-OIDC)", "uri": "mongodb://localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:test", "valid": true, "credential": { @@ -569,6 +569,66 @@ } } }, + { + "description": "should accept a url-encoded TOKEN_RESOURCE (MONGODB-OIDC)", + "uri": "mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:mongodb%3A%2F%2Ftest-cluster", + "valid": true, + "credential": { + "username": "user", + "password": null, + "source": "$external", + "mechanism": "MONGODB-OIDC", + "mechanism_properties": { + "ENVIRONMENT": "azure", + "TOKEN_RESOURCE": "mongodb://test-cluster" + } + } + }, + { + "description": "should accept an un-encoded TOKEN_RESOURCE (MONGODB-OIDC)", + "uri": "mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:mongodb://test-cluster", + "valid": true, + "credential": { + "username": "user", + "password": null, + "source": "$external", + "mechanism": "MONGODB-OIDC", + "mechanism_properties": { + "ENVIRONMENT": "azure", + "TOKEN_RESOURCE": "mongodb://test-cluster" + } + } + }, + { + "description": "should handle a complicated url-encoded TOKEN_RESOURCE (MONGODB-OIDC)", + "uri": "mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:abc%2Cd%25ef%3Ag%26hi", + "valid": true, + "credential": { + "username": "user", + "password": null, + "source": "$external", + "mechanism": "MONGODB-OIDC", + "mechanism_properties": { + "ENVIRONMENT": "azure", + "TOKEN_RESOURCE": "abc,d%ef:g&hi" + } + } + }, + { + "description": "should url-encode a TOKEN_RESOURCE (MONGODB-OIDC)", + "uri": "mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:a$b", + "valid": true, + "credential": { + "username": "user", + "password": null, + "source": "$external", + "mechanism": "MONGODB-OIDC", + "mechanism_properties": { + "ENVIRONMENT": "azure", + "TOKEN_RESOURCE": "a$b" + } + } + }, { "description": "should accept a username and throw an error for a password with azure provider (MONGODB-OIDC)", "uri": "mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:foo", @@ -609,4 +669,4 @@ "credential": null } ] -} +} \ No newline at end of file diff --git a/specifications/auth/tests/legacy/connection-string.yml b/specifications/auth/tests/legacy/connection-string.yml index dcb90b27448..c88eb1edce8 100644 --- a/specifications/auth/tests/legacy/connection-string.yml +++ b/specifications/auth/tests/legacy/connection-string.yml @@ -374,7 +374,7 @@ tests: uri: mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:test valid: false credential: -- description: should throw an exception if username is specified for aws (MONGODB-OIDC) +- description: should throw an exception if username is specified for test (MONGODB-OIDC) uri: mongodb://principalName@localhost/?authMechanism=MONGODB-OIDC&ENVIRONMENT:test valid: false credential: @@ -412,6 +412,50 @@ tests: mechanism_properties: ENVIRONMENT: azure TOKEN_RESOURCE: foo +- description: should accept a url-encoded TOKEN_RESOURCE (MONGODB-OIDC) + uri: mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:mongodb%3A%2F%2Ftest-cluster + valid: true + credential: + username: user + password: null + source: $external + mechanism: MONGODB-OIDC + mechanism_properties: + ENVIRONMENT: azure + TOKEN_RESOURCE: 'mongodb://test-cluster' +- description: should accept an un-encoded TOKEN_RESOURCE (MONGODB-OIDC) + uri: mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:mongodb://test-cluster + valid: true + credential: + username: user + password: null + source: $external + mechanism: MONGODB-OIDC + mechanism_properties: + ENVIRONMENT: azure + TOKEN_RESOURCE: 'mongodb://test-cluster' +- description: should handle a complicated url-encoded TOKEN_RESOURCE (MONGODB-OIDC) + uri: mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:abc%2Cd%25ef%3Ag%26hi + valid: true + credential: + username: user + password: null + source: $external + mechanism: MONGODB-OIDC + mechanism_properties: + ENVIRONMENT: azure + TOKEN_RESOURCE: 'abc,d%ef:g&hi' +- description: should url-encode a TOKEN_RESOURCE (MONGODB-OIDC) + uri: mongodb://user@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:a$b + valid: true + credential: + username: user + password: null + source: $external + mechanism: MONGODB-OIDC + mechanism_properties: + ENVIRONMENT: azure + TOKEN_RESOURCE: a$b - description: should accept a username and throw an error for a password with azure provider (MONGODB-OIDC) uri: mongodb://user:pass@localhost/?authMechanism=MONGODB-OIDC&authMechanismProperties=ENVIRONMENT:azure,TOKEN_RESOURCE:foo valid: false diff --git a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/AzureOidcCallback.cs b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/AzureOidcCallback.cs index 96856af1a9c..c227256a5ae 100644 --- a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/AzureOidcCallback.cs +++ b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/AzureOidcCallback.cs @@ -15,16 +15,13 @@ using System; using System.IO; -using System.Net; -using System.Threading; -using System.Threading.Tasks; using MongoDB.Bson.IO; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Serializers; namespace MongoDB.Driver.Core.Authentication.Oidc { - internal sealed class AzureOidcCallback : IOidcCallback + internal sealed class AzureOidcCallback : HttpRequestOidcCallback { private readonly string _tokenResource; @@ -33,50 +30,20 @@ public AzureOidcCallback(string tokenResource) _tokenResource = tokenResource; } - public OidcAccessToken GetOidcAccessToken(OidcCallbackParameters parameters, CancellationToken cancellationToken) + protected override (Uri Uri, (string Key, string Value)[] headers) GetHttpRequestParams(OidcCallbackParameters parameters) { - var request = CreateMetadataRequest(parameters); - using (cancellationToken.Register(() => request.Abort(), useSynchronizationContext: false)) - { - var response = request.GetResponse(); - return ParseMetadataResponse((HttpWebResponse)response); - } - } - - public async Task GetOidcAccessTokenAsync(OidcCallbackParameters parameters, CancellationToken cancellationToken) - { - var request = CreateMetadataRequest(parameters); - using (cancellationToken.Register(() => request.Abort(), useSynchronizationContext: false)) - { - var response = await request.GetResponseAsync().ConfigureAwait(false); - return ParseMetadataResponse((HttpWebResponse)response); - } - } - - private HttpWebRequest CreateMetadataRequest(OidcCallbackParameters parameters) - { - var metadataUrl = $"http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource={_tokenResource}"; + var metadataUrl = $"http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource={Uri.EscapeDataString(_tokenResource)}"; if (!string.IsNullOrEmpty(parameters.UserName)) { - metadataUrl += $"&client_id={parameters.UserName}"; + metadataUrl += $"&client_id={Uri.EscapeDataString(parameters.UserName)}"; } - var request = WebRequest.CreateHttp(new Uri(metadataUrl)); - request.Headers["Metadata"] = "true"; - request.Accept = "application/json"; - request.Method = "GET"; - - return request; + return (new Uri(metadataUrl), new [] { ("Accept", "application/json"), ("Metadata", "true") }); } - private OidcAccessToken ParseMetadataResponse(HttpWebResponse response) + protected override OidcAccessToken ProcessHttpResponse(Stream responseStream) { - if (response.StatusCode != HttpStatusCode.OK) - { - throw new InvalidOperationException($"Response status code does not indicate success {response.StatusCode}:{response.StatusDescription}"); - } - - using var responseReader = new StreamReader(response.GetResponseStream()); + using var responseReader = new StreamReader(responseStream); using var jsonReader = new JsonReader(responseReader); var context = BsonDeserializationContext.CreateRoot(jsonReader); diff --git a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/GcpOidcCallback.cs b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/GcpOidcCallback.cs new file mode 100644 index 00000000000..4f2480d4d53 --- /dev/null +++ b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/GcpOidcCallback.cs @@ -0,0 +1,43 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.IO; + +namespace MongoDB.Driver.Core.Authentication.Oidc +{ + internal sealed class GcpOidcCallback : HttpRequestOidcCallback + { + private readonly string _tokenResource; + + public GcpOidcCallback(string tokenResource) + { + _tokenResource = tokenResource; + } + + protected override (Uri Uri, (string Key, string Value)[] headers) GetHttpRequestParams(OidcCallbackParameters parameters) + { + var metadataUrl = $"http://metadata/computeMetadata/v1/instance/service-accounts/default/identity?audience={Uri.EscapeDataString(_tokenResource)}"; + + return (new Uri(metadataUrl), new[] { ("Metadata-Flavor", "Google") } ); + } + + protected override OidcAccessToken ProcessHttpResponse(Stream responseStream) + { + using var responseReader = new StreamReader(responseStream); + return new OidcAccessToken(responseReader.ReadToEnd(), null); + } + } +} diff --git a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/HttpRequestOidcCallback.cs b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/HttpRequestOidcCallback.cs new file mode 100644 index 00000000000..a92f5e21d42 --- /dev/null +++ b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/HttpRequestOidcCallback.cs @@ -0,0 +1,81 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; + +namespace MongoDB.Driver.Core.Authentication.Oidc +{ + internal abstract class HttpRequestOidcCallback : IOidcCallback + { + public OidcAccessToken GetOidcAccessToken(OidcCallbackParameters parameters, CancellationToken cancellationToken) + { + var request = CreateRequest(parameters); + using (cancellationToken.Register(() => request.Abort(), useSynchronizationContext: false)) + { + using var response = request.GetResponse(); + return ProcessHttpResponse((HttpWebResponse)response); + } + } + + public async Task GetOidcAccessTokenAsync(OidcCallbackParameters parameters, CancellationToken cancellationToken) + { + var request = CreateRequest(parameters); + using (cancellationToken.Register(() => request.Abort(), useSynchronizationContext: false)) + { + using var response = await request.GetResponseAsync().ConfigureAwait(false); + return ProcessHttpResponse((HttpWebResponse)response); + } + } + + protected abstract (Uri Uri, (string Key, string Value)[] headers) GetHttpRequestParams(OidcCallbackParameters parameters); + protected abstract OidcAccessToken ProcessHttpResponse(Stream responseStream); + + private HttpWebRequest CreateRequest(OidcCallbackParameters parameters) + { + var metadataInfo = GetHttpRequestParams(parameters); + + var request = WebRequest.CreateHttp(metadataInfo.Uri); + request.Method = "GET"; + foreach (var header in metadataInfo.headers) + { + if (string.Equals(header.Key, "Accept", StringComparison.OrdinalIgnoreCase)) + { + request.Accept = header.Value; + } + else + { + request.Headers.Add(header.Key, header.Value); + } + } + + return request; + } + + private OidcAccessToken ProcessHttpResponse(HttpWebResponse response) + { + if (response.StatusCode != HttpStatusCode.OK) + { + throw new InvalidOperationException($"Response status code does not indicate success {response.StatusCode}:{response.StatusDescription}"); + } + + using var responseStream = response.GetResponseStream(); + return ProcessHttpResponse(responseStream); + } + } +} diff --git a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcCallbackAdapterFactory.cs b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcCallbackAdapterFactory.cs index 963841df9b1..7623d83438c 100644 --- a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcCallbackAdapterFactory.cs +++ b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcCallbackAdapterFactory.cs @@ -52,8 +52,9 @@ private IOidcCallbackAdapter CreateCallbackAdapter(OidcConfiguration configurati { callback = configuration.Environment switch { - "test" => FileOidcCallback.CreateFromEnvironmentVariable("OIDC_TOKEN_FILE", _environmentVariableProvider), "azure" => new AzureOidcCallback(configuration.TokenResource), + "gcp" => new GcpOidcCallback(configuration.TokenResource), + "test" => FileOidcCallback.CreateFromEnvironmentVariable("OIDC_TOKEN_FILE", _environmentVariableProvider), _ => throw new NotSupportedException($"Non supported {OidcConfiguration.EnvironmentMechanismPropertyName} value: {configuration.Environment}") }; } diff --git a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcConfiguration.cs b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcConfiguration.cs index e9e12bc486c..bb5628258e9 100644 --- a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcConfiguration.cs +++ b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/OidcConfiguration.cs @@ -28,7 +28,7 @@ internal sealed class OidcConfiguration public const string EnvironmentMechanismPropertyName = "ENVIRONMENT"; public const string TokenResourceMechanismPropertyName = "TOKEN_RESOURCE"; - private static readonly ISet __supportedEnvironments = new HashSet { "test", "azure" }; + private static readonly ISet __supportedEnvironments = new HashSet { "test", "azure", "gcp" }; private readonly int _hashCode; public OidcConfiguration( @@ -125,17 +125,18 @@ private void ValidateOptions() EnvironmentMechanismPropertyName); } - if (!string.IsNullOrEmpty(TokenResource) && Environment != "azure") + var tokenResourceRequired = Environment == "azure" || Environment == "gcp"; + if (!tokenResourceRequired && !string.IsNullOrEmpty(TokenResource)) { throw new ArgumentException( - $"{TokenResourceMechanismPropertyName} mechanism property supported only by azure environment.", + $"{TokenResourceMechanismPropertyName} mechanism property is not supported by {Environment} environment.", TokenResourceMechanismPropertyName); } - if (Environment == "azure" && string.IsNullOrEmpty(TokenResource)) + if (tokenResourceRequired && string.IsNullOrEmpty(TokenResource)) { throw new ArgumentException( - $"{TokenResourceMechanismPropertyName} mechanism property is required by azure environment.", + $"{TokenResourceMechanismPropertyName} mechanism property is required by {Environment} environment.", TokenResourceMechanismPropertyName); } } diff --git a/src/MongoDB.Driver.Core/Core/Configuration/ConnectionString.cs b/src/MongoDB.Driver.Core/Core/Configuration/ConnectionString.cs index c502ccc4a48..67a21dac11c 100644 --- a/src/MongoDB.Driver.Core/Core/Configuration/ConnectionString.cs +++ b/src/MongoDB.Driver.Core/Core/Configuration/ConnectionString.cs @@ -809,8 +809,8 @@ private void ExtractOptions(Match match) { var parts = option.Value.Split('='); var name = parts[0].Trim(); - var value = Uri.UnescapeDataString(parts[1].Trim()); - _allOptions.Add(name, value); + var value = parts[1].Trim(); + _allOptions.Add(name, Uri.UnescapeDataString(value)); ParseOption(name, value); } } @@ -980,6 +980,12 @@ string ProtectConnectionString(string connectionString) private void ParseOption(string name, string value) { + // Should not decode authmechanismproperties before splitting by separator. + if (!string.Equals(name, "authmechanismproperties", StringComparison.OrdinalIgnoreCase)) + { + value = Uri.UnescapeDataString(value); + } + switch (name.ToLowerInvariant()) { case "appname": @@ -1208,12 +1214,17 @@ private static IEnumerable> GetAuthMechanismPropert { foreach (var property in value.Split(',')) { - var parts = property.Split(':'); - if (parts.Length != 2) + var unescapedProperty = Uri.UnescapeDataString(property); + var separatorPosition = unescapedProperty.IndexOf(':'); + if (separatorPosition == -1) { - throw new MongoConfigurationException(string.Format("{0} has an invalid value of {1}.", name, value)); + throw new MongoConfigurationException($"{name} has an invalid value of {value}."); } - yield return new KeyValuePair(parts[0], parts[1]); + + var propertyKey = unescapedProperty.Substring(0, separatorPosition); + var propertyValue = unescapedProperty.Substring(separatorPosition + 1); + + yield return new KeyValuePair(propertyKey, propertyValue); } } diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Authentication/Oidc/OidcConfigurationTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Authentication/Oidc/OidcConfigurationTests.cs index a9a3f5e7f53..6e2d75ac5b2 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Authentication/Oidc/OidcConfigurationTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Authentication/Oidc/OidcConfigurationTests.cs @@ -52,6 +52,7 @@ public void Constructor_should_accept_valid_arguments( new object[] { new[] { new DnsEndPoint("localhost", 27017) }, null, new Dictionary { ["ENVIRONMENT"] = "test" }, "test", null }, new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "test" }, "test", null }, new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure", ["TOKEN_RESOURCE"] = "tr" }, "azure", null }, + new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "gcp", ["TOKEN_RESOURCE"] = "tr" }, "gcp", null }, new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["OIDC_CALLBACK"] = __callbackMock }, null, __callbackMock }, }; @@ -90,6 +91,9 @@ public void Constructor_throws_on_invalid_arguments( new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure" }, "TOKEN_RESOURCE" }, new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure", ["TOKEN_RESOURCE"] = null }, "TOKEN_RESOURCE" }, new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure", ["TOKEN_RESOURCE"] = "" }, "TOKEN_RESOURCE" }, + new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "gcp" }, "TOKEN_RESOURCE" }, + new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "gcp", ["TOKEN_RESOURCE"] = null }, "TOKEN_RESOURCE" }, + new object[] { new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "gcp", ["TOKEN_RESOURCE"] = "" }, "TOKEN_RESOURCE" }, }; [Theory] @@ -162,6 +166,12 @@ public void Equals_should_compare_by_values( new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure", ["TOKEN_RESOURCE"] = "tr" } }, new object[] + { + false, + new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure", ["TOKEN_RESOURCE"] = "tr" }, + new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "gcp", ["TOKEN_RESOURCE"] = "tr" } + }, + new object[] { false, new[] { new DnsEndPoint("localhost", 27017) }, "name", new Dictionary { ["ENVIRONMENT"] = "azure", ["TOKEN_RESOURCE"] = "tr1" }, diff --git a/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs b/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs index 392f5ceee8d..5ddfa797356 100644 --- a/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs +++ b/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs @@ -553,6 +553,7 @@ private IGridFSBucket CreateBucket(BsonDocument entity, Dictionary Date: Tue, 7 May 2024 13:39:32 -0700 Subject: [PATCH 11/38] Push master build to MyGet + validate Api Docs generation script (#1319) --- evergreen/evergreen.yml | 183 +- evergreen/evergreen.yml.bak | 2496 ----------------- evergreen/get-version.sh | 2 +- evergreen/push-packages.sh | 45 +- .../MongoDB.Driver.LambdaTest.csproj | 1 + 5 files changed, 149 insertions(+), 2578 deletions(-) delete mode 100644 evergreen/evergreen.yml.bak diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 67bfa43f2af..a7ecaa4f326 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -46,10 +46,7 @@ functions: CURRENT_VERSION=latest fi - if [ "${BUILD_TARGET}" = "release" ]; then - PACKAGE_VERSION=$(bash ./evergreen/get-version.sh) - fi - + PACKAGE_VERSION=$(bash ./evergreen/get-version.sh) export DOTNET_SDK_PATH="$(pwd)/../.dotnet" export DRIVERS_TOOLS="$(pwd)/../drivers-tools" @@ -925,6 +922,7 @@ functions: build-packages: - command: shell.exec params: + shell: bash working_dir: mongo-csharp-driver script: | ${PREPARE_SHELL} @@ -933,6 +931,7 @@ functions: push-packages: - command: shell.exec params: + shell: bash working_dir: mongo-csharp-driver env: PACKAGES_SOURCE: ${PACKAGES_SOURCE} @@ -941,40 +940,90 @@ functions: ${PREPARE_SHELL} . ./evergreen/push-packages.sh - upload-package: + upload-packages: - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg + local_files_include_filter: + - mongo-csharp-driver/artifacts/nuget/*.${PACKAGE_VERSION}.nupkg + - mongo-csharp-driver/artifacts/nuget/*.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/ + preserve_path: false bucket: mciuploads permissions: public-read content_type: ${content_type|application/octet-stream} - - command: s3.put + + download-packages: + - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Bson.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Bson.${PACKAGE_VERSION}.nupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Bson.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Bson.${PACKAGE_VERSION}.snupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Driver.${PACKAGE_VERSION}.nupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Driver.${PACKAGE_VERSION}.snupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.Core.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Driver.Core.${PACKAGE_VERSION}.nupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.Core.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Driver.Core.${PACKAGE_VERSION}.snupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.GridFS.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Driver.GridFS.${PACKAGE_VERSION}.nupkg + bucket: mciuploads + - command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.GridFS.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/MongoDB.Driver.GridFS.${PACKAGE_VERSION}.snupkg bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/octet-stream} - - download-package: - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg + local_file: mongo-csharp-driver/artifacts/nuget/mongocsharpdriver.${PACKAGE_VERSION}.nupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/mongocsharpdriver.${PACKAGE_VERSION}.nupkg bucket: mciuploads - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg + local_file: mongo-csharp-driver/artifacts/nuget/mongocsharpdriver.${PACKAGE_VERSION}.snupkg + remote_file: ${UPLOAD_BUCKET}/${revision}/mongocsharpdriver.${PACKAGE_VERSION}.snupkg bucket: mciuploads build-apidocs: @@ -984,10 +1033,6 @@ functions: working_dir: mongo-csharp-driver script: | ${PREPARE_SHELL} - if ! [[ "$PACKAGE_VERSION" =~ ^[0-9]+\.[0-9]+\.0$ ]]; then - echo "Skip api docs generating for the patch release" - exit 0 - fi ./evergreen/build-apidocs.sh upload-apidocs: @@ -1001,7 +1046,7 @@ functions: script: | ${PREPARE_SHELL} if ! [[ "$PACKAGE_VERSION" =~ ^[0-9]+\.[0-9]+\.0$ ]]; then - echo "Skip api docs generating for the patch release" + echo "Cannot upload api docs for the patch release" exit 0 fi ./evergreen/upload-apidocs.sh @@ -1721,51 +1766,37 @@ tasks: commands: - func: install-dotnet - func: build-packages - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Bson" - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Driver" - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Driver.Core" - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Driver.GridFS" - - func: upload-package - vars: - PACKAGE_ID: "mongocsharpdriver" + - func: upload-packages - - name: push-packages + - name: push-packages-nuget commands: - func: install-dotnet - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Bson" - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Driver" - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Driver.Core" - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Driver.GridFS" - - func: download-package - vars: - PACKAGE_ID: "mongocsharpdriver" + - func: download-packages - func: push-packages vars: PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json" PACKAGES_SOURCE_KEY: ${nuget_api_key} + - name: push-packages-myget + commands: + - func: install-dotnet + - func: download-packages + - func: push-packages + vars: + PACKAGES_SOURCE: "https://www.myget.org/F/mongodb/api/v3/index.json" + PACKAGES_SOURCE_KEY: ${myget_api_key} + - name: generate-apidocs commands: - func: install-dotnet - func: build-apidocs - func: upload-apidocs + - name: validate-apidocs + commands: + - func: install-dotnet + - func: build-apidocs + axes: - id: version display_name: MongoDB Version @@ -1933,18 +1964,6 @@ axes: variables: VAULT_NAME: "serverless_next" - - id: build-target - display_name: CI build target - values: - - id: "tests" - display_name: "tests" - variables: - BUILD_TARGET: "tests" - - id: "release" - display_name: "release" - variables: - BUILD_TARGET: "release" - task_groups: - name: testazurekms-task-group setup_group_can_fail_task: true @@ -2569,18 +2588,23 @@ buildvariants: # Package release variants - matrix_name: build-packages matrix_spec: - build-target: "release" os: "windows-64" # should produce package on Windows to make sure full framework binaries created. - display_name: "Package Pack" + display_name: "Packages Pack" tags: ["build-packages", "release-tag"] tasks: - name: build-packages - git_tag_only: true priority: 10 +- matrix_name: validate-apidocs + matrix_spec: + os: "ubuntu-2004" + display_name: "Validate API Documentation generation" + tags: ["build-apidocs"] + tasks: + - name: validate-apidocs + - matrix_name: generate-apidocs matrix_spec: - build-target: "release" os: "ubuntu-2004" display_name: "Generate API Documentation" tags: ["build-apidocs", "release-tag"] @@ -2593,17 +2617,28 @@ buildvariants: variant: ".build-packages" ## add dependency onto packages smoke test once it implemented -- matrix_name: push-packages +- matrix_name: push-packages-nuget matrix_spec: - build-target: "release" os: "ubuntu-2004" - display_name: "Package Push" + display_name: "Packages Push (NuGet)" tags: ["push-packages", "release-tag"] tasks: - - name: push-packages + - name: push-packages-nuget git_tag_only: true priority: 10 depends_on: - name: build-packages variant: ".build-packages" ## add dependency onto packages smoke test once it implemented + +- matrix_name: push-packages-myget + matrix_spec: + os: "ubuntu-2004" + display_name: "Packages Push (MyGet)" + tags: ["push-packages-myget"] + tasks: + - name: push-packages-myget + depends_on: + - name: build-packages + variant: ".build-packages" + ## add dependency onto packages smoke test once it implemented diff --git a/evergreen/evergreen.yml.bak b/evergreen/evergreen.yml.bak deleted file mode 100644 index b3f5aaa5805..00000000000 --- a/evergreen/evergreen.yml.bak +++ /dev/null @@ -1,2496 +0,0 @@ -######################################## -# Evergreen Template for MongoDB Drivers -######################################## - -# When a task that used to pass starts to fail -# Go through all versions that may have been skipped to detect -# when the task started failing -stepback: true - -# Mark a failure as a system/bootstrap failure (purple box) rather then a task -# failure by default. -# Actual testing tasks are marked with `type: test` -command_type: system - -# Protect ourself against rogue test case, or curl gone wild, that runs forever -# 60 minutes: 20 minutes is a normal test run + up to 10 minutes for test setup + 15 minutes for longer macOS tests + 15 minutes for longer macOS 1100 tests -exec_timeout_secs: 3600 - -# What to do when evergreen hits the timeout (`post:` tasks are run automatically) -timeout: - - command: shell.exec - params: - script: | - ls -la - df -h - -functions: - - fetch-source: - # Executes git clone and applies the submitted patch, if any - - command: git.get_project - params: - directory: mongo-csharp-driver - # Applies the submitted patch, if any - # Deprecated. Should be removed. But still needed for certain agents (ZAP) - - command: git.apply_patch - # Make an evergreen expansion file with dynamic values - - command: shell.exec - params: - working_dir: mongo-csharp-driver - script: | - # Get the current unique version of this checkout - if [ "${is_patch}" = "true" ]; then - CURRENT_VERSION=$(git describe)-patch-${version_id} - else - CURRENT_VERSION=latest - fi - - if [ "${BUILD_TARGET}" = "release" ]; then - PACKAGE_VERSION=$(bash ./evergreen/get-version.sh) - fi - - export DOTNET_SDK_PATH="$(pwd)/../.dotnet" - export DRIVERS_TOOLS="$(pwd)/../drivers-tools" - - if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin - # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory - export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) - else - # non windows OSs don't have dotnet in the PATH - export PATH=$PATH:/usr/share/dotnet - fi - - export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" - export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" - export PROJECT_DIRECTORY="$(pwd)" - - cat < expansion.yml - CURRENT_VERSION: "$CURRENT_VERSION" - DRIVERS_TOOLS: "$DRIVERS_TOOLS" - MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" - MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" - PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" - PACKAGE_VERSION: "$PACKAGE_VERSION" - DOTNET_SDK_PATH: "$DOTNET_SDK_PATH" - PREPARE_SHELL: | - set -o errexit - set -o xtrace - export DRIVERS_TOOLS="$DRIVERS_TOOLS" - export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" - export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" - export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - export PACKAGE_VERSION="$PACKAGE_VERSION" - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" - export PATH="$DOTNET_SDK_PATH:$MONGODB_BINARIES:$PATH" - export PROJECT="${project}" - EOT - # See what we've done - cat expansion.yml - - # Load the expansion file to make an evergreen variable with the current unique version - - command: expansions.update - params: - file: mongo-csharp-driver/expansion.yml - - install-dotnet: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - bash ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh - - prepare-resources: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - rm -rf $DRIVERS_TOOLS - if [ "${project}" = "drivers-tools" ]; then - # If this was a patch build, doing a fresh clone would not actually test the patch - cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS - else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS - fi - echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config - - # Upload build artifacts that other tasks may depend on - # Note this URL needs to be totally unique, while predictable for the next task - # so it can automatically download the artifacts - upload-build: - # Compress and upload the entire build directory - - command: archive.targz_pack - params: - # Example: mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz - target: "${build_id}.tar.gz" - source_dir: ${PROJECT_DIRECTORY}/ - include: - - "./**" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ${build_id}.tar.gz - # Example: /mciuploads/${UPLOAD_BUCKET}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${task_name}/${build_id}.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - - exec-script: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - ${PROJECT_DIRECTORY}/${file} - - upload-mo-artifacts: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: mongodb-logs.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "mongodb-logs.tar.gz" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: drivers-tools/.evergreen/orchestration/server.log - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log - bucket: mciuploads - permissions: public-read - content_type: ${content_type|text/plain} - display_name: "orchestration.log" - - upload-working-dir: - - command: archive.targz_pack - params: - target: "working-dir.tar.gz" - source_dir: ${PROJECT_DIRECTORY}/ - include: - - "./**" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: working-dir.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "working-dir.tar.gz" - - command: archive.targz_pack - params: - target: "drivers-dir.tar.gz" - source_dir: ${DRIVERS_TOOLS} - include: - - "./**" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: drivers-dir.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: "drivers-dir.tar.gz" - - upload-test-results: - - command: attach.xunit_results - params: - file: ./mongo-csharp-driver/build/test-results/TEST-*.xml - - bootstrap-mongo-orchestration: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - LOAD_BALANCER=${LOAD_BALANCER} \ - MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - STORAGE_ENGINE=${STORAGE_ENGINE} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - SKIP_LEGACY_SHELL=${SKIP_LEGACY_SHELL} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - # run-orchestration generates expansion file with the MONGODB_URI for the cluster - - command: expansions.update - params: - file: mo-expansion.yml - - ocsp-bootstrap-mongo-orchestration: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - MONGODB_VERSION=${VERSION} \ - TOPOLOGY=${TOPOLOGY} \ - AUTH=${AUTH} \ - SSL=${SSL} \ - ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ - bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - - command: expansions.update - params: - file: mo-expansion.yml - - bootstrap-mongohoused: - - command: shell.exec - params: - script: | - DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - - command: shell.exec - params: - script: | - DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh - - run-load-balancer: - - command: shell.exec - params: - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start - - command: expansions.update - params: - file: lb-expansion.yml - - run-load-balancer-tests: - - command: shell.exec - type: test - params: - working_dir: "mongo-csharp-driver" - script: | - ${PREPARE_SHELL} - OS=${OS} evergreen/add-ca-certs.sh - AUTH="${AUTH}" SSL="${SSL}" \ - FRAMEWORK=${FRAMEWORK} \ - OS=${OS} \ - SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ - MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ - evergreen/run-load-balancer-tests.sh - - stop-load-balancer: - - command: shell.exec - params: - script: | - cd ${DRIVERS_TOOLS}/.evergreen - DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop - - run-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "FLE_AWS_ACCESS_KEY_ID" - - "FLE_AWS_SECRET_ACCESS_KEY" - - "FLE_AZURE_TENANT_ID" - - "FLE_AZURE_CLIENT_ID" - - "FLE_AZURE_CLIENT_SECRET" - - "FLE_GCP_EMAIL" - - "FLE_GCP_PRIVATE_KEY" - script: | - . ./evergreen/set-virtualenv.sh - . ./evergreen/set-temp-fle-aws-creds.sh - ${PREPARE_SHELL} - OS=${OS} \ - evergreen/add-ca-certs.sh - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY=${TOPOLOGY} \ - OS=${OS} \ - COMPRESSOR=${COMPRESSOR} \ - CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - FRAMEWORK=${FRAMEWORK} \ - CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ - evergreen/run-tests.sh - echo "Skipping certificate removal..." - OS=${OS} \ - evergreen/cleanup-test-resources.sh - - run-csfle-with-mocked-kms-tests: - - command: shell.exec - type: test - params: - working_dir: "mongo-csharp-driver" - include_expansions_in_env: - - "FLE_AWS_ACCESS_KEY_ID" - - "FLE_AWS_SECRET_ACCESS_KEY" - - "FLE_AZURE_TENANT_ID" - - "FLE_AZURE_CLIENT_ID" - - "FLE_AZURE_CLIENT_SECRET" - - "FLE_GCP_EMAIL" - - "FLE_GCP_PRIVATE_KEY" - script: | - export KMS_MOCK_SERVERS_ENABLED=true - export GCE_METADATA_HOST="localhost:5000" - export AZURE_IMDS_MOCK_ENDPOINT="localhost:8080" - ${PREPARE_SHELL} - OS=${OS} \ - evergreen/add-ca-certs.sh - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY=${TOPOLOGY} \ - OS=${OS} \ - CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \ - FRAMEWORK=${FRAMEWORK} \ - TARGET="TestCsfleWithMockedKms" \ - CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ - evergreen/run-tests.sh - OS=${OS} \ - evergreen/cleanup-test-resources.sh - - run-csfle-with-mongocryptd-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "FLE_AWS_ACCESS_KEY_ID" - - "FLE_AWS_SECRET_ACCESS_KEY" - - "FLE_AZURE_TENANT_ID" - - "FLE_AZURE_CLIENT_ID" - - "FLE_AZURE_CLIENT_SECRET" - - "FLE_GCP_EMAIL" - - "FLE_GCP_PRIVATE_KEY" - script: | - . ./evergreen/set-virtualenv.sh - . ./evergreen/set-temp-fle-aws-creds.sh - ${PREPARE_SHELL} - OS=${OS} \ - evergreen/add-ca-certs.sh - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY=${TOPOLOGY} \ - OS=${OS} \ - COMPRESSOR=${COMPRESSOR} \ - CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - TARGET="TestCsfleWithMongocryptd" \ - FRAMEWORK=${FRAMEWORK} \ - CRYPT_SHARED_LIB_PATH="" \ - evergreen/run-tests.sh - echo "Skipping certificate removal..." - OS=${OS} \ - evergreen/cleanup-test-resources.sh - - run-atlas-connectivity-tests: - - command: shell.exec - type: test - params: - silent: true - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "ATLAS_FREE" - - "ATLAS_FREE_SRV" - - "ATLAS_REPLICA" - - "ATLAS_REPLICA_SRV" - - "ATLAS_SHARDED" - - "ATLAS_SHARDED_SRV" - - "ATLAS_TLS11" - - "ATLAS_TLS11_SRV" - - "ATLAS_TLS12" - - "ATLAS_TLS12_SRV" - - "ATLAS_SERVERLESS" - - "ATLAS_SERVERLESS_SRV" - script: | - . evergreen/run-atlas-connectivity-tests.sh - - run-gssapi-auth-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "AUTH_GSSAPI" - - "AUTH_HOST" - script: | - PROJECT_DIRECTORY=${PROJECT_DIRECTORY} \ - FRAMEWORK=${FRAMEWORK} \ - evergreen/run-gssapi-auth-tests.sh - - run-plain-auth-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - env: - MONGODB_URI: ${plain_auth_mongodb_uri} - script: | - ${PREPARE_SHELL} - OS=${OS} \ - evergreen/run-plain-auth-tests.sh - - run-performance-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver/benchmarks/MongoDB.Driver.Benchmarks - script: | - ${PREPARE_SHELL} - MONGODB_URI="${MONGODB_URI}" ../../evergreen/run-perf-tests.sh - - command: perf.send - params: - file: mongo-csharp-driver/benchmarks/MongoDB.Driver.Benchmarks/Benchmark.Artifacts/results/evergreen-results.json - - assume-ec2-role: - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - - add-aws-auth-variables-to-file: - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - - command: shell.exec - type: test - params: - shell: "bash" - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "AWS_ACCESS_KEY_ID" - - "AWS_SECRET_ACCESS_KEY" - - "AWS_SESSION_TOKEN" - script: | - ${PREPARE_SHELL} - cd $DRIVERS_TOOLS/.evergreen/auth_aws - ./setup_secrets.sh drivers/aws_auth - - run-aws-auth-test-with-regular-aws-credentials: - - command: shell.exec - type: test - params: - env: - IAM_AUTH_ECS_ACCOUNT: ${iam_auth_ecs_account} - IAM_AUTH_ECS_SECRET_ACCESS_KEY: ${iam_auth_ecs_secret_access_key} - working_dir: mongo-csharp-driver - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=${OS} evergreen/run-mongodb-aws-test.sh regular - - run-aws-auth-test-with-assume-role-credentials: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=${OS} evergreen/run-mongodb-aws-test.sh assume-role - - run-aws-auth-test-with-aws-EC2-credentials: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - if [ "${skip_EC2_auth_test}" = "true" ]; then - echo "This platform does not support the EC2 auth test, skipping..." - exit 0 - fi - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=${OS} ASSERT_NO_URI_CREDS=true evergreen/run-mongodb-aws-test.sh ec2 - - run-aws-auth-test-with-aws-ECS-credentials: - - command: shell.exec - type: test - params: - shell: "bash" - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - if [ "${skip_ECS_auth_test}" = "true" ]; then - echo "This platform does not support the ECS auth test, skipping..." - exit 0 - fi - echo "Project Directory: $PROJECT_DIRECTORY" - # SRC_DIRECTORY is workaround since EG_TOOLS expects "src" folder as a root - SRC_DIRECTORY=$(dirname $PROJECT_DIRECTORY)/src - echo "Src Directory: $SRC_DIRECTORY" - cp -r $PROJECT_DIRECTORY $SRC_DIRECTORY - # Workaround. EG_TOOLS scripts for ECS assume that a folder with EG scripts in the driver is named ".evergreen" - mkdir $SRC_DIRECTORY/.evergreen - cp -r $SRC_DIRECTORY/evergreen/run-mongodb-aws-ecs-test.sh $SRC_DIRECTORY/.evergreen/run-mongodb-aws-ecs-test.sh - - export PROJECT_DIRECTORY="$SRC_DIRECTORY" - ${DRIVERS_TOOLS}/.evergreen/auth_aws/aws_setup.sh ecs - - run-aws-auth-test-with-aws-web-identity-credentials: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - if [ "${skip_web_identity_auth_test}" = "true" ]; then - echo "This platform does not support the web identity auth test, skipping..." - exit 0 - fi - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=$OS ASSERT_NO_URI_CREDS=true evergreen/run-mongodb-aws-test.sh web-identity - - command: shell.exec - type: test - params: - shell: "bash" - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - if [ "${skip_web_identity_auth_test}" = "true" ]; then - echo "This platform does not support the web identity auth test, skipping..." - exit 0 - fi - export AWS_ROLE_SESSION_NAME="test" - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=$OS ASSERT_NO_URI_CREDS=true evergreen/run-mongodb-aws-test.sh web-identity - - run-aws-auth-test-with-aws-credentials-as-environment-variables: - - command: shell.exec - type: test - params: - env: - IAM_AUTH_ECS_ACCOUNT: ${iam_auth_ecs_account} - IAM_AUTH_ECS_SECRET_ACCESS_KEY: ${iam_auth_ecs_secret_access_key} - working_dir: mongo-csharp-driver - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=${OS} ASSERT_NO_URI_CREDS=true evergreen/run-mongodb-aws-test.sh env-creds - - run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - DRIVERS_TOOLS=${DRIVERS_TOOLS} OS=${OS} ASSERT_NO_URI_CREDS=true evergreen/run-mongodb-aws-test.sh session-creds - - run-atlas-data-lake-test: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - evergreen/run-atlas-data-lake-test.sh - - run-atlas-search-test: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "ATLAS_SEARCH" - script: | - ${PREPARE_SHELL} - evergreen/run-atlas-search-test.sh - - run-atlas-search-index-helpers-test: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - MONGODB_URI="${MONGODB_URI}" evergreen/run-atlas-search-index-helpers-test.sh - - run-ocsp-test: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ - OCSP_ALGORITHM=${OCSP_ALGORITHM} \ - evergreen/add-ca-certs.sh - set +o xtrace - AUTH="${AUTH}" \ - SSL="ssl" \ - TOPOLOGY="${TOPOLOGY}" \ - MONGODB_URI="${MONGODB_URI}" \ - OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \ - evergreen/run-tests.sh - echo "Skipping certificate removal..." - - run-valid-ocsp-server-ca-responder: - - command: shell.exec - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - evergreen/prepare-ocsp.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - set -o xtrace - cd ${DRIVERS_TOOLS}/.evergreen/ocsp - . ./activate-ocspvenv.sh - nohup python ocsp_mock.py \ - --ca_file ${OCSP_ALGORITHM}/ca.pem \ - --ocsp_responder_cert ${OCSP_ALGORITHM}/ca.crt \ - --ocsp_responder_key ${OCSP_ALGORITHM}/ca.key \ - -p 8100 -v - - run-valid-ocsp-server-delegate-responder: - - command: shell.exec - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - evergreen/prepare-ocsp.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - set -o xtrace - cd ${DRIVERS_TOOLS}/.evergreen/ocsp - . ./activate-ocspvenv.sh - nohup python ocsp_mock.py \ - --ca_file ${OCSP_ALGORITHM}/ca.pem \ - --ocsp_responder_cert ${OCSP_ALGORITHM}/ocsp-responder.crt \ - --ocsp_responder_key ${OCSP_ALGORITHM}/ocsp-responder.key \ - -p 8100 -v - - run-revoked-ocsp-server-ca-responder: - - command: shell.exec - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - evergreen/prepare-ocsp.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - set -o xtrace - cd ${DRIVERS_TOOLS}/.evergreen/ocsp - . ./activate-ocspvenv.sh - nohup python ocsp_mock.py \ - --ca_file ${OCSP_ALGORITHM}/ca.pem \ - --ocsp_responder_cert ${OCSP_ALGORITHM}/ca.crt \ - --ocsp_responder_key ${OCSP_ALGORITHM}/ca.key \ - -p 8100 \ - -v \ - --fault revoked - - run-revoked-ocsp-server-delegate-responder: - - command: shell.exec - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - evergreen/prepare-ocsp.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - set -o xtrace - cd ${DRIVERS_TOOLS}/.evergreen/ocsp - . ./activate-ocspvenv.sh - nohup python ocsp_mock.py \ - --ca_file ${OCSP_ALGORITHM}/ca.pem \ - --ocsp_responder_cert ${OCSP_ALGORITHM}/ocsp-responder.crt \ - --ocsp_responder_key ${OCSP_ALGORITHM}/ocsp-responder.key \ - -p 8100 \ - -v \ - --fault revoked - - run-mongodb-oidc-tests: - - command: subprocess.exec - type: test - params: - working_dir: mongo-csharp-driver - binary: bash - include_expansions_in_env: - - "DRIVERS_TOOLS" - - "OS" - - "FRAMEWORK" - args: - - evergreen/run-mongodb-oidc-tests.sh - - run-serverless-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - include_expansions_in_env: - - "FLE_AWS_ACCESS_KEY_ID" - - "FLE_AWS_SECRET_ACCESS_KEY" - - "FLE_AZURE_TENANT_ID" - - "FLE_AZURE_CLIENT_ID" - - "FLE_AZURE_CLIENT_SECRET" - - "FLE_GCP_EMAIL" - - "FLE_GCP_PRIVATE_KEY" - - "SERVERLESS_ATLAS_USER" - - "SERVERLESS_ATLAS_PASSWORD" - - "SERVERLESS_URI" - script: | - ${PREPARE_SHELL} - AUTH=${AUTH} \ - FRAMEWORK=${FRAMEWORK} \ - SSL=${SSL} \ - CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} \ - evergreen/run-serverless-tests.sh - - run-smoke-tests: - - command: shell.exec - type: test - params: - working_dir: mongo-csharp-driver - script: | - set +x - ${PREPARE_SHELL} - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_URI="${MONGODB_URI}" \ - TOPOLOGY=${TOPOLOGY} \ - OS=${OS} \ - COMPRESSOR=${COMPRESSOR} \ - CLIENT_PEM=${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem \ - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ - TARGET="SmokeTests" \ - FRAMEWORK=${FRAMEWORK} \ - evergreen/run-tests.sh - - create-serverless-instance: - - command: shell.exec - params: - shell: bash - include_expansions_in_env: - - "SERVERLESS_API_PUBLIC_KEY" - - "SERVERLESS_API_PRIVATE_KEY" - script: | - ${PREPARE_SHELL} - if [ "Terminating" = "${SERVERLESS_PROXY_TYPE}" ]; then - SERVERLESS_GROUP="${TERMINATING_PROXY_SERVERLESS_DRIVERS_GROUP}" - else - SERVERLESS_GROUP="${SERVERLESS_DRIVERS_GROUP}" - fi - SERVERLESS_DRIVERS_GROUP="$SERVERLESS_GROUP" \ - LOADBALANCED=ON \ - bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh - - command: expansions.update - params: - file: serverless-expansion.yml - - delete-serverless-instance-if-configured: - - command: shell.exec - params: - shell: bash - include_expansions_in_env: - - "SERVERLESS_API_PUBLIC_KEY" - - "SERVERLESS_API_PRIVATE_KEY" - script: | - if [ "" != "${SERVERLESS}" ]; then - ${PREPARE_SHELL} - if [ "Terminating" = "${SERVERLESS_PROXY_TYPE}" ]; then - SERVERLESS_GROUP="${TERMINATING_PROXY_SERVERLESS_DRIVERS_GROUP}" - else - SERVERLESS_GROUP="${SERVERLESS_DRIVERS_GROUP}" - fi - SERVERLESS_DRIVERS_GROUP="$SERVERLESS_GROUP" \ - SERVERLESS_INSTANCE_NAME=${SERVERLESS_INSTANCE_NAME} \ - bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - fi - - start-kms-mock-servers: - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - #expired client cert - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 - - command: shell.exec - params: - background: true - shell: "bash" - script: | - #wrong-host client cert - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 - - command: shell.exec - params: - background: true - shell: "bash" - script: | - #server.pem client cert - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert - - start-kms-mock-kmip-server: - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - - command: shell.exec - params: - shell: "bash" - background: true - script: | - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python -u kms_kmip_server.py - - start-kms-mock-gcp-server: - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - cd ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms - . ./activate-kmstlsvenv.sh - python -m pip install PyJWT - mkdir ${DRIVERS_TOOLS}/tmp - echo '${GOOGLE_APPLICATION_CREDENTIALS_CONTENT}' > ${DRIVERS_TOOLS}/tmp/testgcpkms_key_file.json - export GOOGLE_APPLICATION_CREDENTIALS=${DRIVERS_TOOLS}/tmp/testgcpkms_key_file.json - python -u mock_server.py - - start-kms-mock-azure-imds-server: - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - - command: shell.exec - params: - background: true - shell: "bash" - script: | - cd ${DRIVERS_TOOLS}/.evergreen/csfle - . ./activate-kmstlsvenv.sh - python bottle.py fake_azure:imds - - cleanup: - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - cd "${DRIVERS_TOOLS}/.evergreen/auth_aws" - if [ -f "./aws_e2e_setup.json" ]; then - . ./activate-authawsvenv.sh - python ./lib/aws_assign_instance_profile.py - fi - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - cd "$MONGO_ORCHESTRATION_HOME" - # source the mongo-orchestration virtualenv if it exists - if [ -f venv/bin/activate ]; then - . venv/bin/activate - elif [ -f venv/Scripts/activate ]; then - . venv/Scripts/activate - fi - mongo-orchestration stop - cd - - rm -rf $DRIVERS_TOOLS || true - - fix-absolute-paths: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for filename in $(find ${DRIVERS_TOOLS} -name \*.json); do - perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename - done - - windows-fix: - - command: shell.exec - params: - script: | - if [ "Windows_NT" = "$OS" ]; then - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/evergreen -name \*.sh); do - cat $i | tr -d '\r' > $i.new - mv $i.new $i - done - # Copy client certificate because symlinks do not work on Windows. - cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem - fi - - make-files-executable: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - for i in $(find ${DRIVERS_TOOLS}/.evergreen ${PROJECT_DIRECTORY}/evergreen -name \*.sh); do - chmod +x $i - done - - init-test-results: - - command: shell.exec - params: - script: | - ${PREPARE_SHELL} - echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json - - build-packages: - - command: shell.exec - params: - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - . ./evergreen/build-packages.sh - - push-packages: - - command: shell.exec - params: - working_dir: mongo-csharp-driver - env: - PACKAGES_SOURCE: ${PACKAGES_SOURCE} - PACKAGES_SOURCE_KEY: ${PACKAGES_SOURCE_KEY} - script: | - ${PREPARE_SHELL} - . ./evergreen/push-packages.sh - - upload-package: - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/octet-stream} - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/octet-stream} - - download-package: - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg - bucket: mciuploads - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: mongo-csharp-driver/artifacts/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg - remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg - bucket: mciuploads - - build-apidocs: - - command: shell.exec - params: - shell: bash - working_dir: mongo-csharp-driver - script: | - ${PREPARE_SHELL} - if ! [[ "$PACKAGE_VERSION" =~ ^[0-9]+\.[0-9]+\.0$ ]]; then - echo "Skip api docs generating for the patch release" - exit 0 - fi - ./evergreen/build-apidocs.sh - - upload-apidocs: - - command: shell.exec - params: - shell: bash - working_dir: mongo-csharp-driver - env: - GITHUB_USER: ${github_user} - GITHUB_APIKEY: ${github_apikey} - script: | - ${PREPARE_SHELL} - if ! [[ "$PACKAGE_VERSION" =~ ^[0-9]+\.[0-9]+\.0$ ]]; then - echo "Skip api docs generating for the patch release" - exit 0 - fi - ./evergreen/upload-apidocs.sh - -pre: - - func: fetch-source - - func: prepare-resources - - func: windows-fix - - func: fix-absolute-paths - - func: init-test-results - - func: make-files-executable - -post: - # Removed, causing timeouts - # - func: upload-working-dir - - func: delete-serverless-instance-if-configured - - func: upload-mo-artifacts - - func: upload-test-results - - func: cleanup - -tasks: - - name: test-net472 - commands: - - func: bootstrap-mongo-orchestration - - func: run-tests - vars: - FRAMEWORK: net472 - - - name: test-netstandard20 - commands: - - func: bootstrap-mongo-orchestration - - func: run-tests - vars: - FRAMEWORK: netstandard20 - - - name: test-netstandard21 - commands: - - func: bootstrap-mongo-orchestration - - func: run-tests - vars: - FRAMEWORK: netstandard21 - - - name: test-csfle-with-mongocryptd-net472 - commands: - - func: bootstrap-mongo-orchestration - - func: run-csfle-with-mongocryptd-tests - vars: - FRAMEWORK: net472 - - - name: test-csfle-with-mongocryptd-netstandard20 - commands: - - func: bootstrap-mongo-orchestration - - func: run-csfle-with-mongocryptd-tests - vars: - FRAMEWORK: netstandard20 - - - name: test-csfle-with-mongocryptd-netstandard21 - commands: - - func: bootstrap-mongo-orchestration - - func: run-csfle-with-mongocryptd-tests - vars: - FRAMEWORK: netstandard21 - - - name: test-csfle-with-mocked-kms-tls-net472 - commands: - - func: start-kms-mock-servers - - func: start-kms-mock-kmip-server - - func: start-kms-mock-gcp-server - - func: start-kms-mock-azure-imds-server - - func: bootstrap-mongo-orchestration - - func: run-csfle-with-mocked-kms-tests - vars: - FRAMEWORK: net472 - - - name: test-csfle-with-mocked-kms-tls-netstandard20 - commands: - - func: start-kms-mock-servers - - func: start-kms-mock-kmip-server - - func: start-kms-mock-gcp-server - - func: start-kms-mock-azure-imds-server - - func: bootstrap-mongo-orchestration - - func: run-csfle-with-mocked-kms-tests - vars: - FRAMEWORK: netstandard20 - - - name: test-csfle-with-mocked-kms-tls-netstandard21 - commands: - - func: start-kms-mock-servers - - func: start-kms-mock-kmip-server - - func: start-kms-mock-gcp-server - - func: start-kms-mock-azure-imds-server - - func: bootstrap-mongo-orchestration - - func: run-csfle-with-mocked-kms-tests - vars: - FRAMEWORK: netstandard21 - - - name: test-load-balancer-netstandard20 - commands: - - func: bootstrap-mongo-orchestration - vars: - LOAD_BALANCER: 'true' - - func: run-load-balancer - - func: run-load-balancer-tests - vars: - FRAMEWORK: netstandard20 - - func: stop-load-balancer - - - name: test-load-balancer-netstandard21 - commands: - - func: bootstrap-mongo-orchestration - vars: - LOAD_BALANCER: 'true' - - func: run-load-balancer - - func: run-load-balancer-tests - vars: - FRAMEWORK: netstandard21 - - func: stop-load-balancer - - - name: atlas-connectivity-tests - commands: - - func: run-atlas-connectivity-tests - - - name: test-gssapi - commands: - - func: run-gssapi-auth-tests - - - name: test-gssapi-net472 - commands: - - func: run-gssapi-auth-tests - vars: - FRAMEWORK: net472 - - - name: test-gssapi-netstandard20 - commands: - - func: run-gssapi-auth-tests - vars: - FRAMEWORK: netstandard20 - - - name: test-gssapi-netstandard21 - commands: - - func: run-gssapi-auth-tests - vars: - FRAMEWORK: netstandard21 - - - name: plain-auth-tests - commands: - - func: run-plain-auth-tests - - - name: aws-auth-tests - commands: - - func: bootstrap-mongo-orchestration - vars: - AUTH: "auth" - ORCHESTRATION_FILE: "auth-aws.json" - TOPOLOGY: "server" - - func: add-aws-auth-variables-to-file - # This step also creates test related users, so don't avoid this step in order to run other tests - - func: run-aws-auth-test-with-regular-aws-credentials - - func: run-aws-auth-test-with-assume-role-credentials - - func: run-aws-auth-test-with-aws-credentials-as-environment-variables - # This step requires running run-aws-auth-test-with-assume-role-credentials before to explicitly set up instance profile - - func: run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables - - func: run-aws-auth-test-with-aws-EC2-credentials - - func: run-aws-auth-test-with-aws-ECS-credentials - - func: run-aws-auth-test-with-aws-web-identity-credentials - - - name: stable-api-tests-net472 - commands: - - func: bootstrap-mongo-orchestration - vars: - REQUIRE_API_VERSION: true - - func: run-tests - vars: - FRAMEWORK: net472 - REQUIRE_API_VERSION: true - - - name: stable-api-tests-netstandard20 - commands: - - func: bootstrap-mongo-orchestration - vars: - REQUIRE_API_VERSION: true - - func: run-tests - vars: - FRAMEWORK: netstandard20 - REQUIRE_API_VERSION: true - - - name: stable-api-tests-netstandard21 - commands: - - func: bootstrap-mongo-orchestration - vars: - REQUIRE_API_VERSION: true - - func: run-tests - vars: - FRAMEWORK: netstandard21 - REQUIRE_API_VERSION: true - - - name: atlas-data-lake-test - commands: - - func: bootstrap-mongohoused - - func: run-atlas-data-lake-test - - - name: atlas-search-test - commands: - - func: run-atlas-search-test - - - name: atlas-search-index-helpers-test - commands: - - func: run-atlas-search-index-helpers-test - - - name: test-oidc-auth-aws - commands: - - func: assume-ec2-role - - func: run-mongodb-oidc-tests - - - name: test-serverless - exec_timeout_secs: 2700 # 45 minutes: 15 for setup + 30 for tests - commands: - - func: create-serverless-instance - - func: run-serverless-tests - - - name: test-ocsp-rsa-valid-cert-server-staples-ca-responder - tags: ["ocsp"] - commands: - - func: run-valid-ocsp-server-ca-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "true" - - - name: test-ocsp-rsa-invalid-cert-server-staples-ca-responder - tags: ["ocsp"] - commands: - - func: run-revoked-ocsp-server-ca-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-ocsp-rsa-valid-cert-server-does-not-staple-ca-responder - tags: ["ocsp"] - commands: - - func: run-valid-ocsp-server-ca-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "true" - - - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-ca-responder - tags: ["ocsp"] - commands: - - func: run-revoked-ocsp-server-ca-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-ocsp-rsa-soft-fail - tags: ["ocsp"] - commands: - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" # Spec mandates true, but .NET on Windows hard fails in this case - - - name: test-ocsp-rsa-malicious-invalid-cert-mustStaple-server-does-not-staple-ca-responder - tags: ["ocsp"] - commands: - - func: run-revoked-ocsp-server-ca-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-ocsp-rsa-malicious-no-responder-mustStaple-server-does-not-staple - tags: ["ocsp"] - commands: - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-ocsp-rsa-valid-cert-server-staples-delegate-responder - tags: ["ocsp"] - commands: - - func: run-valid-ocsp-server-delegate-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "true" - - - name: test-ocsp-rsa-invalid-cert-server-staples-delegate-responder - tags: ["ocsp"] - commands: - - func: run-revoked-ocsp-server-delegate-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-ocsp-rsa-valid-cert-server-does-not-staple-delegate-responder - tags: ["ocsp"] - commands: - - func: run-valid-ocsp-server-delegate-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "true" - - - name: test-ocsp-rsa-invalid-cert-server-does-not-staple-delegate-responder - tags: ["ocsp"] - commands: - - func: run-revoked-ocsp-server-delegate-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-ocsp-rsa-malicious-invalid-cert-mustStaple-server-does-not-staple-delegate-responder - tags: ["ocsp"] - commands: - - func: run-revoked-ocsp-server-delegate-responder - vars: - OCSP_ALGORITHM: "rsa" - - func: ocsp-bootstrap-mongo-orchestration - vars: - ORCHESTRATION_FILE: "rsa-basic-tls-ocsp-mustStaple-disableStapling.json" - - func: run-ocsp-test - vars: - OCSP_ALGORITHM: "rsa" - OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-smoke-tests-net472 - commands: - - func: bootstrap-mongo-orchestration - - func: run-smoke-tests - vars: - FRAMEWORK: net472 - - - name: test-smoke-tests-netcoreapp21 - commands: - - func: bootstrap-mongo-orchestration - - func: run-smoke-tests - vars: - FRAMEWORK: netcoreapp21 - - - name: test-smoke-tests-netcoreapp31 - commands: - - func: bootstrap-mongo-orchestration - - func: run-smoke-tests - vars: - FRAMEWORK: netcoreapp31 - - - name: test-smoke-tests-net50 - commands: - - func: bootstrap-mongo-orchestration - - func: run-smoke-tests - vars: - FRAMEWORK: net50 - - - name: test-smoke-tests-net60 - commands: - - func: bootstrap-mongo-orchestration - - func: run-smoke-tests - vars: - FRAMEWORK: net60 - - - name: test-smoke-tests-net80 - commands: - - func: bootstrap-mongo-orchestration - - func: run-smoke-tests - vars: - FRAMEWORK: net80 - - - name: performance-tests-net60-server-v6.0 - commands: - - func: bootstrap-mongo-orchestration - vars: - VERSION: "v6.0-perf" - TOPOLOGY: "server" - SSL: "nossl" - AUTH: "noauth" - SKIP_LEGACY_SHELL: "true" - - func: install-dotnet - - func: run-performance-tests - - - name: test-aws-lambda-deployed - commands: - - command: ec2.assume_role - params: - role_arn: ${LAMBDA_AWS_ROLE_ARN} - duration_seconds: 3600 - - command: shell.exec - params: - working_dir: mongo-csharp-driver - add_expansions_to_env: true - script: | - ${PREPARE_SHELL} - evergreen/run-deployed-lambda-aws-tests.sh - env: - TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/tests/FaasTests/LambdaTests - AWS_REGION: us-east-1 - - # ECDSA tests - # Disabled until https://jira.mongodb.org/browse/SPEC-1589 is resolved - # - name: test-ocsp-ecdsa-valid-cert-server-staples-ca-responder - # tags: ["ocsp"] - # commands: - # - func: run-valid-ocsp-server-ca-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "true" - - # - name: test-ocsp-ecdsa-invalid-cert-server-staples-ca-responder - # tags: ["ocsp"] - # commands: - # - func: run-revoked-ocsp-server-ca-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - # - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-ca-responder - # tags: ["ocsp"] - # commands: - # - func: run-valid-ocsp-server-ca-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "true" - - # - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-ca-responder - # tags: ["ocsp"] - # commands: - # - func: run-revoked-ocsp-server-ca-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - # - name: test-ocsp-ecdsa-soft-fail - # tags: ["ocsp"] - # commands: - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" # Spec mandates true but .NET on Windows hard fails in this case - - # - name: test-ocsp-ecdsa-malicious-invalid-cert-mustStaple-server-does-not-staple-ca-responder - # tags: ["ocsp"] - # commands: - # - func: run-revoked-ocsp-server-ca-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - # - name: test-ocsp-ecdsa-malicious-no-responder-mustStaple-server-does-not-staple - # tags: ["ocsp"] - # commands: - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - # - name: test-ocsp-ecdsa-valid-cert-server-staples-delegate-responder - # tags: ["ocsp"] - # commands: - # - func: run-valid-ocsp-server-delegate-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "true" - - # - name: test-ocsp-ecdsa-invalid-cert-server-staples-delegate-responder - # tags: ["ocsp"] - # commands: - # - func: run-revoked-ocsp-server-delegate-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - # - name: test-ocsp-ecdsa-valid-cert-server-does-not-staple-delegate-responder - # tags: ["ocsp"] - # commands: - # - func: run-valid-ocsp-server-delegate-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "true" - - # - name: test-ocsp-ecdsa-invalid-cert-server-does-not-staple-delegate-responder - # tags: ["ocsp"] - # commands: - # - func: run-revoked-ocsp-server-delegate-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - # - name: test-ocsp-ecdsa-malicious-invalid-cert-mustStaple-server-does-not-staple-delegate-responder - # tags: ["ocsp"] - # commands: - # - func: run-revoked-ocsp-server-delegate-responder - # vars: - # OCSP_ALGORITHM: "ecdsa" - # - func: ocsp-bootstrap-mongo-orchestration - # vars: - # ORCHESTRATION_FILE: "ecdsa-basic-tls-ocsp-mustStaple-disableStapling.json" - # - func: run-ocsp-test - # vars: - # OCSP_ALGORITHM: "ecdsa" - # OCSP_TLS_SHOULD_SUCCEED: "false" - - - name: test-csfle-with-azure-kms - commands: - - command: shell.exec - type: setup - params: - working_dir: mongo-csharp-driver - shell: "bash" - script: | - ${PREPARE_SHELL} - echo "Copying files ... begin" - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar czf /tmp/mongo-csharp-driver.tgz . - AZUREKMS_SRC=/tmp/mongo-csharp-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh - echo "Copying files ... end" - echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf mongo-csharp-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - echo "Untarring file ... end" - - - command: shell.exec - type: test - params: - working_dir: "mongo-csharp-driver" - shell: "bash" - script: | - ${PREPARE_SHELL} - export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' ./evergreen/run-csfle-azure-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - - - name: test-csfle-with-gcp-kms - commands: - - command: shell.exec - type: setup - params: - working_dir: mongo-csharp-driver - shell: "bash" - script: | - ${PREPARE_SHELL} - echo "Copying files ... begin" - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - tar czf /tmp/mongo-csharp-driver.tgz . - GCPKMS_SRC=/tmp/mongo-csharp-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh - echo "Copying files ... end" - echo "Untarring file ... begin" - GCPKMS_CMD="tar xf mongo-csharp-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - echo "Untarring file ... end" - - - command: shell.exec - type: test - params: - working_dir: "mongo-csharp-driver" - shell: "bash" - script: | - ${PREPARE_SHELL} - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' ./evergreen/run-csfle-gcp-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - - - name: build-packages - commands: - - func: install-dotnet - - func: build-packages - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Bson" - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Driver" - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Driver.Core" - - func: upload-package - vars: - PACKAGE_ID: "MongoDB.Driver.GridFS" - - func: upload-package - vars: - PACKAGE_ID: "mongocsharpdriver" - - - name: push-packages - commands: - - func: install-dotnet - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Bson" - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Driver" - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Driver.Core" - - func: download-package - vars: - PACKAGE_ID: "MongoDB.Driver.GridFS" - - func: download-package - vars: - PACKAGE_ID: "mongocsharpdriver" - - func: push-packages - vars: - PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json" - PACKAGES_SOURCE_KEY: ${nuget_api_key} - - - name: generate-apidocs - commands: - - func: install-dotnet - - func: build-apidocs - - func: upload-apidocs - -axes: - - id: version - display_name: MongoDB Version - values: - - id: "latest" - display_name: "latest" - variables: - VERSION: "latest" - - id: "rapid" - display_name: "rapid" - variables: - VERSION: "rapid" - - id: "7.0" - display_name: "7.0" - variables: - VERSION: "7.0" - - id: "6.0" - display_name: "6.0" - variables: - VERSION: "6.0" - - id: "5.0" - display_name: "5.0" - variables: - VERSION: "5.0" - - id: "4.4" - display_name: "4.4" - variables: - VERSION: "4.4" - - id: "4.2" - display_name: "4.2" - variables: - VERSION: "4.2" - - id: "4.0" - display_name: "4.0" - variables: - VERSION: "4.0" - - id: "3.6" - display_name: "3.6" - variables: - VERSION: "3.6" - - - id: os - display_name: OS - values: - - id: "windows-64" - display_name: "Windows 64-bit" - variables: - OS: "windows-64" - python3_binary: "C:/python/Python38/python.exe" - skip_ECS_auth_test: true - skip_web_identity_auth_test: true - run_on: windows-64-vs2017-test - - id: "ubuntu-1804" - display_name: "Ubuntu 18.04" - variables: - OS: "ubuntu-1804" - python3_binary: "/opt/python/3.8/bin/python3" - run_on: ubuntu1804-test - - id: "ubuntu-2004" - display_name: "Ubuntu 20.04" - variables: - OS: "ubuntu-2004" - python3_binary: "/opt/python/3.8/bin/python3" - run_on: ubuntu2004-small - - id: "macos-1100" - display_name: "macOS 11.00" - variables: - OS: "macos-1100" - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 - skip_EC2_auth_test: true - skip_ECS_auth_test: true - skip_web_identity_auth_test: true - run_on: macos-1100 - - id: "macos-1100-arm64" - display_name: "macOS 11.00 M1" - variables: - OS: "macos-1100-arm64" - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 - skip_EC2_auth_test: true - skip_ECS_auth_test: true - skip_web_identity_auth_test: true - run_on: macos-1100-arm64 - - - id: topology - display_name: Topology - values: - - id: "standalone" - display_name: Standalone - variables: - TOPOLOGY: "server" - - id: "replicaset" - display_name: Replica Set - variables: - TOPOLOGY: "replica_set" - - id: "sharded-cluster" - display_name: Sharded Cluster - variables: - TOPOLOGY: "sharded_cluster" - - - id: auth - display_name: Authentication - values: - - id: "auth" - display_name: Auth - variables: - AUTH: "auth" - - id: "noauth" - display_name: NoAuth - variables: - AUTH: "noauth" - - - id: ssl - display_name: SSL - values: - - id: "ssl" - display_name: SSL - variables: - SSL: "ssl" - - id: "nossl" - display_name: NoSSL - variables: - SSL: "nossl" - - - id: compressor - display_name: Compressor - values: - - id: "zlib" - display_name: Zlib - variables: - COMPRESSOR: "zlib" - - id: "snappy" - display_name: Snappy - variables: - COMPRESSOR: "snappy" - - id: "zstandard" - display_name: Zstandard - variables: - COMPRESSOR: "zstd" - - - id: target_framework - display_name: Target .net framework - values: - - id: "net472" - display_name: net472 - variables: - FRAMEWORK: net472 - - id: "netstandard20" - display_name: netstandard20 - variables: - FRAMEWORK: netstandard20 - - id: "netstandard21" - display_name: netstandard21 - variables: - FRAMEWORK: netstandard21 - - - id: serverless_proxy_type - display_name: Serverless Proxy Type - values: - - id: "Passthrough" - display_name: "Serverless Passthrough Proxy" - variables: - SERVERLESS_PROXY_TYPE: Passthrough - - id: "Terminating" - display_name: "Serverless Terminating Proxy" - variables: - SERVERLESS_PROXY_TYPE: Terminating - - - id: build-target - display_name: CI build target - values: - - id: "tests" - display_name: "tests" - variables: - BUILD_TARGET: "tests" - - id: "release" - display_name: "release" - variables: - BUILD_TARGET: "release" - -task_groups: - - name: testazurekms-task-group - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 # 30 minutes - setup_group: - - func: fetch-source - - func: prepare-resources - - func: windows-fix - - func: fix-absolute-paths - - func: init-test-results - - func: make-files-executable - - command: shell.exec - params: - shell: "bash" - silent: true - env: - AZUREKMS_CLIENTID : ${testazurekms_clientid} - AZUREKMS_TENANTID : ${testazurekms_tenantid} - AZUREKMS_SECRET= : ${testazurekms_secret} - AZUREKMS_RESOURCEGROUP: ${testazurekms_resourcegroup} - AZUREKMS_SCOPE : ${testazurekms_scope} - script: | - ${PREPARE_SHELL} - echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey - echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey - # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open". - chmod 600 /tmp/testazurekms_privatekey - - export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS - export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - export AZUREKMS_VMNAME_PREFIX=CSHARPDRIVER - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - - command: expansions.update - params: - file: testazurekms-expansions.yml - teardown_group: - - func: upload-test-results - # Load expansions again. The setup task may have failed before running `expansions.update`. - - command: expansions.update - params: - file: testazurekms-expansions.yml - - command: shell.exec - params: - shell: "bash" - env: - AZUREKMS_VMNAME : ${AZUREKMS_VMNAME} - AZUREKMS_RESOURCEGROUP : ${testazurekms_resourcegroup} - script: | - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh - tasks: - - test-csfle-with-azure-kms - - - name: testgcpkms-task-group - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 # 30 minutes - setup_group: - - func: fetch-source - - func: prepare-resources - - func: windows-fix - - func: fix-absolute-paths - - func: init-test-results - - func: make-files-executable - - command: shell.exec - params: - shell: "bash" - silent: true - include_expansions_in_env: - - "GCPKMS_SERVICEACCOUNT" - script: | - ${PREPARE_SHELL} - echo '${GOOGLE_APPLICATION_CREDENTIALS_CONTENT}' > /tmp/testgcpkms_key_file.json - export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json - export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS - export GCPKMS_MACHINETYPE="e2-standard-4" - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh - # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. - - command: expansions.update - params: - file: testgcpkms-expansions.yml - teardown_group: - - func: upload-test-results - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh - tasks: - - test-csfle-with-gcp-kms - - - name: atlas-search-index-helpers-task-group - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 # 30 minutes - setup_group: - - func: fetch-source - - func: prepare-resources - - func: fix-absolute-paths - - func: init-test-results - - func: make-files-executable - - command: shell.exec - params: - env: - DRIVERS_ATLAS_PUBLIC_API_KEY: "${DRIVERS_ATLAS_PUBLIC_API_KEY}" - DRIVERS_ATLAS_PRIVATE_API_KEY: "${DRIVERS_ATLAS_PRIVATE_API_KEY}" - DRIVERS_ATLAS_GROUP_ID: "${DRIVERS_ATLAS_GROUP_ID}" - DRIVERS_ATLAS_LAMBDA_USER: "${DRIVERS_ATLAS_LAMBDA_USER}" - DRIVERS_ATLAS_LAMBDA_PASSWORD: "${DRIVERS_ATLAS_LAMBDA_PASSWORD}" - LAMBDA_STACK_NAME: "${LAMBDA_STACK_NAME}" - add_expansions_to_env: true - shell: "bash" - script: | - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: atlas-expansion.yml - teardown_group: - - func: upload-test-results - - command: shell.exec - params: - env: - DRIVERS_ATLAS_PUBLIC_API_KEY: "${DRIVERS_ATLAS_PUBLIC_API_KEY}" - DRIVERS_ATLAS_PRIVATE_API_KEY: "${DRIVERS_ATLAS_PRIVATE_API_KEY}" - DRIVERS_ATLAS_GROUP_ID: "${DRIVERS_ATLAS_GROUP_ID}" - DRIVERS_ATLAS_LAMBDA_USER: "${DRIVERS_ATLAS_LAMBDA_USER}" - DRIVERS_ATLAS_LAMBDA_PASSWORD: "${DRIVERS_ATLAS_LAMBDA_PASSWORD}" - LAMBDA_STACK_NAME: "${LAMBDA_STACK_NAME}" - add_expansions_to_env: true - shell: "bash" - script: | - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/atlas/teardown-atlas-cluster.sh - tasks: - - atlas-search-index-helpers-test - - - name: test-aws-lambda-task-group - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 # 30 minutes - setup_group: - - func: fetch-source - - func: prepare-resources - - func: install-dotnet - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: atlas-expansion.yml - teardown_group: - - command: subprocess.exec - params: - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh - tasks: - - test-aws-lambda-deployed - -buildvariants: -- matrix_name: stable-api-tests - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "standalone", auth: "auth", ssl: "nossl", os: "windows-64" } - display_name: "Stable API ${version} ${topology} ${auth} ${ssl} ${os}" - run_on: - - windows-64-vs2017-test - tasks: - - name: stable-api-tests-net472 - - name: stable-api-tests-netstandard20 - - name: stable-api-tests-netstandard21 - -# Secure tests -- matrix_name: "secure-tests-windows" - matrix_spec: { version: "*", topology: "*", auth: "auth", ssl: "ssl", os: "windows-64" } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-net472 - - name: test-netstandard20 - - name: test-netstandard21 - -- matrix_name: "secure-tests-macOS" - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "replicaset", auth: "auth", ssl: "ssl", os: ["macos-1100", "macos-1100-arm64"] } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-netstandard21 - -- matrix_name: "secure-tests-linux-1804" - matrix_spec: { version: ["4.0", "4.2", "4.4", "5.0", "6.0"], topology: "*", auth: "auth", ssl: "ssl", os: "ubuntu-1804" } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-netstandard20 - - name: test-netstandard21 - -- matrix_name: "secure-tests-linux-2004" - matrix_spec: { version: ["7.0", "rapid", "latest"], topology: "*", auth: "auth", ssl: "ssl", os: "ubuntu-2004" } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-netstandard20 - - name: test-netstandard21 - -# Unsecure tests -- matrix_name: "unsecure-tests-windows" - matrix_spec: { version: "*", topology: "*", auth: "noauth", ssl: "nossl", os: "windows-64" } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-net472 - - name: test-netstandard20 - - name: test-netstandard21 - -- matrix_name: "unsecure-tests-macOS" - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "replicaset", auth: "noauth", ssl: "nossl", os: ["macos-1100", "macos-1100-arm64"] } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-netstandard21 - -- matrix_name: "unsecure-tests-linux-1804" - matrix_spec: { version: ["3.6", "4.0", "4.2", "4.4", "5.0", "6.0"], topology: "*", auth: "noauth", ssl: "nossl", os: "ubuntu-1804" } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-netstandard20 - - name: test-netstandard21 - -- matrix_name: "unsecure-tests-linux-2004" - matrix_spec: { version: ["7.0", "rapid", "latest"], topology: "*", auth: "noauth", ssl: "nossl", os: "ubuntu-2004" } - display_name: "${version} ${topology} ${auth} ${ssl} ${os}" - tags: ["tests-variant"] - tasks: - - name: test-netstandard20 - - name: test-netstandard21 - -# Compression tests -- matrix_name: "tests-compression-windows" - matrix_spec: { compressor: "*", auth: "noauth", ssl: "nossl", version: "*", topology: "standalone", os: "windows-64" } - display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} " - tags: ["tests-variant"] - tasks: - - name: test-net472 - - name: test-netstandard20 - - name: test-netstandard21 - -- matrix_name: "tests-compression-macOS" - matrix_spec: { compressor : ["snappy", "zstandard"], auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "standalone", os: ["macos-1100", "macos-1100-arm64"] } - display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} " - tags: ["tests-variant"] - tasks: - - name: test-netstandard21 - -- matrix_name: "tests-compression-linux-1804" - matrix_spec: { compressor: "*", auth: "noauth", ssl: "nossl", version: ["3.6", "4.0", "4.2", "4.4", "5.0", "6.0"], topology: "standalone", os: "ubuntu-1804" } - display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} " - tags: ["tests-variant"] - tasks: - - name: test-netstandard20 - - name: test-netstandard21 - -- matrix_name: "tests-compression-linux-2004" - matrix_spec: { compressor: "*", auth: "noauth", ssl: "nossl", version: ["7.0", "rapid", "latest"], topology: "standalone", os: "ubuntu-2004" } - display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} " - tags: ["tests-variant"] - tasks: - - name: test-netstandard20 - - name: test-netstandard21 - -# Auth tests -- matrix_name: plain-auth-tests - matrix_spec: { os: "*" } - display_name: "PLAIN (LDAP) Auth Tests ${os}" - tasks: - - name: plain-auth-tests - -- matrix_name: mongodb-oidc-tests - matrix_spec: { os: [ "windows-64", "ubuntu-2004", "macos-1100" ] } - display_name: "MongoDB-OIDC Auth tests - ${os}" - tasks: - - name: test-oidc-auth-aws - -- matrix_name: "ocsp-tests" - matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "rapid", "latest"], auth: "noauth", ssl: "ssl", topology: "standalone", os: "windows-64" } - display_name: "OCSP ${version} ${os}" - batchtime: 20160 # 14 days - tasks: - - name: ".ocsp" - -- matrix_name: aws-auth-tests-windows - matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "rapid", "latest"], topology: "standalone", os: "windows-64" } - display_name: "MONGODB-AWS Auth test ${version} ${os}" - run_on: - - windows-64-vs2017-test - tasks: - - name: aws-auth-tests - -- matrix_name: aws-auth-tests-linux - matrix_spec: { version: ["6.0", "7.0", "rapid", "latest"], topology: "standalone", os: "ubuntu-2004" } - display_name: "MONGODB-AWS Auth test ${version} ${os}" - tasks: - - name: aws-auth-tests - -- matrix_name: aws-auth-tests-macos - matrix_spec: { version: ["6.0", "7.0", "rapid", "latest"], topology: "standalone", os: "macos-1100" } - display_name: "MONGODB-AWS Auth test ${version} ${os}" - run_on: - - macos-1100 - tasks: - - name: aws-auth-tests - -- name: gssapi-auth-tests-windows - run_on: - - windows-64-vs2017-test - display_name: "GSSAPI (Kerberos) Auth tests - Windows" - tasks: - - name: test-gssapi-net472 - - name: test-gssapi-netstandard20 - - name: test-gssapi-netstandard21 - -- name: gssapi-auth-tests-linux - run_on: - - ubuntu1804-test - display_name: "GSSAPI (Kerberos) Auth tests - Linux" - tasks: - - name: test-gssapi-netstandard20 - - name: test-gssapi-netstandard21 - -# Load balancer tests -- matrix_name: load-balancer-tests - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], auth: "noauth", ssl: "nossl", topology: "sharded-cluster", os: "ubuntu-2004" } - display_name: "Load Balancer ${version} ${auth} ${ssl} ${os}" - tasks: - - name: "test-load-balancer-netstandard20" - - name: "test-load-balancer-netstandard21" - -- matrix_name: load-balancer-tests-secure - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], auth: "auth", ssl: "ssl", topology: "sharded-cluster", os: "ubuntu-2004" } - display_name: "Load Balancer ${version} ${auth} ${ssl} ${os}" - tasks: - - name: "test-load-balancer-netstandard20" - - name: "test-load-balancer-netstandard21" - -# Serverless tests -- matrix_name: serverless-tests-windows - matrix_spec: { auth: "auth", ssl: "ssl", compressor: "zlib", os: "windows-64", target_framework: "*", serverless_proxy_type: "*" } - display_name: "${serverless_proxy_type} ${compressor} ${auth} ${ssl} ${os} ${target_framework}" - tasks: - - name: test-serverless - -- matrix_name: serverless-tests-ubuntu - matrix_spec: { auth: "auth", ssl: "ssl", compressor: "zlib", os: "ubuntu-2004", target_framework: ["netstandard20", "netstandard21"], serverless_proxy_type: "*" } - display_name: "${serverless_proxy_type} ${compressor} ${auth} ${ssl} ${os} ${target_framework}" - tasks: - - name: test-serverless - -# Performance tests -- name: driver-performance-tests - display_name: "Driver Performance Tests" - run_on: - - rhel90-dbx-perf-large - tasks: - - name: performance-tests-net60-server-v6.0 - -# AWS Lambda tests -- name: aws-lambda-tests - display_name: "AWS Lambda Tests" - run_on: - - ubuntu2204-small - tasks: - - name: test-aws-lambda-task-group - -# Atlas tests -- name: atlas-connectivity-tests - display_name: "Atlas Connectivity Tests" - run_on: - - windows-64-vs2017-test - tasks: - - name: atlas-connectivity-tests - -- name: atlas-data-lake-test - display_name: "Atlas Data Lake Tests" - run_on: - - ubuntu2004-large - tasks: - - name: atlas-data-lake-test - -- name: atlas-search-test - display_name: "Atlas Search Tests" - run_on: - - windows-64-vs2017-test - tasks: - - name: atlas-search-test - -- name: atlas-search-index-helpers-test - display_name: "Atlas Search Index Helpers Tests" - run_on: - - ubuntu1804-test - tasks: - - name: atlas-search-index-helpers-task-group - -# CSFLE tests -- matrix_name: "csfle-with-mocked-kms-tests-windows" - matrix_spec: { os: "windows-64", ssl: "nossl", version: ["4.2", "4.4", "5.0", "6.0", "7.0", "rapid", "latest"], topology: ["replicaset"] } - display_name: "CSFLE Mocked KMS ${version} ${os}" - tasks: - - name: test-csfle-with-mocked-kms-tls-net472 - - name: test-csfle-with-mocked-kms-tls-netstandard20 - - name: test-csfle-with-mocked-kms-tls-netstandard21 - - name: test-csfle-with-mongocryptd-net472 - - name: test-csfle-with-mongocryptd-netstandard20 - - name: test-csfle-with-mongocryptd-netstandard21 - -- matrix_name: "csfle-with-mocked-kms-tests-linux-1804" - matrix_spec: { os: "ubuntu-1804", ssl: "nossl", version: ["4.2", "4.4", "5.0", "6.0"], topology: ["replicaset"] } - display_name: "CSFLE Mocked KMS ${version} ${os}" - tasks: - - name: test-csfle-with-mocked-kms-tls-netstandard20 - - name: test-csfle-with-mocked-kms-tls-netstandard21 - - name: test-csfle-with-mongocryptd-netstandard20 - - name: test-csfle-with-mongocryptd-netstandard21 - -- matrix_name: "csfle-with-mocked-kms-tests-linux-2004" - matrix_spec: { os: "ubuntu-2004", ssl: "nossl", version: ["7.0", "rapid", "latest"], topology: ["replicaset"] } - display_name: "CSFLE Mocked KMS ${version} ${os}" - tasks: - - name: test-csfle-with-mocked-kms-tls-netstandard20 - - name: test-csfle-with-mocked-kms-tls-netstandard21 - - name: test-csfle-with-mongocryptd-netstandard20 - - name: test-csfle-with-mongocryptd-netstandard21 - -- matrix_name: "csfle-with-mocked-kms-tests-macOS" - matrix_spec: { os: ["macos-1100", "macos-1100-arm64"], ssl: "nossl", version: ["4.2", "4.4", "5.0", "6.0", "7.0", "rapid", "latest"], topology: ["replicaset"] } - display_name: "CSFLE Mocked KMS ${version} ${os}" - tasks: - - name: test-csfle-with-mocked-kms-tls-netstandard21 - - name: test-csfle-with-mongocryptd-netstandard21 - -- matrix_name: "csfle-with-azure-kms-tests-linux" - matrix_spec: { ssl: "nossl", os: "ubuntu-1804" } - display_name: "CSFLE with AZURE KMS ${os}" - batchtime: 20160 # 14 days - tasks: - - name: testazurekms-task-group - - name: test-csfle-with-mongocryptd-netstandard21 - -- matrix_name: "csfle-with-gcp-kms-tests-linux" - matrix_spec: { ssl: "nossl", os: "ubuntu-1804" } - display_name: "CSFLE with GCP KMS ${os}" - batchtime: 20160 # 14 days - tasks: - - name: testgcpkms-task-group - - name: test-csfle-with-mongocryptd-netstandard21 - -# Smoke tests -- matrix_name: "smoke-tests-windows" - matrix_spec: { os: "windows-64", ssl: "nossl", version: ["5.0", "6.0", "7.0", "latest"], topology: ["replicaset"] } - display_name: "smoke-tests ${version} ${os}" - batchtime: 1440 # 1 day - tasks: - - name: test-smoke-tests-net472 - - name: test-smoke-tests-netcoreapp21 - - name: test-smoke-tests-netcoreapp31 - - name: test-smoke-tests-net50 - - name: test-smoke-tests-net60 - - name: test-smoke-tests-net80 - -- matrix_name: "smoke-tests-linux" - matrix_spec: { os: "ubuntu-2004", ssl: "nossl", version: ["5.0", "6.0", "7.0", "latest"], topology: ["replicaset"] } - display_name: "smoke-tests ${version} ${os}" - batchtime: 1440 # 1 day - tasks: - - name: test-smoke-tests-netcoreapp21 - - name: test-smoke-tests-netcoreapp31 - - name: test-smoke-tests-net50 - - name: test-smoke-tests-net60 - - name: test-smoke-tests-net80 - -- matrix_name: "smoke-tests-macOS" - matrix_spec: { os: "macos-1100", ssl: "nossl", version: ["5.0", "6.0", "7.0", "latest"], topology: ["replicaset"] } - display_name: "smoke-tests ${version} ${os}" - batchtime: 1440 # 1 day - tasks: - - name: test-smoke-tests-netcoreapp21 - - name: test-smoke-tests-netcoreapp31 - - name: test-smoke-tests-net50 - - name: test-smoke-tests-net60 - -# Package release variants -- matrix_name: build-packages - matrix_spec: - build-target: "release" - os: "windows-64" # should produce package on Windows to make sure full framework binaries created. - display_name: "Package Pack" - tags: ["build-packages", "release-tag"] - tasks: - - name: build-packages - git_tag_only: true - priority: 10 - -- matrix_name: generate-apidocs - matrix_spec: - build-target: "release" - os: "ubuntu-2004" - display_name: "Generate API Documentation" - tags: ["build-apidocs", "release-tag"] - tasks: - - name: generate-apidocs - git_tag_only: true - priority: 10 - depends_on: - - name: build-packages - variant: ".build-packages" - ## add dependency onto packages smoke test once it implemented - -- matrix_name: push-packages - matrix_spec: - build-target: "release" - os: "ubuntu-2004" - display_name: "Package Push" - tags: ["push-packages", "release-tag"] - tasks: - - name: push-packages - git_tag_only: true - priority: 10 - depends_on: - - name: build-packages - variant: ".build-packages" - ## add dependency onto packages smoke test once it implemented diff --git a/evergreen/get-version.sh b/evergreen/get-version.sh index aa91aae52a4..579e9c5396f 100644 --- a/evergreen/get-version.sh +++ b/evergreen/get-version.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -o errexit # Exit the script with error if any of the commands fail -PACKAGE_VERSION=$(git describe --tags --abbrev=0 --match="v[0-9]*.[0-9]*.[0-9]*") +PACKAGE_VERSION=$(git describe --tags) PACKAGE_VERSION=$(echo $PACKAGE_VERSION | cut -c 2-) echo "$PACKAGE_VERSION" diff --git a/evergreen/push-packages.sh b/evergreen/push-packages.sh index 6f081498376..1cafa221452 100644 --- a/evergreen/push-packages.sh +++ b/evergreen/push-packages.sh @@ -1,6 +1,30 @@ #!/usr/bin/env bash -set -o errexit # Exit the script with error if any of the commands fail -set +o xtrace # Disable tracing. +set -o errexit # Exit the script with error if any of the commands fail +set +o xtrace # Disable tracing. + +wait_until_package_is_available () +{ + package=$1 + version=$2 + query_url="${PACKAGES_SOURCE%index.json}query" + resp="" + count=0 + echo "Checking package availability: ${package}:${version} at ${query_url}" + while [ -z "$resp" ] && [ $count -le 40 ]; do + resp=$(curl -X GET -s "$query_url?prerelease=true&take=1&q=PackageId:$package" | jq --arg jq_version "$version" '.data[0].versions[] | select(.version==$jq_version) | .version') + if [ -z "$resp" ]; then + echo "sleeping for 15 seconds..." + sleep 15 + fi + done + + if [ -z "$resp" ]; then + echo "Timeout while waiting for package availability: ${package}" + exit 1 + else + echo "Package ${package} is available, version: ${resp}" + fi +} if [ -z "$PACKAGES_SOURCE" ]; then echo "PACKAGES_SOURCE variable should be set" @@ -17,10 +41,17 @@ if [ -z "$PACKAGE_VERSION" ]; then exit 1 fi -dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/MongoDB.Bson."$PACKAGE_VERSION".nupkg -dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/MongoDB.Driver.Core."$PACKAGE_VERSION".nupkg -dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/MongoDB.Driver."$PACKAGE_VERSION".nupkg -dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/MongoDB.Driver.GridFS."$PACKAGE_VERSION".nupkg -dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/mongocsharpdriver."$PACKAGE_VERSION".nupkg +clear_version_rx='^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$' +if [ "$PACKAGES_SOURCE" = "https://api.nuget.org/v3/index.json" ] && [[ ! "$PACKAGE_VERSION" =~ $clear_version_rx ]]; then + echo "Cannot push dev version to nuget.org: '$PACKAGE_VERSION'" + exit 1 +fi +PACKAGES=("MongoDB.Bson" "MongoDB.Driver.Core" "MongoDB.Driver" "MongoDB.Driver.GridFS" "mongocsharpdriver") +for package in ${PACKAGES[*]}; do + dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/"$package"."$PACKAGE_VERSION".nupkg +done +for package in ${PACKAGES[*]}; do + wait_until_package_is_available "$package" "$PACKAGE_VERSION" +done diff --git a/tests/FaasTests/LambdaTests/MongoDB.Driver.LambdaTest/MongoDB.Driver.LambdaTest.csproj b/tests/FaasTests/LambdaTests/MongoDB.Driver.LambdaTest/MongoDB.Driver.LambdaTest.csproj index 028573cdd37..688b6b06276 100644 --- a/tests/FaasTests/LambdaTests/MongoDB.Driver.LambdaTest/MongoDB.Driver.LambdaTest.csproj +++ b/tests/FaasTests/LambdaTests/MongoDB.Driver.LambdaTest/MongoDB.Driver.LambdaTest.csproj @@ -4,6 +4,7 @@ net6.0 true Lambda + false From 85c2cf1f01a6e813809ca81d85c1b17ca346e462 Mon Sep 17 00:00:00 2001 From: BorisDog Date: Wed, 8 May 2024 13:04:50 -0700 Subject: [PATCH 12/38] CSHARP-5066: Onboard .NET Driver to Papertrail Service (#1322) --- evergreen/evergreen.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index a7ecaa4f326..b01c24a205c 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -854,6 +854,25 @@ functions: . ./activate-kmstlsvenv.sh python bottle.py fake_azure:imds + trace-artifacts: + - command: papertrail.trace + params: + key_id: ${papertrail_key_id} + secret_key: ${papertrail_secret_key} + product: ${PRODUCT_NAME} + version: ${PACKAGE_VERSION} + filenames: + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Bson.${PACKAGE_VERSION}.nupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Bson.${PACKAGE_VERSION}.snupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.${PACKAGE_VERSION}.nupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.${PACKAGE_VERSION}.snupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.Core.${PACKAGE_VERSION}.nupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.Core.${PACKAGE_VERSION}.snupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.GridFS.${PACKAGE_VERSION}.nupkg" + - "mongo-csharp-driver/artifacts/nuget/MongoDB.Driver.GridFS.${PACKAGE_VERSION}.snupkg" + - "mongo-csharp-driver/artifacts/nuget/mongocsharpdriver.${PACKAGE_VERSION}.nupkg" + - "mongo-csharp-driver/artifacts/nuget/mongocsharpdriver.${PACKAGE_VERSION}.snupkg" + cleanup: - command: shell.exec params: @@ -1776,6 +1795,9 @@ tasks: vars: PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json" PACKAGES_SOURCE_KEY: ${nuget_api_key} + - func: trace-artifacts + vars: + PRODUCT_NAME: "mongo-csharp-driver" - name: push-packages-myget commands: @@ -2641,4 +2663,4 @@ buildvariants: depends_on: - name: build-packages variant: ".build-packages" - ## add dependency onto packages smoke test once it implemented + ## add dependency onto packages smoke test once it implemented \ No newline at end of file From 9eb4dca262de9cc8facc9829b3893cdd1b1cd8d4 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Wed, 8 May 2024 16:48:15 -0400 Subject: [PATCH 13/38] CSHARP-5070: Add SBOM file (#1318) --- purls.txt | 1 + sbom.json | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 purls.txt create mode 100644 sbom.json diff --git a/purls.txt b/purls.txt new file mode 100644 index 00000000000..7c0089a1a10 --- /dev/null +++ b/purls.txt @@ -0,0 +1 @@ +pkg:nuget/MongoDB.Libmongocrypt@1.8.2 diff --git a/sbom.json b/sbom.json new file mode 100644 index 00000000000..d9812068318 --- /dev/null +++ b/sbom.json @@ -0,0 +1,82 @@ +{ + "components": [ + { + "bom-ref": "pkg:nuget/MongoDB.Libmongocrypt@1.8.2", + "externalReferences": [ + { + "type": "distribution", + "url": "https://www.nuget.org/api/v2/package/MongoDB.Libmongocrypt/1.8.2" + }, + { + "type": "website", + "url": "https://www.nuget.org/packages/MongoDB.Libmongocrypt/1.8.2" + } + ], + "licenses": [ + { + "license": { + "name": "Other" + } + } + ], + "name": "MongoDB.Libmongocrypt", + "purl": "pkg:nuget/MongoDB.Libmongocrypt@1.8.2", + "type": "library", + "version": "1.8.2" + } + ], + "dependencies": [ + { + "ref": "pkg:nuget/MongoDB.Libmongocrypt@1.8.2" + } + ], + "metadata": { + "timestamp": "2024-05-03T20:19:34.867974+00:00", + "tools": [ + { + "externalReferences": [ + { + "type": "build-system", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/actions" + }, + { + "type": "distribution", + "url": "https://pypi.org/project/cyclonedx-python-lib/" + }, + { + "type": "documentation", + "url": "https://cyclonedx-python-library.readthedocs.io/" + }, + { + "type": "issue-tracker", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/issues" + }, + { + "type": "license", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE" + }, + { + "type": "release-notes", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md" + }, + { + "type": "vcs", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib" + }, + { + "type": "website", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/#readme" + } + ], + "name": "cyclonedx-python-lib", + "vendor": "CycloneDX", + "version": "6.4.4" + } + ] + }, + "serialNumber": "urn:uuid:137c9f19-2c41-488a-bacb-b367694656ff", + "version": 1, + "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.5" +} From 4672e59963bba8449c7b48ebe444fb78f75b3fd3 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Thu, 16 May 2024 09:16:17 -0700 Subject: [PATCH 14/38] CSHARP-5087 Bump MacOS version used for tests (#1324) --- build.config | 2 +- build.sh | 4 +--- evergreen/evergreen.yml | 44 ++++++++++++++++++++--------------------- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/build.config b/build.config index 719f0942b7d..1c2f8500d19 100644 --- a/build.config +++ b/build.config @@ -1,3 +1,3 @@ #!/usr/bin/env bash CAKE_VERSION=2.2.0 -DOTNET_VERSION=6.0.400 +DOTNET_VERSION=8.0.204 diff --git a/build.sh b/build.sh index e78d14c937f..a5e51a17503 100755 --- a/build.sh +++ b/build.sh @@ -47,9 +47,7 @@ if [ "$DOTNET_VERSION" != "$DOTNET_INSTALLED_VERSION" ]; then bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --channel 2.1 --architecture x64 --install-dir .dotnet --no-path bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --channel 3.1 --architecture x64 --install-dir .dotnet --no-path bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --channel 5.0 --architecture x64 --install-dir .dotnet --no-path -if [[ ! "$OS" =~ macOS|macos ]]; then # net8 is not supported on macOS 11 - bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --channel 8.0 --architecture x64 --install-dir .dotnet --no-path -fi + bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --channel 6.0 --install-dir .dotnet --no-path bash "$SCRIPT_DIR/.dotnet/dotnet-install.sh" --version $DOTNET_VERSION --install-dir .dotnet --no-path export PATH="$SCRIPT_DIR/.dotnet":$PATH export DOTNET_ROOT="$SCRIPT_DIR/.dotnet" diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index b01c24a205c..808f0cfdb73 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -217,6 +217,12 @@ functions: params: script: | ${PREPARE_SHELL} + if [ "${OS}" = "macos-14-arm64" ]; then + # have to set the limit manually for macos-14-arm, can be removed once DEVPROD-7353 will be solved + ulimit -n 64000 + ulimit -a + fi + REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ LOAD_BALANCER=${LOAD_BALANCER} \ MONGODB_VERSION=${VERSION} \ @@ -1867,7 +1873,6 @@ axes: display_name: "Windows 64-bit" variables: OS: "windows-64" - python3_binary: "C:/python/Python38/python.exe" skip_ECS_auth_test: true skip_web_identity_auth_test: true run_on: windows-64-vs2017-test @@ -1875,32 +1880,28 @@ axes: display_name: "Ubuntu 18.04" variables: OS: "ubuntu-1804" - python3_binary: "/opt/python/3.8/bin/python3" run_on: ubuntu1804-test - id: "ubuntu-2004" display_name: "Ubuntu 20.04" variables: OS: "ubuntu-2004" - python3_binary: "/opt/python/3.8/bin/python3" run_on: ubuntu2004-small - - id: "macos-1100" - display_name: "macOS 11.00" + - id: "macos-14" + display_name: "macOS 14.00" variables: - OS: "macos-1100" - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 + OS: "macos-1400" skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - run_on: macos-1100 - - id: "macos-1100-arm64" - display_name: "macOS 11.00 M1" + run_on: macos-14 + - id: "macos-14-arm64" + display_name: "macOS 14.00 ARM" variables: - OS: "macos-1100-arm64" - python3_binary: /Library/Frameworks/Python.framework/Versions/3.8/bin/python3 + OS: "macos-14-arm64" skip_EC2_auth_test: true skip_ECS_auth_test: true skip_web_identity_auth_test: true - run_on: macos-1100-arm64 + run_on: macos-14-arm64 - id: topology display_name: Topology @@ -2283,7 +2284,7 @@ buildvariants: - name: test-netstandard21 - matrix_name: "secure-tests-macOS" - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "replicaset", auth: "auth", ssl: "ssl", os: ["macos-1100", "macos-1100-arm64"] } + matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "replicaset", auth: "auth", ssl: "ssl", os: ["macos-14", "macos-14-arm64"] } display_name: "${version} ${topology} ${auth} ${ssl} ${os}" tags: ["tests-variant"] tasks: @@ -2316,7 +2317,7 @@ buildvariants: - name: test-netstandard21 - matrix_name: "unsecure-tests-macOS" - matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "replicaset", auth: "noauth", ssl: "nossl", os: ["macos-1100", "macos-1100-arm64"] } + matrix_spec: { version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "replicaset", auth: "noauth", ssl: "nossl", os: ["macos-14", "macos-14-arm64"] } display_name: "${version} ${topology} ${auth} ${ssl} ${os}" tags: ["tests-variant"] tasks: @@ -2349,7 +2350,7 @@ buildvariants: - name: test-netstandard21 - matrix_name: "tests-compression-macOS" - matrix_spec: { compressor : ["snappy", "zstandard"], auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "standalone", os: ["macos-1100", "macos-1100-arm64"] } + matrix_spec: { compressor : ["snappy", "zstandard"], auth: "noauth", ssl: "nossl", version: ["5.0", "6.0", "7.0", "rapid", "latest"], topology: "standalone", os: ["macos-14", "macos-14-arm64"] } display_name: "${version} ${compressor} ${topology} ${auth} ${ssl} ${os} " tags: ["tests-variant"] tasks: @@ -2421,10 +2422,8 @@ buildvariants: - name: aws-auth-tests - matrix_name: aws-auth-tests-macos - matrix_spec: { version: ["6.0", "7.0", "rapid", "latest"], topology: "standalone", os: "macos-1100" } + matrix_spec: { version: ["6.0", "7.0", "rapid", "latest"], topology: "standalone", os: "macos-14" } display_name: "MONGODB-AWS Auth test ${version} ${os}" - run_on: - - macos-1100 tasks: - name: aws-auth-tests @@ -2551,7 +2550,7 @@ buildvariants: - name: test-csfle-with-mongocryptd-netstandard21 - matrix_name: "csfle-with-mocked-kms-tests-macOS" - matrix_spec: { os: ["macos-1100", "macos-1100-arm64"], ssl: "nossl", version: ["4.2", "4.4", "5.0", "6.0", "7.0", "rapid", "latest"], topology: ["replicaset"] } + matrix_spec: { os: ["macos-14", "macos-14-arm64"], ssl: "nossl", version: ["4.2", "4.4", "5.0", "6.0", "7.0", "rapid", "latest"], topology: ["replicaset"] } display_name: "CSFLE Mocked KMS ${version} ${os}" tasks: - name: test-csfle-with-mocked-kms-tls-netstandard21 @@ -2598,7 +2597,7 @@ buildvariants: - name: test-smoke-tests-net80 - matrix_name: "smoke-tests-macOS" - matrix_spec: { os: "macos-1100", ssl: "nossl", version: ["5.0", "6.0", "7.0", "latest"], topology: ["replicaset"] } + matrix_spec: { os: "macos-14", ssl: "nossl", version: ["5.0", "6.0", "7.0", "latest"], topology: ["replicaset"] } display_name: "smoke-tests ${version} ${os}" batchtime: 1440 # 1 day tasks: @@ -2606,6 +2605,7 @@ buildvariants: - name: test-smoke-tests-netcoreapp31 - name: test-smoke-tests-net50 - name: test-smoke-tests-net60 + - name: test-smoke-tests-net80 # Package release variants - matrix_name: build-packages @@ -2663,4 +2663,4 @@ buildvariants: depends_on: - name: build-packages variant: ".build-packages" - ## add dependency onto packages smoke test once it implemented \ No newline at end of file + ## add dependency onto packages smoke test once it implemented From 886da07cd6965d14917cd7f189b4f84196f48a4e Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Tue, 21 May 2024 15:32:36 -0700 Subject: [PATCH 15/38] CSHARP-3986: Fix flaky ConnectionPool_cleared_on_failed_hello test (#1326) --- .../ServerDiscoveryAndMonitoringProseTests.cs | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/tests/MongoDB.Driver.Tests/Specifications/server-discovery-and-monitoring/ServerDiscoveryAndMonitoringProseTests.cs b/tests/MongoDB.Driver.Tests/Specifications/server-discovery-and-monitoring/ServerDiscoveryAndMonitoringProseTests.cs index 62556a080ba..cb1dc9f8bb5 100644 --- a/tests/MongoDB.Driver.Tests/Specifications/server-discovery-and-monitoring/ServerDiscoveryAndMonitoringProseTests.cs +++ b/tests/MongoDB.Driver.Tests/Specifications/server-discovery-and-monitoring/ServerDiscoveryAndMonitoringProseTests.cs @@ -241,10 +241,7 @@ public void ConnectionPool_cleared_on_failed_hello() RequireServer.Check().VersionGreaterThanOrEqualTo(minVersion); const string appName = "SDAMPoolManagementTest"; - // Using a 100ms heartbeatInterval can result in sporadic failures of this test if the RTT thread - // consumes both of the configured failpoints before the monitoring thread can run. - // Increasing the heartbeatInterval to 200ms avoids this race condition. - var heartbeatInterval = TimeSpan.FromMilliseconds(200); + var heartbeatInterval = TimeSpan.FromMilliseconds(100); var eventsWaitTimeout = TimeSpan.FromMilliseconds(5000); var failPointCommand = BsonDocument.Parse( @@ -289,15 +286,20 @@ public void ConnectionPool_cleared_on_failed_hello() var failpointServer = DriverTestConfiguration.Client.Cluster.SelectServer(new EndPointServerSelector(new DnsEndPoint(serverAddress.Host, serverAddress.Port)), default); using var failPoint = FailPoint.Configure(failpointServer, NoCoreSession.NewHandle(), failPointCommand); - eventCapturer.WaitForOrThrowIfTimeout(new[] - { - typeof(ServerHeartbeatFailedEvent), - typeof(ConnectionPoolClearedEvent), - typeof(ServerHeartbeatSucceededEvent), - typeof(ConnectionPoolReadyEvent), - typeof(ServerHeartbeatSucceededEvent), - }, - eventsWaitTimeout); + eventCapturer.WaitForEventOrThrowIfTimeout(eventsWaitTimeout); + var events = eventCapturer.Events + .OfType() + // event capturer could have some HeartbeatSucceeded have to skip all of them + .SkipWhile(e => e.Type == EventType.ServerHeartbeatSucceeded); + + events.ElementAt(0).Should().BeOfType(); + events.ElementAt(1).Should().BeOfType(); + + // it could be another ServerHeartbeatFailedEvent, because of the failPoint configuration, should just ignore it. + events = events.Skip(2).SkipWhile(e => e.Type == EventType.ServerHeartbeatFailed); + + events.ElementAt(0).Should().BeOfType(); + events.ElementAt(1).Should().BeOfType(); } // private methods @@ -306,6 +308,7 @@ private DisposableMongoClient CreateClient(MongoClientSettings mongoClientSettin var clonedClientSettings = mongoClientSettings ?? DriverTestConfiguration.Client.Settings.Clone(); clonedClientSettings.ApplicationName = applicationName; clonedClientSettings.HeartbeatInterval = heartbeatInterval; + clonedClientSettings.ServerMonitoringMode = ServerMonitoringMode.Poll; clonedClientSettings.ClusterConfigurator = builder => builder.Subscribe(eventCapturer); return DriverTestConfiguration.CreateDisposableClient(clonedClientSettings); From 774b78fdb14566f41ed0fa0c29ce62be7d21c71e Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Thu, 23 May 2024 11:23:21 -0700 Subject: [PATCH 16/38] CSHARP-5087: Bump MacOS version used for tests (#1327) --- evergreen/evergreen.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 808f0cfdb73..d947e68ce38 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -217,12 +217,6 @@ functions: params: script: | ${PREPARE_SHELL} - if [ "${OS}" = "macos-14-arm64" ]; then - # have to set the limit manually for macos-14-arm, can be removed once DEVPROD-7353 will be solved - ulimit -n 64000 - ulimit -a - fi - REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ LOAD_BALANCER=${LOAD_BALANCER} \ MONGODB_VERSION=${VERSION} \ From a5bcbd17f96878d719b635813f68d8aa797a6d5d Mon Sep 17 00:00:00 2001 From: rstam Date: Tue, 14 May 2024 09:25:02 -0700 Subject: [PATCH 17/38] CSHARP-4684: Make part of IMongoQueryProvider public. --- .../Linq/IMongoQueryProvider.cs | 34 +-- src/MongoDB.Driver/Linq/IMongoQueryable.cs | 5 + .../MongoQueryProviderImpl.cs | 2 +- .../Linq2Implementation/MongoQueryableImpl.cs | 13 +- .../Processors/Pipeline/PipelineBinder.cs | 2 +- .../Processors/SerializationBinder.cs | 2 +- .../ExtensionMethods/ExpressionExtensions.cs | 2 +- .../Linq/Linq3Implementation/MongoQuery.cs | 2 +- .../Linq3Implementation/MongoQueryProvider.cs | 2 +- .../ExpressionToPipelineTranslator.cs | 2 +- .../UnionMethodToPipelineTranslator.cs | 2 +- src/MongoDB.Driver/Linq/MongoQueryable.cs | 196 +++++++++--------- .../Jira/CSharp4684Tests.cs | 36 +++- 13 files changed, 174 insertions(+), 126 deletions(-) diff --git a/src/MongoDB.Driver/Linq/IMongoQueryProvider.cs b/src/MongoDB.Driver/Linq/IMongoQueryProvider.cs index 22887f4091b..7b878a05c0d 100644 --- a/src/MongoDB.Driver/Linq/IMongoQueryProvider.cs +++ b/src/MongoDB.Driver/Linq/IMongoQueryProvider.cs @@ -25,17 +25,32 @@ namespace MongoDB.Driver.Linq /// /// An implementation of for MongoDB. /// - internal interface IMongoQueryProvider : IQueryProvider + public interface IMongoQueryProvider : IQueryProvider { /// - /// Gets the collection namespace. + /// Gets the most recently logged stages. /// - CollectionNamespace CollectionNamespace { get; } + BsonDocument[] LoggedStages { get; } /// - /// Gets the most recently logged stages. + /// Executes the strongly-typed query represented by a specified expression tree. /// - BsonDocument[] LoggedStages { get; } + /// The type of the result. + /// An expression tree that represents a LINQ query. + /// The cancellation token. + /// The value that results from executing the specified query. + Task ExecuteAsync(Expression expression, CancellationToken cancellationToken = default(CancellationToken)); + } + + /// + /// The internal IMongoQueryProvider interface. + /// + internal interface IMongoQueryProviderInternal : IMongoQueryProvider + { + /// + /// Gets the collection namespace. + /// + CollectionNamespace CollectionNamespace { get; } /// /// Gets the pipeline input serializer (the DocumentSerializer for collection queries and NoPipelineInputSerializer for database queries). @@ -48,14 +63,5 @@ internal interface IMongoQueryProvider : IQueryProvider /// The expression. /// The execution model. QueryableExecutionModel GetExecutionModel(Expression expression); - - /// - /// Executes the strongly-typed query represented by a specified expression tree. - /// - /// The type of the result. - /// An expression tree that represents a LINQ query. - /// The cancellation token. - /// The value that results from executing the specified query. - Task ExecuteAsync(Expression expression, CancellationToken cancellationToken = default(CancellationToken)); } } diff --git a/src/MongoDB.Driver/Linq/IMongoQueryable.cs b/src/MongoDB.Driver/Linq/IMongoQueryable.cs index a6266bbd572..e569c74f395 100644 --- a/src/MongoDB.Driver/Linq/IMongoQueryable.cs +++ b/src/MongoDB.Driver/Linq/IMongoQueryable.cs @@ -28,6 +28,11 @@ public interface IMongoQueryable : IQueryable /// BsonDocument[] LoggedStages { get; } + /// + /// Gets the IMongoQueryProvider for this queryable. + /// + new IMongoQueryProvider Provider { get; } + /// /// Gets the execution model. /// diff --git a/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryProviderImpl.cs b/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryProviderImpl.cs index 44e6a90ed99..dcc6b0f3c23 100644 --- a/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryProviderImpl.cs +++ b/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryProviderImpl.cs @@ -29,7 +29,7 @@ namespace MongoDB.Driver.Linq.Linq2Implementation { - internal sealed class MongoQueryProviderImpl : IMongoQueryProvider + internal sealed class MongoQueryProviderImpl : IMongoQueryProviderInternal { private readonly IMongoCollection _collection; private readonly AggregateOptions _options; diff --git a/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryableImpl.cs b/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryableImpl.cs index 208a6c89ddf..165d8609244 100644 --- a/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryableImpl.cs +++ b/src/MongoDB.Driver/Linq/Linq2Implementation/MongoQueryableImpl.cs @@ -26,16 +26,16 @@ namespace MongoDB.Driver.Linq.Linq2Implementation { internal sealed class MongoQueryableImpl : IOrderedMongoQueryable { - private readonly IMongoQueryProvider _queryProvider; + private readonly IMongoQueryProviderInternal _queryProvider; private readonly Expression _expression; - public MongoQueryableImpl(IMongoQueryProvider queryProvider) + public MongoQueryableImpl(IMongoQueryProviderInternal queryProvider) { _queryProvider = Ensure.IsNotNull(queryProvider, nameof(queryProvider)); _expression = Expression.Constant(this, typeof(IMongoQueryable)); } - public MongoQueryableImpl(IMongoQueryProvider queryProvider, Expression expression) + public MongoQueryableImpl(IMongoQueryProviderInternal queryProvider, Expression expression) { _queryProvider = Ensure.IsNotNull(queryProvider, nameof(queryProvider)); _expression = Ensure.IsNotNull(expression, nameof(expression)); @@ -53,7 +53,12 @@ public Expression Expression public BsonDocument[] LoggedStages => _queryProvider.LoggedStages; - public IQueryProvider Provider + public IMongoQueryProvider Provider + { + get { return _queryProvider; } + } + + IQueryProvider IQueryable.Provider { get { return _queryProvider; } } diff --git a/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/Pipeline/PipelineBinder.cs b/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/Pipeline/PipelineBinder.cs index 50fe969ee4c..b4cc64a44e1 100644 --- a/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/Pipeline/PipelineBinder.cs +++ b/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/Pipeline/PipelineBinder.cs @@ -78,7 +78,7 @@ protected override Expression BindNonMethodCall(Expression node) node.Type.GetGenericTypeDefinition() == typeof(IMongoQueryable<>)) { var queryable = (IMongoQueryable)((ConstantExpression)node).Value; - var provider = (IMongoQueryProvider)queryable.Provider; + var provider = (IMongoQueryProviderInternal)queryable.Provider; return new PipelineExpression( new CollectionExpression(provider.CollectionNamespace, provider.PipelineInputSerializer), new DocumentExpression(provider.PipelineInputSerializer)); diff --git a/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/SerializationBinder.cs b/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/SerializationBinder.cs index f5984633917..5770abc83e9 100644 --- a/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/SerializationBinder.cs +++ b/src/MongoDB.Driver/Linq/Linq2Implementation/Processors/SerializationBinder.cs @@ -93,7 +93,7 @@ protected override Expression VisitConstant(ConstantExpression node) node.Type.GetGenericTypeDefinition() == typeof(IMongoQueryable<>)) { var queryable = (IMongoQueryable)node.Value; - var provider = (IMongoQueryProvider)queryable.Provider; + var provider = (IMongoQueryProviderInternal)queryable.Provider; return new CollectionExpression( provider.CollectionNamespace, provider.PipelineInputSerializer); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/ExtensionMethods/ExpressionExtensions.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/ExtensionMethods/ExpressionExtensions.cs index 0ddd3d14ab4..73aabf3f45c 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/ExtensionMethods/ExpressionExtensions.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/ExtensionMethods/ExpressionExtensions.cs @@ -39,7 +39,7 @@ public static (string CollectionName, IBsonSerializer DocumentSerializer) GetCol { if (innerExpression is ConstantExpression constantExpression && constantExpression.Value is IMongoQueryable mongoQueryable && - mongoQueryable.Provider is IMongoQueryProvider mongoQueryProvider && + mongoQueryable.Provider is IMongoQueryProviderInternal mongoQueryProvider && mongoQueryProvider.CollectionNamespace != null) { return (mongoQueryProvider.CollectionNamespace.CollectionName, mongoQueryProvider.PipelineInputSerializer); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQuery.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQuery.cs index 07f3c316585..f4732364430 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQuery.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQuery.cs @@ -57,7 +57,7 @@ public MongoQuery(MongoQueryProvider provider, Expression expression) public BsonDocument[] LoggedStages => _provider.LoggedStages; - public MongoQueryProvider Provider => _provider; + public IMongoQueryProvider Provider => _provider; IQueryProvider IQueryable.Provider => _provider; diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQueryProvider.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQueryProvider.cs index 9016fe225aa..95e303b81c4 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQueryProvider.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/MongoQueryProvider.cs @@ -26,7 +26,7 @@ namespace MongoDB.Driver.Linq.Linq3Implementation { - internal abstract class MongoQueryProvider : IMongoQueryProvider + internal abstract class MongoQueryProvider : IMongoQueryProviderInternal { // protected fields protected readonly AggregateOptions _options; diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/ExpressionToPipelineTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/ExpressionToPipelineTranslator.cs index e38d6fd63b7..7b67b0bc334 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/ExpressionToPipelineTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/ExpressionToPipelineTranslator.cs @@ -27,7 +27,7 @@ public static AstPipeline Translate(TranslationContext context, Expression expre if (expression.NodeType == ExpressionType.Constant) { var query = (IQueryable)((ConstantExpression)expression).Value; - var provider = (IMongoQueryProvider)query.Provider; + var provider = (IMongoQueryProviderInternal)query.Provider; return AstPipeline.Empty(provider.PipelineInputSerializer); } diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/UnionMethodToPipelineTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/UnionMethodToPipelineTranslator.cs index 18411b48537..1b90bc80be4 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/UnionMethodToPipelineTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToPipelineTranslators/UnionMethodToPipelineTranslator.cs @@ -38,7 +38,7 @@ public static AstPipeline Translate(TranslationContext context, MethodCallExpres var secondValue = secondExpression.Evaluate(); if (secondValue is IMongoQueryable secondQueryable) { - var secondProvider = (IMongoQueryProvider)secondQueryable.Provider; + var secondProvider = (IMongoQueryProviderInternal)secondQueryable.Provider; var secondCollectionName = secondProvider.CollectionNamespace.CollectionName; var secondPipelineInputSerializer = secondProvider.PipelineInputSerializer; var secondContext = TranslationContext.Create(secondQueryable.Expression, secondPipelineInputSerializer); diff --git a/src/MongoDB.Driver/Linq/MongoQueryable.cs b/src/MongoDB.Driver/Linq/MongoQueryable.cs index e6bce72c239..90c8b73acdd 100644 --- a/src/MongoDB.Driver/Linq/MongoQueryable.cs +++ b/src/MongoDB.Driver/Linq/MongoQueryable.cs @@ -44,7 +44,7 @@ public static class MongoQueryable { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, bool>(Queryable.Any, source), source.Expression), @@ -66,7 +66,7 @@ public static class MongoQueryable Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, bool>(Queryable.Any, source, predicate), source.Expression, @@ -135,7 +135,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, decimal>(Queryable.Average, source), source.Expression), @@ -152,7 +152,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, decimal?>(Queryable.Average, source), source.Expression), @@ -169,7 +169,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double>(Queryable.Average, source), source.Expression), @@ -186,7 +186,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double?>(Queryable.Average, source), source.Expression), @@ -203,7 +203,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, float>(Queryable.Average, source), source.Expression), @@ -220,7 +220,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, float?>(Queryable.Average, source), source.Expression), @@ -237,7 +237,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double>(Queryable.Average, source), source.Expression), @@ -254,7 +254,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double?>(Queryable.Average, source), source.Expression), @@ -271,7 +271,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double>(Queryable.Average, source), source.Expression), @@ -288,7 +288,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double?>(Queryable.Average, source), source.Expression), @@ -311,7 +311,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, decimal>(Queryable.Average, source, selector), source.Expression, @@ -335,7 +335,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, decimal?>(Queryable.Average, source, selector), source.Expression, @@ -359,7 +359,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double>(Queryable.Average, source, selector), source.Expression, @@ -383,7 +383,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double?>(Queryable.Average, source, selector), source.Expression, @@ -407,7 +407,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, float>(Queryable.Average, source, selector), source.Expression, @@ -431,7 +431,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, float?>(Queryable.Average, source, selector), source.Expression, @@ -455,7 +455,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double>(Queryable.Average, source, selector), source.Expression, @@ -479,7 +479,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double?>(Queryable.Average, source, selector), source.Expression, @@ -503,7 +503,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double>(Queryable.Average, source, selector), source.Expression, @@ -527,7 +527,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double?>(Queryable.Average, source, selector), source.Expression, @@ -548,7 +548,7 @@ public static IMongoQueryable As( { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, int>(Queryable.Count, source), source.Expression), @@ -570,7 +570,7 @@ public static IMongoQueryable As( Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, int>(Queryable.Count, source, predicate), source.Expression, @@ -713,7 +713,7 @@ public static IMongoQueryable Documents(this IMongoQueryab { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, TSource>(Queryable.First, source), source.Expression), @@ -735,7 +735,7 @@ public static IMongoQueryable Documents(this IMongoQueryab Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, TSource>(Queryable.First, source, predicate), source.Expression, @@ -756,7 +756,7 @@ public static IMongoQueryable Documents(this IMongoQueryab { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, TSource>(Queryable.FirstOrDefault, source), source.Expression), @@ -778,7 +778,7 @@ public static IMongoQueryable Documents(this IMongoQueryab Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, TSource>(Queryable.FirstOrDefault, source, predicate), source.Expression, @@ -946,7 +946,7 @@ public static IMongoQueryable Join(this { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, long>(Queryable.LongCount, source), source.Expression), @@ -968,7 +968,7 @@ public static IMongoQueryable Join(this Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, long>(Queryable.LongCount, source, predicate), source.Expression, @@ -989,7 +989,7 @@ public static IMongoQueryable Join(this { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, TSource>(Queryable.Max, source), source.Expression), @@ -1012,7 +1012,7 @@ public static IMongoQueryable Join(this Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, TResult>(Queryable.Max, source, selector), source.Expression, @@ -1033,7 +1033,7 @@ public static IMongoQueryable Join(this { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, TSource>(Queryable.Min, source), source.Expression), @@ -1056,7 +1056,7 @@ public static IMongoQueryable Join(this Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, TResult>(Queryable.Min, source, selector), source.Expression, @@ -1287,7 +1287,7 @@ public static IMongoQueryable SelectMany { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, TSource>(Queryable.Single, source), source.Expression), @@ -1309,7 +1309,7 @@ public static IMongoQueryable SelectMany Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, TSource>(Queryable.Single, source, predicate), source.Expression, @@ -1330,7 +1330,7 @@ public static IMongoQueryable SelectMany { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, TSource>(Queryable.SingleOrDefault, source), source.Expression), @@ -1352,7 +1352,7 @@ public static IMongoQueryable SelectMany Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(predicate, nameof(predicate)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, TSource>(Queryable.SingleOrDefault, source, predicate), source.Expression, @@ -1791,7 +1791,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1810,7 +1810,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1829,7 +1829,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1848,7 +1848,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1867,7 +1867,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1886,7 +1886,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1905,7 +1905,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1924,7 +1924,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1943,7 +1943,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1962,7 +1962,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< { Ensure.IsNotNull(source, nameof(source)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -1984,7 +1984,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2007,7 +2007,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2030,7 +2030,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2053,7 +2053,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2076,7 +2076,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2099,7 +2099,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2122,7 +2122,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2145,7 +2145,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2168,7 +2168,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2191,7 +2191,7 @@ public static decimal StandardDeviationPopulation(this IMongoQueryable< Ensure.IsNotNull(source, nameof(source)); Ensure.IsNotNull(selector, nameof(selector)); - return ((IMongoQueryProvider)source.Provider).ExecuteAsync( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationPopulation, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2591,7 +2591,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2610,7 +2610,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2629,7 +2629,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2648,7 +2648,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2667,7 +2667,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2686,7 +2686,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2705,7 +2705,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2724,7 +2724,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2743,7 +2743,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2762,7 +2762,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source), Expression.Convert(source.Expression, typeof(IMongoQueryable))), @@ -2784,7 +2784,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2807,7 +2807,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2830,7 +2830,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2853,7 +2853,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2876,7 +2876,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2899,7 +2899,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2922,7 +2922,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2945,7 +2945,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2968,7 +2968,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -2991,7 +2991,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo(StandardDeviationSample, source, selector), Expression.Convert(source.Expression, typeof(IMongoQueryable)), @@ -3009,7 +3009,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, decimal>(Queryable.Sum, source), source.Expression), @@ -3026,7 +3026,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, decimal?>(Queryable.Sum, source), source.Expression), @@ -3043,7 +3043,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double>(Queryable.Sum, source), source.Expression), @@ -3060,7 +3060,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, double?>(Queryable.Sum, source), source.Expression), @@ -3077,7 +3077,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, float>(Queryable.Sum, source), source.Expression), @@ -3094,7 +3094,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, float?>(Queryable.Sum, source), source.Expression), @@ -3111,7 +3111,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, int>(Queryable.Sum, source), source.Expression), @@ -3128,7 +3128,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, int?>(Queryable.Sum, source), source.Expression), @@ -3145,7 +3145,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, long>(Queryable.Sum, source), source.Expression), @@ -3162,7 +3162,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, long?>(Queryable.Sum, source), source.Expression), @@ -3185,7 +3185,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, decimal>(Queryable.Sum, source, selector), source.Expression, @@ -3209,7 +3209,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, decimal?>(Queryable.Sum, source, selector), source.Expression, @@ -3233,7 +3233,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double>(Queryable.Sum, source, selector), source.Expression, @@ -3257,7 +3257,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, double?>(Queryable.Sum, source, selector), source.Expression, @@ -3281,7 +3281,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, float>(Queryable.Sum, source, selector), source.Expression, @@ -3305,7 +3305,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, float?>(Queryable.Sum, source, selector), source.Expression, @@ -3329,7 +3329,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, int>(Queryable.Sum, source, selector), source.Expression, @@ -3353,7 +3353,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, int?>(Queryable.Sum, source, selector), source.Expression, @@ -3377,7 +3377,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, long>(Queryable.Sum, source, selector), source.Expression, @@ -3401,7 +3401,7 @@ public static decimal StandardDeviationSample(this IMongoQueryable( + return source.Provider.ExecuteAsync( Expression.Call( GetMethodInfo, Expression>, long?>(Queryable.Sum, source, selector), source.Expression, diff --git a/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp4684Tests.cs b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp4684Tests.cs index 51fd571b498..42c3d323f03 100644 --- a/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp4684Tests.cs +++ b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp4684Tests.cs @@ -27,7 +27,7 @@ public class CSharp4684Tests : Linq3IntegrationTest { [Theory] [ParameterAttributeData] - public async Task Multiple_result_query_logged_stages_can_be_retrieved_using_LoggedStages_property( + public async Task Multiple_result_query_logged_stages_can_be_retrieved_using_IMongoQueryable_LoggedStages_property( [Values(false, true)] bool async) { var collection = GetCollection(); @@ -41,7 +41,7 @@ public async Task Multiple_result_query_logged_stages_can_be_retrieved_using_Log [Theory] [ParameterAttributeData] - public async Task Single_result_query_logged_stages_can_be_retrieved_using_LoggedStages_property( + public async Task Single_result_query_logged_stages_can_be_retrieved_using_IMongoQueryable_LoggedStages_property( [Values(false, true)] bool async) { var collection = GetCollection(); @@ -57,6 +57,38 @@ public async Task Single_result_query_logged_stages_can_be_retrieved_using_Logge result.Id.Should().Be(1); } + [Theory] + [ParameterAttributeData] + public async Task Multiple_result_query_logged_stages_can_be_retrieved_using_IMongoQueryProvider_LoggedStages_property( + [Values(false, true)] bool async) + { + var collection = GetCollection(); + var queryable = collection.AsQueryable().Where(x => x.X == 1); + + var results = async ? await queryable.ToListAsync() : queryable.ToList(); + + AssertStages(queryable.Provider.LoggedStages, "{ $match : { X : 1 } }"); + results.Select(x => x.Id).Should().Equal(1); + } + + [Theory] + [ParameterAttributeData] + public async Task Single_result_query_logged_stages_can_be_retrieved_using_IMongoQueryProvider_LoggedStages_property( + [Values(false, true)] bool async) + { + var collection = GetCollection(); + var queryable = collection.AsQueryable().Where(x => x.X == 1); + + // cast to IQueryable so First below resolves to Queryable.First instead of IAsyncCursorSource.First + var result = async ? await queryable.FirstAsync() : ((IQueryable)queryable).First(); + + AssertStages( + queryable.Provider.LoggedStages, + "{ $match : { X : 1 } }", + "{ $limit : 1 }"); + result.Id.Should().Be(1); + } + private IMongoCollection GetCollection() { var collection = GetCollection(); From 3a71962a34059dcbfbf5abf1796536ccca968047 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Fri, 24 May 2024 12:46:02 -0400 Subject: [PATCH 18/38] CSHARP-4944: Update Libmongocrypt version and sbom lite (#1329) --- purls.txt | 2 +- sbom.json | 23 +++++++------------ .../MongoDB.Driver.Core.csproj | 2 +- src/MongoDB.Driver/MongoDB.Driver.csproj | 2 +- .../Packaging/PackagingTests.cs | 2 +- 5 files changed, 12 insertions(+), 19 deletions(-) diff --git a/purls.txt b/purls.txt index 7c0089a1a10..254713bb3aa 100644 --- a/purls.txt +++ b/purls.txt @@ -1 +1 @@ -pkg:nuget/MongoDB.Libmongocrypt@1.8.2 +pkg:nuget/MongoDB.Libmongocrypt@1.9.0 diff --git a/sbom.json b/sbom.json index d9812068318..82e0bbfdb7f 100644 --- a/sbom.json +++ b/sbom.json @@ -1,37 +1,30 @@ { "components": [ { - "bom-ref": "pkg:nuget/MongoDB.Libmongocrypt@1.8.2", + "bom-ref": "pkg:nuget/MongoDB.Libmongocrypt@1.9.0", "externalReferences": [ { "type": "distribution", - "url": "https://www.nuget.org/api/v2/package/MongoDB.Libmongocrypt/1.8.2" + "url": "https://www.nuget.org/api/v2/package/MongoDB.Libmongocrypt/1.9.0" }, { "type": "website", - "url": "https://www.nuget.org/packages/MongoDB.Libmongocrypt/1.8.2" - } - ], - "licenses": [ - { - "license": { - "name": "Other" - } + "url": "https://www.nuget.org/packages/MongoDB.Libmongocrypt/1.9.0" } ], "name": "MongoDB.Libmongocrypt", - "purl": "pkg:nuget/MongoDB.Libmongocrypt@1.8.2", + "purl": "pkg:nuget/MongoDB.Libmongocrypt@1.9.0", "type": "library", - "version": "1.8.2" + "version": "1.9.0" } ], "dependencies": [ { - "ref": "pkg:nuget/MongoDB.Libmongocrypt@1.8.2" + "ref": "pkg:nuget/MongoDB.Libmongocrypt@1.9.0" } ], "metadata": { - "timestamp": "2024-05-03T20:19:34.867974+00:00", + "timestamp": "2024-05-23T20:46:09.160033+00:00", "tools": [ { "externalReferences": [ @@ -74,7 +67,7 @@ } ] }, - "serialNumber": "urn:uuid:137c9f19-2c41-488a-bacb-b367694656ff", + "serialNumber": "urn:uuid:77687498-fcba-4e54-880d-67b3b53d41af", "version": 1, "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json", "bomFormat": "CycloneDX", diff --git a/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj b/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj index fcf67fd8b62..5b996d1a679 100644 --- a/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj +++ b/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj @@ -23,8 +23,8 @@ - + diff --git a/src/MongoDB.Driver/MongoDB.Driver.csproj b/src/MongoDB.Driver/MongoDB.Driver.csproj index bdcf1db6ed1..c06572f1195 100644 --- a/src/MongoDB.Driver/MongoDB.Driver.csproj +++ b/src/MongoDB.Driver/MongoDB.Driver.csproj @@ -14,8 +14,8 @@ - + diff --git a/tests/MongoDB.Driver.Tests/Packaging/PackagingTests.cs b/tests/MongoDB.Driver.Tests/Packaging/PackagingTests.cs index adc3d7e39e3..aef639b267c 100644 --- a/tests/MongoDB.Driver.Tests/Packaging/PackagingTests.cs +++ b/tests/MongoDB.Driver.Tests/Packaging/PackagingTests.cs @@ -38,7 +38,7 @@ public static void Libmongocrypt_library_should_provide_library_version() { var version = Library.Version; - version.Should().Be("1.8.4"); + version.Should().Be("1.10.0"); } #pragma warning disable IDE0051 // Remove unused private members From 5dd4746d688a57cb0d57408b35397aa9976119dc Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Mon, 27 May 2024 12:21:38 -0700 Subject: [PATCH 19/38] Set open files limit manually for MacOS-14-arm. (#1331) --- build.ps1 | 4 ++-- evergreen/evergreen.yml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build.ps1 b/build.ps1 index 1d07b5e0a19..c4eb3e1ae7d 100644 --- a/build.ps1 +++ b/build.ps1 @@ -102,7 +102,7 @@ if($FoundDotNetCliVersion -ne $DotNetVersion) { & bash $ScriptPath --install-dir "$InstallPath" --channel 2.1 --no-path & bash $ScriptPath --install-dir "$InstallPath" --channel 3.1 --no-path & bash $ScriptPath --install-dir "$InstallPath" --channel 5.0 --no-path - & bash $ScriptPath --install-dir "$InstallPath" --channel 8.0 --no-path + & bash $ScriptPath --install-dir "$InstallPath" --channel 6.0 --no-path & bash $ScriptPath --version "$DotNetVersion" --install-dir "$InstallPath" --channel "$DotNetChannel" --no-path Remove-PathVariable "$InstallPath" @@ -114,7 +114,7 @@ if($FoundDotNetCliVersion -ne $DotNetVersion) { & $ScriptPath -Channel 2.1 -InstallDir $InstallPath; & $ScriptPath -Channel 3.1 -InstallDir $InstallPath; & $ScriptPath -Channel 5.0 -InstallDir $InstallPath; - & $ScriptPath -Channel 8.0 -InstallDir $InstallPath; + & $ScriptPath -Channel 6.0 -InstallDir $InstallPath; & $ScriptPath -Channel $DotNetChannel -Version $DotNetVersion -InstallDir $InstallPath; Remove-PathVariable "$InstallPath" diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index d947e68ce38..c86fea85ed8 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -217,6 +217,11 @@ functions: params: script: | ${PREPARE_SHELL} + if [ "${OS}" = "macos-14-arm64" ]; then + ulimit -a + # have to set the limit manually for macos-14-arm, can be removed once DEVPROD-7353 will be solved + ulimit -n 64000 + fi REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ LOAD_BALANCER=${LOAD_BALANCER} \ MONGODB_VERSION=${VERSION} \ From 0fb592b533505688dfcd11362df9a41ba8c97072 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Mon, 27 May 2024 16:09:31 -0700 Subject: [PATCH 20/38] CSHARP-4996: OIDC Spec Cleanup (#1323) --- evergreen/run-mongodb-oidc-env-tests.sh | 2 + evergreen/run-mongodb-oidc-tests.sh | 10 +- .../tests/unified/mongodb-oidc-no-retry.json | 213 ++---------------- .../tests/unified/mongodb-oidc-no-retry.yml | 100 +------- .../Oidc/MongoOidcAuthenticator.cs | 43 ++-- src/MongoDB.Driver.Core/ServerErrorCode.cs | 1 + .../auth/OidcAuthenticationProseTests.cs | 167 ++++++++++++-- .../UnifiedTestOperations/UnifiedEntityMap.cs | 6 +- 8 files changed, 215 insertions(+), 327 deletions(-) diff --git a/evergreen/run-mongodb-oidc-env-tests.sh b/evergreen/run-mongodb-oidc-env-tests.sh index 94f63e31eed..0a7093e3af8 100644 --- a/evergreen/run-mongodb-oidc-env-tests.sh +++ b/evergreen/run-mongodb-oidc-env-tests.sh @@ -31,4 +31,6 @@ else exit 1 fi +sleep 60 # sleep for 1 minute to let cluster make the master election + dotnet test --no-build --framework net6.0 --filter Category=MongoDbOidc -e OIDC_ENV="$OIDC_ENV" -e TOKEN_RESOURCE="$TOKEN_RESOURCE" -e MONGODB_URI="$MONGODB_URI" --results-directory ./build/test-results --logger "console;verbosity=detailed" ./tests/**/*.Tests.dll diff --git a/evergreen/run-mongodb-oidc-tests.sh b/evergreen/run-mongodb-oidc-tests.sh index 9583f20f873..a51efceb59c 100644 --- a/evergreen/run-mongodb-oidc-tests.sh +++ b/evergreen/run-mongodb-oidc-tests.sh @@ -30,9 +30,13 @@ if [ $OIDC_ENV == "test" ]; then fi source ${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh - if [[ ! $OS =~ ubuntu.* ]]; then - # Ubuntu uses local server with already build admin credentials in connection string - MONGODB_URI="mongodb+srv://${OIDC_ADMIN_USER}:${OIDC_ADMIN_PWD}@${MONGODB_URI:14}?authSource=admin" + if [[ "$MONGODB_URI" =~ ^mongodb:.* ]]; then + MONGODB_URI="mongodb://${OIDC_ADMIN_USER}:${OIDC_ADMIN_PWD}@${MONGODB_URI:10}&authSource=admin" + elif [[ "$MONGODB_URI" =~ ^mongodb\+srv:.* ]]; then + MONGODB_URI="mongodb+srv://${OIDC_ADMIN_USER}:${OIDC_ADMIN_PWD}@${MONGODB_URI:14}&authSource=admin" + else + echo "Unexpected MONGODB_URI format: $MONGODB_URI" + exit 1 fi elif [ $OIDC_ENV == "azure" ]; then source ./env.sh diff --git a/specifications/auth/tests/unified/mongodb-oidc-no-retry.json b/specifications/auth/tests/unified/mongodb-oidc-no-retry.json index 83d73e4e509..0a8658455ec 100644 --- a/specifications/auth/tests/unified/mongodb-oidc-no-retry.json +++ b/specifications/auth/tests/unified/mongodb-oidc-no-retry.json @@ -5,7 +5,8 @@ { "minServerVersion": "7.0", "auth": true, - "authMechanism": "MONGODB-OIDC" + "authMechanism": "MONGODB-OIDC", + "serverless": "forbid" } ], "createEntities": [ @@ -52,9 +53,7 @@ { "collectionName": "collName", "databaseName": "test", - "documents": [ - - ] + "documents": [] } ], "tests": [ @@ -65,12 +64,9 @@ "name": "find", "object": "collection0", "arguments": { - "filter": { - } + "filter": {} }, - "expectResult": [ - - ] + "expectResult": [] } ], "expectEvents": [ @@ -81,8 +77,7 @@ "commandStartedEvent": { "command": { "find": "collName", - "filter": { - } + "filter": {} } } }, @@ -161,12 +156,9 @@ "name": "find", "object": "collection0", "arguments": { - "filter": { - } + "filter": {} }, - "expectResult": [ - - ] + "expectResult": [] } ], "expectEvents": [ @@ -177,8 +169,7 @@ "commandStartedEvent": { "command": { "find": "collName", - "filter": { - } + "filter": {} } } }, @@ -191,8 +182,7 @@ "commandStartedEvent": { "command": { "find": "collName", - "filter": { - } + "filter": {} } } }, @@ -324,12 +314,14 @@ "client": "failPointClient", "failPoint": { "configureFailPoint": "failCommand", - "mode": "alwaysOn", + "mode": { + "times": 1 + }, "data": { "failCommands": [ "saslStart" ], - "errorCode": 20 + "errorCode": 18 } } } @@ -392,52 +384,6 @@ { "description": "Handshake without cached token should not use speculative authentication", "operations": [ - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "failPointClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": "alwaysOn", - "data": { - "failCommands": [ - "saslStart" - ], - "errorCode": 20 - } - } - } - }, - { - "name": "insertOne", - "object": "collection0", - "arguments": { - "document": { - "_id": 1, - "x": 1 - } - }, - "expectError": { - "errorCode": 20 - } - } - ] - }, - { - "description": "Read commands should fail if reauthentication fails", - "operations": [ - { - "name": "find", - "object": "collection0", - "arguments": { - "filter": { - } - }, - "expectResult": [ - - ] - }, { "name": "failPoint", "object": "testRunner", @@ -446,69 +392,17 @@ "failPoint": { "configureFailPoint": "failCommand", "mode": { - "times": 2 + "times": 1 }, "data": { "failCommands": [ - "find", "saslStart" ], - "errorCode": 391 + "errorCode": 18 } } } }, - { - "name": "find", - "object": "collection0", - "arguments": { - "filter": { - } - }, - "expectError": { - "errorCode": 391 - } - } - ], - "expectEvents": [ - { - "client": "client0", - "events": [ - { - "commandStartedEvent": { - "command": { - "find": "collName", - "filter": { - } - } - } - }, - { - "commandSucceededEvent": { - "commandName": "find" - } - }, - { - "commandStartedEvent": { - "command": { - "find": "collName", - "filter": { - } - } - } - }, - { - "commandFailedEvent": { - "commandName": "find" - } - } - ] - } - ] - }, - { - "description": "Write commands should fail if reauthentication fails", - "operations": [ { "name": "insertOne", "object": "collection0", @@ -517,84 +411,11 @@ "_id": 1, "x": 1 } - } - }, - { - "name": "failPoint", - "object": "testRunner", - "arguments": { - "client": "failPointClient", - "failPoint": { - "configureFailPoint": "failCommand", - "mode": { - "times": 2 - }, - "data": { - "failCommands": [ - "insert", - "saslStart" - ], - "errorCode": 391 - } - } - } - }, - { - "name": "insertOne", - "object": "collection0", - "arguments": { - "document": { - "_id": 2, - "x": 2 - } }, "expectError": { - "errorCode": 391 + "errorCode": 18 } } - ], - "expectEvents": [ - { - "client": "client0", - "events": [ - { - "commandStartedEvent": { - "command": { - "insert": "collName", - "documents": [ - { - "_id": 1, - "x": 1 - } - ] - } - } - }, - { - "commandSucceededEvent": { - "commandName": "insert" - } - }, - { - "commandStartedEvent": { - "command": { - "insert": "collName", - "documents": [ - { - "_id": 2, - "x": 2 - } - ] - } - } - }, - { - "commandFailedEvent": { - "commandName": "insert" - } - } - ] - } ] } ] diff --git a/specifications/auth/tests/unified/mongodb-oidc-no-retry.yml b/specifications/auth/tests/unified/mongodb-oidc-no-retry.yml index 8108acb5019..339f8817412 100644 --- a/specifications/auth/tests/unified/mongodb-oidc-no-retry.yml +++ b/specifications/auth/tests/unified/mongodb-oidc-no-retry.yml @@ -5,6 +5,7 @@ runOnRequirements: - minServerVersion: "7.0" auth: true authMechanism: "MONGODB-OIDC" + serverless: forbid createEntities: - client: id: &failPointClient failPointClient @@ -173,11 +174,12 @@ tests: client: failPointClient failPoint: configureFailPoint: failCommand - mode: "alwaysOn" + mode: + times: 1 data: failCommands: - saslStart - errorCode: 20 # IllegalOperation + errorCode: 18 - name: insertOne object: collection0 arguments: @@ -211,11 +213,12 @@ tests: client: failPointClient failPoint: configureFailPoint: failCommand - mode: "alwaysOn" + mode: + times: 1 data: failCommands: - saslStart - errorCode: 20 # IllegalOperation + errorCode: 18 - name: insertOne object: collection0 arguments: @@ -223,91 +226,4 @@ tests: _id: 1 x: 1 expectError: - errorCode: 20 # IllegalOperation -- description: Read commands should fail if reauthentication fails - operations: - - name: find - object: collection0 - arguments: - filter: {} - expectResult: [] - - name: failPoint - object: testRunner - arguments: - client: failPointClient - failPoint: - configureFailPoint: failCommand - mode: - times: 2 - data: - failCommands: - - find - - saslStart - errorCode: 391 # ReauthenticationRequired - - name: find - object: collection0 - arguments: - filter: {} - expectError: { errorCode: 391 } - expectEvents: - - client: client0 - events: - - commandStartedEvent: - command: - find: collName - filter: {} - - commandSucceededEvent: - commandName: find - - commandStartedEvent: - command: - find: collName - filter: {} - - commandFailedEvent: - commandName: find -- description: Write commands should fail if reauthentication fails - operations: - - name: insertOne - object: collection0 - arguments: - document: - _id: 1 - x: 1 - - name: failPoint - object: testRunner - arguments: - client: failPointClient - failPoint: - configureFailPoint: failCommand - mode: - times: 2 - data: - failCommands: - - insert - - saslStart - errorCode: 391 # ReauthenticationRequired - - name: insertOne - object: collection0 - arguments: - document: - _id: 2 - x: 2 - expectError: { errorCode: 391 } - expectEvents: - - client: client0 - events: - - commandStartedEvent: - command: - insert: collName - documents: - - _id: 1 - x: 1 - - commandSucceededEvent: - commandName: insert - - commandStartedEvent: - command: - insert: collName - documents: - - _id: 2 - x: 2 - - commandFailedEvent: - commandName: insert + errorCode: 18 \ No newline at end of file diff --git a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/MongoOidcAuthenticator.cs b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/MongoOidcAuthenticator.cs index 221f11f24fa..e03cc9d9db7 100644 --- a/src/MongoDB.Driver.Core/Core/Authentication/Oidc/MongoOidcAuthenticator.cs +++ b/src/MongoDB.Driver.Core/Core/Authentication/Oidc/MongoOidcAuthenticator.cs @@ -98,17 +98,18 @@ void TryAuthenticate(bool retryOnFailure) } catch (Exception ex) { - ClearCredentialsCache(); - - if (retryOnFailure && ShouldReauthenticateIfSaslError(ex, connection)) - { - Thread.Sleep(100); - TryAuthenticate(false); - } - else + if (IsAuthenticationError(ex)) { - throw UnwrapMongoAuthenticationException(ex); + ClearCredentialsCache(); + if (retryOnFailure) + { + Thread.Sleep(100); + TryAuthenticate(false); + return; + } } + + throw UnwrapMongoAuthenticationException(ex); } } } @@ -129,17 +130,18 @@ async Task TryAuthenticateAsync(bool retryOnFailure) } catch (Exception ex) { - ClearCredentialsCache(); - - if (retryOnFailure && ShouldReauthenticateIfSaslError(ex, connection)) - { - await Task.Delay(100).ConfigureAwait(false); - await TryAuthenticateAsync(false).ConfigureAwait(false); - } - else + if (IsAuthenticationError(ex)) { - throw UnwrapMongoAuthenticationException(ex); + ClearCredentialsCache(); + if (retryOnFailure) + { + await Task.Delay(100, cancellationToken).ConfigureAwait(false); + await TryAuthenticateAsync(false).ConfigureAwait(false); + return; + } } + + throw UnwrapMongoAuthenticationException(ex); } } } @@ -160,12 +162,11 @@ public override BsonDocument CustomizeInitialHelloCommand(BsonDocument helloComm public void ClearCredentialsCache() => OidcMechanism.ClearCache(); - private static bool ShouldReauthenticateIfSaslError(Exception ex, IConnection connection) + private static bool IsAuthenticationError(Exception ex) { return ex is MongoAuthenticationException authenticationException && authenticationException.InnerException is MongoCommandException mongoCommandException && - mongoCommandException.Code == (int)ServerErrorCode.AuthenticationFailed && - !connection.Description.IsInitialized(); + mongoCommandException.Code == (int)ServerErrorCode.AuthenticationFailed; } private static Exception UnwrapMongoAuthenticationException(Exception ex) diff --git a/src/MongoDB.Driver.Core/ServerErrorCode.cs b/src/MongoDB.Driver.Core/ServerErrorCode.cs index 36be4cf388f..254fd2fd10e 100644 --- a/src/MongoDB.Driver.Core/ServerErrorCode.cs +++ b/src/MongoDB.Driver.Core/ServerErrorCode.cs @@ -30,6 +30,7 @@ internal enum ServerErrorCode HostNotFound = 7, HostUnreachable = 6, DuplicateKey = 11000, + IllegalOperation = 20, Interrupted = 11601, InterruptedAtShutdown = 11600, InterruptedDueToReplStateChange = 11602, diff --git a/tests/MongoDB.Driver.Tests/Specifications/auth/OidcAuthenticationProseTests.cs b/tests/MongoDB.Driver.Tests/Specifications/auth/OidcAuthenticationProseTests.cs index f69e76c24b9..380385ee4ec 100644 --- a/tests/MongoDB.Driver.Tests/Specifications/auth/OidcAuthenticationProseTests.cs +++ b/tests/MongoDB.Driver.Tests/Specifications/auth/OidcAuthenticationProseTests.cs @@ -51,7 +51,8 @@ public OidcAuthenticationProseTests(ITestOutputHelper output) : base(output) OidcCallbackAdapterCachingFactory.Instance.Reset(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L37 + // 1.1 Callback is called during authentication + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L39 [Theory] [ParameterAttributeData] public async Task Callback_authentication_callback_called_during_authentication([Values(false, true)]bool async) @@ -73,7 +74,8 @@ public async Task Callback_authentication_callback_called_during_authentication( eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L44 + // 1.2 Callback is called once for multiple connections + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L46 [Theory] [ParameterAttributeData] public async Task Callback_authentication_callback_called_once_for_multiple_connections([Values(false, true)]bool async) @@ -98,7 +100,8 @@ await ThreadingUtilities.ExecuteTasksOnNewThreads(10, async t => VerifyCallbackUsage(callbackMock, async, Times.Once()); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L53 + // 2.1 Valid Callback Inputs + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L55 [Theory] [ParameterAttributeData] public async Task Callback_validation_valid_callback_inputs([Values(false, true)] bool async) @@ -120,7 +123,8 @@ public async Task Callback_validation_valid_callback_inputs([Values(false, true) eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L60 + // 2.2 OIDC Callback Returns Null + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L62 [Theory] [ParameterAttributeData] public async Task Callback_validation_callback_returns_null([Values(false, true)] bool async) @@ -141,7 +145,8 @@ public async Task Callback_validation_callback_returns_null([Values(false, true) eventCapturer.Events.Should().BeEmpty(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L66 + // 2.3 OIDC Callback Returns Missing Data + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L68 [Theory] [ParameterAttributeData] public async Task Callback_validation_callback_returns_missing_data([Values(false, true)] bool async) @@ -165,7 +170,8 @@ public async Task Callback_validation_callback_returns_missing_data([Values(fals eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L73 + // 2.4 Invalid Client Configuration with Callback + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L75 [Theory] [ParameterAttributeData] public async Task Callback_validation_invalid_client_configuration([Values(false, true)] bool async) @@ -188,7 +194,30 @@ public async Task Callback_validation_invalid_client_configuration([Values(false eventCapturer.Events.Should().BeEmpty(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L80 + // 2.5 Invalid use of ALLOWED_HOSTS + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L81 + [Theory] + [ParameterAttributeData] + public async Task Invalid_Allowed_Hosts_Usage([Values(false, true)] bool async) + { + EnsureOidcIsConfigured("azure"); + + var credential = MongoCredential.CreateOidcCredential("azure") + .WithMechanismProperty(OidcConfiguration.TokenResourceMechanismPropertyName, Environment.GetEnvironmentVariable("TOKEN_RESOURCE")) + .WithMechanismProperty("ALLOWED_HOSTS", Array.Empty()); + var eventCapturer = new EventCapturer().CaptureCommandEvents(SaslAuthenticator.SaslStartCommand); + var collection = CreateMongoCollection(credential, eventCapturer); + + var exception = async + ? await Record.ExceptionAsync(() => collection.FindAsync(Builders.Filter.Empty)) + : Record.Exception(() => collection.FindSync(Builders.Filter.Empty)); + + exception.Should().BeOfType(); + eventCapturer.Count.Should().Be(0); + } + + // 3.1 Authentication failure with cached tokens fetch a new token and retry auth + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L89 [Theory] [ParameterAttributeData] public async Task Authentication_failure_with_cached_tokens_fetch_new_and_retry([Values(false, true)] bool async) @@ -211,6 +240,7 @@ public async Task Authentication_failure_with_cached_tokens_fetch_new_and_retry( : callbackAdapter.GetCredentials(callbackParameters, default); // configure mock with valid access token + callbackMock.Reset(); ConfigureOidcCallback(callbackMock, GetAccessTokenValue()); // callbackAdapter should have cached wrong access token at this point. @@ -229,7 +259,8 @@ public async Task Authentication_failure_with_cached_tokens_fetch_new_and_retry( // eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L88 + // 3.2 Authentication failures without cached tokens return an error + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L97 [Theory] [ParameterAttributeData] public async Task Authentication_failure_without_cached_tokens_return_error([Values(false, true)] bool async) @@ -253,7 +284,37 @@ public async Task Authentication_failure_without_cached_tokens_return_error([Val eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L95 + // 3.3 Unexpected error code does not clear the cache + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L104 + [Theory] + [ParameterAttributeData] + public async Task Unexpected_error_does_not_clear_token_cache([Values(false, true)] bool async) + { + EnsureOidcIsConfigured("test"); + + var callbackMock = new Mock(); + ConfigureOidcCallback(callbackMock, GetAccessTokenValue()); + var credential = MongoCredential.CreateOidcCredential(callbackMock.Object); + var collection = CreateMongoCollection(credential); + + Exception exception; + using (ConfigureFailPoint(1, (int)ServerErrorCode.IllegalOperation, "saslStart")) + { + exception = async + ? await Record.ExceptionAsync(() => collection.FindAsync(Builders.Filter.Empty)) + : Record.Exception(() => collection.FindSync(Builders.Filter.Empty)); + + _ = async + ? await collection.FindAsync(Builders.Filter.Empty) + : collection.FindSync(Builders.Filter.Empty); + } + + exception.Should().BeOfType(); + VerifyCallbackUsage(callbackMock, async, Times.Once()); + } + + // 4.1 Reauthentication Succeeds + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L132 [Theory] [ParameterAttributeData] public async Task ReAuthentication([Values(false, true)] bool async) @@ -280,7 +341,88 @@ public async Task ReAuthentication([Values(false, true)] bool async) eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L125 + // 4.2 Read Commands Fail If Reauthentication Fails + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L156 + [Theory] + [ParameterAttributeData] + public async Task Read_commands_fail_if_reauthentication_fails([Values(false, true)] bool async) + { + EnsureOidcIsConfigured("test"); + + var callbackMock = new Mock(); + // configure mock with valid access token + ConfigureOidcCallback(callbackMock, GetAccessTokenValue()); + var credential = MongoCredential.CreateOidcCredential(callbackMock.Object); + var eventCapturer = new EventCapturer().CaptureCommandEvents(SaslAuthenticator.SaslStartCommand); + var collection = CreateMongoCollection(credential, eventCapturer); + + _ = async + ? await collection.FindAsync(Builders.Filter.Empty) + : collection.FindSync(Builders.Filter.Empty); + + // reconfigure mock to return invalid access token + ConfigureOidcCallback(callbackMock, "wrong token"); + Exception exception; + using (ConfigureFailPoint(1, (int)ServerErrorCode.ReauthenticationRequired, "find")) + { + exception = async + ? await Record.ExceptionAsync(() => collection.FindAsync(Builders.Filter.Empty)) + : Record.Exception(() => collection.FindSync(Builders.Filter.Empty)); + } + + exception.Should().BeOfType(); + VerifyCallbackUsage(callbackMock, async, Times.Exactly(2)); + eventCapturer.Next().Should().BeOfType(); + eventCapturer.Next().Should().BeOfType(); + eventCapturer.Next().Should().BeOfType(); + eventCapturer.Next().Should().BeOfType(); + } + + // 4.3 Write Commands Fail If Reauthentication Fails + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L181 + [Theory] + [ParameterAttributeData] + public async Task Write_commands_fail_if_reauthentication_fails([Values(false, true)] bool async) + { + var dummyDocument = new BsonDocument("dummy", "value"); + EnsureOidcIsConfigured("test"); + + var callbackMock = new Mock(); + // configure mock with valid access token + ConfigureOidcCallback(callbackMock, GetAccessTokenValue()); + var credential = MongoCredential.CreateOidcCredential(callbackMock.Object); + var eventCapturer = new EventCapturer().CaptureCommandEvents(SaslAuthenticator.SaslStartCommand); + var collection = CreateMongoCollection(credential, eventCapturer); + + if (async) + { + await collection.InsertOneAsync(dummyDocument); + } + else + { + collection.InsertOne(dummyDocument); + } + + // reconfigure mock to return invalid access token + ConfigureOidcCallback(callbackMock, "wrong token"); + Exception exception; + using (ConfigureFailPoint(1, (int)ServerErrorCode.ReauthenticationRequired, "insert")) + { + exception = async + ? await Record.ExceptionAsync(() => collection.InsertOneAsync(dummyDocument)) + : Record.Exception(() => collection.InsertOne(dummyDocument)); + } + + exception.Should().BeOfType(); + VerifyCallbackUsage(callbackMock, async, Times.Exactly(2)); + eventCapturer.Next().Should().BeOfType(); + eventCapturer.Next().Should().BeOfType(); + eventCapturer.Next().Should().BeOfType(); + eventCapturer.Next().Should().BeOfType(); + } + + // 5.1 Azure With No Username + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L212 [Theory] [ParameterAttributeData] public async Task Azure_auth_no_username([Values(false, true)] bool async) @@ -300,7 +442,8 @@ public async Task Azure_auth_no_username([Values(false, true)] bool async) eventCapturer.Next().Should().BeOfType(); } - // https://github.com/mongodb/specifications/blob/611b12ccbdd012dcd9ab2877a32200b3835c97af/source/auth/tests/mongodb-oidc.md?plain=1#L131 + // 5.2 Azure with Bad Username + // https://github.com/mongodb/specifications/blob/1448ba6eedfa2f16584222e683b427bea07bb085/source/auth/tests/mongodb-oidc.md?plain=1#L218 [Theory] [ParameterAttributeData] public async Task Azure_auth_bad_username_return_error([Values(false, true)] bool async) @@ -320,8 +463,6 @@ public async Task Azure_auth_bad_username_return_error([Values(false, true)] boo private void ConfigureOidcCallback(Mock callbackMock, string accessToken) { - callbackMock.Reset(); - var response = new OidcAccessToken(accessToken, null); callbackMock .Setup(c => c.GetOidcAccessToken(It.IsAny(), It.IsAny())) diff --git a/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs b/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs index 5ddfa797356..832a7f72e0b 100644 --- a/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs +++ b/tests/MongoDB.Driver.Tests/UnifiedTestOperations/UnifiedEntityMap.cs @@ -837,7 +837,10 @@ private IMongoCollection CreateCollection(BsonDocument entity, Dic { string collectionName = null; IMongoDatabase database = null; - MongoCollectionSettings settings = null; + var settings = new MongoCollectionSettings + { + ReadPreference = ReadPreference.Primary + }; foreach (var element in entity) { @@ -854,7 +857,6 @@ private IMongoCollection CreateCollection(BsonDocument entity, Dic collectionName = entity["collectionName"].AsString; break; case "collectionOptions": - settings = new MongoCollectionSettings(); foreach (var option in element.Value.AsBsonDocument) { switch (option.Name) From f8f1198e7be81cb184a987004b360996700ed361 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Wed, 29 May 2024 15:37:43 -0400 Subject: [PATCH 21/38] CSHARP-5065: Deserialize wrappedBytes directly in ExplicitEncryptionLibMongoCryptController.cs (#1321) --- .../ExplicitEncryptionLibMongoCryptController.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/MongoDB.Driver/Encryption/ExplicitEncryptionLibMongoCryptController.cs b/src/MongoDB.Driver/Encryption/ExplicitEncryptionLibMongoCryptController.cs index e202733f482..a3b40994a6f 100644 --- a/src/MongoDB.Driver/Encryption/ExplicitEncryptionLibMongoCryptController.cs +++ b/src/MongoDB.Driver/Encryption/ExplicitEncryptionLibMongoCryptController.cs @@ -103,7 +103,7 @@ public Guid CreateDataKey( { var wrappedKeyBytes = ProcessStates(context, _keyVaultNamespace.DatabaseNamespace.DatabaseName, cancellationToken); - var wrappedKeyDocument = new RawBsonDocument(wrappedKeyBytes); + var wrappedKeyDocument = BsonSerializer.Deserialize(wrappedKeyBytes); var keyId = UnwrapKeyId(wrappedKeyDocument); _keyVaultCollection.Value.InsertOne(wrappedKeyDocument, cancellationToken: cancellationToken); @@ -132,7 +132,7 @@ public async Task CreateDataKeyAsync( { var wrappedKeyBytes = await ProcessStatesAsync(context, _keyVaultNamespace.DatabaseNamespace.DatabaseName, cancellationToken).ConfigureAwait(false); - var wrappedKeyDocument = new RawBsonDocument(wrappedKeyBytes); + var wrappedKeyDocument = BsonSerializer.Deserialize(wrappedKeyBytes); var keyId = UnwrapKeyId(wrappedKeyDocument); await _keyVaultCollection.Value.InsertOneAsync(wrappedKeyDocument, cancellationToken: cancellationToken).ConfigureAwait(false); @@ -556,7 +556,7 @@ private BsonValue RenderFilter(FilterDefinition filter) return filter.Render(serializer, registry); } - private Guid UnwrapKeyId(RawBsonDocument wrappedKeyDocument) + private Guid UnwrapKeyId(BsonDocument wrappedKeyDocument) { var keyId = wrappedKeyDocument["_id"].AsBsonBinaryData; if (keyId.SubType != BsonBinarySubType.UuidStandard) @@ -568,8 +568,8 @@ private Guid UnwrapKeyId(RawBsonDocument wrappedKeyDocument) private BsonValue UnwrapValue(byte[] encryptedWrappedBytes) { - var rawDocument = new RawBsonDocument(encryptedWrappedBytes); - return rawDocument["v"]; + var bsonDocument = BsonSerializer.Deserialize(encryptedWrappedBytes); + return bsonDocument["v"]; } } } From 1b9d98badbaec78942cd2f5800afd9d25cf5f0bd Mon Sep 17 00:00:00 2001 From: rstam Date: Thu, 23 May 2024 09:33:54 -0800 Subject: [PATCH 22/38] CSHARP-5081: Support SelectMany inside Project/Select. --- ...essionToAggregationExpressionTranslator.cs | 1 + ...MethodToAggregationExpressionTranslator.cs | 65 ++++++ .../Jira/CSharp5081Tests.cs | 210 ++++++++++++++++++ 3 files changed, 276 insertions(+) create mode 100644 src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/SelectManyMethodToAggregationExpressionTranslator.cs create mode 100644 tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5081Tests.cs diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs index 0a11578635f..0c57aeddf29 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs @@ -68,6 +68,7 @@ public static AggregationExpression Translate(TranslationContext context, Method case "Reverse": return ReverseMethodToAggregationExpressionTranslator.Translate(context, expression); case "Round": return RoundMethodToAggregationExpressionTranslator.Translate(context, expression); case "Select": return SelectMethodToAggregationExpressionTranslator.Translate(context, expression); + case "SelectMany": return SelectManyMethodToAggregationExpressionTranslator.Translate(context, expression); case "SetEquals": return SetEqualsMethodToAggregationExpressionTranslator.Translate(context, expression); case "Shift": return ShiftMethodToAggregationExpressionTranslator.Translate(context, expression); case "Split": return SplitMethodToAggregationExpressionTranslator.Translate(context, expression); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/SelectManyMethodToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/SelectManyMethodToAggregationExpressionTranslator.cs new file mode 100644 index 00000000000..16e21fa8267 --- /dev/null +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/SelectManyMethodToAggregationExpressionTranslator.cs @@ -0,0 +1,65 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Linq.Expressions; +using System.Reflection; +using MongoDB.Bson; +using MongoDB.Driver.Linq.Linq3Implementation.Ast.Expressions; +using MongoDB.Driver.Linq.Linq3Implementation.Misc; +using MongoDB.Driver.Linq.Linq3Implementation.Reflection; + +namespace MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToAggregationExpressionTranslators.MethodTranslators +{ + internal static class SelectManyMethodToAggregationExpressionTranslator + { + private static readonly MethodInfo[] __selectManyMethods = + { + EnumerableMethod.SelectMany, + QueryableMethod.SelectMany + }; + + public static AggregationExpression Translate(TranslationContext context, MethodCallExpression expression) + { + var method = expression.Method; + var arguments = expression.Arguments; + + if (method.IsOneOf(__selectManyMethods)) + { + var sourceExpression = arguments[0]; + var sourceTranslation = ExpressionToAggregationExpressionTranslator.TranslateEnumerable(context, sourceExpression); + NestedAsQueryableHelper.EnsureQueryableMethodHasNestedAsQueryableSource(expression, sourceTranslation); + var selectorLambda = ExpressionHelper.UnquoteLambdaIfQueryableMethod(method, arguments[1]); + var selectorParameter = selectorLambda.Parameters[0]; + var selectorParameterSerializer = ArraySerializerHelper.GetItemSerializer(sourceTranslation.Serializer); + var selectorParameterSymbol = context.CreateSymbol(selectorParameter, selectorParameterSerializer); + var selectorContext = context.WithSymbol(selectorParameterSymbol); + var selectorTranslation = ExpressionToAggregationExpressionTranslator.Translate(selectorContext, selectorLambda.Body); + var asVar = selectorParameterSymbol.Var; + var valueVar = AstExpression.Var("value"); + var thisVar = AstExpression.Var("this"); + var ast = AstExpression.Reduce( + input: AstExpression.Map( + input: sourceTranslation.Ast, + @as: asVar, + @in: selectorTranslation.Ast), + initialValue: new BsonArray(), + @in: AstExpression.ConcatArrays(valueVar, thisVar)); + return new AggregationExpression(expression, ast, selectorTranslation.Serializer); + } + + throw new ExpressionNotSupportedException(expression); + } + } +} diff --git a/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5081Tests.cs b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5081Tests.cs new file mode 100644 index 00000000000..3a62fd495ed --- /dev/null +++ b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5081Tests.cs @@ -0,0 +1,210 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using MongoDB.Driver.Linq; +using MongoDB.TestHelpers.XunitExtensions; +using Xunit; + +namespace MongoDB.Driver.Tests.Linq.Linq3Implementation.Jira +{ + public class CSharp5081Tests : Linq3IntegrationTest + { + [Theory] + [ParameterAttributeData] + public void SelectMany_chained_should_work( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .SelectMany(series => series.Books) + .SelectMany(book => book.Chapters) + .Select(chapter => chapter.Title); + + var stages = Translate(collection, queryable); + if (linqProvider == LinqProvider.V2) + { + AssertStages( + stages, + "{ $unwind : '$Books' }", + "{ $project : { Books : '$Books', _id : 0 } }", + "{ $unwind : '$Books.Chapters' }", + "{ $project : { Chapters : '$Books.Chapters', _id : 0 } }", + "{ $project : { Title : '$Chapters.Title', _id : 0 } }"); + } + else + { + AssertStages( + stages, + "{ $project : { _v : '$Books', _id : 0 } }", + "{ $unwind : '$_v' }", + "{ $project : { _v : '$_v.Chapters', _id : 0 } }", + "{ $unwind : '$_v' }", + "{ $project : { _v : '$_v.Title', _id : 0 } }"); + } + + var results = queryable.ToList(); + results.Should().Equal( + "Book 1 Chapter 1", + "Book 1 Chapter 2", + "Book 2 Chapter 1", + "Book 2 Chapter 2", + "Book 3 Chapter 1", + "Book 3 Chapter 2"); + } + + [Theory] + [ParameterAttributeData] + public void SelectMany_nested_should_work( + [Values(false, true)] bool withNestedAsQueryable, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = withNestedAsQueryable ? + collection.AsQueryable().SelectMany(series => series.Books.AsQueryable().SelectMany(book => book.Chapters.Select(chapter => chapter.Title))) : + collection.AsQueryable().SelectMany(series => series.Books.SelectMany(book => book.Chapters.Select(chapter => chapter.Title))); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages( + stages, + "{ $project : { _v : { $reduce : { input : { $map : { input : '$Books', as : 'book', in : '$$book.Chapters.Title' } }, initialValue : [], in : { $concatArrays : ['$$value', '$$this'] } } }, _id : 0 } }", + "{ $unwind : '$_v' }"); + + var results = queryable.ToList(); + results.Should().Equal( + "Book 1 Chapter 1", + "Book 1 Chapter 2", + "Book 2 Chapter 1", + "Book 2 Chapter 2", + "Book 3 Chapter 1", + "Book 3 Chapter 2"); + } + } + + [Theory] + [ParameterAttributeData] + public void Aggregate_Project_SelectMany_should_work( + [Values(false, true)] bool withNestedAsQueryable, + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var aggregate = withNestedAsQueryable ? + collection.Aggregate().Project(Series => Series.Books.AsQueryable().SelectMany(book => book.Chapters.Select(chapter => chapter.Title)).ToList()) : + collection.Aggregate().Project(Series => Series.Books.SelectMany(book => book.Chapters.Select(chapter => chapter.Title)).ToList()); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, aggregate)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, aggregate); + AssertStages( + stages, + "{ $project : { _v : { $reduce : { input : { $map : { input : '$Books', as : 'book', in : '$$book.Chapters.Title' } }, initialValue : [], in : { $concatArrays : ['$$value', '$$this'] } } }, _id : 0 } }"); + + var results = aggregate.ToList(); + results.Should().HaveCount(2); + results[0].Should().Equal( + "Book 1 Chapter 1", + "Book 1 Chapter 2", + "Book 2 Chapter 1", + "Book 2 Chapter 2"); + results[1].Should().Equal( + "Book 3 Chapter 1", + "Book 3 Chapter 2"); + } + } + + private IMongoCollection GetCollection(LinqProvider linqProvider) + { + var collection = GetCollection("series", linqProvider); + var document1 = new Series + { + Id = 1, + Books = new List + { + new Book + { + Title = "Book1", + Chapters = new List + { + new Chapter { Title = "Book 1 Chapter 1"}, + new Chapter { Title = "Book 1 Chapter 2"} + } + }, + new Book + { + Title = "Book2", + Chapters = new List + { + new Chapter { Title = "Book 2 Chapter 1"}, + new Chapter { Title = "Book 2 Chapter 2"} + } + } + } + }; + var document2 = new Series + { + Id = 2, + Books = new List + { + new Book + { + Title = "Book3", + Chapters = new List + { + new Chapter { Title = "Book 3 Chapter 1"}, + new Chapter { Title = "Book 3 Chapter 2"} + } + } + } + }; + CreateCollection(collection, document1, document2); + return collection; + } + + public class Series + { + public int Id { get; set; } + public List Books { get; set; } + } + + public class Book + { + public string Title { get; set; } + public List Chapters { get; set; } + } + + public class Chapter + { + public string Title { get; set; } + } + } +} From 56d0ababe94aa66175bdfb3c93b6fe5390170f92 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Thu, 30 May 2024 16:45:47 -0700 Subject: [PATCH 23/38] Use index.json to find base url of packages search service. (#1334) --- evergreen/push-packages.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/evergreen/push-packages.sh b/evergreen/push-packages.sh index 1cafa221452..81b64701c76 100644 --- a/evergreen/push-packages.sh +++ b/evergreen/push-packages.sh @@ -2,16 +2,18 @@ set -o errexit # Exit the script with error if any of the commands fail set +o xtrace # Disable tracing. +# querying nuget source to find search base url +packages_search_url=$(curl -X GET -s "${PACKAGES_SOURCE}" | jq -r 'first(.resources[] | select(."@type"=="SearchQueryService") | ."@id")') + wait_until_package_is_available () { package=$1 version=$2 - query_url="${PACKAGES_SOURCE%index.json}query" resp="" count=0 - echo "Checking package availability: ${package}:${version} at ${query_url}" + echo "Checking package availability: ${package}:${version} at ${packages_search_url}" while [ -z "$resp" ] && [ $count -le 40 ]; do - resp=$(curl -X GET -s "$query_url?prerelease=true&take=1&q=PackageId:$package" | jq --arg jq_version "$version" '.data[0].versions[] | select(.version==$jq_version) | .version') + resp=$(curl -X GET -s "$packages_search_url?prerelease=true&take=1&q=PackageId:$package" | jq --arg jq_version "$version" '.data[0].versions[] | select(.version==$jq_version) | .version') if [ -z "$resp" ]; then echo "sleeping for 15 seconds..." sleep 15 From 6b2537a20170940b5dd1fc14d603c1e9140a30d3 Mon Sep 17 00:00:00 2001 From: BorisDog Date: Fri, 31 May 2024 13:38:14 -0700 Subject: [PATCH 24/38] CSHARP-5075: Add SK test variant to EG (#1320) --- evergreen/evergreen.yml | 28 ++++++++++++++++++++++++++++ evergreen/run-sk.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 evergreen/run-sk.sh diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index c86fea85ed8..74054802314 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -768,6 +768,19 @@ functions: FRAMEWORK=${FRAMEWORK} \ evergreen/run-tests.sh + run-test-SK: + - command: shell.exec + params: + shell: bash + working_dir: mongo-csharp-driver + env: + ATLAS_SK: ${ATLAS_SK} + script: | + ${PREPARE_SHELL} + echo "cloning sk rep" + git clone https://github.com/microsoft/semantic-kernel.git + ./evergreen/run-sk.sh + start-kms-mock-servers: - command: shell.exec params: @@ -1786,6 +1799,11 @@ tasks: export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' ./evergreen/run-csfle-gcp-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + - name: test-SK + commands: + - func: install-dotnet + - func: run-test-SK + - name: build-packages commands: - func: install-dotnet @@ -2638,6 +2656,16 @@ buildvariants: variant: ".build-packages" ## add dependency onto packages smoke test once it implemented +- matrix_name: test-SK + matrix_spec: + os: "ubuntu-2004" + display_name: "Semantic Kernel Connector" + tasks: + - name: test-SK + depends_on: + - name: push-packages-myget + variant: ".push-packages-myget" + - matrix_name: push-packages-nuget matrix_spec: os: "ubuntu-2004" diff --git a/evergreen/run-sk.sh b/evergreen/run-sk.sh new file mode 100644 index 00000000000..a662dbcbe86 --- /dev/null +++ b/evergreen/run-sk.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -o errexit # Exit the script with error if any of the commands fail +set +o xtrace # Disable tracing. + +cd ./semantic-kernel/dotnet/ + +# Can't modify packageSourceMapping via command line yet (https://github.com/NuGet/Home/issues/10735), therefore using nuget.custom.config +echo "Creating nuget.custom.config" +cat > nuget.custom.config << EOL + + + + + + + + + + +EOL + +# Update mongodb version +echo Update MongoDB Driver version to "$PACKAGE_VERSION" +sed -i -e 's/PackageVersion Include="MongoDB.Driver" Version=".\+"/PackageVersion Include="MongoDB.Driver" Version="'"$PACKAGE_VERSION"'"/g' Directory.Packages.props + +echo "MongoDB Driver version updated" + +# Set SkipReason to null to enable integration tests +sed -i -e 's/"MongoDB Atlas cluster is required"/null/g' ./src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBMemoryStoreTests.cs + +dotnet clean +dotnet restore --configfile nuget.custom.config +echo "restored" + +# Run unit tests +dotnet test ./src/Connectors/Connectors.UnitTests/Connectors.UnitTests.csproj --filter SemanticKernel.Connectors.UnitTests.MongoDB.MongoDBMemoryStoreTests --no-restore + +# Run integration tests - Currently Fails +#MongoDB__ConnectionString="$ATLAS_SK" +#dotnet test ./src/IntegrationTests/IntegrationTests.csproj --filter SemanticKernel.IntegrationTests.Connectors.MongoDB.MongoDBMemoryStoreTests From 3384185a7892663786cada33dde261421e5bc616 Mon Sep 17 00:00:00 2001 From: James Kovacs Date: Fri, 31 May 2024 16:39:17 -0600 Subject: [PATCH 25/38] CSHARP-2187: Flatten CombinedProjectionDefinitions to reduce recursive calls to Render. (#1333) --- .../ProjectionDefinitionBuilder.cs | 13 ++++++++++++- .../ProjectionDefinitionBuilderTests.cs | 19 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/MongoDB.Driver/ProjectionDefinitionBuilder.cs b/src/MongoDB.Driver/ProjectionDefinitionBuilder.cs index 5fc7d5c4cf3..34971cd803d 100644 --- a/src/MongoDB.Driver/ProjectionDefinitionBuilder.cs +++ b/src/MongoDB.Driver/ProjectionDefinitionBuilder.cs @@ -789,7 +789,18 @@ internal sealed class CombinedProjectionDefinition : ProjectionDefiniti public CombinedProjectionDefinition(IEnumerable> projections) { - _projections = Ensure.IsNotNull(projections, nameof(projections)).ToList(); + // Unwind CombinedProjectionDefinitions to avoid deep recursion on Render + _projections = Ensure.IsNotNull(projections, nameof(projections)) + .Aggregate(new List>(), (current, projection) => + { + if (projection is CombinedProjectionDefinition combinedProjection) + { + current.AddRange(combinedProjection._projections); + } else + current.Add(projection); + return current; + }) + .ToList(); } public override BsonDocument Render(IBsonSerializer sourceSerializer, IBsonSerializerRegistry serializerRegistry, LinqProvider linqProvider) diff --git a/tests/MongoDB.Driver.Tests/ProjectionDefinitionBuilderTests.cs b/tests/MongoDB.Driver.Tests/ProjectionDefinitionBuilderTests.cs index 5f75bb8f8af..f3320e3723c 100644 --- a/tests/MongoDB.Driver.Tests/ProjectionDefinitionBuilderTests.cs +++ b/tests/MongoDB.Driver.Tests/ProjectionDefinitionBuilderTests.cs @@ -14,6 +14,7 @@ */ using System.Linq; +using System.Text; using FluentAssertions; using MongoDB.Bson; using MongoDB.Bson.Serialization; @@ -61,6 +62,24 @@ public void Combine_with_redundant_fields_using_extension_method() Assert(projection, "{LastName: 0, fn: 1}"); } + [Fact] + public void Combine_many_fields_should_not_overflow_stack_on_Render() + { + var subject = CreateSubject(); + + var projection = subject.Include(x => x.FirstName); + var expectedProjection = new StringBuilder("{fn: 1"); + for (int i = 0; i < 10000; i++) + { + var field = $"Field{i}"; + projection = projection.Include(field); + expectedProjection.Append($", {field}: 1"); + } + expectedProjection.Append("}"); + + Assert(projection, expectedProjection.ToString()); + } + [Fact] public void ElemMatch() { From e74b6320d2c432f9cfab74c368dd4d09bcf982fe Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:08:57 -0700 Subject: [PATCH 26/38] CSHARP-4610: OIDC: Automatic token acquisition for GCP Identity Provider (#1336) --- .../Specifications/auth/AuthTestRunner.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/MongoDB.Driver.Tests/Specifications/auth/AuthTestRunner.cs b/tests/MongoDB.Driver.Tests/Specifications/auth/AuthTestRunner.cs index ac0e90175de..b04d198d19c 100644 --- a/tests/MongoDB.Driver.Tests/Specifications/auth/AuthTestRunner.cs +++ b/tests/MongoDB.Driver.Tests/Specifications/auth/AuthTestRunner.cs @@ -51,11 +51,6 @@ public void RunTestDefinition(JsonDrivenTestCase testCase) throw new SkipException("Test skipped because CANONICALIZE_HOST_NAME is not supported."); } - if (connectionString.Contains("ENVIRONMENT:gcp")) - { - throw new SkipException("Test skipped because ENVIRONMENT:gcp is not supported."); - } - try { mongoCredential = MongoClientSettings.FromConnectionString(connectionString).Credential; From a5aadd53c5b06ecc01a0f5ae923a4a3b3aae5806 Mon Sep 17 00:00:00 2001 From: BorisDog Date: Wed, 5 Jun 2024 14:21:41 -0700 Subject: [PATCH 27/38] CSHARP-5095: Generate ssdlc_compliance_report.md (#1337) --- evergreen/evergreen.yml | 49 +++++++++++++-- evergreen/generate-ssdlc-report.sh | 35 +++++++++++ evergreen/template_ssdlc_compliance_report.md | 59 +++++++++++++++++++ 3 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 evergreen/generate-ssdlc-report.sh create mode 100644 evergreen/template_ssdlc_compliance_report.md diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 74054802314..c376efef5a3 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -237,6 +237,32 @@ functions: params: file: mo-expansion.yml + generate-ssdlc-report: + - command: shell.exec + params: + working_dir: "mongo-csharp-driver" + env: + PRODUCT_NAME: "mongo-csharp-driver" + github_commit: ${github_commit} + script: | + ${PREPARE_SHELL} + ./evergreen/generate-ssdlc-report.sh + - command: ec2.assume_role + params: + role_arn: ${UPLOAD_SSDLC_RELEASE_ASSETS_ROLE_ARN} + - command: s3.put + params: + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + local_file: ./mongo-csharp-driver/artifacts/ssdlc/ssdlc_compliance_report.md + remote_file: mongo-csharp-driver/${PACKAGE_VERSION}/ssdlc_compliance_report.md + bucket: csharp-driver-release-assets + region: us-west-2 + permissions: private + content_type: text/markdown + display_name: ssdlc_compliance_report.md + ocsp-bootstrap-mongo-orchestration: - command: shell.exec params: @@ -877,7 +903,7 @@ functions: params: key_id: ${papertrail_key_id} secret_key: ${papertrail_secret_key} - product: ${PRODUCT_NAME} + product: "mongo-csharp-driver" version: ${PACKAGE_VERSION} filenames: - "mongo-csharp-driver/artifacts/nuget/MongoDB.Bson.${PACKAGE_VERSION}.nupkg" @@ -1818,9 +1844,6 @@ tasks: vars: PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json" PACKAGES_SOURCE_KEY: ${nuget_api_key} - - func: trace-artifacts - vars: - PRODUCT_NAME: "mongo-csharp-driver" - name: push-packages-myget commands: @@ -1837,6 +1860,12 @@ tasks: - func: build-apidocs - func: upload-apidocs + - name: generate-ssdlc-reports + commands: + - func: download-packages + - func: trace-artifacts + - func: generate-ssdlc-report + - name: validate-apidocs commands: - func: install-dotnet @@ -2691,3 +2720,15 @@ buildvariants: - name: build-packages variant: ".build-packages" ## add dependency onto packages smoke test once it implemented + +- matrix_name: ssdlc-reports + matrix_spec: + os: "ubuntu-2004" + display_name: "SSDLC Reports" + tags: ["release-tag"] + tasks: + - name: generate-ssdlc-reports + git_tag_only: true + depends_on: + - name: push-packages-nuget + variant: ".push-packages" \ No newline at end of file diff --git a/evergreen/generate-ssdlc-report.sh b/evergreen/generate-ssdlc-report.sh new file mode 100644 index 00000000000..e28a8958dc1 --- /dev/null +++ b/evergreen/generate-ssdlc-report.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail + +# Environment variables used as input: +# PRODUCT_NAME +# PACKAGE_VERSION +# github_commit + +echo "$PRODUCT_NAME" +echo "$PACKAGE_VERSION" +echo "$github_commit" + +echo "Creating SSDLC reports" + +declare -r SSDLC_PATH="./artifacts/ssdlc" +mkdir -p "${SSDLC_PATH}" + +echo "Creating SSDLC compliance report" +declare -r TEMPLATE_SSDLC_REPORT_PATH="./evergreen/template_ssdlc_compliance_report.md" +declare -r SSDLC_REPORT_PATH="${SSDLC_PATH}/ssdlc_compliance_report.md" +cp "${TEMPLATE_SSDLC_REPORT_PATH}" "${SSDLC_REPORT_PATH}" + +declare -a SED_EDIT_IN_PLACE_OPTION +if [[ "$OSTYPE" == "darwin"* ]]; then + SED_EDIT_IN_PLACE_OPTION=(-i '') +else + SED_EDIT_IN_PLACE_OPTION=(-i) +fi +sed "${SED_EDIT_IN_PLACE_OPTION[@]}" \ + -e "s/\${PRODUCT_NAME}/${PRODUCT_NAME}/g" \ + -e "s/\${PACKAGE_VERSION}/$PACKAGE_VERSION/g" \ + -e "s/\${github_commit}/$github_commit/g" \ + -e "s/\${REPORT_DATE_UTC}/$(date -u +%Y-%m-%d)/g" \ + "${SSDLC_REPORT_PATH}" +ls "${SSDLC_REPORT_PATH}" \ No newline at end of file diff --git a/evergreen/template_ssdlc_compliance_report.md b/evergreen/template_ssdlc_compliance_report.md new file mode 100644 index 00000000000..8c882d89a0e --- /dev/null +++ b/evergreen/template_ssdlc_compliance_report.md @@ -0,0 +1,59 @@ +# ${PRODUCT_NAME} SSDLC compliance report + +This report is available +here. + + + + + + + + + + + + + + +
Product name${PRODUCT_NAME}
Product version${PACKAGE_VERSION}
Report date, UTC${REPORT_DATE_UTC}
+ +## Release creator + +This information is available in multiple ways: + + + + + + + + + + +
Evergreen + See the "Submitted by" field in Evergreen release patch. +
Papertrail + Refer to data in Papertrail. There is currently no official way to serve that data. +
+ +## Process document + +Blocked on . + +The MongoDB SSDLC policy is available at +. + +## Third-darty dependency information + +There are no dependencies to report vulnerabilities of. +Our [SBOM](https://docs.devprod.prod.corp.mongodb.com/mms/python/src/sbom/silkbomb/docs/CYCLONEDX/) lite +is . + +## Static analysis findings + +Coverity static analysis report is available here, under mongodb-csharp-driver project. + +## Signature information + +Blocked on . From 2f5beb8cc114f264530f2331f187fd5d64432ebe Mon Sep 17 00:00:00 2001 From: BorisDog Date: Wed, 5 Jun 2024 15:53:10 -0700 Subject: [PATCH 28/38] CSHARP-4807: Support serialization of Memory and ReadOnlyMemory (#1330) --- .../CollectionsSerializationProvider.cs | 3 +- .../Serializers/MemorySerializer.cs | 334 ++++++++++++++++++ .../Serializers/PrimitivesArrayReader.cs | 226 ++++++++++++ .../Serializers/PrimitivesArrayWriter.cs | 138 ++++++++ tests/BuildProps/Tests.Build.props | 2 +- .../Serializers/MemorySerializerTests.cs | 334 ++++++++++++++++++ 6 files changed, 1035 insertions(+), 2 deletions(-) create mode 100644 src/MongoDB.Bson/Serialization/Serializers/MemorySerializer.cs create mode 100644 src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayReader.cs create mode 100644 src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayWriter.cs create mode 100644 tests/MongoDB.Bson.Tests/Serialization/Serializers/MemorySerializerTests.cs diff --git a/src/MongoDB.Bson/Serialization/CollectionsSerializationProvider.cs b/src/MongoDB.Bson/Serialization/CollectionsSerializationProvider.cs index c9496929852..8224dd63a76 100644 --- a/src/MongoDB.Bson/Serialization/CollectionsSerializationProvider.cs +++ b/src/MongoDB.Bson/Serialization/CollectionsSerializationProvider.cs @@ -20,7 +20,6 @@ using System.Dynamic; using System.Linq; using System.Reflection; -using System.Runtime.InteropServices; using MongoDB.Bson.Serialization.Serializers; namespace MongoDB.Bson.Serialization @@ -43,6 +42,8 @@ static CollectionsSerializationProvider() { typeof(Queue<>), typeof(QueueSerializer<>) }, { typeof(ReadOnlyCollection<>), typeof(ReadOnlyCollectionSerializer<>) }, { typeof(Stack<>), typeof(StackSerializer<>) }, + { typeof(Memory<>), typeof(MemorySerializer<>) }, + { typeof(ReadOnlyMemory<>), typeof(ReadonlyMemorySerializer<>) } }; } diff --git a/src/MongoDB.Bson/Serialization/Serializers/MemorySerializer.cs b/src/MongoDB.Bson/Serialization/Serializers/MemorySerializer.cs new file mode 100644 index 00000000000..9504f871493 --- /dev/null +++ b/src/MongoDB.Bson/Serialization/Serializers/MemorySerializer.cs @@ -0,0 +1,334 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using MongoDB.Bson.IO; + +namespace MongoDB.Bson.Serialization.Serializers +{ + /// + /// Represents a serializer for . + /// + /// The type of the item. Only primitive numeric types are supported. + public sealed class ReadonlyMemorySerializer : MemorySerializerBase> + { + /// + /// Initializes a new instance of the class. + /// + public ReadonlyMemorySerializer() : base() + { + } + + /// + /// Initializes a new instance of the class. + /// + public ReadonlyMemorySerializer(BsonType representation) : base(representation) + { + } + + /// + /// Returns a serializer that has been reconfigured with the specified representation. + /// + /// The representation. + /// The reconfigured serializer. + public override MemorySerializerBase> WithRepresentation(BsonType representation) + { + if (representation == Representation) + { + return this; + } + else + { + return new ReadonlyMemorySerializer(representation); + } + } + + /// + protected override ReadOnlyMemory CreateMemory(TItem[] items) => items; + + /// + protected override Memory GetMemory(ReadOnlyMemory memory) => MemoryMarshal.AsMemory(memory); + } + + /// + /// Represents a serializer for . + /// + /// The type of the item. Only primitive numeric types are supported. + public sealed class MemorySerializer : MemorySerializerBase> + { + /// + /// Initializes a new instance of the class. + /// + public MemorySerializer() : base() + { + } + + /// + /// Initializes a new instance of the class. + /// + public MemorySerializer(BsonType representation) : base(representation) + { + } + + /// + /// Returns a serializer that has been reconfigured with the specified representation. + /// + /// The representation. + /// The reconfigured serializer. + public override MemorySerializerBase> WithRepresentation(BsonType representation) + { + if (representation == Representation) + { + return this; + } + else + { + return new MemorySerializer(representation); + } + } + + /// + protected override Memory CreateMemory(TItem[] items) => items; + + /// + protected override Memory GetMemory(Memory memory) => memory; + } + + /// + /// Represents an abstract base class for and serializers. + /// + /// The type of the item. Only primitive numeric types are supported. + /// The type of the memory struct. + public abstract class MemorySerializerBase : StructSerializerBase, IRepresentationConfigurable> + where TMemory : struct + { + private static readonly bool __isByte = (typeof(TItem) == typeof(byte)); + + private readonly Func _readItems; + private readonly Action> _writeItems; + + /// + public BsonType Representation { get; } + + // constructors + /// + /// Initializes a new instance of the class. + /// + public MemorySerializerBase(BsonType representation) + { + if (representation != BsonType.Array && + !(__isByte && representation == BsonType.Binary)) + { + throw new ArgumentOutOfRangeException(nameof(representation)); + } + + (_readItems, _writeItems) = GetReaderAndWriter(); + Representation = representation; + } + + /// + /// Initializes a new instance of the class. + /// + public MemorySerializerBase() : + this(__isByte ? BsonType.Binary : BsonType.Array) // Match the serialization behavior for arrays + { + } + + // public methods + /// + public override TMemory Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + var reader = context.Reader; + + var bsonType = reader.GetCurrentBsonType(); + switch (bsonType) + { + case BsonType.Array: + var items = _readItems(reader); + + return CreateMemory(items); + case BsonType.Binary: + if (!__isByte) + { + throw CreateCannotDeserializeFromBsonTypeException(bsonType); + } + var bytes = reader.ReadBytes(); + return CreateMemory(bytes as TItem[]); + default: + throw CreateCannotDeserializeFromBsonTypeException(bsonType); + } + } + + /// + public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, TMemory value) + { + var memory = GetMemory(value); + + switch (Representation) + { + case BsonType.Array: + _writeItems(context.Writer, memory); + break; + case BsonType.Binary: + var bytesMemory = Unsafe.As, Memory>(ref memory); + var bytes = MemoryMarshal.AsBytes(bytesMemory.Span); + context.Writer.WriteBytes(bytes.ToArray()); + break; + default: + throw new NotSupportedException(nameof(Representation)); + } + } + + /// + public abstract MemorySerializerBase WithRepresentation(BsonType representation); + + // explicit interface implementations + IBsonSerializer IRepresentationConfigurable.WithRepresentation(BsonType representation) => + WithRepresentation(representation); + + /// + /// Creates the Memory{TITem} structure. + /// + /// The items to initialize the resulting instance with. + /// The created memory structure. + protected abstract TMemory CreateMemory(TItem[] items); + + /// + /// Get the memory structure from TMemory instance. + /// + /// The Memory{TITem} structure. + protected abstract Memory GetMemory(TMemory memory); + + private static (Func reader, Action> writer) GetReaderAndWriter() + { + Func readItems; + Action> writeItems; + + switch (typeof(TItem)) + { + case var t when t == typeof(bool): + readItems = reader => PrimitivesArrayReader.ReadBool(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteBool(writer, span); + }; + break; + case var t when t == typeof(sbyte): + readItems = reader => PrimitivesArrayReader.ReadInt8(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteInt8(writer, span); + }; + break; + case var t when t == typeof(byte): + readItems = reader => PrimitivesArrayReader.ReadUInt8(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteUInt8(writer, span); + }; + break; + case var t when t == typeof(char): + readItems = reader => PrimitivesArrayReader.ReadChar(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteChar(writer, span); + }; + break; + case var t when t == typeof(short): + readItems = reader => PrimitivesArrayReader.ReadInt16(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteInt16(writer, span); + }; + break; + case var t when t == typeof(ushort): + readItems = reader => PrimitivesArrayReader.ReadUInt16(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteUInt16(writer, span); + }; + break; + case var t when t == typeof(int): + readItems = reader => PrimitivesArrayReader.ReadInt32(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteInt32(writer, span); + }; + break; + case var t when t == typeof(uint): + readItems = reader => PrimitivesArrayReader.ReadUInt32(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteUInt32(writer, span); + }; + break; + case var t when t == typeof(long): + readItems = reader => PrimitivesArrayReader.ReadInt64(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteInt64(writer, span); + }; + break; + case var t when t == typeof(ulong): + readItems = reader => PrimitivesArrayReader.ReadUInt64(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteUInt64(writer, span); + }; + break; + case var t when t == typeof(float): + readItems = reader => PrimitivesArrayReader.ReadSingles(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteSingles(writer, span); + }; + break; + case var t when t == typeof(double): + readItems = reader => PrimitivesArrayReader.ReadDoubles(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteDoubles(writer, span); + }; + break; + case var t when t == typeof(decimal): + readItems = reader => PrimitivesArrayReader.ReadDecimal128(reader) as TItem[]; + writeItems = (writer, memory) => + { + var span = Unsafe.As, Memory>(ref memory).Span; + PrimitivesArrayWriter.WriteDecimal128(writer, span); + }; + break; + default: + throw new NotSupportedException($"Not supported memory type {typeof(TItem)}. Only primitive numeric types are supported."); + }; + + return (readItems, writeItems); + } + } +} diff --git a/src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayReader.cs b/src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayReader.cs new file mode 100644 index 00000000000..e811edef230 --- /dev/null +++ b/src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayReader.cs @@ -0,0 +1,226 @@ +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using MongoDB.Bson.IO; + +namespace MongoDB.Bson.Serialization.Serializers +{ + internal static class PrimitivesArrayReader + { + private enum ConversionType + { + BoolToBool, + DoubleToSingle, + DoubleToDouble, + Decimal128ToDecimal128, + Int32ToInt8, + Int32ToUInt8, + Int32ToInt16, + Int32ToUInt16, + Int32ToChar, + Int32ToInt32, + Int32ToUInt32, + Int64ToInt64, + Int64ToUInt64 + } + + public static bool[] ReadBool(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.BoolToBool); + + public static byte[] ReadInt8(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToInt8); + + public static sbyte[] ReadUInt8(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToUInt8); + + public static short[] ReadInt16(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToInt16); + + public static ushort[] ReadUInt16(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToUInt16); + + public static char[] ReadChar(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToChar); + + public static int[] ReadInt32(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToInt32); + + public static int[] ReadUInt32(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int32ToUInt32); + + public static float[] ReadSingles(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.DoubleToSingle); + + public static double[] ReadDoubles(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.DoubleToDouble); + + public static decimal[] ReadDecimal128(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Decimal128ToDecimal128); + + public static long[] ReadInt64(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int64ToInt64); + + public static ulong[] ReadUInt64(IBsonReader bsonReader) => + ReadBsonArray(bsonReader, ConversionType.Int64ToUInt64); + + private static T[] ReadBsonArray( + IBsonReader bsonReader, + ConversionType conversionType) + { + var (bsonDataType, bsonDataSize) = GetBsonDataTypeAndSize(conversionType); + + var array = bsonReader.ReadRawBsonArray(); + using var buffer = ThreadStaticBuffer.RentBuffer(array.Length); + + var bytes = buffer.Bytes; + array.GetBytes(0, bytes, 0, array.Length); + + var result = new List(); + + var index = 4; // 4 first bytes are array object size in bytes + var maxIndex = array.Length - 1; + + while (index < maxIndex) + { + ValidateBsonType(bsonDataType); + + // Skip name + while (bytes[index] != 0) { index++; }; + index++; // Skip string terminating 0 + + T value = default; + + // Read next item + switch (conversionType) + { + case ConversionType.DoubleToSingle: + { + var v = (float)BitConverter.ToDouble(bytes, index); + + value = Unsafe.As(ref v); + break; + } + case ConversionType.DoubleToDouble: + { + var v = BitConverter.ToDouble(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Decimal128ToDecimal128: + { + var lowBits = (ulong)BitConverter.ToInt64(bytes, index); + var highBits = (ulong)BitConverter.ToInt64(bytes, index + 8); + var v = Decimal128.ToDecimal(Decimal128.FromIEEEBits(highBits, lowBits)); + + value = Unsafe.As(ref v); + break; + } + case ConversionType.BoolToBool: + { + var v = bytes[index] != 0; + + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int32ToInt8: + { + var v = (sbyte)BitConverter.ToInt32(bytes, index); + value = Unsafe.As(ref v); + + break; + } + case ConversionType.Int32ToUInt8: + { + var v = (byte)BitConverter.ToInt32(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int32ToInt16: + { + var v = (short)BitConverter.ToInt32(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int32ToUInt16: + { + var v = (ushort)BitConverter.ToInt32(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int32ToChar: + { + var v = BitConverter.ToChar(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int32ToInt32: + { + var v = BitConverter.ToInt32(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int32ToUInt32: + { + var v = BitConverter.ToUInt32(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int64ToInt64: + { + var v = BitConverter.ToInt64(bytes, index); + value = Unsafe.As(ref v); + break; + } + case ConversionType.Int64ToUInt64: + { + var v = BitConverter.ToUInt64(bytes, index); + value = Unsafe.As(ref v); + break; + } + default: + throw new InvalidOperationException(); + } + + result.Add(value); + + index += bsonDataSize; + } + + ValidateBsonType(BsonType.EndOfDocument); + + return result.ToArray(); + + void ValidateBsonType(BsonType bsonType) + { + if ((BsonType)bytes[index] != bsonType) + { + throw new InvalidOperationException(); + } + } + } + + private static (BsonType, int) GetBsonDataTypeAndSize(ConversionType conversionType) => + conversionType switch + { + ConversionType.BoolToBool => (BsonType.Boolean, 1), + + ConversionType.DoubleToSingle or + ConversionType.DoubleToDouble => (BsonType.Double, 8), + + ConversionType.Int32ToUInt8 or + ConversionType.Int32ToInt8 or + ConversionType.Int32ToUInt16 or + ConversionType.Int32ToInt16 or + ConversionType.Int32ToInt32 or + ConversionType.Int32ToChar or + ConversionType.Int32ToUInt32 => (BsonType.Int32, 4), + + ConversionType.Int64ToInt64 or + ConversionType.Int64ToUInt64 => (BsonType.Int64, 8), + + ConversionType.Decimal128ToDecimal128 => (BsonType.Decimal128, 16), + + _ => throw new NotSupportedException() + }; + } +} diff --git a/src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayWriter.cs b/src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayWriter.cs new file mode 100644 index 00000000000..6295f36ca71 --- /dev/null +++ b/src/MongoDB.Bson/Serialization/Serializers/PrimitivesArrayWriter.cs @@ -0,0 +1,138 @@ +using System; +using MongoDB.Bson.IO; + +namespace MongoDB.Bson.Serialization.Serializers +{ + internal static class PrimitivesArrayWriter + { + public static void WriteBool(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteBoolean(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteInt8(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteUInt8(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteInt16(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteUInt16(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteChar(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteInt32(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteUInt32(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt32((int)span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteInt64(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt64(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteUInt64(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteInt64((long)span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteSingles(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteDouble(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteDoubles(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteDouble(span[i]); + } + bsonWriter.WriteEndArray(); + } + + public static void WriteDecimal128(IBsonWriter bsonWriter, Span span) + { + bsonWriter.WriteStartArray(); + for (int i = 0; i < span.Length; i++) + { + bsonWriter.WriteDecimal128(span[i]); + } + bsonWriter.WriteEndArray(); + } + } +} diff --git a/tests/BuildProps/Tests.Build.props b/tests/BuildProps/Tests.Build.props index 7f76f18e209..e6150099b89 100644 --- a/tests/BuildProps/Tests.Build.props +++ b/tests/BuildProps/Tests.Build.props @@ -14,7 +14,7 @@ - 10 + 12 true diff --git a/tests/MongoDB.Bson.Tests/Serialization/Serializers/MemorySerializerTests.cs b/tests/MongoDB.Bson.Tests/Serialization/Serializers/MemorySerializerTests.cs new file mode 100644 index 00000000000..48f28c6fcc7 --- /dev/null +++ b/tests/MongoDB.Bson.Tests/Serialization/Serializers/MemorySerializerTests.cs @@ -0,0 +1,334 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using FluentAssertions; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Bson.Serialization.Serializers; +using Xunit; + +namespace MongoDB.Bson.Tests.Serialization +{ + public class MemorySerializerTests + { + public class ReadonlyMemoryHolder + { + public ReadOnlyMemory Items { get; set; } + } + + public class ReadonlyMemoryHolderBytesAsArray + { + [BsonRepresentation(BsonType.Array)] + public ReadOnlyMemory Items { get; set; } + } + + public class ReadonlyMemoryHolderBytesAsBinary + { + [BsonRepresentation(BsonType.Binary)] + public ReadOnlyMemory Items { get; set; } + } + + public class MemoryHolder + { + public Memory Items { get; set; } + } + + public class MemoryHolderBytesAsArray + { + [BsonRepresentation(BsonType.Array)] + public Memory Items { get; set; } + } + + public class MemoryHolderBytesAsBinary + { + [BsonRepresentation(BsonType.Binary)] + public Memory Items { get; set; } + } + + public class ArrayHolder + { + public T[] Items { get; set; } + } + + public class ArrayHolderDecimal + { + [BsonRepresentation(BsonType.Decimal128)] + public decimal[] Items { get; set; } + } + + public class MultiHolder + { + public Memory ItemsBytes { get; set; } + public Memory ItemsFloat { get; set; } + public ReadOnlyMemory ItemsInt { get; set; } + public ReadOnlyMemory ItemsDouble { get; set; } + } + + [Theory] + [MemberData(nameof(NonSupportedTestData))] + public void Memory_should_throw_on_non_primitive_numeric_types(T[] notSupportedData) + { + var memoryHolder = new MemoryHolder() { Items = notSupportedData }; + + var exception = Record.Exception(() => memoryHolder.ToBson()); + var e = exception.Should() + .BeOfType().Subject.InnerException.Should() + .BeOfType().Subject.InnerException.Should() + .BeOfType().Subject; + + e.Message.Should().StartWith("Not supported memory type"); + } + + [Theory] + [MemberData(nameof(NonSupportedTestData))] + public void ReadonlyMemory_should_throw_on_non_primitive_numeric_types(T[] notSupportedData) + { + var memoryHolder = new ReadonlyMemoryHolder() { Items = notSupportedData }; + + var exception = Record.Exception(() => memoryHolder.ToBson()); + var e = exception.Should() + .BeOfType().Subject.InnerException.Should() + .BeOfType().Subject.InnerException.Should() + .BeOfType().Subject; + + e.Message.Should().StartWith("Not supported memory type"); + } + + [Theory] + [MemberData(nameof(TestData))] + public void Memory_should_roundtrip_equivalent_to_array(T[] array) + { + var memoryHolder = new MemoryHolder() { Items = array }; + var memoryBson = memoryHolder.ToBson(); + var arrayBson = GetArrayHolderBson(array); + + memoryBson.ShouldAllBeEquivalentTo(arrayBson); + + var memoryHolderMaterialized = BsonSerializer.Deserialize>(memoryBson); + memoryHolderMaterialized.Items.ToArray().ShouldAllBeEquivalentTo(memoryHolder.Items.ToArray()); + } + + [Theory] + [MemberData(nameof(TestData))] + public void ReadonlyMemory_should_roundtrip_equivalent_to_array(T[] array) + { + var memoryHolder = new ReadonlyMemoryHolder() { Items = array }; + var memoryBson = memoryHolder.ToBson(); + var arrayBson = GetArrayHolderBson(array); + + memoryBson.ShouldAllBeEquivalentTo(arrayBson); + + var memoryHolderMaterialized = BsonSerializer.Deserialize>(memoryBson); + memoryHolderMaterialized.Items.ToArray().ShouldAllBeEquivalentTo(memoryHolder.Items.ToArray()); + } + + [Theory] + [MemberData(nameof(TestDataSpecialValues))] + public void Memory_should_roundtrip_special_values(T[] array) + { + var memoryHolder = new MemoryHolder() { Items = array }; + var memoryBson = memoryHolder.ToBson(); + + var memoryHolderMaterialized = BsonSerializer.Deserialize>(memoryBson); + memoryHolderMaterialized.Items.ToArray().ShouldAllBeEquivalentTo(memoryHolder.Items.ToArray()); + } + + [Theory] + [MemberData(nameof(TestDataSpecialValues))] + public void ReadonlyMemory_should_roundtrip_special_values(T[] array) + { + var memoryHolder = new ReadonlyMemoryHolder() { Items = array }; + var memoryBson = memoryHolder.ToBson(); + + var memoryHolderMaterialized = BsonSerializer.Deserialize>(memoryBson); + memoryHolderMaterialized.Items.ToArray().ShouldAllBeEquivalentTo(memoryHolder.Items.ToArray()); + } + + [Theory] + [MemberData(nameof(TestData))] + public void Memory_should_roundtrip_special_values_correctly(T[] array) + { + var memoryHolder = new MemoryHolder() { Items = array }; + var memoryBson = memoryHolder.ToBson(); + + var memoryHolderMaterialized = BsonSerializer.Deserialize>(memoryBson); + memoryHolderMaterialized.Items.ToArray().ShouldAllBeEquivalentTo(memoryHolder.Items.ToArray()); + } + + [Theory] + [MemberData(nameof(NonSupportedRepresentationTypes))] + public void MemorySerializer_should_throw_on_not_supported_representation(T item, BsonType representation) + where T : struct + { + var exception = Record.Exception(() => new MemorySerializer(representation)); + var e = exception.Should().BeOfType().Subject; + e.ParamName.Should().Be("representation"); + } + + [Fact] + public void Memory_should_support_array_representation_for_bytes() + { + var bytes = new byte[] { 1, 2, 3 }; + var memoryHolder = new MemoryHolderBytesAsArray() { Items = bytes }; + + var bsonDocument = BsonSerializer.Deserialize(memoryHolder.ToBson()); + var itemsElement = bsonDocument[nameof(memoryHolder.Items)]; + + itemsElement.BsonType.Should().Be(BsonType.Array); + itemsElement.AsBsonArray.ShouldAllBeEquivalentTo(bytes); + } + + [Fact] + public void ReadonlyMemory_should_support_array_representation_for_bytes() + { + var bytes = new byte[] { 1, 2, 3 }; + var memoryHolder = new ReadonlyMemoryHolderBytesAsArray() { Items = bytes }; + + var bsonDocument = BsonSerializer.Deserialize(memoryHolder.ToBson()); + var itemsElement = bsonDocument[nameof(memoryHolder.Items)]; + + itemsElement.BsonType.Should().Be(BsonType.Array); + itemsElement.AsBsonArray.ShouldAllBeEquivalentTo(bytes); + } + + [Fact] + public void Memory_should_support_binary_representation_for_bytes() + { + var bytes = new byte[] { 1, 2, 3 }; + var memoryHolder = new ReadonlyMemoryHolderBytesAsBinary() { Items = bytes }; + + var bsonDocument = BsonSerializer.Deserialize(memoryHolder.ToBson()); + var itemsElement = bsonDocument[nameof(memoryHolder.Items)]; + + itemsElement.BsonType.Should().Be(BsonType.Binary); + itemsElement.AsByteArray.ShouldAllBeEquivalentTo(bytes); + } + + [Fact] + public void ReadonlyMemory_should_support_binary_representation_for_bytes() + { + var bytes = new byte[] { 1, 2, 3 }; + var memoryHolder = new ReadonlyMemoryHolderBytesAsBinary() { Items = bytes }; + + var bsonDocument = BsonSerializer.Deserialize(memoryHolder.ToBson()); + var itemsElement = bsonDocument[nameof(memoryHolder.Items)]; + + itemsElement.BsonType.Should().Be(BsonType.Binary); + itemsElement.AsByteArray.ShouldAllBeEquivalentTo(bytes); + } + + [Fact] + public void Mixed_memory_types_should_roundtrip_correctly() + { + var multiHolder = new MultiHolder() + { + ItemsBytes = new byte[] { 1, 2, 3 }, + ItemsDouble = new double[] { 1.1, 2.2, 3.3 }, + ItemsFloat = new float[] { 11.1f, 22.2f, 33.3f }, + ItemsInt = new int[] { 10, 100, 1000 } + }; + + var bson = multiHolder.ToBson(); + var materialized = BsonSerializer.Deserialize(bson); + + materialized.ItemsBytes.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsBytes.ToArray()); + materialized.ItemsDouble.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsDouble.ToArray()); + materialized.ItemsFloat.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsFloat.ToArray()); + materialized.ItemsInt.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsInt.ToArray()); + } + + [Fact] + public void Empty_memory_should_roundtrip_correctly() + { + var multiHolder = new MultiHolder() + { + ItemsBytes = new byte[0], + ItemsDouble = new double[0] + }; + + var bson = multiHolder.ToBson(); + var materialized = BsonSerializer.Deserialize(bson); + + materialized.ItemsBytes.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsBytes.ToArray()); + materialized.ItemsDouble.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsDouble.ToArray()); + materialized.ItemsFloat.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsFloat.ToArray()); + materialized.ItemsInt.ToArray().ShouldAllBeEquivalentTo(multiHolder.ItemsInt.ToArray()); + } + + public readonly static IEnumerable TestData = + [ + [ GetArray(i => (bool)( i % 2 == 0)) ], + [ GetArray(i => (sbyte)i) ], + [ GetArray(i => (byte)i) ], + [ GetArray(i => (short)i) ], + [ GetArray(i => (ushort)i) ], + [ GetArray(i => (char)i) ], + [ GetArray(i => (int)i) ], + [ GetArray(i => (uint)i) ], + [ GetArray(i => (long)i) ], + [ GetArray(i => (ulong)i) ], + [ GetArray(i => (float)i) ], + [ GetArray(i => (double)i) ], + [ GetArray(i => (decimal)i) ] + ]; + + public static readonly IEnumerable TestDataSpecialValues = + [ + [ new[] { sbyte.MaxValue, sbyte.MinValue } ], + [ new[] { byte.MaxValue, byte.MinValue } ], + [ new[] { short.MaxValue, short.MinValue } ], + [ new[] { ushort.MaxValue, ushort.MinValue } ], + [ new[] { char.MaxValue, char.MinValue } ], + [ new[] { int.MaxValue, int.MinValue } ], + [ new[] { uint.MaxValue, uint.MinValue } ], + [ new[] { long.MaxValue, long.MinValue } ], + [ new[] { ulong.MaxValue, ulong.MinValue } ], + [ new[] { float.MaxValue, float.MinValue, float.Epsilon, float.NegativeInfinity, float.PositiveInfinity, float.NaN } ], + [ new[] { double.MaxValue, double.MinValue, double.Epsilon, double.NegativeInfinity, double.PositiveInfinity, double.NaN } ], + [ new[] { decimal.MaxValue, decimal.MinValue, decimal.One, decimal.Zero, decimal.MinusOne }] + ]; + + public static readonly IEnumerable NonSupportedTestData = + [ + [ new[] { "str" } ], + [ new[] { new object() }], + [ new[] { new TimeSpan() }] + ]; + + public static IEnumerable NonSupportedRepresentationTypes() + { + foreach (var bsonType in Enum.GetValues(typeof(BsonType)).Cast()) + { + if (bsonType == BsonType.Array) + continue; + + yield return new object[] { 1, bsonType }; + } + } + + private static T[] GetArray(Func converter) => + Enumerable.Range(0, 16).Select(converter).ToArray(); + + private static byte[] GetArrayHolderBson(T[] array) => typeof(T) switch + { + var t when t == typeof(decimal) => (new ArrayHolderDecimal() { Items = array as decimal[] }).ToBson(), + _ => (new ArrayHolder() { Items = array }).ToBson(), + }; + } +} From 2b032890508e689c204505538d5008b9d5b45ad8 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Fri, 7 Jun 2024 15:13:06 -0700 Subject: [PATCH 29/38] CSHARP-5117: EG script to run tests from other repos (#1338) --- evergreen/evergreen.yml | 54 ++++++++++++++++++++++++++++++-- evergreen/run-external-script.sh | 37 ++++++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 evergreen/run-external-script.sh diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index c376efef5a3..92841a98bbb 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -53,6 +53,7 @@ functions: if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin # Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS) + export DOTNET_SDK_PATH=$(cygpath -m $DOTNET_SDK_PATH) else # non windows OSs don't have dotnet in the PATH export PATH=$PATH:/usr/share/dotnet @@ -72,6 +73,7 @@ functions: PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" PACKAGE_VERSION: "$PACKAGE_VERSION" DOTNET_SDK_PATH: "$DOTNET_SDK_PATH" + TEST_RESULTS_PATH: "./mongo-csharp-driver/build/test-results/TEST-*.xml" PREPARE_SHELL: | set -o errexit set -o xtrace @@ -146,6 +148,21 @@ functions: ${PREPARE_SHELL} ${PROJECT_DIRECTORY}/${file} + run-external-script: + - command: subprocess.exec + type: test + params: + working_dir: mongo-csharp-driver + binary: bash + include_expansions_in_env: + - "OS" + - "GIT_REPO" + - "GIT_BRANCH" + - "LOCAL_PATH" + - "SCRIPT" + args: + - evergreen/run-external-script.sh + upload-mo-artifacts: - command: shell.exec params: @@ -210,7 +227,7 @@ functions: upload-test-results: - command: attach.xunit_results params: - file: ./mongo-csharp-driver/build/test-results/TEST-*.xml + file: ${TEST_RESULTS_PATH} bootstrap-mongo-orchestration: - command: shell.exec @@ -1871,6 +1888,23 @@ tasks: - func: install-dotnet - func: build-apidocs + - name: test-odata + commands: + - func: bootstrap-mongo-orchestration + - command: expansions.update + params: + updates: + - key: TEST_RESULTS_PATH + value: "./odata/build/test-results/TEST-*.xml" + - func: run-external-script + vars: + GIT_REPO: "https://github.com/mongodb/mongo-aspnetcore-odata.git" + LOCAL_PATH: ${workdir}/odata + SCRIPT: | + ${PREPARE_SHELL} + DOTNET_SDK_PATH="${DOTNET_SDK_PATH}" bash ./evergreen/install-dependencies.sh + DRIVER_VERSION="${PACKAGE_VERSION}" bash ./evergreen/run-tests.sh + axes: - id: version display_name: MongoDB Version @@ -2731,4 +2765,20 @@ buildvariants: git_tag_only: true depends_on: - name: push-packages-nuget - variant: ".push-packages" \ No newline at end of file + variant: ".push-packages" + +- matrix_name: odata-tests + batchtime: 720 # 12 hours + matrix_spec: + os: ["ubuntu-2004", "windows-64"] + version: ["4.2", "7.0", "latest"] + exclude_spec: + # We do not have MongoDB 4.2 binaries for Ubuntu 2004 + - os: "ubuntu-2004" + version: "4.2" + display_name: "OData tests on ${os} with MongoDB ${version}" + tasks: + - name: test-odata + depends_on: + - name: push-packages-myget + variant: ".push-packages-myget" diff --git a/evergreen/run-external-script.sh b/evergreen/run-external-script.sh new file mode 100644 index 00000000000..c046e63bd27 --- /dev/null +++ b/evergreen/run-external-script.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -o xtrace # Write all commands first to stderr +set -o errexit # Exit the script with error if any of the commands fail + +# clone the repo +rm -rf "${LOCAL_PATH}" +mkdir "${LOCAL_PATH}" +cd "${LOCAL_PATH}" || exit + +git clone -b "${GIT_BRANCH:-main}" --single-branch "${GIT_REPO}" . + +# add/adjust nuget.config pointing to myget so intermediate versions could be restored +if [ -f "./nuget.config" ]; then + echo "Adding myget into nuget.config" + dotnet nuget add source https://www.myget.org/F/mongodb/api/v3/index.json -n myget.org --configfile ./nuget.config +else + echo "Creating custom nuget.config" + cat > "nuget.config" << EOL + + + + + + + + +EOL +fi + +# make files executable +for i in $(find "." -name \*.sh); do + chmod +x $i +done + +# execute the provided script +eval "$SCRIPT" From 25c24f6565dae4d1e6a30979e80b217bf4e310f8 Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:36:44 -0700 Subject: [PATCH 30/38] CSHARP-2377: Enable unit tests that are not being run (#1339) --- tests/MongoDB.Driver.Core.Tests/MongoDB.Driver.Core.Tests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/MongoDB.Driver.Core.Tests/MongoDB.Driver.Core.Tests.csproj b/tests/MongoDB.Driver.Core.Tests/MongoDB.Driver.Core.Tests.csproj index 3520a37b3d7..f035381b02a 100644 --- a/tests/MongoDB.Driver.Core.Tests/MongoDB.Driver.Core.Tests.csproj +++ b/tests/MongoDB.Driver.Core.Tests/MongoDB.Driver.Core.Tests.csproj @@ -18,7 +18,6 @@ 1701;1702; - xUnit1000; xUnit1010; xUnit1013; xUnit1014; From 7b9f51eafd73efad1c055589fcd6d1b0989ea7ab Mon Sep 17 00:00:00 2001 From: Oleksandr Poliakov <31327136+sanych-sun@users.noreply.github.com> Date: Tue, 11 Jun 2024 09:01:04 -0700 Subject: [PATCH 31/38] CSHARP-5119: FullDocument of the ChangeStreamDocument should not throw on BsonNull (#1341) --- src/MongoDB.Driver.Core/ChangeStreamDocument.cs | 5 +++++ .../ChangeStreamDocumentTests.cs | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/MongoDB.Driver.Core/ChangeStreamDocument.cs b/src/MongoDB.Driver.Core/ChangeStreamDocument.cs index 2afdfde967f..5e325239ff3 100644 --- a/src/MongoDB.Driver.Core/ChangeStreamDocument.cs +++ b/src/MongoDB.Driver.Core/ChangeStreamDocument.cs @@ -144,6 +144,11 @@ public TDocument FullDocument // if TDocument is BsonDocument avoid deserializing it again to prevent possible duplicate element name errors if (typeof(TDocument) == typeof(BsonDocument) && BackingDocument.TryGetValue("fullDocument", out var fullDocument)) { + if (fullDocument.IsBsonNull) + { + return default; + } + return (TDocument)(object)fullDocument.AsBsonDocument; } else diff --git a/tests/MongoDB.Driver.Core.Tests/ChangeStreamDocumentTests.cs b/tests/MongoDB.Driver.Core.Tests/ChangeStreamDocumentTests.cs index 79a5de7f451..f63d9e32504 100644 --- a/tests/MongoDB.Driver.Core.Tests/ChangeStreamDocumentTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/ChangeStreamDocumentTests.cs @@ -323,10 +323,12 @@ public void FullDocument_should_allow_duplicate_elements() secondElement.Value.Should().Be(2); } - [Fact] - public void FullDocument_should_return_null_when_not_present() + [Theory] + [InlineData("{ other : 1 }")] + [InlineData("{ other : 1, fullDocument : null }")] + public void FullDocument_should_return_null_when_not_present(string changeDocument) { - var backingDocument = new BsonDocument { { "other", 1 } }; + var backingDocument = BsonDocument.Parse(changeDocument); var subject = CreateSubject(backingDocument: backingDocument); var result = subject.FullDocument; From 5282cb0679fe96195a7b02b99fe700e766e3cc8b Mon Sep 17 00:00:00 2001 From: BorisDog Date: Tue, 11 Jun 2024 16:04:19 -0700 Subject: [PATCH 32/38] CSHARP-5050: Sign release artifacts or tags with MongoDB-managed keys --- evergreen/build-packages.sh | 5 ++++- evergreen/evergreen.yml | 21 ++++++++++++++++++++- evergreen/push-packages.sh | 11 +++++++++++ evergreen/sign-packages.sh | 25 +++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 evergreen/sign-packages.sh diff --git a/evergreen/build-packages.sh b/evergreen/build-packages.sh index f09258efcd3..aa140bd5bf9 100644 --- a/evergreen/build-packages.sh +++ b/evergreen/build-packages.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -o errexit # Exit the script with error if any of the commands fail +# Environment variables used as input: +# PACKAGE_VERSION + if [ -z "$PACKAGE_VERSION" ]; then PACKAGE_VERSION=$(bash ./evergreen/get-version.sh) echo Calculated PACKAGE_VERSION value: "$PACKAGE_VERSION" @@ -9,4 +12,4 @@ fi echo Creating nuget package... dotnet clean ./CSharpDriver.sln -dotnet pack ./CSharpDriver.sln -o ./artifacts/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg -p:ContinuousIntegrationBuild=true +dotnet pack ./CSharpDriver.sln -o ./artifacts/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg -p:ContinuousIntegrationBuild=true \ No newline at end of file diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index 92841a98bbb..a87115b75bd 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -1008,12 +1008,29 @@ functions: ${PREPARE_SHELL} . ./evergreen/build-packages.sh + sign-packages: + - command: shell.exec + params: + shell: bash + working_dir: mongo-csharp-driver + env: + ARTIFACTORY_PASSWORD: ${ARTIFACTORY_PASSWORD} + ARTIFACTORY_USERNAME: ${ARTIFACTORY_USERNAME} + AZURE_NUGET_SIGN_TENANT_ID: ${AZURE_NUGET_SIGN_TENANT_ID} + AZURE_NUGET_SIGN_CLIENT_ID: ${AZURE_NUGET_SIGN_CLIENT_ID} + AZURE_NUGET_SIGN_CLIENT_SECRET: ${AZURE_NUGET_SIGN_CLIENT_SECRET} + PACKAGE_VERSION: "$PACKAGE_VERSION" + script: | + ${PREPARE_SHELL} + . ./evergreen/sign-packages.sh + push-packages: - command: shell.exec params: shell: bash working_dir: mongo-csharp-driver env: + NUGET_SIGN_CERTIFICATE_FINGERPRINT: ${NUGET_SIGN_CERTIFICATE_FINGERPRINT} PACKAGES_SOURCE: ${PACKAGES_SOURCE} PACKAGES_SOURCE_KEY: ${PACKAGES_SOURCE_KEY} script: | @@ -1857,6 +1874,7 @@ tasks: commands: - func: install-dotnet - func: download-packages + - func: sign-packages - func: push-packages vars: PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json" @@ -1866,6 +1884,7 @@ tasks: commands: - func: install-dotnet - func: download-packages + - func: sign-packages - func: push-packages vars: PACKAGES_SOURCE: "https://www.myget.org/F/mongodb/api/v3/index.json" @@ -2690,7 +2709,7 @@ buildvariants: # Package release variants - matrix_name: build-packages matrix_spec: - os: "windows-64" # should produce package on Windows to make sure full framework binaries created. + os: "windows-64" # should produce package on Windows to build .NET framework (net472) packages. display_name: "Packages Pack" tags: ["build-packages", "release-tag"] tasks: diff --git a/evergreen/push-packages.sh b/evergreen/push-packages.sh index 81b64701c76..58c32ee0910 100644 --- a/evergreen/push-packages.sh +++ b/evergreen/push-packages.sh @@ -2,6 +2,12 @@ set -o errexit # Exit the script with error if any of the commands fail set +o xtrace # Disable tracing. +# Environment variables used as inpu +# NUGET_SIGN_CERTIFICATE_FINGERPRINT +# PACKAGES_SOURCE +# PACKAGES_SOURCE_KEY +# PACKAGE_VERSION + # querying nuget source to find search base url packages_search_url=$(curl -X GET -s "${PACKAGES_SOURCE}" | jq -r 'first(.resources[] | select(."@type"=="SearchQueryService") | ."@id")') @@ -50,6 +56,11 @@ if [ "$PACKAGES_SOURCE" = "https://api.nuget.org/v3/index.json" ] && [[ ! "$PACK fi PACKAGES=("MongoDB.Bson" "MongoDB.Driver.Core" "MongoDB.Driver" "MongoDB.Driver.GridFS" "mongocsharpdriver") + +for package in ${PACKAGES[*]}; do + dotnet nuget verify ./artifacts/nuget/"$package"."$PACKAGE_VERSION".nupkg --certificate-fingerprint "$NUGET_SIGN_CERTIFICATE_FINGERPRINT" +done + for package in ${PACKAGES[*]}; do dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./artifacts/nuget/"$package"."$PACKAGE_VERSION".nupkg done diff --git a/evergreen/sign-packages.sh b/evergreen/sign-packages.sh new file mode 100644 index 00000000000..3bdfe4fbb2f --- /dev/null +++ b/evergreen/sign-packages.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -o errexit # Exit the script with error if any of the commands fail + +# Environment variables used as input: +# ARTIFACTORY_PASSWORD +# ARTIFACTORY_USERNAME +# AZURE_NUGET_SIGN_TENANT_ID +# AZURE_NUGET_SIGN_CLIENT_ID +# AZURE_NUGET_SIGN_CLIENT_SECRET +# PACKAGE_VERSION + +echo "${ARTIFACTORY_PASSWORD}" | docker login --password-stdin --username "${ARTIFACTORY_USERNAME}" artifactory.corp.mongodb.com + +docker run --platform="linux/amd64" --rm -v $(pwd):/workdir -w /workdir \ + artifactory.corp.mongodb.com/release-tools-container-registry-local/azure-keyvault-nuget \ + NuGetKeyVaultSignTool sign "artifacts/nuget/*.$PACKAGE_VERSION.nupkg" \ + --force \ + --file-digest=sha256 \ + --timestamp-rfc3161=http://timestamp.digicert.com \ + --timestamp-digest=sha256 \ + --azure-key-vault-url=https://mdb-authenticode.vault.azure.net \ + --azure-key-vault-tenant-id="$AZURE_NUGET_SIGN_TENANT_ID" \ + --azure-key-vault-client-secret="$AZURE_NUGET_SIGN_CLIENT_SECRET" \ + --azure-key-vault-client-id="$AZURE_NUGET_SIGN_CLIENT_ID" \ + --azure-key-vault-certificate=authenticode-2021 \ No newline at end of file From fbcf93aac669dff85e28baabd23f8e052e5d0fd1 Mon Sep 17 00:00:00 2001 From: rstam Date: Fri, 3 May 2024 13:20:49 -0700 Subject: [PATCH 33/38] CSHARP-5071: Support string concatenation of mixed types. --- .../Reflection/StringMethod.cs | 12 + ...essionToAggregationExpressionTranslator.cs | 19 +- ...MethodToAggregationExpressionTranslator.cs | 4 +- ...MethodToAggregationExpressionTranslator.cs | 99 +++- .../Jira/CSharp5071Tests.cs | 478 ++++++++++++++++++ 5 files changed, 593 insertions(+), 19 deletions(-) create mode 100644 tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5071Tests.cs diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Reflection/StringMethod.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Reflection/StringMethod.cs index 58884338e00..44d04beb1d0 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Reflection/StringMethod.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Reflection/StringMethod.cs @@ -28,6 +28,10 @@ internal static class StringMethod private static readonly MethodInfo __anyStringInWithParams; private static readonly MethodInfo __anyStringNinWithEnumerable; private static readonly MethodInfo __anyStringNinWithParams; + private static readonly MethodInfo __concatWith1Object; + private static readonly MethodInfo __concatWith2Objects; + private static readonly MethodInfo __concatWith3Objects; + private static readonly MethodInfo __concatWithObjectArray; private static readonly MethodInfo __concatWith2Strings; private static readonly MethodInfo __concatWith3Strings; private static readonly MethodInfo __concatWith4Strings; @@ -108,6 +112,10 @@ static StringMethod() __anyStringInWithParams = ReflectionInfo.Method((IEnumerable s, StringOrRegularExpression[] values) => s.AnyStringIn(values)); __anyStringNinWithEnumerable = ReflectionInfo.Method((IEnumerable s, IEnumerable values) => s.AnyStringNin(values)); __anyStringNinWithParams = ReflectionInfo.Method((IEnumerable s, StringOrRegularExpression[] values) => s.AnyStringNin(values)); + __concatWith1Object = ReflectionInfo.Method((object arg) => string.Concat(arg)); + __concatWith2Objects = ReflectionInfo.Method((object arg0, object arg1) => string.Concat(arg0, arg1)); + __concatWith3Objects = ReflectionInfo.Method((object arg0, object arg1, object arg2) => string.Concat(arg0, arg1, arg2)); + __concatWithObjectArray = ReflectionInfo.Method((object[] args) => string.Concat(args)); __concatWith2Strings = ReflectionInfo.Method((string str0, string str1) => string.Concat(str0, str1)); __concatWith3Strings = ReflectionInfo.Method((string str0, string str1, string str2) => string.Concat(str0, str1, str2)); __concatWith4Strings = ReflectionInfo.Method((string str0, string str1, string str2, string str3) => string.Concat(str0, str1, str2, str3)); @@ -168,6 +176,10 @@ static StringMethod() public static MethodInfo AnyStringInWithParams => __anyStringInWithParams; public static MethodInfo AnyStringNinWithEnumerable => __anyStringNinWithEnumerable; public static MethodInfo AnyStringNinWithParams => __anyStringNinWithParams; + public static MethodInfo ConcatWith1Object => __concatWith1Object; + public static MethodInfo ConcatWith2Objects => __concatWith2Objects; + public static MethodInfo ConcatWith3Objects => __concatWith3Objects; + public static MethodInfo ConcatWithObjectArray => __concatWithObjectArray; public static MethodInfo ConcatWith2Strings => __concatWith2Strings; public static MethodInfo ConcatWith3Strings => __concatWith3Strings; public static MethodInfo ConcatWith4Strings => __concatWith4Strings; diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/BinaryExpressionToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/BinaryExpressionToAggregationExpressionTranslator.cs index c8a3173499a..94251e3dff4 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/BinaryExpressionToAggregationExpressionTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/BinaryExpressionToAggregationExpressionTranslator.cs @@ -22,6 +22,7 @@ using MongoDB.Driver.Linq.Linq3Implementation.ExtensionMethods; using MongoDB.Driver.Linq.Linq3Implementation.Misc; using MongoDB.Driver.Linq.Linq3Implementation.Serializers; +using MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToAggregationExpressionTranslators.MethodTranslators; using MongoDB.Driver.Support; namespace MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToAggregationExpressionTranslators @@ -30,6 +31,11 @@ internal static class BinaryExpressionToAggregationExpressionTranslator { public static AggregationExpression Translate(TranslationContext context, BinaryExpression expression) { + if (StringConcatMethodToAggregationExpressionTranslator.CanTranslate(expression, out var method, out var arguments)) + { + return StringConcatMethodToAggregationExpressionTranslator.Translate(context, expression, method, arguments); + } + if (GetTypeComparisonExpressionToAggregationExpressionTranslator.CanTranslate(expression)) { return GetTypeComparisonExpressionToAggregationExpressionTranslator.Translate(context, expression); @@ -78,9 +84,7 @@ public static AggregationExpression Translate(TranslationContext context, Binary var ast = expression.NodeType switch { - ExpressionType.Add => IsStringConcatenationExpression(expression) ? - AstExpression.Concat(leftTranslation.Ast, rightTranslation.Ast) : - AstExpression.Add(leftTranslation.Ast, rightTranslation.Ast), + ExpressionType.Add => AstExpression.Add(leftTranslation.Ast, rightTranslation.Ast), ExpressionType.And => expression.Type == typeof(bool) ? AstExpression.And(leftTranslation.Ast, rightTranslation.Ast) : AstExpression.BitAnd(leftTranslation.Ast, rightTranslation.Ast), @@ -221,15 +225,6 @@ static bool IsEnumOrConvertEnumToUnderlyingType(Expression expression) return expression.Type.IsEnum || IsConvertEnumToUnderlyingType(expression); } - private static bool IsStringConcatenationExpression(BinaryExpression expression) - { - return - expression.NodeType == ExpressionType.Add && - expression.Type == typeof(string) && - expression.Left.Type == typeof(string) && - expression.Right.Type == typeof(string); - } - private static AstBinaryOperator ToBinaryOperator(ExpressionType nodeType) { return nodeType switch diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ConcatMethodToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ConcatMethodToAggregationExpressionTranslator.cs index 70114d28a91..7dc3f32b209 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ConcatMethodToAggregationExpressionTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ConcatMethodToAggregationExpressionTranslator.cs @@ -26,9 +26,9 @@ public static AggregationExpression Translate(TranslationContext context, Method return EnumerableConcatMethodToAggregationExpressionTranslator.Translate(context, expression); } - if (StringConcatMethodToAggregationExpressionTranslator.CanTranslate(expression)) + if (StringConcatMethodToAggregationExpressionTranslator.CanTranslate(expression, out var method, out var arguments)) { - return StringConcatMethodToAggregationExpressionTranslator.Translate(context, expression); + return StringConcatMethodToAggregationExpressionTranslator.Translate(context, expression, method, arguments); } throw new ExpressionNotSupportedException(expression); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/StringConcatMethodToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/StringConcatMethodToAggregationExpressionTranslator.cs index 78eec7ac8be..4b94471722e 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/StringConcatMethodToAggregationExpressionTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/StringConcatMethodToAggregationExpressionTranslator.cs @@ -14,9 +14,12 @@ */ using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Linq; using System.Linq.Expressions; using System.Reflection; +using MongoDB.Bson; +using MongoDB.Bson.IO; using MongoDB.Bson.Serialization.Serializers; using MongoDB.Driver.Linq.Linq3Implementation.Ast.Expressions; using MongoDB.Driver.Linq.Linq3Implementation.Misc; @@ -27,20 +30,48 @@ internal static class StringConcatMethodToAggregationExpressionTranslator { private static readonly MethodInfo[] __stringConcatMethods = new[] { + StringMethod.ConcatWith1Object, + StringMethod.ConcatWith2Objects, + StringMethod.ConcatWith3Objects, + StringMethod.ConcatWithObjectArray, StringMethod.ConcatWith2Strings, StringMethod.ConcatWith3Strings, StringMethod.ConcatWith4Strings, StringMethod.ConcatWithStringArray }; - public static bool CanTranslate(MethodCallExpression expression) - => expression.Method.IsOneOf(__stringConcatMethods); + public static bool CanTranslate(BinaryExpression expression, out MethodInfo method, out ReadOnlyCollection arguments) + { + if (expression.NodeType == ExpressionType.Add && + expression.Method != null && + expression.Method.IsOneOf(StringMethod.ConcatWith2Objects, StringMethod.ConcatWith2Strings)) + { + method = expression.Method; + arguments = new ReadOnlyCollection(new[] { expression.Left, expression.Right }); + return true; + } + + method = null; + arguments = null; + return false; + } - public static AggregationExpression Translate(TranslationContext context, MethodCallExpression expression) + public static bool CanTranslate(MethodCallExpression expression, out MethodInfo method, out ReadOnlyCollection arguments) { - var method = expression.Method; - var arguments = expression.Arguments; + if (expression.Method.IsOneOf(__stringConcatMethods)) + { + method = expression.Method; + arguments = expression.Arguments; + return true; + } + method = null; + arguments = null; + return false; + } + + public static AggregationExpression Translate(TranslationContext context, Expression expression, MethodInfo method, ReadOnlyCollection arguments) + { IEnumerable argumentsTranslations = null; if (method.IsOneOf( @@ -52,6 +83,16 @@ public static AggregationExpression Translate(TranslationContext context, Method arguments.Select(a => ExpressionToAggregationExpressionTranslator.Translate(context, a).Ast); } + if (method.IsOneOf( + StringMethod.ConcatWith1Object, + StringMethod.ConcatWith2Objects, + StringMethod.ConcatWith3Objects)) + { + argumentsTranslations = arguments + .Select(a => ExpressionToAggregationExpressionTranslator.Translate(context, a)) + .Select(ExpressionToString); + } + if (method.Is(StringMethod.ConcatWithStringArray)) { var argumentTranslation = ExpressionToAggregationExpressionTranslator.Translate(context, arguments.Single()); @@ -61,6 +102,16 @@ public static AggregationExpression Translate(TranslationContext context, Method } } + if (method.Is(StringMethod.ConcatWithObjectArray)) + { + if (arguments.Single() is NewArrayExpression newArrayExpression) + { + argumentsTranslations = newArrayExpression.Expressions + .Select(a => ExpressionToAggregationExpressionTranslator.Translate(context, a)) + .Select(ExpressionToString); + } + } + if (argumentsTranslations != null) { var ast = AstExpression.Concat(argumentsTranslations.ToArray()); @@ -68,6 +119,44 @@ public static AggregationExpression Translate(TranslationContext context, Method } throw new ExpressionNotSupportedException(expression); + + static AstExpression ExpressionToString(AggregationExpression aggregationExpression) + { + var astExpression = aggregationExpression.Ast; + if (aggregationExpression.Serializer.ValueType == typeof(string)) + { + return astExpression; + } + else + { + if (astExpression is AstConstantExpression constantAstExpression) + { + var value = constantAstExpression.Value; + var stringValue = ValueToString(aggregationExpression.Expression, value); + return AstExpression.Constant(stringValue); + } + else + { + return AstExpression.ToString(astExpression); + } + } + } + + static string ValueToString(Expression expression, BsonValue value) + { + return value switch + { + BsonBoolean booleanValue => JsonConvert.ToString(booleanValue.Value), + BsonDateTime dateTimeValue => JsonConvert.ToString(dateTimeValue.ToUniversalTime()), + BsonDecimal128 decimalValue => JsonConvert.ToString(decimalValue.Value), + BsonDouble doubleValue => JsonConvert.ToString(doubleValue.Value), + BsonInt32 int32Value => JsonConvert.ToString(int32Value.Value), + BsonInt64 int64Value => JsonConvert.ToString(int64Value.Value), + BsonObjectId objectIdValue => objectIdValue.Value.ToString(), + BsonString stringValue => stringValue.Value, + _ => throw new ExpressionNotSupportedException(expression, because: $"values represented as BSON type {value.BsonType} are not supported by $toString") + }; + } } } } diff --git a/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5071Tests.cs b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5071Tests.cs new file mode 100644 index 00000000000..0683bb11e32 --- /dev/null +++ b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp5071Tests.cs @@ -0,0 +1,478 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Linq; +using FluentAssertions; +using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.TestHelpers.XunitExtensions; +using MongoDB.Driver.Linq; +using Xunit; + +namespace MongoDB.Driver.Tests.Linq.Linq3Implementation.Jira +{ + public class CSharp5071Tests : Linq3IntegrationTest + { + [Theory] + [InlineData("intconstant+stringproperty", "wrong:{ $project : { __fld0 : { $add : [1, '$B'] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("intconstant+stringproperty", "{ $project : { _v : { $concat : ['1', '$B'] }, _id : 0 } }", "1B", LinqProvider.V3)] + [InlineData("intproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B'] }, _id : 0 } }", "1B", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty", "{ $project : { __fld0 : { $concat : ['X', '$B'] }, _id : 0 } }", "XB", LinqProvider.V2)] + [InlineData("stringconstant+stringproperty", "{ $project : { _v : { $concat : ['X', '$B'] }, _id : 0 } }", "XB", LinqProvider.V3)] + [InlineData("stringproperty+intconstant", "wrong:{ $project : { __fld0 : { $concat : ['$A', 2] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("stringproperty+intconstant", "{ $project : { _v : { $concat : ['$A', '2'] }, _id : 0 } }", "A2", LinqProvider.V3)] + [InlineData("stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+intproperty", "{ $project : { _v : { $concat : ['$A', { $toString : '$J' }] }, _id : 0 } }", "A2", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant", "{ $project : { __fld0 : { $concat : ['$A', 'X'] }, _id : 0 } }", "AX", LinqProvider.V2)] + [InlineData("stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', 'X'] }, _id : 0 } }", "AX", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty", "{ $project : { __fld0 : { $concat : ['$A', '$B'] }, _id : 0 } }", "AB", LinqProvider.V2)] + [InlineData("stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B'] }, _id : 0 } }", "AB", LinqProvider.V3)] + public void Add_with_two_terms_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + if (expectedStage.Contains("$toString")) + { + RequireServer.Check().Supports(Feature.AggregateToString); + } + + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "intconstant+stringproperty" => collection.AsQueryable().Select(x => 1 + x.B), + "intproperty+stringproperty" => collection.AsQueryable().Select(x => x.I + x.B), + "stringconstant+stringproperty" => collection.AsQueryable().Select(x => "X" + x.B), + "stringproperty+intconstant" => collection.AsQueryable().Select(x => x.A + 2), + "stringproperty+intproperty" => collection.AsQueryable().Select(x => x.A + x.J), + "stringproperty+stringconstant" => collection.AsQueryable().Select(x => x.A + "X"), + "stringproperty+stringproperty" => collection.AsQueryable().Select(x => x.A + x.B), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + [Theory] + [InlineData("intconstant+stringproperty+intconstant", "wrong:{ $project : { __fld0 : { $concat : [{ $add : [1, '$B'] }, 3] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("intconstant+stringproperty+intconstant", "{ $project : { _v : { $concat : ['1', '$B', '3'] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+intproperty", "{ $project : { _v : { $concat : ['1', '$B', { $toString : '$K' }] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+stringconstant", "wrong:{ $project : { __fld0 : { $concat : [{ $add : [1, '$B'] }, 'Z'] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("intconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['1', '$B', 'Z'] }, _id : 0 } }", "1BZ", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+stringproperty", "wrong:{ $project : { __fld0 : { $concat : [{ $add : [1, '$B'] }, '$C'] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("intconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['1', '$B', '$C'] }, _id : 0 } }", "1BC", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+intconstant", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+intconstant", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', '3'] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+intproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', { $toString : '$K' }] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+stringconstant", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', 'Z'] }, _id : 0 } }", "1BZ", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', '$C'] }, _id : 0 } }", "1BC", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+intconstant", "wrong:{ $project : { __fld0 : { $concat : ['X', '$B', 3] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+intconstant", "{ $project : { _v : { $concat : ['X', '$B', '3'] }, _id : 0 } }", "XB3", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+intproperty", "{ $project : { _v : { $concat : ['X', '$B', { $toString : '$K' }] }, _id : 0 } }", "XB3", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringconstant", "{ $project : { __fld0 : { $concat : ['X', '$B', 'Z'] }, _id : 0 } }", "XBZ", LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['X', '$B', 'Z'] }, _id : 0 } }", "XBZ", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringproperty", "{ $project : { __fld0 : { $concat : ['X', '$B', '$C'] }, _id : 0 } }", "XBC", LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['X', '$B', '$C'] }, _id : 0 } }", "XBC", LinqProvider.V3)] + [InlineData("stringproperty+intconstant+stringproperty", "wrong:{ $project : { __fld0 : { $concat : ['$A', 2, '$C'] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("stringproperty+intconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', '2', '$C'] }, _id : 0 } }", "A2C", LinqProvider.V3)] + [InlineData("stringproperty+intproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+intproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', { $toString : '$J' }, '$C'] }, _id : 0 } }", "A2C", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant+stringproperty", "{ $project : { __fld0 : { $concat : ['$A', 'Y', '$C'] }, _id : 0 } }", "AYC", LinqProvider.V2)] + [InlineData("stringproperty+stringconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', 'Y', '$C'] }, _id : 0 } }", "AYC", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+intconstant", "wrong:{ $project : { __fld0 : { $concat : ['$A', '$B', 3] }, _id : 0 } }", "throws:MongoCommandException", LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+intconstant", "{ $project : { _v : { $concat : ['$A', '$B', '3'] }, _id : 0 } }", "AB3", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+intproperty", "{ $project : { _v : { $concat : ['$A', '$B', { $toString : '$K' }] }, _id : 0 } }", "AB3", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringconstant", "{ $project : { __fld0 : { $concat : ['$A', '$B', 'Z'] }, _id : 0 } }", "ABZ", LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', '$B', 'Z'] }, _id : 0 } }", "ABZ", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringproperty", "{ $project : { __fld0 : { $concat : ['$A', '$B', '$C'] }, _id : 0 } }", "ABC", LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B', '$C'] }, _id : 0 } }", "ABC", LinqProvider.V3)] + public void Add_with_three_terms_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + if (expectedStage.Contains("$toString")) + { + RequireServer.Check().Supports(Feature.AggregateToString); + } + + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "intconstant+stringproperty+intconstant" => collection.AsQueryable().Select(x => 1 + x.B + 3), + "intconstant+stringproperty+intproperty" => collection.AsQueryable().Select(x => 1 + x.B + x.K), + "intconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => 1 + x.B + "Z"), + "intconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => 1 + x.B + x.C), + "intproperty+stringproperty+intconstant" => collection.AsQueryable().Select(x => x.I + x.B + 3), + "intproperty+stringproperty+intproperty" => collection.AsQueryable().Select(x => x.I + x.B + x.K), + "intproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => x.I + x.B + "Z"), + "intproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => x.I + x.B + x.C), + "stringconstant+stringproperty+intconstant" => collection.AsQueryable().Select(x => "X" + x.B + 3), + "stringconstant+stringproperty+intproperty" => collection.AsQueryable().Select(x => "X" + x.B + x.K), + "stringconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => "X" + x.B + "Z"), + "stringconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => "X" + x.B + x.C), + "stringproperty+intconstant+stringproperty" => collection.AsQueryable().Select(x => x.A + 2 + x.C), + "stringproperty+intproperty+stringproperty" => collection.AsQueryable().Select(x => x.A + x.J + x.C), + "stringproperty+stringconstant+stringproperty" => collection.AsQueryable().Select(x => x.A + "Y" + x.C), + "stringproperty+stringproperty+intconstant" => collection.AsQueryable().Select(x => x.A + x.B + 3), + "stringproperty+stringproperty+intproperty" => collection.AsQueryable().Select(x => x.A + x.B + x.K), + "stringproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => x.A + x.B + "Z"), + "stringproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => x.A + x.B + x.C), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + [Theory] + [InlineData("intproperty", "throws:ArgumentException", null, LinqProvider.V2)] + [InlineData("intproperty", "{ $project : { _v : { $concat : { $toString : '$I' } }, _id : 0 } }", "1", LinqProvider.V3)] + [InlineData("stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty", "{ $project : { _v : { $concat : '$A' }, _id : 0 } }", "A", LinqProvider.V3)] + public void Concat_with_one_argument_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + if (expectedStage.Contains("$toString")) + { + RequireServer.Check().Supports(Feature.AggregateToString); + } + + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "intproperty" => collection.AsQueryable().Select(x => string.Concat(x.I)), + "stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.A)), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + [Theory] + [InlineData("intconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty", "{ $project : { _v : { $concat : ['1', '$B'] }, _id : 0 } }", "1B", LinqProvider.V3)] + [InlineData("intproperty+stringproperty", "throws:ArgumentException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B'] }, _id : 0 } }", "1B", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty", "{ $project : { _v : { $concat : ['X', '$B'] }, _id : 0 } }", "XB", LinqProvider.V3)] + [InlineData("stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+intconstant", "{ $project : { _v : { $concat : ['$A', '2'] }, _id : 0 } }", "A2", LinqProvider.V3)] + [InlineData("stringproperty+intproperty", "throws:ArgumentException", null, LinqProvider.V2)] + [InlineData("stringproperty+intproperty", "{ $project : { _v : { $concat : ['$A', { $toString : '$J' }] }, _id : 0 } }", "A2", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', 'X'] }, _id : 0 } }", "AX", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B'] }, _id : 0 } }", "AB", LinqProvider.V3)] + public void Concat_with_two_arguments_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + if (expectedStage.Contains("$toString")) + { + RequireServer.Check().Supports(Feature.AggregateToString); + } + + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "intconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(1, x.B)), + "intproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.I, x.B)), + "stringconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat("X", x.B)), + "stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(x.A, 2)), + "stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(x.A, x.J)), + "stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(x.A, "X")), + "stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.A, x.B)), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + [Theory] + [InlineData("intconstant+stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+intconstant", "{ $project : { _v : { $concat : ['1', '$B', '3'] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+intproperty", "{ $project : { _v : { $concat : ['1', '$B', { $toString : '$K' }] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['1', '$B', 'Z'] }, _id : 0 } }", "1BZ", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['1', '$B', '$C'] }, _id : 0 } }", "1BC", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+intconstant", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+intconstant", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', '3'] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+intproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', { $toString : '$K' }] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+stringconstant", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', 'Z'] }, _id : 0 } }", "1BZ", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', '$C'] }, _id : 0 } }", "1BC", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+intconstant", "{ $project : { _v : { $concat : ['X', '$B', '3'] }, _id : 0 } }", "XB3", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+intproperty", "{ $project : { _v : { $concat : ['X', '$B', { $toString : '$K' }] }, _id : 0 } }", "XB3", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['X', '$B', 'Z'] }, _id : 0 } }", "XBZ", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['X', '$B', '$C'] }, _id : 0 } }", "XBC", LinqProvider.V3)] + [InlineData("stringproperty+intconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+intconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', '2', '$C'] }, _id : 0 } }", "A2C", LinqProvider.V3)] + [InlineData("stringproperty+intproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+intproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', { $toString : '$J' }, '$C'] }, _id : 0 } }", "A2C", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', 'Y', '$C'] }, _id : 0 } }", "AYC", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+intconstant", "{ $project : { _v : { $concat : ['$A', '$B', '3'] }, _id : 0 } }", "AB3", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+intproperty", "{ $project : { _v : { $concat : ['$A', '$B', { $toString : '$K' }] }, _id : 0 } }", "AB3", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', '$B', 'Z'] }, _id : 0 } }", "ABZ", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B', '$C'] }, _id : 0 } }", "ABC", LinqProvider.V3)] + public void Concat_with_three_arguments_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + if (expectedStage.Contains("$toString")) + { + RequireServer.Check().Supports(Feature.AggregateToString); + } + + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "intconstant+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(1 + x.B + 3)), + "intconstant+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(1 + x.B + x.K)), + "intconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(1 + x.B + "Z")), + "intconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(1 + x.B + x.C)), + "intproperty+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(x.I + x.B + 3)), + "intproperty+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(x.I + x.B + x.K)), + "intproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(x.I + x.B + "Z")), + "intproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.I + x.B + x.C)), + "stringconstant+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat("X" + x.B + 3)), + "stringconstant+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat("X" + x.B + x.K)), + "stringconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat("X" + x.B + "Z")), + "stringconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat("X" + x.B + x.C)), + "stringproperty+intconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.A + 2 + x.C)), + "stringproperty+intproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.A + x.J + x.C)), + "stringproperty+stringconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.A + "Y" + x.C)), + "stringproperty+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(x.A + x.B + 3)), + "stringproperty+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(x.A + x.B + x.K)), + "stringproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(x.A + x.B + "Z")), + "stringproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(x.A + x.B + x.C)), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + [Theory] + [InlineData("intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty", "{ $project : { _v : { $concat : { $toString : '$I' } }, _id : 0 } }", "1", LinqProvider.V3)] + [InlineData("stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty", "{ $project : { _v : { $concat : '$A' }, _id : 0 } }", "A", LinqProvider.V3)] + [InlineData("intconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty", "{ $project : { _v : { $concat : ['1', '$B'] }, _id : 0 } }", "1B", LinqProvider.V3)] + [InlineData("intproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B'] }, _id : 0 } }", "1B", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty", "{ $project : { _v : { $concat : ['X', '$B'] }, _id : 0 } }", "XB", LinqProvider.V3)] + [InlineData("stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+intconstant", "{ $project : { _v : { $concat : ['$A', '2'] }, _id : 0 } }", "A2", LinqProvider.V3)] + [InlineData("stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+intproperty", "{ $project : { _v : { $concat : ['$A', { $toString : '$J' }] }, _id : 0 } }", "A2", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', 'X'] }, _id : 0 } }", "AX", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B'] }, _id : 0 } }", "AB", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+intconstant", "{ $project : { _v : { $concat : ['1', '$B', '3'] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+intproperty", "{ $project : { _v : { $concat : ['1', '$B', { $toString : '$K' }] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['1', '$B', 'Z'] }, _id : 0 } }", "1BZ", LinqProvider.V3)] + [InlineData("intconstant+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("intconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['1', '$B', '$C'] }, _id : 0 } }", "1BC", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+intconstant", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+intconstant", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', '3'] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+intproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', { $toString : '$K' }] }, _id : 0 } }", "1B3", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+stringconstant", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', 'Z'] }, _id : 0 } }", "1BZ", LinqProvider.V3)] + [InlineData("intproperty+stringproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("intproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : [{ $toString : '$I' }, '$B', '$C'] }, _id : 0 } }", "1BC", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+intconstant", "{ $project : { _v : { $concat : ['X', '$B', '3'] }, _id : 0 } }", "XB3", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+intproperty", "{ $project : { _v : { $concat : ['X', '$B', { $toString : '$K' }] }, _id : 0 } }", "XB3", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['X', '$B', 'Z'] }, _id : 0 } }", "XBZ", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['X', '$B', '$C'] }, _id : 0 } }", "XBC", LinqProvider.V3)] + [InlineData("stringproperty+intconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+intconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', '2', '$C'] }, _id : 0 } }", "A2C", LinqProvider.V3)] + [InlineData("stringproperty+intproperty+stringproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+intproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', { $toString : '$J' }, '$C'] }, _id : 0 } }", "A2C", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', 'Y', '$C'] }, _id : 0 } }", "AYC", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+intconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+intconstant", "{ $project : { _v : { $concat : ['$A', '$B', '3'] }, _id : 0 } }", "AB3", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+intproperty", "throws:InvalidOperationException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+intproperty", "{ $project : { _v : { $concat : ['$A', '$B', { $toString : '$K' }] }, _id : 0 } }", "AB3", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', '$B', 'Z'] }, _id : 0 } }", "ABZ", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B', '$C'] }, _id : 0 } }", "ABC", LinqProvider.V3)] + public void Concat_with_array_of_object_argument_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + if (expectedStage.Contains("$toString")) + { + RequireServer.Check().Supports(Feature.AggregateToString); + } + + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "intproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.I })), + "stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A })), + "intconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { 1, x.B })), + "intproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.I, x.B })), + "stringconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { "X", x.B })), + "stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A, 2 })), + "stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A, x.J })), + "stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A, "X" })), + "stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A, x.B })), + "intconstant+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { 1 + x.B + 3 })), + "intconstant+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { 1 + x.B + x.K })), + "intconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { 1 + x.B + "Z" })), + "intconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { 1 + x.B + x.C })), + "intproperty+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.I + x.B + 3 })), + "intproperty+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.I + x.B + x.K })), + "intproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.I + x.B + "Z" })), + "intproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.I + x.B + x.C })), + "stringconstant+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { "X" + x.B + 3 })), + "stringconstant+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { "X" + x.B + x.K })), + "stringconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { "X" + x.B + "Z" })), + "stringconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { "X" + x.B + x.C })), + "stringproperty+intconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + 2 + x.C })), + "stringproperty+intproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + x.J + x.C })), + "stringproperty+stringconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + "Y" + x.C })), + "stringproperty+stringproperty+intconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + x.B + 3 })), + "stringproperty+stringproperty+intproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + x.B + x.K })), + "stringproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + x.B + "Z" })), + "stringproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new object[] { x.A + x.B + x.C })), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + [Theory] + [InlineData("stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty", "{ $project : { _v : { $concat : '$A' }, _id : 0 } }", "A", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty", "{ $project : { _v : { $concat : ['X', '$B'] }, _id : 0 } }", "XB", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', 'X'] }, _id : 0 } }", "AX", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B'] }, _id : 0 } }", "AB", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['X', '$B', 'Z'] }, _id : 0 } }", "XBZ", LinqProvider.V3)] + [InlineData("stringconstant+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringconstant+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['X', '$B', '$C'] }, _id : 0 } }", "XBC", LinqProvider.V3)] + [InlineData("stringproperty+stringconstant+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringconstant+stringproperty", "{ $project : { _v : { $concat : ['$A', 'Y', '$C'] }, _id : 0 } }", "AYC", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringconstant", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringconstant", "{ $project : { _v : { $concat : ['$A', '$B', 'Z'] }, _id : 0 } }", "ABZ", LinqProvider.V3)] + [InlineData("stringproperty+stringproperty+stringproperty", "throws:NotSupportedException", null, LinqProvider.V2)] + [InlineData("stringproperty+stringproperty+stringproperty", "{ $project : { _v : { $concat : ['$A', '$B', '$C'] }, _id : 0 } }", "ABC", LinqProvider.V3)] + public void Concat_with_array_of_string_argument_should_work(string scenario, string expectedStage, string expectedResult, LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = scenario switch + { + "stringproperty" => collection.AsQueryable().Select(x => string.Concat(new string[] { x.A })), + "stringconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new string[] { "X", x.B })), + "stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new string[] { x.A, "X" })), + "stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new string[] { x.A, x.B })), + "stringconstant+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new string[] { "X" + x.B + "Z" })), + "stringconstant+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new string[] { "X" + x.B + x.C })), + "stringproperty+stringconstant+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new string[] { x.A + "Y" + x.C })), + "stringproperty+stringproperty+stringconstant" => collection.AsQueryable().Select(x => string.Concat(new string[] { x.A + x.B + "Z" })), + "stringproperty+stringproperty+stringproperty" => collection.AsQueryable().Select(x => string.Concat(new string[] { x.A + x.B + x.C })), + _ => throw new Exception() + }; + + Assert(collection, queryable, expectedStage, expectedResult); + } + + private void Assert(IMongoCollection collection, IQueryable queryable, string expectedStage, string expectedResult) + { + if (expectedStage.StartsWith("throws:")) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().NotBeNull(); + exception.GetType().Name.Should().Be(expectedStage.Substring(7)); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, expectedStage.StartsWith("wrong:") ? expectedStage.Substring(6) : expectedStage); + + if (expectedResult.StartsWith("throws:")) + { + var exception = Record.Exception(() => queryable.Single()); + exception.Should().NotBeNull(); + exception.GetType().Name.Should().Be(expectedResult.Substring(7)); + } + else + { + var result = queryable.Single(); + result.Should().Be(expectedResult); + } + } + } + + private IMongoCollection GetCollection(LinqProvider linqProvider) + { + var collection = GetCollection("test", linqProvider); + CreateCollection( + collection, + new Document + { + Id = 1, + A = "A", + B = "B", + C = "C", + I = 1, + J = 2, + K = 3 + }); + return collection; + } + + private class Document + { + public int Id { get; set; } + public string A { get; set; } + public string B { get; set; } + public string C { get; set; } + public int I { get; set; } + public int J { get; set; } + public int K { get; set; } + } + } +} From 61d2c774c958d65a8ed9889e6d0ec9de2f8be0ca Mon Sep 17 00:00:00 2001 From: rstam Date: Fri, 31 May 2024 09:33:26 -0700 Subject: [PATCH 34/38] CSHARP-2509: Support Dictionary.ContainsValue in LINQ queries. --- .../Ast/Expressions/AstExpression.cs | 5 + ...essionToAggregationExpressionTranslator.cs | 1 + ...MethodToAggregationExpressionTranslator.cs | 115 +++++++++ .../ContainsValueMethodToFilterTranslator.cs | 84 +++++++ .../MethodCallExpressionToFilterTranslator.cs | 1 + .../Jira/CSharp2509Tests.cs | 220 ++++++++++++++++++ 6 files changed, 426 insertions(+) create mode 100644 src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ContainsValueMethodToAggregationExpressionTranslator.cs create mode 100644 src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/ContainsValueMethodToFilterTranslator.cs create mode 100644 tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp2509Tests.cs diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Ast/Expressions/AstExpression.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Ast/Expressions/AstExpression.cs index 6e8faa2edd8..58040f476c9 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Ast/Expressions/AstExpression.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Ast/Expressions/AstExpression.cs @@ -575,6 +575,11 @@ public static AstExpression NullaryWindowExpression(AstNullaryWindowOperator @op return new AstNullaryWindowExpression(@operator, window); } + public static AstExpression ObjectToArray(AstExpression arg) + { + return new AstUnaryExpression(AstUnaryOperator.ObjectToArray, arg); + } + public static AstExpression Or(params AstExpression[] args) { Ensure.IsNotNull(args, nameof(args)); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs index 0c57aeddf29..c86dbbbb523 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodCallExpressionToAggregationExpressionTranslator.cs @@ -37,6 +37,7 @@ public static AggregationExpression Translate(TranslationContext context, Method case "Concat": return ConcatMethodToAggregationExpressionTranslator.Translate(context, expression); case "Contains": return ContainsMethodToAggregationExpressionTranslator.Translate(context, expression); case "ContainsKey": return ContainsKeyMethodToAggregationExpressionTranslator.Translate(context, expression); + case "ContainsValue": return ContainsValueMethodToAggregationExpressionTranslator.Translate(context, expression); case "CovariancePopulation": return CovariancePopulationMethodToAggregationExpressionTranslator.Translate(context, expression); case "CovarianceSample": return CovarianceSampleMethodToAggregationExpressionTranslator.Translate(context, expression); case "Create": return CreateMethodToAggregationExpressionTranslator.Translate(context, expression); diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ContainsValueMethodToAggregationExpressionTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ContainsValueMethodToAggregationExpressionTranslator.cs new file mode 100644 index 00000000000..c2437521c6d --- /dev/null +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToAggregationExpressionTranslators/MethodTranslators/ContainsValueMethodToAggregationExpressionTranslator.cs @@ -0,0 +1,115 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Linq.Expressions; +using System.Reflection; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Options; +using MongoDB.Bson.Serialization.Serializers; +using MongoDB.Driver.Linq.Linq3Implementation.Ast.Expressions; + +namespace MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToAggregationExpressionTranslators.MethodTranslators +{ + internal static class ContainsValueMethodToAggregationExpressionTranslator + { + // public methods + public static AggregationExpression Translate(TranslationContext context, MethodCallExpression expression) + { + var method = expression.Method; + var arguments = expression.Arguments; + + if (IsContainsValueMethod(method)) + { + var dictionaryExpression = expression.Object; + var valueExpression = arguments[0]; + + var dictionaryTranslation = ExpressionToAggregationExpressionTranslator.Translate(context, dictionaryExpression); + var dictionarySerializer = GetDictionarySerializer(expression, dictionaryTranslation); + var dictionaryRepresentation = dictionarySerializer.DictionaryRepresentation; + + var valueTranslation = ExpressionToAggregationExpressionTranslator.Translate(context, valueExpression); + var (valueBinding, valueAst) = AstExpression.UseVarIfNotSimple("value", valueTranslation.Ast); + + AstExpression ast; + switch (dictionaryRepresentation) + { + case DictionaryRepresentation.Document: + ast = AstExpression.Let( + var: valueBinding, + @in: AstExpression.Reduce( + input: AstExpression.ObjectToArray(dictionaryTranslation.Ast), + initialValue: false, + @in: AstExpression.Cond( + @if: AstExpression.Var("value"), + @then: true, + @else: AstExpression.Eq(AstExpression.GetField(AstExpression.Var("this"), "v"), valueAst)))); + break; + + case DictionaryRepresentation.ArrayOfArrays: + ast = AstExpression.Let( + var: valueBinding, + @in: AstExpression.Reduce( + input: dictionaryTranslation.Ast, + initialValue: false, + @in: AstExpression.Cond( + @if: AstExpression.Var("value"), + @then: true, + @else: AstExpression.Eq(AstExpression.ArrayElemAt(AstExpression.Var("this"), 1), valueAst)))); + break; + + case DictionaryRepresentation.ArrayOfDocuments: + ast = AstExpression.Let( + var: valueBinding, + @in: AstExpression.Reduce( + input: dictionaryTranslation.Ast, + initialValue: false, + @in: AstExpression.Cond( + @if: AstExpression.Var("value"), + @then: true, + @else: AstExpression.Eq(AstExpression.GetField(AstExpression.Var("this"), "v"), valueAst)))); + break; + + default: + throw new ExpressionNotSupportedException(expression, because: $"ContainsValue is not supported when DictionaryRepresentation is: {dictionaryRepresentation}"); + } + + return new AggregationExpression(expression, ast, BooleanSerializer.Instance); + } + + throw new ExpressionNotSupportedException(expression); + } + + private static IBsonDictionarySerializer GetDictionarySerializer(Expression expression, AggregationExpression dictionaryTranslation) + { + if (dictionaryTranslation.Serializer is IBsonDictionarySerializer dictionarySerializer) + { + return dictionarySerializer; + } + + throw new ExpressionNotSupportedException(expression, because: $"class {dictionaryTranslation.Serializer.GetType().FullName} does not implement the IBsonDictionarySerializer interface"); + } + + private static bool IsContainsValueMethod(MethodInfo method) + { + return + !method.IsStatic && + method.IsPublic && + method.ReturnType == typeof(bool) && + method.Name == "ContainsValue" && + method.GetParameters() is var parameters && + parameters.Length == 1; + } + } +} diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/ContainsValueMethodToFilterTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/ContainsValueMethodToFilterTranslator.cs new file mode 100644 index 00000000000..256f320364e --- /dev/null +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/ContainsValueMethodToFilterTranslator.cs @@ -0,0 +1,84 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Linq.Expressions; +using System.Reflection; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Options; +using MongoDB.Driver.Linq.Linq3Implementation.Ast.Filters; +using MongoDB.Driver.Linq.Linq3Implementation.Misc; +using MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToFilterTranslators.ToFilterFieldTranslators; + +namespace MongoDB.Driver.Linq.Linq3Implementation.Translators.ExpressionToFilterTranslators.MethodTranslators +{ + internal static class ContainsValueMethodToFilterTranslator + { + public static AstFilter Translate(TranslationContext context, MethodCallExpression expression) + { + var method = expression.Method; + var arguments = expression.Arguments; + + if (IsContainsValueMethod(method)) + { + var dictionaryExpression = expression.Object; + var valueExpression = arguments[0]; + + var dictionaryField = ExpressionToFilterFieldTranslator.Translate(context, dictionaryExpression); + var dictionarySerializer = GetDictionarySerializer(expression, dictionaryField); + var dictionaryRepresentation = dictionarySerializer.DictionaryRepresentation; + var valueSerializer = dictionarySerializer.ValueSerializer; + + if (valueExpression is ConstantExpression constantValueExpression) + { + var valueField = AstFilter.Field("v", valueSerializer); + var value = constantValueExpression.Value; + var serializedValue = SerializationHelper.SerializeValue(valueSerializer, value); + + switch (dictionaryRepresentation) + { + case DictionaryRepresentation.ArrayOfDocuments: + return AstFilter.ElemMatch(dictionaryField, AstFilter.Eq(valueField, serializedValue)); + + default: + throw new ExpressionNotSupportedException(expression, because: $"ContainsValue is not supported when DictionaryRepresentation is: {dictionaryRepresentation}"); + } + } + } + + throw new ExpressionNotSupportedException(expression); + } + + private static IBsonDictionarySerializer GetDictionarySerializer(Expression expression, AstFilterField field) + { + if (field.Serializer is IBsonDictionarySerializer dictionarySerializer) + { + return dictionarySerializer; + } + + throw new ExpressionNotSupportedException(expression, because: $"class {field.Serializer.GetType().FullName} does not implement the IBsonDictionarySerializer interface"); + } + + private static bool IsContainsValueMethod(MethodInfo method) + { + return + !method.IsStatic && + method.IsPublic && + method.ReturnType == typeof(bool) && + method.Name == "ContainsValue" && + method.GetParameters() is var parameters && + parameters.Length == 1; + } + } +} diff --git a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/MethodCallExpressionToFilterTranslator.cs b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/MethodCallExpressionToFilterTranslator.cs index 9791b572c96..46bec0e63df 100644 --- a/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/MethodCallExpressionToFilterTranslator.cs +++ b/src/MongoDB.Driver/Linq/Linq3Implementation/Translators/ExpressionToFilterTranslators/MethodTranslators/MethodCallExpressionToFilterTranslator.cs @@ -26,6 +26,7 @@ public static AstFilter Translate(TranslationContext context, MethodCallExpressi { case "Contains": return ContainsMethodToFilterTranslator.Translate(context, expression); case "ContainsKey": return ContainsKeyMethodToFilterTranslator.Translate(context, expression); + case "ContainsValue": return ContainsValueMethodToFilterTranslator.Translate(context, expression); case "EndsWith": return EndsWithMethodToFilterTranslator.Translate(context, expression); case "Equals": return EqualsMethodToFilterTranslator.Translate(context, expression); case "Exists": return ExistsMethodToFilterTranslator.Translate(context, expression); diff --git a/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp2509Tests.cs b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp2509Tests.cs new file mode 100644 index 00000000000..5311bb058da --- /dev/null +++ b/tests/MongoDB.Driver.Tests/Linq/Linq3Implementation/Jira/CSharp2509Tests.cs @@ -0,0 +1,220 @@ +/* Copyright 2010-present MongoDB Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Bson.Serialization.Options; +using MongoDB.Driver.Linq; +using MongoDB.TestHelpers.XunitExtensions; +using Xunit; + +namespace MongoDB.Driver.Tests.Linq.Linq3Implementation.Jira +{ + public class CSharp2509Tests : Linq3IntegrationTest + { + [Theory] + [ParameterAttributeData] + public void Where_ContainsValue_should_work_when_representation_is_Dictionary( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .Where(x => x.D1.ContainsValue(1)); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $match : { $expr : { $reduce : { input : { $objectToArray : '$D1' }, initialValue : false, in : { $cond : { if : '$$value', then : true, else : { $eq : ['$$this.v', 1] } } } } } } }"); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(1, 2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_ContainsValue_should_work_when_representation_is_ArrayOfArrays( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .Where(x => x.D2.ContainsValue(1)); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $match : { $expr : { $reduce : { input : '$D2', initialValue : false, in : { $cond : { if : '$$value', then : true, else : { $eq : [{ $arrayElemAt : ['$$this', 1] }, 1] } } } } } } }"); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(1, 2); + } + } + + [Theory] + [ParameterAttributeData] + public void Where_ContainsValue_should_work_when_representation_is_ArrayOfDocuments( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .Where(x => x.D3.ContainsValue(1)); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $match : { D3 : { $elemMatch : { v : 1 } } } }"); + + var results = queryable.ToList(); + results.Select(x => x.Id).Should().Equal(1, 2); + } + } + + [Theory] + [ParameterAttributeData] + public void Select_ContainsValue_should_work_when_representation_is_Dictionary( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .Select(x => x.D1.ContainsValue(1)); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $project : { _v : { $reduce : { input : { $objectToArray : '$D1' }, initialValue : false, in : { $cond : { if : '$$value', then : true, else : { $eq : ['$$this.v', 1] } } } } }, _id : 0 } }"); + + var results = queryable.ToList(); + results.Should().Equal(true, true, false); + } + } + + [Theory] + [ParameterAttributeData] + public void Select_ContainsValue_should_work_when_representation_is_ArrayOfArrays( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .Select(x => x.D2.ContainsValue(1)); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $project : { _v : { $reduce : { input : '$D2', initialValue : false, in : { $cond : { if : '$$value', then : true, else : { $eq : [{ $arrayElemAt : ['$$this', 1] }, 1] } } } } }, _id : 0 } }"); + + var results = queryable.ToList(); + results.Should().Equal(true, true, false); + } + } + + [Theory] + [ParameterAttributeData] + public void Select_ContainsValue_should_work_when_representation_is_ArrayOfDocuments( + [Values(LinqProvider.V2, LinqProvider.V3)] LinqProvider linqProvider) + { + var collection = GetCollection(linqProvider); + + var queryable = collection.AsQueryable() + .Select(x => x.D3.ContainsValue(1)); + + if (linqProvider == LinqProvider.V2) + { + var exception = Record.Exception(() => Translate(collection, queryable)); + exception.Should().BeOfType(); + } + else + { + var stages = Translate(collection, queryable); + AssertStages(stages, "{ $project : { _v : { $reduce : { input : '$D3', initialValue : false, in : { $cond : { if : '$$value', then : true, else : { $eq : ['$$this.v', 1] } } } } }, _id : 0 } }"); + + var results = queryable.ToList(); + results.Should().Equal(true, true, false); + } + } + + private IMongoCollection GetCollection(LinqProvider linqProvider) + { + var collection = GetCollection("test", linqProvider); + CreateCollection( + collection, + new User + { + Id = 1, + D1 = new() { { "A", 1 }, { "B", 2 } }, + D2 = new() { { "A", 1 }, { "B", 2 } }, + D3 = new() { { "A", 1 }, { "B", 2 } } + }, + new User + { + Id = 2, + D1 = new() { { "A", 2 }, { "B", 1 } }, + D2 = new() { { "A", 2 }, { "B", 1 } }, + D3 = new() { { "A", 2 }, { "B", 1 } } + }, + new User + { + Id = 3, + D1 = new() { { "A", 2 }, { "B", 3 } }, + D2 = new() { { "A", 2 }, { "B", 3 } }, + D3 = new() { { "A", 2 }, { "B", 3 } } + }); + return collection; + } + + private class User + { + public int Id { get; set; } + [BsonDictionaryOptions(DictionaryRepresentation.Document)] + public Dictionary D1 { get; set; } + [BsonDictionaryOptions(DictionaryRepresentation.ArrayOfArrays)] + public Dictionary D2 { get; set; } + [BsonDictionaryOptions(DictionaryRepresentation.ArrayOfDocuments)] + public Dictionary D3 { get; set; } + } + } +} From 33e14d01526b77196d65a6b8dafecb03dc42eb05 Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:21:49 -0400 Subject: [PATCH 35/38] CSHARP-3757: Redirect read/write retries to other mongos if possible (#1304) --- .../Core/Bindings/ChannelReadBinding.cs | 17 ++- .../Core/Bindings/ChannelReadWriteBinding.cs | 37 +++++ .../Bindings/ChannelSourceReadWriteBinding.cs | 38 +++++ .../Core/Bindings/IBinding.cs | 51 +++++++ .../Core/Bindings/ReadBindingHandle.cs | 18 ++- .../Core/Bindings/ReadPreferenceBinding.cs | 21 ++- .../Core/Bindings/ReadWriteBindingHandle.cs | 47 ++++++- .../Core/Bindings/SingleServerReadBinding.cs | 13 ++ .../Bindings/SingleServerReadWriteBinding.cs | 37 +++++ .../Core/Bindings/WritableServerBinding.cs | 68 +++++++-- .../Core/Clusters/IClusterExtensions.cs | 11 ++ .../ServerSelectors/PriorityServerSelector.cs | 56 ++++++++ .../RetryableReadOperationExecutor.cs | 4 +- .../RetryableWriteOperationExecutor.cs | 4 +- .../Bindings/WritableServerBindingTests.cs | 42 ++++++ .../Core/Clusters/ClusterTests.cs | 92 ++++++++++++ .../PriorityServerSelectorTests.cs | 83 +++++++++++ .../RetryableReadsProseTests.cs | 103 +++++++++++++- .../prose-tests/RetryWriteOnOtherMongos.cs | 133 ++++++++++++++++++ 19 files changed, 844 insertions(+), 31 deletions(-) create mode 100644 src/MongoDB.Driver.Core/Core/Clusters/ServerSelectors/PriorityServerSelector.cs create mode 100644 tests/MongoDB.Driver.Core.Tests/Core/Clusters/ServerSelectors/PriorityServerSelectorTests.cs create mode 100644 tests/MongoDB.Driver.Tests/Specifications/retryable-writes/prose-tests/RetryWriteOnOtherMongos.cs diff --git a/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadBinding.cs index 11b3c29c3fb..c91671dfa75 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadBinding.cs @@ -14,10 +14,9 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using MongoDB.Driver.Core.Clusters; -using MongoDB.Driver.Core.Connections; using MongoDB.Driver.Core.Misc; using MongoDB.Driver.Core.Servers; @@ -51,7 +50,7 @@ public ChannelReadBinding(IServer server, IChannelHandle channel, ReadPreference _session = Ensure.IsNotNull(session, nameof(session)); } - // properties + // properties /// public ReadPreference ReadPreference { @@ -90,6 +89,18 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return Task.FromResult(GetReadChannelSourceHelper()); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSource(cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(cancellationToken); + } + private IChannelSourceHandle GetReadChannelSourceHelper() { return new ChannelSourceHandle(new ChannelChannelSource(_server, _channel.Fork(), _session.Fork())); diff --git a/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadWriteBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadWriteBinding.cs index 2375dd0560d..2e8bf6695f3 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadWriteBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/ChannelReadWriteBinding.cs @@ -14,6 +14,7 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Misc; @@ -85,6 +86,18 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return Task.FromResult(GetChannelSourceHelper()); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSource(cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellationToken) { @@ -92,12 +105,24 @@ public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellation return GetChannelSourceHelper(); } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetWriteChannelSource(cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { return GetWriteChannelSource(cancellationToken); // ignore mayUseSecondary } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSource(mayUseSecondary, cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(CancellationToken cancellationToken) { @@ -105,12 +130,24 @@ public Task GetWriteChannelSourceAsync(CancellationToken c return Task.FromResult(GetChannelSourceHelper()); } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { return GetWriteChannelSourceAsync(cancellationToken); // ignore mayUseSecondary } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(mayUseSecondary, cancellationToken); + } + // private methods private IChannelSourceHandle GetChannelSourceHelper() { diff --git a/src/MongoDB.Driver.Core/Core/Bindings/ChannelSourceReadWriteBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/ChannelSourceReadWriteBinding.cs index 75b8928e297..e434a36a57b 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/ChannelSourceReadWriteBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/ChannelSourceReadWriteBinding.cs @@ -14,9 +14,11 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.Servers; namespace MongoDB.Driver.Core.Bindings { @@ -73,6 +75,18 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return Task.FromResult(GetChannelSourceHelper()); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSource(cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellationToken) { @@ -80,12 +94,24 @@ public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellation return GetChannelSourceHelper(); } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetWriteChannelSource(cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { return GetWriteChannelSource(cancellationToken); // ignore mayUseSecondary } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSource(mayUseSecondary, cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(CancellationToken cancellationToken) { @@ -93,12 +119,24 @@ public Task GetWriteChannelSourceAsync(CancellationToken c return Task.FromResult(GetChannelSourceHelper()); } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { return GetWriteChannelSourceAsync(cancellationToken); // ignore mayUseSecondary } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(mayUseSecondary, cancellationToken); + } + /// public void Dispose() { diff --git a/src/MongoDB.Driver.Core/Core/Bindings/IBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/IBinding.cs index 1e625a04e3c..40c33ee441c 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/IBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/IBinding.cs @@ -14,6 +14,7 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Servers; @@ -61,6 +62,22 @@ public interface IReadBinding : IBinding /// The cancellation token. /// A channel source. Task GetReadChannelSourceAsync(CancellationToken cancellationToken); + + /// + /// Gets a channel source for read operations while deprioritizing servers in the provided collection. + /// + /// The deprioritized servers. + /// The cancellation token. + /// A channel source. + IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken); + + /// + /// Gets a channel source for read operations while deprioritizing servers in the provided collection. + /// + /// The deprioritized servers. + /// The cancellation token. + /// A channel source. + Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken); } /// @@ -75,6 +92,14 @@ public interface IWriteBinding : IBinding /// A channel source. IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellationToken); + /// + /// Gets a channel source for write operations while deprioritizing servers in the provided collection. + /// + /// The deprioritized servers. + /// The cancellation token. + /// A channel source. + IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken); + /// /// Gets a channel source for write operations that may use a secondary. /// @@ -83,6 +108,15 @@ public interface IWriteBinding : IBinding /// A channel source. IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken); + /// + /// Gets a channel source for write operations that may use a secondary and deprioritizes servers in the provided collection. + /// + /// The deprioritized servers. + /// The may use secondary criteria. + /// The cancellation token. + /// A channel source. + IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken); + /// /// Gets a channel source for write operations. /// @@ -90,6 +124,14 @@ public interface IWriteBinding : IBinding /// A channel source. Task GetWriteChannelSourceAsync(CancellationToken cancellationToken); + /// + /// Gets a channel source for write operations while deprioritizing servers in the provided collection. + /// + /// The deprioritized servers. + /// The cancellation token. + /// A channel source. + Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken); + /// /// Gets a channel source for write operations that may use a secondary. /// @@ -97,6 +139,15 @@ public interface IWriteBinding : IBinding /// The cancellation token. /// A channel source. Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken); + + /// + /// Gets a channel source for write operations that may use a secondary and deprioritizes servers in the provided collection. + /// + /// The deprioritized servers. + /// The may use secondary criteria. + /// The cancellation token. + /// A channel source. + Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken); } /// diff --git a/src/MongoDB.Driver.Core/Core/Bindings/ReadBindingHandle.cs b/src/MongoDB.Driver.Core/Core/Bindings/ReadBindingHandle.cs index f190db358d0..6c8283d2bfd 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/ReadBindingHandle.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/ReadBindingHandle.cs @@ -15,12 +15,10 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; -using MongoDB.Driver.Core.Clusters; using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.Servers; namespace MongoDB.Driver.Core.Bindings { @@ -76,6 +74,20 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return _reference.Instance.GetReadChannelSourceAsync(cancellationToken); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetReadChannelSource(deprioritizedServers, cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetReadChannelSourceAsync(deprioritizedServers, cancellationToken); + } + /// public void Dispose() { diff --git a/src/MongoDB.Driver.Core/Core/Bindings/ReadPreferenceBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/ReadPreferenceBinding.cs index 5790a7e5be9..a84fa4fe932 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/ReadPreferenceBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/ReadPreferenceBinding.cs @@ -14,6 +14,7 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Clusters; @@ -66,17 +67,29 @@ public ICoreSessionHandle Session // methods /// public IChannelSourceHandle GetReadChannelSource(CancellationToken cancellationToken) + { + return GetReadChannelSource(null, cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(null, cancellationToken); + } + + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { ThrowIfDisposed(); - var server = _cluster.SelectServerAndPinIfNeeded(_session, _serverSelector, cancellationToken); + var server = _cluster.SelectServerAndPinIfNeeded(_session, _serverSelector, deprioritizedServers, cancellationToken); return GetChannelSourceHelper(server); } - /// - public async Task GetReadChannelSourceAsync(CancellationToken cancellationToken) + /// + public async Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { ThrowIfDisposed(); - var server = await _cluster.SelectServerAndPinIfNeededAsync(_session, _serverSelector, cancellationToken).ConfigureAwait(false); + var server = await _cluster.SelectServerAndPinIfNeededAsync(_session, _serverSelector, deprioritizedServers, cancellationToken).ConfigureAwait(false); return GetChannelSourceHelper(server); } diff --git a/src/MongoDB.Driver.Core/Core/Bindings/ReadWriteBindingHandle.cs b/src/MongoDB.Driver.Core/Core/Bindings/ReadWriteBindingHandle.cs index c205eb33ade..205e47dfdf4 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/ReadWriteBindingHandle.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/ReadWriteBindingHandle.cs @@ -15,13 +15,10 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; -using MongoDB.Driver.Core.Clusters; using MongoDB.Driver.Core.Misc; -using MongoDB.Driver.Core.Operations; +using MongoDB.Driver.Core.Servers; namespace MongoDB.Driver.Core.Bindings { @@ -77,6 +74,20 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return _reference.Instance.GetReadChannelSourceAsync(cancellationToken); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetReadChannelSource(deprioritizedServers, cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetReadChannelSourceAsync(deprioritizedServers, cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellationToken) { @@ -84,6 +95,13 @@ public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellation return _reference.Instance.GetWriteChannelSource(cancellationToken); } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetWriteChannelSource(deprioritizedServers, cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { @@ -91,6 +109,13 @@ public IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUs return _reference.Instance.GetWriteChannelSource(mayUseSecondary, cancellationToken); } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetWriteChannelSource(deprioritizedServers, mayUseSecondary, cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(CancellationToken cancellationToken) { @@ -98,6 +123,13 @@ public Task GetWriteChannelSourceAsync(CancellationToken c return _reference.Instance.GetWriteChannelSourceAsync(cancellationToken); } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetWriteChannelSourceAsync(deprioritizedServers, cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { @@ -105,6 +137,13 @@ public Task GetWriteChannelSourceAsync(IMayUseSecondaryCri return _reference.Instance.GetWriteChannelSourceAsync(mayUseSecondary, cancellationToken); } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + return _reference.Instance.GetWriteChannelSourceAsync(deprioritizedServers, mayUseSecondary, cancellationToken); + } + /// public void Dispose() { diff --git a/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadBinding.cs index 4415c65d325..7666ef3e719 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadBinding.cs @@ -14,6 +14,7 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Misc; @@ -77,6 +78,18 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return Task.FromResult(GetChannelSourceHelper()); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSource(cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(cancellationToken); + } + /// public void Dispose() { diff --git a/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadWriteBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadWriteBinding.cs index 09acb647105..8fec85d3249 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadWriteBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/SingleServerReadWriteBinding.cs @@ -14,6 +14,7 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Misc; @@ -81,6 +82,18 @@ public Task GetReadChannelSourceAsync(CancellationToken ca return Task.FromResult(GetChannelSourceHelper()); } + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSource(cancellationToken); + } + + /// + public Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellationToken) { @@ -88,12 +101,24 @@ public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellation return GetChannelSourceHelper(); } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetWriteChannelSource(cancellationToken); + } + /// public IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { return GetWriteChannelSource(cancellationToken); // ignore mayUseSecondary } + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSource(mayUseSecondary, cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(CancellationToken cancellationToken) { @@ -101,12 +126,24 @@ public Task GetWriteChannelSourceAsync(CancellationToken c return Task.FromResult(GetChannelSourceHelper()); } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(cancellationToken); + } + /// public Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { return GetWriteChannelSourceAsync(cancellationToken); // ignore mayUseSecondary } + /// + public Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(mayUseSecondary, cancellationToken); + } + private IChannelSourceHandle GetChannelSourceHelper() { return new ChannelSourceHandle(new ServerChannelSource(_server, _session.Fork())); diff --git a/src/MongoDB.Driver.Core/Core/Bindings/WritableServerBinding.cs b/src/MongoDB.Driver.Core/Core/Bindings/WritableServerBinding.cs index 8bb61bb5563..44f323d1360 100644 --- a/src/MongoDB.Driver.Core/Core/Bindings/WritableServerBinding.cs +++ b/src/MongoDB.Driver.Core/Core/Bindings/WritableServerBinding.cs @@ -14,6 +14,7 @@ */ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Clusters; @@ -62,58 +63,103 @@ public ICoreSessionHandle Session /// public IChannelSourceHandle GetReadChannelSource(CancellationToken cancellationToken) { - ThrowIfDisposed(); - var server = _cluster.SelectServerAndPinIfNeeded(_session, WritableServerSelector.Instance, cancellationToken); + return GetReadChannelSource(null, cancellationToken); + } + /// + public Task GetReadChannelSourceAsync(CancellationToken cancellationToken) + { + return GetReadChannelSourceAsync(null, cancellationToken); + } + + /// + public IChannelSourceHandle GetReadChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) + { + ThrowIfDisposed(); + var server = _cluster.SelectServerAndPinIfNeeded(_session, WritableServerSelector.Instance, deprioritizedServers, cancellationToken); return CreateServerChannelSource(server); } - /// - public async Task GetReadChannelSourceAsync(CancellationToken cancellationToken) + /// + public async Task GetReadChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { ThrowIfDisposed(); - var server = await _cluster.SelectServerAndPinIfNeededAsync(_session, WritableServerSelector.Instance, cancellationToken).ConfigureAwait(false); + var server = await _cluster.SelectServerAndPinIfNeededAsync(_session, WritableServerSelector.Instance, deprioritizedServers, cancellationToken).ConfigureAwait(false); return CreateServerChannelSource(server); } /// public IChannelSourceHandle GetWriteChannelSource(CancellationToken cancellationToken) + { + return GetWriteChannelSource(deprioritizedServers: null, cancellationToken); + } + + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { ThrowIfDisposed(); - var server = _cluster.SelectServerAndPinIfNeeded(_session, WritableServerSelector.Instance, cancellationToken); + var server = _cluster.SelectServerAndPinIfNeeded(_session, WritableServerSelector.Instance, deprioritizedServers, cancellationToken); return CreateServerChannelSource(server); } /// public IChannelSourceHandle GetWriteChannelSource(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSource(null, mayUseSecondary, cancellationToken); + } + + /// + public IChannelSourceHandle GetWriteChannelSource(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { if (IsSessionPinnedToServer()) { throw new InvalidOperationException($"This overload of {nameof(GetWriteChannelSource)} cannot be called when pinned to a server."); } - var selector = new WritableServerSelector(mayUseSecondary); + var writableServerSelector = new WritableServerSelector(mayUseSecondary); + + var selector = deprioritizedServers != null + ? (IServerSelector)new CompositeServerSelector(new IServerSelector[] { new PriorityServerSelector(deprioritizedServers), writableServerSelector }) + : writableServerSelector; + var server = _cluster.SelectServer(selector, cancellationToken); return CreateServerChannelSource(server); } /// - public async Task GetWriteChannelSourceAsync(CancellationToken cancellationToken) + public Task GetWriteChannelSourceAsync(CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(deprioritizedServers: null, cancellationToken); + } + + /// + public async Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { ThrowIfDisposed(); - var server = await _cluster.SelectServerAndPinIfNeededAsync(_session, WritableServerSelector.Instance, cancellationToken).ConfigureAwait(false); + var server = await _cluster.SelectServerAndPinIfNeededAsync(_session, WritableServerSelector.Instance, deprioritizedServers, cancellationToken).ConfigureAwait(false); return CreateServerChannelSource(server); } /// - public async Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + public Task GetWriteChannelSourceAsync(IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) + { + return GetWriteChannelSourceAsync(null, mayUseSecondary, cancellationToken); + } + + /// + public async Task GetWriteChannelSourceAsync(IReadOnlyCollection deprioritizedServers, IMayUseSecondaryCriteria mayUseSecondary, CancellationToken cancellationToken) { if (IsSessionPinnedToServer()) { throw new InvalidOperationException($"This overload of {nameof(GetWriteChannelSource)} cannot be called when pinned to a server."); } - var selector = new WritableServerSelector(mayUseSecondary); + var writableServerSelector = new WritableServerSelector(mayUseSecondary); + + IServerSelector selector = deprioritizedServers != null + ? new CompositeServerSelector(new IServerSelector[] { new PriorityServerSelector(deprioritizedServers), writableServerSelector }) + : writableServerSelector; + var server = await _cluster.SelectServerAsync(selector, cancellationToken).ConfigureAwait(false); return CreateServerChannelSource(server); } diff --git a/src/MongoDB.Driver.Core/Core/Clusters/IClusterExtensions.cs b/src/MongoDB.Driver.Core/Core/Clusters/IClusterExtensions.cs index e5cabd39243..6b83a44715d 100644 --- a/src/MongoDB.Driver.Core/Core/Clusters/IClusterExtensions.cs +++ b/src/MongoDB.Driver.Core/Core/Clusters/IClusterExtensions.cs @@ -13,6 +13,7 @@ * limitations under the License. */ +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MongoDB.Driver.Core.Bindings; @@ -30,6 +31,7 @@ public static IServer SelectServerAndPinIfNeeded( this ICluster cluster, ICoreSessionHandle session, IServerSelector selector, + IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { var pinnedServer = GetPinnedServerIfValid(cluster, session); @@ -38,6 +40,10 @@ public static IServer SelectServerAndPinIfNeeded( return pinnedServer; } + selector = deprioritizedServers != null + ? new CompositeServerSelector(new[] { new PriorityServerSelector(deprioritizedServers), selector }) + : selector; + // Server selection also updates the cluster type, allowing us to to determine if the server // should be pinned. var server = cluster.SelectServer(selector, cancellationToken); @@ -49,6 +55,7 @@ public static async Task SelectServerAndPinIfNeededAsync( this ICluster cluster, ICoreSessionHandle session, IServerSelector selector, + IReadOnlyCollection deprioritizedServers, CancellationToken cancellationToken) { var pinnedServer = GetPinnedServerIfValid(cluster, session); @@ -57,6 +64,10 @@ public static async Task SelectServerAndPinIfNeededAsync( return pinnedServer; } + selector = deprioritizedServers != null + ? new CompositeServerSelector(new[] { new PriorityServerSelector(deprioritizedServers), selector }) + : selector; + // Server selection also updates the cluster type, allowing us to to determine if the server // should be pinned. var server = await cluster.SelectServerAsync(selector, cancellationToken).ConfigureAwait(false); diff --git a/src/MongoDB.Driver.Core/Core/Clusters/ServerSelectors/PriorityServerSelector.cs b/src/MongoDB.Driver.Core/Core/Clusters/ServerSelectors/PriorityServerSelector.cs new file mode 100644 index 00000000000..6d125a879a7 --- /dev/null +++ b/src/MongoDB.Driver.Core/Core/Clusters/ServerSelectors/PriorityServerSelector.cs @@ -0,0 +1,56 @@ +/* Copyright 2010-present MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.Servers; + +namespace MongoDB.Driver.Core.Clusters.ServerSelectors +{ + /// + /// Represents a server selector that selects servers based on a collection of servers to deprioritize. + /// + public sealed class PriorityServerSelector : IServerSelector + { + private readonly IReadOnlyCollection _deprioritizedServers; + + /// + /// Initializes a new instance of the class. + /// + /// The collection of servers to deprioritize. + public PriorityServerSelector(IReadOnlyCollection deprioritizedServers) + { + _deprioritizedServers = Ensure.IsNotNullOrEmpty(deprioritizedServers, nameof(deprioritizedServers)) as IReadOnlyCollection; + } + + /// + public IEnumerable SelectServers(ClusterDescription cluster, IEnumerable servers) + { + // according to spec, we only do deprioritization in a sharded cluster. + if (cluster.Type != ClusterType.Sharded) + { + return servers; + } + + var filteredServers = servers.Where(description => _deprioritizedServers.All(d => d.EndPoint != description.EndPoint)).ToList(); + + return filteredServers.Any() ? filteredServers : servers; + } + + /// + public override string ToString() => $"PriorityServerSelector{{{{ Deprioritized servers: {string.Join(", ", _deprioritizedServers.Select(s => s.EndPoint))} }}}}"; + } +} diff --git a/src/MongoDB.Driver.Core/Core/Operations/RetryableReadOperationExecutor.cs b/src/MongoDB.Driver.Core/Core/Operations/RetryableReadOperationExecutor.cs index d17cc437ab3..cdb74827dc0 100644 --- a/src/MongoDB.Driver.Core/Core/Operations/RetryableReadOperationExecutor.cs +++ b/src/MongoDB.Driver.Core/Core/Operations/RetryableReadOperationExecutor.cs @@ -51,7 +51,7 @@ public static TResult Execute(IRetryableReadOperation operatio try { - context.ReplaceChannelSource(context.Binding.GetReadChannelSource(cancellationToken)); + context.ReplaceChannelSource(context.Binding.GetReadChannelSource(new[] { context.ChannelSource.ServerDescription }, cancellationToken)); context.ReplaceChannel(context.ChannelSource.GetChannel(cancellationToken)); } catch @@ -96,7 +96,7 @@ public static async Task ExecuteAsync(IRetryableReadOperation< try { - context.ReplaceChannelSource(context.Binding.GetReadChannelSource(cancellationToken)); + context.ReplaceChannelSource(context.Binding.GetReadChannelSource(new[] { context.ChannelSource.ServerDescription }, cancellationToken)); context.ReplaceChannel(context.ChannelSource.GetChannel(cancellationToken)); } catch diff --git a/src/MongoDB.Driver.Core/Core/Operations/RetryableWriteOperationExecutor.cs b/src/MongoDB.Driver.Core/Core/Operations/RetryableWriteOperationExecutor.cs index 86087929ae7..6e73af1e758 100644 --- a/src/MongoDB.Driver.Core/Core/Operations/RetryableWriteOperationExecutor.cs +++ b/src/MongoDB.Driver.Core/Core/Operations/RetryableWriteOperationExecutor.cs @@ -53,7 +53,7 @@ public static TResult Execute(IRetryableWriteOperation operati try { - context.ReplaceChannelSource(context.Binding.GetWriteChannelSource(cancellationToken)); + context.ReplaceChannelSource(context.Binding.GetWriteChannelSource(new[] { context.ChannelSource.ServerDescription }, cancellationToken)); context.ReplaceChannel(context.ChannelSource.GetChannel(cancellationToken)); } catch @@ -104,7 +104,7 @@ public static async Task ExecuteAsync(IRetryableWriteOperation try { - context.ReplaceChannelSource(await context.Binding.GetWriteChannelSourceAsync(cancellationToken).ConfigureAwait(false)); + context.ReplaceChannelSource(await context.Binding.GetWriteChannelSourceAsync(new[] { context.ChannelSource.ServerDescription }, cancellationToken).ConfigureAwait(false)); context.ReplaceChannel(await context.ChannelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false)); } catch diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Bindings/WritableServerBindingTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Bindings/WritableServerBindingTests.cs index 9ee482c2f6a..55a9d3eca62 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Bindings/WritableServerBindingTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Bindings/WritableServerBindingTests.cs @@ -200,6 +200,48 @@ public void GetWriteChannelSourceAsync_should_use_a_writable_server_selector_to_ } } + [Theory] + [ParameterAttributeData] + public async Task GetWriteChannelSource_should_use_a_composite_server_selector_to_select_the_server_from_the_cluster_when_deprioritized_servers_present( + [Values(false, true)] + bool async) + { + var subject = new WritableServerBinding(_mockCluster.Object, NoCoreSession.NewHandle()); + var selectedServer = new Mock().Object; + + var clusterId = new ClusterId(); + var endPoint = new DnsEndPoint("localhost", 27017); + var server = new ServerDescription(new ServerId(clusterId, endPoint), endPoint); +#pragma warning disable CS0618 // Type or member is obsolete + var initialClusterDescription = new ClusterDescription( + clusterId, + ClusterConnectionMode.Sharded, + ClusterType.Unknown, + new[] { server }); +#pragma warning restore CS0618 // Type or member is obsolete + var finalClusterDescription = initialClusterDescription.WithType(ClusterType.Sharded); + _mockCluster.SetupSequence(c => c.Description).Returns(initialClusterDescription).Returns(finalClusterDescription); + + var deprioritizedServers = new ServerDescription[] { server }; + + if (async) + { + _mockCluster.Setup(c => c.SelectServerAsync(It.Is(cp => cp.ToString().Contains("PriorityServerSelector")), CancellationToken.None)).Returns(Task.FromResult(selectedServer)); + + await subject.GetWriteChannelSourceAsync(deprioritizedServers, CancellationToken.None); + + _mockCluster.Verify(c => c.SelectServerAsync(It.Is(cp => cp.ToString().Contains("PriorityServerSelector")), CancellationToken.None), Times.Once); + } + else + { + _mockCluster.Setup(c => c.SelectServer(It.Is(cp => cp.ToString().Contains("PriorityServerSelector")), CancellationToken.None)).Returns(selectedServer); + + subject.GetWriteChannelSource(deprioritizedServers, CancellationToken.None); + + _mockCluster.Verify(c => c.SelectServer(It.Is(c => c.ToString().Contains("PriorityServerSelector")), CancellationToken.None), Times.Once); + } + } + [Theory] [ParameterAttributeData] public void GetWriteChannelSource_with_mayUseSecondary_should_pass_mayUseSecondary_to_server_selector( diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ClusterTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ClusterTests.cs index d84ab3f6465..d367ab5c0aa 100644 --- a/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ClusterTests.cs +++ b/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ClusterTests.cs @@ -413,6 +413,93 @@ public void SelectServer_should_keep_trying_to_match_by_waiting_on_cluster_descr _capturedEvents.Any().Should().BeFalse(); } + [Theory] + [ParameterAttributeData] + public async Task SelectServer_should_ignore_deprioritized_servers_if_cluster_is_sharded( + [Values(false, true)] + bool async) + { +#pragma warning disable CS0618 // Type or member is obsolete + var subject = CreateSubject(ClusterConnectionMode.Sharded); +#pragma warning restore CS0618 // Type or member is obsolete + + subject.Initialize(); + + var connected1 = ServerDescriptionHelper.Connected(subject.Description.ClusterId, new DnsEndPoint("localhost", 27017)); + var connected2 = ServerDescriptionHelper.Connected(subject.Description.ClusterId, new DnsEndPoint("localhost", 27018)); + var connected3 = ServerDescriptionHelper.Connected(subject.Description.ClusterId, new DnsEndPoint("localhost", 27019)); + + subject.SetServerDescriptions(connected1, connected2, connected3); + + var deprioritizedServers = new List { connected1 }; + + var selector = new PriorityServerSelector(deprioritizedServers); + + for (int i = 0; i < 15; i++) + { + _capturedEvents.Clear(); + + IServer result; + if (async) + { + result = await subject.SelectServerAsync(selector, CancellationToken.None); + } + else + { + result = subject.SelectServer(selector, CancellationToken.None); + } + + result.Should().NotBeNull(); + + deprioritizedServers.Should().NotContain(d => d.EndPoint == result.Description.EndPoint); + + _capturedEvents.Next().Should().BeOfType(); + _capturedEvents.Next().Should().BeOfType(); + _capturedEvents.Any().Should().BeFalse(); + } + } + + [Theory] + [ParameterAttributeData] + public async Task SelectServer_should_return_deprioritized_servers_if_no_other_servers_exist_or_cluster_not_sharded( + [Values(false, true)] bool async, + [Values(false, true)] bool isSharded) + { +#pragma warning disable CS0618 // Type or member is obsolete + StubCluster subject = isSharded ? CreateSubject(ClusterConnectionMode.Sharded) : CreateSubject(); +#pragma warning restore CS0618 // Type or member is obsolete + + subject.Initialize(); + + var connected1 = ServerDescriptionHelper.Connected(subject.Description.ClusterId, new DnsEndPoint("localhost", 27017)); + var connected2 = ServerDescriptionHelper.Connected(subject.Description.ClusterId, new DnsEndPoint("localhost", 27018)); + + subject.SetServerDescriptions(connected1, connected2); + + var deprioritizedServers = new List { connected1, connected2 }; + + var selector = new PriorityServerSelector(deprioritizedServers); + + _capturedEvents.Clear(); + IServer result; + if (async) + { + result = await subject.SelectServerAsync(selector, CancellationToken.None); + } + else + { + result = subject.SelectServer(selector, CancellationToken.None); + } + + result.Should().NotBeNull(); + + deprioritizedServers.Should().Contain(d => d.EndPoint == result.Description.EndPoint); + + _capturedEvents.Next().Should().BeOfType(); + _capturedEvents.Next().Should().BeOfType(); + _capturedEvents.Any().Should().BeFalse(); + } + [Fact] public void StartSession_should_return_expected_result() { @@ -431,6 +518,10 @@ public void DescriptionChanged_should_be_raised_when_the_description_changes() int count = 0; var subject = CreateSubject(); subject.Initialize(); + + // clear the ClusterDescriptionChanged event from initializing the StubCluster + _capturedEvents.Clear(); + subject.DescriptionChanged += (o, e) => count++; subject.SetServerDescriptions(ServerDescriptionHelper.Connected(subject.Description.ClusterId)); @@ -591,6 +682,7 @@ public StubCluster(ClusterSettings settings, public override void Initialize() { base.Initialize(); + UpdateClusterDescription(Description.WithType(Settings.GetInitialClusterType())); } public void RemoveServer(EndPoint endPoint) diff --git a/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ServerSelectors/PriorityServerSelectorTests.cs b/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ServerSelectors/PriorityServerSelectorTests.cs new file mode 100644 index 00000000000..16aa265ad29 --- /dev/null +++ b/tests/MongoDB.Driver.Core.Tests/Core/Clusters/ServerSelectors/PriorityServerSelectorTests.cs @@ -0,0 +1,83 @@ +/* Copyright 2010-present MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Linq; +using System.Net; +using FluentAssertions; +using MongoDB.Driver.Core.Helpers; +using MongoDB.Driver.Core.Servers; +using Xunit; + +namespace MongoDB.Driver.Core.Clusters.ServerSelectors; + +public class PriorityServerSelectorTests +{ + private readonly ClusterDescription _description; + private readonly ServerDescription _server1; + private readonly ServerDescription _server2; + private readonly ServerDescription _server3; + + public PriorityServerSelectorTests() + { + var clusterId = new ClusterId(); + + _server1 = ServerDescriptionHelper.Connected(clusterId, new DnsEndPoint("localhost", 27017)); + _server2 = ServerDescriptionHelper.Connected(clusterId, new DnsEndPoint("localhost", 27018)); + _server3 = ServerDescriptionHelper.Connected(clusterId, new DnsEndPoint("localhost", 27019)); + +#pragma warning disable CS0618 // Type or member is obsolete + _description = new ClusterDescription( + clusterId, + ClusterConnectionMode.Sharded, + ClusterType.Sharded, + new[] { _server1, _server2, _server3 }); +#pragma warning restore CS0618 // Type or member is obsolete + } + + [Fact] + public void Should_select_all_the_servers_not_deprioritized() + { + var subject = new PriorityServerSelector(new[] { _server1, _server2 }); + + var result = subject.SelectServers(_description, _description.Servers).ToList(); + + result.Count.Should().Be(1); + result.Should().BeEquivalentTo(_server3); + } + + [Fact] + public void Should_select_all_the_servers_if_all_servers_are_deprioritized() + { + var subject = new PriorityServerSelector(new[] { _server1, _server2, _server3}); + + var result = subject.SelectServers(_description, _description.Servers).ToList(); + + result.Count.Should().Be(3); + result.Should().BeEquivalentTo(_description.Servers); + } + + [Fact] + public void Should_ignore_deprioritized_servers_if_not_in_sharded_mode() + { + var changedDescription = _description.WithType(ClusterType.Unknown); + + var subject = new PriorityServerSelector(new[] { _server2, _server3 }); + + var result = subject.SelectServers(changedDescription, _description.Servers).ToList(); + + result.Count.Should().Be(3); + result.Should().BeEquivalentTo(_description.Servers); + } +} diff --git a/tests/MongoDB.Driver.Tests/Specifications/retryable-reads/RetryableReadsProseTests.cs b/tests/MongoDB.Driver.Tests/Specifications/retryable-reads/RetryableReadsProseTests.cs index 79f34dbafca..db8330df4ed 100644 --- a/tests/MongoDB.Driver.Tests/Specifications/retryable-reads/RetryableReadsProseTests.cs +++ b/tests/MongoDB.Driver.Tests/Specifications/retryable-reads/RetryableReadsProseTests.cs @@ -20,15 +20,16 @@ using FluentAssertions; using MongoDB.Bson; using MongoDB.Bson.TestHelpers; -using MongoDB.TestHelpers.XunitExtensions; using MongoDB.Driver.Core; using MongoDB.Driver.Core.Bindings; +using MongoDB.Driver.Core.Clusters; using MongoDB.Driver.Core.Clusters.ServerSelectors; using MongoDB.Driver.Core.Events; using MongoDB.Driver.Core.Misc; using MongoDB.Driver.Core.TestHelpers; using MongoDB.Driver.Core.TestHelpers.XunitExtensions; using MongoDB.Driver.TestHelpers; +using MongoDB.TestHelpers.XunitExtensions; using Xunit; namespace MongoDB.Driver.Tests.Specifications.retryable_reads @@ -60,7 +61,7 @@ public async Task PoolClearedError_read_retryablity_test([Values(true, false)] b IServerSelector failPointSelector = new ReadPreferenceServerSelector(ReadPreference.Primary); var settings = DriverTestConfiguration.GetClientSettings(); - if (CoreTestConfiguration.Cluster.Description.Type == Core.Clusters.ClusterType.Sharded) + if (CoreTestConfiguration.Cluster.Description.Type == ClusterType.Sharded) { var serverAddress = settings.Servers.First(); settings.Servers = new[] { serverAddress }; @@ -120,6 +121,104 @@ await ThreadingUtilities.ExecuteTasksOnNewThreads(2, async __ => eventCapturer.Events.OfType().Count().Should().Be(1); } + [Fact] + public void Sharded_cluster_retryable_reads_are_retried_on_different_mongos_if_available() + { + RequireServer.Check() + .Supports(Feature.FailPointsFailCommandForSharded) + .ClusterTypes(ClusterType.Sharded) + .MultipleMongoses(true); + + var failPointCommand = BsonDocument.Parse( + @"{ + configureFailPoint: ""failCommand"", + mode: { times: 1 }, + data: + { + failCommands: [""find""], + errorCode: 6 + } + }"); + + var eventCapturer = new EventCapturer().CaptureCommandEvents("find"); + + using var client = DriverTestConfiguration.CreateDisposableClient( + s => + { + s.RetryReads = true; + s.ClusterConfigurator = b => b.Subscribe(eventCapturer); + } + , null, useMultipleShardRouters: true); + + var failPointServer1 = client.Cluster.SelectServer(new EndPointServerSelector(client.Cluster.Description.Servers[0].EndPoint), default); + var failPointServer2 = client.Cluster.SelectServer(new EndPointServerSelector(client.Cluster.Description.Servers[1].EndPoint), default); + + using var failPoint1 = FailPoint.Configure(failPointServer1, NoCoreSession.NewHandle(), failPointCommand); + using var failPoint2 = FailPoint.Configure(failPointServer2, NoCoreSession.NewHandle(), failPointCommand); + + var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName); + var collection = database.GetCollection(DriverTestConfiguration.CollectionNamespace.CollectionName); + + Assert.Throws(() => + { + collection.Find(Builders.Filter.Empty).ToList(); + }); + + var failedEvents = eventCapturer.Events.OfType().ToArray(); + failedEvents.Length.Should().Be(2); + + failedEvents[0].CommandName.Should().Be(failedEvents[1].CommandName).And.Be("find"); + failedEvents[0].ConnectionId.ServerId.Should().NotBe(failedEvents[1].ConnectionId.ServerId); + } + + [Fact] + public void Sharded_cluster_retryable_reads_are_retried_on_same_mongos_if_no_other_is_available() + { + RequireServer.Check() + .Supports(Feature.FailPointsFailCommandForSharded) + .ClusterTypes(ClusterType.Sharded); + + var failPointCommand = BsonDocument.Parse( + @"{ + configureFailPoint: ""failCommand"", + mode: { times: 1 }, + data: + { + failCommands: [""find""], + errorCode: 6 + } + }"); + + var eventCapturer = new EventCapturer().CaptureCommandEvents("find"); + + using var client = DriverTestConfiguration.CreateDisposableClient( + s => + { + s.RetryReads = true; + s.DirectConnection = false; + s.ClusterConfigurator = b => b.Subscribe(eventCapturer); + } + , null, useMultipleShardRouters: false); + + var failPointServer = client.Cluster.SelectServer(new EndPointServerSelector(client.Cluster.Description.Servers[0].EndPoint), default); + + using var failPoint = FailPoint.Configure(failPointServer, NoCoreSession.NewHandle(), failPointCommand); + + var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName); + var collection = database.GetCollection(DriverTestConfiguration.CollectionNamespace.CollectionName); + + collection.Find(Builders.Filter.Empty).ToList(); + + var failedEvents = eventCapturer.Events.OfType().ToArray(); + var succeededEvents = eventCapturer.Events.OfType().ToArray(); + + failedEvents.Length.Should().Be(1); + succeededEvents.Length.Should().Be(1); + + failedEvents[0].CommandName.Should().Be(succeededEvents[0].CommandName).And.Be("find"); + failedEvents[0].ConnectionId.ServerId.Should().Be(succeededEvents[0].ConnectionId.ServerId); + } + // private methods private DisposableMongoClient CreateClient(MongoClientSettings mongoClientSettings, EventCapturer eventCapturer, TimeSpan heartbeatInterval, string applicationName = null) { diff --git a/tests/MongoDB.Driver.Tests/Specifications/retryable-writes/prose-tests/RetryWriteOnOtherMongos.cs b/tests/MongoDB.Driver.Tests/Specifications/retryable-writes/prose-tests/RetryWriteOnOtherMongos.cs new file mode 100644 index 00000000000..7240eb86c08 --- /dev/null +++ b/tests/MongoDB.Driver.Tests/Specifications/retryable-writes/prose-tests/RetryWriteOnOtherMongos.cs @@ -0,0 +1,133 @@ +/* Copyright 2010-present MongoDB Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Linq; +using FluentAssertions; +using MongoDB.Bson; +using MongoDB.Driver.Core; +using MongoDB.Driver.Core.Bindings; +using MongoDB.Driver.Core.Clusters; +using MongoDB.Driver.Core.Clusters.ServerSelectors; +using MongoDB.Driver.Core.Events; +using MongoDB.Driver.Core.Misc; +using MongoDB.Driver.Core.TestHelpers; +using MongoDB.Driver.Core.TestHelpers.XunitExtensions; +using Xunit; + +namespace MongoDB.Driver.Tests.Specifications.retryable_writes.prose_tests +{ + public class RetryWriteOnOtherMongos + { + [Fact] + public void Sharded_cluster_retryable_writes_are_retried_on_different_mongos_if_available() + { + RequireServer.Check() + .Supports(Feature.FailPointsFailCommandForSharded) + .ClusterTypes(ClusterType.Sharded) + .MultipleMongoses(true); + + var failPointCommand = BsonDocument.Parse( + @"{ + configureFailPoint: ""failCommand"", + mode: { times: 1 }, + data: + { + failCommands: [""insert""], + errorCode: 6, + errorLabels: [""RetryableWriteError""] + } + }"); + + var eventCapturer = new EventCapturer().CaptureCommandEvents("insert"); + + using var client = DriverTestConfiguration.CreateDisposableClient( + s => + { + s.RetryWrites = true; + s.ClusterConfigurator = b => b.Subscribe(eventCapturer); + } + , null, useMultipleShardRouters: true); + + var failPointServer1 = client.Cluster.SelectServer(new EndPointServerSelector(client.Cluster.Description.Servers[0].EndPoint), default); + var failPointServer2 = client.Cluster.SelectServer(new EndPointServerSelector(client.Cluster.Description.Servers[1].EndPoint), default); + + using var failPoint1 = FailPoint.Configure(failPointServer1, NoCoreSession.NewHandle(), failPointCommand); + using var failPoint2 = FailPoint.Configure(failPointServer2, NoCoreSession.NewHandle(), failPointCommand); + + var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName); + var collection = database.GetCollection(DriverTestConfiguration.CollectionNamespace.CollectionName); + + Assert.Throws(() => + { + collection.InsertOne(new BsonDocument("x", 1)); + }); + + var failedEvents = eventCapturer.Events.OfType().ToArray(); + failedEvents.Length.Should().Be(2); + + failedEvents[0].CommandName.Should().Be(failedEvents[1].CommandName).And.Be("insert"); + failedEvents[0].ConnectionId.ServerId.Should().NotBe(failedEvents[1].ConnectionId.ServerId); + } + + [Fact] + public void Sharded_cluster_retryable_writes_are_retried_on_same_mongo_if_no_other_is_available() + { + RequireServer.Check() + .Supports(Feature.FailPointsFailCommandForSharded) + .ClusterTypes(ClusterType.Sharded); + + var failPointCommand = BsonDocument.Parse( + @"{ + configureFailPoint: ""failCommand"", + mode: { times: 1 }, + data: + { + failCommands: [""insert""], + errorCode: 6, + errorLabels: [""RetryableWriteError""] + } + }"); + + var eventCapturer = new EventCapturer().CaptureCommandEvents("insert"); + + using var client = DriverTestConfiguration.CreateDisposableClient( + s => + { + s.RetryWrites = true; + s.DirectConnection = false; + s.ClusterConfigurator = b => b.Subscribe(eventCapturer); + } + , null, useMultipleShardRouters: false); + + var failPointServer = client.Cluster.SelectServer(new EndPointServerSelector(client.Cluster.Description.Servers[0].EndPoint), default); + + using var failPoint = FailPoint.Configure(failPointServer, NoCoreSession.NewHandle(), failPointCommand); + + var database = client.GetDatabase(DriverTestConfiguration.DatabaseNamespace.DatabaseName); + var collection = database.GetCollection(DriverTestConfiguration.CollectionNamespace.CollectionName); + + collection.InsertOne(new BsonDocument("x", 1)); + + var failedEvents = eventCapturer.Events.OfType().ToArray(); + var succeededEvents = eventCapturer.Events.OfType().ToArray(); + + failedEvents.Length.Should().Be(1); + succeededEvents.Length.Should().Be(1); + + failedEvents[0].CommandName.Should().Be(succeededEvents[0].CommandName).And.Be("insert"); + failedEvents[0].ConnectionId.ServerId.Should().Be(succeededEvents[0].ConnectionId.ServerId); + } + } +} From ca17b40cfec6506750ff04ae4aa2dba23bf0636f Mon Sep 17 00:00:00 2001 From: Adelin Owona <51498470+adelinowona@users.noreply.github.com> Date: Wed, 12 Jun 2024 14:16:34 -0400 Subject: [PATCH 36/38] CSHARP-5048: Integrate with silk and get SBOM document for releases (#1340) --- evergreen/download-augmented-sbom.sh | 16 ++++++++++ evergreen/evergreen.yml | 30 +++++++++++++++++-- evergreen/template_ssdlc_compliance_report.md | 8 ++--- 3 files changed, 46 insertions(+), 8 deletions(-) create mode 100755 evergreen/download-augmented-sbom.sh diff --git a/evergreen/download-augmented-sbom.sh b/evergreen/download-augmented-sbom.sh new file mode 100755 index 00000000000..7b277f88055 --- /dev/null +++ b/evergreen/download-augmented-sbom.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# Environment variables used as input: +# SILK_CLIENT_ID +# SILK_CLIENT_SECRET + +declare -r SSDLC_PATH="./artifacts/ssdlc" +mkdir -p "${SSDLC_PATH}" + +echo "Downloading augmented sbom from silk" + +docker run --platform="linux/amd64" --rm -v ${PWD}:/pwd \ + -e SILK_CLIENT_ID \ + -e SILK_CLIENT_SECRET \ + artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0 \ + download --silk-asset-group mongodb-dotnet-csharp-driver --sbom-out /pwd/${SSDLC_PATH}/augmented-sbom.json diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index a87115b75bd..f7b5788d600 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -254,6 +254,29 @@ functions: params: file: mo-expansion.yml + download-and-promote-augmented-sbom-to-s3-bucket: + - command: shell.exec + params: + working_dir: "mongo-csharp-driver" + include_expansions_in_env: + - "SILK_CLIENT_ID" + - "SILK_CLIENT_SECRET" + script: | + ${PREPARE_SHELL} + ./evergreen/download-augmented-sbom.sh + - command: s3.put + params: + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + local_file: ./mongo-csharp-driver/artifacts/ssdlc/augmented-sbom.json + remote_file: mongo-csharp-driver/${PACKAGE_VERSION}/augmented-sbom.json + bucket: csharp-driver-release-assets + region: us-west-2 + permissions: private + content_type: application/json + display_name: augmented-sbom.json + generate-ssdlc-report: - command: shell.exec params: @@ -264,9 +287,6 @@ functions: script: | ${PREPARE_SHELL} ./evergreen/generate-ssdlc-report.sh - - command: ec2.assume_role - params: - role_arn: ${UPLOAD_SSDLC_RELEASE_ASSETS_ROLE_ARN} - command: s3.put params: aws_key: ${AWS_ACCESS_KEY_ID} @@ -1898,8 +1918,12 @@ tasks: - name: generate-ssdlc-reports commands: + - command: ec2.assume_role + params: + role_arn: ${UPLOAD_SSDLC_RELEASE_ASSETS_ROLE_ARN} - func: download-packages - func: trace-artifacts + - func: download-and-promote-augmented-sbom-to-s3-bucket - func: generate-ssdlc-report - name: validate-apidocs diff --git a/evergreen/template_ssdlc_compliance_report.md b/evergreen/template_ssdlc_compliance_report.md index 8c882d89a0e..3dc92df9255 100644 --- a/evergreen/template_ssdlc_compliance_report.md +++ b/evergreen/template_ssdlc_compliance_report.md @@ -1,7 +1,7 @@ # ${PRODUCT_NAME} SSDLC compliance report This report is available -here. +here. @@ -46,13 +46,11 @@ The MongoDB SSDLC policy is available at ## Third-darty dependency information -There are no dependencies to report vulnerabilities of. -Our [SBOM](https://docs.devprod.prod.corp.mongodb.com/mms/python/src/sbom/silkbomb/docs/CYCLONEDX/) lite -is . +Our third party report is available here. ## Static analysis findings -Coverity static analysis report is available here, under mongodb-csharp-driver project. +Coverity static analysis report is available here. ## Signature information From 281960b95b6b22fa8fc1ee3a87b6f68fbe50facc Mon Sep 17 00:00:00 2001 From: BorisDog Date: Wed, 12 Jun 2024 11:46:13 -0700 Subject: [PATCH 37/38] CSHARP-5128 Update template_ssdlc_compliance_report.md with signing information (#1344) --- evergreen/evergreen.yml | 1 + evergreen/generate-ssdlc-report.sh | 2 ++ evergreen/template_ssdlc_compliance_report.md | 11 ++++++++--- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/evergreen/evergreen.yml b/evergreen/evergreen.yml index f7b5788d600..904fa07179e 100644 --- a/evergreen/evergreen.yml +++ b/evergreen/evergreen.yml @@ -282,6 +282,7 @@ functions: params: working_dir: "mongo-csharp-driver" env: + NUGET_SIGN_CERTIFICATE_FINGERPRINT: ${NUGET_SIGN_CERTIFICATE_FINGERPRINT} PRODUCT_NAME: "mongo-csharp-driver" github_commit: ${github_commit} script: | diff --git a/evergreen/generate-ssdlc-report.sh b/evergreen/generate-ssdlc-report.sh index e28a8958dc1..81d58e99269 100644 --- a/evergreen/generate-ssdlc-report.sh +++ b/evergreen/generate-ssdlc-report.sh @@ -2,6 +2,7 @@ set -o errexit # Exit the script with error if any of the commands fail # Environment variables used as input: +# NUGET_SIGN_CERTIFICATE_FINGERPRINT # PRODUCT_NAME # PACKAGE_VERSION # github_commit @@ -31,5 +32,6 @@ sed "${SED_EDIT_IN_PLACE_OPTION[@]}" \ -e "s/\${PACKAGE_VERSION}/$PACKAGE_VERSION/g" \ -e "s/\${github_commit}/$github_commit/g" \ -e "s/\${REPORT_DATE_UTC}/$(date -u +%Y-%m-%d)/g" \ + -e "s/\${NUGET_SIGN_CERTIFICATE_FINGERPRINT}/${NUGET_SIGN_CERTIFICATE_FINGERPRINT}/g" \ "${SSDLC_REPORT_PATH}" ls "${SSDLC_REPORT_PATH}" \ No newline at end of file diff --git a/evergreen/template_ssdlc_compliance_report.md b/evergreen/template_ssdlc_compliance_report.md index 3dc92df9255..80e2d8e983a 100644 --- a/evergreen/template_ssdlc_compliance_report.md +++ b/evergreen/template_ssdlc_compliance_report.md @@ -41,8 +41,7 @@ This information is available in multiple ways: Blocked on . -The MongoDB SSDLC policy is available at -. +The MongoDB SSDLC policy is available here. ## Third-darty dependency information @@ -54,4 +53,10 @@ Coverity static analysis report is available