diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionCondition.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionCondition.cs index fe73b8266e..e891397933 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionCondition.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionCondition.cs @@ -33,7 +33,9 @@ public FaultInjectionCondition( this.region = string.IsNullOrEmpty(region) ? string.Empty : mapper.GetCosmosDBRegionName(region); this.operationType = operationType ?? FaultInjectionOperationType.All; - this.connectionType = connectionType ?? FaultInjectionConnectionType.All; + this.connectionType = this.IsMetadataOperationType() + ? FaultInjectionConnectionType.Gateway + : connectionType ?? FaultInjectionConnectionType.All; this.endpoint = endpoint ?? FaultInjectionEndpoint.Empty; } @@ -87,5 +89,14 @@ public override string ToString() this.region, this.endpoint.ToString()); } + + internal bool IsMetadataOperationType() + { + return this.operationType == FaultInjectionOperationType.MetadataContainer + || this.operationType == FaultInjectionOperationType.MetadataDatabaseAccount + || this.operationType == FaultInjectionOperationType.MetadataPartitionKeyRange + || this.operationType == FaultInjectionOperationType.MetadataRefreshAddresses + || this.operationType == FaultInjectionOperationType.MetadataQueryPlan; + } } } diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionOperationType.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionOperationType.cs index ea8b3d19d1..ee5513dae2 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionOperationType.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionOperationType.cs @@ -53,6 +53,31 @@ public enum FaultInjectionOperationType /// ReadFeed, + /// + /// Metadata operations for containers. + /// + MetadataContainer, + + /// + /// Metadata operations for databases. + /// + MetadataDatabaseAccount, + + /// + /// Metadata operations for partition key ranges. + /// + MetadataPartitionKeyRange, + + /// + /// Metadata operations for addresses. + /// + MetadataRefreshAddresses, + + /// + /// Metadata operations for query plan. + /// + MetadataQueryPlan, + /// /// All operation types. Default value. /// diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionRuleBuilder.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionRuleBuilder.cs index 807155195a..efa166166b 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionRuleBuilder.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionRuleBuilder.cs @@ -96,6 +96,16 @@ public FaultInjectionRuleBuilder IsEnabled(bool enabled) /// the . public FaultInjectionRule Build() { + if (this.condition.GetConnectionType() == FaultInjectionConnectionType.Gateway) + { + this.ValidateGatewayConnection(); + } + + if (this.condition.GetConnectionType() == FaultInjectionConnectionType.Direct) + { + this.ValidateDirectConnection(); + } + return new FaultInjectionRule( this.result, this.condition, @@ -105,5 +115,45 @@ public FaultInjectionRule Build() this.hitLimit, this.enabled); } + + private void ValidateDirectConnection() + { + if (this.result == null) + { + throw new ArgumentNullException(nameof(this.result), "Argument 'result' cannot be null."); + } + + FaultInjectionServerErrorResult? serverErrorResult = this.result as FaultInjectionServerErrorResult; + + if (serverErrorResult?.GetServerErrorType() == FaultInjectionServerErrorType.DatabaseAccountNotFound) + { + throw new ArgumentException("DatabaseAccountNotFound error type is not supported for Direct connection type."); + } + } + + private void ValidateGatewayConnection() + { + if (this.result == null) + { + throw new ArgumentNullException(nameof(this.result), "Argument 'result' cannot be null."); + } + + FaultInjectionServerErrorResult? serverErrorResult = this.result as FaultInjectionServerErrorResult; + + if (serverErrorResult?.GetServerErrorType() == FaultInjectionServerErrorType.Gone) + { + throw new ArgumentException("Gone error type is not supported for Gateway connection type."); + } + + if (this.condition.IsMetadataOperationType()) + { + if (serverErrorResult?.GetServerErrorType() != FaultInjectionServerErrorType.TooManyRequests + && serverErrorResult?.GetServerErrorType() != FaultInjectionServerErrorType.ResponseDelay + && serverErrorResult?.GetServerErrorType() != FaultInjectionServerErrorType.SendDelay) + { + throw new ArgumentException($"{serverErrorResult?.GetServerErrorType()} is not supported for metadata requests."); + } + } + } } } diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionServerErrorType.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionServerErrorType.cs index 7781cb4b87..bb3f7dae2d 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionServerErrorType.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionServerErrorType.cs @@ -9,7 +9,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection public enum FaultInjectionServerErrorType { /// - /// 410: Gone from server + /// 410: Gone from server. Only Applicable for Direct mode calls. /// Gone, @@ -69,6 +69,11 @@ public enum FaultInjectionServerErrorType /// /// 503: Service Unavailable from server /// - ServiceUnavailable + ServiceUnavailable, + + /// + /// 404:1008 Database account not found from gateway + /// + DatabaseAccountNotFound, } } diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionConditionInternal.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionConditionInternal.cs index caf34767fc..8bdbe29a4f 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionConditionInternal.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionConditionInternal.cs @@ -55,11 +55,13 @@ public void SetRegionEndpoints(List regionEndpoints) } } - public void SetPartitionKeyRangeIds(IEnumerable partitionKeyRangeIds) + public void SetPartitionKeyRangeIds(IEnumerable partitionKeyRangeIds, FaultInjectionRule rule) { if (partitionKeyRangeIds != null && partitionKeyRangeIds.Any()) { - this.validators.Add(new PartitionKeyRangeIdValidator(partitionKeyRangeIds)); + this.validators.Add(new PartitionKeyRangeIdValidator( + partitionKeyRangeIds, + rule.GetCondition().GetEndpoint().IsIncludePrimary())); } } @@ -302,10 +304,12 @@ public bool IsApplicable(Uri callUri) private class PartitionKeyRangeIdValidator : IFaultInjectionConditionValidator { private readonly IEnumerable pkRangeIds; + private readonly bool includePrimaryForMetaData; - public PartitionKeyRangeIdValidator(IEnumerable pkRangeIds) + public PartitionKeyRangeIdValidator(IEnumerable pkRangeIds, bool includePrimaryForMetaData) { this.pkRangeIds = pkRangeIds ?? throw new ArgumentNullException(nameof(pkRangeIds)); + this.includePrimaryForMetaData = includePrimaryForMetaData; } public bool IsApplicable(string ruleId, ChannelCallArguments args) @@ -320,7 +324,13 @@ public bool IsApplicable(string ruleId, DocumentServiceRequest request) { PartitionKeyRange pkRange = request.RequestContext.ResolvedPartitionKeyRange; - return this.pkRangeIds.Contains(pkRange.Id); + if (pkRange is null && this.includePrimaryForMetaData) + { + //For metadata operations, rule will apply to all partition key ranges + return true; + } + + return this.pkRangeIds.Contains(pkRange?.Id); } } diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionRuleProcessor.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionRuleProcessor.cs index e9b8b45a30..02413d1183 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionRuleProcessor.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionRuleProcessor.cs @@ -98,9 +98,12 @@ private async Task GetEffectiveServerErrorRule(Faul FaultInjectionOperationType operationType = rule.GetCondition().GetOperationType(); if ((operationType != FaultInjectionOperationType.All) && this.CanErrorLimitToOperation(errorType)) { - effectiveCondition.SetOperationType(this.GetEffectiveOperationType(operationType)); - //Will need to change when introducing metadata operations - effectiveCondition.SetResourceType(ResourceType.Document); + OperationType effectiveOperationType = this.GetEffectiveOperationType(operationType); + if (effectiveOperationType != OperationType.Invalid) + { + effectiveCondition.SetOperationType(this.GetEffectiveOperationType(operationType)); + } + effectiveCondition.SetResourceType(this.GetEffectiveResourceType(operationType)); } List regionEndpoints = this.GetRegionEndpoints(rule.GetCondition()); @@ -129,7 +132,10 @@ await BackoffRetryUtility>.ExecuteAsync( rule.GetCondition().GetEndpoint()), this.retryPolicy()); - effectiveCondition.SetPartitionKeyRangeIds(effectivePKRangeId); + if (!this.IsMetaData(rule.GetCondition().GetOperationType())) + { + effectiveCondition.SetPartitionKeyRangeIds(effectivePKRangeId, rule); + } } } else @@ -246,6 +252,33 @@ private OperationType GetEffectiveOperationType(FaultInjectionOperationType faul FaultInjectionOperationType.PatchItem => OperationType.Patch, FaultInjectionOperationType.Batch => OperationType.Batch, FaultInjectionOperationType.ReadFeed => OperationType.ReadFeed, + FaultInjectionOperationType.MetadataContainer => OperationType.Read, + FaultInjectionOperationType.MetadataDatabaseAccount => OperationType.Read, + FaultInjectionOperationType.MetadataPartitionKeyRange => OperationType.ReadFeed, + FaultInjectionOperationType.MetadataRefreshAddresses => OperationType.Invalid, + FaultInjectionOperationType.MetadataQueryPlan => OperationType.QueryPlan, + _ => throw new ArgumentException($"FaultInjectionOperationType: {faultInjectionOperationType} is not supported"), + }; + } + + private ResourceType GetEffectiveResourceType(FaultInjectionOperationType faultInjectionOperationType) + { + return faultInjectionOperationType switch + { + FaultInjectionOperationType.ReadItem => ResourceType.Document, + FaultInjectionOperationType.CreateItem => ResourceType.Document, + FaultInjectionOperationType.QueryItem => ResourceType.Document, + FaultInjectionOperationType.UpsertItem => ResourceType.Document, + FaultInjectionOperationType.ReplaceItem => ResourceType.Document, + FaultInjectionOperationType.DeleteItem => ResourceType.Document, + FaultInjectionOperationType.PatchItem => ResourceType.Document, + FaultInjectionOperationType.Batch => ResourceType.Document, + FaultInjectionOperationType.ReadFeed => ResourceType.Document, + FaultInjectionOperationType.MetadataContainer => ResourceType.Collection, + FaultInjectionOperationType.MetadataDatabaseAccount => ResourceType.DatabaseAccount, + FaultInjectionOperationType.MetadataPartitionKeyRange => ResourceType.PartitionKeyRange, + FaultInjectionOperationType.MetadataRefreshAddresses => ResourceType.Address, + FaultInjectionOperationType.MetadataQueryPlan => ResourceType.Document, _ => throw new ArgumentException($"FaultInjectionOperationType: {faultInjectionOperationType} is not supported"), }; } @@ -322,6 +355,15 @@ private bool IsWriteOnly(FaultInjectionCondition condition) && this.GetEffectiveOperationType(condition.GetOperationType()).IsWriteOperation(); } + private bool IsMetaData(FaultInjectionOperationType operationType) + { + return operationType == FaultInjectionOperationType.MetadataContainer + || operationType == FaultInjectionOperationType.MetadataDatabaseAccount + || operationType == FaultInjectionOperationType.MetadataPartitionKeyRange + || operationType == FaultInjectionOperationType.MetadataRefreshAddresses + || operationType == FaultInjectionOperationType.MetadataQueryPlan; + } + private async Task> ResolvePhyicalAddresses( List regionEndpoints, FaultInjectionCondition condition, diff --git a/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionServerErrorResultInternal.cs b/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionServerErrorResultInternal.cs index ec4010a4bb..de5d6f7d60 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionServerErrorResultInternal.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionServerErrorResultInternal.cs @@ -466,6 +466,28 @@ public HttpResponseMessage GetInjectedServerError(DocumentServiceRequest dsr, st return httpResponse; + case FaultInjectionServerErrorType.DatabaseAccountNotFound: + + httpResponse = new HttpResponseMessage + { + StatusCode = HttpStatusCode.NotFound, + Content = new FauntInjectionHttpContent( + new MemoryStream( + FaultInjectionResponseEncoding.GetBytes($"Fault Injection Server Error: DatabaseAccountNotFound, rule: {ruleId}"))), + }; + + foreach (string header in headers.AllKeys()) + { + httpResponse.Headers.Add(header, headers.Get(header)); + } + + httpResponse.Headers.Add( + WFConstants.BackendHeaders.SubStatus, + ((int)SubStatusCodes.DatabaseAccountNotFound).ToString(CultureInfo.InvariantCulture)); + httpResponse.Headers.Add(WFConstants.BackendHeaders.LocalLSN, lsn); + + return httpResponse; + default: throw new ArgumentException($"Server error type {this.serverErrorType} is not supported"); } diff --git a/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionMetadataTests.cs b/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionMetadataTests.cs new file mode 100644 index 0000000000..e51d6acf94 --- /dev/null +++ b/Microsoft.Azure.Cosmos/FaultInjection/tests/FaultInjectionMetadataTests.cs @@ -0,0 +1,597 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ +namespace Microsoft.Azure.Cosmos.FaultInjection.Tests +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Linq; + using System.Net; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Cosmos; + using Microsoft.Azure.Cosmos.FaultInjection.Tests.Utils; + using Microsoft.Azure.Cosmos.Routing; + using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Microsoft.Azure.Documents; + using static Microsoft.Azure.Cosmos.FaultInjection.Tests.Utils.TestCommon; + using ConsistencyLevel = ConsistencyLevel; + using CosmosSystemTextJsonSerializer = Utils.TestCommon.CosmosSystemTextJsonSerializer; + using Database = Database; + using PartitionKey = PartitionKey; + + [TestClass] + public class FaultInjectionMetadataTests + { + private const int Timeout = 66000; + + private string connectionString; + private CosmosSystemTextJsonSerializer serializer; + + private CosmosClient client; + private Database database; + private Container container; + + private CosmosClient fiClient; + private Database fiDatabase; + private Container fiContainer; + private Container highThroughputContainer; + + + [TestInitialize] + public async Task Initialize() + { + //tests use a live account with multi-region enabled + this.connectionString = TestCommon.GetConnectionString(); + + if (string.IsNullOrEmpty(this.connectionString)) + { + Assert.Fail("Set environment variable COSMOSDB_MULTI_REGION to run the tests"); + } + + //serializer settings, not needed for fault injection but used for test objects + JsonSerializerOptions jsonSerializerOptions = new JsonSerializerOptions() + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + this.serializer = new CosmosSystemTextJsonSerializer(jsonSerializerOptions); + + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() + { + ConsistencyLevel = ConsistencyLevel.Session, + Serializer = this.serializer, + }; + + this.client = new CosmosClient(this.connectionString, cosmosClientOptions); + + //create a database and container if they do not already exist on test account + //SDK test account uses strong consistency so haivng pre existing databases helps shorten test time with global replication lag + (this.database, this.container) = await TestCommon.GetOrCreateMultiRegionFIDatabaseAndContainersAsync(this.client); + } + + [TestCleanup] + public async Task Cleanup() + { + //deletes the high throughput container if it was created to save costs + if (this.highThroughputContainer != null) + { + await this.highThroughputContainer.DeleteContainerAsync(); + } + + try + { + await this.container.DeleteItemAsync("deleteme", new PartitionKey("deleteme")); + } + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + //ignore + } + finally + { + this.client?.Dispose(); + this.fiClient?.Dispose(); + } + } + + [TestMethod] + public async Task AddressRefreshResponseDelayTest() + { + //create rule + Uri primaryUri = this.client.DocumentClient.GlobalEndpointManager.WriteEndpoints.First(); + string primaryRegion = this.client.DocumentClient.GlobalEndpointManager.GetLocation(primaryUri); + string responseDelayRuleId = "responseDelayRule-" + Guid.NewGuid().ToString(); + FaultInjectionRule delayRule = new FaultInjectionRuleBuilder( + id: responseDelayRuleId, + condition: + new FaultInjectionConditionBuilder() + .WithRegion(primaryRegion) + .WithOperationType(FaultInjectionOperationType.MetadataRefreshAddresses) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromSeconds(15)) + .WithTimes(1) + .Build()) + .WithDuration(TimeSpan.FromMinutes(5)) + .Build(); + + delayRule.Disable(); + + try + { + //create client with fault injection + FaultInjector faultInjector = new FaultInjector(new List { delayRule }); + + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() + { + ConsistencyLevel = ConsistencyLevel.Session, + Serializer = this.serializer, + EnableContentResponseOnWrite = true, + }; + + this.fiClient = new CosmosClient( + this.connectionString, + faultInjector.GetFaultInjectionClientOptions(cosmosClientOptions)); + this.fiDatabase = this.fiClient.GetDatabase(TestCommon.FaultInjectionDatabaseName); + this.fiContainer = this.fiDatabase.GetContainer(TestCommon.FaultInjectionContainerName); + + delayRule.Enable(); + + ValueStopwatch stopwatch = ValueStopwatch.StartNew(); + TimeSpan elapsed; + + ItemResponse _ = await this.fiContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "deleteme", Pk = "deleteme" }); + + elapsed = stopwatch.Elapsed; + stopwatch.Stop(); + delayRule.Disable(); + + this.ValidateRuleHit(delayRule, 1); + + //Check the create time is at least as long as the delay in the rule + Assert.IsTrue(elapsed.TotalSeconds >= 15); + } + finally + { + delayRule.Disable(); + } + } + + [TestMethod] + public async Task AddressRefreshTooManyRequestsTest() + { + //create rule + Uri primaryUri = this.client.DocumentClient.GlobalEndpointManager.WriteEndpoints.First(); + string primaryRegion = this.client.DocumentClient.GlobalEndpointManager.GetLocation(primaryUri); + string responseDelayRuleId = "responseTooManyRule-" + Guid.NewGuid().ToString(); + FaultInjectionRule tooManyRequestRule = new FaultInjectionRuleBuilder( + id: responseDelayRuleId, + condition: + new FaultInjectionConditionBuilder() + .WithRegion(primaryRegion) + .WithOperationType(FaultInjectionOperationType.MetadataRefreshAddresses) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.TooManyRequests) + .WithTimes(1) + .Build()) + .WithDuration(TimeSpan.FromMinutes(5)) + .Build(); + + tooManyRequestRule.Disable(); + + try + { + //create client with fault injection + FaultInjector faultInjector = new FaultInjector(new List { tooManyRequestRule }); + + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() + { + ConsistencyLevel = ConsistencyLevel.Session, + Serializer = this.serializer, + EnableContentResponseOnWrite = true, + }; + + this.fiClient = new CosmosClient( + this.connectionString, + faultInjector.GetFaultInjectionClientOptions(cosmosClientOptions)); + this.fiDatabase = this.fiClient.GetDatabase(TestCommon.FaultInjectionDatabaseName); + this.fiContainer = this.fiDatabase.GetContainer(TestCommon.FaultInjectionContainerName); + + tooManyRequestRule.Enable(); + + ItemResponse response; + try + { + response = await this.fiContainer.ReadItemAsync( + "testId", + new PartitionKey("pk")); + } + catch (CosmosException ex) + { + this.ValidateRuleHit(tooManyRequestRule, 1); + this.ValidateFaultInjectionRuleApplication( + ex, + (int)HttpStatusCode.TooManyRequests, + tooManyRequestRule); + } + + tooManyRequestRule.Disable(); + } + finally + { + tooManyRequestRule.Disable(); + } + } + + [TestMethod] + [Description("Test Partition rule filtering")] + [Owner("ntripician")] + public async Task FIMetadataAddressRefreshPartitionTest() + { + //create container with high throughput to create multiple feed ranges + await this.InitializeHighThroughputContainerAsync(); + + List feedRanges = (List)await this.highThroughputContainer.GetFeedRangesAsync(); + Assert.IsTrue(feedRanges.Count > 1); + + string query = "SELECT * FROM c"; + + FeedIterator feedIterator = this.highThroughputContainer.GetItemQueryIterator(query); + + //get one item from each feed range, since it will be a cross partition query, each page will contain items from different partitions + FaultInjectionTestObject result1 = (await feedIterator.ReadNextAsync()).First(); + FaultInjectionTestObject result2 = (await feedIterator.ReadNextAsync()).First(); + + //create fault injection rule for one of the partitions + string serverErrorFeedRangeRuleId = "serverErrorFeedRangeRule-" + Guid.NewGuid().ToString(); + FaultInjectionRule serverErrorFeedRangeRule = new FaultInjectionRuleBuilder( + id: serverErrorFeedRangeRuleId, + condition: + new FaultInjectionConditionBuilder() + .WithEndpoint( + new FaultInjectionEndpointBuilder( + TestCommon.FaultInjectionDatabaseName, + TestCommon.FaultInjectionHTPContainerName, + feedRanges[0]) + .Build()) + .WithOperationType(FaultInjectionOperationType.MetadataRefreshAddresses) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.TooManyRequests) + .WithTimes(100) + .Build()) + .Build(); + + //disable rule until ready to test + serverErrorFeedRangeRule.Disable(); + + //create client with fault injection + List rules = new List { serverErrorFeedRangeRule }; + FaultInjector faultInjector = new FaultInjector(rules); + + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() + { + ConsistencyLevel = ConsistencyLevel.Session, + Serializer = this.serializer, + MaxRetryAttemptsOnRateLimitedRequests = 0, + }; + + this.fiClient = new CosmosClient( + this.connectionString, + faultInjector.GetFaultInjectionClientOptions(cosmosClientOptions)); + this.fiDatabase = this.fiClient.GetDatabase(TestCommon.FaultInjectionDatabaseName); + this.fiContainer = this.fiDatabase.GetContainer(TestCommon.FaultInjectionHTPContainerName); + + serverErrorFeedRangeRule.Enable(); + + ItemResponse response; + try + { + response = await this.fiContainer.ReadItemAsync( + result1.Id, + new PartitionKey(result1.Pk)); + } + catch (CosmosException ex) + { + this.ValidateHitCount(serverErrorFeedRangeRule, 1); + this.ValidateFaultInjectionRuleApplication( + ex, + (int)HttpStatusCode.TooManyRequests, + serverErrorFeedRangeRule); + } + + //test that rule is applied to other partition, for metadata operations, the rule should not be applied to all partitions becaseu address calls go to primary replica + try + { + response = await this.fiContainer.ReadItemAsync( + result2.Id, + new PartitionKey(result2.Pk)); + } + catch (CosmosException ex) + { + this.ValidateRuleHit(serverErrorFeedRangeRule, 2); + this.ValidateFaultInjectionRuleApplication( + ex, + (int)HttpStatusCode.TooManyRequests, + serverErrorFeedRangeRule); + } + finally + { + serverErrorFeedRangeRule.Disable(); + } + } + + private async Task InitializeHighThroughputContainerAsync() + { + if (this.database != null) + { + ContainerResponse cr = await this.database.CreateContainerIfNotExistsAsync( + id: TestCommon.FaultInjectionHTPContainerName, + partitionKeyPath: "/pk", + throughput: 11000); + + if (cr.StatusCode == HttpStatusCode.Created) + { + this.highThroughputContainer = cr.Container; + List tasks = new List() + { + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId", Pk = "pk" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId2", Pk = "pk2" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId3", Pk = "pk3" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId4", Pk = "pk4" }), + this.highThroughputContainer.CreateItemAsync( + //unsued but needed to create multiple feed ranges + new FaultInjectionTestObject { Id = "testId5", Pk = "qwertyuiop" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId6", Pk = "asdfghjkl" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId7", Pk = "zxcvbnm" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId8", Pk = "2wsx3edc" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId9", Pk = "5tgb6yhn" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId10", Pk = "7ujm8ik" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId11", Pk = "9ol" }), + this.highThroughputContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "testId12", Pk = "1234567890" }) + }; + + await Task.WhenAll(tasks); + } + else + { + this.highThroughputContainer = this.database.GetContainer(TestCommon.FaultInjectionHTPContainerName); + } + } + } + + [TestMethod] + public async Task PKRangeResponseDelayTest() + { + //create rule + Uri primaryUri = this.client.DocumentClient.GlobalEndpointManager.WriteEndpoints.First(); + string primaryRegion = this.client.DocumentClient.GlobalEndpointManager.GetLocation(primaryUri); + string responseDelayRuleId = "responseDelayRule-" + Guid.NewGuid().ToString(); + FaultInjectionRule delayRule = new FaultInjectionRuleBuilder( + id: responseDelayRuleId, + condition: + new FaultInjectionConditionBuilder() + .WithRegion(primaryRegion) + .WithOperationType(FaultInjectionOperationType.MetadataPartitionKeyRange) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromSeconds(15)) + .WithTimes(1) + .Build()) + .WithDuration(TimeSpan.FromMinutes(5)) + .Build(); + + string createRuleId = "createRule-" + Guid.NewGuid().ToString(); + FaultInjectionRule createRule = new FaultInjectionRuleBuilder( + id: createRuleId, + condition: + new FaultInjectionConditionBuilder() + .WithRegion(primaryRegion) + .WithOperationType(FaultInjectionOperationType.CreateItem) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.PartitionIsSplitting) + .WithTimes(1) + .Build()) + .WithDuration(TimeSpan.FromMinutes(5)) + .Build(); + + delayRule.Disable(); + + try + { + //create client with fault injection + FaultInjector faultInjector = new FaultInjector(new List { delayRule, createRule }); + + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() + { + ConsistencyLevel = ConsistencyLevel.Session, + Serializer = this.serializer, + EnableContentResponseOnWrite = true, + }; + + this.fiClient = new CosmosClient( + this.connectionString, + faultInjector.GetFaultInjectionClientOptions(cosmosClientOptions)); + this.fiDatabase = this.fiClient.GetDatabase(TestCommon.FaultInjectionDatabaseName); + this.fiContainer = this.fiDatabase.GetContainer(TestCommon.FaultInjectionContainerName); + + delayRule.Enable(); + + ValueStopwatch stopwatch = ValueStopwatch.StartNew(); + TimeSpan elapsed; + + ItemResponse _ = await this.fiContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "deleteme", Pk = "deleteme" }); + + elapsed = stopwatch.Elapsed; + stopwatch.Stop(); + delayRule.Disable(); + + this.ValidateRuleHit(delayRule, 1); + + //Check the create time is at least as long as the delay in the rule + Assert.IsTrue(elapsed.TotalSeconds >= 15); + } + finally + { + delayRule.Disable(); + } + } + + [TestMethod] + public async Task CollectionReadResponseDelayTest() + { + //create rule + Uri primaryUri = this.client.DocumentClient.GlobalEndpointManager.WriteEndpoints.First(); + string primaryRegion = this.client.DocumentClient.GlobalEndpointManager.GetLocation(primaryUri); + string responseDelayRuleId = "responseDelayRule-" + Guid.NewGuid().ToString(); + FaultInjectionRule delayRule = new FaultInjectionRuleBuilder( + id: responseDelayRuleId, + condition: + new FaultInjectionConditionBuilder() + .WithRegion(primaryRegion) + .WithOperationType(FaultInjectionOperationType.MetadataContainer) + .Build(), + result: + FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay) + .WithDelay(TimeSpan.FromSeconds(15)) + .WithTimes(1) + .Build()) + .WithDuration(TimeSpan.FromMinutes(5)) + .Build(); + + delayRule.Disable(); + + try + { + //create client with fault injection + FaultInjector faultInjector = new FaultInjector(new List { delayRule }); + + CosmosClientOptions cosmosClientOptions = new CosmosClientOptions() + { + ConsistencyLevel = ConsistencyLevel.Session, + Serializer = this.serializer, + EnableContentResponseOnWrite = true, + }; + + this.fiClient = new CosmosClient( + this.connectionString, + faultInjector.GetFaultInjectionClientOptions(cosmosClientOptions)); + this.fiDatabase = this.fiClient.GetDatabase(TestCommon.FaultInjectionDatabaseName); + this.fiContainer = this.fiDatabase.GetContainer(TestCommon.FaultInjectionContainerName); + + delayRule.Enable(); + + ValueStopwatch stopwatch = ValueStopwatch.StartNew(); + TimeSpan elapsed; + + ItemResponse _ = await this.fiContainer.CreateItemAsync( + new FaultInjectionTestObject { Id = "deleteme", Pk = "deleteme" }); + + elapsed = stopwatch.Elapsed; + stopwatch.Stop(); + delayRule.Disable(); + + this.ValidateRuleHit(delayRule, 1); + + //Check the create time is at least as long as the delay in the rule + Assert.IsTrue(elapsed.TotalSeconds >= 6); + } + finally + { + delayRule.Disable(); + } + } + + private async Task<(List, List)> GetReadWriteEndpoints(GlobalEndpointManager globalEndpointManager) + { + AccountProperties accountProperties = await globalEndpointManager.GetDatabaseAccountAsync(); + List writeRegions = accountProperties.WritableRegions.Select(region => region.Name).ToList(); + List readRegions = accountProperties.ReadableRegions.Select(region => region.Name).ToList(); + return (writeRegions, readRegions); + } + + private void ValidateHitCount(FaultInjectionRule rule, long expectedHitCount) + { + Assert.AreEqual(expectedHitCount, rule.GetHitCount()); + } + + private void ValidateRuleHit(FaultInjectionRule rule, long expectedHitCount) + { + Assert.IsTrue(expectedHitCount <= rule.GetHitCount()); + } + + private void ValidateFaultInjectionRuleNotApplied( + ItemResponse response, + FaultInjectionRule rule, + int expectedHitCount = 0) + { + Assert.AreEqual(expectedHitCount, rule.GetHitCount()); + Assert.AreEqual(0, response.Diagnostics.GetFailedRequestCount()); + Assert.IsTrue((int)response.StatusCode < 400); + } + + private void ValidateFaultInjectionRuleApplication( + DocumentClientException ex, + int statusCode, + FaultInjectionRule rule) + { + Assert.IsTrue(1 <= rule.GetHitCount()); + Assert.IsTrue(ex.Message.Contains(rule.GetId())); + Assert.AreEqual(statusCode, (int)ex.StatusCode); + } + + private void ValidateFaultInjectionRuleApplication( + CosmosException ex, + int statusCode, + FaultInjectionRule rule) + { + Assert.IsTrue(1 <= rule.GetHitCount()); + Assert.IsTrue(ex.Message.Contains(rule.GetId())); + Assert.AreEqual(statusCode, (int)ex.StatusCode); + } + + private void ValidateFaultInjectionRuleApplication( + DocumentClientException ex, + int statusCode, + int subStatusCode, + FaultInjectionRule rule) + { + Assert.IsTrue(1 <= rule.GetHitCount()); + Assert.IsTrue(ex.Message.Contains(rule.GetId())); + Assert.AreEqual(statusCode, (int)ex.StatusCode); + Assert.AreEqual(subStatusCode.ToString(), ex.Headers.Get(WFConstants.BackendHeaders.SubStatus)); + } + + private void ValidateFaultInjectionRuleApplication( + CosmosException ex, + int statusCode, + int subStatusCode, + FaultInjectionRule rule) + { + Assert.IsTrue(1 <= rule.GetHitCount()); + Assert.IsTrue(ex.Message.Contains(rule.GetId())); + Assert.AreEqual(statusCode, (int)ex.StatusCode); + Assert.AreEqual(subStatusCode, ex.SubStatusCode); + } + } +} diff --git a/Microsoft.Azure.Cosmos/FaultInjection/tests/Utils/TestCommon.cs b/Microsoft.Azure.Cosmos/FaultInjection/tests/Utils/TestCommon.cs index 394dc45684..97fb5fbd7d 100644 --- a/Microsoft.Azure.Cosmos/FaultInjection/tests/Utils/TestCommon.cs +++ b/Microsoft.Azure.Cosmos/FaultInjection/tests/Utils/TestCommon.cs @@ -23,65 +23,6 @@ internal static string GetConnectionString() { return ConfigurationManager.GetEnvironmentVariable("COSMOSDB_MULTI_REGION", string.Empty); } - - internal static CosmosClient CreateCosmosClient( - bool useGateway, - FaultInjector injector, - bool multiRegion, - List? preferredRegion = null, - Action? customizeClientBuilder = null) - { - CosmosClientBuilder cosmosClientBuilder = GetDefaultConfiguration(multiRegion); - cosmosClientBuilder.WithFaultInjection(injector.GetChaosInterceptorFactory()); - - customizeClientBuilder?.Invoke(cosmosClientBuilder); - - if (useGateway) - { - cosmosClientBuilder.WithConnectionModeGateway(); - } - - if (preferredRegion != null) - { - cosmosClientBuilder.WithApplicationPreferredRegions(preferredRegion); - } - - return cosmosClientBuilder.Build(); - } - - internal static CosmosClient CreateCosmosClient( - bool useGateway, - bool multiRegion, - Action? customizeClientBuilder = null) - { - CosmosClientBuilder cosmosClientBuilder = GetDefaultConfiguration(multiRegion); - - customizeClientBuilder?.Invoke(cosmosClientBuilder); - - if (useGateway) - { - cosmosClientBuilder.WithConnectionModeGateway(); - } - - return cosmosClientBuilder.Build(); - } - - internal static CosmosClientBuilder GetDefaultConfiguration( - bool multiRegion, - string? accountEndpointOverride = null) - { - CosmosClientBuilder clientBuilder = new CosmosClientBuilder( - accountEndpoint: accountEndpointOverride - ?? EndpointMultiRegion, - authKeyOrResourceToken: AuthKeyMultiRegion); - - if (!multiRegion) - { - return clientBuilder.WithApplicationPreferredRegions(new List { "Central US" }); - } - - return clientBuilder; - } internal static async Task<(Database, Container)> GetOrCreateMultiRegionFIDatabaseAndContainersAsync(CosmosClient client) { diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs index 4f0f800d43..00353501fd 100644 --- a/Microsoft.Azure.Cosmos/src/DocumentClient.cs +++ b/Microsoft.Azure.Cosmos/src/DocumentClient.cs @@ -970,7 +970,8 @@ internal virtual void Initialize(Uri serviceEndpoint, this.httpClient, this.ServiceEndpoint, this.GlobalEndpointManager, - this.cancellationTokenSource); + this.cancellationTokenSource, + this.chaosInterceptor is not null); if (sessionContainer != null) { diff --git a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs index 1a2c3e14a4..f332d873fd 100644 --- a/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs +++ b/Microsoft.Azure.Cosmos/src/GatewayAccountReader.cs @@ -52,16 +52,23 @@ await this.cosmosAuthorization.AddAuthorizationHeaderAsync( try { - using (HttpResponseMessage responseMessage = await this.httpClient.GetAsync( - uri: serviceEndpoint, - additionalHeaders: headers, + using (DocumentServiceRequest request = DocumentServiceRequest.Create( + operationType: OperationType.Read, resourceType: ResourceType.DatabaseAccount, - timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance, - clientSideRequestStatistics: stats, - cancellationToken: default)) - using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage)) + authorizationTokenType: AuthorizationTokenType.PrimaryMasterKey)) { - return CosmosResource.FromStream(documentServiceResponse); + using (HttpResponseMessage responseMessage = await this.httpClient.GetAsync( + uri: serviceEndpoint, + additionalHeaders: headers, + resourceType: ResourceType.DatabaseAccount, + timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance, + clientSideRequestStatistics: stats, + cancellationToken: default, + documentServiceRequest: request)) + using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage)) + { + return CosmosResource.FromStream(documentServiceResponse); + } } } catch (ObjectDisposedException) when (this.cancellationToken.IsCancellationRequested) diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClient.cs b/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClient.cs index 375c6c4240..9f1d9dd800 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClient.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClient.cs @@ -14,6 +14,7 @@ namespace Microsoft.Azure.Cosmos internal abstract class CosmosHttpClient : IDisposable { public static readonly TimeSpan GatewayRequestTimeout = TimeSpan.FromSeconds(65); + public abstract bool IsFaultInjectionClient { get; } public abstract HttpMessageHandler HttpMessageHandler { get; } @@ -23,7 +24,8 @@ public abstract Task GetAsync( ResourceType resourceType, HttpTimeoutPolicy timeoutPolicy, IClientSideRequestStatistics clientSideRequestStatistics, - CancellationToken cancellationToken); + CancellationToken cancellationToken, + DocumentServiceRequest documentServiceRequest = null); public abstract Task SendHttpAsync( Func> createRequestMessageAsync, diff --git a/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs b/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs index bfe6fd5b54..399a7b52ad 100644 --- a/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs +++ b/Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs @@ -45,6 +45,8 @@ private CosmosHttpClientCore( this.chaosInterceptor = chaosInterceptor; } + public override bool IsFaultInjectionClient => this.chaosInterceptor is not null; + public override HttpMessageHandler HttpMessageHandler { get; } public static CosmosHttpClient CreateWithConnectionPolicy( @@ -273,7 +275,8 @@ public override Task GetAsync( ResourceType resourceType, HttpTimeoutPolicy timeoutPolicy, IClientSideRequestStatistics clientSideRequestStatistics, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + DocumentServiceRequest documentServiceRequest = null) { if (uri == null) { @@ -304,7 +307,8 @@ ValueTask CreateRequestMessage() resourceType, timeoutPolicy, clientSideRequestStatistics, - cancellationToken); + cancellationToken, + documentServiceRequest); } public override Task SendHttpAsync( diff --git a/Microsoft.Azure.Cosmos/src/Routing/GatewayAddressCache.cs b/Microsoft.Azure.Cosmos/src/Routing/GatewayAddressCache.cs index 4106509ed5..26c4412c9e 100644 --- a/Microsoft.Azure.Cosmos/src/Routing/GatewayAddressCache.cs +++ b/Microsoft.Azure.Cosmos/src/Routing/GatewayAddressCache.cs @@ -727,6 +727,31 @@ private async Task GetMasterAddressesViaGatewayAsync( Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery)); string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint); + + if (this.httpClient.IsFaultInjectionClient) + { + using (DocumentServiceRequest faultInjectionRequest = DocumentServiceRequest.Create( + operationType: OperationType.Read, + resourceType: ResourceType.Address, + authorizationTokenType: AuthorizationTokenType.PrimaryMasterKey)) + { + faultInjectionRequest.RequestContext = request.RequestContext; + using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync( + uri: targetEndpoint, + additionalHeaders: headers, + resourceType: resourceType, + timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance, + clientSideRequestStatistics: request.RequestContext?.ClientRequestStatistics, + cancellationToken: default, + documentServiceRequest: faultInjectionRequest)) + { + DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(httpResponseMessage); + GatewayAddressCache.LogAddressResolutionEnd(request, identifier); + return documentServiceResponse; + } + } + } + using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync( uri: targetEndpoint, additionalHeaders: headers, @@ -808,6 +833,31 @@ private async Task GetServerAddressesViaGatewayAsync( Uri targetEndpoint = UrlUtility.SetQuery(this.addressEndpoint, UrlUtility.CreateQuery(addressQuery)); string identifier = GatewayAddressCache.LogAddressResolutionStart(request, targetEndpoint); + + if (this.httpClient.IsFaultInjectionClient) + { + using (DocumentServiceRequest faultInjectionRequest = DocumentServiceRequest.Create( + operationType: OperationType.Read, + resourceType: ResourceType.Address, + authorizationTokenType: AuthorizationTokenType.PrimaryMasterKey)) + { + faultInjectionRequest.RequestContext = request.RequestContext; + using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync( + uri: targetEndpoint, + additionalHeaders: headers, + resourceType: ResourceType.Document, + timeoutPolicy: HttpTimeoutPolicyControlPlaneRetriableHotPath.Instance, + clientSideRequestStatistics: request.RequestContext?.ClientRequestStatistics, + cancellationToken: default, + documentServiceRequest: faultInjectionRequest)) + { + DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(httpResponseMessage); + GatewayAddressCache.LogAddressResolutionEnd(request, identifier); + return documentServiceResponse; + } + } + } + using (HttpResponseMessage httpResponseMessage = await this.httpClient.GetAsync( uri: targetEndpoint, additionalHeaders: headers, diff --git a/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs b/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs index 4b94811b67..13092d9d59 100644 --- a/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs +++ b/Microsoft.Azure.Cosmos/src/Telemetry/TelemetryToServiceHelper.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Telemetry using System.Net.Http; using System.Threading; using System.Threading.Tasks; + using HdrHistogram.Encoding; using Microsoft.Azure.Cosmos.Core.Trace; using Microsoft.Azure.Cosmos.Handler; using Microsoft.Azure.Cosmos.Query.Core.Monads; @@ -63,7 +64,8 @@ public static TelemetryToServiceHelper CreateAndInitializeClientConfigAndTelemet CosmosHttpClient httpClient, Uri serviceEndpoint, GlobalEndpointManager globalEndpointManager, - CancellationTokenSource cancellationTokenSource) + CancellationTokenSource cancellationTokenSource, + bool faultInjectionClient = false) { #if INTERNAL return new TelemetryToServiceHelper(); @@ -82,13 +84,13 @@ public static TelemetryToServiceHelper CreateAndInitializeClientConfigAndTelemet globalEndpointManager: globalEndpointManager, cancellationTokenSource: cancellationTokenSource); - _ = helper.RetrieveConfigAndInitiateTelemetryAsync(); // Let it run in backgroud + _ = helper.RetrieveConfigAndInitiateTelemetryAsync(faultInjectionClient); // Let it run in backgroud return helper; #endif } - private async Task RetrieveConfigAndInitiateTelemetryAsync() + private async Task RetrieveConfigAndInitiateTelemetryAsync(bool faultInjectionClient) { try { @@ -98,7 +100,8 @@ private async Task RetrieveConfigAndInitiateTelemetryAsync() TryCatch databaseAccountClientConfigs = await this.GetDatabaseAccountClientConfigAsync( cosmosAuthorization: this.cosmosAuthorization, httpClient: this.httpClient, - clientConfigEndpoint: serviceEndpointWithPath); + clientConfigEndpoint: serviceEndpointWithPath, + faultInjectionClient: faultInjectionClient); if (databaseAccountClientConfigs.Succeeded) { @@ -128,7 +131,8 @@ await Task.Delay( private async Task> GetDatabaseAccountClientConfigAsync(AuthorizationTokenProvider cosmosAuthorization, CosmosHttpClient httpClient, - Uri clientConfigEndpoint) + Uri clientConfigEndpoint, + bool faultInjectionClient) { INameValueCollection headers = new RequestNameValueCollection(); await cosmosAuthorization.AddAuthorizationHeaderAsync( @@ -141,6 +145,14 @@ await cosmosAuthorization.AddAuthorizationHeaderAsync( { try { + if (faultInjectionClient) + { + return await this.GetDatabaseAccountClientConfigFaultInjectionHelperAsync( + httpClient: httpClient, + clientConfigEndpoint: clientConfigEndpoint, + headers: headers); + } + using (HttpResponseMessage responseMessage = await httpClient.GetAsync( uri: clientConfigEndpoint, additionalHeaders: headers, @@ -172,6 +184,45 @@ await cosmosAuthorization.AddAuthorizationHeaderAsync( } } + private async Task> GetDatabaseAccountClientConfigFaultInjectionHelperAsync( + CosmosHttpClient httpClient, + Uri clientConfigEndpoint, + INameValueCollection headers) + { + using (DocumentServiceRequest documentServiceRequest = DocumentServiceRequest.Create( + operationType: OperationType.Read, + resourceType: ResourceType.DatabaseAccount, + relativePath: clientConfigEndpoint.AbsolutePath, + headers: headers, + authorizationTokenType: AuthorizationTokenType.PrimaryMasterKey)) + { + using (HttpResponseMessage responseMessage = await httpClient.GetAsync( + uri: clientConfigEndpoint, + additionalHeaders: headers, + resourceType: ResourceType.DatabaseAccount, + timeoutPolicy: HttpTimeoutPolicyControlPlaneRead.Instance, + clientSideRequestStatistics: null, + cancellationToken: default, + documentServiceRequest: documentServiceRequest)) + { + // It means feature flag is off at gateway, then log the exception and retry after defined interval. + // If feature flag is OFF at gateway, SDK won't refresh the latest state of the flag. + if (responseMessage.StatusCode == System.Net.HttpStatusCode.BadRequest) + { + string responseFromGateway = await responseMessage.Content.ReadAsStringAsync(); + return TryCatch.FromException( + new InvalidOperationException($"Client Config API is not enabled at compute gateway. Response is {responseFromGateway}")); + } + + using (DocumentServiceResponse documentServiceResponse = await ClientExtensions.ParseResponseAsync(responseMessage)) + { + return TryCatch.FromResult( + CosmosResource.FromStream(documentServiceResponse)); + } + } + } + } + public ITelemetryCollector GetCollector() { return this.collector; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs index 0f6bf444fb..c3cd96fc74 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAccountReaderTests.cs @@ -145,14 +145,16 @@ public async Task InitializeReaderAsync_WhenCustomEndpointsProvided_ShouldRetryW It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny())) .Callback(( Uri serviceEndpoint, INameValueCollection _, ResourceType _, HttpTimeoutPolicy _, IClientSideRequestStatistics _, - CancellationToken _) => endpointSucceeded = serviceEndpoint) + CancellationToken _, + DocumentServiceRequest _) => endpointSucceeded = serviceEndpoint) .ReturnsAsync(responseMessage); ConnectionPolicy connectionPolicy = new() @@ -207,7 +209,8 @@ public async Task InitializeReaderAsync_WhenRegionalEndpointsProvided_ShouldThro It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny())) .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")); ConnectionPolicy connectionPolicy = new() @@ -247,7 +250,8 @@ private static void SetupMockToThrowException( It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny())) .ThrowsAsync(new Exception("Service is Unavailable at the Moment.")); } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAddressCacheTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAddressCacheTests.cs index e3bca41d0a..cb66708217 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAddressCacheTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/GatewayAddressCacheTests.cs @@ -977,7 +977,8 @@ public async Task OpenConnectionsAsync_WhenSomeAddressResolvingFailsWithExceptio It.IsAny(), It.IsAny(), It.IsAny(), - It.IsAny())) + It.IsAny(), + It.IsAny())) .ThrowsAsync(new Exception("Some random error occurred.")) .ReturnsAsync(responseMessage);