[FaultInjection] Metadata Requests: Adds Metadata request support for FaultInjection

Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionCondition.cs

@@ -33,7 +33,9 @@ public FaultInjectionCondition(
             this.region = string.IsNullOrEmpty(region) ? string.Empty : mapper.GetCosmosDBRegionName(region);
 
             this.operationType = operationType ?? FaultInjectionOperationType.All;
-            this.connectionType = connectionType ?? FaultInjectionConnectionType.All;
+            this.connectionType = this.IsMetadataOperationType()
+                ? FaultInjectionConnectionType.Gateway 
+                : connectionType ?? FaultInjectionConnectionType.All;
             this.endpoint = endpoint ?? FaultInjectionEndpoint.Empty;
         }
 
@@ -87,5 +89,14 @@ public override string ToString()
                 this.region,  
                 this.endpoint.ToString());
         }
+
+        internal bool IsMetadataOperationType()
+        {
+            return this.operationType == FaultInjectionOperationType.MetadataContainer
+                || this.operationType == FaultInjectionOperationType.MetadataDatabaseAccount
+                || this.operationType == FaultInjectionOperationType.MetadataPartitionKeyRange
+                || this.operationType == FaultInjectionOperationType.MetadataRefreshAddresses
+                || this.operationType == FaultInjectionOperationType.MetadataQueryPlan;
+        }
     }
 }

Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionOperationType.cs

@@ -53,6 +53,31 @@ public enum FaultInjectionOperationType
         /// </summary>
         ReadFeed,
 
+        /// <summary>
+        /// Metadata operations for containers.
+        /// </summary>
+        MetadataContainer,
+
+        /// <summary>
+        /// Metadata operations for databases.
+        /// </summary>
+        MetadataDatabaseAccount,
+
+        /// <summary>
+        /// Metadata operations for partition key ranges.
+        /// </summary>
+        MetadataPartitionKeyRange,
+
+        /// <summary>
+        /// Metadata operations for addresses.
+        /// </summary>
+        MetadataRefreshAddresses,
+
+        /// <summary>
+        /// Metadata operations for query plan.
+        /// </summary>
+        MetadataQueryPlan,
+
         /// <summary>
         /// All operation types. Default value.
         /// </summary>

Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionRuleBuilder.cs

@@ -96,6 +96,16 @@ public FaultInjectionRuleBuilder IsEnabled(bool enabled)
         /// <returns>the <see cref="FaultInjectionRule"/>.</returns>
         public FaultInjectionRule Build()
         {
+            if (this.condition.GetConnectionType() == FaultInjectionConnectionType.Gateway)
+            {
+                this.ValidateGatewayConnection();
+            }
+
+            if (this.condition.GetConnectionType() == FaultInjectionConnectionType.Direct)
+            {
+                this.ValidateDirectConnection();
+            }
+
             return new FaultInjectionRule(
                 this.result,
                 this.condition,
@@ -105,5 +115,45 @@ public FaultInjectionRule Build()
                 this.hitLimit,
                 this.enabled);
         }
+
+        private void ValidateDirectConnection()
+        {
+            if (this.result == null)
+            {
+                throw new ArgumentNullException(nameof(this.result), "Argument 'result' cannot be null.");
+            }
+
+            FaultInjectionServerErrorResult? serverErrorResult = this.result as FaultInjectionServerErrorResult;
+
+            if (serverErrorResult?.GetServerErrorType() == FaultInjectionServerErrorType.DatabaseAccountNotFound)
+            {
+                throw new ArgumentException("DatabaseAccountNotFound error type is not supported for Direct connection type.");
+            }
+        }
+
+        private void ValidateGatewayConnection()
+        {
+            if (this.result == null)
+            {
+                throw new ArgumentNullException(nameof(this.result), "Argument 'result' cannot be null.");
+            }
+
+            FaultInjectionServerErrorResult? serverErrorResult = this.result as FaultInjectionServerErrorResult;
+
+            if (serverErrorResult?.GetServerErrorType() == FaultInjectionServerErrorType.Gone)
+            {
+                throw new ArgumentException("Gone error type is not supported for Gateway connection type.");
+            }
+
+            if (this.condition.IsMetadataOperationType())
+            {
+                if (serverErrorResult?.GetServerErrorType() != FaultInjectionServerErrorType.TooManyRequests
+                    && serverErrorResult?.GetServerErrorType() != FaultInjectionServerErrorType.ResponseDelay
+                    && serverErrorResult?.GetServerErrorType() != FaultInjectionServerErrorType.SendDelay)
+                {
+                    throw new ArgumentException($"{serverErrorResult?.GetServerErrorType()} is not supported for metadata requests.");
+                }
+            }
+        }
     }
 }

Microsoft.Azure.Cosmos/FaultInjection/src/FaultInjectionServerErrorType.cs

@@ -9,7 +9,7 @@ namespace Microsoft.Azure.Cosmos.FaultInjection
     public enum FaultInjectionServerErrorType
     {
         /// <summary>
-        /// 410: Gone from server
+        /// 410: Gone from server. Only Applicable for Direct mode calls.
         /// </summary>
         Gone,
 
@@ -69,6 +69,11 @@ public enum FaultInjectionServerErrorType
         /// <summary>
         /// 503: Service Unavailable from server
         /// </summary>
-        ServiceUnavailable
+        ServiceUnavailable,
+
+        /// <summary>
+        /// 404:1008 Database account not found from gateway
+        /// </summary>
+        DatabaseAccountNotFound,
     }
 }

Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionConditionInternal.cs

@@ -55,11 +55,13 @@ public void SetRegionEndpoints(List<Uri> regionEndpoints)
             }
         }
 
-        public void SetPartitionKeyRangeIds(IEnumerable<string> partitionKeyRangeIds)
+        public void SetPartitionKeyRangeIds(IEnumerable<string> partitionKeyRangeIds, FaultInjectionRule rule)
         {
             if (partitionKeyRangeIds != null && partitionKeyRangeIds.Any())
             {
-                this.validators.Add(new PartitionKeyRangeIdValidator(partitionKeyRangeIds));
+                this.validators.Add(new PartitionKeyRangeIdValidator(
+                    partitionKeyRangeIds,
+                    rule.GetCondition().GetEndpoint().IsIncludePrimary()));
             }
         }
 
@@ -302,10 +304,12 @@ public bool IsApplicable(Uri callUri)
         private class PartitionKeyRangeIdValidator : IFaultInjectionConditionValidator
         {
             private readonly IEnumerable<string> pkRangeIds;
+            private readonly bool includePrimaryForMetaData;
 
-            public PartitionKeyRangeIdValidator(IEnumerable<string> pkRangeIds)
+            public PartitionKeyRangeIdValidator(IEnumerable<string> pkRangeIds, bool includePrimaryForMetaData)
             {
                 this.pkRangeIds = pkRangeIds ?? throw new ArgumentNullException(nameof(pkRangeIds));
+                this.includePrimaryForMetaData = includePrimaryForMetaData;
             }
 
             public bool IsApplicable(string ruleId, ChannelCallArguments args)
@@ -320,7 +324,13 @@ public bool IsApplicable(string ruleId, DocumentServiceRequest request)
             {
                 PartitionKeyRange pkRange = request.RequestContext.ResolvedPartitionKeyRange;
 
-                return this.pkRangeIds.Contains(pkRange.Id);
+                if (pkRange is null && this.includePrimaryForMetaData)
+                {
+                    //For metadata operations, rule will apply to all partition key ranges
+                    return true;
+                }
+
+                return this.pkRangeIds.Contains(pkRange?.Id);
             }
         }
 

Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionRuleProcessor.cs

@@ -98,9 +98,12 @@ private async Task<IFaultInjectionRuleInternal> GetEffectiveServerErrorRule(Faul
             FaultInjectionOperationType operationType = rule.GetCondition().GetOperationType();
             if ((operationType != FaultInjectionOperationType.All) && this.CanErrorLimitToOperation(errorType))
             {
-                effectiveCondition.SetOperationType(this.GetEffectiveOperationType(operationType));
-                //Will need to change when introducing metadata operations
-                effectiveCondition.SetResourceType(ResourceType.Document);
+                OperationType effectiveOperationType = this.GetEffectiveOperationType(operationType);
+                if (effectiveOperationType != OperationType.Invalid)
+                {
+                    effectiveCondition.SetOperationType(this.GetEffectiveOperationType(operationType));
+                }
+                effectiveCondition.SetResourceType(this.GetEffectiveResourceType(operationType));
             }
 
             List<Uri> regionEndpoints = this.GetRegionEndpoints(rule.GetCondition());
@@ -129,7 +132,10 @@ await BackoffRetryUtility<IEnumerable<string>>.ExecuteAsync(
                                 rule.GetCondition().GetEndpoint()),
                             this.retryPolicy());
 
-                    effectiveCondition.SetPartitionKeyRangeIds(effectivePKRangeId);
+                    if (!this.IsMetaData(rule.GetCondition().GetOperationType()))
+                    {
+                        effectiveCondition.SetPartitionKeyRangeIds(effectivePKRangeId, rule);
+                    }
                 }
             }
             else
@@ -246,6 +252,33 @@ private OperationType GetEffectiveOperationType(FaultInjectionOperationType faul
                 FaultInjectionOperationType.PatchItem => OperationType.Patch,
                 FaultInjectionOperationType.Batch => OperationType.Batch,
                 FaultInjectionOperationType.ReadFeed => OperationType.ReadFeed,
+                FaultInjectionOperationType.MetadataContainer => OperationType.Read,
+                FaultInjectionOperationType.MetadataDatabaseAccount => OperationType.Read,
+                FaultInjectionOperationType.MetadataPartitionKeyRange => OperationType.ReadFeed,
+                FaultInjectionOperationType.MetadataRefreshAddresses => OperationType.Invalid,
+                FaultInjectionOperationType.MetadataQueryPlan => OperationType.QueryPlan,
+                _ => throw new ArgumentException($"FaultInjectionOperationType: {faultInjectionOperationType} is not supported"),
+            };
+        }
+
+        private ResourceType GetEffectiveResourceType(FaultInjectionOperationType faultInjectionOperationType)
+        {
+            return faultInjectionOperationType switch
+            {
+                FaultInjectionOperationType.ReadItem => ResourceType.Document,
+                FaultInjectionOperationType.CreateItem => ResourceType.Document,
+                FaultInjectionOperationType.QueryItem => ResourceType.Document,
+                FaultInjectionOperationType.UpsertItem => ResourceType.Document,
+                FaultInjectionOperationType.ReplaceItem => ResourceType.Document,
+                FaultInjectionOperationType.DeleteItem => ResourceType.Document,
+                FaultInjectionOperationType.PatchItem => ResourceType.Document,
+                FaultInjectionOperationType.Batch => ResourceType.Document,
+                FaultInjectionOperationType.ReadFeed => ResourceType.Document,
+                FaultInjectionOperationType.MetadataContainer => ResourceType.Collection,
+                FaultInjectionOperationType.MetadataDatabaseAccount => ResourceType.DatabaseAccount,
+                FaultInjectionOperationType.MetadataPartitionKeyRange => ResourceType.PartitionKeyRange,
+                FaultInjectionOperationType.MetadataRefreshAddresses => ResourceType.Address,
+                FaultInjectionOperationType.MetadataQueryPlan => ResourceType.Document,
                 _ => throw new ArgumentException($"FaultInjectionOperationType: {faultInjectionOperationType} is not supported"),
             };
         }
@@ -322,6 +355,15 @@ private bool IsWriteOnly(FaultInjectionCondition condition)
                 && this.GetEffectiveOperationType(condition.GetOperationType()).IsWriteOperation();
         }
 
+        private bool IsMetaData(FaultInjectionOperationType operationType)
+        {
+            return operationType == FaultInjectionOperationType.MetadataContainer
+                || operationType == FaultInjectionOperationType.MetadataDatabaseAccount
+                || operationType == FaultInjectionOperationType.MetadataPartitionKeyRange
+                || operationType == FaultInjectionOperationType.MetadataRefreshAddresses
+                || operationType == FaultInjectionOperationType.MetadataQueryPlan;
+        }
+
         private async Task<List<Uri>> ResolvePhyicalAddresses(
             List<Uri> regionEndpoints,
             FaultInjectionCondition condition,

Microsoft.Azure.Cosmos/FaultInjection/src/implementation/FaultInjectionServerErrorResultInternal.cs

@@ -466,6 +466,28 @@ public HttpResponseMessage GetInjectedServerError(DocumentServiceRequest dsr, st
 
                     return httpResponse;
 
+                case FaultInjectionServerErrorType.DatabaseAccountNotFound:
+
+                    httpResponse = new HttpResponseMessage
+                    {
+                        StatusCode = HttpStatusCode.NotFound,
+                        Content = new FauntInjectionHttpContent(
+                            new MemoryStream(
+                                FaultInjectionResponseEncoding.GetBytes($"Fault Injection Server Error: DatabaseAccountNotFound, rule: {ruleId}"))),
+                    };
+
+                    foreach (string header in headers.AllKeys())
+                    {
+                        httpResponse.Headers.Add(header, headers.Get(header));
+                    }
+
+                    httpResponse.Headers.Add(
+                        WFConstants.BackendHeaders.SubStatus,
+                        ((int)SubStatusCodes.DatabaseAccountNotFound).ToString(CultureInfo.InvariantCulture));
+                    httpResponse.Headers.Add(WFConstants.BackendHeaders.LocalLSN, lsn);
+
+                    return httpResponse;
+
                 default:
                     throw new ArgumentException($"Server error type {this.serverErrorType} is not supported");
             }