Cilium API Rate Limiting Test

modules/python/clusterloader2/slo/config/load-config.yaml

@@ -2,6 +2,9 @@ name: load-config
 
 # Config options for test type
 {{$SERVICE_TEST := DefaultParam .CL2_SERVICE_TEST true}}
+# Mugesh - API Rate Limiting Test config
+{{$API_RATE_LIMITING_TEST := DefaultParam .CL2_API_RATE_LIMITING_TEST false}}
+
 
 # Config options for test parameters
 {{$nodesPerNamespace := DefaultParam .CL2_NODES_PER_NAMESPACE 100}}
@@ -11,6 +14,7 @@ name: load-config
 {{$repeats := DefaultParam .CL2_REPEATS 1}}
 {{$groupName := DefaultParam .CL2_GROUP_NAME "service-discovery"}}
 
+
 # TODO(jshr-w): This should eventually use >1 namespace.
 {{$namespaces := 1}}
 {{$nodes := DefaultParam .CL2_NODES 1000}}
@@ -34,6 +38,13 @@ name: load-config
 {{$smallDeploymentPods := SubtractInt $podsPerNamespace (MultiplyInt $bigDeploymentsPerNamespace $BIG_GROUP_SIZE)}}
 {{$smallDeploymentsPerNamespace := DivideInt $smallDeploymentPods $SMALL_GROUP_SIZE}}
 
+
+# Mugesh - API Rate Limiting Test - Added necessary config for API Rate Limiting Test
+{{$PODS := DefaultParam .CL2_PODS 0}}
+
+
+
+
 namespace:
   number: {{$namespaces}}
   prefix: slo
@@ -51,6 +62,10 @@ tuningSets:
   - name: DeploymentDeleteQps
     qpsLoad:
       qps: {{$deploymentQPS}}
+  # Mugesh - API Rate Limiting Test - Added BurstLoad tuning set
+  - name: BurstLoad
+    qpsLoad:
+      qps: 1000
 
 steps:
   - name: Log - namespaces={{$namespaces}}, nodesPerNamespace={{$nodesPerNamespace}}, podsPerNode={{$podsPerNode}}, totalPods={{$totalPods}}, podsPerNamespace={{$podsPerNamespace}}, deploymentsPerNamespace={{$deploymentsPerNamespace}}, deploymentSize={{$deploymentSize}}, deploymentQPS={{$deploymentQPS}}
@@ -85,9 +100,15 @@ steps:
         bigServicesPerNamespace: {{$bigDeploymentsPerNamespace}}
 {{end}}
 
+# Mugesh - Added a conditional statement to check if API Rate Limiting Test is enabled to proceed with the default load test configuration.
+{{if not $API_RATE_LIMITING_TEST}}
   - module:
       path: /modules/reconcile-objects.yaml
       params:
+
+
+
+
         actionName: "create"
         namespaces: {{$namespaces}}
         tuningSet: DeploymentCreateQps
@@ -131,6 +152,77 @@ steps:
         deploymentLabel: restart
         Group: {{$groupName}}
 
+{{end}}
+
+# Mugesh - API Rate Limiting Test - create 200 pods, delete, restart with 250 pods
+
+{{if $API_RATE_LIMITING_TEST}}
+
+  - module:
+      path: /modules/reconcile-objects.yaml
+      params:
+        actionName: "Create {{$PODS}} pods"
+        namespaces: {{$namespaces}}
+        tuningSet: BurstLoad
+        operationTimeout: {{$operationTimeout}}
+        bigDeploymentSize: {{$PODS}}
+        bigDeploymentsPerNamespace: 1
+        smallDeploymentSize: 0
+        smallDeploymentsPerNamespace: 0
+        CpuRequest: {{$latencyPodCpu}}m
+        MemoryRequest: {{$latencyPodMemory}}M
+        Group: {{$groupName}}
+        deploymentLabel: start
+
+  - module:
+      path: /modules/reconcile-objects.yaml
+      params:
+        actionName: "delete {{$PODS}} pods"
+        namespaces: {{$namespaces}}
+        tuningSet: BurstLoad
+        operationTimeout: {{$operationTimeout}}
+        bigDeploymentSize: {{$PODS}}
+        bigDeploymentsPerNamespace: 0
+        smallDeploymentSize: 0
+        smallDeploymentsPerNamespace: 0
+        deploymentLabel: delete
+        Group: {{$groupName}}
+        waitForPods: false
+
+  - module:
+      path: /modules/reconcile-objects.yaml
+      params:
+        actionName: "recreate {{$PODS}} pods"
+        namespaces: {{$namespaces}}
+        tuningSet: BurstLoad
+        operationTimeout: {{$operationTimeout}}
+        bigDeploymentSize: {{$PODS}}
+        bigDeploymentsPerNamespace: 1
+        smallDeploymentSize: 0
+        smallDeploymentsPerNamespace: 0
+        CpuRequest: {{$latencyPodCpu}}m
+        MemoryRequest: {{$latencyPodMemory}}M
+        Group: {{$groupName}}
+        deploymentLabel: restart
+
+  - module:
+      path: /modules/reconcile-objects.yaml
+      params:
+        actionName: "delete {{$PODS}} pods"
+        namespaces: {{$namespaces}}
+        tuningSet: BurstLoad
+        operationTimeout: {{$operationTimeout}}
+        bigDeploymentSize: {{$PODS}}
+        bigDeploymentsPerNamespace: 0
+        smallDeploymentSize: 0
+        smallDeploymentsPerNamespace: 0
+        deploymentLabel: delete
+        Group: {{$groupName}}
+
+{{end}}
+
+# Mugesh - Added conditional statement to delete services (Previously not there)
+{{if $SERVICE_TEST}}
   - module:
       path: /modules/services.yaml
       params:
@@ -140,6 +232,8 @@ steps:
         bigServicesPerNamespace: 0
 {{end}}
 
+{{end}}
+
 {{if $CILIUM_METRICS_ENABLED}}
   - module:
       path: /modules/cilium-measurements.yaml

modules/python/clusterloader2/slo/config/modules/cilium-measurements.yaml

@@ -150,4 +150,118 @@ steps:
         - name: Perc90
           query: quantile(0.90, avg_over_time(cilium_operator_process_resident_memory_bytes[%v:]) / 1024 / 1024)
         - name: Perc50
-          query: quantile(0.5, avg_over_time(cilium_operator_process_resident_memory_bytes[%v:]) / 1024 / 1024)
+          query: quantile(0.5, avg_over_time(cilium_operator_process_resident_memory_bytes[%v:]) / 1024 / 1024)
+
+    # Cilium API Rate Limiter Metrics
+    - Identifier: CiliumAgentAPIProcessTime
+      Method: GenericPrometheusQuery
+      Params:
+        action: {{$action}}
+        metricName: Cilium Agent API Process Time
+        metricVersion: v1
+        unit: s
+        enableViolations: true
+        queries:
+        - name: Perc99
+          query: histogram_quantile(0.99, sum(rate(cilium_agent_api_process_time_seconds_bucket[%v:])) by (le))
+        - name: Perc95
+          query: histogram_quantile(0.95, sum(rate(cilium_agent_api_process_time_seconds_bucket[%v:])) by (le))
+        - name: Perc50
+          query: histogram_quantile(0.50, sum(rate(cilium_agent_api_process_time_seconds_bucket[%v:])) by (le))
+        - name: Avg
+          query: avg(rate(cilium_agent_api_process_time_seconds_sum[%v:])) / avg(rate(cilium_agent_api_process_time_seconds_count[%v:]))
+        - name: Total
+          query: sum(rate(cilium_agent_api_process_time_seconds_sum[%v:]))
+        - name: Max
+          query: max(histogram_quantile(1.0, sum(rate(cilium_agent_api_process_time_seconds_bucket[%v:])) by (le)))
+        - name: Requests
+          query: sum(increase(cilium_agent_api_process_time_seconds_count[%v:]))
+        - name: TotalTime
+          query: sum(increase(cilium_agent_api_process_time_seconds_sum[%v:]))
+
+
+    # - Identifier: CiliumAPIRateLimiterProcessedRequests
+    #   Method: GenericPrometheusQuery
+    #   Params:
+    #     action: {{$action}}
+    #     metricName: Cilium API Rate Limiter Processed Requests
+    #     metricVersion: v1
+    #     unit: requests/second
+    #     enableViolations: true
+    #     queries:
+    #     - name: Total
+    #       query: sum(cilium_api_limiter_processed_requests_count)
+    #     - name: Rate
+    #       query: sum(rate(cilium_api_limiter_processed_requests_count[%v:]))
+
+
+    # - Identifier: CiliumAPIRateLimiterProcessingDuration
+    #   Method: GenericPrometheusQuery
+    #   Params:
+    #     action: {{$action}}
+    #     metricName: Cilium API Rate Limiter Processing Duration in Seconds
+    #     metricVersion: v1
+    #     unit: s
+    #     enableViolations: true
+    #     queries:
+    #     - name: Avg
+    #       query: avg(sum_over_time(cilium_api_limiter_processing_duration_seconds[0][%v]))
+    #     - name: Max
+    #       query: max_over_time(max(cilium_api_limiter_processing_duration_seconds[0])[%v])
+
+
+    # - Identifier: CiliumAPIRateLimiterRequestsInFlight
+    #   Method: GenericPrometheusQuery
+    #   Params:
+    #     action: {{$action}}
+    #     metricName: Cilium API Rate Limiter Requests In Flight
+    #     metricVersion: v1
+    #     unit: count
+    #     enableViolations: true
+    #     queries:
+    #     - name: Instantaneous
+    #       query: sum(cilium_api_limiter_requests_in_flight)
+    #     - name: MaxOverTimeSum
+    #       query: max_over_time(cilium_api_limiter_requests_in_flight[0][%v:])
+    #     - name: AvgOverTimeSum
+    #       query: avg_over_time(cilium_api_limiter_requests_in_flight[0][%v:])
+    #     - name: SumOverTime
+    #       query: sum_over_time(cilium_api_limiter_requests_in_flight[0][%v:])
+    #     - name: MinOverTimeSum
+    #       query: min_over_time(cilium_api_limiter_requests_in_flight[0][%v:])
+
+
+    # - Identifier: CiliumAPIRateLimiterWaitDuration
+    #   Method: GenericPrometheusQuery
+    #   Params:
+    #     action: {{$action}}
+    #     metricName: Cilium API Rate Limiter Wait Duration in Seconds
+    #     metricVersion: v1
+    #     unit: s
+    #     enableViolations: true
+    #     queries:
+    #     - name: Avg
+    #       query: avg(cilium_api_limiter_wait_duration_seconds{value="mean", api_call="endpoint-create"}[%v])
+    #     - name: Max
+    #       query: max_over_time(cilium_api_limiter_wait_duration_seconds{value="max", api_call="endpoint-create"}[%v])
+    #     - name: Min
+    #       query: min_over_time(cilium_api_limiter_wait_duration_seconds{value="min", api_call="endpoint-create"}[%v])
+
+
+    # - Identifier: CiliumAPIRateLimiterRateLimit
+    #   Method: GenericPrometheusQuery
+    #   Params:
+    #     action: {{$action}}
+    #     metricName: Cilium API Rate Limiter Rate Limit
+    #     metricVersion: v1
+    #     unit: count
+    #     enableViolations: true
+    #     queries:
+    #     - name: Avg
+    #       query: avg(avg_over_time(cilium_api_limiter_rate_limit{value="limit", api_call="endpoint-create"}[%v]))
+    #     - name: Max
+    #       query: max_over_time(cilium_api_limiter_rate_limit{value="limit", api_call="endpoint-create"}[%v])
+    #     - name: Min
+    #       query: min_over_time(cilium_api_limiter_rate_limit{value="limit", api_call="endpoint-create"}[%v])
+
+

modules/python/clusterloader2/slo/config/modules/measurements.yaml

@@ -28,14 +28,14 @@ steps:
           action: {{$action}}
           labelSelector: group = {{.group}}
           threshold: {{$podStartupLatencyThreshold}}
-    {{if $PROMETHEUS_SCRAPE_KUBE_PROXY}}
-      - Identifier: NetworkProgrammingLatency
-        Method: NetworkProgrammingLatency
-        Params:
-          action: {{$action}}
-          enableViolations: {{$ENABLE_VIOLATIONS_FOR_NETWORK_PROGRAMMING_LATENCIES}}
-          threshold: {{$NETWORK_PROGRAMMING_LATENCY_THRESHOLD}}
-    {{end}}
+    # {{if $PROMETHEUS_SCRAPE_KUBE_PROXY}}
+    #   - Identifier: NetworkProgrammingLatency
+    #     Method: NetworkProgrammingLatency
+    #     Params:
+    #       action: {{$action}}
+    #       enableViolations: {{$ENABLE_VIOLATIONS_FOR_NETWORK_PROGRAMMING_LATENCIES}}
+    #       threshold: {{$NETWORK_PROGRAMMING_LATENCY_THRESHOLD}}
+    # {{end}}
     {{if $ENABLE_IN_CLUSTER_NETWORK_LATENCY}}
       - Identifier: InClusterNetworkLatency
         Method: InClusterNetworkLatency

modules/python/clusterloader2/slo/config/modules/reconcile-objects.yaml

@@ -4,15 +4,20 @@
 {{$namespaces := .namespaces}}
 {{$tuningSet := .tuningSet}}
 
+# flag to wait for pods to be deleted
+{{$waitToDelete := DefaultParam .waitForPods true}}
+
 # Derivative variables
 {{$is_deleting := (eq .actionName "delete")}}
 {{$operationTimeout := .operationTimeout}}
 
 # Deployments
 {{$bigDeploymentSize := .bigDeploymentSize}}
 {{$bigDeploymentsPerNamespace := .bigDeploymentsPerNamespace}}
-{{$smallDeploymentSize := .smallDeploymentSize}}
-{{$smallDeploymentsPerNamespace := .smallDeploymentsPerNamespace}}
+
+# Mugesh - Set Default values to 0 if not provided
+{{$smallDeploymentSize := DefaultParam .smallDeploymentSize 0}}
+{{$smallDeploymentsPerNamespace := DefaultParam .smallDeploymentsPerNamespace 0}}
 
 steps:
 - name: Starting measurement for '{{$actionName}}'
@@ -59,11 +64,14 @@ steps:
         Group: {{.Group}}
         deploymentLabel: {{.deploymentLabel}}
 
+# Mugesh - Wait for pods to be deleted
+{{if $waitToDelete}}
 - name: Waiting for '{{$actionName}}' to be completed
   measurements:
   - Method: WaitForControlledPodsRunning
     Instances:
     - Identifier: WaitForRunningDeployments
     Params:
       action: gather
-      refreshInterval: 15s
+      refreshInterval: 15s
+{{end}}