Implementing apiserver flowcontrol metrics (aws#829)

zhihonl · Aug 29, 2023 · 02ac7fb · 02ac7fb
1 parent 773bb41
commit 02ac7fb
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 26 deletions.
diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml
@@ -169,6 +169,10 @@ exporters:
         label_matchers: [ ]
         metric_name_selectors:
           - apiserver_storage_list_duration_seconds
+      - dimensions: [ [ClusterName, priority_level], [ ClusterName ] ]
+        label_matchers: [ ]
+        metric_name_selectors:
+          - apiserver_flowcontrol_request_concurrency_limit
       - dimensions: [ [ ClusterName ] ]
         label_matchers: [ ]
         metric_name_selectors:
@@ -180,6 +184,7 @@ exporters:
           - rest_client_request_duration_seconds
           - rest_client_requests_total
           - etcd_request_duration_seconds
+          - apiserver_flowcontrol_rejected_requests_total
     metric_descriptors:
           - metric_name: apiserver_storage_objects
             unit: Count
@@ -205,6 +210,12 @@ exporters:
           - metric_name: etcd_request_duration_seconds
             unit: Seconds
             overwrite: true
+          - metric_name: apiserver_flowcontrol_rejected_requests_total
+            unit: Count
+            overwrite: true
+          - metric_name: apiserver_flowcontrol_request_concurrency_limit
+            unit: Count
+            overwrite: true
     namespace: ContainerInsights
     no_verify_ssl: false
     num_workers: 8

diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml
@@ -140,6 +140,10 @@ exporters:
         label_matchers: [ ]
         metric_name_selectors:
           - apiserver_storage_list_duration_seconds
+      - dimensions: [ [ClusterName, priority_level], [ ClusterName ] ]
+        label_matchers: [ ]
+        metric_name_selectors:
+          - apiserver_flowcontrol_request_concurrency_limit
       - dimensions: [ [ ClusterName ] ]
         label_matchers: [ ]
         metric_name_selectors:
@@ -151,6 +155,7 @@ exporters:
           - rest_client_request_duration_seconds
           - rest_client_requests_total
           - etcd_request_duration_seconds
+          - apiserver_flowcontrol_rejected_requests_total
     metric_descriptors:
       - metric_name: apiserver_storage_objects
         unit: Count
@@ -176,6 +181,12 @@ exporters:
       - metric_name: etcd_request_duration_seconds
         unit: Seconds
         overwrite: true
+      - metric_name: apiserver_flowcontrol_rejected_requests_total
+        unit: Count
+        overwrite: true
+      - metric_name: apiserver_flowcontrol_request_concurrency_limit
+        unit: Count
+        overwrite: true
     namespace: ContainerInsights
     no_verify_ssl: false
     num_workers: 8

diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml
@@ -169,6 +169,10 @@ exporters:
         label_matchers: [ ]
         metric_name_selectors:
           - apiserver_storage_list_duration_seconds
+      - dimensions: [ [ClusterName, priority_level], [ ClusterName ] ]
+        label_matchers: [ ]
+        metric_name_selectors:
+          - apiserver_flowcontrol_request_concurrency_limit
       - dimensions: [ [ ClusterName ] ]
         label_matchers: [ ]
         metric_name_selectors:
@@ -180,6 +184,7 @@ exporters:
           - rest_client_request_duration_seconds
           - rest_client_requests_total
           - etcd_request_duration_seconds
+          - apiserver_flowcontrol_rejected_requests_total
     metric_descriptors:
       - metric_name: apiserver_storage_objects
         unit: Count
@@ -205,6 +210,12 @@ exporters:
       - metric_name: etcd_request_duration_seconds
         unit: Seconds
         overwrite: true
+      - metric_name: apiserver_flowcontrol_rejected_requests_total
+        unit: Count
+        overwrite: true
+      - metric_name: apiserver_flowcontrol_request_concurrency_limit
+        unit: Count
+        overwrite: true
     namespace: ContainerInsights
     no_verify_ssl: false
     num_workers: 8

diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go
@@ -275,17 +275,24 @@ func getControlPlaneMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Met
 					"apiserver_storage_list_duration_seconds",
 				},
 			},
+			{
+				Dimensions: [][]string{{"ClusterName", "priority_level"}, {"ClusterName"}},
+				MetricNameSelectors: []string{
+					"apiserver_flowcontrol_request_concurrency_limit",
+				},
+			},
 			{
 				Dimensions: [][]string{{"ClusterName"}},
 				MetricNameSelectors: []string{
-					"apiserver_storage_objects",
+					"apiserver_admission_controller_admission_duration_seconds",
+					"apiserver_flowcontrol_rejected_requests_total",
+					"apiserver_request_duration_seconds",
 					"apiserver_request_total",
 					"apiserver_request_total_5xx",
-					"apiserver_request_duration_seconds",
-					"apiserver_admission_controller_admission_duration_seconds",
+					"apiserver_storage_objects",
+					"etcd_request_duration_seconds",
 					"rest_client_request_duration_seconds",
 					"rest_client_requests_total",
-					"etcd_request_duration_seconds",
 				},
 			},
 		}...)
@@ -299,17 +306,17 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri
 		// the control plane metrics do not have units so we need to add them manually
 		return []awsemfexporter.MetricDescriptor{
 			{
-				MetricName: "apiserver_storage_objects",
-				Unit:       "Count",
+				MetricName: "apiserver_admission_controller_admission_duration_seconds",
+				Unit:       "Seconds",
 				Overwrite:  true,
 			},
 			{
-				MetricName: "apiserver_request_total",
+				MetricName: "apiserver_flowcontrol_rejected_requests_total",
 				Unit:       "Count",
 				Overwrite:  true,
 			},
 			{
-				MetricName: "apiserver_request_total_5xx",
+				MetricName: "apiserver_flowcontrol_request_concurrency_limit",
 				Unit:       "Count",
 				Overwrite:  true,
 			},
@@ -319,17 +326,17 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri
 				Overwrite:  true,
 			},
 			{
-				MetricName: "apiserver_admission_controller_admission_duration_seconds",
-				Unit:       "Seconds",
+				MetricName: "apiserver_request_total",
+				Unit:       "Count",
 				Overwrite:  true,
 			},
 			{
-				MetricName: "rest_client_request_duration_seconds",
-				Unit:       "Seconds",
+				MetricName: "apiserver_request_total_5xx",
+				Unit:       "Count",
 				Overwrite:  true,
 			},
 			{
-				MetricName: "rest_client_requests_total",
+				MetricName: "apiserver_storage_objects",
 				Unit:       "Count",
 				Overwrite:  true,
 			},
@@ -338,6 +345,16 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri
 				Unit:       "Seconds",
 				Overwrite:  true,
 			},
+			{
+				MetricName: "rest_client_request_duration_seconds",
+				Unit:       "Seconds",
+				Overwrite:  true,
+			},
+			{
+				MetricName: "rest_client_requests_total",
+				Unit:       "Count",
+				Overwrite:  true,
+			},
 		}
 	}
 	return []awsemfexporter.MetricDescriptor{}

diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go
@@ -346,31 +346,38 @@ func TestTranslator(t *testing.T) {
 						Dimensions:          [][]string{{"ClusterName", "resource"}, {"ClusterName"}},
 						MetricNameSelectors: []string{"apiserver_storage_list_duration_seconds"},
 					},
+					{
+						Dimensions:          [][]string{{"ClusterName", "priority_level"}, {"ClusterName"}},
+						MetricNameSelectors: []string{"apiserver_flowcontrol_request_concurrency_limit"},
+					},
 					{
 						Dimensions: [][]string{{"ClusterName"}},
-						MetricNameSelectors: []string{"apiserver_storage_objects",
+						MetricNameSelectors: []string{
+							"apiserver_admission_controller_admission_duration_seconds",
+							"apiserver_flowcontrol_rejected_requests_total",
+							"apiserver_request_duration_seconds",
 							"apiserver_request_total",
 							"apiserver_request_total_5xx",
-							"apiserver_request_duration_seconds",
-							"apiserver_admission_controller_admission_duration_seconds",
+							"apiserver_storage_objects",
+							"etcd_request_duration_seconds",
 							"rest_client_request_duration_seconds",
 							"rest_client_requests_total",
-							"etcd_request_duration_seconds"},
+						},
 					},
 				},
 				"metric_descriptors": []awsemfexporter.MetricDescriptor{
 					{
-						MetricName: "apiserver_storage_objects",
-						Unit:       "Count",
+						MetricName: "apiserver_admission_controller_admission_duration_seconds",
+						Unit:       "Seconds",
 						Overwrite:  true,
 					},
 					{
-						MetricName: "apiserver_request_total",
+						MetricName: "apiserver_flowcontrol_request_concurrency_limit",
 						Unit:       "Count",
 						Overwrite:  true,
 					},
 					{
-						MetricName: "apiserver_request_total_5xx",
+						MetricName: "apiserver_flowcontrol_rejected_requests_total",
 						Unit:       "Count",
 						Overwrite:  true,
 					},
@@ -380,17 +387,17 @@ func TestTranslator(t *testing.T) {
 						Overwrite:  true,
 					},
 					{
-						MetricName: "apiserver_admission_controller_admission_duration_seconds",
-						Unit:       "Seconds",
+						MetricName: "apiserver_request_total",
+						Unit:       "Count",
 						Overwrite:  true,
 					},
 					{
-						MetricName: "rest_client_request_duration_seconds",
-						Unit:       "Seconds",
+						MetricName: "apiserver_request_total_5xx",
+						Unit:       "Count",
 						Overwrite:  true,
 					},
 					{
-						MetricName: "rest_client_requests_total",
+						MetricName: "apiserver_storage_objects",
 						Unit:       "Count",
 						Overwrite:  true,
 					},
@@ -399,6 +406,16 @@ func TestTranslator(t *testing.T) {
 						Unit:       "Seconds",
 						Overwrite:  true,
 					},
+					{
+						MetricName: "rest_client_request_duration_seconds",
+						Unit:       "Seconds",
+						Overwrite:  true,
+					},
+					{
+						MetricName: "rest_client_requests_total",
+						Unit:       "Count",
+						Overwrite:  true,
+					},
 				},
 			},
 		},