OM126 - 7.0 stats and config changes and vulnerability fixes (#96)

sorted newly added 7.0 stats/configs in gauge-list enhanced mockdata with 7.0 stats and configs updated golang.org/x/net (and http2) version to v0.17.0
aerospike · Nov 7, 2023 · 59cccaf · 59cccaf
1 parent 0b4c4f1
commit 59cccaf
Show file tree

Hide file tree

Showing 11 changed files with 187 additions and 107 deletions.
diff --git a/common.go b/common.go
@@ -40,15 +40,16 @@ const (
 
 // below constant represent the labels we send along with metrics to Prometheus or something
 const (
-	METRIC_LABEL_CLUSTER_NAME = "cluster_name"
-	METRIC_LABEL_SERVICE      = "service"
-	METRIC_LABEL_NS           = "ns"
-	METRIC_LABEL_SET          = "set"
-	METRIC_LABEL_LE           = "le"
-	METRIC_LABEL_DC_NAME      = "dc"
-	METRIC_LABEL_INDEX        = "index"
-	METRIC_LABEL_SINDEX       = "sindex"
-	METRIC_LABEL_USER         = "user"
+	METRIC_LABEL_CLUSTER_NAME   = "cluster_name"
+	METRIC_LABEL_SERVICE        = "service"
+	METRIC_LABEL_NS             = "ns"
+	METRIC_LABEL_SET            = "set"
+	METRIC_LABEL_LE             = "le"
+	METRIC_LABEL_DC_NAME        = "dc"
+	METRIC_LABEL_INDEX          = "index"
+	METRIC_LABEL_SINDEX         = "sindex"
+	METRIC_LABEL_STORAGE_ENGINE = "storage_engine"
+	METRIC_LABEL_USER           = "user"
 )
 
 // constants used to identify type of metrics

diff --git a/gauge_stats_list.toml b/gauge_stats_list.toml
@@ -13,6 +13,11 @@ sets_gauge_stats = [
 	"sindexes",
 	"tombstones",
 	"truncate_lut",
+
+	# 7.0 changes
+	"data_used_bytes",
+	"truncating",
+
 ]
 
 #
@@ -220,4 +225,18 @@ namespace_gauge_stats =[
 	"write_q",
 	"xdr_bin_cemeteries",
 	"xdr_tombstones",
+
+    # added in 7.0
+	"data_avail_pct",
+	"data_compression_ratio",
+	"data_total_bytes",
+	"data_used_bytes",
+	"data_used_pct",
+	"index_mounts_used_pct",
+	"index_used_bytes",
+	"set_index_used_bytes",
+	"sindex_mounts_used_pct",
+	"sindex_used_bytes",
+	"truncating",
+
 ]
diff --git a/gauge_stats_test.go b/gauge_stats_test.go
@@ -29,13 +29,13 @@ func TestGetGaugesCounts(t *testing.T) {
 	initGaugeStats(METRICS_CONFIG_FILE, gaugeList)
 
 	glist := gaugeList.NamespaceStats
-	assert.Equal(t, len(glist), 88)
+	assert.Equal(t, len(glist), 99)
 
 	glist = gaugeList.NodeStats
 	assert.Equal(t, len(glist), 69)
 
 	glist = gaugeList.SetsStats
-	assert.Equal(t, len(glist), 7)
+	assert.Equal(t, len(glist), 9)
 
 	glist = gaugeList.SindexStats
 	assert.Equal(t, len(glist), 13)

diff --git a/go.mod b/go.mod
@@ -24,8 +24,9 @@ require (
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/yuin/gopher-lua v1.1.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -99,7 +99,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -124,14 +125,14 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=

diff --git a/helper_test.go b/helper_test.go
@@ -105,11 +105,12 @@ func splitAndRetrieveStats(s, sep string) map[string]string {
 }
 
 func convertValue(s string) (float64, error) {
-	if f, err := strconv.ParseFloat(s, 64); err == nil {
+	temp_s := strings.TrimSpace(s)
+	if f, err := strconv.ParseFloat(temp_s, 64); err == nil {
 		return f, nil
 	}
 
-	if b, err := strconv.ParseBool(s); err == nil {
+	if b, err := strconv.ParseBool(temp_s); err == nil {
 		if b {
 			return 1, nil
 		}

diff --git a/mock_data_generator_test.go b/mock_data_generator_test.go
@@ -49,7 +49,7 @@ func (md *MockAerospikeServer) initialize() {
 	var fileLines []string
 
 	for fileScanner.Scan() {
-		fileLines = append(fileLines, fileScanner.Text())
+		fileLines = append(fileLines, strings.TrimSpace(fileScanner.Text()))
 	}
 
 	readFile.Close()
@@ -170,9 +170,11 @@ func (md *MockAerospikeServer) getBuild() string {
 			colonIndex := strings.Index(entry, ":")
 			// parts := strings.Split(entry, ":")
 			value := entry[colonIndex+1:]
+			fmt.Println("getBuild(): ", value)
 			return value
 		}
 	}
+
 	return ""
 }
 
@@ -231,7 +233,7 @@ func (md *MockAerospikeServer) createNamespacePassTwoExpectedOutputs() []string
 
 	for metricsGrpKey := range rawMetrics {
 		if strings.HasPrefix(metricsGrpKey, "namespace/") {
-			lNamespaces = append(lNamespaces, metricsGrpKey)
+			lNamespaces = append(lNamespaces, strings.TrimSpace(metricsGrpKey))
 		}
 	}
 	return lNamespaces
@@ -243,7 +245,7 @@ func (md *MockAerospikeServer) createXdrPassTwoExpectedOutputs(passOneOutputs ma
 
 	for k := range rawMetrics {
 		if strings.HasPrefix(k, "get-stats:") || strings.HasPrefix(k, "get-config:") {
-			passTwoOutputs = append(passTwoOutputs, k)
+			passTwoOutputs = append(passTwoOutputs, strings.TrimSpace(k))
 		}
 	}
 	// append namespaces
@@ -258,7 +260,7 @@ func (sim *MockAerospikeServer) createSindexPassTwoExpectedOutputs(mas *MockAero
 
 	for k := range rawMetricsKeys {
 		if strings.HasPrefix(k, "sindex/") {
-			lSindexNames = append(lSindexNames, k)
+			lSindexNames = append(lSindexNames, strings.TrimSpace(k))
 		}
 	}
 
@@ -280,7 +282,7 @@ type MockNamespacePromMetricGenerator struct {
 func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutputs(mas *MockAerospikeServer, nsName string, addNsToKey bool) (map[string][]string, map[string][]string) {
 
 	// regex to check if a stat is storage-engine
-	seDynamicExtractor := regexp.MustCompile(`storage\-engine\.(?P<type>file|device)\[(?P<idx>\d+)\]\.(?P<metric>.+)`)
+	seDynamicExtractor := regexp.MustCompile(`storage\-engine\.(?P<type>file|device|stripe)\[(?P<idx>\d+)\]\.(?P<metric>.+)`)
 	itDynamicExtractor := regexp.MustCompile(`index\-type\.(?P<type>mount)\[(?P<idx>\d+)\]\.(?P<metric>.+)`)
 	sintDynamicExtractor := regexp.MustCompile(`sindex\-type\.(?P<type>mount)\[(?P<idx>\d+)\]\.(?P<metric>.+)`)
 
@@ -302,7 +304,7 @@ func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutpu
 
 			// process stats and create {metricname,label} and {metric,values} map of strings
 			for s, v := range stats {
-				key := s
+				key := strings.TrimSpace(s)
 				convertedValue, err := convertValue(v)
 				if err != nil {
 					fmt.Println("IGNORING, failed converting value ( key: ", key, " - value: ", v, ") = error: ", err)
@@ -331,8 +333,9 @@ func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutpu
 						labelsMap["index"] = stats["index-type"]
 					} else if strings.HasPrefix(key, "sindex-type") {
 						labelsMap["sindex"] = stats["sindex-type"]
+					} else {
+						labelsMap["storage_engine"] = stats["storage-engine"]
 					}
-
 					key = strings.ReplaceAll(key, "-", "_")
 					key = strings.ReplaceAll(key, ".", "_")
 
@@ -354,8 +357,8 @@ func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutpu
 
 					labelByNames := createLabelByNames(labelsMap)
 					sorted_label_str := "[" + labelByNames + " ]"
-					valuesArr = append(valuesArr, value)
-					labelsArr = append(labelsArr, sorted_label_str)
+					valuesArr = append(valuesArr, strings.TrimSpace(value))
+					labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 					labelString := stringifyLabel(labelByNames)
 
@@ -382,6 +385,7 @@ func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutpu
 						metricName = match[3]
 						deviceOrFileName = stats["storage-engine."+metricType+"["+metricIndex+"]"]
 						metric = "aerospike_namespace_storage_engine_" + metricType + "_" + metricName
+						labelsMap["storage_engine"] = stats["storage-engine"]
 
 					} else if strings.HasPrefix(key, "index-type") {
 						match := itDynamicExtractor.FindStringSubmatch(key)
@@ -421,8 +425,8 @@ func (nsw *MockNamespacePromMetricGenerator) createNamespaceWatcherExpectedOutpu
 
 					labelByNames := createLabelByNames(labelsMap)
 					sorted_label_str := "[" + labelByNames + " ]"
-					valuesArr = append(valuesArr, value)
-					labelsArr = append(labelsArr, sorted_label_str)
+					valuesArr = append(valuesArr, strings.TrimSpace(value))
+					labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 					labelString := stringifyLabel(labelByNames)
 
@@ -475,7 +479,7 @@ func (nst *MockNodestatPromMetricGenerator) createNodeStatsWatcherExpectedOutput
 			grpValues := rawMetrics[metricsGrpKey]
 			stats := splitAndRetrieveStats(grpValues, ";")
 			for s, v := range stats {
-				key := s
+				key := strings.TrimSpace(s)
 				convertedValue, err := convertValue(v)
 				if err != nil {
 					fmt.Println("IGNORING, failed converting value ( key: ", key, " - value: ", v, ") = error: ", err)
@@ -514,8 +518,8 @@ func (nst *MockNodestatPromMetricGenerator) createNodeStatsWatcherExpectedOutput
 
 				sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-				valuesArr = append(valuesArr, value)
-				labelsArr = append(labelsArr, sorted_label_str)
+				valuesArr = append(valuesArr, strings.TrimSpace(value))
+				labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 				mapKeyname := makeKeyname(serviceIp, metric, true)
 				nst.lExpectedMetricNamedValues[mapKeyname] = valuesArr
@@ -559,7 +563,7 @@ func (msdg *MockSetsPromMetricGenerator) createSetsWatcherExpectedOutputs(mas *M
 				setName := stats["set"]
 
 				for s, v := range stats {
-					key := s
+					key := strings.TrimSpace(s)
 					convertedValue, err := convertValue(v)
 					if err != nil {
 						fmt.Println("IGNORING, failed converting value ( key: ", key, " - value: ", v, ") = error: ", err)
@@ -603,8 +607,8 @@ func (msdg *MockSetsPromMetricGenerator) createSetsWatcherExpectedOutputs(mas *M
 					labelsMap["set"] = setName
 
 					sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
-					valuesArr = append(valuesArr, value)
-					labelsArr = append(labelsArr, sorted_label_str)
+					valuesArr = append(valuesArr, strings.TrimSpace(value))
+					labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 					msdg.lExpectedMetricNamedValues[mapKeyname] = valuesArr
 					msdg.lExpectedMetricLabels[mapKeyname] = labelsArr
@@ -674,8 +678,8 @@ func (ltc *MockLatencyPromMetricGenerator) createLatencysWatcherExpectedOutputs(
 
 					sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-					cntValuesArr = append(cntValuesArr, allLatencyValues[1])
-					cntLabelsArr = append(cntLabelsArr, sorted_label_str)
+					cntValuesArr = append(cntValuesArr, strings.TrimSpace(allLatencyValues[1]))
+					cntLabelsArr = append(cntLabelsArr, strings.TrimSpace(sorted_label_str))
 
 					ltc.lExpectedMetricNamedValues[mapKeyname] = cntValuesArr
 					ltc.lExpectedMetricLabels[mapKeyname] = cntLabelsArr
@@ -706,8 +710,8 @@ func (ltc *MockLatencyPromMetricGenerator) createLatencysWatcherExpectedOutputs(
 
 							sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-							bktValuesArr = append(bktValuesArr, value)
-							bktLabelsArr = append(bktLabelsArr, sorted_label_str)
+							bktValuesArr = append(bktValuesArr, strings.TrimSpace(value))
+							bktLabelsArr = append(bktLabelsArr, strings.TrimSpace(sorted_label_str))
 
 							ltc.lExpectedMetricNamedValues[mapKeyname] = bktValuesArr
 							ltc.lExpectedMetricLabels[mapKeyname] = bktLabelsArr
@@ -763,7 +767,7 @@ func (xdr *MockXdrPromMetricGenerator) createXdrsWatcherExpectedOutputs(mas *Moc
 				// dcName := strings.Split(grpSplitElements[1], "=")[1]
 
 				for s, v := range stats {
-					key := s
+					key := strings.TrimSpace(s)
 					convertedValue, err := convertValue(v)
 					if err != nil {
 						fmt.Println("IGNORING, failed converting value ( key: ", key, " - value: ", v, ") = error: ", err)
@@ -815,8 +819,8 @@ func (xdr *MockXdrPromMetricGenerator) createXdrsWatcherExpectedOutputs(mas *Moc
 					}
 
 					sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
-					valuesArr = append(valuesArr, value)
-					labelsArr = append(labelsArr, sorted_label_str)
+					valuesArr = append(valuesArr, strings.TrimSpace(value))
+					labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 					mapKeyname := makeKeyname(dcName, metric, true)
 					mapKeyname = makeKeyname(namespace, mapKeyname, true)
@@ -880,7 +884,7 @@ func (sim *MockSindexPromMetricGenerator) createSindexWatcherTestData(mas *MockA
 			namespace, sindexName := sim.extractNamespaceSetSindexname(metricsGrpKey)
 			stats := splitAndRetrieveStats(sindexInfos, ";")
 			for stat, value := range stats {
-				key := stat
+				key := strings.TrimSpace(stat)
 				convertedValue, err := convertValue(value)
 				if err != nil {
 					fmt.Println("IGNORING, failed converting value ( key: ", key, " - value: ", value, ") = error: ", err)
@@ -926,8 +930,8 @@ func (sim *MockSindexPromMetricGenerator) createSindexWatcherTestData(mas *MockA
 
 				sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-				valuesArr = append(valuesArr, value)
-				labelsArr = append(labelsArr, sorted_label_str)
+				valuesArr = append(valuesArr, strings.TrimSpace(value))
+				labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 				sim.lExpectedMetricNamedValues[mapKeyname] = valuesArr
 				sim.lExpectedMetricLabels[mapKeyname] = labelsArr
@@ -1027,8 +1031,8 @@ func (mockUsers *MockUsersPromMetricGenerator) createMockUserData(mas *MockAeros
 
 		sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-		valuesArr = append(valuesArr, value)
-		labelsArr = append(labelsArr, sorted_label_str)
+		valuesArr = append(valuesArr, strings.TrimSpace(value))
+		labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 		mockUsers.lExpectedMetricNamedValues[mapKeyname] = valuesArr
 		mockUsers.lExpectedMetricLabels[mapKeyname] = labelsArr
@@ -1063,8 +1067,8 @@ func (mockUsers *MockUsersPromMetricGenerator) createMockUserData(mas *MockAeros
 
 			sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-			valuesArr = append(valuesArr, value)
-			labelsArr = append(labelsArr, sorted_label_str)
+			valuesArr = append(valuesArr, strings.TrimSpace(value))
+			labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 			mockUsers.lExpectedMetricNamedValues[mapKeyname] = valuesArr
 			mockUsers.lExpectedMetricLabels[mapKeyname] = labelsArr
@@ -1098,8 +1102,8 @@ func (mockUsers *MockUsersPromMetricGenerator) createMockUserData(mas *MockAeros
 
 			sorted_label_str := "[" + createLabelByNames(labelsMap) + " ]"
 
-			valuesArr = append(valuesArr, value)
-			labelsArr = append(labelsArr, sorted_label_str)
+			valuesArr = append(valuesArr, strings.TrimSpace(value))
+			labelsArr = append(labelsArr, strings.TrimSpace(sorted_label_str))
 
 			mockUsers.lExpectedMetricNamedValues[mapKeyname] = valuesArr
 			mockUsers.lExpectedMetricLabels[mapKeyname] = labelsArr