Add more error checking

Adding more error checking of values. Signed-off-by: Joe Talerico <[email protected]>
cloud-bulldozer · Nov 1, 2023 · 138d26a · 138d26a
1 parent 4e93887
commit 138d26a
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 12 deletions.
diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go
@@ -29,6 +29,7 @@ import (
 
 const namespace = "netperf"
 const index = "k8s-netperf"
+const retry = 3
 
 var (
 	cfgfile     string
@@ -389,15 +390,27 @@ func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, ipe
 			nr, err = netperf.ParseResults(&r)
 			if err != nil {
 				log.Error(err)
-				log.Warn("Rerunning test.")
-				r, err := netperf.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP)
-				if err != nil {
-					log.Error(err)
-					os.Exit(1)
+				try := 0
+				success := false
+				// Retry the current test.
+				for try < retry {
+					log.Warn("Rerunning test.")
+					r, err := netperf.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP)
+					if err != nil {
+						log.Error(err)
+						continue
+					}
+					nr, err = netperf.ParseResults(&r)
+					if err != nil {
+						log.Error(err)
+						try++
+					} else {
+						success = true
+						break
+					}
 				}
-				nr, err = netperf.ParseResults(&r)
-				if err != nil {
-					log.Error(err)
+				if !success {
+					fmt.Errorf("test was unsuccessful after retry.")
 					os.Exit(1)
 				}
 			}

diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go
@@ -102,10 +102,34 @@ func BuildDocs(sr result.ScenarioResults, uuid string) ([]interface{}, error) {
 			ClientNodeLabels: r.ClientNodeLabels,
 			AcrossAZ:         r.AcrossAZ,
 		}
-		d.UDPLossPercent, _ = result.Average(r.LossSummary)
-		d.TCPRetransmit, _ = result.Average(r.RetransmitSummary)
-		d.Throughput, _ = result.Average(r.ThroughputSummary)
-		d.Latency, _ = result.Average(r.LatencySummary)
+		UDPLossPercent, e := result.Average(r.LossSummary)
+		if e != nil {
+			logging.Warn("Unable to process udp loss, setting value to zero")
+			d.UDPLossPercent = 0
+		} else {
+			d.UDPLossPercent = UDPLossPercent
+		}
+		TCPRetransmit, e := result.Average(r.RetransmitSummary)
+		if e != nil {
+			logging.Warn("Unable to process tcp retransmits, setting value to zero")
+			d.TCPRetransmit = 0
+		} else {
+			d.TCPRetransmit = TCPRetransmit
+		}
+		Throughput, e := result.Average(r.ThroughputSummary)
+		if e != nil {
+			logging.Warn("Unable to process throughput, setting value to zero")
+			d.Throughput = 0
+		} else {
+			d.Throughput = Throughput
+		}
+		Latency, e := result.Average(r.LatencySummary)
+		if e != nil {
+			logging.Warn("Unable to process latency, setting value to zero")
+			d.Latency = 0
+		} else {
+			d.Latency = Latency
+		}
 		docs = append(docs, d)
 	}
 	return docs, nil

diff --git a/pkg/netperf/netperf.go b/pkg/netperf/netperf.go
@@ -105,6 +105,9 @@ func ParseResults(stdout *bytes.Buffer) (sample.Sample, error) {
 		if strings.Contains(l[0], "THROUGHPUT_UNITS") {
 			sample.Metric = l[1]
 		} else if strings.Contains(l[0], "THROUGHPUT") {
+			if len(strings.TrimSpace(l[1])) < 1 {
+				return sample, fmt.Errorf("Throughput was empty.")
+			}
 			sample.Throughput, _ = strconv.ParseFloat(strings.Trim(l[1], "\r"), 64)
 		} else if strings.Contains(l[0], "P99_LATENCY") {
 			sample.Latency99ptile, _ = strconv.ParseFloat(strings.Trim(l[1], "\r"), 64)
@@ -121,6 +124,9 @@ func ParseResults(stdout *bytes.Buffer) (sample.Sample, error) {
 	if math.IsNaN(sample.Throughput) {
 		return sample, fmt.Errorf("Throughput value is NaN")
 	}
+	if math.IsNaN(sample.Latency99ptile) {
+		return sample, fmt.Errorf("Latency value is NaN")
+	}
 	sample.LossPercent = 100 - (recv / send * 100)
 	return sample, nil
 }