diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 860c8f58..a995c2d6 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -29,6 +29,7 @@ import ( const namespace = "netperf" const index = "k8s-netperf" +const retry = 3 var ( cfgfile string @@ -389,15 +390,27 @@ func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, ipe nr, err = netperf.ParseResults(&r) if err != nil { log.Error(err) - log.Warn("Rerunning test.") - r, err := netperf.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP) - if err != nil { - log.Error(err) - os.Exit(1) + try := 0 + success := false + // Retry the current test. + for try < retry { + log.Warn("Rerunning test.") + r, err := netperf.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP) + if err != nil { + log.Error(err) + continue + } + nr, err = netperf.ParseResults(&r) + if err != nil { + log.Error(err) + try++ + } else { + success = true + break + } } - nr, err = netperf.ParseResults(&r) - if err != nil { - log.Error(err) + if !success { + fmt.Errorf("test was unsuccessful after retry.") os.Exit(1) } } diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index a817f6ba..2eb73b42 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -102,10 +102,34 @@ func BuildDocs(sr result.ScenarioResults, uuid string) ([]interface{}, error) { ClientNodeLabels: r.ClientNodeLabels, AcrossAZ: r.AcrossAZ, } - d.UDPLossPercent, _ = result.Average(r.LossSummary) - d.TCPRetransmit, _ = result.Average(r.RetransmitSummary) - d.Throughput, _ = result.Average(r.ThroughputSummary) - d.Latency, _ = result.Average(r.LatencySummary) + UDPLossPercent, e := result.Average(r.LossSummary) + if e != nil { + logging.Warn("Unable to process udp loss, setting value to zero") + d.UDPLossPercent = 0 + } else { + d.UDPLossPercent = UDPLossPercent + } + TCPRetransmit, e := result.Average(r.RetransmitSummary) + if e != nil { + logging.Warn("Unable to process tcp retransmits, setting value to zero") + d.TCPRetransmit = 0 + } else { + d.TCPRetransmit = TCPRetransmit + } + Throughput, e := result.Average(r.ThroughputSummary) + if e != nil { + logging.Warn("Unable to process throughput, setting value to zero") + d.Throughput = 0 + } else { + d.Throughput = Throughput + } + Latency, e := result.Average(r.LatencySummary) + if e != nil { + logging.Warn("Unable to process latency, setting value to zero") + d.Latency = 0 + } else { + d.Latency = Latency + } docs = append(docs, d) } return docs, nil diff --git a/pkg/netperf/netperf.go b/pkg/netperf/netperf.go index a6d02355..6620744e 100644 --- a/pkg/netperf/netperf.go +++ b/pkg/netperf/netperf.go @@ -105,6 +105,9 @@ func ParseResults(stdout *bytes.Buffer) (sample.Sample, error) { if strings.Contains(l[0], "THROUGHPUT_UNITS") { sample.Metric = l[1] } else if strings.Contains(l[0], "THROUGHPUT") { + if len(strings.TrimSpace(l[1])) < 1 { + return sample, fmt.Errorf("Throughput was empty.") + } sample.Throughput, _ = strconv.ParseFloat(strings.Trim(l[1], "\r"), 64) } else if strings.Contains(l[0], "P99_LATENCY") { sample.Latency99ptile, _ = strconv.ParseFloat(strings.Trim(l[1], "\r"), 64) @@ -121,6 +124,9 @@ func ParseResults(stdout *bytes.Buffer) (sample.Sample, error) { if math.IsNaN(sample.Throughput) { return sample, fmt.Errorf("Throughput value is NaN") } + if math.IsNaN(sample.Latency99ptile) { + return sample, fmt.Errorf("Latency value is NaN") + } sample.LossPercent = 100 - (recv / send * 100) return sample, nil }