prometheus metrics for multiple runners

insolar · Oct 6, 2020 · d4a65f2 · d4a65f2
1 parent 6c2708a
commit d4a65f2
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 31 deletions.
diff --git a/cmd/dummy_test/main.go b/cmd/dummy_test/main.go
@@ -24,10 +24,13 @@ func main() {
 		r := loaderbot.NewRunner(&loaderbot.RunnerConfig{
 			TargetUrl:       target,
 			Name:            fmt.Sprintf("dummy_test_%d", i),
-			SystemMode:      loaderbot.UnboundRPS,
-			Attackers:       270,
+			SystemMode:      loaderbot.BoundRPS,
+			Attackers:       1000,
+			StartRPS:        20000,
+			StepRPS:         5000,
+			StepDurationSec: 3,
 			AttackerTimeout: 25,
-			TestTimeSec:     10,
+			TestTimeSec:     20,
 			SuccessRatio:    0.95,
 		}, &loaderbot.HTTPAttackerExample{}, nil)
 		_, _ = r.Run(context.TODO())

diff --git a/default_http_client.go b/default_http_client.go
@@ -26,7 +26,7 @@ func NewLoggingHTTPClient(debug bool, transportTimeout int) *http.Client {
 	http.DefaultTransport.(*http.Transport).MaxIdleConns = 65535
 	http.DefaultTransport.(*http.Transport).MaxIdleConnsPerHost = 65535
 	http.DefaultTransport.(*http.Transport).DisableCompression = true
-	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = 10 * time.Second
+	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Duration(transportTimeout) * time.Second
 	if debug {
 		transport = &DumpTransport{
 			http.DefaultTransport,

diff --git a/manual_test.go b/manual_test.go
@@ -258,7 +258,7 @@ func TestManualUnboundRPS(t *testing.T) {
 		SystemMode:      UnboundRPS,
 		Attackers:       2,
 		AttackerTimeout: 25,
-		TestTimeSec:     30,
+		TestTimeSec:     300,
 		SuccessRatio:    0.95,
 		Prometheus:      &Prometheus{Enable: true},
 	}, &HTTPAttackerExample{}, nil)

diff --git a/prometheus_metrics.go b/prometheus_metrics.go
@@ -9,53 +9,75 @@ package loaderbot
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
-var (
-	// promGoroutines1 = promauto.NewGauge(prometheus.GaugeOpts{
-	// 	Name: "loaderbot_goroutines_1",
-	// 	Help: "loaderbot_goroutines_1",
-	// })
-	//
-	// promGoroutines2 = promauto.NewGauge(prometheus.GaugeOpts{
-	// 	Name: "loaderbot_goroutines_2",
-	// 	Help: "loaderbot_goroutines_2",
-	// })
+type PromReporter struct {
+	promTickSuccessRatio prometheus.Gauge
+	promTickP50          prometheus.Gauge
+	promTickP95          prometheus.Gauge
+	promTickP99          prometheus.Gauge
+	promTickMax          prometheus.Gauge
+	promRPS              prometheus.Gauge
+}
 
-	promTickSuccessRatio = promauto.NewGauge(prometheus.GaugeOpts{
+func NewPromReporter(label string) *PromReporter {
+	m := &PromReporter{}
+	m.promTickSuccessRatio = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "loaderbot_tick_success_ratio",
 		Help: "Success requests ratio",
+		ConstLabels: prometheus.Labels{
+			"runner_name": label,
+		},
 	})
-	promTickP50 = promauto.NewGauge(prometheus.GaugeOpts{
+	_ = prometheus.Register(m.promTickSuccessRatio)
+	m.promTickP50 = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "loaderbot_tick_p50",
 		Help: "Response time 50 Percentile",
+		ConstLabels: prometheus.Labels{
+			"runner_name": label,
+		},
 	})
-	promTickP95 = promauto.NewGauge(prometheus.GaugeOpts{
+	_ = prometheus.Register(m.promTickP50)
+	m.promTickP95 = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "loaderbot_tick_p95",
 		Help: "Response time 95 Percentile",
+		ConstLabels: prometheus.Labels{
+			"runner_name": label,
+		},
 	})
-	promTickP99 = promauto.NewGauge(prometheus.GaugeOpts{
+	_ = prometheus.Register(m.promTickP95)
+	m.promTickP99 = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "loaderbot_tick_p99",
 		Help: "Response time 99 Percentile",
+		ConstLabels: prometheus.Labels{
+			"runner_name": label,
+		},
 	})
-	promTickMax = promauto.NewGauge(prometheus.GaugeOpts{
+	_ = prometheus.Register(m.promTickP99)
+	m.promTickMax = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "loaderbot_tick_max",
 		Help: "Response time MAX",
+		ConstLabels: prometheus.Labels{
+			"runner_name": label,
+		},
 	})
-	promRPS = promauto.NewGauge(prometheus.GaugeOpts{
+	_ = prometheus.Register(m.promTickMax)
+	m.promRPS = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "loaderbot_tick_rps",
 		Help: "Requests per second rate",
+		ConstLabels: prometheus.Labels{
+			"runner_name": label,
+		},
 	})
-)
-
-type PromReporter struct{}
+	_ = prometheus.Register(m.promRPS)
+	return m
+}
 
 func (m *PromReporter) reportTick(tm *TickMetrics) {
-	promTickP50.Set(float64(tm.Metrics.Latencies.P50.Milliseconds()))
-	promTickP95.Set(float64(tm.Metrics.Latencies.P95.Milliseconds()))
-	promTickP99.Set(float64(tm.Metrics.Latencies.P99.Milliseconds()))
-	promTickMax.Set(float64(tm.Metrics.Latencies.Max.Milliseconds()))
-	promTickSuccessRatio.Set(tm.Metrics.Success)
-	promRPS.Set(tm.Metrics.Rate)
+	m.promTickP50.Set(float64(tm.Metrics.Latencies.P50.Milliseconds()))
+	m.promTickP95.Set(float64(tm.Metrics.Latencies.P95.Milliseconds()))
+	m.promTickP99.Set(float64(tm.Metrics.Latencies.P99.Milliseconds()))
+	m.promTickMax.Set(float64(tm.Metrics.Latencies.Max.Milliseconds()))
+	m.promTickSuccessRatio.Set(tm.Metrics.Success)
+	m.promRPS.Set(tm.Metrics.Rate)
 }
diff --git a/runner.go b/runner.go
@@ -158,6 +158,7 @@ func NewRunner(cfg *RunnerConfig, a Attack, data interface{}) *Runner {
 		r.Report = NewReport(r.Cfg)
 	}
 	if cfg.Prometheus != nil && cfg.Prometheus.Enable {
+		r.PromReporter = NewPromReporter(r.Name)
 		promOnce.Do(func() {
 			go func() {
 				http.Handle("/metrics", promhttp.Handler())