diff --git a/api/accounts/accounts.go b/api/accounts/accounts.go index 27af71fbd..c55cbe916 100644 --- a/api/accounts/accounts.go +++ b/api/accounts/accounts.go @@ -351,20 +351,21 @@ func (a *Accounts) Mount(root *mux.Router, pathPrefix string) { sub.Path("/*"). Methods("POST"). - HandlerFunc(utils.MetricsWrapHandler("accounts_call_batch_code", utils.WrapHandlerFunc(a.handleCallBatchCode))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "accounts_call_batch_code", a.handleCallBatchCode)) sub.Path("/{address}"). Methods(http.MethodGet). - HandlerFunc(utils.MetricsWrapHandler("accounts_get_account", utils.WrapHandlerFunc(a.handleGetAccount))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "accounts_get_account", a.handleGetAccount)) sub.Path("/{address}/code"). Methods(http.MethodGet). - HandlerFunc(utils.MetricsWrapHandler("accounts_get_code", utils.WrapHandlerFunc(a.handleGetCode))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "accounts_get_code", a.handleGetCode)) sub.Path("/{address}/storage/{key}"). Methods("GET"). - HandlerFunc(utils.MetricsWrapHandler("accounts_get_storage", utils.WrapHandlerFunc(a.handleGetStorage))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "accounts_get_storage", a.handleGetStorage)) + // These two methods are currently deprecated sub.Path(""). Methods("POST"). - HandlerFunc(utils.MetricsWrapHandler("accounts_api_call_contract", utils.WrapHandlerFunc(a.handleCallContract))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "accounts_api_call_contract", a.handleCallContract)) sub.Path("/{address}"). Methods("POST"). - HandlerFunc(utils.MetricsWrapHandler("accounts_call_contract_address", utils.WrapHandlerFunc(a.handleCallContract))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "accounts_call_contract_address", a.handleCallContract)) } diff --git a/api/blocks/blocks.go b/api/blocks/blocks.go index a4422a7a0..48f415b97 100644 --- a/api/blocks/blocks.go +++ b/api/blocks/blocks.go @@ -138,5 +138,5 @@ func (b *Blocks) Mount(root *mux.Router, pathPrefix string) { sub := root.PathPrefix(pathPrefix).Subrouter() sub.Path("/{revision}"). Methods("GET"). - HandlerFunc(utils.MetricsWrapHandler("blocks_get_block", utils.WrapHandlerFunc(b.handleGetBlock))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "blocks_get_block", b.handleGetBlock)) } diff --git a/api/debug/debug.go b/api/debug/debug.go index 667417c6e..efa4e782a 100644 --- a/api/debug/debug.go +++ b/api/debug/debug.go @@ -460,11 +460,11 @@ func (d *Debug) Mount(root *mux.Router, pathPrefix string) { sub.Path("/tracers"). Methods(http.MethodPost). - HandlerFunc(utils.MetricsWrapHandler("debug_trace_clause", utils.WrapHandlerFunc(d.handleTraceClause))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "debug_trace_clause", d.handleTraceClause)) sub.Path("/tracers/call"). Methods(http.MethodPost). - HandlerFunc(utils.MetricsWrapHandler("debug_trace_call", utils.WrapHandlerFunc(d.handleTraceCall))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "debug_trace_call", d.handleTraceCall)) sub.Path("/storage-range"). Methods(http.MethodPost). - HandlerFunc(utils.MetricsWrapHandler("debug_debug_storage", utils.WrapHandlerFunc(d.handleDebugStorage))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "debug_debug_storage", d.handleDebugStorage)) } diff --git a/api/events/events.go b/api/events/events.go index 10c89cd7d..f68675299 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -63,5 +63,5 @@ func (e *Events) Mount(root *mux.Router, pathPrefix string) { sub.Path(""). Methods("POST"). - HandlerFunc(utils.MetricsWrapHandler("events_filter", utils.WrapHandlerFunc(e.handleFilter))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "events_filter", e.handleFilter)) } diff --git a/api/node/node.go b/api/node/node.go index 96da6682c..813156b32 100644 --- a/api/node/node.go +++ b/api/node/node.go @@ -35,5 +35,5 @@ func (n *Node) Mount(root *mux.Router, pathPrefix string) { sub.Path("/network/peers"). Methods("Get"). - HandlerFunc(utils.MetricsWrapHandler("node_network", utils.WrapHandlerFunc(n.handleNetwork))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "node_network", n.handleNetwork)) } diff --git a/api/subscriptions/subscriptions.go b/api/subscriptions/subscriptions.go index 92f883fcb..3d8a9d57d 100644 --- a/api/subscriptions/subscriptions.go +++ b/api/subscriptions/subscriptions.go @@ -383,8 +383,8 @@ func (s *Subscriptions) Mount(root *mux.Router, pathPrefix string) { sub.Path("/txpool"). Methods("Get"). - HandlerFunc(utils.MetricsWrapHandler("subscriptions_pending_transactions", utils.WrapHandlerFunc(s.handlePendingTransactions))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "subscriptions_pending_transactions", s.handlePendingTransactions)) sub.Path("/{subject}"). Methods("Get"). - HandlerFunc(utils.MetricsWrapHandler("subscriptions_subject", utils.WrapHandlerFunc(s.handleSubject))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "subscriptions_subject", s.handleSubject)) } diff --git a/api/transactions/transactions.go b/api/transactions/transactions.go index 71abfc5e7..5f0a3d6bd 100644 --- a/api/transactions/transactions.go +++ b/api/transactions/transactions.go @@ -217,11 +217,11 @@ func (t *Transactions) Mount(root *mux.Router, pathPrefix string) { sub.Path(""). Methods("POST"). - HandlerFunc(utils.MetricsWrapHandler("transactions_send_transaction", utils.WrapHandlerFunc(t.handleSendTransaction))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "transactions_send_transaction", t.handleSendTransaction)) sub.Path("/{id}"). Methods("GET"). - HandlerFunc(utils.MetricsWrapHandler("transactions_get_transaction_by_id", utils.WrapHandlerFunc(t.handleGetTransactionByID))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "transactions_get_transaction_by_id", t.handleGetTransactionByID)) sub.Path("/{id}/receipt"). Methods("GET"). - HandlerFunc(utils.MetricsWrapHandler("transactions_get_transaction_by_receipt", utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "transactions_get_transaction_by_receipt", t.handleGetTransactionReceiptByID)) } diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index d5c5d07d7..b343a0b0c 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -69,5 +69,5 @@ func (t *Transfers) Mount(root *mux.Router, pathPrefix string) { sub.Path(""). Methods("POST"). - HandlerFunc(utils.MetricsWrapHandler("transfers_transfer_logs", utils.WrapHandlerFunc(t.handleFilterTransferLogs))) + HandlerFunc(utils.MetricsWrapHandler(pathPrefix, "transfers_transfer_logs", t.handleFilterTransferLogs)) } diff --git a/api/utils/http.go b/api/utils/http.go index a0fd7d4db..0d8cd8003 100644 --- a/api/utils/http.go +++ b/api/utils/http.go @@ -7,11 +7,12 @@ package utils import ( "encoding/json" + "github.com/vechain/thor/v2/telemetry" "io" "net/http" + "strconv" + "strings" "time" - - "github.com/vechain/thor/v2/telemetry" ) type httpError struct { @@ -71,14 +72,32 @@ func WrapHandlerFunc(f HandlerFunc) http.HandlerFunc { } // MetricsWrapHandler wraps a given handler and adds metrics to it -func MetricsWrapHandler(endpoint string, f http.HandlerFunc) http.HandlerFunc { - counter := telemetry.Counter("api_" + endpoint + "_count_requests") - duration := telemetry.HistogramWithHTTPBuckets("api_" + endpoint + "_duration_ms") +func MetricsWrapHandler(pathPrefix, endpoint string, f HandlerFunc) http.HandlerFunc { + fixedPath := strings.ReplaceAll(pathPrefix, "/", "_") // ensure no unexpected slashes + httpReqCounter := telemetry.CounterVec(fixedPath+"_request_count", []string{"path", "code", "method"}) + httpReqDuration := telemetry.HistogramVecWithHTTPBuckets(fixedPath+"_duration_ms", []string{"path", "code", "method"}) + return func(w http.ResponseWriter, r *http.Request) { now := time.Now() - f(w, r) - counter.Add(1) - duration.Observe(time.Since(now).Milliseconds()) + err := f(w, r) + + method := r.Method + status := http.StatusOK + if err != nil { + if he, ok := err.(*httpError); ok { + if he.cause != nil { + http.Error(w, he.cause.Error(), he.status) + } else { + w.WriteHeader(he.status) + } + status = he.status + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + status = http.StatusInternalServerError + } + } + httpReqCounter.AddWithLabel(1, map[string]string{"path": endpoint, "code": strconv.Itoa(status), "method": method}) + httpReqDuration.ObserveWithLabels(time.Since(now).Milliseconds(), map[string]string{"path": endpoint, "code": strconv.Itoa(status), "method": method}) } } diff --git a/telemetry/noop.go b/telemetry/noop.go index 0c258d7c4..57dd5f203 100644 --- a/telemetry/noop.go +++ b/telemetry/noop.go @@ -5,18 +5,36 @@ import "net/http" // noopTelemetry implements a no operations telemetry service type noopTelemetry struct{} +func (n *noopTelemetry) GetOrCreateHistogramVecMeter(name string, labels []string, buckets []int64) HistogramVecMeter { + return &noopTelemetry{} +} + func defaultNoopTelemetry() Telemetry { return &noopTelemetry{} } func (n *noopTelemetry) GetOrCreateHistogramMeter(string, []int64) HistogramMeter { return &noopMetric } func (n *noopTelemetry) GetOrCreateCountMeter(string) CountMeter { return &noopMetric } +func (n *noopTelemetry) GetOrCreateCountVecMeter(_ string, _ []string) CountVecMeter { + return &noopMetric +} + +func (n *noopTelemetry) GetOrCreateGaugeVecMeter(name string, labels []string) GaugeVecMeter { + return &noopMetric +} + func (n *noopTelemetry) GetOrCreateHandler() http.Handler { return nil } var noopMetric = noopMeters{} type noopMeters struct{} +func (n noopMeters) GaugeWithLabel(i int64, m map[string]string) {} + +func (n noopMeters) AddWithLabel(i int64, m map[string]string) {} + func (n noopMeters) Add(int64) {} func (n noopMeters) Observe(int64) {} + +func (n *noopTelemetry) ObserveWithLabels(i int64, m map[string]string) {} diff --git a/telemetry/noop_test.go b/telemetry/noop_test.go index ced569ea9..f0b830de7 100644 --- a/telemetry/noop_test.go +++ b/telemetry/noop_test.go @@ -4,6 +4,7 @@ import ( "math/rand" "net/http" "net/http/httptest" + "strconv" "testing" "github.com/stretchr/testify/require" @@ -33,6 +34,12 @@ func TestNoopTelemetry(t *testing.T) { histTotal += i } + countVect := CounterVec("countVec1", []string{"zeroOrOne"}) + for i := 0; i < rand.Intn(100)+1; i++ { + color := i % 2 + countVect.AddWithLabel(int64(i), map[string]string{"color": strconv.Itoa(color)}) + } + // Make a request to the metrics endpoint resp, err := http.Get(server.URL + "/metrics") if err != nil { diff --git a/telemetry/prometheus.go b/telemetry/prometheus.go index 7d9fbc492..aab4ab528 100644 --- a/telemetry/prometheus.go +++ b/telemetry/prometheus.go @@ -22,14 +22,20 @@ func InitializePrometheusTelemetry() { } type prometheusTelemetry struct { - counters sync.Map - histograms sync.Map + counters sync.Map + counterVecs sync.Map + histograms sync.Map + histogramVecs sync.Map + gaugeVecs sync.Map } func newPrometheusTelemetry() Telemetry { return &prometheusTelemetry{ - counters: sync.Map{}, - histograms: sync.Map{}, + counters: sync.Map{}, + counterVecs: sync.Map{}, + histograms: sync.Map{}, + histogramVecs: sync.Map{}, + gaugeVecs: sync.Map{}, } } @@ -45,6 +51,18 @@ func (o *prometheusTelemetry) GetOrCreateCountMeter(name string) CountMeter { return meter } +func (o *prometheusTelemetry) GetOrCreateCountVecMeter(name string, labels []string) CountVecMeter { + var meter CountVecMeter + mapItem, ok := o.counterVecs.Load(name) + if !ok { + meter = o.newCountVecMeter(name, labels) + o.counterVecs.Store(name, meter) + } else { + meter = mapItem.(CountVecMeter) + } + return meter +} + func (o *prometheusTelemetry) GetOrCreateHandler() http.Handler { return promhttp.Handler() } @@ -61,13 +79,37 @@ func (o *prometheusTelemetry) GetOrCreateHistogramMeter(name string, buckets []i return meter } +func (o *prometheusTelemetry) GetOrCreateHistogramVecMeter(name string, labels []string, buckets []int64) HistogramVecMeter { + var meter HistogramVecMeter + mapItem, ok := o.histogramVecs.Load(name) + if !ok { + meter = o.newHistogramVecMeter(name, labels, buckets) + o.histogramVecs.Store(name, meter) + } else { + meter = mapItem.(HistogramVecMeter) + } + return meter +} + +func (o *prometheusTelemetry) GetOrCreateGaugeVecMeter(name string, labels []string) GaugeVecMeter { + var meter GaugeVecMeter + mapItem, ok := o.gaugeVecs.Load(name) + if !ok { + meter = o.newGaugeVecMeter(name, labels) + o.gaugeVecs.Store(name, meter) + } else { + meter = mapItem.(GaugeVecMeter) + } + return meter +} + func (o *prometheusTelemetry) newHistogramMeter(name string, buckets []int64) HistogramMeter { var floatBuckets []float64 for _, bucket := range buckets { floatBuckets = append(floatBuckets, float64(bucket)) } - histogram := prometheus.NewHistogram( + meter := prometheus.NewHistogram( prometheus.HistogramOpts{ Namespace: namespace, Name: name, @@ -75,13 +117,13 @@ func (o *prometheusTelemetry) newHistogramMeter(name string, buckets []int64) Hi }, ) - err := prometheus.Register(histogram) + err := prometheus.Register(meter) if err != nil { log.Warn("unable to register metric", "err", err) } return &promHistogramMeter{ - histogram: histogram, + histogram: meter, } } @@ -93,20 +135,89 @@ func (c *promHistogramMeter) Observe(i int64) { c.histogram.Observe(float64(i)) } +func (o *prometheusTelemetry) newHistogramVecMeter(name string, labels []string, buckets []int64) HistogramVecMeter { + var floatBuckets []float64 + for _, bucket := range buckets { + floatBuckets = append(floatBuckets, float64(bucket)) + } + + meter := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Name: name, + Buckets: floatBuckets, + }, + labels, + ) + + err := prometheus.Register(meter) + if err != nil { + log.Warn("unable to register metric", "err", err) + } + + return &promHistogramVecMeter{ + histogram: meter, + } +} + +type promHistogramVecMeter struct { + histogram *prometheus.HistogramVec +} + +func (c *promHistogramVecMeter) ObserveWithLabels(i int64, labels map[string]string) { + c.histogram.With(labels).Observe(float64(i)) +} + func (o *prometheusTelemetry) newCountMeter(name string) CountMeter { - counter := prometheus.NewCounter( + meter := prometheus.NewCounter( prometheus.CounterOpts{ Namespace: namespace, Name: name, }, ) - err := prometheus.Register(counter) + err := prometheus.Register(meter) if err != nil { log.Warn("unable to register metric", "err", err) } return &promCountMeter{ - counter: counter, + counter: meter, + } +} + +func (o *prometheusTelemetry) newCountVecMeter(name string, labels []string) CountVecMeter { + meter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: name, + }, + labels, + ) + + err := prometheus.Register(meter) + if err != nil { + log.Warn("unable to register metric", "err", err) + } + return &promCountVecMeter{ + counter: meter, + } +} + +func (o *prometheusTelemetry) newGaugeVecMeter(name string, labels []string) GaugeVecMeter { + meter := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: name, + }, + labels, + ) + + err := prometheus.Register(meter) + if err != nil { + log.Warn("unable to register metric", "err", err) + } + return &promGaugeVecMeter{ + gauge: meter, } } @@ -117,3 +228,19 @@ type promCountMeter struct { func (c *promCountMeter) Add(i int64) { c.counter.Add(float64(i)) } + +type promCountVecMeter struct { + counter *prometheus.CounterVec +} + +func (c *promCountVecMeter) AddWithLabel(i int64, labels map[string]string) { + c.counter.With(labels).Add(float64(i)) +} + +type promGaugeVecMeter struct { + gauge *prometheus.GaugeVec +} + +func (c *promGaugeVecMeter) GaugeWithLabel(i int64, labels map[string]string) { + c.gauge.With(labels).Add(float64(i)) +} diff --git a/telemetry/prometheus_test.go b/telemetry/prometheus_test.go index 3d3ffe803..3b971de4b 100644 --- a/telemetry/prometheus_test.go +++ b/telemetry/prometheus_test.go @@ -4,6 +4,7 @@ import ( "math/rand" "net/http" "net/http/httptest" + "strconv" "testing" "github.com/prometheus/common/expfmt" @@ -35,6 +36,15 @@ func TestOtelPromTelemetry(t *testing.T) { histTotal += i } + countVect := CounterVec("countVec1", []string{"zeroOrOne"}) + totalCountVec := 0 + randCountVec := rand.Intn(100) + 1 + for i := 0; i < randCountVec; i++ { + color := i % 2 + countVect.AddWithLabel(int64(i), map[string]string{"zeroOrOne": strconv.Itoa(color)}) + totalCountVec += i + } + // Make a request to the metrics endpoint resp, err := http.Get(server.URL + "/metrics") if err != nil { @@ -50,4 +60,8 @@ func TestOtelPromTelemetry(t *testing.T) { require.Equal(t, metrics["node_telemetry_count1"].GetMetric()[0].GetCounter().GetValue(), float64(1)) require.Equal(t, metrics["node_telemetry_count2"].GetMetric()[0].GetCounter().GetValue(), float64(randCount2)) require.Equal(t, metrics["node_telemetry_hist1"].GetMetric()[0].GetHistogram().GetSampleSum(), float64(histTotal)) + + sumCountVec := metrics["node_telemetry_countVec1"].GetMetric()[0].GetCounter().GetValue() + + metrics["node_telemetry_countVec1"].GetMetric()[1].GetCounter().GetValue() + require.Equal(t, sumCountVec, float64(totalCountVec)) } diff --git a/telemetry/telemetry.go b/telemetry/telemetry.go index 4e1275d3e..c1e16c207 100644 --- a/telemetry/telemetry.go +++ b/telemetry/telemetry.go @@ -9,7 +9,10 @@ var telemetry = defaultNoopTelemetry() // defaults to a Noop implementation of t // Telemetry defines the interface for telemetry service implementations type Telemetry interface { GetOrCreateCountMeter(name string) CountMeter + GetOrCreateCountVecMeter(name string, labels []string) CountVecMeter + GetOrCreateGaugeVecMeter(name string, labels []string) GaugeVecMeter GetOrCreateHistogramMeter(name string, buckets []int64) HistogramMeter + GetOrCreateHistogramVecMeter(name string, labels []string, buckets []int64) HistogramVecMeter GetOrCreateHandler() http.Handler } @@ -31,6 +34,18 @@ func HistogramWithHTTPBuckets(name string) HistogramMeter { return telemetry.GetOrCreateHistogramMeter(name, defaultHTTPBuckets) } +// HistogramVecMeter //todo +type HistogramVecMeter interface { + ObserveWithLabels(int64, map[string]string) +} + +func HistogramVec(name string, labels []string) HistogramVecMeter { + return telemetry.GetOrCreateHistogramVecMeter(name, labels, nil) +} +func HistogramVecWithHTTPBuckets(name string, labels []string) HistogramVecMeter { + return telemetry.GetOrCreateHistogramVecMeter(name, labels, defaultHTTPBuckets) +} + var defaultHTTPBuckets = []int64{0, 150, 300, 450, 600, 900, 1200, 1500, 3000} // CountMeter is a cumulative metric that represents a single monotonically increasing counter @@ -40,3 +55,22 @@ type CountMeter interface { } func Counter(name string) CountMeter { return telemetry.GetOrCreateCountMeter(name) } + +// CountVecMeter is a cumulative metric that represents a single monotonically increasing counter +// whose value can only increase or be reset to zero on restart with a vector of values. +type CountVecMeter interface { + AddWithLabel(int64, map[string]string) +} + +func CounterVec(name string, labels []string) CountVecMeter { + return telemetry.GetOrCreateCountVecMeter(name, labels) +} + +// GaugeVecMeter ... +type GaugeVecMeter interface { + GaugeWithLabel(int64, map[string]string) +} + +func GaugeVec(name string, labels []string) GaugeVecMeter { + return telemetry.GetOrCreateGaugeVecMeter(name, labels) +}