Adding Metrics to the node

api/accounts/accounts.go

@@ -352,10 +352,23 @@ func (a *Accounts) handleBatchCallData(batchCallData *BatchCallData) (txCtx *xen
 func (a *Accounts) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/*").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(a.handleCallBatchCode))
-	sub.Path("/{address}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetAccount))
-	sub.Path("/{address}/code").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetCode))
-	sub.Path("/{address}/storage/{key}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(a.handleGetStorage))
-	sub.Path("").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract))
-	sub.Path("/{address}").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(a.handleCallContract))
+	sub.Path("/*").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "accounts_call_batch_code", a.handleCallBatchCode))
+	sub.Path("/{address}").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "accounts_get_account", a.handleGetAccount))
+	sub.Path("/{address}/code").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "accounts_get_code", a.handleGetCode))
+	sub.Path("/{address}/storage/{key}").
+		Methods("GET").
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "accounts_get_storage", a.handleGetStorage))
+	// These two methods are currently deprecated
+	sub.Path("").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "accounts_api_call_contract", a.handleCallContract))
+	sub.Path("/{address}").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "accounts_call_contract_address", a.handleCallContract))
 }

api/api.go

@@ -56,7 +56,7 @@ func New(
 
 	router := mux.NewRouter()
 
-	// to serve api docs
+	// to serve stoplight, swagger and api docs
 	router.PathPrefix("/doc").Handler(
 		http.StripPrefix("/doc/", http.FileServer(http.FS(doc.FS))),
 	)

api/blocks/blocks.go

@@ -93,5 +93,7 @@ func (b *Blocks) isTrunk(blkID thor.Bytes32, blkNum uint32) (bool, error) {
 
 func (b *Blocks) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
-	sub.Path("/{revision}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(b.handleGetBlock))
+	sub.Path("/{revision}").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "blocks_get_block", b.handleGetBlock))
 }

api/debug/debug.go

@@ -422,7 +422,13 @@ func (d *Debug) handleTraceCallOption(opt *TraceCallOption) (*xenv.TransactionCo
 func (d *Debug) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/tracers").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleTraceClause))
-	sub.Path("/tracers/call").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleTraceCall))
-	sub.Path("/storage-range").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(d.handleDebugStorage))
+	sub.Path("/tracers").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "debug_trace_clause", d.handleTraceClause))
+	sub.Path("/tracers/call").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "debug_trace_call", d.handleTraceCall))
+	sub.Path("/storage-range").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "debug_debug_storage", d.handleDebugStorage))
 }

api/events/events.go

@@ -61,5 +61,7 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error {
 func (e *Events) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(e.handleFilter))
+	sub.Path("").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "events_filter", e.handleFilter))
 }

api/node/node.go

@@ -33,5 +33,7 @@ func (n *Node) handleNetwork(w http.ResponseWriter, req *http.Request) error {
 func (n *Node) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/network/peers").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(n.handleNetwork))
+	sub.Path("/network/peers").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "node_network", n.handleNetwork))
 }

api/subscriptions/subscriptions.go

@@ -381,6 +381,10 @@ func (s *Subscriptions) Close() {
 func (s *Subscriptions) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("/txpool").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(s.handlePendingTransactions))
-	sub.Path("/{subject}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(s.handleSubject))
+	sub.Path("/txpool").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "subscriptions_pending_transactions", s.handlePendingTransactions))
+	sub.Path("/{subject}").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "subscriptions_subject", s.handleSubject))
 }

api/transactions/transactions.go

@@ -217,7 +217,13 @@ func (t *Transactions) parseHead(head string) (thor.Bytes32, error) {
 func (t *Transactions) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(t.handleSendTransaction))
-	sub.Path("/{id}").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionByID))
-	sub.Path("/{id}/receipt").Methods(http.MethodGet).HandlerFunc(utils.WrapHandlerFunc(t.handleGetTransactionReceiptByID))
+	sub.Path("").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "transactions_send_transaction", t.handleSendTransaction))
+	sub.Path("/{id}").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "transactions_get_transaction_by_id", t.handleGetTransactionByID))
+	sub.Path("/{id}/receipt").
+		Methods(http.MethodGet).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "transactions_get_transaction_by_receipt", t.handleGetTransactionReceiptByID))
 }

api/transfers/transfers.go

@@ -67,5 +67,7 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re
 func (t *Transfers) Mount(root *mux.Router, pathPrefix string) {
 	sub := root.PathPrefix(pathPrefix).Subrouter()
 
-	sub.Path("").Methods(http.MethodPost).HandlerFunc(utils.WrapHandlerFunc(t.handleFilterTransferLogs))
+	sub.Path("").
+		Methods(http.MethodPost).
+		HandlerFunc(utils.MetricsWrapHandlerFunc(pathPrefix, "transfers_transfer_logs", t.handleFilterTransferLogs))
 }

api/utils/http.go

@@ -9,6 +9,11 @@ import (
 	"encoding/json"
 	"io"
 	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/vechain/thor/v2/metrics"
 )
 
 type httpError struct {
@@ -67,6 +72,38 @@ func WrapHandlerFunc(f HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// MetricsWrapHandlerFunc wraps a given handler and adds metrics to it
+func MetricsWrapHandlerFunc(pathPrefix, endpoint string, f HandlerFunc) http.HandlerFunc {
+	fixedPath := strings.ReplaceAll(pathPrefix, "/", "_") // ensure no unexpected slashes
+	httpReqCounter := metrics.CounterVec(fixedPath+"_request_count", []string{"path", "code", "method"})
+	httpReqDuration := metrics.HistogramVec(
+		fixedPath+"_duration_ms", []string{"path", "code", "method"}, metrics.BucketHTTPReqs,
+	)
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		now := time.Now()
+		err := f(w, r)
+
+		method := r.Method
+		status := http.StatusOK
+		if err != nil {
+			if he, ok := err.(*httpError); ok {
+				if he.cause != nil {
+					http.Error(w, he.cause.Error(), he.status)
+				} else {
+					w.WriteHeader(he.status)
+				}
+				status = he.status
+			} else {
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+				status = http.StatusInternalServerError
+			}
+		}
+		httpReqCounter.AddWithLabel(1, map[string]string{"path": endpoint, "code": strconv.Itoa(status), "method": method})
+		httpReqDuration.ObserveWithLabels(time.Since(now).Milliseconds(), map[string]string{"path": endpoint, "code": strconv.Itoa(status), "method": method})
+	}
+}
+
 // content types
 const (
 	JSONContentType = "application/json; charset=utf-8"

bft/engine.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"sync/atomic"
 
-	lru "github.com/hashicorp/golang-lru"
 	"github.com/pkg/errors"
 	"github.com/vechain/thor/v2/block"
 	"github.com/vechain/thor/v2/builtin"
@@ -18,6 +17,8 @@ import (
 	"github.com/vechain/thor/v2/muxdb"
 	"github.com/vechain/thor/v2/state"
 	"github.com/vechain/thor/v2/thor"
+
+	lru "github.com/hashicorp/golang-lru"
 )
 
 const dataStoreName = "bft.engine"
@@ -131,6 +132,7 @@ func (engine *BFTEngine) CommitBlock(header *block.Header, isPacking bool) error
 				return err
 			}
 			engine.finalized.Store(id)
+			metricBlocksCommitted().Add(1)
 		}
 	}
 

bft/metrics.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package bft
+
+import (
+	"github.com/vechain/thor/v2/metrics"
+)
+
+var (
+	metricBlocksCommitted = metrics.LazyLoadCounter("bft_committed_count")
+)

block/block.go

@@ -11,7 +11,7 @@ import (
 	"sync/atomic"
 
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/vechain/thor/v2/metric"
+	"github.com/vechain/thor/v2/thor"
 	"github.com/vechain/thor/v2/tx"
 )
 
@@ -90,16 +90,16 @@ func (b *Block) DecodeRLP(s *rlp.Stream) error {
 		header: &payload.Header,
 		txs:    payload.Txs,
 	}
-	b.cache.size.Store(metric.StorageSize(rlp.ListSize(size)))
+	b.cache.size.Store(thor.StorageSize(rlp.ListSize(size)))
 	return nil
 }
 
 // Size returns block size in bytes.
-func (b *Block) Size() metric.StorageSize {
+func (b *Block) Size() thor.StorageSize {
 	if cached := b.cache.size.Load(); cached != nil {
-		return cached.(metric.StorageSize)
+		return cached.(thor.StorageSize)
 	}
-	var size metric.StorageSize
+	var size thor.StorageSize
 	rlp.Encode(&size, b)
 	b.cache.size.Store(size)
 	return size

cmd/thor/flags.go

@@ -127,6 +127,16 @@ var (
 		Name:  "disable-pruner",
 		Usage: "disable state pruner to keep all history",
 	}
+	metricsEnabledFlag = cli.BoolFlag{
+		Name:  "enable-metrics",
+		Usage: "enables metrics collection",
+	}
+	metricsAddrFlag = cli.StringFlag{
+		Name:  "metrics-addr",
+		Value: "localhost:2112",
+		Usage: "metrics service listening address",
+	}
+
 	// solo mode only flags
 	onDemandFlag = cli.BoolFlag{
 		Name:  "on-demand",

cmd/thor/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/vechain/thor/v2/cmd/thor/solo"
 	"github.com/vechain/thor/v2/genesis"
 	"github.com/vechain/thor/v2/logdb"
+	"github.com/vechain/thor/v2/metrics"
 	"github.com/vechain/thor/v2/muxdb"
 	"github.com/vechain/thor/v2/state"
 	"github.com/vechain/thor/v2/thor"
@@ -89,6 +90,8 @@ func main() {
 			pprofFlag,
 			verifyLogsFlag,
 			disablePrunerFlag,
+			metricsEnabledFlag,
+			metricsAddrFlag,
 		},
 		Action: defaultAction,
 		Commands: []cli.Command{
@@ -117,6 +120,8 @@ func main() {
 					txPoolLimitFlag,
 					txPoolLimitPerAccountFlag,
 					disablePrunerFlag,
+					metricsEnabledFlag,
+					metricsAddrFlag,
 				},
 				Action: soloAction,
 			},
@@ -145,6 +150,19 @@ func defaultAction(ctx *cli.Context) error {
 	defer func() { log.Info("exited") }()
 
 	initLogger(ctx)
+
+	// enable metrics as soon as possible
+	metricsURL := ""
+	if ctx.Bool(metricsEnabledFlag.Name) {
+		metrics.InitializePrometheusMetrics()
+		url, close, err := startMetricsServer(ctx.String(metricsAddrFlag.Name))
+		if err != nil {
+			return fmt.Errorf("unable to start metrics server - %w", err)
+		}
+		metricsURL = url
+		defer func() { log.Info("stopping metrics server..."); close() }()
+	}
+
 	gene, forkConfig, err := selectGenesis(ctx)
 	if err != nil {
 		return err
@@ -223,7 +241,7 @@ func defaultAction(ctx *cli.Context) error {
 	}
 	defer func() { log.Info("stopping API server..."); srvCloser() }()
 
-	printStartupMessage2(apiURL, p2pCommunicator.Enode())
+	printStartupMessage2(gene, apiURL, p2pCommunicator.Enode(), metricsURL)
 
 	if err := p2pCommunicator.Start(); err != nil {
 		return err
@@ -253,6 +271,18 @@ func soloAction(ctx *cli.Context) error {
 
 	initLogger(ctx)
 
+	// enable metrics as soon as possible
+	metricsURL := ""
+	if ctx.Bool(metricsEnabledFlag.Name) {
+		metrics.InitializePrometheusMetrics()
+		url, close, err := startMetricsServer(ctx.String(metricsAddrFlag.Name))
+		if err != nil {
+			return fmt.Errorf("unable to start metrics server - %w", err)
+		}
+		metricsURL = url
+		defer func() { log.Info("stopping metrics server..."); close() }()
+	}
+
 	var (
 		gene       *genesis.Genesis
 		forkConfig thor.ForkConfig
@@ -346,7 +376,7 @@ func soloAction(ctx *cli.Context) error {
 		return errors.New("block-interval cannot be zero")
 	}
 
-	printSoloStartupMessage(gene, repo, instanceDir, apiURL, forkConfig)
+	printSoloStartupMessage(gene, repo, instanceDir, apiURL, forkConfig, metricsURL)
 
 	optimizer := optimizer.New(mainDB, repo, !ctx.Bool(disablePrunerFlag.Name))
 	defer func() { log.Info("stopping optimizer..."); optimizer.Stop() }()

cmd/thor/node/metrics.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2024 The VeChainThor developers
+
+// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying
+// file LICENSE or <https://www.gnu.org/licenses/lgpl-3.0.html>
+
+package node
+
+import (
+	"github.com/vechain/thor/v2/metrics"
+)
+
+var (
+	metricBlockProcessedCount    = metrics.LazyLoadCounterVec("block_processed_count", []string{"type", "success"})
+	metricBlockProcessedTxs      = metrics.LazyLoadCounterVec("block_processed_tx_count", []string{"type"})
+	metricBlockProcessedGas      = metrics.LazyLoadCounterVec("block_processed_gas_count", []string{"type"})
+	metricBlockProcessedDuration = metrics.LazyLoadHistogram("block_processed_duration_ms", metrics.Bucket10s)
+	metricChainForkCount         = metrics.LazyLoadCounter("chain_fork_count")
+	metricChainForkSize          = metrics.LazyLoadGauge("chain_fork_size")
+)