From 3af6aa2906e9357187ef3b18071fd86854997c93 Mon Sep 17 00:00:00 2001
From: Anton Bronnikov <anton@northernforest.nl>
Date: Sun, 17 Mar 2024 20:50:10 +0200
Subject: [PATCH] feat: add block lag metric

---
 server/handlers.go | 15 +++++++++------
 server/metrics.go  | 14 ++++++++++++++
 state/el_group.go  |  2 +-
 3 files changed, 24 insertions(+), 7 deletions(-)

diff --git a/server/handlers.go b/server/handlers.go
index 5794d8c..91013ba 100644
--- a/server/handlers.go
+++ b/server/handlers.go
@@ -86,13 +86,13 @@ func (s *Server) handleEventPrometheusObserve(_ context.Context, o metric.Observ
 			}
 		}
 
-		b, t := g.TimeSinceHighestBlock()
+		blockGroup, tsBlockGroup := g.TimeSinceHighestBlock()
 
 		// group's highest block
-		o.ObserveInt64(s.metrics.highestBlock, b, metric.WithAttributes(attrs...))
+		o.ObserveInt64(s.metrics.highestBlock, blockGroup, metric.WithAttributes(attrs...))
 
 		// group's time since last block
-		o.ObserveFloat64(s.metrics.timeSinceLastBlock, t.Seconds(), metric.WithAttributes(attrs...))
+		o.ObserveFloat64(s.metrics.timeSinceLastBlock, tsBlockGroup.Seconds(), metric.WithAttributes(attrs...))
 
 		g.IterateEndpointsRO(func(ename string, e *state.ELEndpoint) {
 			if gname != "" {
@@ -109,13 +109,16 @@ func (s *Server) handleEventPrometheusObserve(_ context.Context, o metric.Observ
 				}
 			}
 
-			b, t := e.TimeSinceHighestBlock()
+			blockEndpoint, tsBlockEndpoint := e.TimeSinceHighestBlock()
 
 			// endpoint's highest block
-			o.ObserveInt64(s.metrics.highestBlock, b, metric.WithAttributes(attrs...))
+			o.ObserveInt64(s.metrics.highestBlock, blockEndpoint, metric.WithAttributes(attrs...))
+
+			// endpoint's highest block lag
+			o.ObserveInt64(s.metrics.highestBlockLag, blockGroup-blockEndpoint, metric.WithAttributes(attrs...))
 
 			// endpoint's time since last block
-			o.ObserveFloat64(s.metrics.timeSinceLastBlock, t.Seconds(), metric.WithAttributes(attrs...))
+			o.ObserveFloat64(s.metrics.timeSinceLastBlock, tsBlockEndpoint.Seconds(), metric.WithAttributes(attrs...))
 		})
 	})
 
diff --git a/server/metrics.go b/server/metrics.go
index 0cdd2ec..58b94de 100644
--- a/server/metrics.go
+++ b/server/metrics.go
@@ -11,6 +11,7 @@ import (
 
 const (
 	metricHighestBlock       = "highest_block"
+	metricHighestBlockLag    = "highest_block_lag"
 	metricNewBlockLatency    = "new_block_latency"
 	metricTimeSinceLastBlock = "time_since_last_block"
 )
@@ -18,6 +19,7 @@ const (
 var (
 	metricDescriptions = map[string]string{
 		metricHighestBlock:       "The highest known block",
+		metricHighestBlockLag:    "The distance between endpoint's highest known block and its group's one",
 		metricNewBlockLatency:    "Statistics on how late a node receives blocks compared to the earliest observed ones",
 		metricTimeSinceLastBlock: "Time passed since last block was received",
 	}
@@ -29,6 +31,7 @@ var (
 
 type metrics struct {
 	highestBlock       otelapi.Int64ObservableGauge
+	highestBlockLag    otelapi.Int64ObservableGauge
 	newBlockLatency    otelapi.Float64Histogram
 	timeSinceLastBlock otelapi.Float64Observable
 }
@@ -45,6 +48,17 @@ func (m *metrics) setup(meter otelapi.Meter, observe func(ctx context.Context, o
 	}
 	m.highestBlock = highestBlock
 
+	// highest block lag
+	highestBlockLag, err := meter.Int64ObservableGauge(metricHighestBlockLag,
+		otelapi.WithDescription(metricDescriptions[metricHighestBlockLag]),
+	)
+	if err != nil {
+		return fmt.Errorf("%w: %w: %s",
+			ErrSetupMetricsFailed, err, metricHighestBlock,
+		)
+	}
+	m.highestBlockLag = highestBlockLag
+
 	// new block latency
 	newBlockLatency, err := meter.Float64Histogram(metricNewBlockLatency,
 		metric.WithExplicitBucketBoundaries(.005, .01, .025, .05, .075, .1, .25, .5, .75, 1, 1.5, 3, 6, 12),
diff --git a/state/el_group.go b/state/el_group.go
index b9a26bb..f49504e 100644
--- a/state/el_group.go
+++ b/state/el_group.go
@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	maxHistoryBlocks = 16
+	maxHistoryBlocks = 1024
 )
 
 type ELGroup struct {