feat: add block lag metric

server/handlers.go

@@ -86,13 +86,13 @@ func (s *Server) handleEventPrometheusObserve(_ context.Context, o metric.Observ
 			}
 		}
 
-		b, t := g.TimeSinceHighestBlock()
+		blockGroup, tsBlockGroup := g.TimeSinceHighestBlock()
 
 		// group's highest block
-		o.ObserveInt64(s.metrics.highestBlock, b, metric.WithAttributes(attrs...))
+		o.ObserveInt64(s.metrics.highestBlock, blockGroup, metric.WithAttributes(attrs...))
 
 		// group's time since last block
-		o.ObserveFloat64(s.metrics.timeSinceLastBlock, t.Seconds(), metric.WithAttributes(attrs...))
+		o.ObserveFloat64(s.metrics.timeSinceLastBlock, tsBlockGroup.Seconds(), metric.WithAttributes(attrs...))
 
 		g.IterateEndpointsRO(func(ename string, e *state.ELEndpoint) {
 			if gname != "" {
@@ -109,13 +109,16 @@ func (s *Server) handleEventPrometheusObserve(_ context.Context, o metric.Observ
 				}
 			}
 
-			b, t := e.TimeSinceHighestBlock()
+			blockEndpoint, tsBlockEndpoint := e.TimeSinceHighestBlock()
 
 			// endpoint's highest block
-			o.ObserveInt64(s.metrics.highestBlock, b, metric.WithAttributes(attrs...))
+			o.ObserveInt64(s.metrics.highestBlock, blockEndpoint, metric.WithAttributes(attrs...))
+
+			// endpoint's highest block lag
+			o.ObserveInt64(s.metrics.highestBlockLag, blockGroup-blockEndpoint, metric.WithAttributes(attrs...))
 
 			// endpoint's time since last block
-			o.ObserveFloat64(s.metrics.timeSinceLastBlock, t.Seconds(), metric.WithAttributes(attrs...))
+			o.ObserveFloat64(s.metrics.timeSinceLastBlock, tsBlockEndpoint.Seconds(), metric.WithAttributes(attrs...))
 		})
 	})
 

server/metrics.go

@@ -11,13 +11,15 @@ import (
 
 const (
 	metricHighestBlock       = "highest_block"
+	metricHighestBlockLag    = "highest_block_lag"
 	metricNewBlockLatency    = "new_block_latency"
 	metricTimeSinceLastBlock = "time_since_last_block"
 )
 
 var (
 	metricDescriptions = map[string]string{
 		metricHighestBlock:       "The highest known block",
+		metricHighestBlockLag:    "The distance between endpoint's highest known block and its group's one",
 		metricNewBlockLatency:    "Statistics on how late a node receives blocks compared to the earliest observed ones",
 		metricTimeSinceLastBlock: "Time passed since last block was received",
 	}
@@ -29,6 +31,7 @@ var (
 
 type metrics struct {
 	highestBlock       otelapi.Int64ObservableGauge
+	highestBlockLag    otelapi.Int64ObservableGauge
 	newBlockLatency    otelapi.Float64Histogram
 	timeSinceLastBlock otelapi.Float64Observable
 }
@@ -45,6 +48,17 @@ func (m *metrics) setup(meter otelapi.Meter, observe func(ctx context.Context, o
 	}
 	m.highestBlock = highestBlock
 
+	// highest block lag
+	highestBlockLag, err := meter.Int64ObservableGauge(metricHighestBlockLag,
+		otelapi.WithDescription(metricDescriptions[metricHighestBlockLag]),
+	)
+	if err != nil {
+		return fmt.Errorf("%w: %w: %s",
+			ErrSetupMetricsFailed, err, metricHighestBlock,
+		)
+	}
+	m.highestBlockLag = highestBlockLag
+
 	// new block latency
 	newBlockLatency, err := meter.Float64Histogram(metricNewBlockLatency,
 		metric.WithExplicitBucketBoundaries(.005, .01, .025, .05, .075, .1, .25, .5, .75, 1, 1.5, 3, 6, 12),

state/el_group.go

@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	maxHistoryBlocks = 16
+	maxHistoryBlocks = 1024
 )
 
 type ELGroup struct {