Add more metrics

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go

@@ -9,6 +9,7 @@ import (
 	ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation"
 
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics"
 )
 
 // ActiveUpkeepList is a list to manage active upkeep IDs
@@ -49,9 +50,10 @@ func (al *activeList) Reset(ids ...*big.Int) {
 	for _, id := range ids {
 		al.items[id.String()] = true
 	}
+	prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items)))
 }
 
-// Add adds new entries to the list
+// Add adds new entries to the list. Returns the number of items added
 func (al *activeList) Add(ids ...*big.Int) int {
 	al.lock.Lock()
 	defer al.lock.Unlock()
@@ -63,10 +65,11 @@ func (al *activeList) Add(ids ...*big.Int) int {
 			al.items[key] = true
 		}
 	}
+	prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items)))
 	return count
 }
 
-// Remove removes entries from the list
+// Remove removes entries from the list. Returns the number of items removed
 func (al *activeList) Remove(ids ...*big.Int) int {
 	al.lock.Lock()
 	defer al.lock.Unlock()
@@ -79,6 +82,7 @@ func (al *activeList) Remove(ids ...*big.Int) int {
 			delete(al.items, key)
 		}
 	}
+	prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items)))
 	return count
 }
 

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/provider.go

@@ -24,6 +24,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pg"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
 )
@@ -162,6 +163,7 @@ func (p *logEventProvider) GetLatestPayloads(ctx context.Context) ([]ocr2keepers
 	if err != nil {
 		return nil, fmt.Errorf("%w: %s", ErrHeadNotAvailable, err)
 	}
+	prommetrics.AutomationLogProviderLatestBlock.Set(float64(latest.BlockNumber))
 	start := latest.BlockNumber - p.opts.LookbackBlocks
 	if start <= 0 {
 		start = 1

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go

@@ -306,7 +306,7 @@ func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers.
 	var results, pending []ocr2keepers.UpkeepPayload
 	for _, payload := range r.pending {
 		if allLogsCounter >= MaxProposals {
-			// we have enough proposals, pushed the rest are pushed back to pending
+			// we have enough proposals, the rest are pushed back to pending
 			pending = append(pending, payload)
 			continue
 		}
@@ -322,6 +322,7 @@ func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers.
 	}
 
 	r.pending = pending
+	prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending)))
 
 	r.lggr.Debugf("found %d recoverable payloads", len(results))
 
@@ -676,6 +677,7 @@ func (r *logRecoverer) addPending(payload ocr2keepers.UpkeepPayload) error {
 	if !exist {
 		r.pending = append(pending, payload)
 	}
+	prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending)))
 	return nil
 }
 
@@ -689,6 +691,7 @@ func (r *logRecoverer) removePending(workID string) {
 		}
 	}
 	r.pending = updated
+	prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending)))
 }
 
 // sortPending sorts the pending list by a random order based on the normalized latest block number.

core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics/metrics.go

@@ -20,4 +20,19 @@ var (
 		Name:      "num_recoverer_missed_logs",
 		Help:      "How many valid log triggers were identified as being missed by the recoverer",
 	})
+	AutomationRecovererPendingPayloads = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "num_recoverer_pending_payloads",
+		Help:      "How many log trigger payloads are currently pending in the recoverer",
+	})
+	AutomationActiveUpkeeps = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "num_active_upkeeps",
+		Help:      "How many log trigger upkeeps are currently active",
+	})
+	AutomationLogProviderLatestBlock = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: AutomationLogTriggerNamespace,
+		Name:      "log_provider_latest_block",
+		Help:      "The latest block number the log provider has seen",
+	})
 )