diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go index 55c01939cb8..27c13f079b2 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/active_list.go @@ -9,6 +9,7 @@ import ( ocr2keepers "github.com/smartcontractkit/chainlink-common/pkg/types/automation" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics" ) // ActiveUpkeepList is a list to manage active upkeep IDs @@ -49,9 +50,10 @@ func (al *activeList) Reset(ids ...*big.Int) { for _, id := range ids { al.items[id.String()] = true } + prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items))) } -// Add adds new entries to the list +// Add adds new entries to the list. Returns the number of items added func (al *activeList) Add(ids ...*big.Int) int { al.lock.Lock() defer al.lock.Unlock() @@ -63,10 +65,11 @@ func (al *activeList) Add(ids ...*big.Int) int { al.items[key] = true } } + prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items))) return count } -// Remove removes entries from the list +// Remove removes entries from the list. Returns the number of items removed func (al *activeList) Remove(ids ...*big.Int) int { al.lock.Lock() defer al.lock.Unlock() @@ -79,6 +82,7 @@ func (al *activeList) Remove(ids ...*big.Int) int { delete(al.items, key) } } + prommetrics.AutomationActiveUpkeeps.Set(float64(len(al.items))) return count } diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/provider.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/provider.go index d1360faaf6d..e06593a9109 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/provider.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/provider.go @@ -24,6 +24,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics" "github.com/smartcontractkit/chainlink/v2/core/services/pg" "github.com/smartcontractkit/chainlink/v2/core/utils" ) @@ -162,6 +163,7 @@ func (p *logEventProvider) GetLatestPayloads(ctx context.Context) ([]ocr2keepers if err != nil { return nil, fmt.Errorf("%w: %s", ErrHeadNotAvailable, err) } + prommetrics.AutomationLogProviderLatestBlock.Set(float64(latest.BlockNumber)) start := latest.BlockNumber - p.opts.LookbackBlocks if start <= 0 { start = 1 diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go index 06c9b63ebb2..9cdbdd0ac86 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/logprovider/recoverer.go @@ -306,7 +306,7 @@ func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers. var results, pending []ocr2keepers.UpkeepPayload for _, payload := range r.pending { if allLogsCounter >= MaxProposals { - // we have enough proposals, pushed the rest are pushed back to pending + // we have enough proposals, the rest are pushed back to pending pending = append(pending, payload) continue } @@ -322,6 +322,7 @@ func (r *logRecoverer) GetRecoveryProposals(ctx context.Context) ([]ocr2keepers. } r.pending = pending + prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending))) r.lggr.Debugf("found %d recoverable payloads", len(results)) @@ -676,6 +677,7 @@ func (r *logRecoverer) addPending(payload ocr2keepers.UpkeepPayload) error { if !exist { r.pending = append(pending, payload) } + prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending))) return nil } @@ -689,6 +691,7 @@ func (r *logRecoverer) removePending(workID string) { } } r.pending = updated + prommetrics.AutomationRecovererPendingPayloads.Set(float64(len(r.pending))) } // sortPending sorts the pending list by a random order based on the normalized latest block number. diff --git a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics/metrics.go b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics/metrics.go index 9de47941941..cebbac59884 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics/metrics.go +++ b/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/prommetrics/metrics.go @@ -20,4 +20,19 @@ var ( Name: "num_recoverer_missed_logs", Help: "How many valid log triggers were identified as being missed by the recoverer", }) + AutomationRecovererPendingPayloads = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: AutomationLogTriggerNamespace, + Name: "num_recoverer_pending_payloads", + Help: "How many log trigger payloads are currently pending in the recoverer", + }) + AutomationActiveUpkeeps = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: AutomationLogTriggerNamespace, + Name: "num_active_upkeeps", + Help: "How many log trigger upkeeps are currently active", + }) + AutomationLogProviderLatestBlock = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: AutomationLogTriggerNamespace, + Name: "log_provider_latest_block", + Help: "The latest block number the log provider has seen", + }) )