From b3bd8adc1c5ecf39b9588d30d2076fb98dd95ea8 Mon Sep 17 00:00:00 2001 From: Sam Davies Date: Wed, 13 Sep 2023 09:54:27 -0400 Subject: [PATCH] Prom metrics on missing/error count for mercury querying price feed --- core/services/relay/evm/mercury/types/types.go | 17 +++++++++++++++++ .../relay/evm/mercury/v2/data_source.go | 7 +++++++ .../relay/evm/mercury/v3/data_source.go | 7 +++++++ docs/CHANGELOG.md | 7 +++++++ 4 files changed, 38 insertions(+) diff --git a/core/services/relay/evm/mercury/types/types.go b/core/services/relay/evm/mercury/types/types.go index 4c606ed9eae..ca266ca8ccd 100644 --- a/core/services/relay/evm/mercury/types/types.go +++ b/core/services/relay/evm/mercury/types/types.go @@ -4,6 +4,8 @@ import ( "context" "math/big" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" @@ -24,3 +26,18 @@ type DataSourceORM interface { type ReportCodec interface { BenchmarkPriceFromReport(report ocrtypes.Report) (*big.Int, error) } + +var ( + PriceFeedMissingCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "mercury_price_feed_missing", + Help: "Running count of times mercury tried to query a price feed for billing from mercury server, but it was missing", + }, + []string{"queriedFeedID"}, + ) + PriceFeedErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "mercury_price_feed_errors", + Help: "Running count of times mercury tried to query a price feed for billing from mercury server, but got an error", + }, + []string{"queriedFeedID"}, + ) +) diff --git a/core/services/relay/evm/mercury/v2/data_source.go b/core/services/relay/evm/mercury/v2/data_source.go index ecf0fd60a82..25fe279dd43 100644 --- a/core/services/relay/evm/mercury/v2/data_source.go +++ b/core/services/relay/evm/mercury/v2/data_source.go @@ -18,6 +18,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" + mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec" "github.com/smartcontractkit/chainlink/v2/core/utils" @@ -117,8 +118,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam defer wg.Done() obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID) if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil { + mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc() ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", relaymercuryv2.MissingPrice), "linkFeedID", ds.linkFeedID) obs.LinkPrice.Val = relaymercuryv2.MissingPrice + } else if obs.LinkPrice.Err != nil { + mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc() } }() } @@ -131,8 +135,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam defer wg.Done() obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID) if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil { + mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc() ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", relaymercuryv2.MissingPrice), "nativeFeedID", ds.nativeFeedID) obs.NativePrice.Val = relaymercuryv2.MissingPrice + } else if obs.NativePrice.Err != nil { + mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc() } }() } diff --git a/core/services/relay/evm/mercury/v3/data_source.go b/core/services/relay/evm/mercury/v3/data_source.go index 7c12ffe8237..d325f70a002 100644 --- a/core/services/relay/evm/mercury/v3/data_source.go +++ b/core/services/relay/evm/mercury/v3/data_source.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" + mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" "github.com/smartcontractkit/chainlink/v2/core/utils" @@ -120,8 +121,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam defer wg.Done() obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID) if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil { + mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc() ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", relaymercuryv3.MissingPrice), "linkFeedID", ds.linkFeedID) obs.LinkPrice.Val = relaymercuryv3.MissingPrice + } else if obs.LinkPrice.Err != nil { + mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc() } }() } @@ -134,8 +138,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam defer wg.Done() obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID) if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil { + mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc() ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", relaymercuryv3.MissingPrice), "nativeFeedID", ds.nativeFeedID) obs.NativePrice.Val = relaymercuryv3.MissingPrice + } else if obs.NativePrice.Err != nil { + mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc() } }() } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f908915a4e3..d593f30b923 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fixed a bug that was preventing job runs to be displayed when the job `chainID` was disabled. - `chainlink txs evm create` returns a transaction hash for the attempted transaction in the CLI. Previously only the sender, receipient and `unstarted` state were returned. +### Added + +- New prometheus metrics for mercury: + - `mercury_price_feed_missing` + - `mercury_price_feed_errors` + Nops may wish to add alerting on these. + ## 2.5.0 - 2023-09-13