Prom metrics on missing/error count for mercury querying price feed (#…

…10621)

core/services/relay/evm/mercury/types/types.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"math/big"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types"
 
 	evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
@@ -24,3 +26,18 @@ type DataSourceORM interface {
 type ReportCodec interface {
 	BenchmarkPriceFromReport(report ocrtypes.Report) (*big.Int, error)
 }
+
+var (
+	PriceFeedMissingCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "mercury_price_feed_missing",
+		Help: "Running count of times mercury tried to query a price feed for billing from mercury server, but it was missing",
+	},
+		[]string{"queriedFeedID"},
+	)
+	PriceFeedErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "mercury_price_feed_errors",
+		Help: "Running count of times mercury tried to query a price feed for billing from mercury server, but got an error",
+	},
+		[]string{"queriedFeedID"},
+	)
+)

core/services/relay/evm/mercury/v2/data_source.go

@@ -18,6 +18,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pipeline"
 	"github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types"
+	mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types"
 	mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -117,8 +118,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam
 			defer wg.Done()
 			obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID)
 			if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil {
+				mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc()
 				ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", relaymercuryv2.MissingPrice), "linkFeedID", ds.linkFeedID)
 				obs.LinkPrice.Val = relaymercuryv2.MissingPrice
+			} else if obs.LinkPrice.Err != nil {
+				mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc()
 			}
 		}()
 	}
@@ -131,8 +135,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam
 			defer wg.Done()
 			obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID)
 			if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil {
+				mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc()
 				ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", relaymercuryv2.MissingPrice), "nativeFeedID", ds.nativeFeedID)
 				obs.NativePrice.Val = relaymercuryv2.MissingPrice
+			} else if obs.NativePrice.Err != nil {
+				mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc()
 			}
 		}()
 	}

core/services/relay/evm/mercury/v3/data_source.go

@@ -19,6 +19,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon"
 	"github.com/smartcontractkit/chainlink/v2/core/services/pipeline"
 	"github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types"
+	mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types"
 	mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils"
 	"github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec"
 	"github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -120,8 +121,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam
 			defer wg.Done()
 			obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID)
 			if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil {
+				mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc()
 				ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", relaymercuryv3.MissingPrice), "linkFeedID", ds.linkFeedID)
 				obs.LinkPrice.Val = relaymercuryv3.MissingPrice
+			} else if obs.LinkPrice.Err != nil {
+				mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc()
 			}
 		}()
 	}
@@ -134,8 +138,11 @@ func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestam
 			defer wg.Done()
 			obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID)
 			if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil {
+				mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc()
 				ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", relaymercuryv3.MissingPrice), "nativeFeedID", ds.nativeFeedID)
 				obs.NativePrice.Val = relaymercuryv3.MissingPrice
+			} else if obs.NativePrice.Err != nil {
+				mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc()
 			}
 		}()
 	}

docs/CHANGELOG.md

@@ -18,6 +18,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed a bug that was preventing job runs to be displayed when the job `chainID` was disabled.
 - `chainlink txs evm create` returns a transaction hash for the attempted transaction in the CLI. Previously only the sender, receipient and `unstarted` state were returned.
 
+### Added
+
+- New prometheus metrics for mercury:
+    - `mercury_price_feed_missing`
+    - `mercury_price_feed_errors`
+  Nops may wish to add alerting on these.
+
 <!-- unreleasedstop -->
 
 ## 2.5.0 - 2023-09-13