Skip to content

Commit

Permalink
Log NewReportingPlugin errors for Commit and Exec plugins
Browse files Browse the repository at this point in the history
Also output metrics
  • Loading branch information
rstout committed Jul 11, 2024
1 parent 2bb582b commit 610a986
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 2 deletions.
12 changes: 11 additions & 1 deletion core/services/ocr2/plugins/ccip/ccipcommit/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func (rf *CommitReportingPluginFactory) NewReportingPlugin(config types.Reportin
// retried via RetryUntilSuccess. NewReportingPlugin must return successfully in order for the Commit plugin to
// function, hence why we can only keep retrying it until it succeeds.
func (rf *CommitReportingPluginFactory) NewReportingPluginFn(config types.ReportingPluginConfig) func() (reportingPluginAndInfo, error) {
return func() (reportingPluginAndInfo, error) {
newReportingPluginFn := func() (reportingPluginAndInfo, error) {
ctx := context.Background() // todo: consider adding some timeout

destPriceReg, err := rf.config.commitStore.ChangeConfig(ctx, config.OnchainConfig, config.OffchainConfig)
Expand Down Expand Up @@ -147,4 +147,14 @@ func (rf *CommitReportingPluginFactory) NewReportingPluginFn(config types.Report

return reportingPluginAndInfo{plugin, pluginInfo}, nil
}

return func() (reportingPluginAndInfo, error) {
result, err := newReportingPluginFn()
if err != nil {
rf.config.lggr.Errorw("NewReportingPlugin failed", "err", err)
rf.config.metricsCollector.NewReportingPluginError()
}

return result, err
}
}
3 changes: 3 additions & 0 deletions core/services/ocr2/plugins/ccip/ccipcommit/factory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/smartcontractkit/chainlink-common/pkg/types/ccip"
"github.com/smartcontractkit/chainlink/v2/core/logger"
ccip2 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip"
"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata"
ccipdataprovidermocks "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata/ccipdataprovider/mocks"
"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata/mocks"
Expand All @@ -24,6 +25,8 @@ import (
// retries a sufficient number of times to get through the transient errors and eventually succeed.
func TestNewReportingPluginRetriesUntilSuccess(t *testing.T) {
commitConfig := CommitPluginStaticConfig{}
commitConfig.lggr = logger.NullLogger
commitConfig.metricsCollector = ccip2.NoopMetricsCollector

// For this unit test, ensure that there is no delay between retries
commitConfig.newReportingPluginRetryConfig = ccipdata.RetryConfig{
Expand Down
12 changes: 11 additions & 1 deletion core/services/ocr2/plugins/ccip/ccipexec/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func (rf *ExecutionReportingPluginFactory) NewReportingPlugin(config types.Repor
// retried via RetryUntilSuccess. NewReportingPlugin must return successfully in order for the Exec plugin to function,
// hence why we can only keep retrying it until it succeeds.
func (rf *ExecutionReportingPluginFactory) NewReportingPluginFn(config types.ReportingPluginConfig) func() (reportingPluginAndInfo, error) {
return func() (reportingPluginAndInfo, error) {
newReportingPluginFn := func() (reportingPluginAndInfo, error) {
ctx := context.Background() // todo: consider setting a timeout

destPriceRegistry, destWrappedNative, err := rf.config.offRampReader.ChangeConfig(ctx, config.OnchainConfig, config.OffchainConfig)
Expand Down Expand Up @@ -154,4 +154,14 @@ func (rf *ExecutionReportingPluginFactory) NewReportingPluginFn(config types.Rep

return reportingPluginAndInfo{plugin, pluginInfo}, nil
}

return func() (reportingPluginAndInfo, error) {
result, err := newReportingPluginFn()
if err != nil {
rf.config.lggr.Errorw("NewReportingPlugin failed", "err", err)
rf.config.metricsCollector.NewReportingPluginError()
}

return result, err
}
}
3 changes: 3 additions & 0 deletions core/services/ocr2/plugins/ccip/ccipexec/factory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/smartcontractkit/chainlink-common/pkg/types/ccip"
"github.com/smartcontractkit/chainlink/v2/core/logger"
ccip2 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip"
"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata"
ccipdataprovidermocks "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata/ccipdataprovider/mocks"
"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/internal/ccipdata/mocks"
Expand All @@ -23,6 +24,8 @@ import (
// retries a sufficient number of times to get through the transient errors and eventually succeed.
func TestNewReportingPluginRetriesUntilSuccess(t *testing.T) {
execConfig := ExecutionPluginStaticConfig{}
execConfig.lggr = logger.TestLogger(t)
execConfig.metricsCollector = ccip2.NoopMetricsCollector

// For this unit test, ensure that there is no delay between retries
execConfig.newReportingPluginRetryConfig = ccipdata.RetryConfig{
Expand Down
14 changes: 14 additions & 0 deletions core/services/ocr2/plugins/ccip/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ var (
Name: "ccip_sequence_number_counter",
Help: "Sequence number of the last message processed by the plugin",
}, []string{"plugin", "source", "dest", "ocrPhase"})
newReportingPluginErrorCounter = promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "ccip_new_reporting_plugin_error_counter",
Help: "The count of the number of errors when calling NewReportingPlugin",
}, []string{"plugin"})
)

type ocrPhase string
Expand All @@ -35,6 +39,7 @@ type PluginMetricsCollector interface {
NumberOfMessagesBasedOnInterval(phase ocrPhase, seqNrMin, seqNrMax uint64)
UnexpiredCommitRoots(count int)
SequenceNumber(phase ocrPhase, seqNr uint64)
NewReportingPluginError()
}

type pluginMetricsCollector struct {
Expand Down Expand Up @@ -79,6 +84,12 @@ func (p *pluginMetricsCollector) SequenceNumber(phase ocrPhase, seqNr uint64) {
Set(float64(seqNr))
}

func (p *pluginMetricsCollector) NewReportingPluginError() {
newReportingPluginErrorCounter.
WithLabelValues(p.pluginName).
Inc()
}

var (
// NoopMetricsCollector is a no-op implementation of PluginMetricsCollector
NoopMetricsCollector PluginMetricsCollector = noop{}
Expand All @@ -97,3 +108,6 @@ func (d noop) UnexpiredCommitRoots(int) {

func (d noop) SequenceNumber(ocrPhase, uint64) {
}

func (d noop) NewReportingPluginError() {
}

0 comments on commit 610a986

Please sign in to comment.