CCIP-4448 Track observation/outcome length in bytes (#15656)

* Track observation/outcome length * Track observation/outcome length * Post review fixes
smartcontractkit · Dec 12, 2024 · c68dcc8 · c68dcc8
1 parent 1e87a19
commit c68dcc8
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 13 deletions.
diff --git a/core/services/ocr3/promwrapper/factory.go b/core/services/ocr3/promwrapper/factory.go
@@ -47,6 +47,7 @@ func (r ReportingPluginFactory[RI]) NewReportingPlugin(ctx context.Context, conf
 		config.ConfigDigest.String(),
 		promOCR3ReportsGenerated,
 		promOCR3Durations,
+		promOCR3Sizes,
 		promOCR3PluginStatus,
 	)
 	return wrapped, info, err

diff --git a/core/services/ocr3/promwrapper/plugin.go b/core/services/ocr3/promwrapper/plugin.go
@@ -21,6 +21,7 @@ type reportingPlugin[RI any] struct {
 	// Prometheus components for tracking metrics
 	reportsGenerated *prometheus.CounterVec
 	durations        *prometheus.HistogramVec
+	sizes            *prometheus.CounterVec
 	status           *prometheus.GaugeVec
 }
 
@@ -31,6 +32,7 @@ func newReportingPlugin[RI any](
 	configDigest string,
 	reportsGenerated *prometheus.CounterVec,
 	durations *prometheus.HistogramVec,
+	sizes *prometheus.CounterVec,
 	status *prometheus.GaugeVec,
 ) *reportingPlugin[RI] {
 	return &reportingPlugin[RI]{
@@ -40,6 +42,7 @@ func newReportingPlugin[RI any](
 		configDigest:     configDigest,
 		reportsGenerated: reportsGenerated,
 		durations:        durations,
+		sizes:            sizes,
 		status:           status,
 	}
 }
@@ -51,9 +54,11 @@ func (p *reportingPlugin[RI]) Query(ctx context.Context, outctx ocr3types.Outcom
 }
 
 func (p *reportingPlugin[RI]) Observation(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query) (ocrtypes.Observation, error) {
-	return withObservedExecution(p, observation, func() (ocrtypes.Observation, error) {
+	result, err := withObservedExecution(p, observation, func() (ocrtypes.Observation, error) {
 		return p.ReportingPlugin.Observation(ctx, outctx, query)
 	})
+	p.trackSize(observation, len(result), err)
+	return result, err
 }
 
 func (p *reportingPlugin[RI]) ValidateObservation(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query, ao ocrtypes.AttributedObservation) error {
@@ -65,9 +70,11 @@ func (p *reportingPlugin[RI]) ValidateObservation(ctx context.Context, outctx oc
 }
 
 func (p *reportingPlugin[RI]) Outcome(ctx context.Context, outctx ocr3types.OutcomeContext, query ocrtypes.Query, aos []ocrtypes.AttributedObservation) (ocr3types.Outcome, error) {
-	return withObservedExecution(p, outcome, func() (ocr3types.Outcome, error) {
+	result, err := withObservedExecution(p, outcome, func() (ocr3types.Outcome, error) {
 		return p.ReportingPlugin.Outcome(ctx, outctx, query, aos)
 	})
+	p.trackSize(outcome, len(result), err)
+	return result, err
 }
 
 func (p *reportingPlugin[RI]) Reports(ctx context.Context, seqNr uint64, outcome ocr3types.Outcome) ([]ocr3types.ReportPlus[RI], error) {
@@ -111,6 +118,15 @@ func (p *reportingPlugin[RI]) updateStatus(status bool) {
 		Set(float64(boolToInt(status)))
 }
 
+func (p *reportingPlugin[RI]) trackSize(function functionType, size int, err error) {
+	if err != nil {
+		return
+	}
+	p.sizes.
+		WithLabelValues(p.chainID, p.plugin, string(function)).
+		Add(float64(size))
+}
+
 func boolToInt(arg bool) int {
 	if arg {
 		return 1

diff --git a/core/services/ocr3/promwrapper/plugin_test.go b/core/services/ocr3/promwrapper/plugin_test.go
@@ -17,17 +17,20 @@ import (
 )
 
 func Test_ReportsGeneratedGauge(t *testing.T) {
+	pluginObservationSize := 5
+	pluginOutcomeSize := 3
+
 	plugin1 := newReportingPlugin(
 		fakePlugin[uint]{reports: make([]ocr3types.ReportPlus[uint], 2)},
-		"123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus,
+		"123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus,
 	)
 	plugin2 := newReportingPlugin(
-		fakePlugin[bool]{reports: make([]ocr3types.ReportPlus[bool], 10)},
-		"solana", "different_plugin", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus,
+		fakePlugin[bool]{reports: make([]ocr3types.ReportPlus[bool], 10), observationSize: pluginObservationSize, outcomeSize: pluginOutcomeSize},
+		"solana", "different_plugin", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus,
 	)
 	plugin3 := newReportingPlugin(
 		fakePlugin[string]{err: errors.New("error")},
-		"1234", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus,
+		"1234", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus,
 	)
 
 	r1, err := plugin1.Reports(tests.Context(t), 1, nil)
@@ -64,20 +67,33 @@ func Test_ReportsGeneratedGauge(t *testing.T) {
 	require.NoError(t, plugin1.Close())
 	pluginHealth = testutil.ToFloat64(promOCR3PluginStatus.WithLabelValues("123", "empty", "abc"))
 	require.Equal(t, 0, int(pluginHealth))
+
+	iterations := 10
+	for i := 0; i < iterations; i++ {
+		_, err1 := plugin2.Outcome(tests.Context(t), ocr3types.OutcomeContext{}, nil, nil)
+		require.NoError(t, err1)
+	}
+	_, err1 := plugin2.Observation(tests.Context(t), ocr3types.OutcomeContext{}, nil)
+	require.NoError(t, err1)
+
+	outcomesLen := testutil.ToFloat64(promOCR3Sizes.WithLabelValues("solana", "different_plugin", "outcome"))
+	require.Equal(t, pluginOutcomeSize*iterations, int(outcomesLen))
+	observationLen := testutil.ToFloat64(promOCR3Sizes.WithLabelValues("solana", "different_plugin", "observation"))
+	require.Equal(t, pluginObservationSize, int(observationLen))
 }
 
 func Test_DurationHistograms(t *testing.T) {
 	plugin1 := newReportingPlugin(
 		fakePlugin[uint]{},
-		"123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus,
+		"123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus,
 	)
 	plugin2 := newReportingPlugin(
 		fakePlugin[uint]{err: errors.New("error")},
-		"123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus,
+		"123", "empty", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus,
 	)
 	plugin3 := newReportingPlugin(
 		fakePlugin[uint]{},
-		"solana", "commit", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3PluginStatus,
+		"solana", "commit", "abc", promOCR3ReportsGenerated, promOCR3Durations, promOCR3Sizes, promOCR3PluginStatus,
 	)
 
 	for _, p := range []*reportingPlugin[uint]{plugin1, plugin2, plugin3} {
@@ -102,8 +118,10 @@ func Test_DurationHistograms(t *testing.T) {
 }
 
 type fakePlugin[RI any] struct {
-	reports []ocr3types.ReportPlus[RI]
-	err     error
+	reports         []ocr3types.ReportPlus[RI]
+	observationSize int
+	outcomeSize     int
+	err             error
 }
 
 func (f fakePlugin[RI]) Query(context.Context, ocr3types.OutcomeContext) (ocrtypes.Query, error) {
@@ -117,7 +135,7 @@ func (f fakePlugin[RI]) Observation(context.Context, ocr3types.OutcomeContext, o
 	if f.err != nil {
 		return nil, f.err
 	}
-	return ocrtypes.Observation{}, nil
+	return make([]byte, f.observationSize), nil
 }
 
 func (f fakePlugin[RI]) ValidateObservation(context.Context, ocr3types.OutcomeContext, ocrtypes.Query, ocrtypes.AttributedObservation) error {
@@ -132,7 +150,7 @@ func (f fakePlugin[RI]) Outcome(context.Context, ocr3types.OutcomeContext, ocrty
 	if f.err != nil {
 		return nil, f.err
 	}
-	return ocr3types.Outcome{}, nil
+	return make([]byte, f.outcomeSize), nil
 }
 
 func (f fakePlugin[RI]) Reports(context.Context, uint64, ocr3types.Outcome) ([]ocr3types.ReportPlus[RI], error) {

diff --git a/core/services/ocr3/promwrapper/types.go b/core/services/ocr3/promwrapper/types.go
@@ -48,6 +48,13 @@ var (
 		},
 		[]string{"chainID", "plugin", "function", "success"},
 	)
+	promOCR3Sizes = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "ocr3_reporting_plugin_data_sizes",
+			Help: "Tracks the size of the data produced by OCR3 plugin in bytes (e.g. reports, observations etc.)",
+		},
+		[]string{"chainID", "plugin", "function"},
+	)
 	promOCR3PluginStatus = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "ocr3_reporting_plugin_status",