Merge pull request #10151 from vegaprotocol/feature/add-eth-rpc-metri…

…cs-and-delay

fix: add metrics for ethereum rpc request + add initial delay for ethereum validation

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.73.6
+
+### 🐛 Fixes
+
+- Generate GraphQL binding properly.
+- [10153](https://github.com/vegaprotocol/vega/issues/10153) - Add metrics and reduce amount of request sent to the Ethereum `RPC`.
+
 ## 0.73.5
 
 ### 🐛 Fixes

core/datasource/external/ethverifier/verifier.go

@@ -25,20 +25,19 @@ import (
 
 	"code.vegaprotocol.io/vega/core/datasource/common"
 	"code.vegaprotocol.io/vega/core/datasource/errors"
-	"code.vegaprotocol.io/vega/core/events"
-	vegapb "code.vegaprotocol.io/vega/protos/vega"
-	datapb "code.vegaprotocol.io/vega/protos/vega/data/v1"
-
 	"code.vegaprotocol.io/vega/core/datasource/external/ethcall"
-
+	"code.vegaprotocol.io/vega/core/events"
+	"code.vegaprotocol.io/vega/core/metrics"
 	"code.vegaprotocol.io/vega/core/types"
 	"code.vegaprotocol.io/vega/core/validators"
 	"code.vegaprotocol.io/vega/logging"
+	vegapb "code.vegaprotocol.io/vega/protos/vega"
+	datapb "code.vegaprotocol.io/vega/protos/vega/data/v1"
 )
 
 //go:generate go run github.com/golang/mock/mockgen -destination mocks/witness_mock.go -package mocks code.vegaprotocol.io/vega/core/datasource/external/ethverifier Witness
 type Witness interface {
-	StartCheck(validators.Resource, func(interface{}, bool), time.Time) error
+	StartCheckWithDelay(validators.Resource, func(interface{}, bool), time.Time, int64) error
 	RestoreResource(validators.Resource, func(interface{}, bool)) error
 }
 
@@ -122,6 +121,7 @@ func (p pendingCallEvent) GetID() string { return p.callEvent.Hash() }
 func (p pendingCallEvent) GetType() types.NodeVoteType {
 	return types.NodeVoteTypeEthereumContractCallResult
 }
+
 func (p *pendingCallEvent) Check(ctx context.Context) error { return p.check(ctx) }
 
 func New(
@@ -181,23 +181,32 @@ func (s *Verifier) ProcessEthereumContractCallResult(callEvent ethcall.ContractC
 		check:     func(ctx context.Context) error { return s.checkCallEventResult(ctx, callEvent) },
 	}
 
+	confirmations, err := s.ethEngine.GetRequiredConfirmations(callEvent.SpecId)
+	if err != nil {
+		return err
+	}
+
 	s.pendingCallEvents = append(s.pendingCallEvents, pending)
 
 	s.log.Info("ethereum call event received, starting validation",
 		logging.String("call-event", fmt.Sprintf("%+v", callEvent)))
 
 	// Timeout for the check set to 1 day, to allow for validator outage scenarios
-	err := s.witness.StartCheck(
-		pending, s.onCallEventVerified, s.timeService.GetTimeNow().Add(24*time.Hour))
+	err = s.witness.StartCheckWithDelay(
+		pending, s.onCallEventVerified, s.timeService.GetTimeNow().Add(30*time.Minute), int64(confirmations))
 	if err != nil {
 		s.log.Error("could not start witness routine", logging.String("id", pending.GetID()))
 		s.removePendingCallEvent(pending.GetID())
 	}
 
+	metrics.DataSourceEthVerifierCallGaugeAdd(1, callEvent.SpecId)
+
 	return err
 }
 
 func (s *Verifier) checkCallEventResult(ctx context.Context, callEvent ethcall.ContractCallEvent) error {
+	metrics.DataSourceEthVerifierCallCounterInc(callEvent.SpecId)
+
 	// Ensure that the ethtime on the call event matches the block number on the eth chain
 	// (submitting call events with malicious times could subvert, e.g. TWAPs on perp markets)
 	checkedTime, err := s.ethEngine.GetEthTime(ctx, callEvent.BlockHeight)
@@ -210,6 +219,7 @@ func (s *Verifier) checkCallEventResult(ctx context.Context, callEvent ethcall.C
 			callEvent.BlockHeight, callEvent.BlockTime, checkedTime)
 	}
 
+	metrics.DataSourceEthVerifierCallCounterInc(callEvent.SpecId)
 	checkResult, err := s.ethEngine.CallSpec(ctx, callEvent.SpecId, callEvent.BlockHeight)
 	if callEvent.Error != nil {
 		if err != nil {
@@ -273,6 +283,8 @@ func (s *Verifier) onCallEventVerified(event interface{}, ok bool) {
 
 	if err := s.removePendingCallEvent(pv.GetID()); err != nil {
 		s.log.Error("could not remove pending call event", logging.Error(err))
+	} else {
+		metrics.DataSourceEthVerifierCallGaugeAdd(-1, pv.callEvent.SpecId)
 	}
 
 	if ok {

core/datasource/external/ethverifier/verifier_snapshot.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 
 	"code.vegaprotocol.io/vega/core/datasource/external/ethcall"
+	"code.vegaprotocol.io/vega/core/metrics"
 	"code.vegaprotocol.io/vega/core/types"
 	"code.vegaprotocol.io/vega/logging"
 
@@ -159,7 +160,15 @@ func (s *Verifier) restorePendingCallEvents(_ context.Context,
 	s.log.Debug("restoring pending call events snapshot", logging.Int("n_pending", len(results)))
 	s.pendingCallEvents = make([]*pendingCallEvent, 0, len(results))
 
+	// clear up all the metrics
+	seenSpecId := map[string]struct{}{}
+
 	for _, callEvent := range results {
+		if _, ok := seenSpecId[callEvent.SpecId]; !ok {
+			metrics.DataSourceEthVerifierCallGaugeReset(callEvent.SpecId)
+			seenSpecId[callEvent.SpecId] = struct{}{}
+		}
+
 		// this populates the id/hash structs
 		if !s.ensureNotDuplicate(callEvent.Hash()) {
 			s.log.Panic("pendingCallEvents's unexpectedly pre-populated when restoring from snapshot")
@@ -175,6 +184,8 @@ func (s *Verifier) restorePendingCallEvents(_ context.Context,
 		if err := s.witness.RestoreResource(pending, s.onCallEventVerified); err != nil {
 			s.log.Panic("unable to restore pending call event resource", logging.String("ID", pending.GetID()), logging.Error(err))
 		}
+
+		metrics.DataSourceEthVerifierCallGaugeAdd(1, callEvent.SpecId)
 	}
 
 	var err error

core/datasource/external/ethverifier/verifier_snapshot_test.go

@@ -83,15 +83,15 @@ func TestEthereumOracleVerifierWithPendingQueryResults(t *testing.T) {
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(5)).Return(uint64(100), nil)
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(5)).Return(result, nil)
 	eov.ethCallEngine.EXPECT().GetInitialTriggerTime("testspec").Return(uint64(90), nil)
-	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil).Times(2)
 
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 	eov.ethConfirmations.EXPECT().CheckRequiredConfirmations(uint64(5), uint64(5)).Return(nil)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})

core/datasource/external/ethverifier/verifier_test.go

@@ -103,12 +103,13 @@ func testSpoofedEthTimeFails(t *testing.T) {
 	assert.NotNil(t, eov)
 
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(1)).Return(uint64(50), nil)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations(gomock.Any()).Return(uint64(0), nil)
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -126,16 +127,17 @@ func testProcessEthereumOracleChainEventWithGlobalError(t *testing.T) {
 	testError := "test error"
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(1)).Return(uint64(100), nil)
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(1)).Return(ethcall.Result{}, errors.New(testError))
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations(gomock.Any()).Return(uint64(0), nil)
 
 	now := time.Now()
 	eov.ts.EXPECT().GetTimeNow().Return(now).Times(1)
 
 	var onQueryResultVerified func(interface{}, bool)
 	var checkResult error
 	var resourceToCheck interface{}
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			resourceToCheck = toCheck
 			onQueryResultVerified = fn
 			checkResult = toCheck.Check(context.Background())
@@ -191,11 +193,12 @@ func testProcessEthereumOracleChainEventWithLocalError(t *testing.T) {
 
 	now := time.Now()
 	eov.ts.EXPECT().GetTimeNow().Return(now).Times(1)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations(gomock.Any()).Return(uint64(0), nil)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -225,11 +228,12 @@ func testProcessEthereumOracleChainEventWithMismatchedError(t *testing.T) {
 
 	now := time.Now()
 	eov.ts.EXPECT().GetTimeNow().Return(now).Times(1)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations(gomock.Any()).Return(uint64(0), nil)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -257,7 +261,7 @@ func testProcessEthereumOracleQueryOK(t *testing.T) {
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(1)).Return(result, nil)
 	eov.ethCallEngine.EXPECT().MakeResult("testspec", []byte("testbytes")).Return(result, nil)
 
-	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil).Times(2)
 
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 	eov.ethCallEngine.EXPECT().GetInitialTriggerTime("testspec").Return(uint64(90), nil)
@@ -266,9 +270,9 @@ func testProcessEthereumOracleQueryOK(t *testing.T) {
 	var onQueryResultVerified func(interface{}, bool)
 	var checkResult error
 	var resourceToCheck interface{}
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			resourceToCheck = toCheck
 			onQueryResultVerified = fn
 			checkResult = toCheck.Check(context.Background())
@@ -311,10 +315,12 @@ func testProcessEthereumOracleQueryWithBlockTimeBeforeInitialTime(t *testing.T)
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 	eov.ethCallEngine.EXPECT().GetInitialTriggerTime("testspec").Return(uint64(110), nil)
 
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations(gomock.Any()).Return(uint64(0), nil)
+
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -334,11 +340,12 @@ func testProcessEthereumOracleQueryResultMismatch(t *testing.T) {
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(1)).Return(uint64(100), nil)
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(1)).Return(result, nil)
 	eov.ts.EXPECT().GetTimeNow().Times(1)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations(gomock.Any()).Return(uint64(0), nil)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -356,16 +363,16 @@ func testProcessEthereumOracleFilterMismatch(t *testing.T) {
 	result := filterMismatchResult()
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(1)).Return(uint64(100), nil)
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(1)).Return(result, nil)
-	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil).Times(2)
 
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 	eov.ethCallEngine.EXPECT().GetInitialTriggerTime("testspec").Return(uint64(90), nil)
 	eov.ethConfirmations.EXPECT().CheckRequiredConfirmations(uint64(1), uint64(5)).Return(nil)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -383,16 +390,16 @@ func testProcessEthereumOracleInsufficientConfirmations(t *testing.T) {
 	result := okResult()
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(1)).Return(uint64(100), nil)
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(1)).Return(result, nil)
-	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil).Times(2)
 
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 	eov.ethCallEngine.EXPECT().GetInitialTriggerTime("testspec").Return(uint64(90), nil)
 	eov.ethConfirmations.EXPECT().CheckRequiredConfirmations(uint64(1), uint64(5)).Return(eth.ErrMissingConfirmations)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})
@@ -411,16 +418,16 @@ func testProcessEthereumOracleQueryDuplicateIgnored(t *testing.T) {
 	result := okResult()
 	eov.ethCallEngine.EXPECT().GetEthTime(gomock.Any(), uint64(1)).Return(uint64(100), nil)
 	eov.ethCallEngine.EXPECT().CallSpec(gomock.Any(), "testspec", uint64(1)).Return(result, nil)
-	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil)
+	eov.ethCallEngine.EXPECT().GetRequiredConfirmations("testspec").Return(uint64(5), nil).Times(2)
 
 	eov.ts.EXPECT().GetTimeNow().Times(1)
 	eov.ethCallEngine.EXPECT().GetInitialTriggerTime("testspec").Return(uint64(90), nil)
 	eov.ethConfirmations.EXPECT().CheckRequiredConfirmations(uint64(1), uint64(5)).Return(nil)
 
 	var checkResult error
-	eov.witness.EXPECT().StartCheck(gomock.Any(), gomock.Any(), gomock.Any()).
+	eov.witness.EXPECT().StartCheckWithDelay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
 		Times(1).
-		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time) error {
+		DoAndReturn(func(toCheck validators.Resource, fn func(interface{}, bool), _ time.Time, _ int64) error {
 			checkResult = toCheck.Check(context.Background())
 			return nil
 		})