diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3ba0f2976ec..1f3e093cfdc 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -563,12 +563,12 @@ jobs: pyroscope_env: ci-smoke-vrf-evm-simulated - name: vrfv2 id: vrfv2 - nodes: 4 + nodes: 5 os: ubuntu-latest pyroscope_env: ci-smoke-vrf2-evm-simulated - name: vrfv2plus id: vrfv2plus - nodes: 7 + nodes: 8 os: ubuntu-latest pyroscope_env: ci-smoke-vrf2plus-evm-simulated - name: forwarder_ocr diff --git a/integration-tests/actions/actions.go b/integration-tests/actions/actions.go index f5c91b63527..ee4309c3613 100644 --- a/integration-tests/actions/actions.go +++ b/integration-tests/actions/actions.go @@ -8,14 +8,18 @@ import ( "errors" "fmt" "math/big" + "os" "strings" "sync" "testing" "time" "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/go-resty/resty/v2" + "github.com/rs/zerolog" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -33,6 +37,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" + "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" ) // ContractDeploymentInterval After how many contract actions to wait before starting any more @@ -562,3 +567,100 @@ func WaitForBlockNumberToBe( } } } + +// todo - move to EVMClient +func RewindSimulatedChainToBlockNumber( + ctx context.Context, + evmClient blockchain.EVMClient, + rpcURL string, + rewindChainToBlockNumber uint64, + l zerolog.Logger, +) (uint64, error) { + latestBlockNumberBeforeReorg, err := evmClient.LatestBlockNumber(ctx) + if err != nil { + return 0, fmt.Errorf("error getting latest block number: %w", err) + } + + l.Info(). + Str("RPC URL", rpcURL). + Uint64("Latest Block Number before Reorg", latestBlockNumberBeforeReorg). + Uint64("Rewind Chain to Block Number", rewindChainToBlockNumber). + Msg("Performing Reorg on chain by rewinding chain to specific block number") + + _, err = NewRPCRawClient(rpcURL).SetHeadForSimulatedChain(rewindChainToBlockNumber) + + if err != nil { + return 0, fmt.Errorf("error making reorg: %w", err) + } + + err = evmClient.WaitForEvents() + if err != nil { + return 0, fmt.Errorf("error waiting for events: %w", err) + } + + latestBlockNumberAfterReorg, err := evmClient.LatestBlockNumber(ctx) + if err != nil { + return 0, fmt.Errorf("error getting latest block number: %w", err) + } + + l.Info(). + Uint64("Block Number", latestBlockNumberAfterReorg). + Msg("Latest Block Number after Reorg") + return latestBlockNumberAfterReorg, nil +} + +func GetRPCUrl(env *test_env.CLClusterTestEnv, chainID int64) (string, error) { + provider, err := env.GetRpcProvider(chainID) + if err != nil { + return "", err + } + return provider.PublicHttpUrls()[0], nil +} + +// RPCRawClient +// created separate client since method evmClient.RawJsonRPCCall fails on "invalid argument 0: json: cannot unmarshal non-string into Go value of type hexutil.Uint64" +type RPCRawClient struct { + resty *resty.Client +} + +func NewRPCRawClient(url string) *RPCRawClient { + isDebug := os.Getenv("DEBUG_RESTY") == "true" + restyClient := resty.New().SetDebug(isDebug).SetBaseURL(url) + return &RPCRawClient{ + resty: restyClient, + } +} + +func (g *RPCRawClient) SetHeadForSimulatedChain(setHeadToBlockNumber uint64) (JsonRPCResponse, error) { + var responseObject JsonRPCResponse + postBody, _ := json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "debug_setHead", + "params": []string{hexutil.EncodeUint64(setHeadToBlockNumber)}, + }) + resp, err := g.resty.R(). + SetHeader("Content-Type", "application/json"). + SetBody(postBody). + SetResult(&responseObject). + Post("") + + if err != nil { + return JsonRPCResponse{}, fmt.Errorf("error making API request: %w", err) + } + statusCode := resp.StatusCode() + if statusCode != 200 && statusCode != 201 { + return JsonRPCResponse{}, fmt.Errorf("error invoking debug_setHead method, received unexpected status code %d: %s", statusCode, resp.String()) + } + if responseObject.Error != "" { + return JsonRPCResponse{}, fmt.Errorf("received non-empty error field: %v", responseObject.Error) + } + return responseObject, nil +} + +type JsonRPCResponse struct { + Version string `json:"jsonrpc"` + Id int `json:"id"` + Result string `json:"result,omitempty"` + Error string `json:"error,omitempty"` +} diff --git a/integration-tests/actions/vrf/vrfv2/contract_steps.go b/integration-tests/actions/vrf/vrfv2/contract_steps.go index fef780b695b..7c1ee634f98 100644 --- a/integration-tests/actions/vrf/vrfv2/contract_steps.go +++ b/integration-tests/actions/vrf/vrfv2/contract_steps.go @@ -506,7 +506,7 @@ func RequestRandomnessAndWaitForFulfillment( randomnessRequestCountPerRequest uint16, randomnessRequestCountPerRequestDeviation uint16, randomWordsFulfilledEventTimeout time.Duration, -) (*contracts.CoordinatorRandomWordsFulfilled, error) { +) (*contracts.CoordinatorRandomWordsRequested, *contracts.CoordinatorRandomWordsFulfilled, error) { randomWordsRequestedEvent, err := RequestRandomness( l, consumer, @@ -520,18 +520,18 @@ func RequestRandomnessAndWaitForFulfillment( randomnessRequestCountPerRequestDeviation, ) if err != nil { - return nil, err + return nil, nil, err } - fulfillmentEvents, err := WaitRandomWordsFulfilledEvent( + randomWordsFulfilledEvent, err := WaitRandomWordsFulfilledEvent( coordinator, randomWordsRequestedEvent.RequestId, randomWordsFulfilledEventTimeout, l, ) if err != nil { - return nil, err + return nil, nil, err } - return fulfillmentEvents, nil + return randomWordsRequestedEvent, randomWordsFulfilledEvent, nil } func RequestRandomness( diff --git a/integration-tests/actions/vrf/vrfv2plus/contract_steps.go b/integration-tests/actions/vrf/vrfv2plus/contract_steps.go index 143b1e2cd9b..26532e9b8e8 100644 --- a/integration-tests/actions/vrf/vrfv2plus/contract_steps.go +++ b/integration-tests/actions/vrf/vrfv2plus/contract_steps.go @@ -376,7 +376,7 @@ func RequestRandomnessAndWaitForFulfillment( isNativeBilling bool, config *vrfv2plus_config.General, l zerolog.Logger, -) (*contracts.CoordinatorRandomWordsFulfilled, error) { +) (*contracts.CoordinatorRandomWordsRequested, *contracts.CoordinatorRandomWordsFulfilled, error) { randomWordsRequestedEvent, err := RequestRandomness( consumer, coordinator, @@ -387,7 +387,7 @@ func RequestRandomnessAndWaitForFulfillment( l, ) if err != nil { - return nil, err + return nil, nil, err } randomWordsFulfilledEvent, err := WaitRandomWordsFulfilledEvent( @@ -399,9 +399,9 @@ func RequestRandomnessAndWaitForFulfillment( l, ) if err != nil { - return nil, err + return nil, nil, err } - return randomWordsFulfilledEvent, nil + return randomWordsRequestedEvent, randomWordsFulfilledEvent, nil } diff --git a/integration-tests/load/vrfv2/gun.go b/integration-tests/load/vrfv2/gun.go index 71d3113e60f..d15ee18d451 100644 --- a/integration-tests/load/vrfv2/gun.go +++ b/integration-tests/load/vrfv2/gun.go @@ -87,7 +87,7 @@ func (m *SingleHashGun) Call(_ *wasp.Generator) *wasp.Response { vrfv2Config := m.testConfig.General //randomly increase/decrease randomness request count per TX randomnessRequestCountPerRequest := deviateValue(*vrfv2Config.RandomnessRequestCountPerRequest, *vrfv2Config.RandomnessRequestCountPerRequestDeviation) - _, err := vrfv2.RequestRandomnessAndWaitForFulfillment( + _, _, err := vrfv2.RequestRandomnessAndWaitForFulfillment( m.logger, //the same consumer is used for all requests and in all subs m.contracts.VRFV2Consumers[0], diff --git a/integration-tests/load/vrfv2plus/gun.go b/integration-tests/load/vrfv2plus/gun.go index 8a99392fa57..6879fbe32dc 100644 --- a/integration-tests/load/vrfv2plus/gun.go +++ b/integration-tests/load/vrfv2plus/gun.go @@ -99,7 +99,7 @@ func (m *SingleHashGun) Call(_ *wasp.Generator) *wasp.Response { //randomly increase/decrease randomness request count per TX reqCount := deviateValue(*m.testConfig.General.RandomnessRequestCountPerRequest, *m.testConfig.General.RandomnessRequestCountPerRequestDeviation) m.testConfig.General.RandomnessRequestCountPerRequest = &reqCount - _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( //the same consumer is used for all requests and in all subs m.contracts.VRFV2PlusConsumer[0], m.contracts.CoordinatorV2Plus, diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index 62c5dc84fe6..4dd23bce26f 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -63,7 +63,7 @@ func TestVRFv2Basic(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -104,7 +104,7 @@ func TestVRFv2Basic(t *testing.T) { subBalanceBeforeRequest := subscription.Balance // test and assert - randomWordsFulfilledEvent, err := vrfv2.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2.RequestRandomnessAndWaitForFulfillment( l, consumers[0], vrfContracts.CoordinatorV2, @@ -137,6 +137,54 @@ func TestVRFv2Basic(t *testing.T) { } }) + t.Run("VRF Node waits block confirmation number specified by the consumer in the rand request before sending fulfilment on-chain", func(t *testing.T) { + configCopy := config.MustCopy().(tc.TestConfig) + testConfig := configCopy.VRFv2.General + + consumers, subIDs, err := vrfv2.SetupNewConsumersAndSubs( + testEnv, + chainID, + vrfContracts.CoordinatorV2, + configCopy, + vrfContracts.LinkToken, + 1, + 1, + l, + ) + require.NoError(t, err, "error setting up new consumers and subs") + subID := subIDs[0] + subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) + require.NoError(t, err, "error getting subscription information") + vrfcommon.LogSubDetails(l, subscription, strconv.FormatUint(subID, 10), vrfContracts.CoordinatorV2) + subIDsForCancellingAfterTest = append(subIDsForCancellingAfterTest, subIDs...) + + expectedBlockNumberWait := uint16(10) + testConfig.MinimumConfirmations = ptr.Ptr[uint16](expectedBlockNumberWait) + randomWordsRequestedEvent, randomWordsFulfilledEvent, err := vrfv2.RequestRandomnessAndWaitForFulfillment( + l, + consumers[0], + vrfContracts.CoordinatorV2, + subID, + vrfKey, + *testConfig.MinimumConfirmations, + *testConfig.CallbackGasLimit, + *testConfig.NumberOfWords, + *testConfig.RandomnessRequestCountPerRequest, + *testConfig.RandomnessRequestCountPerRequestDeviation, + testConfig.RandomWordsFulfilledEventTimeout.Duration, + ) + require.NoError(t, err, "error requesting randomness and waiting for fulfilment") + + // check that VRF node waited at least the number of blocks specified by the consumer in the rand request min confs field + blockNumberWait := randomWordsRequestedEvent.Raw.BlockNumber - randomWordsFulfilledEvent.Raw.BlockNumber + require.GreaterOrEqual(t, blockNumberWait, uint64(expectedBlockNumberWait)) + + status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) + require.NoError(t, err, "error getting rand request status") + require.True(t, status.Fulfilled) + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + }) + t.Run("CL Node VRF Job Runs", func(t *testing.T) { configCopy := config.MustCopy().(tc.TestConfig) consumers, subIDsForJobRuns, err := vrfv2.SetupNewConsumersAndSubs( @@ -161,7 +209,7 @@ func TestVRFv2Basic(t *testing.T) { require.NoError(t, err, "error reading job runs") // test and assert - _, err = vrfv2.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2.RequestRandomnessAndWaitForFulfillment( l, consumers[0], vrfContracts.CoordinatorV2, @@ -281,7 +329,7 @@ func TestVRFv2Basic(t *testing.T) { vrfcommon.LogSubDetails(l, subscription, strconv.FormatUint(subIDForOracleWithdraw, 10), vrfContracts.CoordinatorV2) subIDsForCancellingAfterTest = append(subIDsForCancellingAfterTest, subIDsForOracleWithDraw...) - fulfilledEventLink, err := vrfv2.RequestRandomnessAndWaitForFulfillment( + _, fulfilledEventLink, err := vrfv2.RequestRandomnessAndWaitForFulfillment( l, consumers[0], vrfContracts.CoordinatorV2, @@ -436,7 +484,7 @@ func TestVRFv2Basic(t *testing.T) { // Request randomness - should fail due to underfunded subscription randomWordsFulfilledEventTimeout := 5 * time.Second - _, err = vrfv2.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2.RequestRandomnessAndWaitForFulfillment( l, consumers[0], vrfContracts.CoordinatorV2, @@ -556,7 +604,7 @@ func TestVRFv2MultipleSendingKeys(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -602,7 +650,7 @@ func TestVRFv2MultipleSendingKeys(t *testing.T) { var fulfillmentTxFromAddresses []string for i := 0; i < newEnvConfig.NumberOfTxKeysToCreate+1; i++ { - randomWordsFulfilledEvent, err := vrfv2.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2.RequestRandomnessAndWaitForFulfillment( l, consumers[0], vrfContracts.CoordinatorV2, @@ -664,7 +712,7 @@ func TestVRFOwner(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -807,7 +855,7 @@ func TestVRFV2WithBHS(t *testing.T) { } } if !*vrfv2Config.General.UseExistingEnv { - if err := testEnv.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := testEnv.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -984,3 +1032,164 @@ func TestVRFV2WithBHS(t *testing.T) { require.Equal(t, 0, randomWordsRequestedEvent.Raw.BlockHash.Cmp(randRequestBlockHash)) }) } + +func TestVRFV2NodeReorg(t *testing.T) { + t.Parallel() + var ( + env *test_env.CLClusterTestEnv + vrfContracts *vrfcommon.VRFContracts + subIDsForCancellingAfterTest []uint64 + defaultWalletAddress string + vrfKey *vrfcommon.VRFKeyData + ) + l := logging.GetTestLogger(t) + + config, err := tc.GetConfig("Smoke", tc.VRFv2) + require.NoError(t, err, "Error getting config") + vrfv2Config := config.VRFv2 + chainID := networks.MustGetSelectedNetworkConfig(config.GetNetworkConfig())[0].ChainID + + cleanupFn := func() { + evmClient, err := env.GetEVMClient(chainID) + require.NoError(t, err, "Getting EVM client shouldn't fail") + + if evmClient.NetworkSimulated() { + l.Info(). + Str("Network Name", evmClient.GetNetworkName()). + Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") + } else { + if *vrfv2Config.General.CancelSubsAfterTestRun { + //cancel subs and return funds to sub owner + vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l) + } + } + if !*vrfv2Config.General.UseExistingEnv { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { + l.Error().Err(err).Msg("Error cleaning up test environment") + } + } + } + newEnvConfig := vrfcommon.NewEnvConfig{ + NodesToCreate: []vrfcommon.VRFNodeType{vrfcommon.VRF}, + NumberOfTxKeysToCreate: 0, + UseVRFOwner: false, + UseTestCoordinator: false, + } + + env, vrfContracts, vrfKey, _, err = vrfv2.SetupVRFV2Universe(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + require.NoError(t, err, "Error setting up VRFv2 universe") + + evmClient, err := env.GetEVMClient(chainID) + require.NoError(t, err, "Getting EVM client shouldn't fail") + defaultWalletAddress = evmClient.GetDefaultWallet().Address() + + consumers, subIDs, err := vrfv2.SetupNewConsumersAndSubs( + env, + chainID, + vrfContracts.CoordinatorV2, + config, + vrfContracts.LinkToken, + 1, + 1, + l, + ) + require.NoError(t, err, "error setting up new consumers and subs") + subID := subIDs[0] + subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) + require.NoError(t, err, "error getting subscription information") + vrfcommon.LogSubDetails(l, subscription, strconv.FormatUint(subID, 10), vrfContracts.CoordinatorV2) + subIDsForCancellingAfterTest = append(subIDsForCancellingAfterTest, subIDs...) + + t.Run("Reorg on fulfillment", func(t *testing.T) { + configCopy := config.MustCopy().(tc.TestConfig) + configCopy.VRFv2.General.MinimumConfirmations = ptr.Ptr[uint16](10) + + //1. request randomness and wait for fulfillment for blockhash from Reorged Fork + randomWordsRequestedEvent, randomWordsFulfilledEventOnReorgedFork, err := vrfv2.RequestRandomnessAndWaitForFulfillment( + l, + consumers[0], + vrfContracts.CoordinatorV2, + subID, + vrfKey, + *configCopy.VRFv2.General.MinimumConfirmations, + *configCopy.VRFv2.General.CallbackGasLimit, + *configCopy.VRFv2.General.NumberOfWords, + *configCopy.VRFv2.General.RandomnessRequestCountPerRequest, + *configCopy.VRFv2.General.RandomnessRequestCountPerRequestDeviation, + configCopy.VRFv2.General.RandomWordsFulfilledEventTimeout.Duration, + ) + require.NoError(t, err) + + // rewind chain to block number after the request was made, but before the request was fulfilled + rewindChainToBlock := randomWordsRequestedEvent.Raw.BlockNumber + 1 + + rpcUrl, err := actions.GetRPCUrl(env, chainID) + require.NoError(t, err, "error getting rpc url") + + //2. rewind chain by n number of blocks - basically, mimicking reorg scenario + latestBlockNumberAfterReorg, err := actions.RewindSimulatedChainToBlockNumber(testcontext.Get(t), evmClient, rpcUrl, rewindChainToBlock, l) + require.NoError(t, err, fmt.Sprintf("error rewinding chain to block number %d", rewindChainToBlock)) + + //3.1 ensure that chain is reorged and latest block number is greater than the block number when request was made + require.Greater(t, latestBlockNumberAfterReorg, randomWordsRequestedEvent.Raw.BlockNumber) + + //3.2 ensure that chain is reorged and latest block number is less than the block number when fulfilment was performed + require.Less(t, latestBlockNumberAfterReorg, randomWordsFulfilledEventOnReorgedFork.Raw.BlockNumber) + + //4. wait for the fulfillment which VRF Node will generate for Canonical chain + _, err = vrfv2.WaitRandomWordsFulfilledEvent( + vrfContracts.CoordinatorV2, + randomWordsRequestedEvent.RequestId, + configCopy.VRFv2.General.RandomWordsFulfilledEventTimeout.Duration, + l, + ) + + require.NoError(t, err, "error waiting for randomness fulfilled event") + }) + + t.Run("Reorg on rand request", func(t *testing.T) { + configCopy := config.MustCopy().(tc.TestConfig) + //1. set minimum confirmations to higher value so that we can be sure that request won't be fulfilled before reorg + configCopy.VRFv2.General.MinimumConfirmations = ptr.Ptr[uint16](6) + + //2. request randomness + randomWordsRequestedEvent, err := vrfv2.RequestRandomness( + l, + consumers[0], + vrfContracts.CoordinatorV2, + subID, + vrfKey, + *configCopy.VRFv2.General.MinimumConfirmations, + *configCopy.VRFv2.General.CallbackGasLimit, + *configCopy.VRFv2.General.NumberOfWords, + *configCopy.VRFv2.General.RandomnessRequestCountPerRequest, + *configCopy.VRFv2.General.RandomnessRequestCountPerRequestDeviation, + ) + require.NoError(t, err) + + // rewind chain to block number before the randomness request was made + rewindChainToBlockNumber := randomWordsRequestedEvent.Raw.BlockNumber - 3 + + rpcUrl, err := actions.GetRPCUrl(env, chainID) + require.NoError(t, err, "error getting rpc url") + + //3. rewind chain by n number of blocks - basically, mimicking reorg scenario + latestBlockNumberAfterReorg, err := actions.RewindSimulatedChainToBlockNumber(testcontext.Get(t), evmClient, rpcUrl, rewindChainToBlockNumber, l) + require.NoError(t, err, fmt.Sprintf("error rewinding chain to block number %d", rewindChainToBlockNumber)) + + //4. ensure that chain is reorged and latest block number is less than the block number when request was made + require.Less(t, latestBlockNumberAfterReorg, randomWordsRequestedEvent.Raw.BlockNumber) + + //5. ensure that rand request is not fulfilled for the request which was made on reorged fork + // For context - when performing debug_setHead on geth simulated chain and therefore rewinding chain to a previous block, + //then tx that was mined after reorg will not appear in canonical chain contrary to real world scenario + //Hence, we only verify that VRF node will not generate fulfillment for the reorged fork request + _, err = vrfContracts.CoordinatorV2.WaitForRandomWordsFulfilledEvent( + contracts.RandomWordsFulfilledEventFilter{ + RequestIds: []*big.Int{randomWordsRequestedEvent.RequestId}, + Timeout: time.Second * 10, + }, + ) + require.Error(t, err, "fulfillment should not be generated for the request which was made on reorged fork on Simulated Chain") + }) +} diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index c535e77ef62..ae190e060ee 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -19,6 +19,7 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/networks" "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" "github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext" + "github.com/smartcontractkit/chainlink/integration-tests/actions" vrfcommon "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/common" "github.com/smartcontractkit/chainlink/integration-tests/actions/vrf/vrfv2plus" "github.com/smartcontractkit/chainlink/integration-tests/client" @@ -26,7 +27,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/blockhash_store" - "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" it_utils "github.com/smartcontractkit/chainlink/integration-tests/utils" @@ -64,7 +64,7 @@ func TestVRFv2Plus(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -106,7 +106,7 @@ func TestVRFv2Plus(t *testing.T) { subBalanceBeforeRequest := subscription.Balance // test and assert - randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -163,7 +163,7 @@ func TestVRFv2Plus(t *testing.T) { subNativeTokenBalanceBeforeRequest := subscription.NativeBalance // test and assert - randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -193,6 +193,52 @@ func TestVRFv2Plus(t *testing.T) { require.Equal(t, 1, w.Cmp(big.NewInt(0)), "Expected the VRF job give an answer bigger than 0") } }) + + t.Run("VRF Node waits block confirmation number specified by the consumer in the rand request before sending fulfilment on-chain", func(t *testing.T) { + configCopy := config.MustCopy().(tc.TestConfig) + testConfig := configCopy.VRFv2Plus.General + var isNativeBilling = true + + consumers, subIDs, err := vrfv2plus.SetupNewConsumersAndSubs( + env, + chainID, + vrfContracts.CoordinatorV2Plus, + configCopy, + vrfContracts.LinkToken, + 1, + 1, + l, + ) + require.NoError(t, err, "error setting up new consumers and subs") + subID := subIDs[0] + subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) + require.NoError(t, err, "error getting subscription information") + vrfcommon.LogSubDetails(l, subscription, subID.String(), vrfContracts.CoordinatorV2Plus) + subIDsForCancellingAfterTest = append(subIDsForCancellingAfterTest, subIDs...) + + expectedBlockNumberWait := uint16(10) + testConfig.MinimumConfirmations = ptr.Ptr[uint16](expectedBlockNumberWait) + randomWordsRequestedEvent, randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + consumers[0], + vrfContracts.CoordinatorV2Plus, + vrfKey, + subID, + isNativeBilling, + testConfig, + l, + ) + require.NoError(t, err, "error requesting randomness and waiting for fulfilment") + + // check that VRF node waited at least the number of blocks specified by the consumer in the rand request min confs field + blockNumberWait := randomWordsRequestedEvent.Raw.BlockNumber - randomWordsFulfilledEvent.Raw.BlockNumber + require.GreaterOrEqual(t, blockNumberWait, uint64(expectedBlockNumberWait)) + + status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) + require.NoError(t, err, "error getting rand request status") + require.True(t, status.Fulfilled) + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + }) + t.Run("CL Node VRF Job Runs", func(t *testing.T) { configCopy := config.MustCopy().(tc.TestConfig) var isNativeBilling = false @@ -217,7 +263,7 @@ func TestVRFv2Plus(t *testing.T) { require.NoError(t, err, "error reading job runs") // test and assert - _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -482,7 +528,7 @@ func TestVRFv2Plus(t *testing.T) { require.False(t, pendingRequestsExist, "Pending requests should not exist") configCopy.VRFv2Plus.General.RandomWordsFulfilledEventTimeout = ptr.Ptr(blockchain.StrDuration{Duration: 5 * time.Second}) - _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -494,7 +540,7 @@ func TestVRFv2Plus(t *testing.T) { require.Error(t, err, "error should occur for waiting for fulfilment due to low sub balance") - _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -622,7 +668,7 @@ func TestVRFv2Plus(t *testing.T) { vrfcommon.LogSubDetails(l, subscription, subID.String(), vrfContracts.CoordinatorV2Plus) subIDsForCancellingAfterTest = append(subIDsForCancellingAfterTest, subIDs...) - fulfilledEventLink, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, fulfilledEventLink, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -633,7 +679,7 @@ func TestVRFv2Plus(t *testing.T) { ) require.NoError(t, err) - fulfilledEventNative, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, fulfilledEventNative, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -719,7 +765,7 @@ func TestVRFv2PlusMultipleSendingKeys(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -766,7 +812,7 @@ func TestVRFv2PlusMultipleSendingKeys(t *testing.T) { var fulfillmentTxFromAddresses []string for i := 0; i < newEnvConfig.NumberOfTxKeysToCreate+1; i++ { - randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -826,7 +872,7 @@ func TestVRFv2PlusMigration(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1002,7 +1048,7 @@ func TestVRFv2PlusMigration(t *testing.T) { require.Equal(t, 0, expectedEthTotalBalanceForOldCoordinator.Cmp(oldCoordinatorEthTotalBalanceAfterMigration)) //Verify rand requests fulfills with Link Token billing - _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], newCoordinator, vrfKey, @@ -1014,7 +1060,7 @@ func TestVRFv2PlusMigration(t *testing.T) { require.NoError(t, err, "error requesting randomness and waiting for fulfilment") //Verify rand requests fulfills with Native Token billing - _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, _, err = vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[1], newCoordinator, vrfKey, @@ -1249,7 +1295,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1470,7 +1516,7 @@ func TestVRFV2PlusWithBHF(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1618,7 +1664,7 @@ func TestVRFv2PlusReplayAfterTimeout(t *testing.T) { } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1694,7 +1740,7 @@ func TestVRFv2PlusReplayAfterTimeout(t *testing.T) { 1, ) require.NoError(t, err, "error creating funded sub in replay test") - randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[1], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -1817,7 +1863,7 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T) } } if !*vrfv2PlusConfig.General.UseExistingEnv { - if err := env.Cleanup(test_env.CleanupOpts{}); err != nil { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { l.Error().Err(err).Msg("Error cleaning up test environment") } } @@ -1862,7 +1908,7 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T) l.Info().Uint16("minimumConfirmationDelay", *config.VRFv2Plus.General.MinimumConfirmations).Msg("Minimum Confirmation Delay") // test and assert - randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + _, randomWordsFulfilledEvent, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( consumers[0], vrfContracts.CoordinatorV2Plus, vrfKey, @@ -1878,3 +1924,162 @@ func TestVRFv2PlusPendingBlockSimulationAndZeroConfirmationDelays(t *testing.T) require.True(t, status.Fulfilled) l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") } + +func TestVRFv2PlusNodeReorg(t *testing.T) { + t.Parallel() + var ( + env *test_env.CLClusterTestEnv + vrfContracts *vrfcommon.VRFContracts + subIDsForCancellingAfterTest []*big.Int + defaultWalletAddress string + vrfKey *vrfcommon.VRFKeyData + ) + l := logging.GetTestLogger(t) + + config, err := tc.GetConfig("Smoke", tc.VRFv2Plus) + require.NoError(t, err, "Error getting config") + vrfv2PlusConfig := config.VRFv2Plus + chainID := networks.MustGetSelectedNetworkConfig(config.GetNetworkConfig())[0].ChainID + + cleanupFn := func() { + evmClient, err := env.GetEVMClient(chainID) + require.NoError(t, err, "Getting EVM client shouldn't fail") + + if evmClient.NetworkSimulated() { + l.Info(). + Str("Network Name", evmClient.GetNetworkName()). + Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") + } else { + if *vrfv2PlusConfig.General.CancelSubsAfterTestRun { + //cancel subs and return funds to sub owner + vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l) + } + } + if !*vrfv2PlusConfig.General.UseExistingEnv { + if err := env.Cleanup(test_env.CleanupOpts{TestName: t.Name()}); err != nil { + l.Error().Err(err).Msg("Error cleaning up test environment") + } + } + } + newEnvConfig := vrfcommon.NewEnvConfig{ + NodesToCreate: []vrfcommon.VRFNodeType{vrfcommon.VRF}, + NumberOfTxKeysToCreate: 0, + UseVRFOwner: false, + UseTestCoordinator: false, + } + + env, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, config, chainID, cleanupFn, newEnvConfig, l) + require.NoError(t, err, "Error setting up VRFv2Plus universe") + + evmClient, err := env.GetEVMClient(chainID) + require.NoError(t, err, "Getting EVM client shouldn't fail") + defaultWalletAddress = evmClient.GetDefaultWallet().Address() + + var isNativeBilling = true + + consumers, subIDs, err := vrfv2plus.SetupNewConsumersAndSubs( + env, + chainID, + vrfContracts.CoordinatorV2Plus, + config, + vrfContracts.LinkToken, + 1, + 1, + l, + ) + require.NoError(t, err, "error setting up new consumers and subs") + subID := subIDs[0] + subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) + require.NoError(t, err, "error getting subscription information") + vrfcommon.LogSubDetails(l, subscription, subID.String(), vrfContracts.CoordinatorV2Plus) + subIDsForCancellingAfterTest = append(subIDsForCancellingAfterTest, subIDs...) + + t.Run("Reorg on fulfillment", func(t *testing.T) { + configCopy := config.MustCopy().(tc.TestConfig) + configCopy.VRFv2Plus.General.MinimumConfirmations = ptr.Ptr[uint16](10) + + //1. request randomness and wait for fulfillment for blockhash from Reorged Fork + randomWordsRequestedEvent, randomWordsFulfilledEventOnReorgedFork, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + consumers[0], + vrfContracts.CoordinatorV2Plus, + vrfKey, + subID, + isNativeBilling, + configCopy.VRFv2Plus.General, + l, + ) + require.NoError(t, err) + + // rewind chain to block number after the request was made, but before the request was fulfilled + rewindChainToBlock := randomWordsRequestedEvent.Raw.BlockNumber + 1 + + rpcUrl, err := actions.GetRPCUrl(env, chainID) + require.NoError(t, err, "error getting rpc url") + + //2. rewind chain by n number of blocks - basically, mimicking reorg scenario + latestBlockNumberAfterReorg, err := actions.RewindSimulatedChainToBlockNumber(testcontext.Get(t), evmClient, rpcUrl, rewindChainToBlock, l) + require.NoError(t, err, fmt.Sprintf("error rewinding chain to block number %d", rewindChainToBlock)) + + //3.1 ensure that chain is reorged and latest block number is greater than the block number when request was made + require.Greater(t, latestBlockNumberAfterReorg, randomWordsRequestedEvent.Raw.BlockNumber) + + //3.2 ensure that chain is reorged and latest block number is less than the block number when fulfilment was performed + require.Less(t, latestBlockNumberAfterReorg, randomWordsFulfilledEventOnReorgedFork.Raw.BlockNumber) + + //4. wait for the fulfillment which VRF Node will generate for Canonical chain + _, err = vrfv2plus.WaitRandomWordsFulfilledEvent( + vrfContracts.CoordinatorV2Plus, + randomWordsRequestedEvent.RequestId, + subID, + isNativeBilling, + configCopy.VRFv2Plus.General.RandomWordsFulfilledEventTimeout.Duration, + l, + ) + require.NoError(t, err, "error waiting for randomness fulfilled event") + }) + + t.Run("Reorg on rand request", func(t *testing.T) { + configCopy := config.MustCopy().(tc.TestConfig) + //1. set minimum confirmations to higher value so that we can be sure that request won't be fulfilled before reorg + configCopy.VRFv2Plus.General.MinimumConfirmations = ptr.Ptr[uint16](6) + + //2. request randomness + randomWordsRequestedEvent, err := vrfv2plus.RequestRandomness( + consumers[0], + vrfContracts.CoordinatorV2Plus, + vrfKey, + subID, + isNativeBilling, + configCopy.VRFv2Plus.General, + l, + ) + require.NoError(t, err) + + // rewind chain to block number before the randomness request was made + rewindChainToBlockNumber := randomWordsRequestedEvent.Raw.BlockNumber - 3 + + rpcUrl, err := actions.GetRPCUrl(env, chainID) + require.NoError(t, err, "error getting rpc url") + + //3. rewind chain by n number of blocks - basically, mimicking reorg scenario + latestBlockNumberAfterReorg, err := actions.RewindSimulatedChainToBlockNumber(testcontext.Get(t), evmClient, rpcUrl, rewindChainToBlockNumber, l) + require.NoError(t, err, fmt.Sprintf("error rewinding chain to block number %d", rewindChainToBlockNumber)) + + //4. ensure that chain is reorged and latest block number is less than the block number when request was made + require.Less(t, latestBlockNumberAfterReorg, randomWordsRequestedEvent.Raw.BlockNumber) + + //5. ensure that rand request is not fulfilled for the request which was made on reorged fork + // For context - when performing debug_setHead on geth simulated chain and therefore rewinding chain to a previous block, + //then tx that was mined after reorg will not appear in canonical chain contrary to real world scenario + //Hence, we only verify that VRF node will not generate fulfillment for the reorged fork request + _, err = vrfContracts.CoordinatorV2Plus.WaitForRandomWordsFulfilledEvent( + contracts.RandomWordsFulfilledEventFilter{ + RequestIds: []*big.Int{randomWordsRequestedEvent.RequestId}, + SubIDs: []*big.Int{subID}, + Timeout: time.Second * 10, + }, + ) + require.Error(t, err, "fulfillment should not be generated for the request which was made on reorged fork on Simulated Chain") + }) + +}