From 2a0c7b80e98d96af2d0359da84edd01638eee1d2 Mon Sep 17 00:00:00 2001 From: Ilja Pavlovs Date: Thu, 18 Apr 2024 23:53:42 +0300 Subject: [PATCH] VRF-1025: adjust BHS load test to be used as a smoke test (#12884) * VRF-1025: adjust BHS load test to be used as a smoke test * VRF-1025: fix lint * VRF-1025: reverting change of log level; fixing lint --- .../on-demand-vrfv2-performance-test.yml | 6 +- .../on-demand-vrfv2plus-performance-test.yml | 9 +- .../actions/vrf/vrfv2plus/contract_steps.go | 5 +- integration-tests/load/vrfv2/vrfv2_test.go | 112 ++++++++++-------- .../load/vrfv2plus/vrfv2plus_test.go | 106 ++++++++++------- integration-tests/smoke/vrfv2_test.go | 8 +- integration-tests/smoke/vrfv2plus_test.go | 17 +-- 7 files changed, 150 insertions(+), 113 deletions(-) diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml index 860120972d5..f520e2307d9 100644 --- a/.github/workflows/on-demand-vrfv2-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2-performance-test.yml @@ -15,6 +15,10 @@ on: - "Load" - "Stress" - "Spike" + test_list_regex: + description: "Regex for tests to run" + required: false + default: "(TestVRFV2Performance)" jobs: vrfv2_performance_test: name: VRFV2 Performance Test @@ -71,7 +75,7 @@ jobs: - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 with: - test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run TestVRFV2Performance ./vrfv2 + test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2 test_download_vendor_packages_command: cd ./integration-tests && go mod download cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml index 463a7512b0c..16d37617a68 100644 --- a/.github/workflows/on-demand-vrfv2plus-performance-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-performance-test.yml @@ -14,8 +14,11 @@ on: - "Soak" - "Load" - "Stress" - - "Spike" - + - "Spike" + test_list_regex: + description: "Regex for tests to run" + required: false + default: "(TestVRFV2PlusPerformance)" jobs: vrfv2plus_performance_test: name: VRFV2 Plus Performance Test @@ -72,7 +75,7 @@ jobs: - name: Run Tests uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11 with: - test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run TestVRFV2PlusPerformance ./vrfv2plus + test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2plus test_download_vendor_packages_command: cd ./integration-tests && go mod download cl_repo: ${{ env.CHAINLINK_IMAGE }} cl_image_tag: ${{ env.CHAINLINK_VERSION }} diff --git a/integration-tests/actions/vrf/vrfv2plus/contract_steps.go b/integration-tests/actions/vrf/vrfv2plus/contract_steps.go index 5df33a6d997..acc509fb58b 100644 --- a/integration-tests/actions/vrf/vrfv2plus/contract_steps.go +++ b/integration-tests/actions/vrf/vrfv2plus/contract_steps.go @@ -762,15 +762,12 @@ func SetupNewConsumersAndSubs( ) ([]contracts.VRFv2PlusLoadTestConsumer, []*big.Int, error) { consumers, err := DeployVRFV2PlusConsumers(env.ContractDeployer, coordinator, consumerContractsAmount) if err != nil { - if err != nil { - return nil, nil, fmt.Errorf("err: %w", err) - } + return nil, nil, fmt.Errorf("err: %w", err) } evmClient, err := env.GetEVMClient(chainID) if err != nil { return nil, nil, err } - err = evmClient.WaitForEvents() if err != nil { return nil, nil, fmt.Errorf("%s, err: %w", vrfcommon.ErrWaitTXsComplete, err) diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go index 0edf35df8fa..b27395769db 100644 --- a/integration-tests/load/vrfv2/vrfv2_test.go +++ b/integration-tests/load/vrfv2/vrfv2_test.go @@ -26,12 +26,6 @@ import ( ) var ( - testEnv *test_env.CLClusterTestEnv - vrfContracts *vrfcommon.VRFContracts - vrfKey *vrfcommon.VRFKeyData - subIDs []uint64 - eoaWalletAddress string - labels = map[string]string{ "branch": "vrfv2_healthcheck", "commit": "vrfv2_healthcheck", @@ -39,24 +33,30 @@ var ( ) func TestVRFV2Performance(t *testing.T) { + var ( + testEnv *test_env.CLClusterTestEnv + vrfContracts *vrfcommon.VRFContracts + subIDsForCancellingAfterTest []uint64 + defaultWalletAddress string + vrfKey *vrfcommon.VRFKeyData + ) l := logging.GetTestLogger(t) - testType, err := tc.GetConfigurationNameFromEnv() require.NoError(t, err) testConfig, err := tc.GetConfig(testType, tc.VRFv2) require.NoError(t, err) + cfgl := testConfig.Logging.Loki - testReporter := &testreporters.VRFV2TestReporter{} vrfv2Config := testConfig.VRFv2 + testReporter := &testreporters.VRFV2TestReporter{} - cfgl := testConfig.Logging.Loki lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) lc, err := wasp.NewLokiClient(lokiConfig) if err != nil { l.Error().Err(err).Msg(ErrLokiClient) return } - + chainID := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0].ChainID updatedLabels := UpdateLabels(labels, t) l.Info(). @@ -68,9 +68,6 @@ func TestVRFV2Performance(t *testing.T) { Uint16("RandomnessRequestCountPerRequestDeviation", *vrfv2Config.General.RandomnessRequestCountPerRequestDeviation). Bool("UseExistingEnv", *vrfv2Config.General.UseExistingEnv). Msg("Performance Test Configuration") - - chainID := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0].ChainID - cleanupFn := func() { teardown(t, vrfContracts.VRFV2Consumers[0], lc, updatedLabels, testReporter, testType, &testConfig) @@ -84,7 +81,7 @@ func TestVRFV2Performance(t *testing.T) { } else { if *vrfv2Config.General.CancelSubsAfterTestRun { //cancel subs and return funds to sub owner - vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, eoaWalletAddress, subIDs, l) + vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l) } } if !*vrfv2Config.General.UseExistingEnv { @@ -108,7 +105,7 @@ func TestVRFV2Performance(t *testing.T) { require.NoError(t, err, "error getting EVM client") var consumers []contracts.VRFv2LoadTestConsumer - subIDs, consumers, err = vrfv2.SetupSubsAndConsumersForExistingEnv( + subIDs, consumers, err := vrfv2.SetupSubsAndConsumersForExistingEnv( testEnv, chainID, vrfContracts.CoordinatorV2, @@ -118,16 +115,17 @@ func TestVRFV2Performance(t *testing.T) { testConfig, l, ) - vrfContracts.VRFV2Consumers = consumers - - eoaWalletAddress = evmClient.GetDefaultWallet().Address() - - l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") + require.NoError(t, err, "error setting up new consumers and subs") for _, subID := range subIDs { subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) require.NoError(t, err, "error getting subscription information for subscription %d", subID) vrfv2.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2) } + subIDsForCancellingAfterTest = subIDs + l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") + + vrfContracts.VRFV2Consumers = consumers + defaultWalletAddress = evmClient.GetDefaultWallet().Address() // is our "job" stable at all, no memory leaks, no flaking performance under some RPS? t.Run("vrfv2 performance test", func(t *testing.T) { @@ -177,16 +175,21 @@ func TestVRFV2Performance(t *testing.T) { } func TestVRFV2BHSPerformance(t *testing.T) { + var ( + testEnv *test_env.CLClusterTestEnv + vrfContracts *vrfcommon.VRFContracts + subIDsForCancellingAfterTest []uint64 + defaultWalletAddress string + vrfKey *vrfcommon.VRFKeyData + ) l := logging.GetTestLogger(t) testType, err := tc.GetConfigurationNameFromEnv() require.NoError(t, err) testConfig, err := tc.GetConfig(testType, tc.VRFv2) require.NoError(t, err) - - testReporter := &testreporters.VRFV2TestReporter{} vrfv2Config := testConfig.VRFv2 - + testReporter := &testreporters.VRFV2TestReporter{} cfgl := testConfig.Logging.Loki lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) lc, err := wasp.NewLokiClient(lokiConfig) @@ -222,7 +225,7 @@ func TestVRFV2BHSPerformance(t *testing.T) { } else { if *vrfv2Config.General.CancelSubsAfterTestRun { //cancel subs and return funds to sub owner - vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, eoaWalletAddress, subIDs, l) + vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l) } } if !*vrfv2Config.General.UseExistingEnv { @@ -245,43 +248,29 @@ func TestVRFV2BHSPerformance(t *testing.T) { evmClient, err := testEnv.GetEVMClient(chainID) require.NoError(t, err, "error getting EVM client") - var consumers []contracts.VRFv2LoadTestConsumer - subIDs, consumers, err = vrfv2.SetupSubsAndConsumersForExistingEnv( - testEnv, - chainID, - vrfContracts.CoordinatorV2, - vrfContracts.LinkToken, - 1, - *vrfv2Config.General.NumberOfSubToCreate, - testConfig, - l, - ) - vrfContracts.VRFV2Consumers = consumers - - eoaWalletAddress = evmClient.GetDefaultWallet().Address() - - l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") - for _, subID := range subIDs { - subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) - require.NoError(t, err, "error getting subscription information for subscription %d", subID) - vrfv2.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2) - } - + defaultWalletAddress = evmClient.GetDefaultWallet().Address() t.Run("vrfv2 and bhs performance test", func(t *testing.T) { configCopy := testConfig.MustCopy().(tc.TestConfig) //Underfund Subscription configCopy.VRFv2.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0)) - consumers, subIDs, err = vrfv2.SetupNewConsumersAndSubs( + underfundedSubIDs, consumers, err := vrfv2.SetupSubsAndConsumersForExistingEnv( testEnv, chainID, vrfContracts.CoordinatorV2, - configCopy, vrfContracts.LinkToken, 1, - *configCopy.VRFv2.General.NumberOfSubToCreate, + *vrfv2Config.General.NumberOfSubToCreate, + configCopy, l, ) - require.NoError(t, err, "error setting up new consumers and subscriptions") + require.NoError(t, err, "error setting up new consumers and subs") + for _, subID := range underfundedSubIDs { + subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID) + require.NoError(t, err, "error getting subscription information for subscription %d", subID) + vrfv2.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2) + } + subIDsForCancellingAfterTest = underfundedSubIDs + l.Debug().Int("Number of Subs", len(underfundedSubIDs)).Msg("Subs involved in the test") vrfContracts.VRFV2Consumers = consumers require.Len(t, vrfContracts.VRFV2Consumers, 1, "only one consumer should be created for Load Test") err = vrfContracts.VRFV2Consumers[0].ResetMetrics() @@ -296,7 +285,7 @@ func TestVRFV2BHSPerformance(t *testing.T) { Gun: NewBHSTestGun( vrfContracts, vrfKey.KeyHash, - subIDs, + underfundedSubIDs, configCopy.VRFv2, l, ), @@ -322,8 +311,27 @@ func TestVRFV2BHSPerformance(t *testing.T) { _, err = actions.WaitForBlockNumberToBe(latestBlockNumber+uint64(256), evmClient, &wgBlockNumberTobe, configCopy.VRFv2.General.WaitFor256BlocksTimeout.Duration, t) wgBlockNumberTobe.Wait() require.NoError(t, err, "error waiting for block number to be") - err = vrfv2.FundSubscriptions(testEnv, chainID, big.NewFloat(*configCopy.VRFv2.General.SubscriptionRefundingAmountLink), vrfContracts.LinkToken, vrfContracts.CoordinatorV2, subIDs) + + metrics, err := consumers[0].GetLoadTestMetrics(testcontext.Get(t)) + require.NoError(t, err) + require.Equal(t, 0, metrics.FulfilmentCount.Cmp(big.NewInt(0)), "Fulfilment count should be 0 since sub is underfunded. Check if the sub is actually funded") + + var subIDsString []uint64 + subIDsString = append(subIDsString, underfundedSubIDs...) + l.Info(). + Float64("SubscriptionRefundingAmountLink", *configCopy.VRFv2.General.SubscriptionRefundingAmountLink). + Uints64("SubIDs", subIDsString). + Msg("Funding Subscriptions with Link and Native Tokens") + err = vrfv2.FundSubscriptions( + testEnv, + chainID, + big.NewFloat(*configCopy.VRFv2.General.SubscriptionRefundingAmountLink), + vrfContracts.LinkToken, + vrfContracts.CoordinatorV2, + underfundedSubIDs, + ) require.NoError(t, err, "error funding subscriptions") + var wgAllRequestsFulfilled sync.WaitGroup wgAllRequestsFulfilled.Add(1) requestCount, fulfilmentCount, err := vrfcommon.WaitForRequestCountEqualToFulfilmentCount(testcontext.Get(t), vrfContracts.VRFV2Consumers[0], 2*time.Minute, &wgAllRequestsFulfilled) diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go index 1589123c77e..bb9c33f43a6 100644 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go @@ -26,12 +26,6 @@ import ( ) var ( - testEnv *test_env.CLClusterTestEnv - vrfContracts *vrfcommon.VRFContracts - vrfKey *vrfcommon.VRFKeyData - subIDs []*big.Int - eoaWalletAddress string - labels = map[string]string{ "branch": "vrfv2Plus_healthcheck", "commit": "vrfv2Plus_healthcheck", @@ -39,8 +33,14 @@ var ( ) func TestVRFV2PlusPerformance(t *testing.T) { + var ( + testEnv *test_env.CLClusterTestEnv + vrfContracts *vrfcommon.VRFContracts + subIDsForCancellingAfterTest []*big.Int + defaultWalletAddress string + vrfKey *vrfcommon.VRFKeyData + ) l := logging.GetTestLogger(t) - testType, err := tc.GetConfigurationNameFromEnv() require.NoError(t, err) testConfig, err := tc.GetConfig(testType, tc.VRFv2Plus) @@ -56,8 +56,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { l.Error().Err(err).Msg(ErrLokiClient) return } - - networkConfig := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0] + chainID := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0].ChainID updatedLabels := UpdateLabels(labels, t) l.Info(). @@ -73,8 +72,8 @@ func TestVRFV2PlusPerformance(t *testing.T) { cleanupFn := func() { teardown(t, vrfContracts.VRFV2PlusConsumer[0], lc, updatedLabels, testReporter, testType, &testConfig) - evmClient, err := testEnv.GetEVMClient(networkConfig.ChainID) - require.NoError(t, err, "Getting EVM client shouldn't fail") + evmClient, err := testEnv.GetEVMClient(chainID) + require.NoError(t, err, "error getting EVM client") if evmClient.NetworkSimulated() { l.Info(). @@ -83,7 +82,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { } else { if *testConfig.VRFv2Plus.General.CancelSubsAfterTestRun { //cancel subs and return funds to sub owner - vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, eoaWalletAddress, subIDs, l) + vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l) } } if !*testConfig.VRFv2Plus.General.UseExistingEnv { @@ -98,16 +97,16 @@ func TestVRFV2PlusPerformance(t *testing.T) { NumberOfTxKeysToCreate: *vrfv2PlusConfig.General.NumberOfSendingKeysToCreate, } - testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, networkConfig.ChainID, cleanupFn, newEnvConfig, l) + testEnv, vrfContracts, vrfKey, _, err = vrfv2plus.SetupVRFV2PlusUniverse(testcontext.Get(t), t, testConfig, chainID, cleanupFn, newEnvConfig, l) require.NoError(t, err, "error setting up VRFV2Plus universe") - evmClient, err := testEnv.GetEVMClient(networkConfig.ChainID) + evmClient, err := testEnv.GetEVMClient(chainID) require.NoError(t, err, "Getting EVM client shouldn't fail") var consumers []contracts.VRFv2PlusLoadTestConsumer - subIDs, consumers, err = vrfv2plus.SetupSubsAndConsumersForExistingEnv( + subIDs, consumers, err := vrfv2plus.SetupSubsAndConsumersForExistingEnv( testEnv, - networkConfig.ChainID, + chainID, vrfContracts.CoordinatorV2Plus, vrfContracts.LinkToken, 1, @@ -115,19 +114,26 @@ func TestVRFV2PlusPerformance(t *testing.T) { testConfig, l, ) - vrfContracts.VRFV2PlusConsumer = consumers - - eoaWalletAddress = evmClient.GetDefaultWallet().Address() - - l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") + require.NoError(t, err, "error setting up new consumers and subs") for _, subID := range subIDs { subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) require.NoError(t, err, "error getting subscription information for subscription %s", subID.String()) vrfv2plus.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2Plus) } + subIDsForCancellingAfterTest = subIDs + l.Info().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") + + vrfContracts.VRFV2PlusConsumer = consumers + defaultWalletAddress = evmClient.GetDefaultWallet().Address() // is our "job" stable at all, no memory leaks, no flaking performance under some RPS? t.Run("vrfv2plus performance test", func(t *testing.T) { + require.Len(t, vrfContracts.VRFV2PlusConsumer, 1, "only one consumer should be created for Load Test") + consumer := vrfContracts.VRFV2PlusConsumer[0] + err = consumer.ResetMetrics() + require.NoError(t, err) + MonitorLoadStats(testcontext.Get(t), lc, consumer, updatedLabels) + singleFeedConfig := &wasp.Config{ T: t, LoadType: wasp.RPS, @@ -144,11 +150,6 @@ func TestVRFV2PlusPerformance(t *testing.T) { LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), CallTimeout: 2 * time.Minute, } - require.Len(t, vrfContracts.VRFV2PlusConsumer, 1, "only one consumer should be created for Load Test") - consumer := vrfContracts.VRFV2PlusConsumer[0] - err = consumer.ResetMetrics() - require.NoError(t, err) - MonitorLoadStats(testcontext.Get(t), lc, consumer, updatedLabels) singleFeedConfig.Schedule = wasp.Plain( *vrfv2PlusConfig.Performance.RPS, @@ -174,17 +175,22 @@ func TestVRFV2PlusPerformance(t *testing.T) { } func TestVRFV2PlusBHSPerformance(t *testing.T) { + var ( + testEnv *test_env.CLClusterTestEnv + vrfContracts *vrfcommon.VRFContracts + subIDsForCancellingAfterTest []*big.Int + defaultWalletAddress string + vrfKey *vrfcommon.VRFKeyData + ) l := logging.GetTestLogger(t) testType, err := tc.GetConfigurationNameFromEnv() require.NoError(t, err) testConfig, err := tc.GetConfig(testType, tc.VRFv2Plus) require.NoError(t, err) - cfgl := testConfig.Logging.Loki - vrfv2PlusConfig := testConfig.VRFv2Plus testReporter := &testreporters.VRFV2PlusTestReporter{} - + cfgl := testConfig.Logging.Loki lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken) lc, err := wasp.NewLokiClient(lokiConfig) if err != nil { @@ -219,7 +225,7 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { } else { if *testConfig.VRFv2Plus.General.CancelSubsAfterTestRun { //cancel subs and return funds to sub owner - vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, eoaWalletAddress, subIDs, l) + vrfv2plus.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l) } } if !*testConfig.VRFv2Plus.General.UseExistingEnv { @@ -240,30 +246,32 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { evmClient, err := testEnv.GetEVMClient(chainID) require.NoError(t, err, "error getting EVM client") - eoaWalletAddress = evmClient.GetDefaultWallet().Address() - - l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test") - for _, subID := range subIDs { - subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) - require.NoError(t, err, "error getting subscription information for subscription %s", subID.String()) - vrfv2plus.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2Plus) - } + defaultWalletAddress = evmClient.GetDefaultWallet().Address() t.Run("vrfv2plus and bhs performance test", func(t *testing.T) { configCopy := testConfig.MustCopy().(tc.TestConfig) //Underfund Subscription configCopy.VRFv2Plus.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0)) - consumers, underfundedSubIDs, err := vrfv2plus.SetupNewConsumersAndSubs( + configCopy.VRFv2Plus.General.SubscriptionFundingAmountNative = ptr.Ptr(float64(0)) + + underfundedSubIDs, consumers, err := vrfv2plus.SetupSubsAndConsumersForExistingEnv( testEnv, chainID, vrfContracts.CoordinatorV2Plus, - configCopy, vrfContracts.LinkToken, 1, - *configCopy.VRFv2Plus.General.NumberOfSubToCreate, + *vrfv2PlusConfig.General.NumberOfSubToCreate, + configCopy, l, ) require.NoError(t, err, "error setting up new consumers and subs for Load Test") + for _, subID := range underfundedSubIDs { + subscription, err := vrfContracts.CoordinatorV2Plus.GetSubscription(testcontext.Get(t), subID) + require.NoError(t, err, "error getting subscription information for subscription %s", subID.String()) + vrfv2plus.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2Plus) + } + subIDsForCancellingAfterTest = underfundedSubIDs + l.Debug().Int("Number of Subs", len(underfundedSubIDs)).Msg("Subs involved in the test") vrfContracts.VRFV2PlusConsumer = consumers require.Len(t, vrfContracts.VRFV2PlusConsumer, 1, "only one consumer should be created for Load Test") consumer := vrfContracts.VRFV2PlusConsumer[0] @@ -306,6 +314,20 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { wgBlockNumberTobe.Wait() require.NoError(t, err, "error waiting for block number to be") + metrics, err := consumers[0].GetLoadTestMetrics(testcontext.Get(t)) + require.NoError(t, err) + require.Equal(t, 0, metrics.FulfilmentCount.Cmp(big.NewInt(0)), "Fulfilment count should be 0 since sub is underfunded. Check if the sub is actually funded") + + var subIDsString []string + for _, subID := range underfundedSubIDs { + subIDsString = append(subIDsString, subID.String()) + } + + l.Info(). + Float64("SubscriptionRefundingAmountNative", *configCopy.VRFv2Plus.General.SubscriptionRefundingAmountNative). + Float64("SubscriptionRefundingAmountLink", *configCopy.VRFv2Plus.General.SubscriptionRefundingAmountLink). + Strs("SubIDs", subIDsString). + Msg("Funding Subscriptions with Link and Native Tokens") err = vrfv2plus.FundSubscriptions( testEnv, chainID, @@ -313,7 +335,7 @@ func TestVRFV2PlusBHSPerformance(t *testing.T) { big.NewFloat(*configCopy.VRFv2Plus.General.SubscriptionRefundingAmountLink), vrfContracts.LinkToken, vrfContracts.CoordinatorV2Plus, - subIDs, + underfundedSubIDs, ) require.NoError(t, err, "error funding subscriptions") diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index 9834cd77973..a834d1be757 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -126,7 +126,7 @@ func TestVRFv2Basic(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") require.Equal(t, *configCopy.VRFv2.General.NumberOfWords, uint32(len(status.RandomWords))) for _, w := range status.RandomWords { @@ -748,7 +748,7 @@ func TestVRFOwner(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") require.Equal(t, *configCopy.VRFv2.General.NumberOfWords, uint32(len(status.RandomWords))) for _, w := range status.RandomWords { @@ -886,7 +886,7 @@ func TestVRFV2WithBHS(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") }) t.Run("BHS Job should fill in blockhashes into BHS contract for unfulfilled requests", func(t *testing.T) { @@ -949,7 +949,7 @@ func TestVRFV2WithBHS(t *testing.T) { gom.Eventually(func(g gomega.Gomega) { clNodeTxs, _, err = nodeTypeToNodeMap[vrfcommon.BHS].CLNode.API.ReadTransactions() g.Expect(err).ShouldNot(gomega.HaveOccurred(), "error getting CL Node transactions") - l.Debug().Int("Number of TXs", len(clNodeTxs.Data)).Msg("BHS Node txs") + l.Info().Int("Number of TXs", len(clNodeTxs.Data)).Msg("BHS Node txs") g.Expect(len(clNodeTxs.Data)).Should(gomega.BeNumerically("==", 1), "Expected 1 tx posted by BHS Node, but found %d", len(clNodeTxs.Data)) txHash = clNodeTxs.Data[0].Attributes.Hash }, "2m", "1s").Should(gomega.Succeed()) diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index c5a35704b1a..2f58c2d0003 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -136,7 +136,7 @@ func TestVRFv2Plus(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") require.Equal(t, *configCopy.VRFv2Plus.General.NumberOfWords, uint32(len(status.RandomWords))) for _, w := range status.RandomWords { @@ -198,7 +198,7 @@ func TestVRFv2Plus(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") require.Equal(t, *testConfig.NumberOfWords, uint32(len(status.RandomWords))) for _, w := range status.RandomWords { @@ -939,7 +939,7 @@ func TestVRFv2PlusMigration(t *testing.T) { coordinatorAddressInConsumerAfterMigration, err := consumer.GetCoordinator(testcontext.Get(t)) require.NoError(t, err, "error getting Coordinator from Consumer contract") require.Equal(t, newCoordinator.Address(), coordinatorAddressInConsumerAfterMigration.String()) - l.Debug(). + l.Info(). Str("Consumer", consumer.Address()). Str("Coordinator", coordinatorAddressInConsumerAfterMigration.String()). Msg("Coordinator Address in Consumer After Migration") @@ -1123,7 +1123,7 @@ func TestVRFv2PlusMigration(t *testing.T) { coordinatorAddressInConsumerAfterMigration, err := wrapperContracts.VRFV2PlusWrapper.Coordinator(testcontext.Get(t)) require.NoError(t, err, "error getting Coordinator from Consumer contract- VRFV2PlusWrapper") require.Equal(t, newCoordinator.Address(), coordinatorAddressInConsumerAfterMigration.String()) - l.Debug(). + l.Info(). Str("Consumer-VRFV2PlusWrapper", wrapperContracts.VRFV2PlusWrapper.Address()). Str("Coordinator", coordinatorAddressInConsumerAfterMigration.String()). Msg("Coordinator Address in VRFV2PlusWrapper After Migration") @@ -1318,7 +1318,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") randRequestBlockHash, err := vrfContracts.BHS.GetBlockHash(testcontext.Get(t), big.NewInt(int64(randRequestBlockNumber))) require.NoError(t, err, "error getting blockhash for a blocknumber which was stored in BHS contract") @@ -1385,6 +1385,9 @@ func TestVRFV2PlusWithBHS(t *testing.T) { err = evmClient.WaitForEvents() require.NoError(t, err, vrfcommon.ErrWaitTXsComplete) + metrics, err := consumers[0].GetLoadTestMetrics(testcontext.Get(t)) + require.Equal(t, 0, metrics.RequestCount.Cmp(big.NewInt(1))) + require.Equal(t, 0, metrics.FulfilmentCount.Cmp(big.NewInt(0))) var clNodeTxs *client.TransactionsData var txHash string @@ -1392,7 +1395,7 @@ func TestVRFV2PlusWithBHS(t *testing.T) { gom.Eventually(func(g gomega.Gomega) { clNodeTxs, _, err = nodeTypeToNodeMap[vrfcommon.BHS].CLNode.API.ReadTransactions() g.Expect(err).ShouldNot(gomega.HaveOccurred(), "error getting CL Node transactions") - l.Debug().Int("Number of TXs", len(clNodeTxs.Data)).Msg("BHS Node txs") + l.Info().Int("Number of TXs", len(clNodeTxs.Data)).Msg("BHS Node txs") g.Expect(len(clNodeTxs.Data)).Should(gomega.BeNumerically("==", 1), "Expected 1 tx posted by BHS Node, but found %d", len(clNodeTxs.Data)) txHash = clNodeTxs.Data[0].Attributes.Hash }, "2m", "1s").Should(gomega.Succeed()) @@ -1556,7 +1559,7 @@ func TestVRFV2PlusWithBHF(t *testing.T) { status, err := consumers[0].GetRequestStatus(testcontext.Get(t), randomWordsFulfilledEvent.RequestId) require.NoError(t, err, "error getting rand request status") require.True(t, status.Fulfilled) - l.Debug().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") + l.Info().Bool("Fulfilment Status", status.Fulfilled).Msg("Random Words Request Fulfilment Status") clNodeTxs, _, err := nodeTypeToNodeMap[vrfcommon.BHF].CLNode.API.ReadTransactions() require.NoError(t, err, "error fetching txns from BHF node")