VRF-1025: adjust BHS load test to be used as a smoke test (#12884)

* VRF-1025: adjust BHS load test to be used as a smoke test * VRF-1025: fix lint * VRF-1025: reverting change of log level; fixing lint
smartcontractkit · Apr 18, 2024 · 2a0c7b8 · 2a0c7b8
1 parent 2b38bd8
commit 2a0c7b8
Show file tree

Hide file tree

Showing 7 changed files with 150 additions and 113 deletions.
diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml
@@ -15,6 +15,10 @@ on:
           - "Load"
           - "Stress"
           - "Spike"
+      test_list_regex:
+        description: "Regex for tests to run"
+        required: false
+        default: "(TestVRFV2Performance)"
 jobs:
   vrfv2_performance_test:
     name: VRFV2 Performance Test
@@ -71,7 +75,7 @@ jobs:
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
         with:
-          test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run TestVRFV2Performance ./vrfv2
+          test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}

diff --git a/.github/workflows/on-demand-vrfv2plus-performance-test.yml b/.github/workflows/on-demand-vrfv2plus-performance-test.yml
@@ -14,8 +14,11 @@ on:
           - "Soak"
           - "Load"
           - "Stress"
-          - "Spike"          
-
+          - "Spike"
+      test_list_regex:
+        description: "Regex for tests to run"
+        required: false
+        default: "(TestVRFV2PlusPerformance)"
 jobs:
   vrfv2plus_performance_test:
     name: VRFV2 Plus Performance Test
@@ -72,7 +75,7 @@ jobs:
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
         with:
-          test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run TestVRFV2PlusPerformance ./vrfv2plus
+          test_command_to_run: cd ./integration-tests/load && go test -v -count=1 -timeout 24h -run "${{ inputs.test_list_regex }}" ./vrfv2plus
           test_download_vendor_packages_command: cd ./integration-tests && go mod download
           cl_repo: ${{ env.CHAINLINK_IMAGE }}
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}

diff --git a/integration-tests/actions/vrf/vrfv2plus/contract_steps.go b/integration-tests/actions/vrf/vrfv2plus/contract_steps.go
@@ -762,15 +762,12 @@ func SetupNewConsumersAndSubs(
 ) ([]contracts.VRFv2PlusLoadTestConsumer, []*big.Int, error) {
 	consumers, err := DeployVRFV2PlusConsumers(env.ContractDeployer, coordinator, consumerContractsAmount)
 	if err != nil {
-		if err != nil {
-			return nil, nil, fmt.Errorf("err: %w", err)
-		}
+		return nil, nil, fmt.Errorf("err: %w", err)
 	}
 	evmClient, err := env.GetEVMClient(chainID)
 	if err != nil {
 		return nil, nil, err
 	}
-
 	err = evmClient.WaitForEvents()
 	if err != nil {
 		return nil, nil, fmt.Errorf("%s, err: %w", vrfcommon.ErrWaitTXsComplete, err)

diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go
@@ -26,37 +26,37 @@ import (
 )
 
 var (
-	testEnv          *test_env.CLClusterTestEnv
-	vrfContracts     *vrfcommon.VRFContracts
-	vrfKey           *vrfcommon.VRFKeyData
-	subIDs           []uint64
-	eoaWalletAddress string
-
 	labels = map[string]string{
 		"branch": "vrfv2_healthcheck",
 		"commit": "vrfv2_healthcheck",
 	}
 )
 
 func TestVRFV2Performance(t *testing.T) {
+	var (
+		testEnv                      *test_env.CLClusterTestEnv
+		vrfContracts                 *vrfcommon.VRFContracts
+		subIDsForCancellingAfterTest []uint64
+		defaultWalletAddress         string
+		vrfKey                       *vrfcommon.VRFKeyData
+	)
 	l := logging.GetTestLogger(t)
-
 	testType, err := tc.GetConfigurationNameFromEnv()
 	require.NoError(t, err)
 	testConfig, err := tc.GetConfig(testType, tc.VRFv2)
 	require.NoError(t, err)
+	cfgl := testConfig.Logging.Loki
 
-	testReporter := &testreporters.VRFV2TestReporter{}
 	vrfv2Config := testConfig.VRFv2
+	testReporter := &testreporters.VRFV2TestReporter{}
 
-	cfgl := testConfig.Logging.Loki
 	lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)
 	lc, err := wasp.NewLokiClient(lokiConfig)
 	if err != nil {
 		l.Error().Err(err).Msg(ErrLokiClient)
 		return
 	}
-
+	chainID := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0].ChainID
 	updatedLabels := UpdateLabels(labels, t)
 
 	l.Info().
@@ -68,9 +68,6 @@ func TestVRFV2Performance(t *testing.T) {
 		Uint16("RandomnessRequestCountPerRequestDeviation", *vrfv2Config.General.RandomnessRequestCountPerRequestDeviation).
 		Bool("UseExistingEnv", *vrfv2Config.General.UseExistingEnv).
 		Msg("Performance Test Configuration")
-
-	chainID := networks.MustGetSelectedNetworkConfig(testConfig.GetNetworkConfig())[0].ChainID
-
 	cleanupFn := func() {
 		teardown(t, vrfContracts.VRFV2Consumers[0], lc, updatedLabels, testReporter, testType, &testConfig)
 
@@ -84,7 +81,7 @@ func TestVRFV2Performance(t *testing.T) {
 		} else {
 			if *vrfv2Config.General.CancelSubsAfterTestRun {
 				//cancel subs and return funds to sub owner
-				vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, eoaWalletAddress, subIDs, l)
+				vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l)
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
@@ -108,7 +105,7 @@ func TestVRFV2Performance(t *testing.T) {
 	require.NoError(t, err, "error getting EVM client")
 
 	var consumers []contracts.VRFv2LoadTestConsumer
-	subIDs, consumers, err = vrfv2.SetupSubsAndConsumersForExistingEnv(
+	subIDs, consumers, err := vrfv2.SetupSubsAndConsumersForExistingEnv(
 		testEnv,
 		chainID,
 		vrfContracts.CoordinatorV2,
@@ -118,16 +115,17 @@ func TestVRFV2Performance(t *testing.T) {
 		testConfig,
 		l,
 	)
-	vrfContracts.VRFV2Consumers = consumers
-
-	eoaWalletAddress = evmClient.GetDefaultWallet().Address()
-
-	l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test")
+	require.NoError(t, err, "error setting up new consumers and subs")
 	for _, subID := range subIDs {
 		subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID)
 		require.NoError(t, err, "error getting subscription information for subscription %d", subID)
 		vrfv2.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2)
 	}
+	subIDsForCancellingAfterTest = subIDs
+	l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test")
+
+	vrfContracts.VRFV2Consumers = consumers
+	defaultWalletAddress = evmClient.GetDefaultWallet().Address()
 
 	// is our "job" stable at all, no memory leaks, no flaking performance under some RPS?
 	t.Run("vrfv2 performance test", func(t *testing.T) {
@@ -177,16 +175,21 @@ func TestVRFV2Performance(t *testing.T) {
 }
 
 func TestVRFV2BHSPerformance(t *testing.T) {
+	var (
+		testEnv                      *test_env.CLClusterTestEnv
+		vrfContracts                 *vrfcommon.VRFContracts
+		subIDsForCancellingAfterTest []uint64
+		defaultWalletAddress         string
+		vrfKey                       *vrfcommon.VRFKeyData
+	)
 	l := logging.GetTestLogger(t)
 
 	testType, err := tc.GetConfigurationNameFromEnv()
 	require.NoError(t, err)
 	testConfig, err := tc.GetConfig(testType, tc.VRFv2)
 	require.NoError(t, err)
-
-	testReporter := &testreporters.VRFV2TestReporter{}
 	vrfv2Config := testConfig.VRFv2
-
+	testReporter := &testreporters.VRFV2TestReporter{}
 	cfgl := testConfig.Logging.Loki
 	lokiConfig := wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken)
 	lc, err := wasp.NewLokiClient(lokiConfig)
@@ -222,7 +225,7 @@ func TestVRFV2BHSPerformance(t *testing.T) {
 		} else {
 			if *vrfv2Config.General.CancelSubsAfterTestRun {
 				//cancel subs and return funds to sub owner
-				vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, eoaWalletAddress, subIDs, l)
+				vrfv2.CancelSubsAndReturnFunds(testcontext.Get(t), vrfContracts, defaultWalletAddress, subIDsForCancellingAfterTest, l)
 			}
 		}
 		if !*vrfv2Config.General.UseExistingEnv {
@@ -245,43 +248,29 @@ func TestVRFV2BHSPerformance(t *testing.T) {
 	evmClient, err := testEnv.GetEVMClient(chainID)
 	require.NoError(t, err, "error getting EVM client")
 
-	var consumers []contracts.VRFv2LoadTestConsumer
-	subIDs, consumers, err = vrfv2.SetupSubsAndConsumersForExistingEnv(
-		testEnv,
-		chainID,
-		vrfContracts.CoordinatorV2,
-		vrfContracts.LinkToken,
-		1,
-		*vrfv2Config.General.NumberOfSubToCreate,
-		testConfig,
-		l,
-	)
-	vrfContracts.VRFV2Consumers = consumers
-
-	eoaWalletAddress = evmClient.GetDefaultWallet().Address()
-
-	l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs involved in the test")
-	for _, subID := range subIDs {
-		subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID)
-		require.NoError(t, err, "error getting subscription information for subscription %d", subID)
-		vrfv2.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2)
-	}
-
+	defaultWalletAddress = evmClient.GetDefaultWallet().Address()
 	t.Run("vrfv2 and bhs performance test", func(t *testing.T) {
 		configCopy := testConfig.MustCopy().(tc.TestConfig)
 		//Underfund Subscription
 		configCopy.VRFv2.General.SubscriptionFundingAmountLink = ptr.Ptr(float64(0))
-		consumers, subIDs, err = vrfv2.SetupNewConsumersAndSubs(
+		underfundedSubIDs, consumers, err := vrfv2.SetupSubsAndConsumersForExistingEnv(
 			testEnv,
 			chainID,
 			vrfContracts.CoordinatorV2,
-			configCopy,
 			vrfContracts.LinkToken,
 			1,
-			*configCopy.VRFv2.General.NumberOfSubToCreate,
+			*vrfv2Config.General.NumberOfSubToCreate,
+			configCopy,
 			l,
 		)
-		require.NoError(t, err, "error setting up new consumers and subscriptions")
+		require.NoError(t, err, "error setting up new consumers and subs")
+		for _, subID := range underfundedSubIDs {
+			subscription, err := vrfContracts.CoordinatorV2.GetSubscription(testcontext.Get(t), subID)
+			require.NoError(t, err, "error getting subscription information for subscription %d", subID)
+			vrfv2.LogSubDetails(l, subscription, subID, vrfContracts.CoordinatorV2)
+		}
+		subIDsForCancellingAfterTest = underfundedSubIDs
+		l.Debug().Int("Number of Subs", len(underfundedSubIDs)).Msg("Subs involved in the test")
 		vrfContracts.VRFV2Consumers = consumers
 		require.Len(t, vrfContracts.VRFV2Consumers, 1, "only one consumer should be created for Load Test")
 		err = vrfContracts.VRFV2Consumers[0].ResetMetrics()
@@ -296,7 +285,7 @@ func TestVRFV2BHSPerformance(t *testing.T) {
 			Gun: NewBHSTestGun(
 				vrfContracts,
 				vrfKey.KeyHash,
-				subIDs,
+				underfundedSubIDs,
 				configCopy.VRFv2,
 				l,
 			),
@@ -322,8 +311,27 @@ func TestVRFV2BHSPerformance(t *testing.T) {
 		_, err = actions.WaitForBlockNumberToBe(latestBlockNumber+uint64(256), evmClient, &wgBlockNumberTobe, configCopy.VRFv2.General.WaitFor256BlocksTimeout.Duration, t)
 		wgBlockNumberTobe.Wait()
 		require.NoError(t, err, "error waiting for block number to be")
-		err = vrfv2.FundSubscriptions(testEnv, chainID, big.NewFloat(*configCopy.VRFv2.General.SubscriptionRefundingAmountLink), vrfContracts.LinkToken, vrfContracts.CoordinatorV2, subIDs)
+
+		metrics, err := consumers[0].GetLoadTestMetrics(testcontext.Get(t))
+		require.NoError(t, err)
+		require.Equal(t, 0, metrics.FulfilmentCount.Cmp(big.NewInt(0)), "Fulfilment count should be 0 since sub is underfunded. Check if the sub is actually funded")
+
+		var subIDsString []uint64
+		subIDsString = append(subIDsString, underfundedSubIDs...)
+		l.Info().
+			Float64("SubscriptionRefundingAmountLink", *configCopy.VRFv2.General.SubscriptionRefundingAmountLink).
+			Uints64("SubIDs", subIDsString).
+			Msg("Funding Subscriptions with Link and Native Tokens")
+		err = vrfv2.FundSubscriptions(
+			testEnv,
+			chainID,
+			big.NewFloat(*configCopy.VRFv2.General.SubscriptionRefundingAmountLink),
+			vrfContracts.LinkToken,
+			vrfContracts.CoordinatorV2,
+			underfundedSubIDs,
+		)
 		require.NoError(t, err, "error funding subscriptions")
+
 		var wgAllRequestsFulfilled sync.WaitGroup
 		wgAllRequestsFulfilled.Add(1)
 		requestCount, fulfilmentCount, err := vrfcommon.WaitForRequestCountEqualToFulfilmentCount(testcontext.Get(t), vrfContracts.VRFV2Consumers[0], 2*time.Minute, &wgAllRequestsFulfilled)