diff --git a/.github/workflows/on-demand-vrfv2plus-load-test.yml b/.github/workflows/on-demand-vrfv2plus-load-test.yml index 1faf1b84908..66c51fdbe0c 100644 --- a/.github/workflows/on-demand-vrfv2plus-load-test.yml +++ b/.github/workflows/on-demand-vrfv2plus-load-test.yml @@ -38,21 +38,25 @@ on: chainlinkVersion: description: Container image version for the Chainlink nodes required: true - default: "2.6.0-beta0" + default: "2.6.0" + performanceTestType: + description: Performance Test Type of test to run + type: choice + options: + - "Soak" + - "Load" + - "Stress" + - "Spike" testDuration: description: Duration of the test (time string) required: false default: 1m - randomnessRequestCountPerRequest: - description: Randomness request Count per one TX request - required: false - default: 1 useExistingEnv: - description: Whether to deploy a new contracts or use an existing one + description: Set `true` to use existing environment or `false` to deploy CL node and all contracts required: false default: false configBase64: - description: TOML config in base64 + description: TOML config in base64 (Needed when overriding config or providing contract addresses for existing env) required: false jobs: vrfv2plus_load_test: @@ -68,10 +72,8 @@ jobs: LOKI_URL: ${{ secrets.LOKI_URL }} LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }} SELECTED_NETWORKS: ${{ inputs.network }} + TEST_TYPE: ${{ inputs.performanceTestType }} VRFV2PLUS_TEST_DURATION: ${{ inputs.testDuration }} - VRFV2PLUS_RATE_LIMIT_UNIT_DURATION: 1m - VRFV2PLUS_RPS: 1 - VRFV2PLUS_RANDOMNESS_REQUEST_COUNT_PER_REQUEST: ${{ inputs.randomnessRequestCountPerRequest }} VRFV2PLUS_USE_EXISTING_ENV: ${{ inputs.useExistingEnv }} CONFIG: ${{ inputs.configBase64 }} TEST_LOG_LEVEL: debug diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go b/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go index 0ff9b4afa7d..7a1221eaf8b 100644 --- a/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go +++ b/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go @@ -18,7 +18,8 @@ type VRFV2PlusConfig struct { FulfillmentFlatFeeLinkPPM uint32 `envconfig:"FULFILLMENT_FLAT_FEE_LINK_PPM" default:"500"` // Flat fee in ppm for LINK for the VRF Coordinator config FulfillmentFlatFeeNativePPM uint32 `envconfig:"FULFILLMENT_FLAT_FEE_NATIVE_PPM" default:"500"` // Flat fee in ppm for native currency for the VRF Coordinator config - RandomnessRequestCountPerRequest uint16 `envconfig:"RANDOMNESS_REQUEST_COUNT_PER_REQUEST" default:"1"` // How many randomness requests to send per request + RandomnessRequestCountPerRequest uint16 `envconfig:"RANDOMNESS_REQUEST_COUNT_PER_REQUEST" default:"1"` // How many randomness requests to send per request + RandomnessRequestCountPerRequestDeviation uint16 `envconfig:"RANDOMNESS_REQUEST_COUNT_PER_REQUEST_DEVIATION" default:"0"` // How many randomness requests to send per request //Wrapper Config WrapperGasOverhead uint32 `envconfig:"WRAPPER_GAS_OVERHEAD" default:"50000"` diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go index 17e321ea63a..3bfa5d4f416 100644 --- a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go +++ b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/big" + "sync" "time" "github.com/smartcontractkit/chainlink/v2/core/assets" @@ -232,15 +233,17 @@ func FundVRFCoordinatorV2_5Subscription(linkToken contracts.LinkToken, coordinat return chainClient.WaitForEvents() } +// SetupVRFV2_5Environment will create specified number of subscriptions and add the same conumer/s to each of them func SetupVRFV2_5Environment( env *test_env.CLClusterTestEnv, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, linkToken contracts.LinkToken, mockNativeLINKFeed contracts.MockETHLINKFeed, - consumerContractsAmount int, -) (*VRFV2_5Contracts, *big.Int, *VRFV2PlusData, error) { + numberOfConsumers int, + numberOfSubToCreate int, +) (*VRFV2_5Contracts, []*big.Int, *VRFV2PlusData, error) { - vrfv2_5Contracts, err := DeployVRFV2_5Contracts(env.ContractDeployer, env.EVMClient, consumerContractsAmount) + vrfv2_5Contracts, err := DeployVRFV2_5Contracts(env.ContractDeployer, env.EVMClient, numberOfConsumers) if err != nil { return nil, nil, nil, errors.Wrap(err, ErrDeployVRFV2_5Contracts) } @@ -260,34 +263,39 @@ func SetupVRFV2_5Environment( return nil, nil, nil, errors.Wrap(err, ErrSetVRFCoordinatorConfig) } - subID, err := CreateSubAndFindSubID(env, vrfv2_5Contracts.Coordinator) + err = vrfv2_5Contracts.Coordinator.SetLINKAndLINKNativeFeed(linkToken.Address(), mockNativeLINKFeed.Address()) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, errors.Wrap(err, ErrSetLinkNativeLinkFeed) } - err = env.EVMClient.WaitForEvents() if err != nil { return nil, nil, nil, errors.Wrap(err, ErrWaitTXsComplete) } - for _, consumer := range vrfv2_5Contracts.LoadTestConsumers { - err = vrfv2_5Contracts.Coordinator.AddConsumer(subID, consumer.Address()) - if err != nil { - return nil, nil, nil, errors.Wrap(err, ErrAddConsumerToSub) - } + + subIDs, err := CreateSubsAndFund(env, vrfv2PlusConfig, linkToken, vrfv2_5Contracts, numberOfSubToCreate) + if err != nil { + return nil, nil, nil, err } - err = vrfv2_5Contracts.Coordinator.SetLINKAndLINKNativeFeed(linkToken.Address(), mockNativeLINKFeed.Address()) + subToConsumersMap := map[*big.Int][]contracts.VRFv2PlusLoadTestConsumer{} + + //each subscription will have the same consumers + for _, subID := range subIDs { + subToConsumersMap[subID] = vrfv2_5Contracts.LoadTestConsumers + } + + err = AddConsumersToSubs( + subToConsumersMap, + vrfv2_5Contracts.Coordinator, + ) if err != nil { - return nil, nil, nil, errors.Wrap(err, ErrSetLinkNativeLinkFeed) + return nil, nil, nil, err } + err = env.EVMClient.WaitForEvents() if err != nil { return nil, nil, nil, errors.Wrap(err, ErrWaitTXsComplete) } - err = FundSubscription(env, vrfv2PlusConfig, linkToken, vrfv2_5Contracts.Coordinator, subID) - if err != nil { - return nil, nil, nil, err - } vrfKey, err := env.ClCluster.NodeAPIs()[0].MustCreateVRFKey() if err != nil { @@ -350,12 +358,66 @@ func SetupVRFV2_5Environment( chainID, } - return vrfv2_5Contracts, subID, &data, nil + return vrfv2_5Contracts, subIDs, &data, nil +} + +func CreateSubsAndFund( + env *test_env.CLClusterTestEnv, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, + linkToken contracts.LinkToken, + vrfv2_5Contracts *VRFV2_5Contracts, + subAmountToCreate int, +) ([]*big.Int, error) { + subs, err := CreateSubs(env, vrfv2_5Contracts.Coordinator, subAmountToCreate) + if err != nil { + return nil, err + } + err = env.EVMClient.WaitForEvents() + if err != nil { + return nil, errors.Wrap(err, ErrWaitTXsComplete) + } + err = FundSubscriptions(env, vrfv2PlusConfig, linkToken, vrfv2_5Contracts.Coordinator, subs) + if err != nil { + return nil, err + } + return subs, nil +} + +func CreateSubs( + env *test_env.CLClusterTestEnv, + coordinator contracts.VRFCoordinatorV2_5, + subAmountToCreate int, +) ([]*big.Int, error) { + var subIDArr []*big.Int + + for i := 0; i < subAmountToCreate; i++ { + subID, err := CreateSubAndFindSubID(env, coordinator) + if err != nil { + return nil, err + } + subIDArr = append(subIDArr, subID) + } + return subIDArr, nil +} + +func AddConsumersToSubs( + subToConsumerMap map[*big.Int][]contracts.VRFv2PlusLoadTestConsumer, + coordinator contracts.VRFCoordinatorV2_5, +) error { + for subID, consumers := range subToConsumerMap { + for _, consumer := range consumers { + err := coordinator.AddConsumer(subID, consumer.Address()) + if err != nil { + return errors.Wrap(err, ErrAddConsumerToSub) + } + } + } + return nil } func SetupVRFV2PlusWrapperEnvironment( env *test_env.CLClusterTestEnv, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, linkToken contracts.LinkToken, mockNativeLINKFeed contracts.MockETHLINKFeed, coordinator contracts.VRFCoordinatorV2_5, @@ -411,7 +473,7 @@ func SetupVRFV2PlusWrapperEnvironment( return nil, nil, errors.Wrap(err, ErrWaitTXsComplete) } - err = FundSubscription(env, vrfv2PlusConfig, linkToken, coordinator, wrapperSubID) + err = FundSubscriptions(env, vrfv2PlusConfig, linkToken, coordinator, []*big.Int{wrapperSubID}) if err != nil { return nil, nil, err } @@ -445,15 +507,17 @@ func CreateSubAndFindSubID(env *test_env.CLClusterTestEnv, coordinator contracts if err != nil { return nil, errors.Wrap(err, ErrCreateVRFSubscription) } - err = env.EVMClient.WaitForEvents() + + sub, err := coordinator.WaitForSubscriptionCreatedEvent(time.Second * 10) if err != nil { - return nil, errors.Wrap(err, ErrWaitTXsComplete) + return nil, errors.Wrap(err, ErrFindSubID) } - subID, err := coordinator.FindSubscriptionID() + + err = env.EVMClient.WaitForEvents() if err != nil { - return nil, errors.Wrap(err, ErrFindSubID) + return nil, errors.Wrap(err, ErrWaitTXsComplete) } - return subID, nil + return sub.SubId, nil } func GetUpgradedCoordinatorTotalBalance(coordinator contracts.VRFCoordinatorV2PlusUpgradedVersion) (linkTotalBalance *big.Int, nativeTokenTotalBalance *big.Int, err error) { @@ -480,28 +544,29 @@ func GetCoordinatorTotalBalance(coordinator contracts.VRFCoordinatorV2_5) (linkT return } -func FundSubscription( +func FundSubscriptions( env *test_env.CLClusterTestEnv, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, linkAddress contracts.LinkToken, coordinator contracts.VRFCoordinatorV2_5, - subID *big.Int, + subIDs []*big.Int, ) error { - //Native Billing - err := coordinator.FundSubscriptionWithNative(subID, big.NewInt(0).Mul(big.NewInt(vrfv2PlusConfig.SubscriptionFundingAmountNative), big.NewInt(1e18))) - if err != nil { - return errors.Wrap(err, ErrFundSubWithNativeToken) - } - - err = FundVRFCoordinatorV2_5Subscription(linkAddress, coordinator, env.EVMClient, subID, big.NewInt(vrfv2PlusConfig.SubscriptionFundingAmountLink)) - if err != nil { - return errors.Wrap(err, ErrFundSubWithLinkToken) + for _, subID := range subIDs { + //Native Billing + err := coordinator.FundSubscriptionWithNative(subID, big.NewInt(0).Mul(big.NewInt(vrfv2PlusConfig.SubscriptionFundingAmountNative), big.NewInt(1e18))) + if err != nil { + return errors.Wrap(err, ErrFundSubWithNativeToken) + } + //Link Billing + err = FundVRFCoordinatorV2_5Subscription(linkAddress, coordinator, env.EVMClient, subID, big.NewInt(vrfv2PlusConfig.SubscriptionFundingAmountLink)) + if err != nil { + return errors.Wrap(err, ErrFundSubWithLinkToken) + } } - err = env.EVMClient.WaitForEvents() + err := env.EVMClient.WaitForEvents() if err != nil { return errors.Wrap(err, ErrWaitTXsComplete) } - return nil } @@ -511,7 +576,8 @@ func RequestRandomnessAndWaitForFulfillment( vrfv2PlusData *VRFV2PlusData, subID *big.Int, isNativeBilling bool, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + randomnessRequestCountPerRequest uint16, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, l zerolog.Logger, ) (*vrf_coordinator_v2_5.VRFCoordinatorV25RandomWordsFulfilled, error) { logRandRequest(consumer.Address(), coordinator.Address(), subID, isNativeBilling, vrfv2PlusConfig, l) @@ -522,7 +588,7 @@ func RequestRandomnessAndWaitForFulfillment( vrfv2PlusConfig.CallbackGasLimit, isNativeBilling, vrfv2PlusConfig.NumberOfWords, - vrfv2PlusConfig.RandomnessRequestCountPerRequest, + randomnessRequestCountPerRequest, ) if err != nil { return nil, errors.Wrap(err, ErrRequestRandomness) @@ -537,7 +603,7 @@ func RequestRandomnessAndWaitForFulfillmentUpgraded( vrfv2PlusData *VRFV2PlusData, subID *big.Int, isNativeBilling bool, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, l zerolog.Logger, ) (*vrf_v2plus_upgraded_version.VRFCoordinatorV2PlusUpgradedVersionRandomWordsFulfilled, error) { logRandRequest(consumer.Address(), coordinator.Address(), subID, isNativeBilling, vrfv2PlusConfig, l) @@ -585,7 +651,7 @@ func DirectFundingRequestRandomnessAndWaitForFulfillment( vrfv2PlusData *VRFV2PlusData, subID *big.Int, isNativeBilling bool, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, l zerolog.Logger, ) (*vrf_coordinator_v2_5.VRFCoordinatorV25RandomWordsFulfilled, error) { logRandRequest(consumer.Address(), coordinator.Address(), subID, isNativeBilling, vrfv2PlusConfig, l) @@ -650,6 +716,51 @@ func WaitForRequestAndFulfillmentEvents( return randomWordsFulfilledEvent, err } +func WaitForRequestCountEqualToFulfilmentCount(consumer contracts.VRFv2PlusLoadTestConsumer, timeout time.Duration, wg *sync.WaitGroup) (*big.Int, *big.Int, error) { + metricsChannel := make(chan *contracts.VRFLoadTestMetrics) + metricsErrorChannel := make(chan error) + + testContext, testCancel := context.WithTimeout(context.Background(), timeout) + defer testCancel() + + ticker := time.NewTicker(time.Second * 1) + var metrics *contracts.VRFLoadTestMetrics + for { + select { + case <-testContext.Done(): + ticker.Stop() + wg.Done() + return metrics.RequestCount, metrics.FulfilmentCount, + fmt.Errorf("timeout waiting for rand request and fulfilments to be equal AFTER performance test was executed. Request Count: %d, Fulfilment Count: %d", + metrics.RequestCount.Uint64(), metrics.FulfilmentCount.Uint64()) + case <-ticker.C: + go getLoadTestMetrics(consumer, metricsChannel, metricsErrorChannel) + case metrics = <-metricsChannel: + if metrics.RequestCount.Cmp(metrics.FulfilmentCount) == 0 { + ticker.Stop() + wg.Done() + return metrics.RequestCount, metrics.FulfilmentCount, nil + } + case err := <-metricsErrorChannel: + ticker.Stop() + wg.Done() + return nil, nil, err + } + } +} + +func getLoadTestMetrics( + consumer contracts.VRFv2PlusLoadTestConsumer, + metricsChannel chan *contracts.VRFLoadTestMetrics, + metricsErrorChannel chan error, +) { + metrics, err := consumer.GetLoadTestMetrics(context.Background()) + if err != nil { + metricsErrorChannel <- err + } + metricsChannel <- metrics +} + func LogSubDetails(l zerolog.Logger, subscription vrf_coordinator_v2_5.GetSubscription, subID *big.Int, coordinator contracts.VRFCoordinatorV2_5) { l.Debug(). Str("Coordinator", coordinator.Address()). @@ -795,14 +906,16 @@ func logRandRequest( coordinator string, subID *big.Int, isNativeBilling bool, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, l zerolog.Logger) { l.Debug(). Str("Consumer", consumer). Str("Coordinator", coordinator). - Str("subID", subID.String()). + Str("SubID", subID.String()). Bool("IsNativePayment", isNativeBilling). Uint16("MinimumConfirmations", vrfv2PlusConfig.MinimumConfirmations). + Uint32("CallbackGasLimit", vrfv2PlusConfig.CallbackGasLimit). Uint16("RandomnessRequestCountPerRequest", vrfv2PlusConfig.RandomnessRequestCountPerRequest). + Uint16("RandomnessRequestCountPerRequestDeviation", vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation). Msg("Requesting randomness") } diff --git a/integration-tests/contracts/contract_vrf_models.go b/integration-tests/contracts/contract_vrf_models.go index ed46b08dbbf..ea8b6ae3cac 100644 --- a/integration-tests/contracts/contract_vrf_models.go +++ b/integration-tests/contracts/contract_vrf_models.go @@ -83,6 +83,7 @@ type VRFCoordinatorV2_5 interface { GetNativeTokenTotalBalance(ctx context.Context) (*big.Int, error) GetLinkTotalBalance(ctx context.Context) (*big.Int, error) FindSubscriptionID() (*big.Int, error) + WaitForSubscriptionCreatedEvent(timeout time.Duration) (*vrf_coordinator_v2_5.VRFCoordinatorV25SubscriptionCreated, error) WaitForRandomWordsFulfilledEvent(subID []*big.Int, requestID []*big.Int, timeout time.Duration) (*vrf_coordinator_v2_5.VRFCoordinatorV25RandomWordsFulfilled, error) WaitForRandomWordsRequestedEvent(keyHash [][32]byte, subID []*big.Int, sender []common.Address, timeout time.Duration) (*vrf_coordinator_v2_5.VRFCoordinatorV25RandomWordsRequested, error) WaitForMigrationCompletedEvent(timeout time.Duration) (*vrf_coordinator_v2_5.VRFCoordinatorV25MigrationCompleted, error) diff --git a/integration-tests/contracts/ethereum_vrfv2plus_contracts.go b/integration-tests/contracts/ethereum_vrfv2plus_contracts.go index 5ea9bc3f5b1..df7cca54b2e 100644 --- a/integration-tests/contracts/ethereum_vrfv2plus_contracts.go +++ b/integration-tests/contracts/ethereum_vrfv2plus_contracts.go @@ -267,6 +267,26 @@ func (v *EthereumVRFCoordinatorV2_5) FindSubscriptionID() (*big.Int, error) { return subscriptionIterator.Event.SubId, nil } +func (v *EthereumVRFCoordinatorV2_5) WaitForSubscriptionCreatedEvent(timeout time.Duration) (*vrf_coordinator_v2_5.VRFCoordinatorV25SubscriptionCreated, error) { + eventsChannel := make(chan *vrf_coordinator_v2_5.VRFCoordinatorV25SubscriptionCreated) + subscription, err := v.coordinator.WatchSubscriptionCreated(nil, eventsChannel, nil) + if err != nil { + return nil, err + } + defer subscription.Unsubscribe() + + for { + select { + case err := <-subscription.Err(): + return nil, err + case <-time.After(timeout): + return nil, fmt.Errorf("timeout waiting for SubscriptionCreated event") + case sub := <-eventsChannel: + return sub, nil + } + } +} + func (v *EthereumVRFCoordinatorV2_5) WaitForRandomWordsFulfilledEvent(subID []*big.Int, requestID []*big.Int, timeout time.Duration) (*vrf_coordinator_v2_5.VRFCoordinatorV25RandomWordsFulfilled, error) { randomWordsFulfilledEventsChannel := make(chan *vrf_coordinator_v2_5.VRFCoordinatorV25RandomWordsFulfilled) subscription, err := v.coordinator.WatchRandomWordsFulfilled(nil, randomWordsFulfilledEventsChannel, requestID, subID) diff --git a/integration-tests/load/vrfv2plus/config.go b/integration-tests/load/vrfv2plus/config.go index 3b76d64cc7f..5f3babfeab0 100644 --- a/integration-tests/load/vrfv2plus/config.go +++ b/integration-tests/load/vrfv2plus/config.go @@ -5,6 +5,7 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/pkg/errors" "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink/integration-tests/actions/vrfv2plus/vrfv2plus_config" "github.com/smartcontractkit/chainlink/v2/core/store/models" "os" ) @@ -12,17 +13,20 @@ import ( const ( DefaultConfigFilename = "config.toml" - ErrReadPerfConfig = "failed to read TOML config for performance tests" - ErrUnmarshalPerfConfig = "failed to unmarshal TOML config for performance tests" + ErrReadPerfConfig = "failed to read TOML config for performance tests" + ErrUnmarshalPerfConfig = "failed to unmarshal TOML config for performance tests" + ErrDeviationShouldBeLessThanOriginal = "`RandomnessRequestCountPerRequestDeviation` should be less than `RandomnessRequestCountPerRequest`" ) type PerformanceConfig struct { - Soak *Soak `toml:"Soak"` - Load *Load `toml:"Load"` - SoakVolume *SoakVolume `toml:"SoakVolume"` - LoadVolume *LoadVolume `toml:"LoadVolume"` + Soak *Soak `toml:"Soak"` + Load *Load `toml:"Load"` + Stress *Stress `toml:"Stress"` + Spike *Spike `toml:"Spike"` + Common *Common `toml:"Common"` ExistingEnvConfig *ExistingEnvConfig `toml:"ExistingEnvConfig"` + NewEnvConfig *NewEnvConfig `toml:"NewEnvConfig"` } type ExistingEnvConfig struct { @@ -32,9 +36,12 @@ type ExistingEnvConfig struct { KeyHash string `toml:"key_hash"` } -type Common struct { +type NewEnvConfig struct { Funding - IsNativePayment bool `toml:"is_native_payment"` + NumberOfSubToCreate int `toml:"number_of_sub_to_create"` +} + +type Common struct { MinimumConfirmations uint16 `toml:"minimum_confirmations"` } @@ -45,29 +52,27 @@ type Funding struct { } type Soak struct { - RPS int64 `toml:"rps"` - Duration *models.Duration `toml:"duration"` + PerformanceTestConfig } -type SoakVolume struct { - Products int64 `toml:"products"` - Pace *models.Duration `toml:"pace"` - Duration *models.Duration `toml:"duration"` +type Load struct { + PerformanceTestConfig } -type Load struct { - RPSFrom int64 `toml:"rps_from"` - RPSIncrease int64 `toml:"rps_increase"` - RPSSteps int `toml:"rps_steps"` - Duration *models.Duration `toml:"duration"` +type Stress struct { + PerformanceTestConfig } -type LoadVolume struct { - ProductsFrom int64 `toml:"products_from"` - ProductsIncrease int64 `toml:"products_increase"` - ProductsSteps int `toml:"products_steps"` - Pace *models.Duration `toml:"pace"` - Duration *models.Duration `toml:"duration"` +type Spike struct { + PerformanceTestConfig +} + +type PerformanceTestConfig struct { + RPS int64 `toml:"rps"` + //Duration *models.Duration `toml:"duration"` + RateLimitUnitDuration *models.Duration `toml:"rate_limit_unit_duration"` + RandomnessRequestCountPerRequest uint16 `toml:"randomness_request_count_per_request"` + RandomnessRequestCountPerRequestDeviation uint16 `toml:"randomness_request_count_per_request_deviation"` } func ReadConfig() (*PerformanceConfig, error) { @@ -88,6 +93,35 @@ func ReadConfig() (*PerformanceConfig, error) { return nil, errors.Wrap(err, ErrUnmarshalPerfConfig) } + if cfg.Soak.RandomnessRequestCountPerRequest <= cfg.Soak.RandomnessRequestCountPerRequestDeviation { + return nil, errors.Wrap(err, ErrDeviationShouldBeLessThanOriginal) + } + log.Debug().Interface("Config", cfg).Msg("Parsed config") return cfg, nil } + +func SetPerformanceTestConfig(vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, cfg *PerformanceConfig) { + switch os.Getenv("TEST_TYPE") { + case "Soak": + vrfv2PlusConfig.RPS = cfg.Soak.RPS + vrfv2PlusConfig.RateLimitUnitDuration = cfg.Soak.RateLimitUnitDuration.Duration() + vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Soak.RandomnessRequestCountPerRequest + vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Soak.RandomnessRequestCountPerRequestDeviation + case "Load": + vrfv2PlusConfig.RPS = cfg.Load.RPS + vrfv2PlusConfig.RateLimitUnitDuration = cfg.Load.RateLimitUnitDuration.Duration() + vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Load.RandomnessRequestCountPerRequest + vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Load.RandomnessRequestCountPerRequestDeviation + case "Stress": + vrfv2PlusConfig.RPS = cfg.Stress.RPS + vrfv2PlusConfig.RateLimitUnitDuration = cfg.Stress.RateLimitUnitDuration.Duration() + vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Stress.RandomnessRequestCountPerRequest + vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Stress.RandomnessRequestCountPerRequestDeviation + case "Spike": + vrfv2PlusConfig.RPS = cfg.Spike.RPS + vrfv2PlusConfig.RateLimitUnitDuration = cfg.Spike.RateLimitUnitDuration.Duration() + vrfv2PlusConfig.RandomnessRequestCountPerRequest = cfg.Spike.RandomnessRequestCountPerRequest + vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation = cfg.Spike.RandomnessRequestCountPerRequestDeviation + } +} diff --git a/integration-tests/load/vrfv2plus/config.toml b/integration-tests/load/vrfv2plus/config.toml index d77c22badb8..1208423dc0d 100644 --- a/integration-tests/load/vrfv2plus/config.toml +++ b/integration-tests/load/vrfv2plus/config.toml @@ -1,10 +1,12 @@ [Common] -node_funds = 0.1 -is_native_payment = false minimum_confirmations = 3 -sub_funds_link = 10 -sub_funds_native = 1 + +[NewEnvConfig] +sub_funds_link = 1000 +sub_funds_native = 1000 +node_funds = 10 +number_of_sub_to_create = 10 [ExistingEnvConfig] coordinator_address = "0x4931Ce2e341398c8eD8A5D0F6ADb920476D6DaBb" @@ -12,3 +14,31 @@ consumer_address = "0x087F232165D9bA1A602f148025e5D0666953F64a" sub_id = "52116875585187328970776211988181422347535732407068188096422095950800466618218" key_hash = "0x4c422465ed6a06cfc84575a5437fef7b9dc6263133f648afbe6ae7b2c694d3b3" + +# 10 RPM - 1 tx request with 1 rand request in each tx every 6 seconds +[Soak] +rate_limit_unit_duration = "6s" +rps = 1 +randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting + +# approx 60 RPM - 1 tx request with 4 rand requests in each tx every 3 seconds +[Load] +rate_limit_unit_duration = "3s" +rps = 1 +randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting + +# approx 540 RPM - 3 tx requests per second with 4 rand requests in each tx +[Stress] +rate_limit_unit_duration = "0" +rps = 3 +randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting + +# approx 150 RPM - 1 tx request with 150 rand requests in each tx every 60 seconds +[Spike] +rate_limit_unit_duration = "1m" +rps = 1 +randomness_request_count_per_request = 150 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting diff --git a/integration-tests/load/vrfv2plus/gun.go b/integration-tests/load/vrfv2plus/gun.go index f1b15bb5fef..c9947fa32f5 100644 --- a/integration-tests/load/vrfv2plus/gun.go +++ b/integration-tests/load/vrfv2plus/gun.go @@ -6,6 +6,7 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/actions/vrfv2plus/vrfv2plus_config" "github.com/smartcontractkit/wasp" "math/big" + "math/rand" ) /* SingleHashGun is a gun that constantly requests randomness for one feed */ @@ -13,22 +14,22 @@ import ( type SingleHashGun struct { contracts *vrfv2plus.VRFV2_5Contracts keyHash [32]byte - subID *big.Int - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig + subIDs []*big.Int + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig logger zerolog.Logger } func NewSingleHashGun( contracts *vrfv2plus.VRFV2_5Contracts, keyHash [32]byte, - subID *big.Int, - vrfv2PlusConfig vrfv2plus_config.VRFV2PlusConfig, + subIDs []*big.Int, + vrfv2PlusConfig *vrfv2plus_config.VRFV2PlusConfig, logger zerolog.Logger, ) *SingleHashGun { return &SingleHashGun{ contracts: contracts, keyHash: keyHash, - subID: subID, + subIDs: subIDs, vrfv2PlusConfig: vrfv2PlusConfig, logger: logger, } @@ -37,20 +38,40 @@ func NewSingleHashGun( // Call implements example gun call, assertions on response bodies should be done here func (m *SingleHashGun) Call(l *wasp.Generator) *wasp.CallResult { //todo - should work with multiple consumers and consumers having different keyhashes and wallets + + //randomly increase/decrease randomness request count per TX + randomnessRequestCountPerRequest := deviateValue(m.vrfv2PlusConfig.RandomnessRequestCountPerRequest, m.vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation) _, err := vrfv2plus.RequestRandomnessAndWaitForFulfillment( + //the same consumer is used for all requests and in all subs m.contracts.LoadTestConsumers[0], m.contracts.Coordinator, &vrfv2plus.VRFV2PlusData{VRFV2PlusKeyData: vrfv2plus.VRFV2PlusKeyData{KeyHash: m.keyHash}}, - m.subID, - //todo - make this configurable - m.vrfv2PlusConfig.IsNativePayment, + //randomly pick a subID from pool of subIDs + m.subIDs[randInRange(0, len(m.subIDs)-1)], + //randomly pick payment type + randBool(), + randomnessRequestCountPerRequest, m.vrfv2PlusConfig, m.logger, ) - if err != nil { return &wasp.CallResult{Error: err.Error(), Failed: true} } - return &wasp.CallResult{} } + +func deviateValue(requestCountPerTX uint16, deviation uint16) uint16 { + if randBool() && requestCountPerTX > deviation { + requestCountPerTX -= uint16(randInRange(0, int(deviation))) + } else { + requestCountPerTX += uint16(randInRange(0, int(deviation))) + } + return requestCountPerTX +} + +func randBool() bool { + return rand.Intn(2) == 1 +} +func randInRange(min int, max int) int { + return rand.Intn(max-min+1) + min +} diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go index 5221a9860f6..5c3ea6e8c6d 100644 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go @@ -2,13 +2,13 @@ package loadvrfv2plus import ( "context" - "fmt" "github.com/ethereum/go-ethereum/common" "github.com/kelseyhightower/envconfig" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/wasp" "github.com/stretchr/testify/require" "math/big" + "os" "sync" "testing" "time" @@ -27,18 +27,27 @@ func TestVRFV2PlusLoad(t *testing.T) { err = envconfig.Process("VRFV2PLUS", &vrfv2PlusConfig) require.NoError(t, err) + SetPerformanceTestConfig(&vrfv2PlusConfig, cfg) + l := logging.GetTestLogger(t) //todo: temporary solution with envconfig and toml config until VRF-662 is implemented - vrfv2PlusConfig.ChainlinkNodeFunding = cfg.Common.NodeFunds - vrfv2PlusConfig.IsNativePayment = cfg.Common.IsNativePayment vrfv2PlusConfig.MinimumConfirmations = cfg.Common.MinimumConfirmations - vrfv2PlusConfig.SubscriptionFundingAmountLink = cfg.Common.Funding.SubFundsLink - vrfv2PlusConfig.SubscriptionFundingAmountNative = cfg.Common.Funding.SubFundsNative + + l.Info(). + Str("Test Type", os.Getenv("TEST_TYPE")). + Str("Test Duration", vrfv2PlusConfig.TestDuration.Truncate(time.Second).String()). + Int64("RPS", vrfv2PlusConfig.RPS). + Str("RateLimitUnitDuration", vrfv2PlusConfig.RateLimitUnitDuration.String()). + Uint16("RandomnessRequestCountPerRequest", vrfv2PlusConfig.RandomnessRequestCountPerRequest). + Uint16("RandomnessRequestCountPerRequestDeviation", vrfv2PlusConfig.RandomnessRequestCountPerRequestDeviation). + Bool("UseExistingEnv", vrfv2PlusConfig.UseExistingEnv). + Msg("Performance Test Configuration") var env *test_env.CLClusterTestEnv var vrfv2PlusContracts *vrfv2plus.VRFV2_5Contracts - var subID *big.Int var vrfv2PlusData *vrfv2plus.VRFV2PlusData + var subIDs []*big.Int + if vrfv2PlusConfig.UseExistingEnv { //todo: temporary solution with envconfig and toml config until VRF-662 is implemented vrfv2PlusConfig.CoordinatorAddress = cfg.ExistingEnvConfig.CoordinatorAddress @@ -48,6 +57,7 @@ func TestVRFV2PlusLoad(t *testing.T) { env, err = test_env.NewCLTestEnvBuilder(). WithTestLogger(t). + WithoutCleanup(). Build() require.NoError(t, err, "error creating test env") @@ -64,7 +74,7 @@ func TestVRFV2PlusLoad(t *testing.T) { BHS: nil, } var ok bool - subID, ok = new(big.Int).SetString(vrfv2PlusConfig.SubID, 10) + subID, ok := new(big.Int).SetString(vrfv2PlusConfig.SubID, 10) require.True(t, ok) vrfv2PlusData = &vrfv2plus.VRFV2PlusData{ @@ -77,12 +87,19 @@ func TestVRFV2PlusLoad(t *testing.T) { PrimaryEthAddress: "", ChainID: nil, } + subIDs = append(subIDs, subID) } else { + //todo: temporary solution with envconfig and toml config until VRF-662 is implemented + vrfv2PlusConfig.ChainlinkNodeFunding = cfg.NewEnvConfig.NodeFunds + vrfv2PlusConfig.SubscriptionFundingAmountLink = cfg.NewEnvConfig.Funding.SubFundsLink + vrfv2PlusConfig.SubscriptionFundingAmountNative = cfg.NewEnvConfig.Funding.SubFundsNative + numberOfSubToCreate := cfg.NewEnvConfig.NumberOfSubToCreate env, err = test_env.NewCLTestEnvBuilder(). WithTestLogger(t). WithGeth(). WithCLNodes(1). WithFunding(big.NewFloat(vrfv2PlusConfig.ChainlinkNodeFunding)). + WithStandardCleanup(). WithLogWatcher(). Build() @@ -96,28 +113,22 @@ func TestVRFV2PlusLoad(t *testing.T) { linkToken, err := actions.DeployLINKToken(env.ContractDeployer) require.NoError(t, err, "error deploying LINK contract") - vrfv2PlusContracts, subID, vrfv2PlusData, err = vrfv2plus.SetupVRFV2_5Environment(env, vrfv2PlusConfig, linkToken, mockETHLinkFeed, 1) + vrfv2PlusContracts, subIDs, vrfv2PlusData, err = vrfv2plus.SetupVRFV2_5Environment(env, &vrfv2PlusConfig, linkToken, mockETHLinkFeed, 1, numberOfSubToCreate) require.NoError(t, err, "error setting up VRF v2_5 env") } - subscription, err := vrfv2PlusContracts.Coordinator.GetSubscription(context.Background(), subID) - require.NoError(t, err, "error getting subscription information") - - vrfv2plus.LogSubDetails(l, subscription, subID, vrfv2PlusContracts.Coordinator) + l.Debug().Int("Number of Subs", len(subIDs)).Msg("Subs Involved in Load Test") + for _, subID := range subIDs { + subscription, err := vrfv2PlusContracts.Coordinator.GetSubscription(context.Background(), subID) + require.NoError(t, err, "error getting subscription information for subscription %s", subID.String()) + vrfv2plus.LogSubDetails(l, subscription, subID, vrfv2PlusContracts.Coordinator) + } labels := map[string]string{ "branch": "vrfv2Plus_healthcheck", "commit": "vrfv2Plus_healthcheck", } - l.Info(). - Str("Test Duration", vrfv2PlusConfig.TestDuration.Truncate(time.Second).String()). - Int64("RPS", vrfv2PlusConfig.RPS). - Str("RateLimitUnitDuration", vrfv2PlusConfig.RateLimitUnitDuration.String()). - Uint16("RandomnessRequestCountPerRequest", vrfv2PlusConfig.RandomnessRequestCountPerRequest). - Bool("UseExistingEnv", vrfv2PlusConfig.UseExistingEnv). - Msg("Load Test Configs") - lokiConfig := wasp.NewEnvLokiConfig() lc, err := wasp.NewLokiClient(lokiConfig) if err != nil { @@ -133,14 +144,15 @@ func TestVRFV2PlusLoad(t *testing.T) { Gun: NewSingleHashGun( vrfv2PlusContracts, vrfv2PlusData.KeyHash, - subID, - vrfv2PlusConfig, + subIDs, + &vrfv2PlusConfig, l, ), Labels: labels, LokiConfig: lokiConfig, CallTimeout: 2 * time.Minute, } + require.Len(t, vrfv2PlusContracts.LoadTestConsumers, 1, "only one consumer should be created for Load Test") consumer := vrfv2PlusContracts.LoadTestConsumers[0] err = consumer.ResetMetrics() require.NoError(t, err) @@ -162,7 +174,7 @@ func TestVRFV2PlusLoad(t *testing.T) { var wg sync.WaitGroup wg.Add(1) - requestCount, fulfilmentCount, err := WaitForRequestCountEqualToFulfilmentCount(vrfv2PlusContracts.LoadTestConsumers[0], 30*time.Second, &wg) + requestCount, fulfilmentCount, err := vrfv2plus.WaitForRequestCountEqualToFulfilmentCount(consumer, 30*time.Second, &wg) l.Info(). Interface("Request Count", requestCount). Interface("Fulfilment Count", fulfilmentCount). @@ -174,48 +186,3 @@ func TestVRFV2PlusLoad(t *testing.T) { }) } - -func WaitForRequestCountEqualToFulfilmentCount(consumer contracts.VRFv2PlusLoadTestConsumer, timeout time.Duration, wg *sync.WaitGroup) (*big.Int, *big.Int, error) { - metricsChannel := make(chan *contracts.VRFLoadTestMetrics) - metricsErrorChannel := make(chan error) - - testContext, testCancel := context.WithTimeout(context.Background(), timeout) - defer testCancel() - - ticker := time.NewTicker(time.Second * 1) - var metrics *contracts.VRFLoadTestMetrics - for { - select { - case <-testContext.Done(): - ticker.Stop() - wg.Done() - return metrics.RequestCount, metrics.FulfilmentCount, - fmt.Errorf("timeout waiting for rand request and fulfilments to be equal AFTER performance test was executed. Request Count: %d, Fulfilment Count: %d", - metrics.RequestCount.Uint64(), metrics.FulfilmentCount.Uint64()) - case <-ticker.C: - go getLoadTestMetrics(consumer, metricsChannel, metricsErrorChannel) - case metrics = <-metricsChannel: - if metrics.RequestCount.Cmp(metrics.FulfilmentCount) == 0 { - ticker.Stop() - wg.Done() - return metrics.RequestCount, metrics.FulfilmentCount, nil - } - case err := <-metricsErrorChannel: - ticker.Stop() - wg.Done() - return nil, nil, err - } - } -} - -func getLoadTestMetrics( - consumer contracts.VRFv2PlusLoadTestConsumer, - metricsChannel chan *contracts.VRFLoadTestMetrics, - metricsErrorChannel chan error, -) { - metrics, err := consumer.GetLoadTestMetrics(context.Background()) - if err != nil { - metricsErrorChannel <- err - } - metricsChannel <- metrics -} diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index b9854d59ad6..c2cc0878b66 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -46,9 +46,11 @@ func TestVRFv2Plus(t *testing.T) { linkToken, err := actions.DeployLINKToken(env.ContractDeployer) require.NoError(t, err, "error deploying LINK contract") - vrfv2PlusContracts, subID, vrfv2PlusData, err := vrfv2plus.SetupVRFV2_5Environment(env, vrfv2PlusConfig, linkToken, mockETHLinkFeed, 1) + vrfv2PlusContracts, subIDs, vrfv2PlusData, err := vrfv2plus.SetupVRFV2_5Environment(env, &vrfv2PlusConfig, linkToken, mockETHLinkFeed, 1, 1) require.NoError(t, err, "error setting up VRF v2_5 env") + subID := subIDs[0] + subscription, err := vrfv2PlusContracts.Coordinator.GetSubscription(context.Background(), subID) require.NoError(t, err, "error getting subscription information") @@ -68,7 +70,8 @@ func TestVRFv2Plus(t *testing.T) { vrfv2PlusData, subID, isNativeBilling, - vrfv2PlusConfig, + vrfv2PlusConfig.RandomnessRequestCountPerRequest, + &vrfv2PlusConfig, l, ) require.NoError(t, err, "error requesting randomness and waiting for fulfilment") @@ -109,7 +112,8 @@ func TestVRFv2Plus(t *testing.T) { vrfv2PlusData, subID, isNativeBilling, - vrfv2PlusConfig, + vrfv2PlusConfig.RandomnessRequestCountPerRequest, + &vrfv2PlusConfig, l, ) require.NoError(t, err, "error requesting randomness and waiting for fulfilment") @@ -137,7 +141,7 @@ func TestVRFv2Plus(t *testing.T) { wrapperContracts, wrapperSubID, err := vrfv2plus.SetupVRFV2PlusWrapperEnvironment( env, - vrfv2PlusConfig, + &vrfv2PlusConfig, linkToken, mockETHLinkFeed, vrfv2PlusContracts.Coordinator, @@ -162,7 +166,7 @@ func TestVRFv2Plus(t *testing.T) { vrfv2PlusData, wrapperSubID, isNativeBilling, - vrfv2PlusConfig, + &vrfv2PlusConfig, l, ) require.NoError(t, err, "error requesting randomness and waiting for fulfilment") @@ -210,7 +214,7 @@ func TestVRFv2Plus(t *testing.T) { vrfv2PlusData, wrapperSubID, isNativeBilling, - vrfv2PlusConfig, + &vrfv2PlusConfig, l, ) require.NoError(t, err, "error requesting randomness and waiting for fulfilment") @@ -267,9 +271,11 @@ func TestVRFv2PlusMigration(t *testing.T) { linkAddress, err := actions.DeployLINKToken(env.ContractDeployer) require.NoError(t, err, "error deploying LINK contract") - vrfv2PlusContracts, subID, vrfv2PlusData, err := vrfv2plus.SetupVRFV2_5Environment(env, vrfv2PlusConfig, linkAddress, mockETHLinkFeedAddress, 2) + vrfv2PlusContracts, subIDs, vrfv2PlusData, err := vrfv2plus.SetupVRFV2_5Environment(env, &vrfv2PlusConfig, linkAddress, mockETHLinkFeedAddress, 2, 1) require.NoError(t, err, "error setting up VRF v2_5 env") + subID := subIDs[0] + subscription, err := vrfv2PlusContracts.Coordinator.GetSubscription(context.Background(), subID) require.NoError(t, err, "error getting subscription information") @@ -402,7 +408,7 @@ func TestVRFv2PlusMigration(t *testing.T) { vrfv2PlusData, subID, false, - vrfv2PlusConfig, + &vrfv2PlusConfig, l, ) require.NoError(t, err, "error requesting randomness and waiting for fulfilment") @@ -414,7 +420,7 @@ func TestVRFv2PlusMigration(t *testing.T) { vrfv2PlusData, subID, true, - vrfv2PlusConfig, + &vrfv2PlusConfig, l, ) require.NoError(t, err, "error requesting randomness and waiting for fulfilment")