From b614278db965542ee3ab9694ea3edb0d58693c2c Mon Sep 17 00:00:00 2001 From: Ilja Pavlovs Date: Sat, 11 Nov 2023 00:38:17 +0200 Subject: [PATCH] VRF-745: adding VRF V2 GH Actions workflow for WASP load test --- .../on-demand-vrfv2-performance-test.yml | 135 ++++++++++++++++++ integration-tests/load/vrfv2/config.toml | 65 ++++++--- integration-tests/load/vrfv2/vrfv2_test.go | 63 +++++++- 3 files changed, 240 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/on-demand-vrfv2-performance-test.yml diff --git a/.github/workflows/on-demand-vrfv2-performance-test.yml b/.github/workflows/on-demand-vrfv2-performance-test.yml new file mode 100644 index 00000000000..ed0fcd3cdc7 --- /dev/null +++ b/.github/workflows/on-demand-vrfv2-performance-test.yml @@ -0,0 +1,135 @@ +name: On Demand VRFV2 Performance Test +on: + workflow_dispatch: + inputs: + network: + description: Network to run tests on + type: choice + options: + - "ETHEREUM_MAINNET" + - "SIMULATED" + - "SEPOLIA" + - "OPTIMISM_MAINNET" + - "OPTIMISM_GOERLI" + - "ARBITRUM_MAINNET" + - "ARBITRUM_GOERLI" + - "BSC_MAINNET" + - "BSC_TESTNET" + - "POLYGON_MAINNET" + - "MUMBAI" + - "AVALANCHE_FUJI" + - "AVALANCHE_MAINNET" + fundingPrivateKey: + description: Private funding key (Skip for Simulated) + required: false + type: string + wsURL: + description: WS URL for the network (Skip for Simulated) + required: false + type: string + httpURL: + description: HTTP URL for the network (Skip for Simulated) + required: false + type: string + chainlinkImage: + description: Container image location for the Chainlink nodes + required: true + default: public.ecr.aws/chainlink/chainlink + chainlinkVersion: + description: Container image version for the Chainlink nodes + required: true + default: "2.6.0" + performanceTestType: + description: Performance Test Type of test to run + type: choice + options: + - "Soak" + - "Load" + - "Stress" + - "Spike" + testDuration: + description: Duration of the test (time string) + required: true + default: 5m + useExistingEnv: + description: Set `true` to use existing environment or `false` to deploy CL node and all contracts + required: false + default: false + configBase64: + description: TOML config in base64 (Needed when overriding config or providing contract addresses for existing env) + required: false +jobs: + vrfv2_performance_test: + name: ${{ inputs.network }} VRFV2 Performance Test + environment: integration + runs-on: ubuntu20.04-8cores-32GB + permissions: + checks: write + pull-requests: write + id-token: write + contents: read + env: + LOKI_URL: ${{ secrets.LOKI_URL }} + LOKI_TOKEN: ${{ secrets.LOKI_TOKEN }} + SELECTED_NETWORKS: ${{ inputs.network }} + TEST_TYPE: ${{ inputs.performanceTestType }} + VRFV2_TEST_DURATION: ${{ inputs.testDuration }} + VRFV2_USE_EXISTING_ENV: ${{ inputs.useExistingEnv }} + CONFIG: ${{ inputs.configBase64 }} + TEST_LOG_LEVEL: debug + REF_NAME: ${{ github.head_ref || github.ref_name }} + CHAINLINK_IMAGE: ${{ inputs.chainlinkImage }} + CHAINLINK_VERSION: ${{ inputs.chainlinkVersion }} + SLACK_API_KEY: ${{ secrets.QA_SLACK_API_KEY }} + SLACK_CHANNEL: ${{ secrets.QA_VRF_SLACK_CHANNEL }} + WASP_LOG_LEVEL: info + steps: + - name: Collect Metrics + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: ${{ inputs.network }} VRFV2 Performance Test + continue-on-error: true + - name: Setup Push Tag + shell: bash + run: | + echo "### chainlink image used for this test run :link:" >>$GITHUB_STEP_SUMMARY + echo "\`${{ inputs.chainlinkVersion }}\`" >>$GITHUB_STEP_SUMMARY + echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY + echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY + - name: Get Inputs + run: | + EVM_URLS=$(jq -r '.inputs.wsURL' $GITHUB_EVENT_PATH) + EVM_HTTP_URLS=$(jq -r '.inputs.httpURL' $GITHUB_EVENT_PATH) + EVM_KEYS=$(jq -r '.inputs.fundingPrivateKey' $GITHUB_EVENT_PATH) + + echo ::add-mask::$EVM_URLS + echo ::add-mask::$EVM_HTTP_URLS + echo ::add-mask::$EVM_KEYS + + echo EVM_URLS=$EVM_URLS >> $GITHUB_ENV + echo EVM_HTTP_URLS=$EVM_HTTP_URLS >> $GITHUB_ENV + echo EVM_KEYS=$EVM_KEYS >> $GITHUB_ENV + + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Run Tests + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@7d541cbbca52d45b8a718257af86d9cf49774d1f # v2.2.15 + with: + test_command_to_run: cd ./integration-tests && go test -v -count=1 -timeout 6h -run TestVRFV2Performance/vrfv2_performance_test ./load/vrfv2 + test_download_vendor_packages_command: cd ./integration-tests && go mod download + cl_repo: ${{ inputs.chainlinkImage }} + cl_image_tag: ${{ inputs.chainlinkVersion }} + aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} + artifacts_name: vrf-test-logs + artifacts_location: ./integration-tests/load/logs/ + token: ${{ secrets.GITHUB_TOKEN }} + go_mod_path: ./integration-tests/go.mod + should_cleanup: false + QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} + QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} + QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }} diff --git a/integration-tests/load/vrfv2/config.toml b/integration-tests/load/vrfv2/config.toml index 8917db88cc2..e3200fafe22 100644 --- a/integration-tests/load/vrfv2/config.toml +++ b/integration-tests/load/vrfv2/config.toml @@ -1,27 +1,50 @@ -# testing one product (jobs + contracts) by varying RPS + +[Common] +minimum_confirmations = 3 + +[NewEnvConfig] +sub_funds_link = 1 +sub_funds_native = 1 +node_funds = 10 + +[ExistingEnvConfig] +coordinator_address = "0x27b61f155F772b291D1d9B478BeAd37B2Ae447b0" +#consumer_address = "0x087F232165D9bA1A602f148025e5D0666953F64a" +#sub_id = "52116875585187328970776211988181422347535732407068188096422095950800466618218" +key_hash = "0x787d74caea10b2b357790d5b5247c2f63d1d91572a9846f780606e4d953677ae" +create_fund_subs_and_add_consumers = true +link_address = "0x779877A7B0D9E8603169DdbD7836e478b4624789" +sub_funds_link = 3 +sub_funds_native = 1 + +# 10 RPM - 1 tx request with 1 rand request in each tx every 6 seconds [Soak] +rate_limit_unit_duration = "6s" rps = 1 -duration = "3m" +randomness_request_count_per_request = 1 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting +number_of_sub_to_create = 1 +# approx 60 RPM - 1 tx request with 4 rand requests in each tx every 3 seconds [Load] -rps_from = 1 -rps_increase = 1 -rps_steps = 10 -duration = "3m" - -# testing multiple products (jobs + contracts) by varying instances, deploying more of the same type with stable RPS for each product -[SoakVolume] -products = 5 -pace = "1s" -duration = "3m" +rate_limit_unit_duration = "3s" +rps = 1 +randomness_request_count_per_request = 3 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 2 #NOTE - deviation should be less than randomness_request_count_per_request setting +number_of_sub_to_create = 1 -[LoadVolume] -products_from = 1 -products_increase = 1 -products_steps = 10 -pace = "1s" -duration = "3m" +# approx 540 RPM - 3 tx requests per second with 4 rand requests in each tx +[Stress] +rate_limit_unit_duration = "1s" +rps = 3 +randomness_request_count_per_request = 4 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting +number_of_sub_to_create = 1 -[Common] -node_funds = 10 -sub_funds = 100 \ No newline at end of file +# approx 150 RPM - 1 tx request with 150 rand requests in each tx every 60 seconds +[Spike] +rate_limit_unit_duration = "1m" +rps = 1 +randomness_request_count_per_request = 150 # amount of randomness requests to make per one TX request +randomness_request_count_per_request_deviation = 0 #NOTE - deviation should be less than randomness_request_count_per_request setting +number_of_sub_to_create = 1 \ No newline at end of file diff --git a/integration-tests/load/vrfv2/vrfv2_test.go b/integration-tests/load/vrfv2/vrfv2_test.go index c9102b22e46..a1480038935 100644 --- a/integration-tests/load/vrfv2/vrfv2_test.go +++ b/integration-tests/load/vrfv2/vrfv2_test.go @@ -6,6 +6,7 @@ import ( "github.com/kelseyhightower/envconfig" "github.com/rs/zerolog/log" "github.com/smartcontractkit/chainlink-testing-framework/logging" + "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/actions/vrfv2/vrfv2_config" "github.com/smartcontractkit/chainlink/integration-tests/contracts" "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" @@ -36,7 +37,7 @@ var ( testType = os.Getenv("TEST_TYPE") ) -func TestVRFV2Load(t *testing.T) { +func TestVRFV2Performance(t *testing.T) { cfg, err := ReadConfig() require.NoError(t, err) var vrfv2Config vrfv2_config.VRFV2Config @@ -158,7 +159,65 @@ func TestVRFV2Load(t *testing.T) { } } else { - //todo + //todo: temporary solution with envconfig and toml config until VRF-662 is implemented + vrfv2Config.ChainlinkNodeFunding = cfg.NewEnvConfig.NodeFunds + vrfv2Config.SubscriptionFundingAmountLink = cfg.NewEnvConfig.Funding.SubFundsLink + env, err = test_env.NewCLTestEnvBuilder(). + WithTestLogger(t). + WithGeth(). + WithCLNodes(1). + WithFunding(big.NewFloat(vrfv2Config.ChainlinkNodeFunding)). + WithCustomCleanup( + func() { + teardown(t, vrfv2Contracts.LoadTestConsumers[0], lc, updatedLabels, testReporter, testType, vrfv2Config) + + if env.EVMClient.NetworkSimulated() { + l.Info(). + Str("Network Name", env.EVMClient.GetNetworkName()). + Msg("Network is a simulated network. Skipping fund return for Coordinator Subscriptions.") + } else { + for _, subID := range subIDs { + l.Info(). + Uint64("Returning funds from SubID", subID). + Str("Returning funds to", eoaWalletAddress). + Msg("Canceling subscription and returning funds to subscription owner") + _, err := vrfv2Contracts.Coordinator.CancelSubscription(subID, common.HexToAddress(eoaWalletAddress)) + if err != nil { + l.Error().Err(err).Msg("Error canceling subscription") + } + } + //err = vrfv2.ReturnFundsForFulfilledRequests(env.EVMClient, vrfv2Contracts.Coordinator, l) + //l.Error().Err(err).Msg("Error returning funds for fulfilled requests") + } + if err := env.Cleanup(); err != nil { + l.Error().Err(err).Msg("Error cleaning up test environment") + } + }). + WithLogWatcher(). + Build() + + require.NoError(t, err, "error creating test env") + + env.ParallelTransactions(true) + + mockETHLinkFeed, err := actions.DeployMockETHLinkFeed(env.ContractDeployer, big.NewInt(vrfv2Config.LinkNativeFeedResponse)) + require.NoError(t, err, "error deploying mock ETH/LINK feed") + + linkToken, err := actions.DeployLINKToken(env.ContractDeployer) + require.NoError(t, err, "error deploying LINK contract") + + vrfv2Contracts, subIDs, vrfv2Data, err = vrfv2.SetupVRFV2Environment( + env, + vrfv2Config, + linkToken, + mockETHLinkFeed, + //register proving key against EOA address in order to return funds to this address + env.EVMClient.GetDefaultWallet().Address(), + 1, + vrfv2Config.NumberOfSubToCreate, + l, + ) + require.NoError(t, err, "error setting up VRF v2 env") } eoaWalletAddress = env.EVMClient.GetDefaultWallet().Address()