From 15e7b79c92208feee41d8cf51ca4e5bb8843a531 Mon Sep 17 00:00:00 2001 From: Anirudh Warrier <12178754+anirudhwarrier@users.noreply.github.com> Date: Fri, 1 Dec 2023 14:36:38 +0400 Subject: [PATCH] [AUTO-7451] Move load test and smoke test to use new actions (#11364) * move load test to use new common action * move smoke test to use common action * fix smoke tests * decrease batch size in load test report prep * add info logs in TestAutomationKeeperNodesDown * remove duplicate registry config in smoke test * use AutomationTest struct in setupAutomationTestDocker * add and use SetupMercuryMock --- .../actions/automationv2/actions.go | 23 ++ .../automationv2_1/automationv2_1_test.go | 233 ++++------- integration-tests/smoke/automation_test.go | 381 +++++++++++------- 3 files changed, 346 insertions(+), 291 deletions(-) diff --git a/integration-tests/actions/automationv2/actions.go b/integration-tests/actions/automationv2/actions.go index f97c17bb94f..febdd892150 100644 --- a/integration-tests/actions/automationv2/actions.go +++ b/integration-tests/actions/automationv2/actions.go @@ -11,6 +11,8 @@ import ( "testing" "time" + "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/lib/pq" @@ -36,6 +38,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" "github.com/smartcontractkit/chainlink/v2/core/store/models" "github.com/smartcontractkit/chainlink/v2/core/utils" + + ctfTestEnv "github.com/smartcontractkit/chainlink-testing-framework/docker/test_env" ) type NodeDetails struct { @@ -70,6 +74,8 @@ type AutomationTest struct { ChainlinkNodesk8s []*client.ChainlinkK8sClient ChainlinkNodes []*client.ChainlinkClient + DockerEnv *test_env.CLClusterTestEnv + NodeDetails []NodeDetails DefaultP2Pv2Bootstrapper string MercuryCredentialName string @@ -135,6 +141,10 @@ func (a *AutomationTest) SetUpkeepPrivilegeManager(address string) { a.UpkeepPrivilegeManager = common.HexToAddress(address) } +func (a *AutomationTest) SetDockerEnv(env *test_env.CLClusterTestEnv) { + a.DockerEnv = env +} + func (a *AutomationTest) DeployLINK() error { linkToken, err := a.Deployer.DeployLinkTokenContract() if err != nil { @@ -646,6 +656,19 @@ func (a *AutomationTest) AddJobsAndSetConfig(t *testing.T) { l.Info().Str("Registry Address", a.Registry.Address()).Msg("Successfully setConfig on registry") } +func (a *AutomationTest) SetupMercuryMock(t *testing.T, imposters []ctfTestEnv.KillgraveImposter) { + if a.IsOnk8s { + t.Error("mercury mock is not supported on k8s") + } + if a.DockerEnv == nil { + t.Error("docker env is not set") + } + err := a.DockerEnv.MockAdapter.AddImposter(imposters) + if err != nil { + require.NoError(t, err, "Error adding mock imposter") + } +} + func (a *AutomationTest) SetupAutomationDeployment(t *testing.T) { l := logging.GetTestLogger(t) err := a.CollectNodeDetails() diff --git a/integration-tests/load/automationv2_1/automationv2_1_test.go b/integration-tests/load/automationv2_1/automationv2_1_test.go index 1f39d5e4b0e..5fde9befba5 100644 --- a/integration-tests/load/automationv2_1/automationv2_1_test.go +++ b/integration-tests/load/automationv2_1/automationv2_1_test.go @@ -2,8 +2,8 @@ package automationv2_1 import ( "context" - "encoding/json" "fmt" + "math" "math/big" "os" "strconv" @@ -31,12 +31,12 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" - registrar21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_registrar_wrapper2_1" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/log_emitter" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/simple_log_upkeep_counter_wrapper" "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/actions/automationv2" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" contractseth "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum" @@ -284,31 +284,14 @@ func TestLogTrigger(t *testing.T) { chainClient.ParallelTransactions(true) - linkToken, err := contractDeployer.DeployLinkTokenContract() - require.NoError(t, err, "Error deploying link token contract") - - err = chainClient.WaitForEvents() - require.NoError(t, err, "Failed waiting for contracts to deploy") - - registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar( - t, contractseth.RegistryVersion_2_1, *registrySettings, linkToken, contractDeployer, chainClient, - ) - - err = actions.FundChainlinkNodesAddress(chainlinkNodes[1:], chainClient, big.NewFloat(100), 0) - require.NoError(t, err, "Error funding chainlink nodes") - - actions.CreateOCRKeeperJobs( - t, - chainlinkNodes, - registry.Address(), - chainClient.GetChainID().Int64(), - 0, - contractseth.RegistryVersion_2_1, - ) - - S, oracleIdentities, err := actions.GetOracleIdentities(chainlinkNodes) - require.NoError(t, err, "Error getting oracle identities") - offC, err := json.Marshal(ocr2keepers30config.OffchainConfig{ + a := automationv2.NewAutomationTestK8s(chainClient, contractDeployer, chainlinkNodes) + a.RegistrySettings = *registrySettings + a.RegistrarSettings = contracts.KeeperRegistrarSettings{ + AutoApproveConfigType: uint8(2), + AutoApproveMaxAllowed: math.MaxUint16, + MinLinkJuels: big.NewInt(0), + } + a.PluginConfig = ocr2keepers30config.OffchainConfig{ TargetProbability: "0.999", TargetInRounds: 1, PerformLockoutWindow: 80_000, // Copied from arbitrum mainnet prod value @@ -316,64 +299,33 @@ func TestLogTrigger(t *testing.T) { GasOverheadPerUpkeep: 300_000, MinConfirmations: 0, MaxUpkeepBatchSize: 10, - }) - require.NoError(t, err, "Error marshalling offchain config") - - signerOnchainPublicKeys, transmitterAccounts, f, _, offchainConfigVersion, offchainConfig, err := ocr3.ContractSetConfigArgsForTests( - 10*time.Second, // deltaProgress time.Duration, - 15*time.Second, // deltaResend time.Duration, - 500*time.Millisecond, // deltaInitial time.Duration, - 1000*time.Millisecond, // deltaRound time.Duration, - 200*time.Millisecond, // deltaGrace time.Duration, - 300*time.Millisecond, // deltaCertifiedCommitRequest time.Duration - 15*time.Second, // deltaStage time.Duration, - 24, // rMax uint64, - S, // s []int, - oracleIdentities, // oracles []OracleIdentityExtra, - offC, // reportingPluginConfig []byte, - 20*time.Millisecond, // maxDurationQuery time.Duration, - 20*time.Millisecond, // maxDurationObservation time.Duration, // good to here - 1200*time.Millisecond, // maxDurationShouldAcceptAttestedReport time.Duration, - 20*time.Millisecond, // maxDurationShouldTransmitAcceptedReport time.Duration, - 1, // f int, - nil, // onchainConfig []byte, - ) - require.NoError(t, err, "Error setting OCR config vars") - - var signers []common.Address - for _, signer := range signerOnchainPublicKeys { - require.Equal(t, 20, len(signer), "OnChainPublicKey '%v' has wrong length for address", signer) - signers = append(signers, common.BytesToAddress(signer)) } - - var transmitters []common.Address - for _, transmitter := range transmitterAccounts { - require.True(t, common.IsHexAddress(string(transmitter)), "TransmitAccount '%s' is not a valid Ethereum address", string(transmitter)) - transmitters = append(transmitters, common.HexToAddress(string(transmitter))) + a.PublicConfig = ocr3.PublicConfig{ + DeltaProgress: 10 * time.Second, + DeltaResend: 15 * time.Second, + DeltaInitial: 500 * time.Millisecond, + DeltaRound: 1000 * time.Millisecond, + DeltaGrace: 200 * time.Millisecond, + DeltaCertifiedCommitRequest: 300 * time.Millisecond, + DeltaStage: 15 * time.Second, + RMax: 24, + MaxDurationQuery: 20 * time.Millisecond, + MaxDurationObservation: 20 * time.Millisecond, + MaxDurationShouldAcceptAttestedReport: 1200 * time.Millisecond, + MaxDurationShouldTransmitAcceptedReport: 20 * time.Millisecond, + F: 1, } - onchainConfig, err := registrySettings.EncodeOnChainConfig(registrar.Address(), common.HexToAddress(chainClient.GetDefaultWallet().Address())) - require.NoError(t, err, "Error encoding onchain config") - l.Info().Msg("Done building OCR config") - ocrConfig := contracts.OCRv2Config{ - Signers: signers, - Transmitters: transmitters, - F: f, - OnchainConfig: onchainConfig, - OffchainConfigVersion: offchainConfigVersion, - OffchainConfig: offchainConfig, - } + a.SetupAutomationDeployment(t) - err = registry.SetConfig(*registrySettings, ocrConfig) - require.NoError(t, err, "Error setting registry config") + err = actions.FundChainlinkNodesAddress(chainlinkNodes[1:], chainClient, big.NewFloat(100), 0) + require.NoError(t, err, "Error funding chainlink nodes") consumerContracts := make([]contracts.KeeperConsumer, 0) triggerContracts := make([]contracts.LogEmitter, 0) utilsABI, err := automation_utils_2_1.AutomationUtilsMetaData.GetAbi() require.NoError(t, err, "Error getting automation utils abi") - registrarABI, err := registrar21.AutomationRegistrarMetaData.GetAbi() - require.NoError(t, err, "Error getting automation registrar abi") emitterABI, err := log_emitter.LogEmitterMetaData.GetAbi() require.NoError(t, err, "Error getting log emitter abi") consumerABI, err := simple_log_upkeep_counter_wrapper.SimpleLogUpkeepCounterMetaData.GetAbi() @@ -382,8 +334,8 @@ func TestLogTrigger(t *testing.T) { var bytes0 = [32]byte{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } - registrationTxHashes := make([]common.Hash, 0) - upkeepIds := make([]*big.Int, 0) + + upkeepConfigs := make([]automationv2.UpkeepConfig, 0) for i := 0; i < numberOfUpkeeps; i++ { consumerContract, err := contractDeployer.DeployAutomationSimpleLogTriggerConsumer() @@ -427,49 +379,32 @@ func TestLogTrigger(t *testing.T) { require.NoError(t, err, "Error encoding log trigger config") l.Debug().Bytes("Encoded Log Trigger Config", encodedLogTriggerConfig).Msg("Encoded Log Trigger Config") - registrationRequest, err := registrarABI.Pack( - "register", - fmt.Sprintf("LogTriggerUpkeep-%d", i), - []byte("test@mail.com"), - common.HexToAddress(consumerContract.Address()), - automationDefaultUpkeepGasLimit, - common.HexToAddress(chainClient.GetDefaultWallet().Address()), - uint8(1), - []byte("0"), - encodedLogTriggerConfig, - []byte("0"), - automationDefaultLinkFunds, - common.HexToAddress(chainClient.GetDefaultWallet().Address()), - ) - require.NoError(t, err, "Error encoding upkeep registration request") - tx, err := linkToken.TransferAndCall(registrar.Address(), automationDefaultLinkFunds, registrationRequest) - require.NoError(t, err, "Error sending upkeep registration request") - registrationTxHashes = append(registrationTxHashes, tx.Hash()) + upkeepConfig := automationv2.UpkeepConfig{ + UpkeepName: fmt.Sprintf("LogTriggerUpkeep-%d", i), + EncryptedEmail: []byte("test@mail.com"), + UpkeepContract: common.HexToAddress(consumerContract.Address()), + GasLimit: automationDefaultUpkeepGasLimit, + AdminAddress: common.HexToAddress(chainClient.GetDefaultWallet().Address()), + TriggerType: uint8(1), + CheckData: []byte("0"), + TriggerConfig: encodedLogTriggerConfig, + OffchainConfig: []byte("0"), + FundingAmount: automationDefaultLinkFunds, + } + upkeepConfigs = append(upkeepConfigs, upkeepConfig) } + registrationTxHashes, err := a.RegisterUpkeeps(upkeepConfigs) + require.NoError(t, err, "Error registering upkeeps") + err = chainClient.WaitForEvents() - require.NoError(t, err, "Failed waiting for upkeeps to be registered") - - for _, txHash := range registrationTxHashes { - receipt, err := chainClient.GetTxReceipt(txHash) - require.NoError(t, err, "Registration tx should be completed") - var upkeepId *big.Int - for _, rawLog := range receipt.Logs { - parsedUpkeepId, err := registry.ParseUpkeepIdFromRegisteredLog(rawLog) - if err == nil { - upkeepId = parsedUpkeepId - break - } - } - require.NotNil(t, upkeepId, "Upkeep ID should be found after registration") - l.Debug(). - Str("TxHash", txHash.String()). - Str("Upkeep ID", upkeepId.String()). - Msg("Found upkeepId in tx hash") - upkeepIds = append(upkeepIds, upkeepId) - } - l.Info().Msg("Successfully registered all Automation Consumer Contracts") - l.Info().Interface("Upkeep IDs", upkeepIds).Msg("Upkeep IDs") + require.NoError(t, err, "Failed waiting for upkeeps to register") + + upkeepIds, err := a.ConfirmUpkeepsRegistered(registrationTxHashes) + require.NoError(t, err, "Error confirming upkeeps registered") + + l.Info().Msg("Successfully registered all Automation Upkeeps") + l.Info().Interface("Upkeep IDs", upkeepIds).Msg("Upkeeps Registered") l.Info().Str("STARTUP_WAIT_TIME", StartupWaitTime.String()).Msg("Waiting for plugin to start") time.Sleep(StartupWaitTime) @@ -523,7 +458,7 @@ func TestLogTrigger(t *testing.T) { upkeepDelays := make([][]int64, 0) var numberOfEventsEmitted int - var batchSize = 500 + var batchSize uint64 = 500 for _, gen := range p.Generators { numberOfEventsEmitted += len(gen.GetData().OKData.Data) @@ -531,38 +466,47 @@ func TestLogTrigger(t *testing.T) { numberOfEventsEmitted = numberOfEventsEmitted * numberOfEvents l.Info().Int("Number of Events Emitted", numberOfEventsEmitted).Msg("Number of Events Emitted") - if endBlock-startBlock < uint64(batchSize) { - batchSize = int(endBlock - startBlock) + if endBlock-startBlock < batchSize { + batchSize = endBlock - startBlock } - for cIter, consumerContract := range consumerContracts { + for _, consumerContract := range consumerContracts { var ( logs []types.Log address = common.HexToAddress(consumerContract.Address()) timeout = 5 * time.Second ) - for fromBlock := startBlock; fromBlock < endBlock; fromBlock += uint64(batchSize) + 1 { - var ( - filterQuery = geth.FilterQuery{ - Addresses: []common.Address{address}, - FromBlock: big.NewInt(0).SetUint64(fromBlock), - ToBlock: big.NewInt(0).SetUint64(fromBlock + uint64(batchSize)), - Topics: [][]common.Hash{{consumerABI.Events["PerformingUpkeep"].ID}}, + for fromBlock := startBlock; fromBlock < endBlock; fromBlock += batchSize + 1 { + filterQuery := geth.FilterQuery{ + Addresses: []common.Address{address}, + FromBlock: big.NewInt(0).SetUint64(fromBlock), + ToBlock: big.NewInt(0).SetUint64(fromBlock + batchSize), + Topics: [][]common.Hash{{consumerABI.Events["PerformingUpkeep"].ID}}, + } + err = fmt.Errorf("initial error") // to ensure our for loop runs at least once + for err != nil { + var ( + logsInBatch []types.Log + ) + ctx2, cancel := context.WithTimeout(ctx, timeout) + logsInBatch, err = chainClient.FilterLogs(ctx2, filterQuery) + cancel() + if err != nil { + l.Error().Err(err). + Interface("FilterQuery", filterQuery). + Str("Contract Address", consumerContract.Address()). + Str("Timeout", timeout.String()). + Msg("Error getting logs") + timeout = time.Duration(math.Min(float64(timeout)*2, float64(2*time.Minute))) + continue } - ) - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, timeout) - logsInBatch, err := chainClient.FilterLogs(ctx, filterQuery) - cancel() - if err != nil { - l.Error().Err(err). + l.Info(). Interface("FilterQuery", filterQuery). Str("Contract Address", consumerContract.Address()). Str("Timeout", timeout.String()). - Msg("Error getting logs") + Msg("Collected logs") + logs = append(logs, logsInBatch...) } - logs = append(logs, logsInBatch...) - time.Sleep(time.Millisecond * 500) } if len(logs) > 0 { @@ -582,9 +526,6 @@ func TestLogTrigger(t *testing.T) { } upkeepDelays = append(upkeepDelays, delay) } - if (cIter+1)%batchSize == 0 { - time.Sleep(time.Millisecond * 500) - } } l.Info().Interface("Upkeep Delays", upkeepDelays).Msg("Upkeep Delays") @@ -596,6 +537,8 @@ func TestLogTrigger(t *testing.T) { } avg, median, ninetyPct, ninetyNinePct, maximum := testreporters.IntListStats(allUpkeepDelays) + eventsMissed := numberOfEventsEmitted - len(allUpkeepDelays) + percentMissed := float64(eventsMissed) / float64(numberOfEventsEmitted) * 100 l.Info(). Float64("Average", avg).Int64("Median", median). Int64("90th Percentile", ninetyPct).Int64("99th Percentile", ninetyNinePct). @@ -604,13 +547,15 @@ func TestLogTrigger(t *testing.T) { l.Info(). Int("Total Perform Count", len(allUpkeepDelays)). Int("Total Events Emitted", numberOfEventsEmitted). - Int("Total Events Missed", numberOfEventsEmitted-len(allUpkeepDelays)). + Int("Total Events Missed", eventsMissed). + Float64("Percent Missed", percentMissed). Msg("Test completed") testReport := fmt.Sprintf("Upkeep Delays in seconds\nAverage: %f\nMedian: %d\n90th Percentile: %d\n"+ - "99th Percentile: %d\nMax: %d\nTotal Perform Count: %d\n\nTotal Events Emitted: %d\nTotal Events Missed: %d\nTest Duration: %s\n", + "99th Percentile: %d\nMax: %d\nTotal Perform Count: %d\n\nTotal Events Emitted: %d\nTotal Events Missed: %d\n"+ + "Percent Missed: %f\nTest Duration: %s\n", avg, median, ninetyPct, ninetyNinePct, maximum, len(allUpkeepDelays), numberOfEventsEmitted, - numberOfEventsEmitted-len(allUpkeepDelays), testDuration.String()) + eventsMissed, percentMissed, testDuration.String()) err = sendSlackNotification("Finished", l, testEnvironment.Cfg.Namespace, strconv.Itoa(numberofNodes), strconv.FormatInt(startTime.UnixMilli(), 10), strconv.FormatInt(endTime.UnixMilli(), 10), diff --git a/integration-tests/smoke/automation_test.go b/integration-tests/smoke/automation_test.go index 81b484f4e73..f72843e77e8 100644 --- a/integration-tests/smoke/automation_test.go +++ b/integration-tests/smoke/automation_test.go @@ -10,12 +10,20 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/common" + ctfTestEnv "github.com/smartcontractkit/chainlink-testing-framework/docker/test_env" + + ocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + + ocr2keepers30config "github.com/smartcontractkit/chainlink-automation/pkg/v3/config" + + "github.com/smartcontractkit/chainlink/integration-tests/actions/automationv2" + "github.com/kelseyhightower/envconfig" + + "github.com/ethereum/go-ethereum/common" "github.com/onsi/gomega" "github.com/stretchr/testify/require" - "github.com/smartcontractkit/chainlink-testing-framework/blockchain" "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" @@ -27,7 +35,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/store/models" "github.com/smartcontractkit/chainlink/integration-tests/actions" - "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" "github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum" "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" @@ -37,28 +44,13 @@ import ( var utilsABI = cltypes.MustGetABI(automation_utils_2_1.AutomationUtilsABI) const ( - automationDefaultUpkeepGasLimit = uint32(2500000) - automationDefaultLinkFunds = int64(9e18) - automationDefaultUpkeepsToDeploy = 10 - automationExpectedData = "abcdef" - defaultAmountOfUpkeeps = 2 + automationDefaultUpkeepGasLimit = uint32(2500000) + automationDefaultLinkFunds = int64(9e18) + automationExpectedData = "abcdef" + defaultAmountOfUpkeeps = 2 ) var ( - defaultOCRRegistryConfig = contracts.KeeperRegistrySettings{ - PaymentPremiumPPB: uint32(200000000), - FlatFeeMicroLINK: uint32(0), - BlockCountPerTurn: big.NewInt(10), - CheckGasLimit: uint32(2500000), - StalenessSeconds: big.NewInt(90000), - GasCeilingMultiplier: uint16(1), - MinUpkeepSpend: big.NewInt(0), - MaxPerformGas: uint32(5000000), - FallbackGasPrice: big.NewInt(2e11), - FallbackLinkPrice: big.NewInt(2e18), - MaxCheckDataSize: uint32(5000), - MaxPerformDataSize: uint32(5000), - } automationDefaultRegistryConfig = contracts.KeeperRegistrySettings{ PaymentPremiumPPB: uint32(200000000), FlatFeeMicroLINK: uint32(0), @@ -124,17 +116,17 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) { isMercuryV03 := name == "registry_2_1_with_mercury_v03" isMercury := isMercuryV02 || isMercuryV03 - chainClient, _, contractDeployer, linkToken, registry, registrar, testEnv := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, isMercuryV02, isMercuryV03, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, isMercuryV02, isMercuryV03, ) consumers, upkeepIDs := actions.DeployConsumers( t, - registry, - registrar, - linkToken, - contractDeployer, - chainClient, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, @@ -155,7 +147,7 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) { privilegeConfigBytes, _ := json.Marshal(streams.UpkeepPrivilegeConfig{ MercuryEnabled: true, }) - if err := registry.SetUpkeepPrivilegeConfig(upkeepIDs[i], privilegeConfigBytes); err != nil { + if err := a.Registry.SetUpkeepPrivilegeConfig(upkeepIDs[i], privilegeConfigBytes); err != nil { l.Error().Msg("Error when setting upkeep privilege config") return } @@ -184,7 +176,7 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) { expect := 5 // Upgrade the nodes one at a time and check that the upkeeps are still being performed for i := 0; i < 5; i++ { - err = actions.UpgradeChainlinkNodeVersionsLocal(upgradeImage, upgradeVersion, testEnv.ClCluster.Nodes[i]) + err = actions.UpgradeChainlinkNodeVersionsLocal(upgradeImage, upgradeVersion, a.DockerEnv.ClCluster.Nodes[i]) require.NoError(t, err, "Error when upgrading node %d", i) time.Sleep(time.Second * 10) expect = expect + 5 @@ -203,11 +195,11 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) { // Cancel all the registered upkeeps via the registry for i := 0; i < len(upkeepIDs); i++ { - err := registry.CancelUpkeep(upkeepIDs[i]) + err := a.Registry.CancelUpkeep(upkeepIDs[i]) require.NoError(t, err, "Could not cancel upkeep at index %d", i) } - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error encountered when waiting for upkeeps to be cancelled") var countersAfterCancellation = make([]*big.Int, len(upkeepIDs)) @@ -238,17 +230,17 @@ func TestSetUpkeepTriggerConfig(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, ethereum.RegistryVersion_2_1, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, ethereum.RegistryVersion_2_1, automationDefaultRegistryConfig, false, false, ) consumers, upkeepIDs := actions.DeployConsumers( t, - registry, - registrar, - linkToken, - contractDeployer, - chainClient, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, @@ -313,11 +305,11 @@ func TestSetUpkeepTriggerConfig(t *testing.T) { return } - err = registry.SetUpkeepTriggerConfig(upkeepIDs[i], encodedLogTriggerConfig) + err = a.Registry.SetUpkeepTriggerConfig(upkeepIDs[i], encodedLogTriggerConfig) require.NoError(t, err, "Could not set upkeep trigger config at index %d", i) } - err := chainClient.WaitForEvents() + err := a.ChainClient.WaitForEvents() require.NoError(t, err, "Error encountered when waiting for setting trigger config for upkeeps") var countersAfterSetNoMatch = make([]*big.Int, len(upkeepIDs)) @@ -361,11 +353,11 @@ func TestSetUpkeepTriggerConfig(t *testing.T) { return } - err = registry.SetUpkeepTriggerConfig(upkeepIDs[i], encodedLogTriggerConfig) + err = a.Registry.SetUpkeepTriggerConfig(upkeepIDs[i], encodedLogTriggerConfig) require.NoError(t, err, "Could not set upkeep trigger config at index %d", i) } - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error encountered when waiting for setting trigger config for upkeeps") var countersAfterSetMatch = make([]*big.Int, len(upkeepIDs)) @@ -413,11 +405,23 @@ func TestAutomationAddFunds(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) - consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, defaultAmountOfUpkeeps, big.NewInt(1), automationDefaultUpkeepGasLimit, false, false) + consumers, upkeepIDs := actions.DeployConsumers( + t, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, + defaultAmountOfUpkeeps, + big.NewInt(1), + automationDefaultUpkeepGasLimit, + false, + false, + ) gom := gomega.NewGomegaWithT(t) // Since the upkeep is currently underfunded, check that it doesn't get executed @@ -429,15 +433,15 @@ func TestAutomationAddFunds(t *testing.T) { }, "2m", "1s").Should(gomega.Succeed()) // ~1m for setup, 1m assertion // Grant permission to the registry to fund the upkeep - err := linkToken.Approve(registry.Address(), big.NewInt(9e18)) + err := a.LinkToken.Approve(a.Registry.Address(), big.NewInt(9e18)) require.NoError(t, err, "Could not approve permissions for the registry on the link token contract") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for events") // Add funds to the upkeep whose ID we know from above - err = registry.AddUpkeepFunds(upkeepIDs[0], big.NewInt(9e18)) + err = a.Registry.AddUpkeepFunds(upkeepIDs[0], big.NewInt(9e18)) require.NoError(t, err, "Unable to add upkeep") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for events") // Now the new upkeep should be performing because we added enough funds @@ -464,11 +468,23 @@ func TestAutomationPauseUnPause(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) - consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, false, false) + consumers, upkeepIDs := actions.DeployConsumers( + t, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, + defaultAmountOfUpkeeps, + big.NewInt(automationDefaultLinkFunds), + automationDefaultUpkeepGasLimit, + false, + false, + ) gom := gomega.NewGomegaWithT(t) gom.Eventually(func(g gomega.Gomega) { @@ -484,11 +500,11 @@ func TestAutomationPauseUnPause(t *testing.T) { // pause all the registered upkeeps via the registry for i := 0; i < len(upkeepIDs); i++ { - err := registry.PauseUpkeep(upkeepIDs[i]) + err := a.Registry.PauseUpkeep(upkeepIDs[i]) require.NoError(t, err, "Could not pause upkeep at index %d", i) } - err := chainClient.WaitForEvents() + err := a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for upkeeps to be paused") var countersAfterPause = make([]*big.Int, len(upkeepIDs)) @@ -513,11 +529,11 @@ func TestAutomationPauseUnPause(t *testing.T) { // unpause all the registered upkeeps via the registry for i := 0; i < len(upkeepIDs); i++ { - err := registry.UnpauseUpkeep(upkeepIDs[i]) + err := a.Registry.UnpauseUpkeep(upkeepIDs[i]) require.NoError(t, err, "Could not unpause upkeep at index %d", i) } - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for upkeeps to be unpaused") gom.Eventually(func(g gomega.Gomega) { @@ -547,11 +563,23 @@ func TestAutomationRegisterUpkeep(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) - consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, false, false) + consumers, upkeepIDs := actions.DeployConsumers( + t, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, + defaultAmountOfUpkeeps, + big.NewInt(automationDefaultLinkFunds), + automationDefaultUpkeepGasLimit, + false, + false, + ) var initialCounters = make([]*big.Int, len(upkeepIDs)) gom := gomega.NewGomegaWithT(t) @@ -571,8 +599,8 @@ func TestAutomationRegisterUpkeep(t *testing.T) { } }, "4m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~1m for performing each upkeep once, ~2m buffer - newConsumers, _ := actions.RegisterNewUpkeeps(t, contractDeployer, chainClient, linkToken, - registry, registrar, automationDefaultUpkeepGasLimit, 1) + newConsumers, _ := actions.RegisterNewUpkeeps(t, a.Deployer, a.ChainClient, a.LinkToken, + a.Registry, a.Registrar, automationDefaultUpkeepGasLimit, 1) // We know that newConsumers has size 1, so we can just use the newly registered upkeep. newUpkeep := newConsumers[0] @@ -618,11 +646,23 @@ func TestAutomationPauseRegistry(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) - consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, false, false) + consumers, upkeepIDs := actions.DeployConsumers( + t, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, + defaultAmountOfUpkeeps, + big.NewInt(automationDefaultLinkFunds), + automationDefaultUpkeepGasLimit, + false, + false, + ) gom := gomega.NewGomegaWithT(t) // Observe that the upkeeps which are initially registered are performing @@ -636,9 +676,9 @@ func TestAutomationPauseRegistry(t *testing.T) { }, "4m", "1s").Should(gomega.Succeed()) // ~1m for cluster setup, ~1m for performing each upkeep once, ~2m buffer // Pause the registry - err := registry.Pause() + err := a.Registry.Pause() require.NoError(t, err, "Error pausing registry") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for registry to pause") // Store how many times each upkeep performed once the registry was successfully paused @@ -676,13 +716,25 @@ func TestAutomationKeeperNodesDown(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) - consumers, upkeepIDs := actions.DeployConsumers(t, registry, registrar, linkToken, contractDeployer, chainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, false, false) + consumers, upkeepIDs := actions.DeployConsumers( + t, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, + defaultAmountOfUpkeeps, + big.NewInt(automationDefaultLinkFunds), + automationDefaultUpkeepGasLimit, + false, + false, + ) gom := gomega.NewGomegaWithT(t) - nodesWithoutBootstrap := chainlinkNodes[1:] + nodesWithoutBootstrap := a.ChainlinkNodes[1:] var initialCounters = make([]*big.Int, len(upkeepIDs)) @@ -691,6 +743,7 @@ func TestAutomationKeeperNodesDown(t *testing.T) { for i := 0; i < len(upkeepIDs); i++ { counter, err := consumers[i].Counter(testcontext.Get(t)) initialCounters[i] = counter + l.Info().Int64("Upkeeps Performed", counter.Int64()).Int("Upkeep Index", i).Msg("Number of upkeeps performed") g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Failed to retrieve consumer counter for upkeep at index %d", i) g.Expect(counter.Int64()).Should(gomega.BeNumerically(">", int64(0)), "Expected consumer counter to be greater than 0, but got %d", counter.Int64()) @@ -700,7 +753,7 @@ func TestAutomationKeeperNodesDown(t *testing.T) { // Take down 1 node. Currently, using 4 nodes so f=1 and is the max nodes that can go down. err := nodesWithoutBootstrap[0].MustDeleteJob("1") require.NoError(t, err, "Error deleting job from Chainlink node") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for blockchain events") l.Info().Msg("Successfully managed to take down the first half of the nodes") @@ -709,6 +762,7 @@ func TestAutomationKeeperNodesDown(t *testing.T) { gom.Eventually(func(g gomega.Gomega) { for i := 0; i < len(upkeepIDs); i++ { currentCounter, err := consumers[i].Counter(testcontext.Get(t)) + l.Info().Int64("Upkeeps Performed", currentCounter.Int64()).Int("Upkeep Index", i).Msg("Number of upkeeps performed") g.Expect(err).ShouldNot(gomega.HaveOccurred(), "Failed to retrieve consumer counter for upkeep at index %d", i) g.Expect(currentCounter.Int64()).Should(gomega.BeNumerically(">", initialCounters[i].Int64()), "Expected counter to have increased from initial value of %s, but got %s", @@ -721,7 +775,7 @@ func TestAutomationKeeperNodesDown(t *testing.T) { for _, nodeToTakeDown := range restOfNodesDown { err = nodeToTakeDown.MustDeleteJob("1") require.NoError(t, err, "Error deleting job from Chainlink node") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for blockchain events") } l.Info().Msg("Successfully managed to take down the second half of the nodes") @@ -761,17 +815,17 @@ func TestAutomationPerformSimulation(t *testing.T) { registryVersion := rv t.Run(name, func(t *testing.T) { t.Parallel() - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) consumersPerformance, _ := actions.DeployPerformanceConsumers( t, - registry, - registrar, - linkToken, - contractDeployer, - chainClient, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, @@ -797,7 +851,7 @@ func TestAutomationPerformSimulation(t *testing.T) { // Set performGas on consumer to be low, so that performUpkeep starts becoming successful err := consumerPerformance.SetPerformGasToBurn(testcontext.Get(t), big.NewInt(100000)) require.NoError(t, err, "Perform gas should be set successfully on consumer") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for set perform gas tx") // Upkeep should now start performing @@ -825,17 +879,17 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - chainClient, chainlinkNodes, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) consumersPerformance, upkeepIDs := actions.DeployPerformanceConsumers( t, - registry, - registrar, - linkToken, - contractDeployer, - chainClient, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, @@ -846,7 +900,7 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) { ) gom := gomega.NewGomegaWithT(t) - nodesWithoutBootstrap := chainlinkNodes[1:] + nodesWithoutBootstrap := a.ChainlinkNodes[1:] consumerPerformance := consumersPerformance[0] upkeepID := upkeepIDs[0] @@ -861,9 +915,9 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) { }, "2m", "1s").Should(gomega.Succeed()) // ~1m for setup, 1m assertion // Increase gas limit for the upkeep, higher than the performGasBurn - err := registry.SetUpkeepGasLimit(upkeepID, uint32(4500000)) + err := a.Registry.SetUpkeepGasLimit(upkeepID, uint32(4500000)) require.NoError(t, err, "Error setting upkeep gas limit") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for SetUpkeepGasLimit tx") // Upkeep should now start performing @@ -878,7 +932,7 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) { // Now increase the checkGasBurn on consumer, upkeep should stop performing err = consumerPerformance.SetCheckGasToBurn(testcontext.Get(t), big.NewInt(3000000)) require.NoError(t, err, "Check gas burn should be set successfully on consumer") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for SetCheckGasToBurn tx") // Get existing performed count @@ -906,12 +960,12 @@ func TestAutomationCheckPerformGasLimit(t *testing.T) { highCheckGasLimit.CheckGasLimit = uint32(5000000) highCheckGasLimit.RegistryVersion = registryVersion - ocrConfig, err := actions.BuildAutoOCR2ConfigVarsLocal(l, nodesWithoutBootstrap, highCheckGasLimit, registrar.Address(), 30*time.Second, registry.RegistryOwnerAddress()) + ocrConfig, err := actions.BuildAutoOCR2ConfigVarsLocal(l, nodesWithoutBootstrap, highCheckGasLimit, a.Registrar.Address(), 30*time.Second, a.Registry.RegistryOwnerAddress()) require.NoError(t, err, "Error building OCR config") - err = registry.SetConfig(highCheckGasLimit, ocrConfig) + err = a.Registry.SetConfig(highCheckGasLimit, ocrConfig) require.NoError(t, err, "Registry config should be set successfully!") - err = chainClient.WaitForEvents() + err = a.ChainClient.WaitForEvents() require.NoError(t, err, "Error waiting for set config tx") // Upkeep should start performing again, and it should get regularly performed @@ -939,17 +993,17 @@ func TestUpdateCheckData(t *testing.T) { t.Run(name, func(t *testing.T) { t.Parallel() l := logging.GetTestLogger(t) - chainClient, _, contractDeployer, linkToken, registry, registrar, _ := setupAutomationTestDocker( - t, registryVersion, defaultOCRRegistryConfig, false, false, + a := setupAutomationTestDocker( + t, registryVersion, automationDefaultRegistryConfig, false, false, ) performDataChecker, upkeepIDs := actions.DeployPerformDataCheckerConsumers( t, - registry, - registrar, - linkToken, - contractDeployer, - chainClient, + a.Registry, + a.Registrar, + a.LinkToken, + a.Deployer, + a.ChainClient, defaultAmountOfUpkeeps, big.NewInt(automationDefaultLinkFunds), automationDefaultUpkeepGasLimit, @@ -970,16 +1024,16 @@ func TestUpdateCheckData(t *testing.T) { }, "2m", "1s").Should(gomega.Succeed()) // ~1m for setup, 1m assertion for i := 0; i < len(upkeepIDs); i++ { - err := registry.UpdateCheckData(upkeepIDs[i], []byte(automationExpectedData)) + err := a.Registry.UpdateCheckData(upkeepIDs[i], []byte(automationExpectedData)) require.NoError(t, err, "Could not update check data for upkeep at index %d", i) } - err := chainClient.WaitForEvents() + err := a.ChainClient.WaitForEvents() require.NoError(t, err, "Error while waiting for check data update") // retrieve new check data for all upkeeps for i := 0; i < len(upkeepIDs); i++ { - upkeep, err := registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) + upkeep, err := a.Registry.GetUpkeepInfo(testcontext.Get(t), upkeepIDs[i]) require.NoError(t, err, "Failed to get upkeep info at index %d", i) require.Equal(t, []byte(automationExpectedData), upkeep.CheckData, "Upkeep data not as expected") } @@ -1009,15 +1063,7 @@ func setupAutomationTestDocker( registryConfig contracts.KeeperRegistrySettings, isMercuryV02 bool, isMercuryV03 bool, -) ( - blockchain.EVMClient, - []*client.ChainlinkClient, - contracts.ContractDeployer, - contracts.LinkToken, - contracts.KeeperRegistry, - contracts.KeeperRegistrar, - *test_env.CLClusterTestEnv, -) { +) automationv2.AutomationTest { require.False(t, isMercuryV02 && isMercuryV03, "Cannot run test with both Mercury V02 and V03 on") l := logging.GetTestLogger(t) @@ -1080,16 +1126,6 @@ func setupAutomationTestDocker( err = env.FundChainlinkNodes(big.NewFloat(testConfig.ChainlinkNodeFunding)) require.NoError(t, err, "Error funding CL nodes") - if isMercuryV02 { - output := `{"chainlinkBlob":"0x0001c38d71fed6c320b90e84b6f559459814d068e2a1700adc931ca9717d4fe70000000000000000000000000000000000000000000000000000000001a80b52b4bf1233f9cb71144a253a1791b202113c4ab4a92fa1b176d684b4959666ff8200000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000260000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001004254432d5553442d415242495452554d2d544553544e4554000000000000000000000000000000000000000000000000000000000000000000000000645570be000000000000000000000000000000000000000000000000000002af2b818dc5000000000000000000000000000000000000000000000000000002af2426faf3000000000000000000000000000000000000000000000000000002af32dc209700000000000000000000000000000000000000000000000000000000012130f8df0a9745bb6ad5e2df605e158ba8ad8a33ef8a0acf9851f0f01668a3a3f2b68600000000000000000000000000000000000000000000000000000000012130f60000000000000000000000000000000000000000000000000000000000000002c4a7958dce105089cf5edb68dad7dcfe8618d7784eb397f97d5a5fade78c11a58275aebda478968e545f7e3657aba9dcbe8d44605e4c6fde3e24edd5e22c94270000000000000000000000000000000000000000000000000000000000000002459c12d33986018a8959566d145225f0c4a4e61a9a3f50361ccff397899314f0018162cf10cd89897635a0bb62a822355bd199d09f4abe76e4d05261bb44733d"}` - err = env.MockAdapter.SetStringValuePath("/client", []string{http.MethodGet, http.MethodPost}, map[string]string{"Content-Type": "application/json"}, output) - require.NoError(t, err) - } - if isMercuryV03 { - output := `{"reports":[{"feedID":"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000","validFromTimestamp":0,"observationsTimestamp":0,"fullReport":"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}]}` - err = env.MockAdapter.SetStringValuePath("/api/v1/reports/bulk", []string{http.MethodGet, http.MethodPost}, map[string]string{"Content-Type": "application/json"}, output) - require.NoError(t, err) - } } else { env, err = test_env.NewCLTestEnvBuilder(). WithTestLogger(t). @@ -1105,31 +1141,82 @@ func setupAutomationTestDocker( env.ParallelTransactions(true) nodeClients := env.ClCluster.NodeAPIs() - workerNodes := nodeClients[1:] - linkToken, err := env.ContractDeployer.DeployLinkTokenContract() - require.NoError(t, err, "Error deploying LINK token") + a := automationv2.NewAutomationTestDocker(env.EVMClient, env.ContractDeployer, nodeClients) + a.MercuryCredentialName = "cred1" + a.RegistrySettings = registryConfig + a.RegistrarSettings = contracts.KeeperRegistrarSettings{ + AutoApproveConfigType: uint8(2), + AutoApproveMaxAllowed: 1000, + MinLinkJuels: big.NewInt(0), + } + a.PluginConfig = ocr2keepers30config.OffchainConfig{ + TargetProbability: "0.999", + TargetInRounds: 1, + PerformLockoutWindow: 3_600_000, // Intentionally set to be higher than in prod for testing purpose + GasLimitPerReport: 10_300_000, + GasOverheadPerUpkeep: 300_000, + MinConfirmations: 0, + MaxUpkeepBatchSize: 10, + } + a.PublicConfig = ocr3.PublicConfig{ + DeltaProgress: 10 * time.Second, + DeltaResend: 15 * time.Second, + DeltaInitial: 500 * time.Millisecond, + DeltaRound: 1000 * time.Millisecond, + DeltaGrace: 200 * time.Millisecond, + DeltaCertifiedCommitRequest: 300 * time.Millisecond, + DeltaStage: 30 * time.Second, + RMax: 24, + MaxDurationQuery: 20 * time.Millisecond, + MaxDurationObservation: 20 * time.Millisecond, + MaxDurationShouldAcceptAttestedReport: 1200 * time.Millisecond, + MaxDurationShouldTransmitAcceptedReport: 20 * time.Millisecond, + F: 1, + } - registry, registrar := actions.DeployAutoOCRRegistryAndRegistrar( - t, - registryVersion, - registryConfig, - linkToken, - env.ContractDeployer, - env.EVMClient, - ) + a.SetupAutomationDeployment(t) + a.SetDockerEnv(env) - // Fund the registry with LINK - err = linkToken.Transfer(registry.Address(), big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(int64(defaultAmountOfUpkeeps)))) - require.NoError(t, err, "Funding keeper registry contract shouldn't fail") + if isMercuryV02 || isMercuryV03 { + var imposters []ctfTestEnv.KillgraveImposter + mercuryv03Mock200 := ctfTestEnv.KillgraveImposter{ + Request: ctfTestEnv.KillgraveRequest{ + Method: http.MethodGet, + Endpoint: "/api/v1/reports/bulk", + SchemaFile: nil, + Params: &map[string]string{"feedIDs": "0x00028c915d6af0fd66bba2d0fc9405226bca8d6806333121a7d9832103d1563c", "timestamp": "{[\\d+]}"}, + Headers: nil, + }, + Response: ctfTestEnv.KillgraveResponse{ + Status: 200, + Body: `{"reports":[{"feedID":"0x00028c915d6af0fd66bba2d0fc9405226bca8d6806333121a7d9832103d1563c","validFromTimestamp":0,"observationsTimestamp":0,"fullReport":"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}]}`, + BodyFile: nil, + Headers: nil, + Delay: nil, + }, + } - err = actions.CreateOCRKeeperJobsLocal(l, nodeClients, registry.Address(), network.ChainID, 0, registryVersion) - require.NoError(t, err, "Error creating OCR Keeper Jobs") - ocrConfig, err := actions.BuildAutoOCR2ConfigVarsLocal(l, workerNodes, registryConfig, registrar.Address(), 30*time.Second, registry.RegistryOwnerAddress()) - require.NoError(t, err, "Error building OCR config vars") - err = registry.SetConfig(automationDefaultRegistryConfig, ocrConfig) - require.NoError(t, err, "Registry config should be set successfully") - require.NoError(t, env.EVMClient.WaitForEvents(), "Waiting for config to be set") + mercuryv02Mock200 := ctfTestEnv.KillgraveImposter{ + Request: ctfTestEnv.KillgraveRequest{ + Method: http.MethodGet, + Endpoint: "/client", + SchemaFile: nil, + Params: &map[string]string{"feedIdHex": "{0x00028c915d6af0fd66bba2d0fc9405226bca8d6806333121a7d9832103d1563c|0x4554482d5553442d415242495452554d2d544553544e45540000000000000000}", "blockNumber": "{[\\d+]}"}, + Headers: nil, + }, + Response: ctfTestEnv.KillgraveResponse{ + Status: 200, + Body: `{"chainlinkBlob":"0x0001c38d71fed6c320b90e84b6f559459814d068e2a1700adc931ca9717d4fe70000000000000000000000000000000000000000000000000000000001a80b52b4bf1233f9cb71144a253a1791b202113c4ab4a92fa1b176d684b4959666ff8200000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000260000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001004254432d5553442d415242495452554d2d544553544e4554000000000000000000000000000000000000000000000000000000000000000000000000645570be000000000000000000000000000000000000000000000000000002af2b818dc5000000000000000000000000000000000000000000000000000002af2426faf3000000000000000000000000000000000000000000000000000002af32dc209700000000000000000000000000000000000000000000000000000000012130f8df0a9745bb6ad5e2df605e158ba8ad8a33ef8a0acf9851f0f01668a3a3f2b68600000000000000000000000000000000000000000000000000000000012130f60000000000000000000000000000000000000000000000000000000000000002c4a7958dce105089cf5edb68dad7dcfe8618d7784eb397f97d5a5fade78c11a58275aebda478968e545f7e3657aba9dcbe8d44605e4c6fde3e24edd5e22c94270000000000000000000000000000000000000000000000000000000000000002459c12d33986018a8959566d145225f0c4a4e61a9a3f50361ccff397899314f0018162cf10cd89897635a0bb62a822355bd199d09f4abe76e4d05261bb44733d"}`, + BodyFile: nil, + Headers: nil, + Delay: nil, + }, + } + + imposters = append(imposters, mercuryv03Mock200, mercuryv02Mock200) + a.SetupMercuryMock(t, imposters) + } - return env.EVMClient, nodeClients, env.ContractDeployer, linkToken, registry, registrar, env + return *a }