Test updates for release branch

.github/workflows/integration-tests-publish.yml

@@ -36,7 +36,6 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
       - name: Setup Other Tags If Not Workflow Dispatch
         id: tags
-        if: github.event_name != 'workflow_dispatch'
         run: |
           echo "other_tags=${ECR_TAG}" >> $GITHUB_OUTPUT
           echo 'release_tag="${{ format('{0}.dkr.ecr.{1}.amazonaws.com/chainlink-ccip-tests:{2}', secrets.QA_AWS_ACCOUNT_NUMBER, secrets.QA_AWS_REGION, github.ref_name) }}"' >> $GITHUB_OUTPUT

integration-tests/.tool-versions

@@ -1,4 +1,4 @@
-golang 1.21.5
+golang 1.21.7
 k3d 5.4.6
 kubectl 1.25.5
 nodejs 18.13.0

integration-tests/Makefile

@@ -130,7 +130,7 @@ test_node_migrations: install_gotestfmt ## Run all node migration tests.
 	go test -timeout 1h -count=1 -json $(args) ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt
 
 .PHONY: test_node_migrations_simulated
-test_node_migrations_simulated: install_gotestfmt 
+test_node_migrations_simulated: install_gotestfmt
 	TEST_LOG_LEVEL="disabled" \
 	go test -timeout 1h -count=1 -json $(args) ./migration 2>&1 | tee /tmp/gotest.log | gotestfmt
 
@@ -145,30 +145,22 @@ test_node_migrations_simulated_verbose:
 # Soak
 .PHONY: test_soak_ocr
 test_soak_ocr:
-	go test -v -count=1 -run TestOCRSoak ./soak
-
-.PHONY: test_soak_ocr_simulated
-test_soak_ocr_simulated:
+	. ./scripts/check_base64_env_var.sh
 	go test -v -count=1 -run TestOCRSoak ./soak
 
 .PHONY: test_soak_forwarder_ocr
 test_soak_forwarder_ocr:
-	go test -v -count=1 -run TestForwarderOCRSoak ./soak
-
-.PHONY: test_soak_forwarder_ocr_simulated
-test_soak_forwarder_ocr_simulated:
+	. ./scripts/check_base64_env_var.sh
 	go test -v -count=1 -run TestForwarderOCRSoak ./soak
 
 .PHONY: test_soak_automation
 test_soak_automation:
-	go test -v -run ^TestAutomationBenchmark$$ ./benchmark -count=1
-
-.PHONY: test_soak_automation_simulated
-test_soak_automation_simulated:
+	. ./scripts/check_base64_env_var.sh
 	go test -v -run ^TestAutomationBenchmark$$ ./benchmark -count=1
 
 .PHONY: test_benchmark_automation
 test_benchmark_automation: ## Run the automation benchmark tests
+	. ./scripts/check_base64_env_var.sh
 	go test -timeout 30m -v -run ^TestAutomationBenchmark$$ ./benchmark -count=1
 
 .PHONY: test_reorg_automation

integration-tests/README.md

@@ -14,7 +14,18 @@ If you have previously run these smoke tests using GitHub Actions or some sort o
 
 ## Configure
 
-See the [example.env](./example.env) file for environment variables you can set to configure things like network settings, Chainlink version, and log level. Remember to use `source .env` to activate your settings.
+We have finished the first pass at moving all test configuration from env vars to TOML files. All product-related configuration is already in TOML files, but env vars are still used to control the log level, Slack notifications, and Kubernetes-related settings. See the [example.env](./example.env) file for how to set these environment variables.
+
+We have defined some sensible defaults for all products, you can find them in `./testconfig/<product>/<product>.toml` files. Each product folder contains an `example.toml` file that describes all options. If you wish to override these values, you can do so by creating a `./testconfig/overrides.toml`. A detailed description of TOML configuration can be found in the [testconfig README](./testconfig/README.md), but if you want to run some tests using default values all you need to do is provide the Chainlink image and version you want to run tests on:
+```toml
+# ./testconfig/overrides.toml
+
+[ChainlinkImage]
+image = "your image name"
+version = "your tag"
+```
+
+The `./testconfig/overrides.toml` file **should never be committed** and has been added to the [.gitignore](../.gitignore) file as it can often contain secrets like private keys and RPC URLs.
 
 ## Build
 
@@ -26,10 +37,10 @@ e.g.
 
 `make build_docker_image image=chainlink tag=test-tag`
 
-You'll want to set the `CHAINLINK_IMAGE` and `CHAINLINK_VERSION` env values appropriately as well. See [example.env](./example.env) for more details.
-
 ## Run
 
+Ensure you have created a `./testconfig/overrides.toml` file with your desired Chainlink image and version.
+
 `go test ./smoke/<product>_test.go`
 
 Most test files have a couple of tests, it's recommended to look into the file and focus on a specific one if possible. 90% of the time this will probably be the `Basic` test. See [ocr_test.go](./smoke/ocr_test.go) for example, which contains the `TestOCRBasic` test.

integration-tests/actions/actions.go

@@ -2,14 +2,18 @@
 package actions
 
 import (
+	"context"
 	"crypto/ecdsa"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
 	"strings"
+	"sync"
 	"testing"
+	"time"
 
+	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 
@@ -26,7 +30,7 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/smartcontractkit/chainlink-testing-framework/testreporters"
 	"github.com/smartcontractkit/chainlink-testing-framework/utils/conversions"
-
+	"github.com/smartcontractkit/chainlink-testing-framework/utils/testcontext"
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 )
@@ -35,7 +39,8 @@ import (
 // Example: When deploying 1000 contracts, stop every ContractDeploymentInterval have been deployed to wait before continuing
 var ContractDeploymentInterval = 200
 
-// FundChainlinkNodes will fund all of the provided Chainlink nodes with a set amount of native currency
+// FundChainlinkNodes will fund all of the provided Chainlink nodes with a set amountCreateOCRv2Jobs of native currency
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.FundChainlinkNodes
 func FundChainlinkNodes(
 	nodes []*client.ChainlinkK8sClient,
 	client blockchain.EVMClient,
@@ -299,6 +304,7 @@ func TeardownSuite(
 
 // TeardownRemoteSuite is used when running a test within a remote-test-runner, like for long-running performance and
 // soak tests
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.TeardownRemoteSuite
 func TeardownRemoteSuite(
 	t *testing.T,
 	namespace string,
@@ -484,3 +490,75 @@ func GetTxFromAddress(tx *types.Transaction) (string, error) {
 	from, err := types.Sender(types.LatestSignerForChainID(tx.ChainId()), tx)
 	return from.String(), err
 }
+
+// todo - move to CTF
+func GetTxByHash(ctx context.Context, client blockchain.EVMClient, hash common.Hash) (*types.Transaction, bool, error) {
+	return client.(*blockchain.EthereumMultinodeClient).
+		DefaultClient.(*blockchain.EthereumClient).
+		Client.
+		TransactionByHash(ctx, hash)
+}
+
+// todo - move to CTF
+func DecodeTxInputData(abiString string, data []byte) (map[string]interface{}, error) {
+	jsonABI, err := abi.JSON(strings.NewReader(abiString))
+	if err != nil {
+		return nil, err
+	}
+	methodSigData := data[:4]
+	inputsSigData := data[4:]
+	method, err := jsonABI.MethodById(methodSigData)
+	if err != nil {
+		return nil, err
+	}
+	inputsMap := make(map[string]interface{})
+	if err := method.Inputs.UnpackIntoMap(inputsMap, inputsSigData); err != nil {
+		return nil, err
+	}
+	return inputsMap, nil
+}
+
+// todo - move to EVMClient
+func WaitForBlockNumberToBe(
+	waitForBlockNumberToBe uint64,
+	client blockchain.EVMClient,
+	wg *sync.WaitGroup,
+	timeout time.Duration,
+	t testing.TB,
+) (uint64, error) {
+	blockNumberChannel := make(chan uint64)
+	errorChannel := make(chan error)
+	testContext, testCancel := context.WithTimeout(context.Background(), timeout)
+	defer testCancel()
+
+	ticker := time.NewTicker(time.Second * 1)
+	var blockNumber uint64
+	for {
+		select {
+		case <-testContext.Done():
+			ticker.Stop()
+			wg.Done()
+			return blockNumber,
+				fmt.Errorf("timeout waiting for Block Number to be: %d. Last recorded block number was: %d",
+					waitForBlockNumberToBe, blockNumber)
+		case <-ticker.C:
+			go func() {
+				currentBlockNumber, err := client.LatestBlockNumber(testcontext.Get(t))
+				if err != nil {
+					errorChannel <- err
+				}
+				blockNumberChannel <- currentBlockNumber
+			}()
+		case blockNumber = <-blockNumberChannel:
+			if blockNumber == waitForBlockNumberToBe {
+				ticker.Stop()
+				wg.Done()
+				return blockNumber, nil
+			}
+		case err := <-errorChannel:
+			ticker.Stop()
+			wg.Done()
+			return 0, err
+		}
+	}
+}

integration-tests/actions/actions_local.go

@@ -1,9 +1,7 @@
 // Package actions enables common chainlink interactions
 package actions
 
-import (
-	"github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
-)
+import "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
 
 // UpgradeChainlinkNodeVersions upgrades all Chainlink nodes to a new version, and then runs the test environment
 // to apply the upgrades

integration-tests/actions/automation_ocr_helpers.go

@@ -20,17 +20,15 @@ import (
 
 	ocr2keepers20config "github.com/smartcontractkit/chainlink-automation/pkg/v2/config"
 	ocr2keepers30config "github.com/smartcontractkit/chainlink-automation/pkg/v3/config"
-
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 
-	"github.com/smartcontractkit/chainlink/v2/core/services/job"
-	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
-	"github.com/smartcontractkit/chainlink/v2/core/store/models"
-
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts/ethereum"
+	"github.com/smartcontractkit/chainlink/v2/core/services/job"
+	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
+	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
 
 func BuildAutoOCR2ConfigVars(
@@ -65,7 +63,7 @@ func BuildAutoOCR2ConfigVarsWithKeyIndex(
 	var offchainConfigVersion uint64
 	var offchainConfig []byte
 
-	if registryConfig.RegistryVersion == ethereum.RegistryVersion_2_1 {
+	if registryConfig.RegistryVersion == ethereum.RegistryVersion_2_1 || registryConfig.RegistryVersion == ethereum.RegistryVersion_2_2 {
 		offC, err = json.Marshal(ocr2keepers30config.OffchainConfig{
 			TargetProbability:    "0.999",
 			TargetInRounds:       1,
@@ -183,12 +181,12 @@ func CreateOCRKeeperJobs(
 	bootstrapP2PId := bootstrapP2PIds.Data[0].Attributes.PeerID
 
 	var contractVersion string
-	if registryVersion == ethereum.RegistryVersion_2_1 {
+	if registryVersion == ethereum.RegistryVersion_2_2 {
+		contractVersion = "v2.1+"
+	} else if registryVersion == ethereum.RegistryVersion_2_1 {
 		contractVersion = "v2.1"
-	} else if registryVersion == ethereum.RegistryVersion_2_0 {
-		contractVersion = "v2.0"
 	} else {
-		require.FailNow(t, "v2.0 and v2.1 are the only supported versions")
+		require.FailNow(t, "v2.0, v2.1, and v2.2 are the only supported versions")
 	}
 
 	bootstrapSpec := &client.OCR2TaskJobSpec{

integration-tests/actions/ocr2_helpers.go

@@ -33,6 +33,7 @@ import (
 )
 
 // DeployOCRv2Contracts deploys a number of OCRv2 contracts and configures them with defaults
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.DeployOCRv2Contracts
 func DeployOCRv2Contracts(
 	numberOfContracts int,
 	linkTokenContract contracts.LinkToken,
@@ -85,6 +86,7 @@ func DeployOCRv2Contracts(
 	return ocrInstances, client.WaitForEvents()
 }
 
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.ConfigureOCRv2AggregatorContracts
 func ConfigureOCRv2AggregatorContracts(
 	client blockchain.EVMClient,
 	contractConfig *contracts.OCRv2Config,
@@ -266,7 +268,7 @@ func CreateOCRv2Jobs(
 	workerChainlinkNodes []*client.ChainlinkK8sClient,
 	mockserver *ctfClient.MockserverClient,
 	mockServerValue int, // Value to get from the mock server when querying the path
-	chainId uint64, // EVM chain ID
+	chainId int64, // EVM chain ID
 	forwardingAllowed bool,
 ) error {
 	// Collect P2P ID

integration-tests/actions/ocr_helpers.go

@@ -22,7 +22,7 @@ import (
 // This actions file often returns functions, rather than just values. These are used as common test helpers, and are
 // handy to have returning as functions so that Ginkgo can use them in an aesthetically pleasing way.
 
-// DeployOCRContracts deploys and funds a certain number of offchain aggregator contracts
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.DeployOCRContracts
 func DeployOCRContracts(
 	numberOfContracts int,
 	linkTokenContract contracts.LinkToken,
@@ -90,7 +90,7 @@ func DeployOCRContracts(
 	for contractCount, ocrInstance := range ocrInstances {
 		// Exclude the first node, which will be used as a bootstrapper
 		err = ocrInstance.SetConfig(
-			workerNodes,
+			contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes),
 			contracts.DefaultOffChainAggregatorConfig(len(workerNodes)),
 			transmitterAddresses,
 		)
@@ -113,6 +113,7 @@ func DeployOCRContracts(
 
 // DeployOCRContractsForwarderFlow deploys and funds a certain number of offchain
 // aggregator contracts with forwarders as effectiveTransmitters
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.DeployOCRContractsForwarderFlow
 func DeployOCRContractsForwarderFlow(
 	t *testing.T,
 	numberOfContracts int,
@@ -163,7 +164,7 @@ func DeployOCRContractsForwarderFlow(
 	for contractCount, ocrInstance := range ocrInstances {
 		// Exclude the first node, which will be used as a bootstrapper
 		err = ocrInstance.SetConfig(
-			workerNodes,
+			contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes),
 			contracts.DefaultOffChainAggregatorConfig(len(workerNodes)),
 			forwarderAddresses,
 		)
@@ -267,7 +268,7 @@ func CreateOCRJobsWithForwarder(
 	workerNodes []*client.ChainlinkK8sClient,
 	mockValue int,
 	mockserver *ctfClient.MockserverClient,
-	evmChainID string,
+	evmChainID int64,
 ) {
 	for _, ocrInstance := range ocrInstances {
 		bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys()
@@ -276,7 +277,7 @@ func CreateOCRJobsWithForwarder(
 		bootstrapSpec := &client.OCRBootstrapJobSpec{
 			Name:            fmt.Sprintf("bootstrap-%s", uuid.New().String()),
 			ContractAddress: ocrInstance.Address(),
-			EVMChainID:      evmChainID,
+			EVMChainID:      fmt.Sprint(evmChainID),
 			P2PPeerID:       bootstrapP2PId,
 			IsBootstrapPeer: true,
 		}
@@ -307,7 +308,7 @@ func CreateOCRJobsWithForwarder(
 			bootstrapPeers := []*client.ChainlinkClient{bootstrapNode.ChainlinkClient}
 			ocrSpec := &client.OCRTaskJobSpec{
 				ContractAddress:    ocrInstance.Address(),
-				EVMChainID:         evmChainID,
+				EVMChainID:         fmt.Sprint(evmChainID),
 				P2PPeerID:          nodeP2PId,
 				P2PBootstrapPeers:  bootstrapPeers,
 				KeyBundleID:        nodeOCRKeyId,
@@ -322,6 +323,7 @@ func CreateOCRJobsWithForwarder(
 }
 
 // StartNewRound requests a new round from the ocr contracts and waits for confirmation
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.StartNewRound
 func StartNewRound(
 	roundNumber int64,
 	ocrInstances []contracts.OffchainAggregator,
@@ -345,6 +347,7 @@ func StartNewRound(
 
 // WatchNewRound watches for a new OCR round, similarly to StartNewRound, but it does not explicitly request a new
 // round from the contract, as this can cause some odd behavior in some cases
+// Deprecated: we are moving away from blockchain.EVMClient, use actions_seth.WatchNewRound
 func WatchNewRound(
 	roundNumber int64,
 	ocrInstances []contracts.OffchainAggregator,
@@ -430,7 +433,7 @@ func SetAllAdapterResponsesToDifferentValues(
 }
 
 // BuildNodeContractPairID builds a UUID based on a related pair of a Chainlink node and OCR contract
-func BuildNodeContractPairID(node *client.ChainlinkK8sClient, ocrInstance contracts.OffchainAggregator) (string, error) {
+func BuildNodeContractPairID(node contracts.ChainlinkNodeWithKeysAndAddress, ocrInstance contracts.OffchainAggregator) (string, error) {
 	if node == nil {
 		return "", fmt.Errorf("chainlink node is nil")
 	}