Refactor E2E docker wrappers to use ClCluster (#10945)

* Use E2E wrapper for node and cluster from CTF

* Fix

* Fix go mod

* Fix

* Move back node and cluster wrappers to core

* Update go mod

* Update go mod

integration-tests/actions/actions_local.go

@@ -3,7 +3,6 @@ package actions
 
 import (
 	"github.com/pkg/errors"
-
 	"github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
 )
 

integration-tests/actions/vrfv2_actions/vrfv2_steps.go

@@ -212,21 +212,21 @@ func SetupLocalLoadTestEnv(nodesFunding *big.Float, subFundingLINK *big.Int) (*t
 	if err != nil {
 		return nil, nil, [32]byte{}, err
 	}
-	jobs, err := CreateVRFV2Jobs(env.GetAPIs(), vrfv2Contracts.Coordinator, env.EVMClient, vrfConst.MinimumConfirmations)
+	jobs, err := CreateVRFV2Jobs(env.ClCluster.NodeAPIs(), vrfv2Contracts.Coordinator, env.EVMClient, vrfConst.MinimumConfirmations)
 	if err != nil {
 		return nil, nil, [32]byte{}, err
 	}
 	// this part is here because VRFv2 can work with only a specific key
 	// [[EVM.KeySpecific]]
 	//	Key = '...'
-	addr, err := env.CLNodes[0].API.PrimaryEthAddress()
+	addr, err := env.ClCluster.Nodes[0].API.PrimaryEthAddress()
 	if err != nil {
 		return nil, nil, [32]byte{}, err
 	}
-	nodeConfig := node.NewConfig(env.CLNodes[0].NodeConfig,
+	nodeConfig := node.NewConfig(env.ClCluster.Nodes[0].NodeConfig,
 		node.WithVRFv2EVMEstimator(addr),
 	)
-	err = env.CLNodes[0].Restart(nodeConfig)
+	err = env.ClCluster.Nodes[0].Restart(nodeConfig)
 	if err != nil {
 		return nil, nil, [32]byte{}, err
 	}

integration-tests/actions/vrfv2plus/vrfv2plus_steps.go

@@ -3,11 +3,12 @@ package vrfv2plus
 import (
 	"context"
 	"fmt"
-	"github.com/smartcontractkit/chainlink/v2/core/assets"
-	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrfv2plus_wrapper_load_test_consumer"
 	"math/big"
 	"time"
 
+	"github.com/smartcontractkit/chainlink/v2/core/assets"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrfv2plus_wrapper_load_test_consumer"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
@@ -288,13 +289,13 @@ func SetupVRFV2_5Environment(
 		return nil, nil, nil, err
 	}
 
-	vrfKey, err := env.GetAPIs()[0].MustCreateVRFKey()
+	vrfKey, err := env.ClCluster.NodeAPIs()[0].MustCreateVRFKey()
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, ErrCreatingVRFv2PlusKey)
 	}
 	pubKeyCompressed := vrfKey.Data.ID
 
-	nativeTokenPrimaryKeyAddress, err := env.GetAPIs()[0].PrimaryEthAddress()
+	nativeTokenPrimaryKeyAddress, err := env.ClCluster.NodeAPIs()[0].PrimaryEthAddress()
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, ErrNodePrimaryKey)
 	}
@@ -310,7 +311,7 @@ func SetupVRFV2_5Environment(
 	chainID := env.EVMClient.GetChainID()
 
 	job, err := CreateVRFV2PlusJob(
-		env.GetAPIs()[0],
+		env.ClCluster.NodeAPIs()[0],
 		vrfv2_5Contracts.Coordinator.Address(),
 		nativeTokenPrimaryKeyAddress,
 		pubKeyCompressed,
@@ -324,14 +325,14 @@ func SetupVRFV2_5Environment(
 	// this part is here because VRFv2 can work with only a specific key
 	// [[EVM.KeySpecific]]
 	//	Key = '...'
-	addr, err := env.CLNodes[0].API.PrimaryEthAddress()
+	addr, err := env.ClCluster.Nodes[0].API.PrimaryEthAddress()
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, ErrGetPrimaryKey)
 	}
-	nodeConfig := node.NewConfig(env.CLNodes[0].NodeConfig,
+	nodeConfig := node.NewConfig(env.ClCluster.Nodes[0].NodeConfig,
 		node.WithVRFv2EVMEstimator(addr),
 	)
-	err = env.CLNodes[0].Restart(nodeConfig)
+	err = env.ClCluster.Nodes[0].Restart(nodeConfig)
 	if err != nil {
 		return nil, nil, nil, errors.Wrap(err, ErrRestartCLNode)
 	}

integration-tests/docker/test_env/cl_node.go

@@ -45,15 +45,13 @@ var (
 
 type ClNode struct {
 	test_env.EnvComponent
-	API                   *client.ChainlinkClient
-	NodeConfig            *chainlink.Config
-	NodeSecretsConfigTOML string
-	PostgresDb            *test_env.PostgresDb
-	lw                    *logwatch.LogWatch
-	ContainerImage        string
-	ContainerVersion      string
+	API                   *client.ChainlinkClient `json:"-"`
+	NodeConfig            *chainlink.Config       `json:"-"`
+	NodeSecretsConfigTOML string                  `json:"-"`
+	PostgresDb            *test_env.PostgresDb    `json:"postgresDb"`
 	t                     *testing.T
 	l                     zerolog.Logger
+	lw                    *logwatch.LogWatch
 }
 
 type ClNodeOption = func(c *ClNode)
@@ -107,11 +105,10 @@ func NewClNode(networks []string, nodeConfig *chainlink.Config, opts ...ClNodeOp
 	return n
 }
 
-func (n *ClNode) WithTestLogger(t *testing.T) *ClNode {
+func (n *ClNode) SetTestLogger(t *testing.T) {
 	n.l = logging.GetTestLogger(t)
 	n.t = t
 	n.PostgresDb.WithTestLogger(t)
-	return n
 }
 
 // Restart restarts only CL node, DB container is reused

integration-tests/docker/test_env/cl_node_cluster.go

@@ -0,0 +1,68 @@
+package test_env
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/chainlink/integration-tests/client"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	ErrGetNodeCSAKeys = "failed get CL node CSA keys"
+)
+
+type ClCluster struct {
+	Nodes []*ClNode `json:"nodes"`
+}
+
+// Start all nodes in the cluster./docker/tests/functional/api
+func (c *ClCluster) Start() error {
+	eg := &errgroup.Group{}
+	nodes := c.Nodes
+
+	for i := 0; i < len(nodes); i++ {
+		nodeIndex := i
+		eg.Go(func() error {
+			err := nodes[nodeIndex].StartContainer()
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+	}
+
+	return eg.Wait()
+}
+
+func (c *ClCluster) NodeAPIs() []*client.ChainlinkClient {
+	clients := make([]*client.ChainlinkClient, 0)
+	for _, c := range c.Nodes {
+		clients = append(clients, c.API)
+	}
+	return clients
+}
+
+// Return all the on-chain wallet addresses for a set of Chainlink nodes
+func (c *ClCluster) NodeAddresses() ([]common.Address, error) {
+	addresses := make([]common.Address, 0)
+	for _, n := range c.Nodes {
+		primaryAddress, err := n.ChainlinkNodeAddress()
+		if err != nil {
+			return nil, err
+		}
+		addresses = append(addresses, primaryAddress)
+	}
+	return addresses, nil
+}
+
+func (c *ClCluster) NodeCSAKeys() ([]string, error) {
+	var keys []string
+	for _, n := range c.Nodes {
+		csaKeys, err := n.GetNodeCSAKeys()
+		if err != nil {
+			return nil, errors.Wrap(err, ErrGetNodeCSAKeys)
+		}
+		keys = append(keys, csaKeys.Data[0].ID)
+	}
+	return keys, nil
+}

integration-tests/docker/test_env/test_env.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/keystore"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -25,16 +24,14 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
 
-	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
-
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/integration-tests/utils"
+	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 )
 
 var (
-	ErrFundCLNode     = "failed to fund CL node"
-	ErrGetNodeCSAKeys = "failed get CL node CSA keys"
+	ErrFundCLNode = "failed to fund CL node"
 )
 
 type CLClusterTestEnv struct {
@@ -43,7 +40,7 @@ type CLClusterTestEnv struct {
 	LogWatch *logwatch.LogWatch
 
 	/* components */
-	CLNodes          []*ClNode
+	ClCluster        *ClCluster
 	Geth             *test_env.Geth          // for tests using --dev networks
 	PrivateChain     []test_env.PrivateChain // for tests using non-dev networks
 	MockAdapter      *test_env.Killgrave
@@ -139,92 +136,40 @@ func (te *CLClusterTestEnv) StartMockAdapter() error {
 	return te.MockAdapter.StartContainer()
 }
 
-func (te *CLClusterTestEnv) GetAPIs() []*client.ChainlinkClient {
-	clients := make([]*client.ChainlinkClient, 0)
-	for _, c := range te.CLNodes {
-		clients = append(clients, c.API)
-	}
-	return clients
-}
-
-// StartClNodes start one bootstrap node and {count} OCR nodes
-func (te *CLClusterTestEnv) StartClNodes(nodeConfig *chainlink.Config, count int, secretsConfig string) error {
-	eg := &errgroup.Group{}
-	nodes := make(chan *ClNode, count)
-
-	// Start nodes
-	for i := 0; i < count; i++ {
-		nodeIndex := i
-		eg.Go(func() error {
-			var nodeContainerName, dbContainerName string
-			if te.Cfg != nil {
-				nodeContainerName = te.Cfg.Nodes[nodeIndex].NodeContainerName
-				dbContainerName = te.Cfg.Nodes[nodeIndex].DbContainerName
-			}
-			n := NewClNode([]string{te.Network.Name}, nodeConfig,
+func (te *CLClusterTestEnv) StartClCluster(nodeConfig *chainlink.Config, count int, secretsConfig string) error {
+	if te.Cfg != nil && te.Cfg.ClCluster != nil {
+		te.ClCluster = te.Cfg.ClCluster
+	} else {
+		te.ClCluster = &ClCluster{}
+		for i := 0; i < count; i++ {
+			ocrNode := NewClNode([]string{te.Network.Name}, nodeConfig,
 				WithSecrets(secretsConfig),
-				WithNodeContainerName(nodeContainerName),
-				WithDbContainerName(dbContainerName),
 			)
-			if te.t != nil {
-				n.WithTestLogger(te.t)
-			}
-			err := n.StartContainer()
-			if err != nil {
-				return err
-			}
-			nodes <- n
-			return nil
-		})
-	}
-
-	if err := eg.Wait(); err != nil {
-		return err
-	}
-	close(nodes)
-
-	for node := range nodes {
-		te.CLNodes = append(te.CLNodes, node)
+			te.ClCluster.Nodes = append(te.ClCluster.Nodes, ocrNode)
+		}
 	}
 
-	return nil
-}
-
-// ChainlinkNodeAddresses will return all the on-chain wallet addresses for a set of Chainlink nodes
-func (te *CLClusterTestEnv) ChainlinkNodeAddresses() ([]common.Address, error) {
-	addresses := make([]common.Address, 0)
-	for _, n := range te.CLNodes {
-		primaryAddress, err := n.ChainlinkNodeAddress()
-		if err != nil {
-			return nil, err
+	// Set test logger
+	if te.t != nil {
+		for _, n := range te.ClCluster.Nodes {
+			n.SetTestLogger(te.t)
 		}
-		addresses = append(addresses, primaryAddress)
 	}
-	return addresses, nil
+
+	// Start/attach node containers
+	return te.ClCluster.Start()
 }
 
 // FundChainlinkNodes will fund all the provided Chainlink nodes with a set amount of native currency
 func (te *CLClusterTestEnv) FundChainlinkNodes(amount *big.Float) error {
-	for _, cl := range te.CLNodes {
+	for _, cl := range te.ClCluster.Nodes {
 		if err := cl.Fund(te.EVMClient, amount); err != nil {
 			return errors.Wrap(err, ErrFundCLNode)
 		}
 	}
 	return te.EVMClient.WaitForEvents()
 }
 
-func (te *CLClusterTestEnv) GetNodeCSAKeys() ([]string, error) {
-	var keys []string
-	for _, n := range te.CLNodes {
-		csaKeys, err := n.GetNodeCSAKeys()
-		if err != nil {
-			return nil, errors.Wrap(err, ErrGetNodeCSAKeys)
-		}
-		keys = append(keys, csaKeys.Data[0].ID)
-	}
-	return keys, nil
-}
-
 func (te *CLClusterTestEnv) Terminate() error {
 	// TESTCONTAINERS_RYUK_DISABLED=false by default so ryuk will remove all
 	// the containers and the Network
@@ -237,7 +182,7 @@ func (te *CLClusterTestEnv) Cleanup() error {
 	if te.t == nil {
 		return errors.New("cannot cleanup test environment without a testing.T")
 	}
-	if te.CLNodes == nil {
+	if te.ClCluster == nil || len(te.ClCluster.Nodes) == 0 {
 		return errors.New("chainlink nodes are nil, unable cleanup chainlink nodes")
 	}
 
@@ -279,7 +224,7 @@ func (te *CLClusterTestEnv) collectTestLogs() error {
 	}
 
 	eg := &errgroup.Group{}
-	for _, n := range te.CLNodes {
+	for _, n := range te.ClCluster.Nodes {
 		node := n
 		eg.Go(func() error {
 			logFileName := filepath.Join(folder, fmt.Sprintf("node-%s.log", node.ContainerName))
@@ -311,7 +256,7 @@ func (te *CLClusterTestEnv) collectTestLogs() error {
 
 func (te *CLClusterTestEnv) returnFunds() error {
 	te.l.Info().Msg("Attempting to return Chainlink node funds to default network wallets")
-	for _, chainlinkNode := range te.CLNodes {
+	for _, chainlinkNode := range te.ClCluster.Nodes {
 		fundedKeys, err := chainlinkNode.API.ExportEVMKeysForChain(te.EVMClient.GetChainID().String())
 		if err != nil {
 			return err

integration-tests/docker/test_env/test_env_builder.go

@@ -197,7 +197,7 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 			return nil, errors.New("cannot create nodes with custom config without nonDevNetworks")
 		}
 
-		err = b.te.StartClNodes(b.clNodeConfig, b.clNodesCount, b.secretsConfig)
+		err = b.te.StartClCluster(b.clNodeConfig, b.clNodesCount, b.secretsConfig)
 		if err != nil {
 			return nil, err
 		}
@@ -261,12 +261,12 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 			node.SetChainConfig(cfg, wsUrls, httpUrls, networkConfig, b.hasForwarders)
 		}
 
-		err := b.te.StartClNodes(cfg, b.clNodesCount, b.secretsConfig)
+		err := b.te.StartClCluster(cfg, b.clNodesCount, b.secretsConfig)
 		if err != nil {
 			return nil, err
 		}
 
-		nodeCsaKeys, err = b.te.GetNodeCSAKeys()
+		nodeCsaKeys, err = b.te.ClCluster.NodeCSAKeys()
 		if err != nil {
 			return nil, err
 		}