fix load test for CRIB, update docs (#13764)

* fix load test for CRIB, update docs

* update docs

* review fixes

integration-tests/client/chainlink_k8s.go

@@ -10,6 +10,11 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/k8s/environment"
 )
 
+const (
+	CLNodeTestEmail    = "[email protected]"
+	CLNodeTestPassword = "fj293fbBnlQ!f9vNs"
+)
+
 type ChainlinkK8sClient struct {
 	ChartName string
 	PodName   string
@@ -74,8 +79,8 @@ func ConnectChainlinkNodes(e *environment.Environment) ([]*ChainlinkK8sClient, e
 	for _, nodeDetails := range e.ChainlinkNodeDetails {
 		c, err := NewChainlinkK8sClient(&ChainlinkConfig{
 			URL:        nodeDetails.LocalIP,
-			Email:      "[email protected]",
-			Password:   "fj293fbBnlQ!f9vNs",
+			Email:      CLNodeTestEmail,
+			Password:   CLNodeTestPassword,
 			InternalIP: parseHostname(nodeDetails.InternalIP),
 		}, nodeDetails.PodName, nodeDetails.ChartName)
 		if err != nil {
@@ -100,8 +105,8 @@ func ReconnectChainlinkNodes(testEnvironment *environment.Environment, nodes []*
 			if details.ChartName == node.ChartName { // Make the link from client to pod consistent
 				node, err = NewChainlinkK8sClient(&ChainlinkConfig{
 					URL:        details.LocalIP,
-					Email:      "[email protected]",
-					Password:   "fj293fbBnlQ!f9vNs",
+					Email:      CLNodeTestEmail,
+					Password:   CLNodeTestPassword,
 					InternalIP: parseHostname(details.InternalIP),
 				}, details.PodName, details.ChartName)
 				if err != nil {
@@ -136,8 +141,8 @@ func ConnectChainlinkNodeURLs(urls []string) ([]*ChainlinkK8sClient, error) {
 func ConnectChainlinkNodeURL(url string) (*ChainlinkK8sClient, error) {
 	return NewChainlinkK8sClient(&ChainlinkConfig{
 		URL:        url,
-		Email:      "[email protected]",
-		Password:   "fj293fbBnlQ!f9vNs",
+		Email:      CLNodeTestEmail,
+		Password:   CLNodeTestPassword,
 		InternalIP: parseHostname(url),
 	},
 		parseHostname(url),   // a decent guess

integration-tests/contracts/ethereum_contracts.go

@@ -99,11 +99,6 @@ func DefaultOffChainAggregatorOptions() OffchainOptions {
 
 // DefaultOffChainAggregatorConfig returns some base defaults for configuring an OCR contract
 func DefaultOffChainAggregatorConfig(numberNodes int) OffChainAggregatorConfig {
-	if numberNodes <= 4 {
-		log.Err(fmt.Errorf("insufficient number of nodes (%d) supplied for OCR, need at least 5", numberNodes)).
-			Int("Number Chainlink Nodes", numberNodes).
-			Msg("You likely need more chainlink nodes to properly configure OCR, try 5 or more.")
-	}
 	s := []int{1}
 	// First node's stage already inputted as a 1 in line above, so numberNodes-1.
 	for i := 0; i < numberNodes-1; i++ {

integration-tests/crib/connect.go

@@ -0,0 +1,120 @@
+package crib
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/seth"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/utils/ptr"
+	seth_utils "github.com/smartcontractkit/chainlink-testing-framework/utils/seth"
+	tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig"
+
+	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
+	msClient "github.com/smartcontractkit/chainlink-testing-framework/client"
+	"github.com/smartcontractkit/chainlink/integration-tests/client"
+)
+
+const (
+	// these are constants for simulated CRIB that should never change
+	// CRIB: https://github.com/smartcontractkit/crib/tree/main/core
+	// Core Chart: https://github.com/smartcontractkit/infra-charts/tree/main/chainlink-cluster
+	mockserverCRIBTemplate        = "https://%s-mockserver%s"
+	internalNodeDNSTemplate       = "app-node%d"
+	ingressNetworkWSURLTemplate   = "wss://%s-geth-1337-ws%s"
+	ingressNetworkHTTPURLTemplate = "https://%s-geth-1337-http%s"
+)
+
+func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) {
+	netName := "CRIB_SIMULATED"
+	cfg.Network.SelectedNetworks = []string{netName}
+	cfg.Network.RpcHttpUrls = map[string][]string{}
+	cfg.Network.RpcHttpUrls[netName] = []string{netHTTPURL}
+	cfg.Network.RpcWsUrls = map[string][]string{}
+	cfg.Network.RpcWsUrls[netName] = []string{netWSURL}
+	cfg.Seth.EphemeralAddrs = ptr.Ptr(int64(0))
+}
+
+// ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core
+// connects to default CRIB network if simulated = true
+func ConnectRemote(simulated bool) (
+	*seth.Client,
+	*msClient.MockserverClient,
+	*client.ChainlinkK8sClient,
+	[]*client.ChainlinkK8sClient,
+	error,
+) {
+	ingressSuffix := os.Getenv("K8S_STAGING_INGRESS_SUFFIX")
+	if ingressSuffix == "" {
+		return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses")
+	}
+	config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	if config.CRIB.CLNodesNum < 2 {
+		return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]")
+	}
+	cfg := config.CRIB
+	mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cfg.Namespace, ingressSuffix)
+	var sethClient *seth.Client
+	if simulated {
+		netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cfg.Namespace, ingressSuffix)
+		netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cfg.Namespace, ingressSuffix)
+		setSethConfig(config, netWSURL, netHTTPURL)
+		net := blockchain.EVMNetwork{
+			Name:                 cfg.NetworkName,
+			Simulated:            true,
+			SupportsEIP1559:      true,
+			ClientImplementation: blockchain.EthereumClientImplementation,
+			ChainID:              1337,
+			PrivateKeys: []string{
+				"ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80",
+			},
+			URLs:                      []string{netWSURL},
+			HTTPURLs:                  []string{netHTTPURL},
+			ChainlinkTransactionLimit: 500000,
+			Timeout:                   blockchain.StrDuration{Duration: 2 * time.Minute},
+			MinimumConfirmations:      1,
+			GasEstimationBuffer:       10000,
+		}
+		sethClient, err = seth_utils.GetChainClient(config, net)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+	}
+	// bootstrap node
+	clClients := make([]*client.ChainlinkK8sClient, 0)
+	c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{
+		URL:        fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, 1, ingressSuffix),
+		Email:      client.CLNodeTestEmail,
+		InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1),
+		Password:   client.CLNodeTestPassword,
+	}, fmt.Sprintf(internalNodeDNSTemplate, 1), cfg.Namespace)
+	if err != nil {
+		return nil, nil, nil, nil, err
+	}
+	clClients = append(clClients, c)
+	// all the other nodes, indices of nodes in CRIB starts with 1
+	for i := 2; i <= cfg.CLNodesNum; i++ {
+		cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{
+			URL:        fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, i, ingressSuffix),
+			Email:      client.CLNodeTestEmail,
+			InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i),
+			Password:   client.CLNodeTestPassword,
+		}, fmt.Sprintf(internalNodeDNSTemplate, i), cfg.Namespace)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		clClients = append(clClients, cl)
+	}
+	mockServerClient := msClient.NewMockserverClient(&msClient.MockserverConfig{
+		LocalURL:   mockserverURL,
+		ClusterURL: mockserverURL,
+	})
+
+	//nolint:gosec // G602 - false positive https://github.com/securego/gosec/issues/1005
+	return sethClient, mockServerClient, clClients[0], clClients[1:], nil
+}

integration-tests/load/ocr/README.md

@@ -1,28 +1,31 @@
 ### OCR Load tests
 
-## Setup
-These tests can connect to any cluster create with [chainlink-cluster](../../../charts/chainlink-cluster/README.md)
-
-Create your cluster, if you already have one just use `kubefwd`
-```
-kubectl create ns cl-cluster
-devspace use namespace cl-cluster
-devspace deploy
-sudo kubefwd svc -n cl-cluster
+## Setup CRIB
+CRIB CORE user documentation is available in [CORE CRIB - Deploy & Access Instructions](https://smartcontract-it.atlassian.net/wiki/spaces/TT/pages/597197209/CORE+CRIB+-+Deploy+Access+Instructions)
+```shell
+devspace deploy --debug --profile local-dev-simulated-core-ocr1 --skip-build
 ```
 
-Change environment connection configuration [here](../../../charts/chainlink-cluster/connect.toml)
-
-If you haven't changed anything in [devspace.yaml](../../crib/devspace.yaml) then default connection configuration will work
-
 ## Usage
 
+Create `overrides.toml` in this directory
+```toml
+[CRIB]
+namespace = "$your_crib_namespace_here"
+# only Geth is supported right now
+network_name = "geth"
+nodes = 5
+
+[Logging.Loki]
+tenant_id="promtail"
+endpoint="..."
+basic_auth_secret="..."
 ```
-export LOKI_TOKEN=...
-export LOKI_URL=...
+Run the tests
 
+Set `K8S_STAGING_INGRESS_SUFFIX` when run locally (`export K8S_STAGING_INGRESS_SUFFIX=$(op read op://CRIB/secrets/K8S_STAGING_INGRESS_SUFFIX)`)
+
+```
 go test -v -run TestOCRLoad
 go test -v -run TestOCRVolume
-```
-
-Check test configuration [here](config.toml)
+```