From f0ab15f15202f137eabb80243f98c68994c2c0fb Mon Sep 17 00:00:00 2001 From: Sergey Kudasov Date: Fri, 5 Jul 2024 15:11:14 +0200 Subject: [PATCH] fix load test for CRIB, update docs (#13764) * fix load test for CRIB, update docs * update docs * review fixes --- integration-tests/client/chainlink_k8s.go | 17 ++- .../contracts/ethereum_contracts.go | 5 - integration-tests/crib/connect.go | 120 ++++++++++++++++++ integration-tests/k8s/connect.go | 99 --------------- integration-tests/load/connect.toml | 13 -- integration-tests/load/ocr/README.md | 39 +++--- integration-tests/load/ocr/ocr_test.go | 24 ++-- integration-tests/testconfig/testconfig.go | 7 + 8 files changed, 167 insertions(+), 157 deletions(-) create mode 100644 integration-tests/crib/connect.go delete mode 100644 integration-tests/k8s/connect.go delete mode 100644 integration-tests/load/connect.toml diff --git a/integration-tests/client/chainlink_k8s.go b/integration-tests/client/chainlink_k8s.go index 147238b7950..794e93f7276 100644 --- a/integration-tests/client/chainlink_k8s.go +++ b/integration-tests/client/chainlink_k8s.go @@ -10,6 +10,11 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/k8s/environment" ) +const ( + CLNodeTestEmail = "notreal@fakeemail.ch" + CLNodeTestPassword = "fj293fbBnlQ!f9vNs" +) + type ChainlinkK8sClient struct { ChartName string PodName string @@ -74,8 +79,8 @@ func ConnectChainlinkNodes(e *environment.Environment) ([]*ChainlinkK8sClient, e for _, nodeDetails := range e.ChainlinkNodeDetails { c, err := NewChainlinkK8sClient(&ChainlinkConfig{ URL: nodeDetails.LocalIP, - Email: "notreal@fakeemail.ch", - Password: "fj293fbBnlQ!f9vNs", + Email: CLNodeTestEmail, + Password: CLNodeTestPassword, InternalIP: parseHostname(nodeDetails.InternalIP), }, nodeDetails.PodName, nodeDetails.ChartName) if err != nil { @@ -100,8 +105,8 @@ func ReconnectChainlinkNodes(testEnvironment *environment.Environment, nodes []* if details.ChartName == node.ChartName { // Make the link from client to pod consistent node, err = NewChainlinkK8sClient(&ChainlinkConfig{ URL: details.LocalIP, - Email: "notreal@fakeemail.ch", - Password: "fj293fbBnlQ!f9vNs", + Email: CLNodeTestEmail, + Password: CLNodeTestPassword, InternalIP: parseHostname(details.InternalIP), }, details.PodName, details.ChartName) if err != nil { @@ -136,8 +141,8 @@ func ConnectChainlinkNodeURLs(urls []string) ([]*ChainlinkK8sClient, error) { func ConnectChainlinkNodeURL(url string) (*ChainlinkK8sClient, error) { return NewChainlinkK8sClient(&ChainlinkConfig{ URL: url, - Email: "notreal@fakeemail.ch", - Password: "fj293fbBnlQ!f9vNs", + Email: CLNodeTestEmail, + Password: CLNodeTestPassword, InternalIP: parseHostname(url), }, parseHostname(url), // a decent guess diff --git a/integration-tests/contracts/ethereum_contracts.go b/integration-tests/contracts/ethereum_contracts.go index ebe99da2d92..f2dc7e85ee7 100644 --- a/integration-tests/contracts/ethereum_contracts.go +++ b/integration-tests/contracts/ethereum_contracts.go @@ -99,11 +99,6 @@ func DefaultOffChainAggregatorOptions() OffchainOptions { // DefaultOffChainAggregatorConfig returns some base defaults for configuring an OCR contract func DefaultOffChainAggregatorConfig(numberNodes int) OffChainAggregatorConfig { - if numberNodes <= 4 { - log.Err(fmt.Errorf("insufficient number of nodes (%d) supplied for OCR, need at least 5", numberNodes)). - Int("Number Chainlink Nodes", numberNodes). - Msg("You likely need more chainlink nodes to properly configure OCR, try 5 or more.") - } s := []int{1} // First node's stage already inputted as a 1 in line above, so numberNodes-1. for i := 0; i < numberNodes-1; i++ { diff --git a/integration-tests/crib/connect.go b/integration-tests/crib/connect.go new file mode 100644 index 00000000000..446c53d8f38 --- /dev/null +++ b/integration-tests/crib/connect.go @@ -0,0 +1,120 @@ +package crib + +import ( + "fmt" + "os" + "time" + + "github.com/pkg/errors" + "github.com/smartcontractkit/seth" + + "github.com/smartcontractkit/chainlink-testing-framework/utils/ptr" + seth_utils "github.com/smartcontractkit/chainlink-testing-framework/utils/seth" + tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" + + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + msClient "github.com/smartcontractkit/chainlink-testing-framework/client" + "github.com/smartcontractkit/chainlink/integration-tests/client" +) + +const ( + // these are constants for simulated CRIB that should never change + // CRIB: https://github.com/smartcontractkit/crib/tree/main/core + // Core Chart: https://github.com/smartcontractkit/infra-charts/tree/main/chainlink-cluster + mockserverCRIBTemplate = "https://%s-mockserver%s" + internalNodeDNSTemplate = "app-node%d" + ingressNetworkWSURLTemplate = "wss://%s-geth-1337-ws%s" + ingressNetworkHTTPURLTemplate = "https://%s-geth-1337-http%s" +) + +func setSethConfig(cfg tc.TestConfig, netWSURL string, netHTTPURL string) { + netName := "CRIB_SIMULATED" + cfg.Network.SelectedNetworks = []string{netName} + cfg.Network.RpcHttpUrls = map[string][]string{} + cfg.Network.RpcHttpUrls[netName] = []string{netHTTPURL} + cfg.Network.RpcWsUrls = map[string][]string{} + cfg.Network.RpcWsUrls[netName] = []string{netWSURL} + cfg.Seth.EphemeralAddrs = ptr.Ptr(int64(0)) +} + +// ConnectRemote connects to a local environment, see https://github.com/smartcontractkit/crib/tree/main/core +// connects to default CRIB network if simulated = true +func ConnectRemote(simulated bool) ( + *seth.Client, + *msClient.MockserverClient, + *client.ChainlinkK8sClient, + []*client.ChainlinkK8sClient, + error, +) { + ingressSuffix := os.Getenv("K8S_STAGING_INGRESS_SUFFIX") + if ingressSuffix == "" { + return nil, nil, nil, nil, errors.New("K8S_STAGING_INGRESS_SUFFIX must be set to connect to k8s ingresses") + } + config, err := tc.GetConfig([]string{"CRIB"}, tc.OCR) + if err != nil { + return nil, nil, nil, nil, err + } + if config.CRIB.CLNodesNum < 2 { + return nil, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, TOML key: [CRIB.nodes]") + } + cfg := config.CRIB + mockserverURL := fmt.Sprintf(mockserverCRIBTemplate, cfg.Namespace, ingressSuffix) + var sethClient *seth.Client + if simulated { + netWSURL := fmt.Sprintf(ingressNetworkWSURLTemplate, cfg.Namespace, ingressSuffix) + netHTTPURL := fmt.Sprintf(ingressNetworkHTTPURLTemplate, cfg.Namespace, ingressSuffix) + setSethConfig(config, netWSURL, netHTTPURL) + net := blockchain.EVMNetwork{ + Name: cfg.NetworkName, + Simulated: true, + SupportsEIP1559: true, + ClientImplementation: blockchain.EthereumClientImplementation, + ChainID: 1337, + PrivateKeys: []string{ + "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80", + }, + URLs: []string{netWSURL}, + HTTPURLs: []string{netHTTPURL}, + ChainlinkTransactionLimit: 500000, + Timeout: blockchain.StrDuration{Duration: 2 * time.Minute}, + MinimumConfirmations: 1, + GasEstimationBuffer: 10000, + } + sethClient, err = seth_utils.GetChainClient(config, net) + if err != nil { + return nil, nil, nil, nil, err + } + } + // bootstrap node + clClients := make([]*client.ChainlinkK8sClient, 0) + c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ + URL: fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, 1, ingressSuffix), + Email: client.CLNodeTestEmail, + InternalIP: fmt.Sprintf(internalNodeDNSTemplate, 1), + Password: client.CLNodeTestPassword, + }, fmt.Sprintf(internalNodeDNSTemplate, 1), cfg.Namespace) + if err != nil { + return nil, nil, nil, nil, err + } + clClients = append(clClients, c) + // all the other nodes, indices of nodes in CRIB starts with 1 + for i := 2; i <= cfg.CLNodesNum; i++ { + cl, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ + URL: fmt.Sprintf("https://%s-node%d%s", cfg.Namespace, i, ingressSuffix), + Email: client.CLNodeTestEmail, + InternalIP: fmt.Sprintf(internalNodeDNSTemplate, i), + Password: client.CLNodeTestPassword, + }, fmt.Sprintf(internalNodeDNSTemplate, i), cfg.Namespace) + if err != nil { + return nil, nil, nil, nil, err + } + clClients = append(clClients, cl) + } + mockServerClient := msClient.NewMockserverClient(&msClient.MockserverConfig{ + LocalURL: mockserverURL, + ClusterURL: mockserverURL, + }) + + //nolint:gosec // G602 - false positive https://github.com/securego/gosec/issues/1005 + return sethClient, mockServerClient, clClients[0], clClients[1:], nil +} diff --git a/integration-tests/k8s/connect.go b/integration-tests/k8s/connect.go deleted file mode 100644 index be1f9a25f9d..00000000000 --- a/integration-tests/k8s/connect.go +++ /dev/null @@ -1,99 +0,0 @@ -package k8s - -import ( - "fmt" - "os" - "time" - - "github.com/pelletier/go-toml/v2" - "github.com/rs/zerolog/log" - - "github.com/smartcontractkit/chainlink-testing-framework/blockchain" - client2 "github.com/smartcontractkit/chainlink-testing-framework/client" - "github.com/smartcontractkit/chainlink/integration-tests/client" -) - -const ( - DefaultConfigFilePath = "../connect.toml" - ErrReadConnectionConfig = "failed to read TOML environment connection config" - ErrUnmarshalConnectionConfig = "failed to unmarshal TOML environment connection config" -) - -type ConnectionVars struct { - Namespace string `toml:"namespace"` - NetworkName string `toml:"network_name"` - NetworkChainID int64 `toml:"network_chain_id"` - NetworkPrivateKey string `toml:"network_private_key"` - NetworkWSURL string `toml:"network_ws_url"` - NetworkHTTPURL string `toml:"network_http_url"` - CLNodesNum int `toml:"cl_nodes_num"` - CLNodeURLTemplate string `toml:"cl_node_url_template"` - CLNodeInternalDNSRecordTemplate string `toml:"cl_node_internal_dns_record_template"` - CLNodeUser string `toml:"cl_node_user"` - CLNodePassword string `toml:"cl_node_password"` - MockServerURL string `toml:"mockserver_url"` -} - -// ConnectRemote connects to a local environment, see charts/chainlink-cluster -func ConnectRemote() (*blockchain.EVMNetwork, *client2.MockserverClient, *client.ChainlinkK8sClient, []*client.ChainlinkK8sClient, error) { - cfg, err := ReadConfig() - if err != nil { - return &blockchain.EVMNetwork{}, nil, nil, nil, err - } - net := &blockchain.EVMNetwork{ - Name: cfg.NetworkName, - Simulated: true, - SupportsEIP1559: true, - ClientImplementation: blockchain.EthereumClientImplementation, - ChainID: 1337, - PrivateKeys: []string{ - cfg.NetworkPrivateKey, - }, - URLs: []string{cfg.NetworkWSURL}, - HTTPURLs: []string{cfg.NetworkHTTPURL}, - ChainlinkTransactionLimit: 500000, - Timeout: blockchain.StrDuration{Duration: 2 * time.Minute}, - MinimumConfirmations: 1, - GasEstimationBuffer: 10000, - } - clClients := make([]*client.ChainlinkK8sClient, 0) - for i := 1; i <= cfg.CLNodesNum; i++ { - c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ - URL: fmt.Sprintf(cfg.CLNodeURLTemplate, i), - Email: cfg.CLNodeUser, - InternalIP: fmt.Sprintf(cfg.CLNodeInternalDNSRecordTemplate, i), - Password: cfg.CLNodePassword, - }, fmt.Sprintf(cfg.CLNodeInternalDNSRecordTemplate, i), cfg.Namespace) - if err != nil { - return &blockchain.EVMNetwork{}, nil, nil, nil, err - } - clClients = append(clClients, c) - } - msClient := client2.NewMockserverClient(&client2.MockserverConfig{ - LocalURL: cfg.MockServerURL, - ClusterURL: cfg.MockServerURL, - }) - - if len(clClients) < 2 { - return &blockchain.EVMNetwork{}, nil, nil, nil, fmt.Errorf("not enough chainlink nodes, need at least 2, got %d", len(clClients)) - } - - //nolint:gosec // G602 - how is this potentially causing slice out of bounds is beyond me - return net, msClient, clClients[0], clClients[1:], nil -} - -func ReadConfig() (*ConnectionVars, error) { - var cfg *ConnectionVars - var d []byte - var err error - d, err = os.ReadFile(DefaultConfigFilePath) - if err != nil { - return nil, fmt.Errorf("%s, err: %w", ErrReadConnectionConfig, err) - } - err = toml.Unmarshal(d, &cfg) - if err != nil { - return nil, fmt.Errorf("%s, err: %w", ErrUnmarshalConnectionConfig, err) - } - log.Info().Interface("Config", cfg).Msg("Connecting to environment from config") - return cfg, nil -} diff --git a/integration-tests/load/connect.toml b/integration-tests/load/connect.toml deleted file mode 100644 index 919c5102c84..00000000000 --- a/integration-tests/load/connect.toml +++ /dev/null @@ -1,13 +0,0 @@ -# this is a static configuration to connect with CRIB k8s environment (Default Geth) -namespace = "cl-cluster" -network_name = "geth" -network_chain_id = 1337 -network_private_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" -network_ws_url = "ws://geth-1337:8546" -network_http_url = "http://geth:8544" -cl_nodes_num = 6 -cl_node_url_template = "http://app-node-%d:6688" -cl_node_internal_dns_record_template = "app-node-%d" -cl_node_user = "notreal@fakeemail.ch" -cl_node_password = "fj293fbBnlQ!f9vNs" -mockserver_url = "http://mockserver:1080" \ No newline at end of file diff --git a/integration-tests/load/ocr/README.md b/integration-tests/load/ocr/README.md index e63828a7d93..ad0df73218e 100644 --- a/integration-tests/load/ocr/README.md +++ b/integration-tests/load/ocr/README.md @@ -1,28 +1,31 @@ ### OCR Load tests -## Setup -These tests can connect to any cluster create with [chainlink-cluster](../../../charts/chainlink-cluster/README.md) - -Create your cluster, if you already have one just use `kubefwd` -``` -kubectl create ns cl-cluster -devspace use namespace cl-cluster -devspace deploy -sudo kubefwd svc -n cl-cluster +## Setup CRIB +CRIB CORE user documentation is available in [CORE CRIB - Deploy & Access Instructions](https://smartcontract-it.atlassian.net/wiki/spaces/TT/pages/597197209/CORE+CRIB+-+Deploy+Access+Instructions) +```shell +devspace deploy --debug --profile local-dev-simulated-core-ocr1 --skip-build ``` -Change environment connection configuration [here](../../../charts/chainlink-cluster/connect.toml) - -If you haven't changed anything in [devspace.yaml](../../crib/devspace.yaml) then default connection configuration will work - ## Usage +Create `overrides.toml` in this directory +```toml +[CRIB] +namespace = "$your_crib_namespace_here" +# only Geth is supported right now +network_name = "geth" +nodes = 5 + +[Logging.Loki] +tenant_id="promtail" +endpoint="..." +basic_auth_secret="..." ``` -export LOKI_TOKEN=... -export LOKI_URL=... +Run the tests +Set `K8S_STAGING_INGRESS_SUFFIX` when run locally (`export K8S_STAGING_INGRESS_SUFFIX=$(op read op://CRIB/secrets/K8S_STAGING_INGRESS_SUFFIX)`) + +``` go test -v -run TestOCRLoad go test -v -run TestOCRVolume -``` - -Check test configuration [here](config.toml) \ No newline at end of file +``` \ No newline at end of file diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index db90f67b3c6..8d4638f9771 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -3,15 +3,13 @@ package ocr import ( "testing" - seth_utils "github.com/smartcontractkit/chainlink-testing-framework/utils/seth" - "github.com/stretchr/testify/require" "github.com/smartcontractkit/wasp" "github.com/smartcontractkit/chainlink-testing-framework/logging" - "github.com/smartcontractkit/chainlink/integration-tests/k8s" + "github.com/smartcontractkit/chainlink/integration-tests/crib" tc "github.com/smartcontractkit/chainlink/integration-tests/testconfig" ) @@ -28,15 +26,12 @@ func TestOCRLoad(t *testing.T) { config, err := tc.GetConfig([]string{"Load"}, tc.OCR) require.NoError(t, err) - evmNetwork, msClient, bootstrapNode, workerNodes, err := k8s.ConnectRemote() + sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) require.NoError(t, err) - seth, err := seth_utils.GetChainClient(config, *evmNetwork) - require.NoError(t, err, "Error creating seth client") - - lta, err := SetupCluster(l, seth, workerNodes) + lta, err := SetupCluster(l, sethClient, workerNodes) require.NoError(t, err) - ocrInstances, err := SetupFeed(l, seth, lta, msClient, bootstrapNode, workerNodes) + ocrInstances, err := SetupFeed(l, sethClient, lta, msClient, bootstrapNode, workerNodes) require.NoError(t, err) cfg := config.OCR @@ -51,7 +46,7 @@ func TestOCRLoad(t *testing.T) { CallTimeout: cfg.Load.VerificationTimeout.Duration, RateLimitUnitDuration: cfg.Load.RateLimitUnitDuration.Duration, Schedule: wasp.Plain(*cfg.Load.Rate, cfg.Load.TestDuration.Duration), - Gun: NewGun(l, seth, ocrInstances), + Gun: NewGun(l, sethClient, ocrInstances), Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), })) @@ -64,13 +59,10 @@ func TestOCRVolume(t *testing.T) { config, err := tc.GetConfig([]string{"Volume"}, tc.OCR) require.NoError(t, err) - evmNetwork, msClient, bootstrapNode, workerNodes, err := k8s.ConnectRemote() + sethClient, msClient, bootstrapNode, workerNodes, err := crib.ConnectRemote(true) require.NoError(t, err) - seth, err := seth_utils.GetChainClient(config, *evmNetwork) - require.NoError(t, err, "Error creating seth client") - - lta, err := SetupCluster(l, seth, workerNodes) + lta, err := SetupCluster(l, sethClient, workerNodes) require.NoError(t, err) cfg := config.OCR @@ -83,7 +75,7 @@ func TestOCRVolume(t *testing.T) { LoadType: wasp.VU, CallTimeout: cfg.Volume.VerificationTimeout.Duration, Schedule: wasp.Plain(*cfg.Volume.Rate, cfg.Volume.TestDuration.Duration), - VU: NewVU(l, seth, *cfg.Volume.VURequestsPerUnit, cfg.Volume.RateLimitUnitDuration.Duration, lta, bootstrapNode, workerNodes, msClient), + VU: NewVU(l, sethClient, *cfg.Volume.VURequestsPerUnit, cfg.Volume.RateLimitUnitDuration.Duration, lta, bootstrapNode, workerNodes, msClient), Labels: CommonTestLabels, LokiConfig: wasp.NewLokiConfig(cfgl.Endpoint, cfgl.TenantId, cfgl.BasicAuth, cfgl.BearerToken), })) diff --git a/integration-tests/testconfig/testconfig.go b/integration-tests/testconfig/testconfig.go index 8f06433fb07..3a73d39dfe9 100644 --- a/integration-tests/testconfig/testconfig.go +++ b/integration-tests/testconfig/testconfig.go @@ -82,10 +82,17 @@ type TestConfig struct { VRF *vrf_config.Config `toml:"VRF"` VRFv2 *vrfv2_config.Config `toml:"VRFv2"` VRFv2Plus *vrfv2plus_config.Config `toml:"VRFv2Plus"` + CRIB *CRIB `toml:"CRIB"` ConfigurationNames []string `toml:"-"` } +type CRIB struct { + Namespace string `toml:"namespace"` + NetworkName string `toml:"network_name"` + CLNodesNum int `toml:"nodes"` +} + var embeddedConfigs embed.FS var areConfigsEmbedded bool