From b65801c1af31ebc6dc08a24136c4f64e37c69202 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 25 Jun 2024 12:19:44 +0200 Subject: [PATCH 1/5] fix forwarder tests + update GHA --- .github/workflows/on-demand-ocr-soak-test.yml | 9 ++++++--- integration-tests/soak/forwarder_ocr_test.go | 4 ---- integration-tests/testsetups/ocr.go | 4 +++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/on-demand-ocr-soak-test.yml b/.github/workflows/on-demand-ocr-soak-test.yml index ae24786b437..c4415ac85d5 100644 --- a/.github/workflows/on-demand-ocr-soak-test.yml +++ b/.github/workflows/on-demand-ocr-soak-test.yml @@ -8,7 +8,10 @@ on: default: TestOCRSoak type: choice options: - - TestOCRSoak + - TestOCRv1Soak + - TestOCRv2Soak + - TestForwarderOCRv1Soak + - TestForwarderOCRv2Soak - TestOCRSoak_GethReorgBelowFinality_FinalityTagDisabled - TestOCRSoak_GethReorgBelowFinality_FinalityTagEnabled - TestOCRSoak_GasSpike @@ -18,7 +21,7 @@ on: base64Config: description: base64-ed config required: true - type: string + type: string slackMemberID: description: Slack Member ID (Not your @) required: true @@ -78,7 +81,7 @@ jobs: echo "### chainlink-tests image tag for this test run :ship:" >>$GITHUB_STEP_SUMMARY echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY - echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY + echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY - name: Build Image uses: ./.github/actions/build-test-image with: diff --git a/integration-tests/soak/forwarder_ocr_test.go b/integration-tests/soak/forwarder_ocr_test.go index fa9752a9a9c..ce2772899a7 100644 --- a/integration-tests/soak/forwarder_ocr_test.go +++ b/integration-tests/soak/forwarder_ocr_test.go @@ -13,8 +13,6 @@ import ( ) func TestForwarderOCRv1Soak(t *testing.T) { - //nolint:revive - t.Fatalf("This test is disabled because the implementation is broken") config, err := tc.GetConfig("Soak", tc.ForwarderOcr) require.NoError(t, err, "Error getting config") @@ -22,8 +20,6 @@ func TestForwarderOCRv1Soak(t *testing.T) { } func TestForwarderOCRv2Soak(t *testing.T) { - //nolint:revive - t.Fatalf("This test is disabled because the implementation is broken") config, err := tc.GetConfig("Soak", tc.ForwarderOcr2) require.NoError(t, err, "Error getting config") diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index fe4d15fc96f..e325b08908e 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -311,7 +311,9 @@ func (o *OCRSoakTest) Setup(ocrTestConfig tt.OcrTestConfig) { require.NoError(o.t, err, "Accepting Authorize Receivers on Operator shouldn't fail") actions.TrackForwarder(o.t, o.seth, forwarders[i], o.workerNodes[i]) } - } else if o.OCRVersion == "1" { + } + + if o.OCRVersion == "1" { if o.OperatorForwarderFlow { o.ocrV1Instances, err = actions.DeployOCRContractsForwarderFlow( o.log, From 2f233646eadfda772a0f1e435bdc46cd87e952f2 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 25 Jun 2024 12:40:27 +0200 Subject: [PATCH 2/5] update readme.md --- integration-tests/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/integration-tests/README.md b/integration-tests/README.md index d34b4426fd8..86be8b6eb35 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -61,7 +61,15 @@ You can see the results of each test in the terminal with normal `go test` outpu These tests remain bound to a Kubernetes run environment, and require more complex setup and running instructions not documented here. We endeavor to make these easier to run and configure, but for the time being please seek a member of the QA/Test Tooling team if you want to run these. +We will only mention that once you have prepared `overrides.toml` (with at least the Chainlink image entry) you can run OCR-related test using `make` command: +* OCRv1 -> `make test_soak_ocr1` +* OCRv2 -> `make test_soak_ocr2` +* OCRv1 with forwarders -> `make test_soak_forwarder_ocr1` +* OCRv2 with forwarders -> `make test_soak_forwarder_ocr2` + ### How to run reorg tests +It's crucial to remember that these tests currently run on **OCRv1**. + Run soak/ocr_test.go with reorg below finality and `FinalityTagEnabled=false` ```bash @@ -127,4 +135,4 @@ Run soak/ocr_test.go with RPC network chaos by bringing down network to RPC node ```bash make test_soak_ocr_rpc_down_half_cl_nodes -``` \ No newline at end of file +``` From c21d439ce4a57521aff2f82a4527574692341966 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 25 Jun 2024 16:53:27 +0200 Subject: [PATCH 3/5] fix namespace creation in ocrv2 soak tests --- integration-tests/testsetups/ocr.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index fe4d15fc96f..749ca3beac9 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -21,7 +21,6 @@ import ( geth "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/google/uuid" "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog" "github.com/smartcontractkit/seth" @@ -152,22 +151,17 @@ func NewOCRSoakTest(t *testing.T, config *tc.TestConfig, opts ...OCRSoakTestOpti func (o *OCRSoakTest) DeployEnvironment(ocrTestConfig tt.OcrTestConfig) { nodeNetwork := networks.MustGetSelectedNetworkConfig(ocrTestConfig.GetNetworkConfig())[0] // Environment currently being used to soak test on - // Define namespace if default not set - if o.namespace == "" { - nsPre := fmt.Sprintf("soak-ocr-v%s-", o.OCRVersion) - if o.OperatorForwarderFlow { - nsPre = fmt.Sprintf("%sforwarder-", nsPre) - } - - nsPre = fmt.Sprintf("%s%s", nsPre, strings.ReplaceAll(strings.ToLower(nodeNetwork.Name), " ", "-")) - nsPre = strings.ReplaceAll(nsPre, "_", "-") - - o.namespace = fmt.Sprintf("%s-%s", nsPre, uuid.NewString()[0:5]) + nsPre := fmt.Sprintf("soak-ocr-v%s-", o.OCRVersion) + if o.OperatorForwarderFlow { + nsPre = fmt.Sprintf("%sforwarder-", nsPre) } + nsPre = fmt.Sprintf("%s%s", nsPre, strings.ReplaceAll(strings.ToLower(nodeNetwork.Name), " ", "-")) + nsPre = strings.ReplaceAll(nsPre, "_", "-") + baseEnvironmentConfig := &environment.Config{ TTL: time.Hour * 720, // 30 days, - Namespace: o.namespace, + NamespacePrefix: nsPre, Test: o.t, PreventPodEviction: true, } From 6d96172f80203178a1df369a5e9d16dacb3cf103 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 25 Jun 2024 17:37:19 +0200 Subject: [PATCH 4/5] make sure 1 and only 1 OCR config is set in TOML --- integration-tests/testsetups/ocr.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index 81de546c2b8..8d797926607 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -116,6 +116,14 @@ func WithForwarderFlow(forwarderFlow bool) OCRSoakTestOption { // NewOCRSoakTest creates a new OCR soak test to setup and run func NewOCRSoakTest(t *testing.T, config *tc.TestConfig, opts ...OCRSoakTestOption) (*OCRSoakTest, error) { + if config.OCR2 != nil && config.OCR != nil { + return nil, fmt.Errorf("both OCR and OCR2 are set. Please check your TOML config and make sure you only have one of them set") + } + + if config.GetActiveOCRConfig() == nil { + return nil, fmt.Errorf("no OCR[1/2] config not set. Please check your TOML config and make sure you have an OCR[1/2] config set") + } + test := &OCRSoakTest{ Config: config, TestReporter: testreporters.OCRSoakTestReporter{ From 1779ee9b56f7d3dff2c9c4e280087d494e124ab0 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Wed, 26 Jun 2024 19:34:55 +0200 Subject: [PATCH 5/5] another fix for forwarder tests --- integration-tests/actions/ocr2_helpers.go | 4 +- integration-tests/actions/ocr_helpers.go | 73 ++--------------------- integration-tests/chaos/ocr_chaos_test.go | 2 +- integration-tests/load/ocr/helper.go | 3 +- integration-tests/load/ocr/vu.go | 3 +- integration-tests/testsetups/ocr.go | 9 +-- 6 files changed, 12 insertions(+), 82 deletions(-) diff --git a/integration-tests/actions/ocr2_helpers.go b/integration-tests/actions/ocr2_helpers.go index 0f8dd9b29f2..fa659d587b3 100644 --- a/integration-tests/actions/ocr2_helpers.go +++ b/integration-tests/actions/ocr2_helpers.go @@ -190,7 +190,7 @@ func CreateOCRv2Jobs( mockserver *ctfClient.MockserverClient, mockServerValue int, // Value to get from the mock server when querying the path chainId int64, // EVM chain ID - forwardingAllowed bool, + withForwarders bool, ) error { // Collect P2P ID bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys() @@ -266,7 +266,7 @@ func CreateOCRv2Jobs( JobType: "offchainreporting2", MaxTaskDuration: "1m", ObservationSource: client.ObservationSourceSpecBridge(bta), - ForwardingAllowed: forwardingAllowed, + ForwardingAllowed: withForwarders, OCR2OracleSpec: job.OCR2OracleSpec{ PluginType: "median", Relay: "evm", diff --git a/integration-tests/actions/ocr_helpers.go b/integration-tests/actions/ocr_helpers.go index 6e54e81cd84..5350e0e6b23 100644 --- a/integration-tests/actions/ocr_helpers.go +++ b/integration-tests/actions/ocr_helpers.go @@ -3,10 +3,8 @@ package actions import ( "fmt" "strings" - "testing" "github.com/google/uuid" - "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" ctfClient "github.com/smartcontractkit/chainlink-testing-framework/client" @@ -26,7 +24,8 @@ func CreateOCRJobs( workerNodes []*client.ChainlinkK8sClient, mockValue int, mockserver *ctfClient.MockserverClient, - evmChainID string, + evmChainID int64, + withForwarders bool, ) error { for _, ocrInstance := range ocrInstances { bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys() @@ -37,7 +36,7 @@ func CreateOCRJobs( bootstrapSpec := &client.OCRBootstrapJobSpec{ Name: fmt.Sprintf("bootstrap-%s", uuid.New().String()), ContractAddress: ocrInstance.Address(), - EVMChainID: evmChainID, + EVMChainID: fmt.Sprint(evmChainID), P2PPeerID: bootstrapP2PId, IsBootstrapPeer: true, } @@ -82,12 +81,13 @@ func CreateOCRJobs( bootstrapPeers := []*client.ChainlinkClient{bootstrapNode.ChainlinkClient} ocrSpec := &client.OCRTaskJobSpec{ ContractAddress: ocrInstance.Address(), - EVMChainID: evmChainID, + EVMChainID: fmt.Sprint(evmChainID), P2PPeerID: nodeP2PId, P2PBootstrapPeers: bootstrapPeers, KeyBundleID: nodeOCRKeyId, TransmitterAddress: nodeTransmitterAddress, ObservationSource: client.ObservationSourceSpecBridge(bta), + ForwardingAllowed: withForwarders, } _, err = node.MustCreateJob(ocrSpec) if err != nil { @@ -98,69 +98,6 @@ func CreateOCRJobs( return nil } -// CreateOCRJobsWithForwarder bootstraps the first node and to the other nodes sends ocr jobs that -// read from different adapters, to be used in combination with SetAdapterResponses -func CreateOCRJobsWithForwarder( - t *testing.T, - ocrInstances []contracts.OffchainAggregator, - bootstrapNode *client.ChainlinkK8sClient, - workerNodes []*client.ChainlinkK8sClient, - mockValue int, - mockserver *ctfClient.MockserverClient, - evmChainID int64, -) { - for _, ocrInstance := range ocrInstances { - bootstrapP2PIds, err := bootstrapNode.MustReadP2PKeys() - require.NoError(t, err, "Shouldn't fail reading P2P keys from bootstrap node") - bootstrapP2PId := bootstrapP2PIds.Data[0].Attributes.PeerID - bootstrapSpec := &client.OCRBootstrapJobSpec{ - Name: fmt.Sprintf("bootstrap-%s", uuid.New().String()), - ContractAddress: ocrInstance.Address(), - EVMChainID: fmt.Sprint(evmChainID), - P2PPeerID: bootstrapP2PId, - IsBootstrapPeer: true, - } - _, err = bootstrapNode.MustCreateJob(bootstrapSpec) - require.NoError(t, err, "Shouldn't fail creating bootstrap job on bootstrap node") - - for nodeIndex, node := range workerNodes { - nodeP2PIds, err := node.MustReadP2PKeys() - require.NoError(t, err, "Shouldn't fail reading P2P keys from OCR node %d", nodeIndex+1) - nodeP2PId := nodeP2PIds.Data[0].Attributes.PeerID - nodeTransmitterAddress, err := node.PrimaryEthAddress() - require.NoError(t, err, "Shouldn't fail getting primary ETH address from OCR node %d", nodeIndex+1) - nodeOCRKeys, err := node.MustReadOCRKeys() - require.NoError(t, err, "Shouldn't fail getting OCR keys from OCR node %d", nodeIndex+1) - nodeOCRKeyId := nodeOCRKeys.Data[0].ID - - nodeContractPairID, err := BuildNodeContractPairID(node, ocrInstance) - require.NoError(t, err, "Failed building node contract pair ID for mockserver") - bta := &client.BridgeTypeAttributes{ - Name: nodeContractPairID, - URL: fmt.Sprintf("%s/%s", mockserver.Config.ClusterURL, strings.TrimPrefix(nodeContractPairID, "/")), - } - err = SetAdapterResponse(mockValue, ocrInstance, node, mockserver) - require.NoError(t, err, "Failed setting adapter responses for node %d", nodeIndex+1) - err = node.MustCreateBridge(bta) - require.NoError(t, err, "Failed creating bridge on OCR node %d", nodeIndex+1) - - bootstrapPeers := []*client.ChainlinkClient{bootstrapNode.ChainlinkClient} - ocrSpec := &client.OCRTaskJobSpec{ - ContractAddress: ocrInstance.Address(), - EVMChainID: fmt.Sprint(evmChainID), - P2PPeerID: nodeP2PId, - P2PBootstrapPeers: bootstrapPeers, - KeyBundleID: nodeOCRKeyId, - TransmitterAddress: nodeTransmitterAddress, - ObservationSource: client.ObservationSourceSpecBridge(bta), - ForwardingAllowed: true, - } - _, err = node.MustCreateJob(ocrSpec) - require.NoError(t, err, "Shouldn't fail creating OCR Task job on OCR node %d", nodeIndex+1) - } - } -} - // SetAdapterResponse sets a single adapter response that correlates with an ocr contract and a chainlink node func SetAdapterResponse( response int, diff --git a/integration-tests/chaos/ocr_chaos_test.go b/integration-tests/chaos/ocr_chaos_test.go index 3b7eeac888c..4f97ac3c31a 100644 --- a/integration-tests/chaos/ocr_chaos_test.go +++ b/integration-tests/chaos/ocr_chaos_test.go @@ -189,7 +189,7 @@ func TestOCRChaos(t *testing.T) { ocrInstances, err := actions.DeployOCRv1Contracts(l, seth, 1, common.HexToAddress(linkContract.Address()), contracts.ChainlinkK8sClientToChainlinkNodeWithKeysAndAddress(workerNodes)) require.NoError(t, err) - err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, ms, fmt.Sprint(seth.ChainID)) + err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, ms, seth.ChainID, false) require.NoError(t, err) chaosApplied := false diff --git a/integration-tests/load/ocr/helper.go b/integration-tests/load/ocr/helper.go index f95bf143bce..920a9b5348f 100644 --- a/integration-tests/load/ocr/helper.go +++ b/integration-tests/load/ocr/helper.go @@ -1,7 +1,6 @@ package ocr import ( - "fmt" "math/big" "math/rand" "time" @@ -45,7 +44,7 @@ func SetupFeed( if err != nil { return nil, err } - err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, fmt.Sprint(seth.ChainID)) + err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, seth.ChainID, false) if err != nil { return nil, err } diff --git a/integration-tests/load/ocr/vu.go b/integration-tests/load/ocr/vu.go index 88f186c5ee1..af6cb2265d9 100644 --- a/integration-tests/load/ocr/vu.go +++ b/integration-tests/load/ocr/vu.go @@ -2,7 +2,6 @@ package ocr import ( "context" - "fmt" "sync/atomic" "time" @@ -82,7 +81,7 @@ func (m *VU) Setup(_ *wasp.Generator) error { if err != nil { return err } - err = actions.CreateOCRJobs(ocrInstances, m.bootstrapNode, m.workerNodes, 5, m.msClient, fmt.Sprint(m.seth.ChainID)) + err = actions.CreateOCRJobs(ocrInstances, m.bootstrapNode, m.workerNodes, 5, m.msClient, m.seth.ChainID, false) if err != nil { return err } diff --git a/integration-tests/testsetups/ocr.go b/integration-tests/testsetups/ocr.go index 8d797926607..413073181e7 100644 --- a/integration-tests/testsetups/ocr.go +++ b/integration-tests/testsetups/ocr.go @@ -392,14 +392,9 @@ func (o *OCRSoakTest) Run() { o.startingBlockNum = latestBlockNum startingValue := 5 - if o.OperatorForwarderFlow { - actions.CreateOCRJobsWithForwarder(o.t, o.ocrV1Instances, o.bootstrapNode, o.workerNodes, startingValue, o.mockServer, o.seth.ChainID) - } else if o.OCRVersion == "1" { - ctx, cancel := context.WithTimeout(testcontext.Get(o.t), time.Second*5) - chainId, err := o.seth.Client.ChainID(ctx) - cancel() + if o.OCRVersion == "1" { require.NoError(o.t, err, "Error getting chain ID") - err = actions.CreateOCRJobs(o.ocrV1Instances, o.bootstrapNode, o.workerNodes, startingValue, o.mockServer, chainId.String()) + err = actions.CreateOCRJobs(o.ocrV1Instances, o.bootstrapNode, o.workerNodes, startingValue, o.mockServer, o.seth.ChainID, o.OperatorForwarderFlow) require.NoError(o.t, err, "Error creating OCR jobs") } else if o.OCRVersion == "2" { err := actions.CreateOCRv2Jobs(o.ocrV2Instances, o.bootstrapNode, o.workerNodes, o.mockServer, startingValue, o.seth.ChainID, o.OperatorForwarderFlow)