diff --git a/core/services/ocr3/plugins/ccip_integration_tests/helpers.go b/core/services/ocr3/plugins/ccip_integration_tests/helpers.go index 2666e732ae..7a22dfc4bf 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/helpers.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/helpers.go @@ -3,12 +3,19 @@ package ccip_integration_tests import ( "encoding/hex" "math/big" + "strconv" "testing" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ocr3_config_encoder" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/weth9" + cctypes "github.com/smartcontractkit/chainlink/v2/core/services/ccipcapability/types" + confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" @@ -23,7 +30,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/price_registry" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/token_admin_registry" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/weth9" kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/shared/generated/link_token" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" @@ -38,6 +44,7 @@ var ( const ( CapabilityLabelledName = "ccip" CapabilityVersion = "v1.0.0" + NodeOperatorID = 1 ) func e18Mult(amount uint64) *big.Int { @@ -53,7 +60,7 @@ type homeChain struct { owner *bind.TransactOpts chainID uint64 capabilityRegistry *kcr.CapabilitiesRegistry - ccipConfigContract common.Address + ccipConfig *ccip_config.CCIPConfig } type onchainUniverse struct { @@ -255,15 +262,154 @@ func setupHomeChain(t *testing.T, owner *bind.TransactOpts, backend *backends.Si require.NoError(t, err, "failed to add capabilities to the capability registry") backend.Commit() + // Add NodeOperator, for simplicity we'll add one NodeOperator only + // First NodeOperator will have NodeOperatorId = 1 + _, err = capabilityRegistry.AddNodeOperators(owner, []kcr.CapabilitiesRegistryNodeOperator{ + { + Admin: owner.From, + Name: "NodeOperator", + }, + }) + require.NoError(t, err, "failed to add node operator to the capability registry") + backend.Commit() + return homeChain{ backend: backend, owner: owner, chainID: homeChainID, capabilityRegistry: capabilityRegistry, - ccipConfigContract: capabilityConfig.Address(), + ccipConfig: capabilityConfig, + } +} + +// bubble sort, n^2 but p2pIDs never should be more than a handful +func sortP2pIDS(p2pIDs [][32]byte) { + for i := 0; i < len(p2pIDs); i++ { + for j := i + 1; j < len(p2pIDs); j++ { + if hex.EncodeToString(p2pIDs[i][:]) > hex.EncodeToString(p2pIDs[j][:]) { + p2pIDs[i], p2pIDs[j] = p2pIDs[j], p2pIDs[i] + } + } } } +func (h *homeChain) AddNodes( + t *testing.T, + p2pIDs [][32]byte, + capabilityIDs [][32]byte, +) { + // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail + sortP2pIDS(p2pIDs) + var nodeParams []kcr.CapabilitiesRegistryNodeParams + for _, p2pID := range p2pIDs { + nodeParam := kcr.CapabilitiesRegistryNodeParams{ + NodeOperatorId: NodeOperatorID, + Signer: p2pID, // Not used in tests + P2pId: p2pID, + HashedCapabilityIds: capabilityIDs, + } + nodeParams = append(nodeParams, nodeParam) + } + _, err := h.capabilityRegistry.AddNodes(h.owner, nodeParams) + require.NoError(t, err, "failed to add node operator oracles") + h.backend.Commit() +} + +func (h *homeChain) AddDON( + t *testing.T, + ccipCapabilityID [32]byte, + chainSelector uint64, + OfframpAddress []byte, + f uint8, + bootstrapP2PID [32]byte, + p2pIDs [][32]byte, + oracles []confighelper2.OracleIdentityExtra, +) { + // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail + sortP2pIDS(p2pIDs) + // First Add ChainConfig that includes all p2pIDs as readers + chainConfig := SetupConfigInfo(chainSelector, p2pIDs, FChainA, []byte(strconv.FormatUint(chainSelector, 10))) + inputConfig := []ccip_config.CCIPConfigTypesChainConfigInfo{ + chainConfig, + } + _, err := h.ccipConfig.ApplyChainConfigUpdates(h.owner, nil, inputConfig) + require.NoError(t, err) + h.backend.Commit() + + // Get OCR3 Config from helper + var schedule []int + for range oracles { + schedule = append(schedule, 1) + } + offchainConfig, onchainConfig := []byte{}, []byte{} + signers, transmitters, f, onchainConfig, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTests( + 30*time.Second, // deltaProgress + 10*time.Second, // deltaResend + 20*time.Second, // deltaInitial + 2*time.Second, // deltaRound + 20*time.Second, // deltaGrace + 10*time.Second, // deltaCertifiedCommitRequest + 10*time.Second, // deltaStage + 3, // rmax + schedule, + oracles, + offchainConfig, + 50*time.Millisecond, // maxDurationQuery + 5*time.Second, // maxDurationObservation + 10*time.Second, // maxDurationShouldAcceptAttestedReport + 10*time.Second, // maxDurationShouldTransmitAcceptedReport + int(f), + onchainConfig) + require.NoError(t, err, "failed to create contract config") + + tabi, err := ocr3_config_encoder.IOCR3ConfigEncoderMetaData.GetAbi() + require.NoError(t, err) + + signersBytes := make([][]byte, len(signers)) + for i, signer := range signers { + signersBytes[i] = signer + } + + transmittersBytes := make([][]byte, len(transmitters)) + for i, transmitter := range transmitters { + parsed, err := common.ParseHexOrString(string(transmitter)) + require.NoError(t, err) + transmittersBytes[i] = parsed + } + + // Add DON on capability registry contract + var ocr3Configs []ocr3_config_encoder.CCIPConfigTypesOCR3Config + for _, pluginType := range []cctypes.PluginType{cctypes.PluginTypeCCIPCommit, cctypes.PluginTypeCCIPExec} { + ocr3Configs = append(ocr3Configs, ocr3_config_encoder.CCIPConfigTypesOCR3Config{ + PluginType: uint8(pluginType), + ChainSelector: chainSelector, + F: f, + OffchainConfigVersion: offchainConfigVersion, + OfframpAddress: OfframpAddress, + BootstrapP2PIds: [][32]byte{bootstrapP2PID}, + P2pIds: p2pIDs, + Signers: signersBytes, + Transmitters: transmittersBytes, + OffchainConfig: offchainConfig, + }) + } + + encodedCall, err := tabi.Pack("exposeOCR3Config", ocr3Configs) + require.NoError(t, err) + + // Trim first four bytes to remove function selector. + encodedConfigs := encodedCall[4:] + + _, err = h.capabilityRegistry.AddDON(h.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{ + { + CapabilityId: ccipCapabilityID, + Config: encodedConfigs, + }, + }, false, false, f) + require.NoError(t, err) + h.backend.Commit() +} + func connectUniverses( t *testing.T, universes map[uint64]onchainUniverse, @@ -331,6 +477,17 @@ func setupUniverseBasics(t *testing.T, uni onchainUniverse) { uni.backend.Commit() } +func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo { + return ccip_config.CCIPConfigTypesChainConfigInfo{ + ChainSelector: chainSelector, + ChainConfig: ccip_config.CCIPConfigTypesChainConfig{ + Readers: readers, + FChain: fChain, + Config: cfg, + }, + } +} + // As we can't change router contract. The contract was expecting onRamp and offRamp per lane and not per chain // In the new architecture we have only one onRamp and one offRamp per chain. // hence we add the mapping for all remote chains to the onRamp/offRamp contract of the local chain diff --git a/core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go b/core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go similarity index 83% rename from core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go rename to core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go index ab8629f8c1..1a7c24bd76 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go @@ -1,4 +1,4 @@ -package home_chain +package ccip_integration_tests import ( "testing" @@ -14,27 +14,25 @@ import ( capcfg "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/logger" - it "github.com/smartcontractkit/chainlink/v2/core/services/ocr3/plugins/ccip_integration_tests" - "github.com/stretchr/testify/require" ) func TestHomeChainReader(t *testing.T) { ctx := testutils.Context(t) lggr := logger.TestLogger(t) - uni := it.NewTestUniverse(ctx, t, lggr) + uni := NewTestUniverse(ctx, t, lggr) // We need 3*f + 1 p2pIDs to have enough nodes to bootstrap var arr []int64 - n := int(it.FChainA*3 + 1) + n := int(FChainA*3 + 1) for i := 0; i <= n; i++ { arr = append(arr, int64(i)) } - p2pIDs := it.P2pIDsFromInts(arr) + p2pIDs := P2pIDsFromInts(arr) uni.AddCapability(p2pIDs) //==============================Apply configs to Capability Contract================================= - chainAConf := it.SetupConfigInfo(it.ChainA, p2pIDs, it.FChainA, []byte("ChainA")) - chainBConf := it.SetupConfigInfo(it.ChainB, p2pIDs[1:], it.FChainB, []byte("ChainB")) - chainCConf := it.SetupConfigInfo(it.ChainC, p2pIDs[2:], it.FChainC, []byte("ChainC")) + chainAConf := SetupConfigInfo(ChainA, p2pIDs, FChainA, []byte("ChainA")) + chainBConf := SetupConfigInfo(ChainB, p2pIDs[1:], FChainB, []byte("ChainB")) + chainCConf := SetupConfigInfo(ChainC, p2pIDs[2:], FChainC, []byte("ChainC")) inputConfig := []capcfg.CCIPConfigTypesChainConfigInfo{ chainAConf, chainBConf, @@ -66,13 +64,13 @@ func TestHomeChainReader(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedChainConfigs, configs) //=================================Remove ChainC from OnChainConfig========================================= - _, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{it.ChainC}, nil) + _, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{ChainC}, nil) require.NoError(t, err) uni.Backend.Commit() time.Sleep(pollDuration * 5) // Wait for the chain reader to update configs, err = homeChain.GetAllChainConfigs() require.NoError(t, err) - delete(expectedChainConfigs, cciptypes.ChainSelector(it.ChainC)) + delete(expectedChainConfigs, cciptypes.ChainSelector(ChainC)) require.Equal(t, expectedChainConfigs, configs) //================================Close HomeChain Reader=============================== //require.NoError(t, homeChain.Close()) diff --git a/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go b/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go index a595bb2378..93942dbc18 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go @@ -294,14 +294,3 @@ func (t *TestUniverse) AddDONToRegistry( require.NoError(t.TestingT, err) t.Backend.Commit() } - -func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo { - return ccip_config.CCIPConfigTypesChainConfigInfo{ - ChainSelector: chainSelector, - ChainConfig: ccip_config.CCIPConfigTypesChainConfig{ - Readers: readers, - FChain: fChain, - Config: cfg, - }, - } -} diff --git a/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go b/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go index 8bdfb4afc2..7935b2a403 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go @@ -34,20 +34,12 @@ func TestIntegration_OCR3Nodes(t *testing.T) { // The bootstrap node will be the first node (index 0) bootstrapPort int bootstrapP2PID p2pkey.PeerID + bootStrappers []commontypes.BootstrapperLocator ) - bootstrapNodePort := freeport.GetOne(t) - bootstrapNode := setupNodeOCR3(t, bootstrapNodePort, nil, universes) - ports := freeport.GetN(t, numNodes) for i := 0; i < numNodes; i++ { - // Supply the bootstrap IP and port as a V2 peer address - bootstrappers := []commontypes.BootstrapperLocator{ - {PeerID: bootstrapNode.peerID, Addrs: []string{ - fmt.Sprintf("127.0.0.1:%d", bootstrapNodePort), - }}, - } - node := setupNodeOCR3(t, ports[i], bootstrappers, universes) + node := setupNodeOCR3(t, ports[i], bootStrappers, universes) apps = append(apps, node.app) for chainID, transmitter := range node.transmitters { @@ -69,6 +61,17 @@ func TestIntegration_OCR3Nodes(t *testing.T) { peerID, err := p2pkey.MakePeerID(node.peerID) require.NoError(t, err) p2pIDs = append(p2pIDs, peerID) + + // First Node is the bootstrap node + if i == 0 { + bootstrapPort = ports[i] + bootstrapP2PID = peerID + bootStrappers = []commontypes.BootstrapperLocator{ + {PeerID: node.peerID, Addrs: []string{ + fmt.Sprintf("127.0.0.1:%d", bootstrapPort), + }}, + } + } } t.Log("starting ticker to commit blocks") @@ -96,6 +99,8 @@ func TestIntegration_OCR3Nodes(t *testing.T) { wg.Wait() }) + bootstrapNode := nodes[0] + t.Log("adding bootstrap node job") err := bootstrapNode.app.Start(testutils.Context(t)) require.NoError(t, err, "failed to start bootstrap node") @@ -109,15 +114,16 @@ func TestIntegration_OCR3Nodes(t *testing.T) { t.Log("creating ocr3 jobs") for i := 0; i < numNodes; i++ { - err := apps[i].Start(testutils.Context(t)) - require.NoError(t, err) - tapp := apps[i] - t.Cleanup(func() { - require.NoError(t, tapp.Stop()) - }) + //err := apps[i].Start(testutils.Context(t)) + //require.NoError(t, err) + //tApp := apps[i] + //t.Cleanup(func() { + // require.NoError(t, tApp.Stop()) + //}) + // + //ccipSpecToml := createCCIPSpecToml(nodes[i].peerID, bootstrapP2PID.String(), bootstrapPort, nodes[i].keybundle.ID()) + //t.Log("Creating ccip job with spec:\n", ccipSpecToml) - ccipSpecToml := createCCIPSpecToml(nodes[i].peerID, bootstrapP2PID.String(), bootstrapPort, nodes[i].keybundle.ID()) - t.Log("Creating ccip job with spec:\n", ccipSpecToml) //ccipJob, err2 := ccipcapability(ccipSpecToml) //require.NoError(t, err2, "failed to validate ccip job") //err2 = apps[i].AddJobV2(testutils.Context(t), &ccipJob) @@ -128,13 +134,20 @@ func TestIntegration_OCR3Nodes(t *testing.T) { ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId(nil, CapabilityLabelledName, CapabilityVersion) require.NoError(t, err, "failed to get hashed capability id for ccip") require.NotEqual(t, [32]byte{}, ccipCapabilityID, "ccip capability id is empty") + + homeChainUni.AddNodes(t, p2pIDs, [][32]byte{ccipCapabilityID}) // create a DON for each chain - //for _, uni := range universes { - // homeChainUni.capabilityRegistry.AddDON(uni.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{ - // { - // CapabilityId: ccipCapabilityID, - // Config: donOCRConfig(t, uni, oracles[uni.chainID]), - // }, - // }, false, false, 1 /* f value: unused for ccip */) - //} + for _, uni := range universes { + // Add nodes and give them the capability + t.Log("AddingDON for universe: ", uni.chainID) + homeChainUni.AddDON(t, + ccipCapabilityID, + uni.chainID, + uni.offramp.Address().Bytes(), + 1, // f + bootstrapP2PID, + p2pIDs, + oracles[uni.chainID], + ) + } } diff --git a/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go b/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go index 2287cb2835..ea4f9e2b63 100644 --- a/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go +++ b/core/services/ocr3/plugins/ccip_integration_tests/ocr_node_helper.go @@ -12,15 +12,14 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" gethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/jmoiron/sqlx" - chainsel "github.com/smartcontractkit/chain-selectors" "github.com/smartcontractkit/chainlink-common/pkg/config" "github.com/smartcontractkit/chainlink-common/pkg/loop" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" v2toml "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml" - encodeutils "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" evmutils "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest/heavyweight" @@ -34,8 +33,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/utils" "github.com/smartcontractkit/chainlink/v2/plugins" "github.com/smartcontractkit/libocr/commontypes" - confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" ) @@ -85,7 +82,7 @@ func setupNodeOCR3( }) lggr := logger.TestLogger(t) - lggr.SetLogLevel(zapcore.DebugLevel) + lggr.SetLogLevel(zapcore.InfoLevel) ctx := testutils.Context(t) clients := make(map[uint64]client.Client) @@ -253,97 +250,11 @@ func createConfigV2Chain(chainID *big.Int) *v2toml.EVMConfig { } } -func donOCRConfig(t *testing.T, uni onchainUniverse, oracles []confighelper2.OracleIdentityExtra) []byte { - var schedule []int - for range oracles { - schedule = append(schedule, 1) - } - offchainConfig, onchainConfig := []byte{}, []byte{} - f := uint8(1) - _, _, f, _, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTests( - 30*time.Second, // deltaProgress - 10*time.Second, // deltaResend - 20*time.Second, // deltaInitial - 2*time.Second, // deltaRound - 20*time.Second, // deltaGrace - 10*time.Second, // deltaCertifiedCommitRequest - 10*time.Second, // deltaStage - 3, // rmax - schedule, - oracles, - offchainConfig, - 50*time.Millisecond, // maxDurationQuery - 5*time.Second, // maxDurationObservation - 10*time.Second, // maxDurationShouldAcceptAttestedReport - 10*time.Second, // maxDurationShouldTransmitAcceptedReport - int(f), - onchainConfig) - require.NoError(t, err, "failed to create contract config") - /* - struct OCR3Config { - PluginType pluginType; // ────────╮ The plugin that the configuration is for. - uint64 chainSelector; // | The (remote) chain that the configuration is for. - uint8 F; // | The "big F" parameter for the role DON. - uint64 offchainConfigVersion; // ─╯ The version of the offchain configuration. - bytes32 offrampAddress; // The remote chain combined (offramp|commit store) address. - bytes32[2][] signers; // An associative array that contains (p2p id, onchain signer public key) pairs. - bytes32[2][] transmitters; // An associative array that contains (p2p id, transmitter) pairs. - bytes offchainConfig; // The offchain configuration for the OCR3 protocol. Protobuf encoded. - } - */ - ocrConfigABI := ` -{ - [ - { - "type": "uint8" - }, - { - "type": "uint64" - }, - { - "type": "uint8" - }, - { - "type": "uint64" - }, - { - "type": "bytes32" - }, - { - "type": "bytes32[2][]" - }, - { - "type": "bytes32[2][]" - }, - { - "type": "bytes" - } - ] -}` - chainSelector, ok := chainsel.EvmChainIdToChainSelector()[uni.chainID] - require.True(t, ok, "chain selector not found for chain id", uni.chainID) - var offrampAddressBytes32 [32]byte - copy(offrampAddressBytes32[:], uni.offramp.Address().Bytes()) - commitConfig, err := encodeutils.ABIEncode(ocrConfigABI, - uint8(0), // pluginType - chainSelector, // chainSelector - f, // F - offchainConfigVersion, // offchainConfigVersion - offrampAddressBytes32, // offrampAddress - nil, // TODO signers - nil, // TODO transmitters - offchainConfig, // offchainConfig - ) - require.NoError(t, err, "failed to encode commit OCR3 config") - // TODO: implement - return commitConfig -} - func createCCIPSpecToml(nodeP2PID, bootstrapP2PID string, bootstrapPort int, ocrKeyBundleID string) string { return fmt.Sprintf(` type = "ccip" -capabilityVersion = "v1.0.0" -capabilityLabelledName = "ccip" +capabilityVersion = "%s" +capabilityLabelledName = "%s" p2pKeyID = "%s" p2pV2Bootstrappers = ["%s"] [ocrKeyBundleIDs] @@ -356,6 +267,8 @@ chainSpecificName = "getStuffEVM" [pluginConfig] tokenPricesPipeline = "the pipeline"`, + CapabilityVersion, + CcipCapabilityLabelledName, nodeP2PID, fmt.Sprintf("%s@127.0.0.1:%d", bootstrapP2PID, bootstrapPort,