Adding DON successfully

core/services/ocr3/plugins/ccip_integration_tests/helpers.go

@@ -3,12 +3,19 @@ package ccip_integration_tests
 import (
 	"encoding/hex"
 	"math/big"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ocr3_config_encoder"
+	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/weth9"
+	cctypes "github.com/smartcontractkit/chainlink/v2/core/services/ccipcapability/types"
+	confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
+	"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper"
 
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config"
@@ -23,7 +30,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/price_registry"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/token_admin_registry"
-	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/weth9"
 	kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/shared/generated/link_token"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
@@ -38,6 +44,7 @@ var (
 const (
 	CapabilityLabelledName = "ccip"
 	CapabilityVersion      = "v1.0.0"
+	NodeOperatorID         = 1
 )
 
 func e18Mult(amount uint64) *big.Int {
@@ -53,7 +60,7 @@ type homeChain struct {
 	owner              *bind.TransactOpts
 	chainID            uint64
 	capabilityRegistry *kcr.CapabilitiesRegistry
-	ccipConfigContract common.Address
+	ccipConfig         *ccip_config.CCIPConfig
 }
 
 type onchainUniverse struct {
@@ -255,15 +262,154 @@ func setupHomeChain(t *testing.T, owner *bind.TransactOpts, backend *backends.Si
 	require.NoError(t, err, "failed to add capabilities to the capability registry")
 	backend.Commit()
 
+	// Add NodeOperator, for simplicity we'll add one NodeOperator only
+	// First NodeOperator will have NodeOperatorId = 1
+	_, err = capabilityRegistry.AddNodeOperators(owner, []kcr.CapabilitiesRegistryNodeOperator{
+		{
+			Admin: owner.From,
+			Name:  "NodeOperator",
+		},
+	})
+	require.NoError(t, err, "failed to add node operator to the capability registry")
+	backend.Commit()
+
 	return homeChain{
 		backend:            backend,
 		owner:              owner,
 		chainID:            homeChainID,
 		capabilityRegistry: capabilityRegistry,
-		ccipConfigContract: capabilityConfig.Address(),
+		ccipConfig:         capabilityConfig,
+	}
+}
+
+// bubble sort, n^2 but p2pIDs never should be more than a handful
+func sortP2pIDS(p2pIDs [][32]byte) {
+	for i := 0; i < len(p2pIDs); i++ {
+		for j := i + 1; j < len(p2pIDs); j++ {
+			if hex.EncodeToString(p2pIDs[i][:]) > hex.EncodeToString(p2pIDs[j][:]) {
+				p2pIDs[i], p2pIDs[j] = p2pIDs[j], p2pIDs[i]
+			}
+		}
 	}
 }
 
+func (h *homeChain) AddNodes(
+	t *testing.T,
+	p2pIDs [][32]byte,
+	capabilityIDs [][32]byte,
+) {
+	// Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail
+	sortP2pIDS(p2pIDs)
+	var nodeParams []kcr.CapabilitiesRegistryNodeParams
+	for _, p2pID := range p2pIDs {
+		nodeParam := kcr.CapabilitiesRegistryNodeParams{
+			NodeOperatorId:      NodeOperatorID,
+			Signer:              p2pID, // Not used in tests
+			P2pId:               p2pID,
+			HashedCapabilityIds: capabilityIDs,
+		}
+		nodeParams = append(nodeParams, nodeParam)
+	}
+	_, err := h.capabilityRegistry.AddNodes(h.owner, nodeParams)
+	require.NoError(t, err, "failed to add node operator oracles")
+	h.backend.Commit()
+}
+
+func (h *homeChain) AddDON(
+	t *testing.T,
+	ccipCapabilityID [32]byte,
+	chainSelector uint64,
+	OfframpAddress []byte,
+	f uint8,
+	bootstrapP2PID [32]byte,
+	p2pIDs [][32]byte,
+	oracles []confighelper2.OracleIdentityExtra,
+) {
+	// Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail
+	sortP2pIDS(p2pIDs)
+	// First Add ChainConfig that includes all p2pIDs as readers
+	chainConfig := SetupConfigInfo(chainSelector, p2pIDs, FChainA, []byte(strconv.FormatUint(chainSelector, 10)))
+	inputConfig := []ccip_config.CCIPConfigTypesChainConfigInfo{
+		chainConfig,
+	}
+	_, err := h.ccipConfig.ApplyChainConfigUpdates(h.owner, nil, inputConfig)
+	require.NoError(t, err)
+	h.backend.Commit()
+
+	// Get OCR3 Config from helper
+	var schedule []int
+	for range oracles {
+		schedule = append(schedule, 1)
+	}
+	offchainConfig, onchainConfig := []byte{}, []byte{}
+	signers, transmitters, f, onchainConfig, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTests(
+		30*time.Second, // deltaProgress
+		10*time.Second, // deltaResend
+		20*time.Second, // deltaInitial
+		2*time.Second,  // deltaRound
+		20*time.Second, // deltaGrace
+		10*time.Second, // deltaCertifiedCommitRequest
+		10*time.Second, // deltaStage
+		3,              // rmax
+		schedule,
+		oracles,
+		offchainConfig,
+		50*time.Millisecond, // maxDurationQuery
+		5*time.Second,       // maxDurationObservation
+		10*time.Second,      // maxDurationShouldAcceptAttestedReport
+		10*time.Second,      // maxDurationShouldTransmitAcceptedReport
+		int(f),
+		onchainConfig)
+	require.NoError(t, err, "failed to create contract config")
+
+	tabi, err := ocr3_config_encoder.IOCR3ConfigEncoderMetaData.GetAbi()
+	require.NoError(t, err)
+
+	signersBytes := make([][]byte, len(signers))
+	for i, signer := range signers {
+		signersBytes[i] = signer
+	}
+
+	transmittersBytes := make([][]byte, len(transmitters))
+	for i, transmitter := range transmitters {
+		parsed, err := common.ParseHexOrString(string(transmitter))
+		require.NoError(t, err)
+		transmittersBytes[i] = parsed
+	}
+
+	// Add DON on capability registry contract
+	var ocr3Configs []ocr3_config_encoder.CCIPConfigTypesOCR3Config
+	for _, pluginType := range []cctypes.PluginType{cctypes.PluginTypeCCIPCommit, cctypes.PluginTypeCCIPExec} {
+		ocr3Configs = append(ocr3Configs, ocr3_config_encoder.CCIPConfigTypesOCR3Config{
+			PluginType:            uint8(pluginType),
+			ChainSelector:         chainSelector,
+			F:                     f,
+			OffchainConfigVersion: offchainConfigVersion,
+			OfframpAddress:        OfframpAddress,
+			BootstrapP2PIds:       [][32]byte{bootstrapP2PID},
+			P2pIds:                p2pIDs,
+			Signers:               signersBytes,
+			Transmitters:          transmittersBytes,
+			OffchainConfig:        offchainConfig,
+		})
+	}
+
+	encodedCall, err := tabi.Pack("exposeOCR3Config", ocr3Configs)
+	require.NoError(t, err)
+
+	// Trim first four bytes to remove function selector.
+	encodedConfigs := encodedCall[4:]
+
+	_, err = h.capabilityRegistry.AddDON(h.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{
+		{
+			CapabilityId: ccipCapabilityID,
+			Config:       encodedConfigs,
+		},
+	}, false, false, f)
+	require.NoError(t, err)
+	h.backend.Commit()
+}
+
 func connectUniverses(
 	t *testing.T,
 	universes map[uint64]onchainUniverse,
@@ -331,6 +477,17 @@ func setupUniverseBasics(t *testing.T, uni onchainUniverse) {
 	uni.backend.Commit()
 }
 
+func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo {
+	return ccip_config.CCIPConfigTypesChainConfigInfo{
+		ChainSelector: chainSelector,
+		ChainConfig: ccip_config.CCIPConfigTypesChainConfig{
+			Readers: readers,
+			FChain:  fChain,
+			Config:  cfg,
+		},
+	}
+}
+
 // As we can't change router contract. The contract was expecting onRamp and offRamp per lane and not per chain
 // In the new architecture we have only one onRamp and one offRamp per chain.
 // hence we add the mapping for all remote chains to the onRamp/offRamp contract of the local chain

core/services/ocr3/plugins/ccip_integration_tests/home_chain/home_chain_test.go → core/services/ocr3/plugins/ccip_integration_tests/home_chain_test.go

@@ -1,4 +1,4 @@
-package home_chain
+package ccip_integration_tests
 
 import (
 	"testing"
@@ -14,27 +14,25 @@ import (
 	capcfg "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/ccip_config"
 	"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
 	"github.com/smartcontractkit/chainlink/v2/core/logger"
-	it "github.com/smartcontractkit/chainlink/v2/core/services/ocr3/plugins/ccip_integration_tests"
-
 	"github.com/stretchr/testify/require"
 )
 
 func TestHomeChainReader(t *testing.T) {
 	ctx := testutils.Context(t)
 	lggr := logger.TestLogger(t)
-	uni := it.NewTestUniverse(ctx, t, lggr)
+	uni := NewTestUniverse(ctx, t, lggr)
 	// We need 3*f + 1 p2pIDs to have enough nodes to bootstrap
 	var arr []int64
-	n := int(it.FChainA*3 + 1)
+	n := int(FChainA*3 + 1)
 	for i := 0; i <= n; i++ {
 		arr = append(arr, int64(i))
 	}
-	p2pIDs := it.P2pIDsFromInts(arr)
+	p2pIDs := P2pIDsFromInts(arr)
 	uni.AddCapability(p2pIDs)
 	//==============================Apply configs to Capability Contract=================================
-	chainAConf := it.SetupConfigInfo(it.ChainA, p2pIDs, it.FChainA, []byte("ChainA"))
-	chainBConf := it.SetupConfigInfo(it.ChainB, p2pIDs[1:], it.FChainB, []byte("ChainB"))
-	chainCConf := it.SetupConfigInfo(it.ChainC, p2pIDs[2:], it.FChainC, []byte("ChainC"))
+	chainAConf := SetupConfigInfo(ChainA, p2pIDs, FChainA, []byte("ChainA"))
+	chainBConf := SetupConfigInfo(ChainB, p2pIDs[1:], FChainB, []byte("ChainB"))
+	chainCConf := SetupConfigInfo(ChainC, p2pIDs[2:], FChainC, []byte("ChainC"))
 	inputConfig := []capcfg.CCIPConfigTypesChainConfigInfo{
 		chainAConf,
 		chainBConf,
@@ -66,13 +64,13 @@ func TestHomeChainReader(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, expectedChainConfigs, configs)
 	//=================================Remove ChainC from OnChainConfig=========================================
-	_, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{it.ChainC}, nil)
+	_, err = uni.CcipCfg.ApplyChainConfigUpdates(uni.Transactor, []uint64{ChainC}, nil)
 	require.NoError(t, err)
 	uni.Backend.Commit()
 	time.Sleep(pollDuration * 5) // Wait for the chain reader to update
 	configs, err = homeChain.GetAllChainConfigs()
 	require.NoError(t, err)
-	delete(expectedChainConfigs, cciptypes.ChainSelector(it.ChainC))
+	delete(expectedChainConfigs, cciptypes.ChainSelector(ChainC))
 	require.Equal(t, expectedChainConfigs, configs)
 	//================================Close HomeChain Reader===============================
 	//require.NoError(t, homeChain.Close())

core/services/ocr3/plugins/ccip_integration_tests/integration_helpers.go

@@ -294,14 +294,3 @@ func (t *TestUniverse) AddDONToRegistry(
 	require.NoError(t.TestingT, err)
 	t.Backend.Commit()
 }
-
-func SetupConfigInfo(chainSelector uint64, readers [][32]byte, fChain uint8, cfg []byte) ccip_config.CCIPConfigTypesChainConfigInfo {
-	return ccip_config.CCIPConfigTypesChainConfigInfo{
-		ChainSelector: chainSelector,
-		ChainConfig: ccip_config.CCIPConfigTypesChainConfig{
-			Readers: readers,
-			FChain:  fChain,
-			Config:  cfg,
-		},
-	}
-}

core/services/ocr3/plugins/ccip_integration_tests/ocr3_node_test.go

@@ -34,20 +34,12 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
 		// The bootstrap node will be the first node (index 0)
 		bootstrapPort  int
 		bootstrapP2PID p2pkey.PeerID
+		bootStrappers  []commontypes.BootstrapperLocator
 	)
 
-	bootstrapNodePort := freeport.GetOne(t)
-	bootstrapNode := setupNodeOCR3(t, bootstrapNodePort, nil, universes)
-
 	ports := freeport.GetN(t, numNodes)
 	for i := 0; i < numNodes; i++ {
-		// Supply the bootstrap IP and port as a V2 peer address
-		bootstrappers := []commontypes.BootstrapperLocator{
-			{PeerID: bootstrapNode.peerID, Addrs: []string{
-				fmt.Sprintf("127.0.0.1:%d", bootstrapNodePort),
-			}},
-		}
-		node := setupNodeOCR3(t, ports[i], bootstrappers, universes)
+		node := setupNodeOCR3(t, ports[i], bootStrappers, universes)
 
 		apps = append(apps, node.app)
 		for chainID, transmitter := range node.transmitters {
@@ -69,6 +61,17 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
 		peerID, err := p2pkey.MakePeerID(node.peerID)
 		require.NoError(t, err)
 		p2pIDs = append(p2pIDs, peerID)
+
+		// First Node is the bootstrap node
+		if i == 0 {
+			bootstrapPort = ports[i]
+			bootstrapP2PID = peerID
+			bootStrappers = []commontypes.BootstrapperLocator{
+				{PeerID: node.peerID, Addrs: []string{
+					fmt.Sprintf("127.0.0.1:%d", bootstrapPort),
+				}},
+			}
+		}
 	}
 
 	t.Log("starting ticker to commit blocks")
@@ -96,6 +99,8 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
 		wg.Wait()
 	})
 
+	bootstrapNode := nodes[0]
+
 	t.Log("adding bootstrap node job")
 	err := bootstrapNode.app.Start(testutils.Context(t))
 	require.NoError(t, err, "failed to start bootstrap node")
@@ -109,15 +114,16 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
 
 	t.Log("creating ocr3 jobs")
 	for i := 0; i < numNodes; i++ {
-		err := apps[i].Start(testutils.Context(t))
-		require.NoError(t, err)
-		tapp := apps[i]
-		t.Cleanup(func() {
-			require.NoError(t, tapp.Stop())
-		})
+		//err := apps[i].Start(testutils.Context(t))
+		//require.NoError(t, err)
+		//tApp := apps[i]
+		//t.Cleanup(func() {
+		//	require.NoError(t, tApp.Stop())
+		//})
+		//
+		//ccipSpecToml := createCCIPSpecToml(nodes[i].peerID, bootstrapP2PID.String(), bootstrapPort, nodes[i].keybundle.ID())
+		//t.Log("Creating ccip job with spec:\n", ccipSpecToml)
 
-		ccipSpecToml := createCCIPSpecToml(nodes[i].peerID, bootstrapP2PID.String(), bootstrapPort, nodes[i].keybundle.ID())
-		t.Log("Creating ccip job with spec:\n", ccipSpecToml)
 		//ccipJob, err2 := ccipcapability(ccipSpecToml)
 		//require.NoError(t, err2, "failed to validate ccip job")
 		//err2 = apps[i].AddJobV2(testutils.Context(t), &ccipJob)
@@ -128,13 +134,20 @@ func TestIntegration_OCR3Nodes(t *testing.T) {
 	ccipCapabilityID, err := homeChainUni.capabilityRegistry.GetHashedCapabilityId(nil, CapabilityLabelledName, CapabilityVersion)
 	require.NoError(t, err, "failed to get hashed capability id for ccip")
 	require.NotEqual(t, [32]byte{}, ccipCapabilityID, "ccip capability id is empty")
+
+	homeChainUni.AddNodes(t, p2pIDs, [][32]byte{ccipCapabilityID})
 	// create a DON for each chain
-	//for _, uni := range universes {
-	//	homeChainUni.capabilityRegistry.AddDON(uni.owner, p2pIDs, []kcr.CapabilitiesRegistryCapabilityConfiguration{
-	//		{
-	//			CapabilityId: ccipCapabilityID,
-	//			Config:       donOCRConfig(t, uni, oracles[uni.chainID]),
-	//		},
-	//	}, false, false, 1 /* f value: unused for ccip */)
-	//}
+	for _, uni := range universes {
+		// Add nodes and give them the capability
+		t.Log("AddingDON for universe: ", uni.chainID)
+		homeChainUni.AddDON(t,
+			ccipCapabilityID,
+			uni.chainID,
+			uni.offramp.Address().Bytes(),
+			1, // f
+			bootstrapP2PID,
+			p2pIDs,
+			oracles[uni.chainID],
+		)
+	}
 }