Merge branch 'develop' into MERC-3331-draft-implement-link-w-eth-bill…

…ing-contract-change-dev

.github/workflows/ci-core.yml

@@ -82,6 +82,7 @@ jobs:
           pushd $(go list -m -f "{{.Dir}}" github.com/smartcontractkit/chainlink-starknet/relayer)
           go install ./pkg/chainlink/cmd/chainlink-starknet
           popd
+          make install-mercury-loop
       - name: Increase Race Timeout
         if: github.event.schedule != ''
         run: |
@@ -257,6 +258,8 @@ jobs:
         uses: ./.github/actions/setup-go
         with:
           only-modules: "true"
+      - name: Install protoc-gen-go-wsrpc
+        run: curl https://github.com/smartcontractkit/wsrpc/raw/main/cmd/protoc-gen-go-wsrpc/protoc-gen-go-wsrpc --output $HOME/go/bin/protoc-gen-go-wsrpc && chmod +x $HOME/go/bin/protoc-gen-go-wsrpc
       - name: Setup NodeJS
         uses: ./.github/actions/setup-nodejs
       - run: make generate # generate install go deps

.golangci.yml

@@ -71,7 +71,7 @@ linters-settings:
       - name: identical-branches
       - name: get-return
       # - name: flag-parameter
-      # - name: early-return
+      - name: early-return
       - name: defer
       - name: constant-logical-expr
       # - name: confusing-naming

.tool-versions

@@ -5,4 +5,4 @@ postgres 13.3
 helm 3.10.3
 zig 0.10.1
 golangci-lint 1.55.2
-protoc 23.2
+protoc 25.1

GNUmakefile

@@ -88,7 +88,7 @@ abigen: ## Build & install abigen.
 	./tools/bin/build_abigen
 
 .PHONY: generate
-generate: abigen codecgen mockery ## Execute all go:generate commands.
+generate: abigen codecgen mockery protoc ## Execute all go:generate commands.
 	go generate -x ./...
 
 .PHONY: testscripts
@@ -124,6 +124,11 @@ mockery: $(mockery) ## Install mockery.
 codecgen: $(codecgen) ## Install codecgen
 	go install github.com/ugorji/go/codec/[email protected]
 
+.PHONY: protoc
+protoc: ## Install protoc
+	core/scripts/install-protoc.sh 25.1 /
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@`go list -m -json google.golang.org/protobuf | jq -r .Version`
+
 .PHONY: telemetry-protobuf
 telemetry-protobuf: $(telemetry-protobuf) ## Generate telemetry protocol buffers.
 	protoc \

common/client/multi_node.go

@@ -406,6 +406,10 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 			// main node is used at the end for the return value
 			continue
 		}
+
+		if n.State() != nodeStateAlive {
+			continue
+		}
 		// Parallel call made to all other nodes with ignored return value
 		wg.Add(1)
 		go func(n SendOnlyNode[CHAIN_ID, RPC_CLIENT]) {
@@ -575,11 +579,14 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 }
 
 // collectTxResults - refer to SendTransaction comment for implementation details,
-func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT]) collectTxResults(ctx context.Context, tx TX, txResults <-chan sendTxResult) error {
+func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT]) collectTxResults(ctx context.Context, tx TX, healthyNodesNum int, txResults <-chan sendTxResult) error {
+	if healthyNodesNum == 0 {
+		return ErroringNodeError
+	}
 	// combine context and stop channel to ensure we stop, when signal received
 	ctx, cancel := c.chStop.Ctx(ctx)
 	defer cancel()
-	requiredResults := int(math.Ceil(float64(len(c.nodes)) * sendTxQuorum))
+	requiredResults := int(math.Ceil(float64(healthyNodesNum) * sendTxQuorum))
 	errorsByCode := map[SendTxReturnCode][]error{}
 	var softTimeoutChan <-chan time.Time
 	var resultsCount int
@@ -685,22 +692,31 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 		return ErroringNodeError
 	}
 
+	healthyNodesNum := 0
 	txResults := make(chan sendTxResult, len(c.nodes))
 	// Must wrap inside IfNotStopped to avoid waitgroup racing with Close
 	ok := c.IfNotStopped(func() {
-		c.wg.Add(len(c.sendonlys))
 		// fire-n-forget, as sendOnlyNodes can not be trusted with result reporting
 		for _, n := range c.sendonlys {
+			if n.State() != nodeStateAlive {
+				continue
+			}
+			c.wg.Add(1)
 			go func(n SendOnlyNode[CHAIN_ID, RPC_CLIENT]) {
 				defer c.wg.Done()
 				c.broadcastTxAsync(ctx, n, tx)
 			}(n)
 		}
 
 		var primaryBroadcastWg sync.WaitGroup
-		primaryBroadcastWg.Add(len(c.nodes))
 		txResultsToReport := make(chan sendTxResult, len(c.nodes))
 		for _, n := range c.nodes {
+			if n.State() != nodeStateAlive {
+				continue
+			}
+
+			healthyNodesNum++
+			primaryBroadcastWg.Add(1)
 			go func(n SendOnlyNode[CHAIN_ID, RPC_CLIENT]) {
 				defer primaryBroadcastWg.Done()
 				result := c.broadcastTxAsync(ctx, n, tx)
@@ -727,7 +743,7 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP
 		return fmt.Errorf("aborted while broadcasting tx - multiNode is stopped: %w", context.Canceled)
 	}
 
-	return c.collectTxResults(ctx, tx, txResults)
+	return c.collectTxResults(ctx, tx, healthyNodesNum, txResults)
 }
 
 // findFirstIn - returns first existing value for the slice of keys

common/client/multi_node_test.go

@@ -535,8 +535,10 @@ func TestMultiNode_BatchCallContextAll(t *testing.T) {
 		// setup ok and failed auxiliary nodes
 		okNode := newMockSendOnlyNode[types.ID, multiNodeRPCClient](t)
 		okNode.On("RPC").Return(okRPC).Once()
+		okNode.On("State").Return(nodeStateAlive)
 		failedNode := newMockNode[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
 		failedNode.On("RPC").Return(failedRPC).Once()
+		failedNode.On("State").Return(nodeStateAlive)
 
 		// setup main node
 		mainNode := newMockNode[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
@@ -557,6 +559,34 @@ func TestMultiNode_BatchCallContextAll(t *testing.T) {
 		require.NoError(t, err)
 		tests.RequireLogMessage(t, observedLogs, "Secondary node BatchCallContext failed")
 	})
+	t.Run("Does not call BatchCallContext for unhealthy nodes", func(t *testing.T) {
+		// setup RPCs
+		okRPC := newMultiNodeRPCClient(t)
+		okRPC.On("BatchCallContext", mock.Anything, mock.Anything).Return(nil).Twice()
+
+		// setup ok and failed auxiliary nodes
+		healthyNode := newMockSendOnlyNode[types.ID, multiNodeRPCClient](t)
+		healthyNode.On("RPC").Return(okRPC).Once()
+		healthyNode.On("State").Return(nodeStateAlive)
+		deadNode := newMockNode[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
+		deadNode.On("State").Return(nodeStateUnreachable)
+
+		// setup main node
+		mainNode := newMockNode[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
+		mainNode.On("RPC").Return(okRPC)
+		nodeSelector := newMockNodeSelector[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
+		nodeSelector.On("Select").Return(mainNode).Once()
+		mn := newTestMultiNode(t, multiNodeOpts{
+			selectionMode: NodeSelectionModeRoundRobin,
+			chainID:       types.RandomID(),
+			nodes:         []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{deadNode, mainNode},
+			sendonlys:     []SendOnlyNode[types.ID, multiNodeRPCClient]{healthyNode, deadNode},
+		})
+		mn.nodeSelector = nodeSelector
+
+		err := mn.BatchCallContextAll(tests.Context(t), nil)
+		require.NoError(t, err)
+	})
 }
 
 func TestMultiNode_SendTransaction(t *testing.T) {
@@ -568,15 +598,20 @@ func TestMultiNode_SendTransaction(t *testing.T) {
 
 		return Successful
 	}
-	newNode := func(t *testing.T, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, types.Head[Hashable], multiNodeRPCClient] {
+	newNodeWithState := func(t *testing.T, state nodeState, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, types.Head[Hashable], multiNodeRPCClient] {
 		rpc := newMultiNodeRPCClient(t)
 		rpc.On("SendTransaction", mock.Anything, mock.Anything).Return(txErr).Run(sendTxRun).Maybe()
 		node := newMockNode[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
 		node.On("String").Return("node name").Maybe()
 		node.On("RPC").Return(rpc).Maybe()
+		node.On("State").Return(state).Maybe()
 		node.On("Close").Return(nil).Once()
 		return node
 	}
+
+	newNode := func(t *testing.T, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, types.Head[Hashable], multiNodeRPCClient] {
+		return newNodeWithState(t, nodeStateAlive, txErr, sendTxRun)
+	}
 	newStartedMultiNode := func(t *testing.T, opts multiNodeOpts) testMultiNode {
 		mn := newTestMultiNode(t, opts)
 		err := mn.StartOnce("startedTestMultiNode", func() error { return nil })
@@ -714,6 +749,39 @@ func TestMultiNode_SendTransaction(t *testing.T) {
 		err = mn.SendTransaction(tests.Context(t), nil)
 		require.EqualError(t, err, "aborted while broadcasting tx - multiNode is stopped: context canceled")
 	})
+	t.Run("Returns error if there is no healthy primary nodes", func(t *testing.T) {
+		mn := newStartedMultiNode(t, multiNodeOpts{
+			selectionMode:       NodeSelectionModeRoundRobin,
+			chainID:             types.RandomID(),
+			nodes:               []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{newNodeWithState(t, nodeStateUnreachable, nil, nil)},
+			sendonlys:           []SendOnlyNode[types.ID, multiNodeRPCClient]{newNodeWithState(t, nodeStateUnreachable, nil, nil)},
+			classifySendTxError: classifySendTxError,
+		})
+		err := mn.SendTransaction(tests.Context(t), nil)
+		assert.EqualError(t, err, ErroringNodeError.Error())
+	})
+	t.Run("Transaction success even if one of the nodes is unhealthy", func(t *testing.T) {
+		chainID := types.RandomID()
+		mainNode := newNode(t, nil, nil)
+		unexpectedCall := func(args mock.Arguments) {
+			panic("SendTx must not be called for unhealthy node")
+		}
+		unhealthyNode := newNodeWithState(t, nodeStateUnreachable, nil, unexpectedCall)
+		unhealthySendOnlyNode := newNodeWithState(t, nodeStateUnreachable, nil, unexpectedCall)
+		lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel)
+		mn := newStartedMultiNode(t, multiNodeOpts{
+			selectionMode:       NodeSelectionModeRoundRobin,
+			chainID:             chainID,
+			nodes:               []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{mainNode, unhealthyNode},
+			sendonlys:           []SendOnlyNode[types.ID, multiNodeRPCClient]{unhealthySendOnlyNode, newNode(t, errors.New("unexpected error"), nil)},
+			classifySendTxError: classifySendTxError,
+			logger:              lggr,
+		})
+		err := mn.SendTransaction(tests.Context(t), nil)
+		require.NoError(t, err)
+		tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2)
+		tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 1)
+	})
 }
 
 func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {

contracts/package.json

@@ -18,7 +18,7 @@
     "prepublishOnly": "pnpm compile && ./scripts/prepublish_generate_abi_folder",
     "publish-beta": "pnpm publish --tag beta",
     "publish-prod": "npm dist-tag add @chainlink/[email protected] latest",
-    "solhint": "solhint --max-warnings 33 \"./src/v0.8/**/*.sol\""
+    "solhint": "solhint --max-warnings 20 \"./src/v0.8/**/*.sol\""
   },
   "files": [
     "src/v0.8",
@@ -51,17 +51,17 @@
     "@types/debug": "^4.1.12",
     "@types/deep-equal-in-any-order": "^1.0.3",
     "@types/mocha": "^10.0.6",
-    "@types/node": "^16.18.68",
-    "@typescript-eslint/eslint-plugin": "^6.14.0",
-    "@typescript-eslint/parser": "^6.14.0",
+    "@types/node": "^16.18.80",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
+    "@typescript-eslint/parser": "^6.21.0",
     "abi-to-sol": "^0.6.6",
     "cbor": "^5.2.0",
     "chai": "^4.3.10",
     "debug": "^4.3.4",
-    "eslint": "^8.55.0",
+    "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "deep-equal-in-any-order": "^2.0.6",
-    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-prettier": "^5.1.3",
     "ethereum-waffle": "^3.4.4",
     "ethers": "~5.7.2",
     "hardhat": "~2.19.2",
@@ -71,11 +71,11 @@
     "hardhat-ignore-warnings": "^0.2.6",
     "istanbul": "^0.4.5",
     "moment": "^2.29.4",
-    "prettier": "^3.1.1",
-    "prettier-plugin-solidity": "1.2.0",
+    "prettier": "^3.2.5",
+    "prettier-plugin-solidity": "1.3.1",
     "rlp": "^2.2.7",
-    "solhint": "^4.0.0",
-    "solhint-plugin-chainlink-solidity": "git+https://github.com/smartcontractkit/chainlink-solhint-rules.git#v1.2.0",
+    "solhint": "^4.1.1",
+    "solhint-plugin-chainlink-solidity": "git+https://github.com/smartcontractkit/chainlink-solhint-rules.git#v1.2.1",
     "solhint-plugin-prettier": "^0.1.0",
     "solidity-coverage": "^0.8.5",
     "ts-node": "^10.9.2",