Move deathDeclarationDelay to config

common/client/multi_node.go

@@ -138,6 +138,7 @@ func NewMultiNode[
 	chainFamily string,
 	classifySendTxError func(tx TX, err error) SendTxReturnCode,
 	sendTxSoftTimeout time.Duration,
+	deathDeclarationDelay time.Duration,
 ) MultiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM] {
 	nodeSelector := newNodeSelector(selectionMode, nodes)
 	// Prometheus' default interval is 15s, set this to under 7.5s to avoid
@@ -159,7 +160,7 @@ func NewMultiNode[
 		chainFamily:           chainFamily,
 		classifySendTxError:   classifySendTxError,
 		reportInterval:        reportInterval,
-		deathDeclarationDelay: reportInterval,
+		deathDeclarationDelay: deathDeclarationDelay,
 		sendTxSoftTimeout:     sendTxSoftTimeout,
 	}
 

common/client/multi_node_test.go

@@ -29,16 +29,17 @@ type testMultiNode struct {
 }
 
 type multiNodeOpts struct {
-	logger              logger.Logger
-	selectionMode       string
-	leaseDuration       time.Duration
-	noNewHeadsThreshold time.Duration
-	nodes               []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]
-	sendonlys           []SendOnlyNode[types.ID, multiNodeRPCClient]
-	chainID             types.ID
-	chainFamily         string
-	classifySendTxError func(tx any, err error) SendTxReturnCode
-	sendTxSoftTimeout   time.Duration
+	logger                logger.Logger
+	selectionMode         string
+	leaseDuration         time.Duration
+	noNewHeadsThreshold   time.Duration
+	nodes                 []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]
+	sendonlys             []SendOnlyNode[types.ID, multiNodeRPCClient]
+	chainID               types.ID
+	chainFamily           string
+	classifySendTxError   func(tx any, err error) SendTxReturnCode
+	sendTxSoftTimeout     time.Duration
+	deathDeclarationDelay time.Duration
 }
 
 func newTestMultiNode(t *testing.T, opts multiNodeOpts) testMultiNode {
@@ -49,7 +50,7 @@ func newTestMultiNode(t *testing.T, opts multiNodeOpts) testMultiNode {
 	result := NewMultiNode[types.ID, *big.Int, Hashable, Hashable, any, Hashable, any, any,
 		types.Receipt[Hashable, Hashable], Hashable, types.Head[Hashable], multiNodeRPCClient, any](opts.logger,
 		opts.selectionMode, opts.leaseDuration, opts.noNewHeadsThreshold, opts.nodes, opts.sendonlys,
-		opts.chainID, opts.chainFamily, opts.classifySendTxError, opts.sendTxSoftTimeout)
+		opts.chainID, opts.chainFamily, opts.classifySendTxError, opts.sendTxSoftTimeout, opts.deathDeclarationDelay)
 	return testMultiNode{
 		result.(*multiNode[types.ID, *big.Int, Hashable, Hashable, any, Hashable, any, any,
 			types.Receipt[Hashable, Hashable], Hashable, types.Head[Hashable], multiNodeRPCClient, any]),

common/client/node.go

@@ -44,6 +44,7 @@ type NodeConfig interface {
 	NodeIsSyncingEnabled() bool
 	FinalizedBlockPollInterval() time.Duration
 	EnforceRepeatableRead() bool
+	DeathDeclarationDelay() time.Duration
 }
 
 type ChainConfig interface {

common/client/node_test.go

@@ -19,6 +19,7 @@ type testNodeConfig struct {
 	nodeIsSyncingEnabled       bool
 	enforceRepeatableRead      bool
 	finalizedBlockPollInterval time.Duration
+	deathDeclarationDelay      time.Duration
 }
 
 func (n testNodeConfig) PollFailureThreshold() uint32 {
@@ -49,6 +50,10 @@ func (n testNodeConfig) EnforceRepeatableRead() bool {
 	return n.enforceRepeatableRead
 }
 
+func (n testNodeConfig) DeathDeclarationDelay() time.Duration {
+	return n.deathDeclarationDelay
+}
+
 type testNode struct {
 	*node[types.ID, Head, NodeClient[types.ID, Head]]
 }

core/chains/evm/client/chain_client.go

@@ -132,6 +132,7 @@ func NewChainClient(
 	chainID *big.Int,
 	chainType chaintype.ChainType,
 	clientErrors evmconfig.ClientErrors,
+	deathDeclarationDelay time.Duration,
 ) Client {
 	multiNode := commonclient.NewMultiNode(
 		lggr,
@@ -146,6 +147,7 @@ func NewChainClient(
 			return ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, chainType.IsL2())
 		},
 		0, // use the default value provided by the implementation
+		deathDeclarationDelay,
 	)
 	return &chainClient{
 		multiNode:    multiNode,

core/chains/evm/client/config_builder.go

@@ -40,6 +40,7 @@ func NewClientConfigs(
 	finalityTagEnabled *bool,
 	finalityBlockOffset *uint32,
 	enforceRepeatableRead *bool,
+	deathDeclarationDelay time.Duration,
 
 ) (commonclient.ChainConfig, evmconfig.NodePool, []*toml.Node, error) {
 	nodes, err := parseNodeConfigs(nodeCfgs)
@@ -54,6 +55,7 @@ func NewClientConfigs(
 		SyncThreshold:         syncThreshold,
 		NodeIsSyncingEnabled:  nodeIsSyncingEnabled,
 		EnforceRepeatableRead: enforceRepeatableRead,
+		DeathDeclarationDelay: commonconfig.MustNewDuration(deathDeclarationDelay),
 	}
 	nodePoolCfg := &evmconfig.NodePoolConfig{C: nodePool}
 	chainConfig := &evmconfig.EVMConfig{

core/chains/evm/client/config_builder_test.go

@@ -25,6 +25,7 @@ func TestClientConfigBuilder(t *testing.T) {
 	chainTypeStr := ""
 	finalizedBlockOffset := ptr[uint32](16)
 	enforceRepeatableRead := ptr(true)
+	deathDeclarationDelay := time.Second * 3
 	nodeConfigs := []client.NodeConfig{
 		{
 			Name:    ptr("foo"),
@@ -37,7 +38,7 @@ func TestClientConfigBuilder(t *testing.T) {
 	noNewHeadsThreshold := time.Second
 	chainCfg, nodePool, nodes, err := client.NewClientConfigs(selectionMode, leaseDuration, chainTypeStr, nodeConfigs,
 		pollFailureThreshold, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth,
-		finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead)
+		finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead, deathDeclarationDelay)
 	require.NoError(t, err)
 
 	// Validate node pool configs
@@ -48,6 +49,7 @@ func TestClientConfigBuilder(t *testing.T) {
 	require.Equal(t, *syncThreshold, nodePool.SyncThreshold())
 	require.Equal(t, *nodeIsSyncingEnabled, nodePool.NodeIsSyncingEnabled())
 	require.Equal(t, *enforceRepeatableRead, nodePool.EnforceRepeatableRead())
+	require.Equal(t, deathDeclarationDelay, nodePool.DeathDeclarationDelay())
 
 	// Validate node configs
 	require.Equal(t, *nodeConfigs[0].Name, *nodes[0].Name)

core/chains/evm/client/evm_client.go

@@ -35,5 +35,5 @@ func NewEvmClient(cfg evmconfig.NodePool, chainCfg commonclient.ChainConfig, cli
 	}
 
 	return NewChainClient(lggr, cfg.SelectionMode(), cfg.LeaseDuration(), chainCfg.NodeNoNewHeadsThreshold(),
-		primaries, sendonlys, chainID, chainType, clientErrors)
+		primaries, sendonlys, chainID, chainType, clientErrors, cfg.DeathDeclarationDelay())
 }

core/chains/evm/client/evm_client_test.go

@@ -26,6 +26,7 @@ func TestNewEvmClient(t *testing.T) {
 	chainTypeStr := ""
 	finalizedBlockOffset := ptr[uint32](16)
 	enforceRepeatableRead := ptr(true)
+	deathDeclarationDelay := time.Second * 3
 	nodeConfigs := []client.NodeConfig{
 		{
 			Name:    ptr("foo"),
@@ -37,7 +38,7 @@ func TestNewEvmClient(t *testing.T) {
 	finalityTagEnabled := ptr(true)
 	chainCfg, nodePool, nodes, err := client.NewClientConfigs(selectionMode, leaseDuration, chainTypeStr, nodeConfigs,
 		pollFailureThreshold, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth,
-		finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead)
+		finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead, deathDeclarationDelay)
 	require.NoError(t, err)
 
 	client := client.NewEvmClient(nodePool, chainCfg, nil, logger.Test(t), testutils.FixtureChainID, nodes, chaintype.ChainType(chainTypeStr))

core/chains/evm/client/helpers_test.go

@@ -88,6 +88,7 @@ type TestNodePoolConfig struct {
 	NodeFinalizedBlockPollInterval time.Duration
 	NodeErrors                     config.ClientErrors
 	EnforceRepeatableReadVal       bool
+	NodeDeathDeclarationDelay      time.Duration
 }
 
 func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold }
@@ -114,6 +115,10 @@ func (tc TestNodePoolConfig) EnforceRepeatableRead() bool {
 	return tc.EnforceRepeatableReadVal
 }
 
+func (tc TestNodePoolConfig) DeathDeclarationDelay() time.Duration {
+	return tc.NodeDeathDeclarationDelay
+}
+
 func NewChainClientWithTestNode(
 	t *testing.T,
 	nodeCfg commonclient.NodeConfig,
@@ -155,7 +160,7 @@ func NewChainClientWithTestNode(
 
 	var chainType chaintype.ChainType
 	clientErrors := NewTestClientErrors()
-	c := NewChainClient(lggr, nodeCfg.SelectionMode(), leaseDuration, noNewHeadsThreshold, primaries, sendonlys, chainID, chainType, &clientErrors)
+	c := NewChainClient(lggr, nodeCfg.SelectionMode(), leaseDuration, noNewHeadsThreshold, primaries, sendonlys, chainID, chainType, &clientErrors, 0)
 	t.Cleanup(c.Close)
 	return c, nil
 }
@@ -170,7 +175,7 @@ func NewChainClientWithEmptyNode(
 	lggr := logger.Test(t)
 
 	var chainType chaintype.ChainType
-	c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, nil, nil, chainID, chainType, nil)
+	c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, nil, nil, chainID, chainType, nil, 0)
 	t.Cleanup(c.Close)
 	return c
 }
@@ -196,7 +201,7 @@ func NewChainClientWithMockedRpc(
 		cfg, clientMocks.ChainConfig{NoNewHeadsThresholdVal: noNewHeadsThreshold}, lggr, *parsed, nil, "eth-primary-node-0", 1, chainID, 1, rpc, "EVM")
 	primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient]{n}
 	clientErrors := NewTestClientErrors()
-	c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, primaries, nil, chainID, chainType, &clientErrors)
+	c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, primaries, nil, chainID, chainType, &clientErrors, 0)
 	t.Cleanup(c.Close)
 	return c
 }

core/chains/evm/config/chain_scoped_node_pool.go

@@ -43,3 +43,7 @@ func (n *NodePoolConfig) Errors() ClientErrors { return &clientErrorsConfig{c: n
 func (n *NodePoolConfig) EnforceRepeatableRead() bool {
 	return *n.C.EnforceRepeatableRead
 }
+
+func (n *NodePoolConfig) DeathDeclarationDelay() time.Duration {
+	return n.C.DeathDeclarationDelay.Duration()
+}

core/chains/evm/config/config.go

@@ -172,6 +172,7 @@ type NodePool interface {
 	FinalizedBlockPollInterval() time.Duration
 	Errors() ClientErrors
 	EnforceRepeatableRead() bool
+	DeathDeclarationDelay() time.Duration
 }
 
 // TODO BCF-2509 does the chainscopedconfig really need the entire app config?

core/chains/evm/config/config_test.go

@@ -327,6 +327,7 @@ func TestNodePoolConfig(t *testing.T) {
 	require.Equal(t, uint32(5), cfg.EVM().NodePool().PollFailureThreshold())
 	require.Equal(t, false, cfg.EVM().NodePool().NodeIsSyncingEnabled())
 	require.Equal(t, false, cfg.EVM().NodePool().EnforceRepeatableRead())
+	require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().DeathDeclarationDelay())
 }
 
 func TestClientErrorsConfig(t *testing.T) {

core/chains/evm/config/toml/config.go

@@ -848,6 +848,7 @@ type NodePool struct {
 	FinalizedBlockPollInterval *commonconfig.Duration
 	Errors                     ClientErrors `toml:",omitempty"`
 	EnforceRepeatableRead      *bool
+	DeathDeclarationDelay      *commonconfig.Duration
 }
 
 func (p *NodePool) setFrom(f *NodePool) {
@@ -876,6 +877,10 @@ func (p *NodePool) setFrom(f *NodePool) {
 	if v := f.EnforceRepeatableRead; v != nil {
 		p.EnforceRepeatableRead = v
 	}
+
+	if v := f.DeathDeclarationDelay; v != nil {
+		p.DeathDeclarationDelay = v
+	}
 	p.Errors.setFrom(&f.Errors)
 }
 

core/chains/evm/config/toml/defaults/fallback.toml

@@ -70,6 +70,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [OCR]
 ContractConfirmations = 4

core/config/docs/chains-evm.toml

@@ -88,6 +88,9 @@ RPCBlockQueryDelay = 1 # Default
 # FinalizedBlockOffset defines the number of blocks by which the latest finalized block will be shifted. E.g if RPC returns block 100 as latest finalized, node will treat block `100 - FinalizedBlockOffset` as the latest finalized block.
 # In case of `FinalityTagEnabled = false` Node will treat block `latest - FinalityDepth - FinalizedBlockOffset` as the latest finalized block.
 # With `EnforceRepeatableRead = true`, RPC is considered healthy only if its most recent finalized block is larger or equal to the highest finalized block observed by the Node minus `FinalizedBlockOffset`.
+# CAUTION: Setting this to values higher than 0 increases the number of RPCs considered healthy but can delay transaction production.
+# PoS chains with FinalityTags enabled (e.g., Ethereum Mainnet) must be handled with special care, as blocks are finalized in batches (epochs).
+# Thus, the `FinalizedBlockOffset=1` configuration will result in larger delays (e.g., 32 blocks for Ethereum Mainnet).
 FinalizedBlockOffset = 0 # Default
 
 [EVM.Transactions]
@@ -378,6 +381,11 @@ FinalizedBlockPollInterval = '5s' # Default
 #
 # Set false to disable
 EnforceRepeatableRead = false # Default
+# DeathDeclarationDelay defines the minimum duration an RPC must be in unhealthy state before producing an error log message.
+# Larger values might be helpful to reduce the noisiness of health checks like `EnforceRepeatableRead = true', which might be falsely
+# trigger declaration of `FinalizedBlockOutOfSync` due to insignificant network delays in broadcasting of the finalized state among RPCs.
+# RPC will not be picked to handle a request even if this option is set to a nonzero value.
+DeathDeclarationDelay = '10s' # Default
 
 # **ADVANCED**
 # Errors enable the node to provide custom regex patterns to match against error messages from RPCs.

core/services/chainlink/config_test.go

@@ -591,6 +591,7 @@ func TestConfig_Marshal(t *testing.T) {
 					NodeIsSyncingEnabled:       ptr(true),
 					FinalizedBlockPollInterval: &second,
 					EnforceRepeatableRead:      ptr(true),
+					DeathDeclarationDelay:      &minute,
 					Errors: evmcfg.ClientErrors{
 						NonceTooLow:                       ptr[string]("(: |^)nonce too low"),
 						NonceTooHigh:                      ptr[string]("(: |^)nonce too high"),
@@ -1060,6 +1061,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = true
 FinalizedBlockPollInterval = '1s'
 EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 
 [EVM.NodePool.Errors]
 NonceTooLow = '(: |^)nonce too low'

core/services/chainlink/testdata/config-full.toml

@@ -354,6 +354,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = true
 FinalizedBlockPollInterval = '1s'
 EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 
 [EVM.NodePool.Errors]
 NonceTooLow = '(: |^)nonce too low'

core/services/chainlink/testdata/config-multi-chain-effective.toml

@@ -325,6 +325,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.OCR]
 ContractConfirmations = 4
@@ -423,6 +424,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.OCR]
 ContractConfirmations = 4
@@ -515,6 +517,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.OCR]
 ContractConfirmations = 4

core/web/resolver/testdata/config-full.toml

@@ -353,6 +353,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.NodePool.Errors]
 NonceTooLow = '(: |^)nonce too low'

core/web/resolver/testdata/config-multi-chain-effective.toml

@@ -325,6 +325,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.OCR]
 ContractConfirmations = 4
@@ -423,6 +424,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.OCR]
 ContractConfirmations = 4
@@ -515,6 +517,7 @@ LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
 EnforceRepeatableRead = false
+DeathDeclarationDelay = '10s'
 
 [EVM.OCR]
 ContractConfirmations = 4