From 399af4109b13bef778a4271ee485d8ad4e70c990 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Mon, 10 Jun 2024 17:37:54 +0200 Subject: [PATCH] Move deathDeclarationDelay to config --- common/client/multi_node.go | 3 +- common/client/multi_node_test.go | 23 +++--- common/client/node.go | 1 + common/client/node_test.go | 5 ++ core/chains/evm/client/chain_client.go | 2 + core/chains/evm/client/config_builder.go | 2 + core/chains/evm/client/config_builder_test.go | 4 +- core/chains/evm/client/evm_client.go | 2 +- core/chains/evm/client/evm_client_test.go | 3 +- core/chains/evm/client/helpers_test.go | 11 ++- .../evm/config/chain_scoped_node_pool.go | 4 ++ core/chains/evm/config/config.go | 1 + core/chains/evm/config/config_test.go | 1 + core/chains/evm/config/toml/config.go | 5 ++ .../evm/config/toml/defaults/fallback.toml | 1 + core/config/docs/chains-evm.toml | 8 +++ core/services/chainlink/config_test.go | 2 + .../chainlink/testdata/config-full.toml | 1 + .../config-multi-chain-effective.toml | 3 + core/web/resolver/testdata/config-full.toml | 1 + .../config-multi-chain-effective.toml | 3 + docs/CONFIG.md | 72 +++++++++++++++++++ .../disk-based-logging-disabled.txtar | 1 + .../validate/disk-based-logging-no-dir.txtar | 1 + .../node/validate/disk-based-logging.txtar | 1 + testdata/scripts/node/validate/invalid.txtar | 1 + testdata/scripts/node/validate/valid.txtar | 1 + 27 files changed, 145 insertions(+), 18 deletions(-) diff --git a/common/client/multi_node.go b/common/client/multi_node.go index 4502bb40edd..82837f983bf 100644 --- a/common/client/multi_node.go +++ b/common/client/multi_node.go @@ -138,6 +138,7 @@ func NewMultiNode[ chainFamily string, classifySendTxError func(tx TX, err error) SendTxReturnCode, sendTxSoftTimeout time.Duration, + deathDeclarationDelay time.Duration, ) MultiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM] { nodeSelector := newNodeSelector(selectionMode, nodes) // Prometheus' default interval is 15s, set this to under 7.5s to avoid @@ -159,7 +160,7 @@ func NewMultiNode[ chainFamily: chainFamily, classifySendTxError: classifySendTxError, reportInterval: reportInterval, - deathDeclarationDelay: reportInterval, + deathDeclarationDelay: deathDeclarationDelay, sendTxSoftTimeout: sendTxSoftTimeout, } diff --git a/common/client/multi_node_test.go b/common/client/multi_node_test.go index 6d41325425a..dc280e0e178 100644 --- a/common/client/multi_node_test.go +++ b/common/client/multi_node_test.go @@ -29,16 +29,17 @@ type testMultiNode struct { } type multiNodeOpts struct { - logger logger.Logger - selectionMode string - leaseDuration time.Duration - noNewHeadsThreshold time.Duration - nodes []Node[types.ID, types.Head[Hashable], multiNodeRPCClient] - sendonlys []SendOnlyNode[types.ID, multiNodeRPCClient] - chainID types.ID - chainFamily string - classifySendTxError func(tx any, err error) SendTxReturnCode - sendTxSoftTimeout time.Duration + logger logger.Logger + selectionMode string + leaseDuration time.Duration + noNewHeadsThreshold time.Duration + nodes []Node[types.ID, types.Head[Hashable], multiNodeRPCClient] + sendonlys []SendOnlyNode[types.ID, multiNodeRPCClient] + chainID types.ID + chainFamily string + classifySendTxError func(tx any, err error) SendTxReturnCode + sendTxSoftTimeout time.Duration + deathDeclarationDelay time.Duration } func newTestMultiNode(t *testing.T, opts multiNodeOpts) testMultiNode { @@ -49,7 +50,7 @@ func newTestMultiNode(t *testing.T, opts multiNodeOpts) testMultiNode { result := NewMultiNode[types.ID, *big.Int, Hashable, Hashable, any, Hashable, any, any, types.Receipt[Hashable, Hashable], Hashable, types.Head[Hashable], multiNodeRPCClient, any](opts.logger, opts.selectionMode, opts.leaseDuration, opts.noNewHeadsThreshold, opts.nodes, opts.sendonlys, - opts.chainID, opts.chainFamily, opts.classifySendTxError, opts.sendTxSoftTimeout) + opts.chainID, opts.chainFamily, opts.classifySendTxError, opts.sendTxSoftTimeout, opts.deathDeclarationDelay) return testMultiNode{ result.(*multiNode[types.ID, *big.Int, Hashable, Hashable, any, Hashable, any, any, types.Receipt[Hashable, Hashable], Hashable, types.Head[Hashable], multiNodeRPCClient, any]), diff --git a/common/client/node.go b/common/client/node.go index e8eaa94c5cb..cc97d7980f7 100644 --- a/common/client/node.go +++ b/common/client/node.go @@ -44,6 +44,7 @@ type NodeConfig interface { NodeIsSyncingEnabled() bool FinalizedBlockPollInterval() time.Duration EnforceRepeatableRead() bool + DeathDeclarationDelay() time.Duration } type ChainConfig interface { diff --git a/common/client/node_test.go b/common/client/node_test.go index 4a8531197ec..3b971e84902 100644 --- a/common/client/node_test.go +++ b/common/client/node_test.go @@ -19,6 +19,7 @@ type testNodeConfig struct { nodeIsSyncingEnabled bool enforceRepeatableRead bool finalizedBlockPollInterval time.Duration + deathDeclarationDelay time.Duration } func (n testNodeConfig) PollFailureThreshold() uint32 { @@ -49,6 +50,10 @@ func (n testNodeConfig) EnforceRepeatableRead() bool { return n.enforceRepeatableRead } +func (n testNodeConfig) DeathDeclarationDelay() time.Duration { + return n.deathDeclarationDelay +} + type testNode struct { *node[types.ID, Head, NodeClient[types.ID, Head]] } diff --git a/core/chains/evm/client/chain_client.go b/core/chains/evm/client/chain_client.go index b16b7df1002..0fb465f816a 100644 --- a/core/chains/evm/client/chain_client.go +++ b/core/chains/evm/client/chain_client.go @@ -132,6 +132,7 @@ func NewChainClient( chainID *big.Int, chainType chaintype.ChainType, clientErrors evmconfig.ClientErrors, + deathDeclarationDelay time.Duration, ) Client { multiNode := commonclient.NewMultiNode( lggr, @@ -146,6 +147,7 @@ func NewChainClient( return ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, chainType.IsL2()) }, 0, // use the default value provided by the implementation + deathDeclarationDelay, ) return &chainClient{ multiNode: multiNode, diff --git a/core/chains/evm/client/config_builder.go b/core/chains/evm/client/config_builder.go index a88dadde090..115d660a90f 100644 --- a/core/chains/evm/client/config_builder.go +++ b/core/chains/evm/client/config_builder.go @@ -40,6 +40,7 @@ func NewClientConfigs( finalityTagEnabled *bool, finalityBlockOffset *uint32, enforceRepeatableRead *bool, + deathDeclarationDelay time.Duration, ) (commonclient.ChainConfig, evmconfig.NodePool, []*toml.Node, error) { nodes, err := parseNodeConfigs(nodeCfgs) @@ -54,6 +55,7 @@ func NewClientConfigs( SyncThreshold: syncThreshold, NodeIsSyncingEnabled: nodeIsSyncingEnabled, EnforceRepeatableRead: enforceRepeatableRead, + DeathDeclarationDelay: commonconfig.MustNewDuration(deathDeclarationDelay), } nodePoolCfg := &evmconfig.NodePoolConfig{C: nodePool} chainConfig := &evmconfig.EVMConfig{ diff --git a/core/chains/evm/client/config_builder_test.go b/core/chains/evm/client/config_builder_test.go index c488511c839..7c08bf18c1d 100644 --- a/core/chains/evm/client/config_builder_test.go +++ b/core/chains/evm/client/config_builder_test.go @@ -25,6 +25,7 @@ func TestClientConfigBuilder(t *testing.T) { chainTypeStr := "" finalizedBlockOffset := ptr[uint32](16) enforceRepeatableRead := ptr(true) + deathDeclarationDelay := time.Second * 3 nodeConfigs := []client.NodeConfig{ { Name: ptr("foo"), @@ -37,7 +38,7 @@ func TestClientConfigBuilder(t *testing.T) { noNewHeadsThreshold := time.Second chainCfg, nodePool, nodes, err := client.NewClientConfigs(selectionMode, leaseDuration, chainTypeStr, nodeConfigs, pollFailureThreshold, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth, - finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead) + finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead, deathDeclarationDelay) require.NoError(t, err) // Validate node pool configs @@ -48,6 +49,7 @@ func TestClientConfigBuilder(t *testing.T) { require.Equal(t, *syncThreshold, nodePool.SyncThreshold()) require.Equal(t, *nodeIsSyncingEnabled, nodePool.NodeIsSyncingEnabled()) require.Equal(t, *enforceRepeatableRead, nodePool.EnforceRepeatableRead()) + require.Equal(t, deathDeclarationDelay, nodePool.DeathDeclarationDelay()) // Validate node configs require.Equal(t, *nodeConfigs[0].Name, *nodes[0].Name) diff --git a/core/chains/evm/client/evm_client.go b/core/chains/evm/client/evm_client.go index 4d309440590..fd7fa5868a4 100644 --- a/core/chains/evm/client/evm_client.go +++ b/core/chains/evm/client/evm_client.go @@ -35,5 +35,5 @@ func NewEvmClient(cfg evmconfig.NodePool, chainCfg commonclient.ChainConfig, cli } return NewChainClient(lggr, cfg.SelectionMode(), cfg.LeaseDuration(), chainCfg.NodeNoNewHeadsThreshold(), - primaries, sendonlys, chainID, chainType, clientErrors) + primaries, sendonlys, chainID, chainType, clientErrors, cfg.DeathDeclarationDelay()) } diff --git a/core/chains/evm/client/evm_client_test.go b/core/chains/evm/client/evm_client_test.go index 9c8f9d66619..9ad25f96025 100644 --- a/core/chains/evm/client/evm_client_test.go +++ b/core/chains/evm/client/evm_client_test.go @@ -26,6 +26,7 @@ func TestNewEvmClient(t *testing.T) { chainTypeStr := "" finalizedBlockOffset := ptr[uint32](16) enforceRepeatableRead := ptr(true) + deathDeclarationDelay := time.Second * 3 nodeConfigs := []client.NodeConfig{ { Name: ptr("foo"), @@ -37,7 +38,7 @@ func TestNewEvmClient(t *testing.T) { finalityTagEnabled := ptr(true) chainCfg, nodePool, nodes, err := client.NewClientConfigs(selectionMode, leaseDuration, chainTypeStr, nodeConfigs, pollFailureThreshold, pollInterval, syncThreshold, nodeIsSyncingEnabled, noNewHeadsThreshold, finalityDepth, - finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead) + finalityTagEnabled, finalizedBlockOffset, enforceRepeatableRead, deathDeclarationDelay) require.NoError(t, err) client := client.NewEvmClient(nodePool, chainCfg, nil, logger.Test(t), testutils.FixtureChainID, nodes, chaintype.ChainType(chainTypeStr)) diff --git a/core/chains/evm/client/helpers_test.go b/core/chains/evm/client/helpers_test.go index 39c66271a31..e1017a5564f 100644 --- a/core/chains/evm/client/helpers_test.go +++ b/core/chains/evm/client/helpers_test.go @@ -88,6 +88,7 @@ type TestNodePoolConfig struct { NodeFinalizedBlockPollInterval time.Duration NodeErrors config.ClientErrors EnforceRepeatableReadVal bool + NodeDeathDeclarationDelay time.Duration } func (tc TestNodePoolConfig) PollFailureThreshold() uint32 { return tc.NodePollFailureThreshold } @@ -114,6 +115,10 @@ func (tc TestNodePoolConfig) EnforceRepeatableRead() bool { return tc.EnforceRepeatableReadVal } +func (tc TestNodePoolConfig) DeathDeclarationDelay() time.Duration { + return tc.NodeDeathDeclarationDelay +} + func NewChainClientWithTestNode( t *testing.T, nodeCfg commonclient.NodeConfig, @@ -155,7 +160,7 @@ func NewChainClientWithTestNode( var chainType chaintype.ChainType clientErrors := NewTestClientErrors() - c := NewChainClient(lggr, nodeCfg.SelectionMode(), leaseDuration, noNewHeadsThreshold, primaries, sendonlys, chainID, chainType, &clientErrors) + c := NewChainClient(lggr, nodeCfg.SelectionMode(), leaseDuration, noNewHeadsThreshold, primaries, sendonlys, chainID, chainType, &clientErrors, 0) t.Cleanup(c.Close) return c, nil } @@ -170,7 +175,7 @@ func NewChainClientWithEmptyNode( lggr := logger.Test(t) var chainType chaintype.ChainType - c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, nil, nil, chainID, chainType, nil) + c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, nil, nil, chainID, chainType, nil, 0) t.Cleanup(c.Close) return c } @@ -196,7 +201,7 @@ func NewChainClientWithMockedRpc( cfg, clientMocks.ChainConfig{NoNewHeadsThresholdVal: noNewHeadsThreshold}, lggr, *parsed, nil, "eth-primary-node-0", 1, chainID, 1, rpc, "EVM") primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient]{n} clientErrors := NewTestClientErrors() - c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, primaries, nil, chainID, chainType, &clientErrors) + c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, primaries, nil, chainID, chainType, &clientErrors, 0) t.Cleanup(c.Close) return c } diff --git a/core/chains/evm/config/chain_scoped_node_pool.go b/core/chains/evm/config/chain_scoped_node_pool.go index 3a6d5602589..a4974366486 100644 --- a/core/chains/evm/config/chain_scoped_node_pool.go +++ b/core/chains/evm/config/chain_scoped_node_pool.go @@ -43,3 +43,7 @@ func (n *NodePoolConfig) Errors() ClientErrors { return &clientErrorsConfig{c: n func (n *NodePoolConfig) EnforceRepeatableRead() bool { return *n.C.EnforceRepeatableRead } + +func (n *NodePoolConfig) DeathDeclarationDelay() time.Duration { + return n.C.DeathDeclarationDelay.Duration() +} diff --git a/core/chains/evm/config/config.go b/core/chains/evm/config/config.go index f0287e65431..ffb2a496baf 100644 --- a/core/chains/evm/config/config.go +++ b/core/chains/evm/config/config.go @@ -172,6 +172,7 @@ type NodePool interface { FinalizedBlockPollInterval() time.Duration Errors() ClientErrors EnforceRepeatableRead() bool + DeathDeclarationDelay() time.Duration } // TODO BCF-2509 does the chainscopedconfig really need the entire app config? diff --git a/core/chains/evm/config/config_test.go b/core/chains/evm/config/config_test.go index 8f1921f551f..ba362bda981 100644 --- a/core/chains/evm/config/config_test.go +++ b/core/chains/evm/config/config_test.go @@ -327,6 +327,7 @@ func TestNodePoolConfig(t *testing.T) { require.Equal(t, uint32(5), cfg.EVM().NodePool().PollFailureThreshold()) require.Equal(t, false, cfg.EVM().NodePool().NodeIsSyncingEnabled()) require.Equal(t, false, cfg.EVM().NodePool().EnforceRepeatableRead()) + require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().DeathDeclarationDelay()) } func TestClientErrorsConfig(t *testing.T) { diff --git a/core/chains/evm/config/toml/config.go b/core/chains/evm/config/toml/config.go index b16b3e469ac..394cc403472 100644 --- a/core/chains/evm/config/toml/config.go +++ b/core/chains/evm/config/toml/config.go @@ -848,6 +848,7 @@ type NodePool struct { FinalizedBlockPollInterval *commonconfig.Duration Errors ClientErrors `toml:",omitempty"` EnforceRepeatableRead *bool + DeathDeclarationDelay *commonconfig.Duration } func (p *NodePool) setFrom(f *NodePool) { @@ -876,6 +877,10 @@ func (p *NodePool) setFrom(f *NodePool) { if v := f.EnforceRepeatableRead; v != nil { p.EnforceRepeatableRead = v } + + if v := f.DeathDeclarationDelay; v != nil { + p.DeathDeclarationDelay = v + } p.Errors.setFrom(&f.Errors) } diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index b207dd6c0dd..a11e646e08b 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -70,6 +70,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 0170d37f51c..19c74913bf3 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -88,6 +88,9 @@ RPCBlockQueryDelay = 1 # Default # FinalizedBlockOffset defines the number of blocks by which the latest finalized block will be shifted. E.g if RPC returns block 100 as latest finalized, node will treat block `100 - FinalizedBlockOffset` as the latest finalized block. # In case of `FinalityTagEnabled = false` Node will treat block `latest - FinalityDepth - FinalizedBlockOffset` as the latest finalized block. # With `EnforceRepeatableRead = true`, RPC is considered healthy only if its most recent finalized block is larger or equal to the highest finalized block observed by the Node minus `FinalizedBlockOffset`. +# CAUTION: Setting this to values higher than 0 increases the number of RPCs considered healthy but can delay transaction production. +# PoS chains with FinalityTags enabled (e.g., Ethereum Mainnet) must be handled with special care, as blocks are finalized in batches (epochs). +# Thus, the `FinalizedBlockOffset=1` configuration will result in larger delays (e.g., 32 blocks for Ethereum Mainnet). FinalizedBlockOffset = 0 # Default [EVM.Transactions] @@ -378,6 +381,11 @@ FinalizedBlockPollInterval = '5s' # Default # # Set false to disable EnforceRepeatableRead = false # Default +# DeathDeclarationDelay defines the minimum duration an RPC must be in unhealthy state before producing an error log message. +# Larger values might be helpful to reduce the noisiness of health checks like `EnforceRepeatableRead = true', which might be falsely +# trigger declaration of `FinalizedBlockOutOfSync` due to insignificant network delays in broadcasting of the finalized state among RPCs. +# RPC will not be picked to handle a request even if this option is set to a nonzero value. +DeathDeclarationDelay = '10s' # Default # **ADVANCED** # Errors enable the node to provide custom regex patterns to match against error messages from RPCs. diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index f413c4959e9..dc9bd9d5cc0 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -591,6 +591,7 @@ func TestConfig_Marshal(t *testing.T) { NodeIsSyncingEnabled: ptr(true), FinalizedBlockPollInterval: &second, EnforceRepeatableRead: ptr(true), + DeathDeclarationDelay: &minute, Errors: evmcfg.ClientErrors{ NonceTooLow: ptr[string]("(: |^)nonce too low"), NonceTooHigh: ptr[string]("(: |^)nonce too high"), @@ -1060,6 +1061,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = true FinalizedBlockPollInterval = '1s' EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' [EVM.NodePool.Errors] NonceTooLow = '(: |^)nonce too low' diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 07f80d94fcd..c76ae58f425 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -354,6 +354,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = true FinalizedBlockPollInterval = '1s' EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' [EVM.NodePool.Errors] NonceTooLow = '(: |^)nonce too low' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 5a1657dc1c4..6bffb85de86 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -325,6 +325,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 @@ -423,6 +424,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 @@ -515,6 +517,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index fbecbc2d8a7..a3c3dd4aa1d 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -353,6 +353,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.NodePool.Errors] NonceTooLow = '(: |^)nonce too low' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 815d78e7682..22469c5a166 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -325,6 +325,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 @@ -423,6 +424,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 @@ -515,6 +517,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 4a69f114547..ef2c3a5a1b6 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1805,6 +1805,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -1897,6 +1898,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -1989,6 +1991,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2081,6 +2084,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2174,6 +2178,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -2266,6 +2271,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2358,6 +2364,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2451,6 +2458,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2543,6 +2551,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2634,6 +2643,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2725,6 +2735,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2817,6 +2828,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -2910,6 +2922,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3002,6 +3015,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3094,6 +3108,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3186,6 +3201,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -3278,6 +3294,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -3370,6 +3387,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3462,6 +3480,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -3554,6 +3573,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3646,6 +3666,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3738,6 +3759,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -3831,6 +3853,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -3923,6 +3946,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4014,6 +4038,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4106,6 +4131,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4198,6 +4224,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4290,6 +4317,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4382,6 +4410,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4473,6 +4502,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4565,6 +4595,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4657,6 +4688,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4749,6 +4781,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -4841,6 +4874,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -4932,6 +4966,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5024,6 +5059,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5116,6 +5152,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -5209,6 +5246,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5301,6 +5339,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5393,6 +5432,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5485,6 +5525,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5577,6 +5618,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -5668,6 +5710,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -5759,6 +5802,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -5850,6 +5894,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -5942,6 +5987,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6034,6 +6080,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -6125,6 +6172,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -6217,6 +6265,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6309,6 +6358,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6402,6 +6452,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6495,6 +6546,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6587,6 +6639,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6679,6 +6732,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6771,6 +6825,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -6863,6 +6918,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -6955,6 +7011,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 1 @@ -7047,6 +7104,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -7139,6 +7197,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [OCR] ContractConfirmations = 4 @@ -7334,6 +7393,9 @@ FinalizedBlockOffset = 0 # Default FinalizedBlockOffset defines the number of blocks by which the latest finalized block will be shifted. E.g if RPC returns block 100 as latest finalized, node will treat block `100 - FinalizedBlockOffset` as the latest finalized block. In case of `FinalityTagEnabled = false` Node will treat block `latest - FinalityDepth - FinalizedBlockOffset` as the latest finalized block. With `EnforceRepeatableRead = true`, RPC is considered healthy only if its most recent finalized block is larger or equal to the highest finalized block observed by the Node minus `FinalizedBlockOffset`. +CAUTION: Setting this to values higher than 0 increases the number of RPCs considered healthy but can delay transaction production. +PoS chains with FinalityTags enabled (e.g., Ethereum Mainnet) must be handled with special care, as blocks are finalized in batches (epochs). +Thus, the `FinalizedBlockOffset=1` configuration will result in larger delays (e.g., 32 blocks for Ethereum Mainnet). ## EVM.Transactions ```toml @@ -7843,6 +7905,7 @@ LeaseDuration = '0s' # Default NodeIsSyncingEnabled = false # Default FinalizedBlockPollInterval = '5s' # Default EnforceRepeatableRead = false # Default +DeathDeclarationDelay = '10s' # Default ``` The node pool manages multiple RPC endpoints. @@ -7926,6 +7989,15 @@ block. Set false to disable +### DeathDeclarationDelay +```toml +DeathDeclarationDelay = '10s' # Default +``` +DeathDeclarationDelay defines the minimum duration an RPC must be in unhealthy state before producing an error log message. +Larger values might be helpful to reduce the noisiness of health checks like `EnforceRepeatableRead = true', which might be falsely +trigger declaration of `FinalizedBlockOutOfSync` due to insignificant network delays in broadcasting of the finalized state among RPCs. +RPC will not be picked to handle a request even if this option is set to a nonzero value. + ## EVM.NodePool.Errors :warning: **_ADVANCED_**: _Do not change these settings unless you know what you are doing._ ```toml diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index bd6e6b2e0b4..0454b9c4e6e 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -381,6 +381,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 1fa25e25e21..9b9a939b970 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -381,6 +381,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index b07d5092854..69a166f9e40 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -381,6 +381,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 72bfbfa4952..fefe6b3a627 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -371,6 +371,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4 diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 33775f52e3f..9f59e065f41 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -378,6 +378,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false +DeathDeclarationDelay = '10s' [EVM.OCR] ContractConfirmations = 4