From 65351c3a6fffcff03d249e64cae223daabd86e3c Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko <34754799+dhaidashenko@users.noreply.github.com> Date: Fri, 1 Nov 2024 00:56:06 +0100 Subject: [PATCH] BCFR-144 enforce repeatable read (#14978) * enforce repeatable read by default * fix tests * celo soak test config * Improve logging * Increase finalized block offset for Celo to avoid false death declaration of RPCs * rollback config * additional logs on finalized block lag * Increase finalized block offset for celo & wemix * add jitter to poller * Set FinalizedBlockOffset for chains with fast finality * fix tests * fix lint * regen docs * disable EnforceRepeatableRead for simulated * disable repeatable read for simulated chain * fix docs --- .changeset/famous-ligers-heal.md | 5 + common/client/multi_node.go | 7 +- common/client/node_fsm.go | 13 +- common/client/poller.go | 2 +- core/chains/evm/config/config_test.go | 6 +- .../config/toml/defaults/Avalanche_Fuji.toml | 1 + .../toml/defaults/Avalanche_Mainnet.toml | 1 + .../evm/config/toml/defaults/BSC_Mainnet.toml | 1 + .../evm/config/toml/defaults/BSC_Testnet.toml | 1 + .../config/toml/defaults/Celo_Mainnet.toml | 1 + .../config/toml/defaults/Celo_Testnet.toml | 1 + .../config/toml/defaults/Gnosis_Chiado.toml | 1 + .../config/toml/defaults/Gnosis_Mainnet.toml | 1 + .../config/toml/defaults/Hedera_Mainnet.toml | 1 + .../config/toml/defaults/Hedera_Testnet.toml | 1 + .../config/toml/defaults/Kroma_Mainnet.toml | 1 + .../config/toml/defaults/Kroma_Sepolia.toml | 1 + .../evm/config/toml/defaults/Simulated.toml | 3 + .../config/toml/defaults/WeMix_Mainnet.toml | 1 + .../config/toml/defaults/WeMix_Testnet.toml | 1 + .../config/toml/defaults/XLayer_Mainnet.toml | 1 + .../config/toml/defaults/XLayer_Sepolia.toml | 1 + .../evm/config/toml/defaults/fallback.toml | 4 +- .../config/toml/defaults/zkSync_Mainnet.toml | 1 + .../config/toml/defaults/zkSync_Sepolia.toml | 1 + core/config/docs/chains-evm.toml | 7 +- .../config-multi-chain-effective.toml | 12 +- core/web/resolver/testdata/config-full.toml | 4 +- .../config-multi-chain-effective.toml | 12 +- docs/CONFIG.md | 305 +++++++++--------- .../ocr2/overrides/celo_alfajores.toml | 15 + .../node/validate/defaults-override.txtar | 4 +- .../disk-based-logging-disabled.txtar | 4 +- .../validate/disk-based-logging-no-dir.txtar | 4 +- .../node/validate/disk-based-logging.txtar | 4 +- testdata/scripts/node/validate/invalid.txtar | 4 +- testdata/scripts/node/validate/valid.txtar | 4 +- 37 files changed, 245 insertions(+), 192 deletions(-) create mode 100644 .changeset/famous-ligers-heal.md create mode 100644 integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml diff --git a/.changeset/famous-ligers-heal.md b/.changeset/famous-ligers-heal.md new file mode 100644 index 00000000000..e469ea85785 --- /dev/null +++ b/.changeset/famous-ligers-heal.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Set `NodePool.EnforceRepeatableRead = true` by default for all chains. This forces Core to stop using RPCs behind on the latest finalized block. #changed #nops diff --git a/common/client/multi_node.go b/common/client/multi_node.go index 9594743f6bd..7d631954c54 100644 --- a/common/client/multi_node.go +++ b/common/client/multi_node.go @@ -13,6 +13,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink/v2/common/types" ) @@ -224,18 +225,20 @@ func (c *MultiNode[CHAIN_ID, RPC]) selectNode() (node Node[CHAIN_ID, RPC], err e return // another goroutine beat us here } + var prevNodeName string if c.activeNode != nil { + prevNodeName = c.activeNode.String() c.activeNode.UnsubscribeAllExceptAliveLoop() } c.activeNode = c.nodeSelector.Select() - if c.activeNode == nil { c.lggr.Criticalw("No live RPC nodes available", "NodeSelectionMode", c.nodeSelector.Name()) errmsg := fmt.Errorf("no live nodes available for chain %s", c.chainID.String()) c.SvcErrBuffer.Append(errmsg) - err = ErroringNodeError + return nil, ErroringNodeError } + c.lggr.Debugw("Switched to a new active node due to prev node heath issues", "prevNode", prevNodeName, "newNode", c.activeNode.String()) return c.activeNode, err } diff --git a/common/client/node_fsm.go b/common/client/node_fsm.go index 4a80e4fae9b..b707e9f4375 100644 --- a/common/client/node_fsm.go +++ b/common/client/node_fsm.go @@ -148,12 +148,19 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool { } highestObservedByCaller := n.poolInfoProvider.HighestUserObservations() - latest, _ := n.rpc.GetInterceptedChainInfo() + latest, rpcHighest := n.rpc.GetInterceptedChainInfo() + isOutOfSync := false if n.chainCfg.FinalityTagEnabled() { - return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + isOutOfSync = latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + } else { + isOutOfSync = latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + } + + if isOutOfSync { + n.lfcLog.Debugw("finalized block is out of sync", "rpcLatestChainInfo", latest, "rpcHighest", rpcHighest, "highestObservedByCaller", highestObservedByCaller) } - return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset()) + return isOutOfSync } // StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle. diff --git a/common/client/poller.go b/common/client/poller.go index eeb6c3af576..bdd3d36f16f 100644 --- a/common/client/poller.go +++ b/common/client/poller.go @@ -67,7 +67,7 @@ func (p *Poller[T]) Err() <-chan error { } func (p *Poller[T]) pollingLoop(ctx context.Context) { - ticker := time.NewTicker(p.pollingInterval) + ticker := services.NewTicker(p.pollingInterval) // reduce possibility of sending two exactly the same request at once defer ticker.Stop() for { diff --git a/core/chains/evm/config/config_test.go b/core/chains/evm/config/config_test.go index 53155d7e7d2..7eb13458f56 100644 --- a/core/chains/evm/config/config_test.go +++ b/core/chains/evm/config/config_test.go @@ -353,9 +353,9 @@ func TestNodePoolConfig(t *testing.T) { require.Equal(t, uint32(5), cfg.EVM().NodePool().SyncThreshold()) require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().PollInterval()) require.Equal(t, uint32(5), cfg.EVM().NodePool().PollFailureThreshold()) - require.Equal(t, false, cfg.EVM().NodePool().NodeIsSyncingEnabled()) - require.Equal(t, false, cfg.EVM().NodePool().EnforceRepeatableRead()) - require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().DeathDeclarationDelay()) + require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled()) + require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead()) + require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay()) } func TestClientErrorsConfig(t *testing.T) { diff --git a/core/chains/evm/config/toml/defaults/Avalanche_Fuji.toml b/core/chains/evm/config/toml/defaults/Avalanche_Fuji.toml index 6e95be2c7f7..fbcc8e6f058 100644 --- a/core/chains/evm/config/toml/defaults/Avalanche_Fuji.toml +++ b/core/chains/evm/config/toml/defaults/Avalanche_Fuji.toml @@ -7,6 +7,7 @@ MinIncomingConfirmations = 1 NoNewHeadsThreshold = '30s' OCR.ContractConfirmations = 1 RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/Avalanche_Mainnet.toml b/core/chains/evm/config/toml/defaults/Avalanche_Mainnet.toml index 930f9a910f6..309fdab1db9 100644 --- a/core/chains/evm/config/toml/defaults/Avalanche_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/Avalanche_Mainnet.toml @@ -7,6 +7,7 @@ MinIncomingConfirmations = 1 NoNewHeadsThreshold = '30s' OCR.ContractConfirmations = 1 RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/BSC_Mainnet.toml b/core/chains/evm/config/toml/defaults/BSC_Mainnet.toml index b3cf458ad87..710ef34d309 100644 --- a/core/chains/evm/config/toml/defaults/BSC_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/BSC_Mainnet.toml @@ -7,6 +7,7 @@ LinkContractAddress = '0x404460C6A5EdE2D891e8297795264fDe62ADBB75' LogPollInterval = '3s' NoNewHeadsThreshold = '30s' RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '45s' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/BSC_Testnet.toml b/core/chains/evm/config/toml/defaults/BSC_Testnet.toml index 7b8fc481d7b..95eef689d00 100644 --- a/core/chains/evm/config/toml/defaults/BSC_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/BSC_Testnet.toml @@ -7,6 +7,7 @@ LinkContractAddress = '0x84b9B910527Ad5C03A9Ca831909E21e236EA7b06' LogPollInterval = '3s' NoNewHeadsThreshold = '30s' RPCBlockQueryDelay = 2 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/Celo_Mainnet.toml b/core/chains/evm/config/toml/defaults/Celo_Mainnet.toml index a4948620370..b773f2d19f5 100644 --- a/core/chains/evm/config/toml/defaults/Celo_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/Celo_Mainnet.toml @@ -5,6 +5,7 @@ LogPollInterval = '5s' MinIncomingConfirmations = 1 NoNewHeadsThreshold = '1m' OCR.ContractConfirmations = 1 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/Celo_Testnet.toml b/core/chains/evm/config/toml/defaults/Celo_Testnet.toml index eb43f080b7d..8bee5323f4d 100644 --- a/core/chains/evm/config/toml/defaults/Celo_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/Celo_Testnet.toml @@ -5,6 +5,7 @@ LogPollInterval = '5s' MinIncomingConfirmations = 1 NoNewHeadsThreshold = '1m' OCR.ContractConfirmations = 1 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/Gnosis_Chiado.toml b/core/chains/evm/config/toml/defaults/Gnosis_Chiado.toml index 379377a2266..4e54210455f 100644 --- a/core/chains/evm/config/toml/defaults/Gnosis_Chiado.toml +++ b/core/chains/evm/config/toml/defaults/Gnosis_Chiado.toml @@ -3,6 +3,7 @@ ChainID = '10200' FinalityDepth = 100 ChainType = 'gnosis' LogPollInterval = '5s' +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/Gnosis_Mainnet.toml b/core/chains/evm/config/toml/defaults/Gnosis_Mainnet.toml index 628646364f5..862485b4d30 100644 --- a/core/chains/evm/config/toml/defaults/Gnosis_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/Gnosis_Mainnet.toml @@ -9,6 +9,7 @@ ChainID = '100' ChainType = 'gnosis' LinkContractAddress = '0xE2e73A1c69ecF83F464EFCE6A5be353a37cA09b2' LogPollInterval = '5s' +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m' [GasEstimator] diff --git a/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml index 7f15a8cf13d..591e486a8ec 100644 --- a/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml @@ -6,6 +6,7 @@ FinalityDepth = 10 # Hedera has high TPS, so polling less often LogPollInterval = '10s' MinIncomingConfirmations = 1 +FinalizedBlockOffset = 2 [BalanceMonitor] Enabled = true diff --git a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml index e15b4e98d69..0e2f2c7cd2c 100644 --- a/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/Hedera_Testnet.toml @@ -6,6 +6,7 @@ FinalityDepth = 10 # Hedera has high TPS, so polling less often LogPollInterval = '10s' MinIncomingConfirmations = 1 +FinalizedBlockOffset = 2 [BalanceMonitor] Enabled = true diff --git a/core/chains/evm/config/toml/defaults/Kroma_Mainnet.toml b/core/chains/evm/config/toml/defaults/Kroma_Mainnet.toml index 327c8b0975c..c832d666aa6 100644 --- a/core/chains/evm/config/toml/defaults/Kroma_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/Kroma_Mainnet.toml @@ -5,6 +5,7 @@ FinalityTagEnabled = true LogPollInterval = '2s' NoNewHeadsThreshold = '40s' MinIncomingConfirmations = 1 +FinalizedBlockOffset = 2 [GasEstimator] EIP1559DynamicFees = true diff --git a/core/chains/evm/config/toml/defaults/Kroma_Sepolia.toml b/core/chains/evm/config/toml/defaults/Kroma_Sepolia.toml index da0dc36541c..b0864d86c90 100644 --- a/core/chains/evm/config/toml/defaults/Kroma_Sepolia.toml +++ b/core/chains/evm/config/toml/defaults/Kroma_Sepolia.toml @@ -5,6 +5,7 @@ FinalityTagEnabled = true LogPollInterval = '2s' NoNewHeadsThreshold = '40s' MinIncomingConfirmations = 1 +FinalizedBlockOffset = 2 [GasEstimator] EIP1559DynamicFees = true diff --git a/core/chains/evm/config/toml/defaults/Simulated.toml b/core/chains/evm/config/toml/defaults/Simulated.toml index 52e78c94edf..65e627a7abd 100644 --- a/core/chains/evm/config/toml/defaults/Simulated.toml +++ b/core/chains/evm/config/toml/defaults/Simulated.toml @@ -20,5 +20,8 @@ HistoryDepth = 10 MaxBufferSize = 100 SamplingInterval = '0s' +[NodePool] +EnforceRepeatableRead = false # disable for simulation to prevent failure of tests with manual commit and reorgs + [OCR] ContractConfirmations = 1 diff --git a/core/chains/evm/config/toml/defaults/WeMix_Mainnet.toml b/core/chains/evm/config/toml/defaults/WeMix_Mainnet.toml index 4a55450e47e..8af3f1bb9e5 100644 --- a/core/chains/evm/config/toml/defaults/WeMix_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/WeMix_Mainnet.toml @@ -6,6 +6,7 @@ MinIncomingConfirmations = 1 # WeMix emits a block every 1 second, regardless of transactions LogPollInterval = '3s' NoNewHeadsThreshold = '30s' +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' [OCR] diff --git a/core/chains/evm/config/toml/defaults/WeMix_Testnet.toml b/core/chains/evm/config/toml/defaults/WeMix_Testnet.toml index d4f829bcfd1..451d22a1557 100644 --- a/core/chains/evm/config/toml/defaults/WeMix_Testnet.toml +++ b/core/chains/evm/config/toml/defaults/WeMix_Testnet.toml @@ -6,6 +6,7 @@ MinIncomingConfirmations = 1 # WeMix emits a block every 1 second, regardless of transactions LogPollInterval = '3s' NoNewHeadsThreshold = '30s' +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' [OCR] diff --git a/core/chains/evm/config/toml/defaults/XLayer_Mainnet.toml b/core/chains/evm/config/toml/defaults/XLayer_Mainnet.toml index 4096a4db244..37dcae91baf 100644 --- a/core/chains/evm/config/toml/defaults/XLayer_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/XLayer_Mainnet.toml @@ -6,6 +6,7 @@ MinIncomingConfirmations = 1 LogPollInterval = '30s' RPCBlockQueryDelay = 15 RPCDefaultBatchSize = 100 +FinalizedBlockOffset = 2 [OCR] ContractConfirmations = 1 diff --git a/core/chains/evm/config/toml/defaults/XLayer_Sepolia.toml b/core/chains/evm/config/toml/defaults/XLayer_Sepolia.toml index 62e2c1e8ad0..f39c9dab704 100644 --- a/core/chains/evm/config/toml/defaults/XLayer_Sepolia.toml +++ b/core/chains/evm/config/toml/defaults/XLayer_Sepolia.toml @@ -6,6 +6,7 @@ MinIncomingConfirmations = 1 LogPollInterval = '30s' RPCBlockQueryDelay = 15 RPCDefaultBatchSize = 100 +FinalizedBlockOffset = 2 [OCR] ContractConfirmations = 1 diff --git a/core/chains/evm/config/toml/defaults/fallback.toml b/core/chains/evm/config/toml/defaults/fallback.toml index 1e18f4a4ebf..ab349ee4688 100644 --- a/core/chains/evm/config/toml/defaults/fallback.toml +++ b/core/chains/evm/config/toml/defaults/fallback.toml @@ -76,8 +76,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m' NewHeadsPollInterval = '0s' [OCR] diff --git a/core/chains/evm/config/toml/defaults/zkSync_Mainnet.toml b/core/chains/evm/config/toml/defaults/zkSync_Mainnet.toml index d4602a85479..85282ea81b3 100644 --- a/core/chains/evm/config/toml/defaults/zkSync_Mainnet.toml +++ b/core/chains/evm/config/toml/defaults/zkSync_Mainnet.toml @@ -4,6 +4,7 @@ FinalityDepth = 10 LogPollInterval = '5s' MinIncomingConfirmations = 1 NoNewHeadsThreshold = '1m' +FinalizedBlockOffset = 2 [GasEstimator] LimitDefault = 100_000_000 diff --git a/core/chains/evm/config/toml/defaults/zkSync_Sepolia.toml b/core/chains/evm/config/toml/defaults/zkSync_Sepolia.toml index 4ea175bfdb8..78ed3c0768d 100644 --- a/core/chains/evm/config/toml/defaults/zkSync_Sepolia.toml +++ b/core/chains/evm/config/toml/defaults/zkSync_Sepolia.toml @@ -4,6 +4,7 @@ FinalityDepth = 10 LogPollInterval = '5s' MinIncomingConfirmations = 1 NoNewHeadsThreshold = '1m' +FinalizedBlockOffset = 2 [GasEstimator] LimitDefault = 100_000_000 diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index 7d0de98acea..b67c57ba40e 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -416,12 +416,13 @@ FinalizedBlockPollInterval = '5s' # Default # block. # # Set false to disable -EnforceRepeatableRead = false # Default +EnforceRepeatableRead = true # Default # DeathDeclarationDelay defines the minimum duration an RPC must be in unhealthy state before producing an error log message. # Larger values might be helpful to reduce the noisiness of health checks like `EnforceRepeatableRead = true', which might be falsely # trigger declaration of `FinalizedBlockOutOfSync` due to insignificant network delays in broadcasting of the finalized state among RPCs. -# RPC will not be picked to handle a request even if this option is set to a nonzero value. -DeathDeclarationDelay = '10s' # Default +# Should be greater than `FinalizedBlockPollInterval`. +# Unhealthy RPC will not be picked to handle a request even if this option is set to a nonzero value. +DeathDeclarationDelay = '1m' # Default # NewHeadsPollInterval define an interval for polling new block periodically using http client rather than subscribe to ws feed # # Set to 0 to disable. diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index f9d34ffbde6..86612f05798 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -369,8 +369,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] @@ -479,8 +479,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] @@ -583,8 +583,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index c319a7bf8ec..ebdafb22c5e 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -401,8 +401,8 @@ SyncThreshold = 13 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.NodePool.Errors] diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index e29c763b74c..2a0ebcf1039 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -369,8 +369,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] @@ -479,8 +479,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] @@ -583,8 +583,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/docs/CONFIG.md b/docs/CONFIG.md index ae22482951b..a1f583ec2d7 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -2041,8 +2041,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2145,8 +2145,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2249,8 +2249,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2353,8 +2353,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2462,8 +2462,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2566,8 +2566,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2670,8 +2670,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2775,8 +2775,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2818,7 +2818,7 @@ NoNewHeadsThreshold = '30s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 2 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '45s' [Transactions] @@ -2879,8 +2879,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -2982,8 +2982,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3085,8 +3085,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3128,7 +3128,7 @@ NoNewHeadsThreshold = '30s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 2 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' [Transactions] @@ -3189,8 +3189,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3233,7 +3233,7 @@ NoNewHeadsThreshold = '3m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' [Transactions] @@ -3294,8 +3294,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3398,8 +3398,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3502,8 +3502,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3545,7 +3545,7 @@ NoNewHeadsThreshold = '12m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 100 RPCBlockQueryDelay = 15 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -3606,8 +3606,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3649,7 +3649,7 @@ NoNewHeadsThreshold = '6m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 100 RPCBlockQueryDelay = 15 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -3710,8 +3710,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3814,8 +3814,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -3857,7 +3857,7 @@ NoNewHeadsThreshold = '40s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -3922,8 +3922,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4029,8 +4029,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4072,7 +4072,7 @@ NoNewHeadsThreshold = '3m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -4133,8 +4133,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4176,7 +4176,7 @@ NoNewHeadsThreshold = '3m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -4237,8 +4237,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4280,7 +4280,7 @@ NoNewHeadsThreshold = '1m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -4344,8 +4344,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4387,7 +4387,7 @@ NoNewHeadsThreshold = '1m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -4451,8 +4451,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4560,8 +4560,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4664,8 +4664,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4767,8 +4767,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4871,8 +4871,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -4975,8 +4975,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5018,7 +5018,7 @@ NoNewHeadsThreshold = '30s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' [Transactions] @@ -5079,8 +5079,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5122,7 +5122,7 @@ NoNewHeadsThreshold = '30s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '40s' [Transactions] @@ -5183,8 +5183,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5287,7 +5287,7 @@ LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5395,8 +5395,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5438,7 +5438,7 @@ NoNewHeadsThreshold = '40s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '0s' [Transactions] @@ -5503,8 +5503,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5607,8 +5607,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5711,8 +5711,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5814,8 +5814,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5922,8 +5922,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -5965,7 +5965,7 @@ NoNewHeadsThreshold = '3m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '2m0s' [Transactions] @@ -6026,8 +6026,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6134,8 +6134,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6242,8 +6242,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6350,8 +6350,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6393,7 +6393,7 @@ NoNewHeadsThreshold = '1m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' [Transactions] @@ -6454,8 +6454,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6497,7 +6497,7 @@ NoNewHeadsThreshold = '30s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 2 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' [Transactions] @@ -6558,8 +6558,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6601,7 +6601,7 @@ NoNewHeadsThreshold = '30s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 2 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' [Transactions] @@ -6662,8 +6662,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6705,7 +6705,7 @@ NoNewHeadsThreshold = '1m0s' LogBroadcasterEnabled = true RPCDefaultBatchSize = 250 RPCBlockQueryDelay = 1 -FinalizedBlockOffset = 0 +FinalizedBlockOffset = 2 NoNewFinalizedHeadsThreshold = '1m0s' [Transactions] @@ -6766,8 +6766,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6877,8 +6877,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -6988,8 +6988,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7091,8 +7091,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7194,8 +7194,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7297,8 +7297,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7401,8 +7401,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7505,8 +7505,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7608,8 +7608,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7716,8 +7716,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7824,8 +7824,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -7932,8 +7932,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8040,8 +8040,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8147,8 +8147,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8255,8 +8255,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8363,8 +8363,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8467,8 +8467,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8575,8 +8575,8 @@ SyncThreshold = 10 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8679,8 +8679,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -8783,8 +8783,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [OCR] @@ -9577,8 +9577,8 @@ SyncThreshold = 5 # Default LeaseDuration = '0s' # Default NodeIsSyncingEnabled = false # Default FinalizedBlockPollInterval = '5s' # Default -EnforceRepeatableRead = false # Default -DeathDeclarationDelay = '10s' # Default +EnforceRepeatableRead = true # Default +DeathDeclarationDelay = '1m' # Default NewHeadsPollInterval = '0s' # Default ``` The node pool manages multiple RPC endpoints. @@ -9655,7 +9655,7 @@ Set to 0 to disable. ### EnforceRepeatableRead ```toml -EnforceRepeatableRead = false # Default +EnforceRepeatableRead = true # Default ``` EnforceRepeatableRead defines if Core should only use RPCs whose most recently finalized block is greater or equal to `highest finalized block - FinalizedBlockOffset`. In other words, exclude RPCs lagging on latest finalized @@ -9665,12 +9665,13 @@ Set false to disable ### DeathDeclarationDelay ```toml -DeathDeclarationDelay = '10s' # Default +DeathDeclarationDelay = '1m' # Default ``` DeathDeclarationDelay defines the minimum duration an RPC must be in unhealthy state before producing an error log message. Larger values might be helpful to reduce the noisiness of health checks like `EnforceRepeatableRead = true', which might be falsely trigger declaration of `FinalizedBlockOutOfSync` due to insignificant network delays in broadcasting of the finalized state among RPCs. -RPC will not be picked to handle a request even if this option is set to a nonzero value. +Should be greater than `FinalizedBlockPollInterval`. +Unhealthy RPC will not be picked to handle a request even if this option is set to a nonzero value. ### NewHeadsPollInterval ```toml diff --git a/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml new file mode 100644 index 00000000000..73995ddc0b5 --- /dev/null +++ b/integration-tests/testconfig/ocr2/overrides/celo_alfajores.toml @@ -0,0 +1,15 @@ +[Network] +selected_networks = ["CELO_ALFAJORES"] + +[Soak.Common] +chainlink_node_funding = 2 + +[Soak.OCR2] +[Soak.OCR2.Common] +test_duration = "24h" + +[Soak.OCR2.Soak] +time_between_rounds = "2m" + +[OCR2.Common] +number_of_contracts = 2 diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index 5c36fddeaf1..14c26bf85cd 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -442,8 +442,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 9a9f5e88811..bff1c74c97b 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -425,8 +425,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 8b4089832eb..6b0b5db9eaf 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -425,8 +425,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index ef496ddfd8b..59c8d7b1c5d 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -425,8 +425,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 81c5a494440..739f427317e 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -415,8 +415,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR] diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 063436f1079..6543a53c43d 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -422,8 +422,8 @@ SyncThreshold = 5 LeaseDuration = '0s' NodeIsSyncingEnabled = false FinalizedBlockPollInterval = '5s' -EnforceRepeatableRead = false -DeathDeclarationDelay = '10s' +EnforceRepeatableRead = true +DeathDeclarationDelay = '1m0s' NewHeadsPollInterval = '0s' [EVM.OCR]