BCFR-144 enforce repeatable read (#14978)

* enforce repeatable read by default

* fix tests

* celo soak test config

* Improve logging

* Increase finalized block offset for Celo to avoid false death declaration of RPCs

* rollback config

* additional logs on finalized block lag

* Increase finalized block offset for celo & wemix

* add jitter to poller

* Set FinalizedBlockOffset for chains with fast finality

* fix tests

* fix lint

* regen docs

* disable EnforceRepeatableRead for simulated

* disable repeatable read for simulated chain

* fix docs

.changeset/famous-ligers-heal.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Set `NodePool.EnforceRepeatableRead = true` by default for all chains. This forces Core to stop using RPCs behind on the latest finalized block. #changed #nops

common/client/multi_node.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
+
 	"github.com/smartcontractkit/chainlink/v2/common/types"
 )
 
@@ -224,18 +225,20 @@ func (c *MultiNode[CHAIN_ID, RPC]) selectNode() (node Node[CHAIN_ID, RPC], err e
 		return // another goroutine beat us here
 	}
 
+	var prevNodeName string
 	if c.activeNode != nil {
+		prevNodeName = c.activeNode.String()
 		c.activeNode.UnsubscribeAllExceptAliveLoop()
 	}
 	c.activeNode = c.nodeSelector.Select()
-
 	if c.activeNode == nil {
 		c.lggr.Criticalw("No live RPC nodes available", "NodeSelectionMode", c.nodeSelector.Name())
 		errmsg := fmt.Errorf("no live nodes available for chain %s", c.chainID.String())
 		c.SvcErrBuffer.Append(errmsg)
-		err = ErroringNodeError
+		return nil, ErroringNodeError
 	}
 
+	c.lggr.Debugw("Switched to a new active node due to prev node heath issues", "prevNode", prevNodeName, "newNode", c.activeNode.String())
 	return c.activeNode, err
 }
 

common/client/node_fsm.go

@@ -148,12 +148,19 @@ func (n *node[CHAIN_ID, HEAD, RPC]) isFinalizedBlockOutOfSync() bool {
 	}
 
 	highestObservedByCaller := n.poolInfoProvider.HighestUserObservations()
-	latest, _ := n.rpc.GetInterceptedChainInfo()
+	latest, rpcHighest := n.rpc.GetInterceptedChainInfo()
+	isOutOfSync := false
 	if n.chainCfg.FinalityTagEnabled() {
-		return latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset())
+		isOutOfSync = latest.FinalizedBlockNumber < highestObservedByCaller.FinalizedBlockNumber-int64(n.chainCfg.FinalizedBlockOffset())
+	} else {
+		isOutOfSync = latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset())
+	}
+
+	if isOutOfSync {
+		n.lfcLog.Debugw("finalized block is out of sync", "rpcLatestChainInfo", latest, "rpcHighest", rpcHighest, "highestObservedByCaller", highestObservedByCaller)
 	}
 
-	return latest.BlockNumber < highestObservedByCaller.BlockNumber-int64(n.chainCfg.FinalizedBlockOffset())
+	return isOutOfSync
 }
 
 // StateAndLatest returns nodeState with the latest ChainInfo observed by Node during current lifecycle.

common/client/poller.go

@@ -67,7 +67,7 @@ func (p *Poller[T]) Err() <-chan error {
 }
 
 func (p *Poller[T]) pollingLoop(ctx context.Context) {
-	ticker := time.NewTicker(p.pollingInterval)
+	ticker := services.NewTicker(p.pollingInterval) // reduce possibility of sending two exactly the same request at once
 	defer ticker.Stop()
 
 	for {

core/chains/evm/config/config_test.go

@@ -353,9 +353,9 @@ func TestNodePoolConfig(t *testing.T) {
 	require.Equal(t, uint32(5), cfg.EVM().NodePool().SyncThreshold())
 	require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().PollInterval())
 	require.Equal(t, uint32(5), cfg.EVM().NodePool().PollFailureThreshold())
-	require.Equal(t, false, cfg.EVM().NodePool().NodeIsSyncingEnabled())
-	require.Equal(t, false, cfg.EVM().NodePool().EnforceRepeatableRead())
-	require.Equal(t, time.Duration(10000000000), cfg.EVM().NodePool().DeathDeclarationDelay())
+	require.False(t, cfg.EVM().NodePool().NodeIsSyncingEnabled())
+	require.True(t, cfg.EVM().NodePool().EnforceRepeatableRead())
+	require.Equal(t, time.Minute, cfg.EVM().NodePool().DeathDeclarationDelay())
 }
 
 func TestClientErrorsConfig(t *testing.T) {

core/chains/evm/config/toml/defaults/Avalanche_Fuji.toml

@@ -7,6 +7,7 @@ MinIncomingConfirmations = 1
 NoNewHeadsThreshold = '30s'
 OCR.ContractConfirmations = 1
 RPCBlockQueryDelay = 2
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '1m'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/Avalanche_Mainnet.toml

@@ -7,6 +7,7 @@ MinIncomingConfirmations = 1
 NoNewHeadsThreshold = '30s'
 OCR.ContractConfirmations = 1
 RPCBlockQueryDelay = 2
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '1m'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/BSC_Mainnet.toml

@@ -7,6 +7,7 @@ LinkContractAddress = '0x404460C6A5EdE2D891e8297795264fDe62ADBB75'
 LogPollInterval = '3s'
 NoNewHeadsThreshold = '30s'
 RPCBlockQueryDelay = 2
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '45s'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/BSC_Testnet.toml

@@ -7,6 +7,7 @@ LinkContractAddress = '0x84b9B910527Ad5C03A9Ca831909E21e236EA7b06'
 LogPollInterval = '3s'
 NoNewHeadsThreshold = '30s'
 RPCBlockQueryDelay = 2
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '40s'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/Celo_Mainnet.toml

@@ -5,6 +5,7 @@ LogPollInterval = '5s'
 MinIncomingConfirmations = 1
 NoNewHeadsThreshold = '1m'
 OCR.ContractConfirmations = 1
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '1m'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/Celo_Testnet.toml

@@ -5,6 +5,7 @@ LogPollInterval = '5s'
 MinIncomingConfirmations = 1
 NoNewHeadsThreshold = '1m'
 OCR.ContractConfirmations = 1
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '1m'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/Gnosis_Chiado.toml

@@ -3,6 +3,7 @@ ChainID = '10200'
 FinalityDepth = 100
 ChainType = 'gnosis'
 LogPollInterval = '5s'
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '2m'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/Gnosis_Mainnet.toml

@@ -9,6 +9,7 @@ ChainID = '100'
 ChainType = 'gnosis'
 LinkContractAddress = '0xE2e73A1c69ecF83F464EFCE6A5be353a37cA09b2'
 LogPollInterval = '5s'
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '2m'
 
 [GasEstimator]

core/chains/evm/config/toml/defaults/Hedera_Mainnet.toml

@@ -6,6 +6,7 @@ FinalityDepth = 10
 # Hedera has high TPS, so polling less often 
 LogPollInterval = '10s'
 MinIncomingConfirmations = 1
+FinalizedBlockOffset = 2
 
 [BalanceMonitor]
 Enabled = true

core/chains/evm/config/toml/defaults/Hedera_Testnet.toml

@@ -6,6 +6,7 @@ FinalityDepth = 10
 # Hedera has high TPS, so polling less often 
 LogPollInterval = '10s'
 MinIncomingConfirmations = 1
+FinalizedBlockOffset = 2
 
 [BalanceMonitor]
 Enabled = true

core/chains/evm/config/toml/defaults/Kroma_Mainnet.toml

@@ -5,6 +5,7 @@ FinalityTagEnabled = true
 LogPollInterval = '2s'
 NoNewHeadsThreshold = '40s'
 MinIncomingConfirmations = 1
+FinalizedBlockOffset = 2
 
 [GasEstimator]
 EIP1559DynamicFees = true

core/chains/evm/config/toml/defaults/Kroma_Sepolia.toml

@@ -5,6 +5,7 @@ FinalityTagEnabled = true
 LogPollInterval = '2s'
 NoNewHeadsThreshold = '40s'
 MinIncomingConfirmations = 1
+FinalizedBlockOffset = 2
 
 [GasEstimator]
 EIP1559DynamicFees = true

core/chains/evm/config/toml/defaults/Simulated.toml

@@ -20,5 +20,8 @@ HistoryDepth = 10
 MaxBufferSize = 100
 SamplingInterval = '0s'
 
+[NodePool]
+EnforceRepeatableRead = false # disable for simulation to prevent failure of tests with manual commit and reorgs
+
 [OCR]
 ContractConfirmations = 1

core/chains/evm/config/toml/defaults/WeMix_Mainnet.toml

@@ -6,6 +6,7 @@ MinIncomingConfirmations = 1
 # WeMix emits a block every 1 second, regardless of transactions
 LogPollInterval = '3s'
 NoNewHeadsThreshold = '30s'
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '40s'
 
 [OCR]

core/chains/evm/config/toml/defaults/WeMix_Testnet.toml

@@ -6,6 +6,7 @@ MinIncomingConfirmations = 1
 # WeMix emits a block every 1 second, regardless of transactions
 LogPollInterval = '3s'
 NoNewHeadsThreshold = '30s'
+FinalizedBlockOffset = 2
 NoNewFinalizedHeadsThreshold = '40s'
 
 [OCR]

core/chains/evm/config/toml/defaults/XLayer_Mainnet.toml

@@ -6,6 +6,7 @@ MinIncomingConfirmations = 1
 LogPollInterval = '30s'
 RPCBlockQueryDelay = 15
 RPCDefaultBatchSize = 100
+FinalizedBlockOffset = 2
 
 [OCR]
 ContractConfirmations = 1

core/chains/evm/config/toml/defaults/XLayer_Sepolia.toml

@@ -6,6 +6,7 @@ MinIncomingConfirmations = 1
 LogPollInterval = '30s'
 RPCBlockQueryDelay = 15
 RPCDefaultBatchSize = 100
+FinalizedBlockOffset = 2
 
 [OCR]
 ContractConfirmations = 1

core/chains/evm/config/toml/defaults/fallback.toml

@@ -76,8 +76,8 @@ SyncThreshold = 5
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m'
 NewHeadsPollInterval = '0s'
 
 [OCR]

core/chains/evm/config/toml/defaults/zkSync_Mainnet.toml

@@ -4,6 +4,7 @@ FinalityDepth = 10
 LogPollInterval = '5s'
 MinIncomingConfirmations = 1
 NoNewHeadsThreshold = '1m'
+FinalizedBlockOffset = 2
 
 [GasEstimator]
 LimitDefault = 100_000_000

core/chains/evm/config/toml/defaults/zkSync_Sepolia.toml

@@ -4,6 +4,7 @@ FinalityDepth = 10
 LogPollInterval = '5s'
 MinIncomingConfirmations = 1
 NoNewHeadsThreshold = '1m'
+FinalizedBlockOffset = 2
 
 [GasEstimator]
 LimitDefault = 100_000_000

core/config/docs/chains-evm.toml

@@ -416,12 +416,13 @@ FinalizedBlockPollInterval = '5s' # Default
 # block.
 #
 # Set false to disable
-EnforceRepeatableRead = false # Default
+EnforceRepeatableRead = true # Default
 # DeathDeclarationDelay defines the minimum duration an RPC must be in unhealthy state before producing an error log message.
 # Larger values might be helpful to reduce the noisiness of health checks like `EnforceRepeatableRead = true', which might be falsely
 # trigger declaration of `FinalizedBlockOutOfSync` due to insignificant network delays in broadcasting of the finalized state among RPCs.
-# RPC will not be picked to handle a request even if this option is set to a nonzero value.
-DeathDeclarationDelay = '10s' # Default
+# Should be greater than `FinalizedBlockPollInterval`.
+# Unhealthy RPC will not be picked to handle a request even if this option is set to a nonzero value.
+DeathDeclarationDelay = '1m' # Default
 # NewHeadsPollInterval define an interval for polling new block periodically using http client rather than subscribe to ws feed
 #
 # Set to 0 to disable.

core/services/chainlink/testdata/config-multi-chain-effective.toml

@@ -369,8 +369,8 @@ SyncThreshold = 5
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.OCR]
@@ -479,8 +479,8 @@ SyncThreshold = 5
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.OCR]
@@ -583,8 +583,8 @@ SyncThreshold = 10
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.OCR]

core/web/resolver/testdata/config-full.toml

@@ -401,8 +401,8 @@ SyncThreshold = 13
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.NodePool.Errors]

core/web/resolver/testdata/config-multi-chain-effective.toml

@@ -369,8 +369,8 @@ SyncThreshold = 5
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.OCR]
@@ -479,8 +479,8 @@ SyncThreshold = 5
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.OCR]
@@ -583,8 +583,8 @@ SyncThreshold = 10
 LeaseDuration = '0s'
 NodeIsSyncingEnabled = false
 FinalizedBlockPollInterval = '5s'
-EnforceRepeatableRead = false
-DeathDeclarationDelay = '10s'
+EnforceRepeatableRead = true
+DeathDeclarationDelay = '1m0s'
 NewHeadsPollInterval = '0s'
 
 [EVM.OCR]