Added config options to control HeadTracker's support of finality tags

.changeset/sour-owls-grab.md

@@ -0,0 +1,7 @@
+---
+"chainlink": patch
+---
+
+Added config option `HeadTracker.FinalityTagSupportDisabled` to force `HeadTracker` to track blocks up to `FinalityDepth` even if `FinalityTagsEnabled = true`. This option is a temporary measure to address high CPU usage on chains with extremely large actual finality depth (gap between the current head and the latest finalized block). #added
+
+Added config option `HeadTracker.MaxAllowedFinalityDepth` maximum gap between current head to the latest finalized block that `HeadTracker` considers healthy. #added

common/headtracker/head_tracker.go

@@ -119,7 +119,7 @@ func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) Start(ctx context.Context) error
 			if ctx.Err() != nil {
 				return ctx.Err()
 			}
-			ht.log.Errorw("Error handling initial head", "err", err)
+			ht.log.Errorw("Error handling initial head", "err", err.Error())
 		}
 
 		ht.wgDone.Add(3)
@@ -338,9 +338,24 @@ func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) backfillLoop() {
 // calculateLatestFinalized - returns latest finalized block. It's expected that currentHeadNumber - is the head of
 // canonical chain. There is no guaranties that returned block belongs to the canonical chain. Additional verification
 // must be performed before usage.
-func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.Context, currentHead HTH) (h HTH, err error) {
-	if ht.config.FinalityTagEnabled() {
-		return ht.client.LatestFinalizedBlock(ctx)
+func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.Context, currentHead HTH) (latestFinalized HTH, err error) {
+	if ht.config.FinalityTagEnabled() && !ht.htConfig.FinalityTagSupportDisabled() {
+		latestFinalized, err = ht.client.LatestFinalizedBlock(ctx)
+		if err != nil {
+			return latestFinalized, fmt.Errorf("failed to get latest finalized block: %w", err)
+		}
+
+		if !latestFinalized.IsValid() {
+			return latestFinalized, fmt.Errorf("failed to get valid latest finalized block")
+		}
+
+		if currentHead.BlockNumber()-latestFinalized.BlockNumber() > int64(ht.htConfig.MaxAllowedFinalityDepth()) {
+			return latestFinalized, fmt.Errorf("gap between latest finalized block (%d) and current head (%d) is too large (> %d)",
+				latestFinalized.BlockNumber(), currentHead.BlockNumber(), ht.htConfig.MaxAllowedFinalityDepth())
+		}
+
+		return latestFinalized, nil
+
 	}
 	// no need to make an additional RPC call on chains with instant finality
 	if ht.config.FinalityDepth() == 0 {

common/headtracker/types/config.go

@@ -12,4 +12,6 @@ type HeadTrackerConfig interface {
 	HistoryDepth() uint32
 	MaxBufferSize() uint32
 	SamplingInterval() time.Duration
+	FinalityTagSupportDisabled() bool
+	MaxAllowedFinalityDepth() uint32
 }

core/chains/evm/config/chain_scoped_head_tracker.go

@@ -21,3 +21,11 @@ func (h *headTrackerConfig) MaxBufferSize() uint32 {
 func (h *headTrackerConfig) SamplingInterval() time.Duration {
 	return h.c.SamplingInterval.Duration()
 }
+
+func (h *headTrackerConfig) FinalityTagSupportDisabled() bool {
+	return *h.c.FinalityTagSupportDisabled
+}
+
+func (h *headTrackerConfig) MaxAllowedFinalityDepth() uint32 {
+	return *h.c.MaxAllowedFinalityDepth
+}

core/chains/evm/config/config.go

@@ -70,6 +70,8 @@ type HeadTracker interface {
 	HistoryDepth() uint32
 	MaxBufferSize() uint32
 	SamplingInterval() time.Duration
+	FinalityTagSupportDisabled() bool
+	MaxAllowedFinalityDepth() uint32
 }
 
 type BalanceMonitor interface {

core/chains/evm/config/config_test.go

@@ -376,6 +376,8 @@ func TestChainScopedConfig_HeadTracker(t *testing.T) {
 	assert.Equal(t, uint32(100), ht.HistoryDepth())
 	assert.Equal(t, uint32(3), ht.MaxBufferSize())
 	assert.Equal(t, time.Second, ht.SamplingInterval())
+	assert.Equal(t, true, ht.FinalityTagSupportDisabled())
+	assert.Equal(t, uint32(10000), ht.MaxAllowedFinalityDepth())
 }
 
 func Test_chainScopedConfig_Validate(t *testing.T) {

core/chains/evm/config/toml/config.go

@@ -677,9 +677,11 @@ func (e *KeySpecificGasEstimator) setFrom(f *KeySpecificGasEstimator) {
 }
 
 type HeadTracker struct {
-	HistoryDepth     *uint32
-	MaxBufferSize    *uint32
-	SamplingInterval *commonconfig.Duration
+	HistoryDepth               *uint32
+	MaxBufferSize              *uint32
+	SamplingInterval           *commonconfig.Duration
+	MaxAllowedFinalityDepth    *uint32
+	FinalityTagSupportDisabled *bool
 }
 
 func (t *HeadTracker) setFrom(f *HeadTracker) {
@@ -692,6 +694,21 @@ func (t *HeadTracker) setFrom(f *HeadTracker) {
 	if v := f.SamplingInterval; v != nil {
 		t.SamplingInterval = v
 	}
+	if v := f.MaxAllowedFinalityDepth; v != nil {
+		t.MaxAllowedFinalityDepth = v
+	}
+	if v := f.FinalityTagSupportDisabled; v != nil {
+		t.FinalityTagSupportDisabled = v
+	}
+}
+
+func (t *HeadTracker) ValidateConfig() (err error) {
+	if *t.MaxAllowedFinalityDepth < 1 {
+		err = multierr.Append(err, commonconfig.ErrInvalid{Name: "MaxAllowedFinalityDepth", Value: *t.MaxAllowedFinalityDepth,
+			Msg: "must be greater than or equal to 1"})
+	}
+
+	return
 }
 
 type ClientErrors struct {

core/chains/evm/config/toml/defaults/Avalanche_Fuji.toml

@@ -15,3 +15,6 @@ PriceMin = '25 gwei'
 
 [GasEstimator.BlockHistory]
 BlockHistorySize = 24
+
+[HeadTracker]
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/BSC_Testnet.toml

@@ -27,6 +27,7 @@ BlockHistorySize = 24
 [HeadTracker]
 HistoryDepth = 100
 SamplingInterval = '1s'
+FinalityTagSupportDisabled = false
 
 [OCR]
 DatabaseTimeout = '2s'

core/chains/evm/config/toml/defaults/Celo_Testnet.toml

@@ -17,3 +17,4 @@ BlockHistorySize = 24
 
 [HeadTracker]
 HistoryDepth = 50
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/Ethereum_Sepolia.toml

@@ -12,3 +12,6 @@ TransactionPercentile = 50
 
 [OCR2.Automation]
 GasLimit = 10500000
+
+[HeadTracker]
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/Gnosis_Chiado.toml

@@ -9,3 +9,6 @@ EIP1559DynamicFees = true
 PriceMax = '500 gwei'
 # 15s delay since feeds update every minute in volatile situations
 BumpThreshold = 3
+
+[HeadTracker]
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/Linea_Sepolia.toml

@@ -11,3 +11,4 @@ ResendAfterThreshold = '3m'
 
 [HeadTracker]
 HistoryDepth = 1000
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/Metis_Sepolia.toml

@@ -18,3 +18,6 @@ BlockHistorySize = 0
 
 [NodePool]
 SyncThreshold = 10
+
+[HeadTracker]
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/Polygon_Amoy.toml

@@ -26,6 +26,7 @@ BlockHistorySize = 24
 [HeadTracker]
 HistoryDepth = 2000
 SamplingInterval = '1s'
+FinalityTagSupportDisabled = false
 
 [NodePool]
 SyncThreshold = 10

core/chains/evm/config/toml/defaults/WeMix_Testnet.toml

@@ -12,3 +12,6 @@ ContractConfirmations = 1
 [GasEstimator]
 EIP1559DynamicFees = true
 TipCapDefault = '100 gwei'
+
+[HeadTracker]
+FinalityTagSupportDisabled = false

core/chains/evm/config/toml/defaults/fallback.toml

@@ -54,6 +54,8 @@ TransactionPercentile = 60
 HistoryDepth = 100
 MaxBufferSize = 3
 SamplingInterval = '1s'
+FinalityTagSupportDisabled = true
+MaxAllowedFinalityDepth = 10000
 
 [NodePool]
 PollFailureThreshold = 5

core/chains/evm/headtracker/head_saver_test.go

@@ -36,6 +36,13 @@ func (h *headTrackerConfig) MaxBufferSize() uint32 {
 	return uint32(0)
 }
 
+func (h *headTrackerConfig) FinalityTagSupportDisabled() bool {
+	return false
+}
+func (h *headTrackerConfig) MaxAllowedFinalityDepth() uint32 {
+	return 10000
+}
+
 type config struct {
 	finalityDepth                     uint32
 	blockEmissionIdleWarningThreshold time.Duration

core/chains/evm/headtracker/head_tracker_test.go

@@ -205,11 +205,27 @@ func TestHeadTracker_Start(t *testing.T) {
 	t.Parallel()
 
 	const historyDepth = 100
-	newHeadTracker := func(t *testing.T) *headTrackerUniverse {
+	const finalityDepth = 50
+	type opts struct {
+		FinalityTagEnable          *bool
+		MaxAllowedFinalityDepth    *uint32
+		FinalityTagSupportDisabled *bool
+	}
+	newHeadTracker := func(t *testing.T, opts opts) *headTrackerUniverse {
 		db := pgtest.NewSqlxDB(t)
 		gCfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, _ *chainlink.Secrets) {
-			c.EVM[0].FinalityTagEnabled = ptr[bool](true)
+			if opts.FinalityTagEnable != nil {
+				c.EVM[0].FinalityTagEnabled = opts.FinalityTagEnable
+			}
 			c.EVM[0].HeadTracker.HistoryDepth = ptr[uint32](historyDepth)
+			c.EVM[0].FinalityDepth = ptr[uint32](finalityDepth)
+			if opts.MaxAllowedFinalityDepth != nil {
+				c.EVM[0].HeadTracker.MaxAllowedFinalityDepth = opts.MaxAllowedFinalityDepth
+			}
+
+			if opts.FinalityTagSupportDisabled != nil {
+				c.EVM[0].HeadTracker.FinalityTagSupportDisabled = opts.FinalityTagSupportDisabled
+			}
 		})
 		config := evmtest.NewChainScopedConfig(t, gCfg)
 		orm := headtracker.NewORM(cltest.FixtureChainID, db)
@@ -219,44 +235,59 @@ func TestHeadTracker_Start(t *testing.T) {
 
 	t.Run("Fail start if context was canceled", func(t *testing.T) {
 		ctx, cancel := context.WithCancel(testutils.Context(t))
-		ht := newHeadTracker(t)
+		ht := newHeadTracker(t, opts{})
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Run(func(args mock.Arguments) {
 			cancel()
 		}).Return(cltest.Head(0), context.Canceled)
 		err := ht.headTracker.Start(ctx)
 		require.ErrorIs(t, err, context.Canceled)
 	})
 	t.Run("Starts even if failed to get initialHead", func(t *testing.T) {
-		ht := newHeadTracker(t)
+		ht := newHeadTracker(t, opts{})
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), errors.New("failed to get init head"))
 		ht.Start(t)
 		tests.AssertLogEventually(t, ht.observer, "Error handling initial head")
 	})
 	t.Run("Starts even if received invalid head", func(t *testing.T) {
-		ht := newHeadTracker(t)
+		ht := newHeadTracker(t, opts{})
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(nil, nil)
 		ht.Start(t)
 		tests.AssertLogEventually(t, ht.observer, "Got nil initial head")
 	})
 	t.Run("Starts even if fails to get finalizedHead", func(t *testing.T) {
-		ht := newHeadTracker(t)
+		ht := newHeadTracker(t, opts{FinalityTagEnable: ptr(true), FinalityTagSupportDisabled: ptr(false)})
 		head := cltest.Head(1000)
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once()
 		ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New("failed to load latest finalized")).Once()
 		ht.Start(t)
 		tests.AssertLogEventually(t, ht.observer, "Error handling initial head")
 	})
 	t.Run("Starts even if latest finalizedHead is nil", func(t *testing.T) {
-		ht := newHeadTracker(t)
+		ht := newHeadTracker(t, opts{FinalityTagEnable: ptr(true), FinalityTagSupportDisabled: ptr(false)})
 		head := cltest.Head(1000)
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once()
 		ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, nil).Once()
+		ht.ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(nil, errors.New("failed to connect")).Maybe()
 		ht.Start(t)
 		tests.AssertLogEventually(t, ht.observer, "Error handling initial head")
 	})
-	t.Run("Happy path", func(t *testing.T) {
+	t.Run("Logs error if finality gap is too big", func(t *testing.T) {
+		ht := newHeadTracker(t, opts{FinalityTagEnable: ptr(true), FinalityTagSupportDisabled: ptr(false), MaxAllowedFinalityDepth: ptr(uint32(10))})
+		head := cltest.Head(1000)
+		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once()
+		ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(cltest.Head(989), nil).Once()
+		ht.ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(nil, errors.New("failed to connect")).Maybe()
+		ht.Start(t)
+		tests.AssertEventually(t, func() bool {
+			// must exactly match the error passed to logger
+			field := zap.String("err", "failed to calculate latest finalized head: gap between latest finalized block (989) and current head (1000) is too large (> 10)")
+			filtered := ht.observer.FilterMessage("Error handling initial head").FilterField(field)
+			return filtered.Len() > 0
+		})
+	})
+	t.Run("Happy path (finality tag)", func(t *testing.T) {
 		head := cltest.Head(1000)
-		ht := newHeadTracker(t)
+		ht := newHeadTracker(t, opts{FinalityTagEnable: ptr(true), FinalityTagSupportDisabled: ptr(false)})
 		ctx := testutils.Context(t)
 		require.NoError(t, ht.orm.IdempotentInsertHead(ctx, cltest.Head(799)))
 		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once()
@@ -265,9 +296,46 @@ func TestHeadTracker_Start(t *testing.T) {
 		ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(finalizedHead, nil).Once()
 		// on backfill
 		ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New("backfill call to finalized failed")).Maybe()
+		ht.ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(nil, errors.New("failed to connect")).Maybe()
 		ht.Start(t)
 		tests.AssertLogEventually(t, ht.observer, "Loaded chain from DB")
 	})
+	happyPathFD := func(t *testing.T, opts opts) {
+		head := cltest.Head(1000)
+		ht := newHeadTracker(t, opts)
+		ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once()
+		finalizedHead := cltest.Head(head.Number - finalityDepth)
+		ht.ethClient.On("HeadByNumber", mock.Anything, big.NewInt(finalizedHead.Number)).Return(finalizedHead, nil).Once()
+		ctx := testutils.Context(t)
+		require.NoError(t, ht.orm.IdempotentInsertHead(ctx, cltest.Head(finalizedHead.Number-1)))
+		// on backfill
+		ht.ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(nil, errors.New("backfill call to finalized failed")).Maybe()
+		ht.ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything).Return(nil, errors.New("failed to connect")).Maybe()
+		ht.Start(t)
+		tests.AssertLogEventually(t, ht.observer, "Loaded chain from DB")
+	}
+	testCases := []struct {
+		Name string
+		Opts opts
+	}{
+		{
+			Name: "Happy path (Chain FT is disabled & HeadTracker's FT is disabled)",
+			Opts: opts{FinalityTagEnable: ptr(false), FinalityTagSupportDisabled: ptr(true)},
+		},
+		{
+			Name: "Happy path (Chain FT is disabled & HeadTracker's FT is enabled, but ignored)",
+			Opts: opts{FinalityTagEnable: ptr(false), FinalityTagSupportDisabled: ptr(false)},
+		},
+		{
+			Name: "Happy path (Chain FT is enabled & HeadTracker's FT is disabled)",
+			Opts: opts{FinalityTagEnable: ptr(true), FinalityTagSupportDisabled: ptr(true)},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			happyPathFD(t, tc.Opts)
+		})
+	}
 }
 
 func TestHeadTracker_CallsHeadTrackableCallbacks(t *testing.T) {

core/config/docs/chains-evm.toml

@@ -301,6 +301,14 @@ MaxBufferSize = 3 # Default
 # **ADVANCED**
 # SamplingInterval means that head tracker callbacks will at maximum be made once in every window of this duration. This is a performance optimisation for fast chains. Set to 0 to disable sampling entirely.
 SamplingInterval = '1s' # Default
+# FinalityTagSupportDisabled disables FinalityTag support in HeadTracker and makes it track blocks up to FinalityDepth from the most recent head.
+# It should only be used on chains with an extremely large actual finality depth (the number of blocks between the most recent head and the latest finalized block).
+# Has no effect if `FinalityTagsEnabled` = false
+FinalityTagSupportDisabled = true # Default
+# MaxAllowedFinalityDepth - defines maximum number of blocks between the most recent head and the latest finalized block.
+# If actual finality depth exceeds this number, HeadTracker aborts backfill and returns an error.
+# Has no effect if `FinalityTagsEnabled` = false
+MaxAllowedFinalityDepth = 10000 # Default
 
 [[EVM.KeySpecific]]
 # Key is the account to apply these settings to