Head Tracker Finality Violation Detection

evm/go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/prometheus/client_model v0.6.1
 	github.com/shopspring/decimal v1.4.0
 	github.com/smartcontractkit/chainlink-common v0.4.2-0.20250130202959-6f1f48342e36
-	github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20250207205350-420ccacab78a
+	github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20250221183912-28742cefdd20
 	github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250211162441-3d6cea220efb
 	github.com/stretchr/testify v1.10.0
 	github.com/tidwall/gjson v1.18.0

evm/go.sum

@@ -555,8 +555,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartcontractkit/chainlink-common v0.4.2-0.20250130202959-6f1f48342e36 h1:bS51NFGHVjkCy7yu9L2Ss4sBsCW6jpa5GuhRAdWWxzM=
 github.com/smartcontractkit/chainlink-common v0.4.2-0.20250130202959-6f1f48342e36/go.mod h1:Z2e1ynSJ4pg83b4Qldbmryc5lmnrI3ojOdg1FUloa68=
-github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20250207205350-420ccacab78a h1:zllQ6pOs1T0oiDNK3EHr7ABy1zHp+2oxoCuVE/hK+uI=
-github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20250207205350-420ccacab78a/go.mod h1:tHem58EihQh63kR2LlAOKDAs9Vbghf1dJKZRGy6LG8g=
+github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20250221183912-28742cefdd20 h1:GhXBLadkf/MhLjvwCPEfjpEN7bP+Kd6sAhYdKrp7hpk=
+github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20250221183912-28742cefdd20/go.mod h1:tHem58EihQh63kR2LlAOKDAs9Vbghf1dJKZRGy6LG8g=
 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250211162441-3d6cea220efb h1:LWijSyJ2lhppkFLN19EGsLHZXQ5wen2DEk1cyR0tV+o=
 github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20250211162441-3d6cea220efb/go.mod h1:4JqpgFy01LaqG1yM2iFTzwX3ZgcAvW9WdstBZQgPHzU=
 github.com/smartcontractkit/libocr v0.0.0-20241223215956-e5b78d8e3919 h1:IpGoPTXpvllN38kT2z2j13sifJMz4nbHglidvop7mfg=

evm/heads/broadcaster_test.go

@@ -60,7 +60,9 @@ func TestHeadBroadcaster_Subscribe(t *testing.T) {
 			chchHeaders <- chHead
 		}).
 		Return((<-chan *evmtypes.Head)(chHead), sub, nil)
-	ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(testutils.Head(1), nil)
+
+	h := testutils.Head(1)
+	ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(h, nil)
 
 	sub.On("Unsubscribe").Return()
 	sub.On("Err").Return(nil)
@@ -82,8 +84,7 @@ func TestHeadBroadcaster_Subscribe(t *testing.T) {
 	assert.Equal(t, (*evmtypes.Head)(nil), latest1)
 
 	headers := <-chchHeaders
-	h := evmtypes.Head{Number: 1, Hash: utils.NewHash(), ParentHash: utils.NewHash(), EVMChainID: big.New(testutils.FixtureChainID)}
-	headers <- &h
+	headers <- h
 	g.Eventually(checker1.OnNewLongestChainCount).Should(gomega.Equal(int32(1)))
 
 	latest2, _ := hb.Subscribe(checker2)
@@ -93,7 +94,9 @@ func TestHeadBroadcaster_Subscribe(t *testing.T) {
 
 	unsubscribe1()
 
-	headers <- &evmtypes.Head{Number: 2, Hash: utils.NewHash(), ParentHash: h.Hash, EVMChainID: big.New(testutils.FixtureChainID)}
+	h2 := &evmtypes.Head{Number: 2, Hash: utils.NewHash(), ParentHash: h.Hash, EVMChainID: big.New(testutils.FixtureChainID)}
+	h2.Parent.Store(h)
+	headers <- h2
 	g.Eventually(checker2.OnNewLongestChainCount).Should(gomega.Equal(int32(1)))
 }
 

evm/heads/tracker.go

@@ -44,7 +44,7 @@ func (*nullTracker) Ready() error                   { return nil }
 func (*nullTracker) HealthReport() map[string]error { return map[string]error{} }
 func (*nullTracker) Name() string                   { return "" }
 func (*nullTracker) SetLogLevel(zapcore.Level)      {}
-func (*nullTracker) Backfill(ctx context.Context, headWithChain *evmtypes.Head) (err error) {
+func (*nullTracker) Backfill(ctx context.Context, headWithChain *evmtypes.Head, prevHeadWithChain *evmtypes.Head) (err error) {
 	return nil
 }
 func (*nullTracker) LatestChain() *evmtypes.Head { return nil }

evm/heads/tracker_test.go

@@ -24,6 +24,7 @@ import (
 	commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink-common/pkg/services"
+	"github.com/smartcontractkit/chainlink-common/pkg/types"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox/mailboxtest"
 	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
@@ -205,6 +206,18 @@ func TestHeadTracker_Start_NewHeads(t *testing.T) {
 	ht.Start(t)
 
 	<-chStarted
+
+	t.Run("Finality violation", func(t *testing.T) {
+		ch <- testutils.Head(1) // Deliver head with finalized block hash mismatch
+
+		g := gomega.NewWithT(t)
+		g.Eventually(func() bool {
+			report := ht.headTracker.HealthReport()
+			return slices.ContainsFunc(maps.Values(report), func(e error) bool {
+				return errors.Is(e, types.ErrFinalityViolated)
+			})
+		}, 5*time.Second, tests.TestInterval).Should(gomega.BeTrue())
+	})
 }
 
 func TestHeadTracker_Start(t *testing.T) {
@@ -892,35 +905,35 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		const expectedError = "failed to fetch latest finalized block"
 		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New(expectedError)).Once()
 
-		err := htu.headTracker.Backfill(ctx, h12)
+		err := htu.Backfill(ctx, h12)
 		require.ErrorContains(t, err, expectedError)
 	})
 	t.Run("returns error if latestFinalized is not valid", func(t *testing.T) {
 		htu := newHeadTrackerUniverse(t, opts{FinalityTagEnabled: true})
 		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, nil).Once()
 
-		err := htu.headTracker.Backfill(ctx, h12)
+		err := htu.Backfill(ctx, h12)
 		require.EqualError(t, err, "failed to calculate finalized block: failed to get valid latest finalized block")
 	})
 	t.Run("Returns error if finality gap is too big", func(t *testing.T) {
 		htu := newHeadTrackerUniverse(t, opts{FinalityTagEnabled: true, MaxAllowedFinalityDepth: 2})
 		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(h9, nil).Once()
 
-		err := htu.headTracker.Backfill(ctx, h12)
+		err := htu.Backfill(ctx, h12)
 		require.EqualError(t, err, "gap between latest finalized block (9) and current head (12) is too large (> 2)")
 	})
 	t.Run("Returns error if finalized head is ahead of canonical", func(t *testing.T) {
 		htu := newHeadTrackerUniverse(t, opts{FinalityTagEnabled: true})
 		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(h14Orphaned, nil).Once()
 
-		err := htu.headTracker.Backfill(ctx, h12)
+		err := htu.Backfill(ctx, h12)
 		require.EqualError(t, err, "invariant violation: expected head of canonical chain to be ahead of the latestFinalized")
 	})
 	t.Run("Returns error if finalizedHead is not present in the canonical chain", func(t *testing.T) {
 		htu := newHeadTrackerUniverse(t, opts{Heads: hs, FinalityTagEnabled: true})
-		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(h14Orphaned, nil).Once()
+		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(h14Orphaned, nil)
 
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.ErrorAs(t, err, &heads.FinalizedMissingError[common.Hash]{})
 	})
 	t.Run("Marks all blocks in chain that are older than finalized", func(t *testing.T) {
@@ -934,7 +947,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		}
 
 		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(h14, nil).Once()
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.NoError(t, err)
 		assertFinalized(true, "expected heads to be marked as finalized after backfill", h14, h13, h12, h11)
 		assertFinalized(false, "expected heads to remain unfinalized", h15, &head10)
@@ -946,7 +959,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash).
 			Return(&head10, nil)
 
-		err := htu.headTracker.Backfill(ctx, h12)
+		err := htu.Backfill(ctx, h12)
 		require.NoError(t, err)
 
 		h := htu.headSaver.Chain(h12.Hash)
@@ -970,7 +983,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		htu.ethClient.On("HeadByHash", mock.Anything, head8.Hash).
 			Return(&head8, nil)
 
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.NoError(t, err)
 
 		h := htu.headSaver.Chain(h15.Hash)
@@ -991,7 +1004,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 			Return(nil, ethereum.NotFound).
 			Once()
 
-		err := htu.headTracker.Backfill(ctx, h12)
+		err := htu.Backfill(ctx, h12)
 		require.Error(t, err)
 		require.ErrorContains(t, err, "fetchAndSaveHead failed: not found")
 
@@ -1013,7 +1026,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 			cancel()
 		})
 
-		err := htu.headTracker.Backfill(lctx, h12)
+		err := htu.headTracker.Backfill(lctx, h12, nil)
 		require.Error(t, err)
 		require.ErrorContains(t, err, "fetchAndSaveHead failed: context canceled")
 
@@ -1030,7 +1043,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		htu.ethClient.On("HeadByHash", mock.Anything, h13.Hash).Return(h13, nil).Once()
 		htu.ethClient.On("HeadByHash", mock.Anything, h12.Hash).Return(nil, errors.New("not found")).Once()
 
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 
 		require.Error(t, err)
 		require.ErrorContains(t, err, "fetchAndSaveHead failed: not found")
@@ -1045,7 +1058,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		htu := newHeadTrackerUniverse(t, opts{Heads: []*evmtypes.Head{h15}, FinalityTagEnabled: true})
 		finalizedH15 := h15 // copy h15 to have different addresses
 		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(finalizedH15, nil).Once()
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.NoError(t, err)
 
 		h := htu.headSaver.LatestChain()
@@ -1065,7 +1078,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		htu.ethClient.On("HeadByHash", mock.Anything, h12.Hash).Return(h12, nil).Once()
 		htu.ethClient.On("HeadByHash", mock.Anything, h13.Hash).Return(h13, nil).Once()
 		htu.ethClient.On("HeadByHash", mock.Anything, h14.Hash).Return(h14, nil).Once()
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.NoError(t, err)
 
 		h := htu.headSaver.LatestChain()
@@ -1087,7 +1100,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		htu.ethClient.On("HeadByHash", mock.Anything, h14.Hash).Return(h14, nil).Once()
 		htu.ethClient.On("HeadByHash", mock.Anything, h13.Hash).Return(h13, nil).Once()
 		htu.ethClient.On("HeadByHash", mock.Anything, h12.Hash).Return(h12, nil).Once()
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.NoError(t, err)
 
 		h := htu.headSaver.LatestChain()
@@ -1108,7 +1121,7 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		// backfill from 15 to 13
 		htu.ethClient.On("HeadByHash", mock.Anything, h14.Hash).Return(h14, nil).Once()
 		htu.ethClient.On("HeadByHash", mock.Anything, h13.Hash).Return(h13, nil).Once()
-		err := htu.headTracker.Backfill(ctx, h15)
+		err := htu.Backfill(ctx, h15)
 		require.NoError(t, err)
 
 		h := htu.headSaver.LatestChain()
@@ -1122,6 +1135,35 @@ func testHeadTrackerBackfill(t *testing.T, newORM func(t *testing.T) evmheads.OR
 		assert.Equal(t, h13.BlockNumber(), h.BlockNumber())
 		assert.Equal(t, h13.Hash, h.Hash)
 	})
+
+	t.Run("finality violation error on finalized block hash mismatch", func(t *testing.T) {
+		htu := newHeadTrackerUniverse(t, opts{Heads: []*evmtypes.Head{h15}, FinalityTagEnabled: true, FinalizedBlockOffset: 2})
+		htu.ethClient.On("HeadByNumber", mock.Anything, big.NewInt(12)).Return(h12, nil).Maybe()
+		htu.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(h14, nil)
+		htu.Start(t)
+
+		// Invalid chain with block mismatch
+		invalid11 := testutils.Head(11)
+		invalid11.IsFinalized.Store(true)
+		invalid11.Parent.Store(h1) // Mismatch with incorrect parent
+		invalid11.ParentHash = h1.Hash
+
+		invalid12 := testutils.Head(12)
+		invalid12.Hash = h12.Hash // Use hash from valid head
+		invalid12.Parent.Store(invalid11)
+		invalid12.ParentHash = invalid11.Hash
+
+		err := htu.headTracker.Backfill(ctx, h12, invalid12)
+		require.ErrorIs(t, err, types.ErrFinalityViolated)
+
+		g := gomega.NewWithT(t)
+		g.Eventually(func() bool {
+			report := htu.headTracker.HealthReport()
+			return slices.ContainsFunc(maps.Values(report), func(e error) bool {
+				return errors.Is(e, types.ErrFinalityViolated)
+			})
+		}, 5*time.Second, tests.TestInterval).Should(gomega.BeTrue())
+	})
 }
 
 func TestHeadTracker_LatestAndFinalizedBlock(t *testing.T) {
@@ -1308,7 +1350,7 @@ type headTrackerUniverse struct {
 }
 
 func (u *headTrackerUniverse) Backfill(ctx context.Context, head *evmtypes.Head) error {
-	return u.headTracker.Backfill(ctx, head)
+	return u.headTracker.Backfill(ctx, head, head) // Passing head as prevHead should always verify hashes correctly
 }
 
 func (u *headTrackerUniverse) Start(t *testing.T) {