From e3e02d04d731d122f063b52ca5f8d9398b376967 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 13 Feb 2025 13:39:36 -0500 Subject: [PATCH 01/16] verify finalized block hashes --- chains/heads/tracker.go | 48 +++++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 640368f..2c67113 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -12,6 +12,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" + "github.com/smartcontractkit/chainlink-common/pkg/types" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox" "github.com/smartcontractkit/chainlink-framework/chains" @@ -36,7 +37,7 @@ const HeadsBufferSize = 10 type Tracker[H chains.Head[BLOCK_HASH], BLOCK_HASH chains.Hashable] interface { services.Service // Backfill given a head will fill in any missing heads up to latestFinalized - Backfill(ctx context.Context, headWithChain H) (err error) + Backfill(ctx context.Context, headWithChain H, prevHeadWithChain H) (err error) LatestChain() H // LatestAndFinalizedBlock - returns latest and latest finalized blocks. // NOTE: Returns latest finalized block as is, ignoring the FinalityTagBypass feature flag. @@ -59,6 +60,11 @@ type TrackerConfig interface { PersistenceEnabled() bool } +type headPair[HTH any] struct { + head HTH + prevHead HTH +} + type tracker[ HTH Head[BLOCK_HASH, ID], S chains.Subscription, @@ -77,7 +83,7 @@ type tracker[ config ChainConfig htConfig TrackerConfig - backfillMB *mailbox.Mailbox[HTH] + backfillMB *mailbox.Mailbox[headPair[HTH]] broadcastMB *mailbox.Mailbox[HTH] headListener Listener[HTH, BLOCK_HASH] getNilHead func() HTH @@ -105,7 +111,7 @@ func NewTracker[ chainID: client.ConfiguredChainID(), config: config, htConfig: htConfig, - backfillMB: mailbox.NewSingle[HTH](), + backfillMB: mailbox.NewSingle[headPair[HTH, ID, BLOCK_HASH]](), broadcastMB: mailbox.New[HTH](HeadsBufferSize), headSaver: headSaver, mailMon: mailMon, @@ -194,7 +200,30 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) close() error { return t.broadcastMB.Close() } -func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain HTH) (err error) { +// verifyFinalizedBlockHashes returns finality violated error if a mismatch is found in finalized block hashes +func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(latestFinalizedHeadWithChain HTH, prevHeadWithChain HTH) error { + if !t.config.FinalityTagEnabled() { + return nil // Bypass if using finality depth + } + + latestFinalizedBlockNum := latestFinalizedHeadWithChain.BlockNumber() + prevFinalizedBlockNum := prevHeadWithChain.LatestFinalizedHead().BlockNumber() + if latestFinalizedBlockNum < prevFinalizedBlockNum { + return fmt.Errorf( + "latest finalized block (%d) is behind previously seen finalized block (%d): %w", + latestFinalizedBlockNum, prevFinalizedBlockNum, types.ErrFinalityViolated, + ) + } + + for blockNum := prevFinalizedBlockNum; blockNum >= 0; blockNum-- { + if latestFinalizedHeadWithChain.HashAtHeight(blockNum) != prevHeadWithChain.HashAtHeight(blockNum) { + return fmt.Errorf("block hash mismatch at height %d: %w", blockNum, types.ErrFinalityViolated) + } + } + return nil +} + +func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain HTH, prevHeadWithChain HTH) (err error) { latestFinalized, err := t.calculateLatestFinalized(ctx, headWithChain, t.htConfig.FinalityTagBypass()) if err != nil { return fmt.Errorf("failed to calculate finalized block: %w", err) @@ -217,6 +246,10 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWith latestFinalized.BlockNumber(), headWithChain.BlockNumber(), t.htConfig.MaxAllowedFinalityDepth()) } + if err = t.verifyFinalizedBlockHashes(latestFinalized, prevHeadWithChain); err != nil { + return err + } + return t.backfill(ctx, headWithChain, latestFinalized) } @@ -248,7 +281,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea if !headWithChain.IsValid() { return fmt.Errorf("heads.tracker#handleNewHighestHead headWithChain was unexpectedly nil") } - t.backfillMB.Deliver(headWithChain) + t.backfillMB.Deliver(headPair[HTH]{headWithChain, prevHead}) t.broadcastMB.Deliver(headWithChain) } else if head.BlockNumber() == prevHead.BlockNumber() { if head.BlockHash() != prevHead.BlockHash() { @@ -265,6 +298,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) t.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) t.eng.EmitHealthErr(err) + return types.ErrFinalityViolated } } return nil @@ -314,12 +348,12 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) backfillLoop(ctx context.Context) { return case <-t.backfillMB.Notify(): for { - head, exists := t.backfillMB.Retrieve() + backfillHeadPair, exists := t.backfillMB.Retrieve() if !exists { break } { - err := t.Backfill(ctx, head) + err := t.Backfill(ctx, backfillHeadPair.head, backfillHeadPair.prevHead) if err != nil { t.log.Warnw("Unexpected error while backfilling heads", "err", err) } else if ctx.Err() != nil { From ace447f94ae5c55ef0ff15e150b719f9ce98008d Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 13 Feb 2025 13:44:15 -0500 Subject: [PATCH 02/16] Fix spacing --- chains/heads/tracker.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 2c67113..55ca142 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -209,10 +209,8 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(latestFinal latestFinalizedBlockNum := latestFinalizedHeadWithChain.BlockNumber() prevFinalizedBlockNum := prevHeadWithChain.LatestFinalizedHead().BlockNumber() if latestFinalizedBlockNum < prevFinalizedBlockNum { - return fmt.Errorf( - "latest finalized block (%d) is behind previously seen finalized block (%d): %w", - latestFinalizedBlockNum, prevFinalizedBlockNum, types.ErrFinalityViolated, - ) + return fmt.Errorf("latest finalized block (%d) is behind previously seen finalized block (%d): %w", + latestFinalizedBlockNum, prevFinalizedBlockNum, types.ErrFinalityViolated) } for blockNum := prevFinalizedBlockNum; blockNum >= 0; blockNum-- { From 1db72ce814b0d29f840b6142e61c0bdff8b58063 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 13 Feb 2025 13:48:54 -0500 Subject: [PATCH 03/16] lint --- chains/heads/tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 55ca142..5b7c1b5 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -111,7 +111,7 @@ func NewTracker[ chainID: client.ConfiguredChainID(), config: config, htConfig: htConfig, - backfillMB: mailbox.NewSingle[headPair[HTH, ID, BLOCK_HASH]](), + backfillMB: mailbox.NewSingle[headPair[HTH]](), broadcastMB: mailbox.New[HTH](HeadsBufferSize), headSaver: headSaver, mailMon: mailMon, From 9b33491fd8194983285e657b85881c05bff0e939 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 13 Feb 2025 14:40:02 -0500 Subject: [PATCH 04/16] Allow nil prevHeadWithChain --- chains/heads/tracker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 5b7c1b5..e653bc6 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -202,6 +202,9 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) close() error { // verifyFinalizedBlockHashes returns finality violated error if a mismatch is found in finalized block hashes func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(latestFinalizedHeadWithChain HTH, prevHeadWithChain HTH) error { + if prevHeadWithChain == nil { + return nil + } if !t.config.FinalityTagEnabled() { return nil // Bypass if using finality depth } From 1347e201aad4cc845e45124c2af881751bde2f08 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Thu, 13 Feb 2025 14:44:55 -0500 Subject: [PATCH 05/16] Update tracker.go --- chains/heads/tracker.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index e653bc6..5b7c1b5 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -202,9 +202,6 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) close() error { // verifyFinalizedBlockHashes returns finality violated error if a mismatch is found in finalized block hashes func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(latestFinalizedHeadWithChain HTH, prevHeadWithChain HTH) error { - if prevHeadWithChain == nil { - return nil - } if !t.config.FinalityTagEnabled() { return nil // Bypass if using finality depth } From 8c7c0dded51e5f7c22714aeeb87c25d0b766d7ef Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 09:59:33 -0500 Subject: [PATCH 06/16] Check prevLatestFinalized is not nil --- chains/heads/tracker.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 5b7c1b5..9599b51 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -206,8 +206,13 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(latestFinal return nil // Bypass if using finality depth } + prevLatestFinalized := prevHeadWithChain.LatestFinalizedHead() + if prevLatestFinalized == nil { + return nil + } + latestFinalizedBlockNum := latestFinalizedHeadWithChain.BlockNumber() - prevFinalizedBlockNum := prevHeadWithChain.LatestFinalizedHead().BlockNumber() + prevFinalizedBlockNum := prevLatestFinalized.BlockNumber() if latestFinalizedBlockNum < prevFinalizedBlockNum { return fmt.Errorf("latest finalized block (%d) is behind previously seen finalized block (%d): %w", latestFinalizedBlockNum, prevFinalizedBlockNum, types.ErrFinalityViolated) From 9bb0cdd5f8c41eb46c0debf369e469ee505f7258 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 12:43:00 -0500 Subject: [PATCH 07/16] Update tracker.go --- chains/heads/tracker.go | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 9599b51..b54ae61 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -200,26 +200,13 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) close() error { return t.broadcastMB.Close() } -// verifyFinalizedBlockHashes returns finality violated error if a mismatch is found in finalized block hashes -func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(latestFinalizedHeadWithChain HTH, prevHeadWithChain HTH) error { - if !t.config.FinalityTagEnabled() { - return nil // Bypass if using finality depth - } - - prevLatestFinalized := prevHeadWithChain.LatestFinalizedHead() - if prevLatestFinalized == nil { - return nil - } - - latestFinalizedBlockNum := latestFinalizedHeadWithChain.BlockNumber() - prevFinalizedBlockNum := prevLatestFinalized.BlockNumber() - if latestFinalizedBlockNum < prevFinalizedBlockNum { - return fmt.Errorf("latest finalized block (%d) is behind previously seen finalized block (%d): %w", - latestFinalizedBlockNum, prevFinalizedBlockNum, types.ErrFinalityViolated) - } - - for blockNum := prevFinalizedBlockNum; blockNum >= 0; blockNum-- { - if latestFinalizedHeadWithChain.HashAtHeight(blockNum) != prevHeadWithChain.HashAtHeight(blockNum) { +// verifyBlockHashes returns finality violated error if a block hash mismatch is found in provided chains +func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyBlockHashes(headWithChain chains.Head[BLOCK_HASH], prevHeadWithChain chains.Head[BLOCK_HASH]) error { + // Verify hashes from previous finalized chain until reaching the corresponding chain length + prevBlockNum := prevHeadWithChain.BlockNumber() + chainLength := int64(prevHeadWithChain.ChainLength()) + for blockNum := prevBlockNum; blockNum > prevBlockNum-chainLength; blockNum-- { + if headWithChain.HashAtHeight(blockNum) != prevHeadWithChain.HashAtHeight(blockNum) { return fmt.Errorf("block hash mismatch at height %d: %w", blockNum, types.ErrFinalityViolated) } } @@ -249,7 +236,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWith latestFinalized.BlockNumber(), headWithChain.BlockNumber(), t.htConfig.MaxAllowedFinalityDepth()) } - if err = t.verifyFinalizedBlockHashes(latestFinalized, prevHeadWithChain); err != nil { + if err = t.verifyBlockHashes(latestFinalized, prevHeadWithChain.LatestFinalizedHead()); err != nil { return err } @@ -301,7 +288,15 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) t.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) t.eng.EmitHealthErr(err) - return types.ErrFinalityViolated + if prevLatestFinalized.BlockNumber() == head.BlockNumber() { + err = t.verifyBlockHashes(head, prevLatestFinalized) + if err == nil { + return nil // Hashes match, no need to return finality violation error + } + } + finalityErr := fmt.Errorf("%w: %w", types.ErrFinalityViolated, err) + t.eng.EmitHealthErr(finalityErr) + return finalityErr } } return nil From 81b0b5ceedb466b61262eb20649aba8f7147b023 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 12:50:40 -0500 Subject: [PATCH 08/16] Update tracker.go --- chains/heads/tracker.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index b54ae61..1a2396c 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -202,6 +202,10 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) close() error { // verifyBlockHashes returns finality violated error if a block hash mismatch is found in provided chains func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyBlockHashes(headWithChain chains.Head[BLOCK_HASH], prevHeadWithChain chains.Head[BLOCK_HASH]) error { + if prevHeadWithChain == nil { + return nil + } + // Verify hashes from previous finalized chain until reaching the corresponding chain length prevBlockNum := prevHeadWithChain.BlockNumber() chainLength := int64(prevHeadWithChain.ChainLength()) From a6816292b13efb00d20fb06d2d783b196eb5f95b Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 13:31:03 -0500 Subject: [PATCH 09/16] Add instant finality check --- chains/heads/tracker.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 1a2396c..2c87311 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -217,6 +217,10 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyBlockHashes(headWithChain chains return nil } +func (t *tracker[HTH, S, ID, BLOCK_HASH]) isInstantFinality() bool { + return !t.config.FinalityTagEnabled() && t.config.FinalityDepth() == 0 && t.config.FinalizedBlockOffset() == 0 +} + func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain HTH, prevHeadWithChain HTH) (err error) { latestFinalized, err := t.calculateLatestFinalized(ctx, headWithChain, t.htConfig.FinalityTagBypass()) if err != nil { @@ -240,8 +244,12 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWith latestFinalized.BlockNumber(), headWithChain.BlockNumber(), t.htConfig.MaxAllowedFinalityDepth()) } - if err = t.verifyBlockHashes(latestFinalized, prevHeadWithChain.LatestFinalizedHead()); err != nil { - return err + if !t.isInstantFinality() { + // verify block hashes since calculateLatestFinalized made an additional RPC call + err = t.verifyBlockHashes(latestFinalized, prevHeadWithChain.LatestFinalizedHead()) + if err != nil { + return err + } } return t.backfill(ctx, headWithChain, latestFinalized) @@ -435,7 +443,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.C return t.getHeadAtHeight(ctx, latestFinalized.BlockHash(), finalizedBlockNumber) } // no need to make an additional RPC call on chains with instant finality - if t.config.FinalityDepth() == 0 && t.config.FinalizedBlockOffset() == 0 { + if t.isInstantFinality() { return currentHead, nil } finalizedBlockNumber := currentHead.BlockNumber() - int64(t.config.FinalityDepth()) - int64(t.config.FinalizedBlockOffset()) From 2eddef0951cdd5b225c1cbba46233adea4497b41 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 13:33:41 -0500 Subject: [PATCH 10/16] Update tracker.go --- chains/heads/tracker.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 2c87311..c0f554d 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -217,7 +217,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyBlockHashes(headWithChain chains return nil } -func (t *tracker[HTH, S, ID, BLOCK_HASH]) isInstantFinality() bool { +func (t *tracker[HTH, S, ID, BLOCK_HASH]) instantFinality() bool { return !t.config.FinalityTagEnabled() && t.config.FinalityDepth() == 0 && t.config.FinalizedBlockOffset() == 0 } @@ -244,7 +244,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWith latestFinalized.BlockNumber(), headWithChain.BlockNumber(), t.htConfig.MaxAllowedFinalityDepth()) } - if !t.isInstantFinality() { + if !t.instantFinality() { // verify block hashes since calculateLatestFinalized made an additional RPC call err = t.verifyBlockHashes(latestFinalized, prevHeadWithChain.LatestFinalizedHead()) if err != nil { @@ -443,7 +443,7 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.C return t.getHeadAtHeight(ctx, latestFinalized.BlockHash(), finalizedBlockNumber) } // no need to make an additional RPC call on chains with instant finality - if t.isInstantFinality() { + if t.instantFinality() { return currentHead, nil } finalizedBlockNumber := currentHead.BlockNumber() - int64(t.config.FinalityDepth()) - int64(t.config.FinalizedBlockOffset()) From 211dbd7170843576a8998e9afa720fa5e8ae70c3 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 15:13:29 -0500 Subject: [PATCH 11/16] Update finality error handling --- chains/heads/tracker.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index c0f554d..d277748 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -300,15 +300,17 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) t.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) t.eng.EmitHealthErr(err) + + var finalityErr error if prevLatestFinalized.BlockNumber() == head.BlockNumber() { - err = t.verifyBlockHashes(head, prevLatestFinalized) - if err == nil { - return nil // Hashes match, no need to return finality violation error - } + finalityErr = t.verifyBlockHashes(head, prevLatestFinalized) + } else { + finalityErr = fmt.Errorf("latest finalized block is behind previously seen finalized block: %w", types.ErrFinalityViolated) + } + if finalityErr != nil { + t.eng.EmitHealthErr(finalityErr) + return finalityErr } - finalityErr := fmt.Errorf("%w: %w", types.ErrFinalityViolated, err) - t.eng.EmitHealthErr(finalityErr) - return finalityErr } } return nil From a8fe13628103fbe3865e55f6f43a54c7b47f7349 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 19 Feb 2025 15:29:17 -0500 Subject: [PATCH 12/16] Emit finality health error --- chains/heads/tracker.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index d277748..13f7863 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -299,7 +299,6 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea promOldHead.WithLabelValues(t.chainID.String()).Inc() err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) t.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) - t.eng.EmitHealthErr(err) var finalityErr error if prevLatestFinalized.BlockNumber() == head.BlockNumber() { @@ -308,9 +307,11 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea finalityErr = fmt.Errorf("latest finalized block is behind previously seen finalized block: %w", types.ErrFinalityViolated) } if finalityErr != nil { - t.eng.EmitHealthErr(finalityErr) - return finalityErr + err = fmt.Errorf("%w: %w", finalityErr, err) + t.eng.EmitHealthErr(err) + return err } + t.eng.EmitHealthErr(err) } } return nil From 8f059e2ba6cd59c4ac93fd24544916a2bd6b3827 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 21 Feb 2025 11:19:11 -0500 Subject: [PATCH 13/16] Verify hashes before saving head --- chains/heads/tracker.go | 60 ++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 13f7863..0ab85cc 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -200,19 +200,20 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) close() error { return t.broadcastMB.Close() } -// verifyBlockHashes returns finality violated error if a block hash mismatch is found in provided chains -func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyBlockHashes(headWithChain chains.Head[BLOCK_HASH], prevHeadWithChain chains.Head[BLOCK_HASH]) error { +// verifyFinalizedBlockHashes returns finality violated error if a block hash mismatch is found in provided chains +func (t *tracker[HTH, S, ID, BLOCK_HASH]) verifyFinalizedBlockHashes(finalizedHeadWithChain chains.Head[BLOCK_HASH], prevHeadWithChain chains.Head[BLOCK_HASH]) error { if prevHeadWithChain == nil { return nil } - // Verify hashes from previous finalized chain until reaching the corresponding chain length - prevBlockNum := prevHeadWithChain.BlockNumber() - chainLength := int64(prevHeadWithChain.ChainLength()) - for blockNum := prevBlockNum; blockNum > prevBlockNum-chainLength; blockNum-- { - if headWithChain.HashAtHeight(blockNum) != prevHeadWithChain.HashAtHeight(blockNum) { - return fmt.Errorf("block hash mismatch at height %d: %w", blockNum, types.ErrFinalityViolated) - } + prevLatestFinalized := prevHeadWithChain.LatestFinalizedHead() + if prevLatestFinalized == nil { + return nil + } + + prevLatestFinalizedBlockNum := prevLatestFinalized.BlockNumber() + if finalizedHeadWithChain.HashAtHeight(prevLatestFinalizedBlockNum) != prevLatestFinalized.BlockHash() { + return fmt.Errorf("block hash mismatch at height %d: %w", prevLatestFinalizedBlockNum, types.ErrFinalityViolated) } return nil } @@ -246,8 +247,9 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWith if !t.instantFinality() { // verify block hashes since calculateLatestFinalized made an additional RPC call - err = t.verifyBlockHashes(latestFinalized, prevHeadWithChain.LatestFinalizedHead()) + err = t.verifyFinalizedBlockHashes(latestFinalized, prevHeadWithChain.LatestFinalizedHead()) if err != nil { + t.eng.EmitHealthErr(err) return err } } @@ -270,6 +272,17 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea "blockDifficulty", head.BlockDifficulty(), ) + if err := t.verifyFinalizedBlockHashes(head, prevHead); err != nil { + if head.BlockNumber() < prevHead.LatestFinalizedHead().BlockNumber() { + promOldHead.WithLabelValues(t.chainID.String()).Inc() + t.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) + oldBlockErr := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) + err = fmt.Errorf("%w: %w", oldBlockErr, err) + } + t.eng.EmitHealthErr(err) + return err + } + if err := t.headSaver.Save(ctx, head); ctx.Err() != nil { return nil } else if err != nil { @@ -292,26 +305,19 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea t.log.Debugw("Head already in the database", "head", head.BlockHash()) } } else { - t.log.Debugw("Got out of order head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockNumber()) prevLatestFinalized := prevHead.LatestFinalizedHead() + if prevLatestFinalized == nil { + return nil + } - if prevLatestFinalized != nil && head.BlockNumber() <= prevLatestFinalized.BlockNumber() { - promOldHead.WithLabelValues(t.chainID.String()).Inc() - err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) - t.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) - - var finalityErr error - if prevLatestFinalized.BlockNumber() == head.BlockNumber() { - finalityErr = t.verifyBlockHashes(head, prevLatestFinalized) - } else { - finalityErr = fmt.Errorf("latest finalized block is behind previously seen finalized block: %w", types.ErrFinalityViolated) - } - if finalityErr != nil { - err = fmt.Errorf("%w: %w", finalityErr, err) - t.eng.EmitHealthErr(err) - return err - } + t.log.Debugw("Got out of order head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockNumber()) + promOldHead.WithLabelValues(t.chainID.String()).Inc() + if head.BlockNumber() != prevLatestFinalized.BlockNumber() { + // This should never happen since we verified finalized block hashes before saving the head + err := fmt.Errorf("head is behind previously seen latest finalized head: %w", types.ErrFinalityViolated) + t.log.Critical(err) t.eng.EmitHealthErr(err) + return err } } return nil From d89b1815981ef350f9d253feea59f3fb6264cfe1 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 21 Feb 2025 13:21:59 -0500 Subject: [PATCH 14/16] Handle re-org pre finality --- chains/heads/tracker.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 0ab85cc..1e2e6db 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -313,7 +313,11 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea t.log.Debugw("Got out of order head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockNumber()) promOldHead.WithLabelValues(t.chainID.String()).Inc() if head.BlockNumber() != prevLatestFinalized.BlockNumber() { - // This should never happen since we verified finalized block hashes before saving the head + if !t.config.FinalityTagEnabled() && prevLatestFinalized.BlockNumber() < int64(t.config.FinalityDepth()) { + return nil // Re-org occurred before reaching finality depth + } + + // This should never happen since we verified block hashes before storing new head err := fmt.Errorf("head is behind previously seen latest finalized head: %w", types.ErrFinalityViolated) t.log.Critical(err) t.eng.EmitHealthErr(err) From 7878e11ceeb64e03aa43a6fd3ee9989c87c75bf9 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 21 Feb 2025 13:29:34 -0500 Subject: [PATCH 15/16] Update tracker.go --- chains/heads/tracker.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 1e2e6db..496d169 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -309,20 +309,8 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea if prevLatestFinalized == nil { return nil } - t.log.Debugw("Got out of order head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockNumber()) promOldHead.WithLabelValues(t.chainID.String()).Inc() - if head.BlockNumber() != prevLatestFinalized.BlockNumber() { - if !t.config.FinalityTagEnabled() && prevLatestFinalized.BlockNumber() < int64(t.config.FinalityDepth()) { - return nil // Re-org occurred before reaching finality depth - } - - // This should never happen since we verified block hashes before storing new head - err := fmt.Errorf("head is behind previously seen latest finalized head: %w", types.ErrFinalityViolated) - t.log.Critical(err) - t.eng.EmitHealthErr(err) - return err - } } return nil } From 28742cefdd20e98b5c49a567abebc6213c5f3056 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 21 Feb 2025 13:39:12 -0500 Subject: [PATCH 16/16] ignore out of order heads --- chains/heads/tracker.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/chains/heads/tracker.go b/chains/heads/tracker.go index 496d169..339bf60 100644 --- a/chains/heads/tracker.go +++ b/chains/heads/tracker.go @@ -305,10 +305,6 @@ func (t *tracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, hea t.log.Debugw("Head already in the database", "head", head.BlockHash()) } } else { - prevLatestFinalized := prevHead.LatestFinalizedHead() - if prevLatestFinalized == nil { - return nil - } t.log.Debugw("Got out of order head", "blockNum", head.BlockNumber(), "head", head.BlockHash(), "prevHead", prevHead.BlockNumber()) promOldHead.WithLabelValues(t.chainID.String()).Inc() }