Skip to content

Commit

Permalink
common/headtracker: improve health error (#13966)
Browse files Browse the repository at this point in the history
  • Loading branch information
jmank88 authored Aug 2, 2024
1 parent 82accff commit af7dab6
Show file tree
Hide file tree
Showing 4 changed files with 7 additions and 7 deletions.
8 changes: 4 additions & 4 deletions common/headtracker/head_tracker.go
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,7 @@ func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context
"blockDifficulty", head.BlockDifficulty(),
)

err := ht.headSaver.Save(ctx, head)
if ctx.Err() != nil {
if err := ht.headSaver.Save(ctx, head); ctx.Err() != nil {
return nil
} else if err != nil {
return fmt.Errorf("failed to save head: %#v: %w", head, err)
Expand All @@ -264,8 +263,9 @@ func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context

if prevLatestFinalized != nil && head.BlockNumber() <= prevLatestFinalized.BlockNumber() {
promOldHead.WithLabelValues(ht.chainID.String()).Inc()
ht.log.Criticalf("Got very old block with number %d (highest seen was %d). This is a problem and either means a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", head.BlockNumber(), prevHead.BlockNumber())
ht.SvcErrBuffer.Append(errors.New("got very old block"))
err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber())
ht.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err)
ht.SvcErrBuffer.Append(err)
}
}
return nil
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/reorg/automation_reorg_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ var (
)

var logScannerSettings = test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage(
"Got very old block with number",
"Got very old block.",
"It is expected, because we are causing reorgs",
zapcore.DPanicLevel,
testreporters.WarnAboutAllowedMsgs_No,
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/smoke/vrfv2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1063,7 +1063,7 @@ func TestVRFV2NodeReorg(t *testing.T) {

chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(
testreporters.NewAllowedLogMessage(
"This is a problem and either means a very deep re-org occurred",
"Got very old block.",
"Test is expecting a reorg to occur",
zapcore.DPanicLevel,
testreporters.WarnAboutAllowedMsgs_No),
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/smoke/vrfv2plus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1959,7 +1959,7 @@ func TestVRFv2PlusNodeReorg(t *testing.T) {
}
chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(
testreporters.NewAllowedLogMessage(
"This is a problem and either means a very deep re-org occurred",
"Got very old block.",
"Test is expecting a reorg to occur",
zapcore.DPanicLevel,
testreporters.WarnAboutAllowedMsgs_No),
Expand Down

0 comments on commit af7dab6

Please sign in to comment.