From af7dab653b10d8c201a2e6a8b3a587e43abd220c Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Fri, 2 Aug 2024 19:06:45 +0200 Subject: [PATCH] common/headtracker: improve health error (#13966) --- common/headtracker/head_tracker.go | 8 ++++---- integration-tests/reorg/automation_reorg_test.go | 2 +- integration-tests/smoke/vrfv2_test.go | 2 +- integration-tests/smoke/vrfv2plus_test.go | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index afa5d931ee6..851458591b8 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -236,8 +236,7 @@ func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context "blockDifficulty", head.BlockDifficulty(), ) - err := ht.headSaver.Save(ctx, head) - if ctx.Err() != nil { + if err := ht.headSaver.Save(ctx, head); ctx.Err() != nil { return nil } else if err != nil { return fmt.Errorf("failed to save head: %#v: %w", head, err) @@ -264,8 +263,9 @@ func (ht *headTracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context if prevLatestFinalized != nil && head.BlockNumber() <= prevLatestFinalized.BlockNumber() { promOldHead.WithLabelValues(ht.chainID.String()).Inc() - ht.log.Criticalf("Got very old block with number %d (highest seen was %d). This is a problem and either means a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", head.BlockNumber(), prevHead.BlockNumber()) - ht.SvcErrBuffer.Append(errors.New("got very old block")) + err := fmt.Errorf("got very old block with number %d (highest seen was %d)", head.BlockNumber(), prevHead.BlockNumber()) + ht.log.Critical("Got very old block. Either a very deep re-org occurred, one of the RPC nodes has gotten far out of sync, or the chain went backwards in block numbers. This node may not function correctly without manual intervention.", "err", err) + ht.SvcErrBuffer.Append(err) } } return nil diff --git a/integration-tests/reorg/automation_reorg_test.go b/integration-tests/reorg/automation_reorg_test.go index 1b9cf5819b9..808e394d69b 100644 --- a/integration-tests/reorg/automation_reorg_test.go +++ b/integration-tests/reorg/automation_reorg_test.go @@ -43,7 +43,7 @@ var ( ) var logScannerSettings = test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages(testreporters.NewAllowedLogMessage( - "Got very old block with number", + "Got very old block.", "It is expected, because we are causing reorgs", zapcore.DPanicLevel, testreporters.WarnAboutAllowedMsgs_No, diff --git a/integration-tests/smoke/vrfv2_test.go b/integration-tests/smoke/vrfv2_test.go index 7a53d2c57c8..48fbc0071c5 100644 --- a/integration-tests/smoke/vrfv2_test.go +++ b/integration-tests/smoke/vrfv2_test.go @@ -1063,7 +1063,7 @@ func TestVRFV2NodeReorg(t *testing.T) { chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages( testreporters.NewAllowedLogMessage( - "This is a problem and either means a very deep re-org occurred", + "Got very old block.", "Test is expecting a reorg to occur", zapcore.DPanicLevel, testreporters.WarnAboutAllowedMsgs_No), diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index f519aa6cd5f..da2989d8fc9 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -1959,7 +1959,7 @@ func TestVRFv2PlusNodeReorg(t *testing.T) { } chainlinkNodeLogScannerSettings := test_env.GetDefaultChainlinkNodeLogScannerSettingsWithExtraAllowedMessages( testreporters.NewAllowedLogMessage( - "This is a problem and either means a very deep re-org occurred", + "Got very old block.", "Test is expecting a reorg to occur", zapcore.DPanicLevel, testreporters.WarnAboutAllowedMsgs_No),