From af480ab1162510c11a470c2c2770f61ae7cedba0 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Mon, 19 Feb 2024 17:37:19 +0100 Subject: [PATCH 01/28] Add LatestFinalizedBlock to HeadTracker --- common/client/mock_rpc_test.go | 28 ++ common/client/multi_node.go | 9 + common/client/types.go | 2 + common/headtracker/head_tracker.go | 58 ++++- common/headtracker/types/client.go | 3 + common/headtracker/types/config.go | 1 + common/mocks/head_tracker.go | 10 +- common/types/head_tracker.go | 8 +- core/chains/evm/client/chain_client.go | 4 + core/chains/evm/client/client.go | 5 + core/chains/evm/client/mocks/client.go | 30 +++ core/chains/evm/client/null_client.go | 5 + core/chains/evm/client/rpc_client.go | 10 +- .../evm/client/simulated_backend_client.go | 11 + core/chains/evm/headtracker/config.go | 1 + .../evm/headtracker/head_broadcaster_test.go | 5 +- core/chains/evm/headtracker/head_saver.go | 21 +- .../chains/evm/headtracker/head_saver_test.go | 6 + core/chains/evm/headtracker/head_tracker.go | 2 +- .../evm/headtracker/head_tracker_test.go | 244 ++++++++---------- core/chains/evm/headtracker/heads.go | 38 ++- core/chains/evm/headtracker/heads_test.go | 48 ++++ core/chains/evm/headtracker/mocks/config.go | 18 ++ core/chains/evm/types/models.go | 2 + core/internal/cltest/cltest.go | 3 +- core/internal/testutils/evmtest/evmtest.go | 2 +- 26 files changed, 401 insertions(+), 173 deletions(-) diff --git a/common/client/mock_rpc_test.go b/common/client/mock_rpc_test.go index d87a02d47c1..7be4d580029 100644 --- a/common/client/mock_rpc_test.go +++ b/common/client/mock_rpc_test.go @@ -426,6 +426,34 @@ func (_m *mockRPC[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS return r0, r1 } +// LatestFinalizedBlock provides a mock function with given fields: ctx +func (_m *mockRPC[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD]) LatestFinalizedBlock(ctx context.Context) (HEAD, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for LatestFinalizedBlock") + } + + var r0 HEAD + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (HEAD, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) HEAD); ok { + r0 = rf(ctx) + } else { + r0 = ret.Get(0).(HEAD) + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // PendingSequenceAt provides a mock function with given fields: ctx, addr func (_m *mockRPC[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD]) PendingSequenceAt(ctx context.Context, addr ADDR) (SEQ, error) { ret := _m.Called(ctx, addr) diff --git a/common/client/multi_node.go b/common/client/multi_node.go index c03c3fbcd61..9306f040ea5 100644 --- a/common/client/multi_node.go +++ b/common/client/multi_node.go @@ -788,3 +788,12 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP } return n.RPC().TransactionReceipt(ctx, txHash) } + +func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT]) LatestFinalizedBlock(ctx context.Context) (head HEAD, err error) { + n, err := c.selectNode() + if err != nil { + return head, err + } + + return n.RPC().LatestFinalizedBlock(ctx) +} diff --git a/common/client/types.go b/common/client/types.go index 32d4da98b50..d10e6ea90d1 100644 --- a/common/client/types.go +++ b/common/client/types.go @@ -5,6 +5,7 @@ import ( "math/big" "github.com/smartcontractkit/chainlink-common/pkg/assets" + feetypes "github.com/smartcontractkit/chainlink/v2/common/fee/types" "github.com/smartcontractkit/chainlink/v2/common/types" ) @@ -113,6 +114,7 @@ type clientAPI[ BlockByNumber(ctx context.Context, number *big.Int) (HEAD, error) BlockByHash(ctx context.Context, hash BLOCK_HASH) (HEAD, error) LatestBlockHeight(context.Context) (*big.Int, error) + LatestFinalizedBlock(ctx context.Context) (HEAD, error) // Events FilterEvents(ctx context.Context, query EVENT_OPS) ([]EVENT, error) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 4cc152fb9fe..6ae38297687 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math/big" "sync" "time" @@ -159,17 +160,20 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) HealthReport() map[string]error { return report } -func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain HTH, depth uint) (err error) { - if uint(headWithChain.ChainLength()) >= depth { - return nil +func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain, latestFinalized HTH) (err error) { + if !latestFinalized.IsValid() { + return errors.New("can not perform backfill without a valid latestFinalized head") } - baseHeight := headWithChain.BlockNumber() - int64(depth-1) - if baseHeight < 0 { - baseHeight = 0 + if headWithChain.BlockNumber() < latestFinalized.BlockNumber() { + const errMsg = "invariant violation: expected head of canonical chain to be ahead of the latestFinalized" + ht.log.With("head_block_num", headWithChain.BlockNumber(), + "latest_finalized_block_number", latestFinalized.BlockNumber()). + Criticalf(errMsg) + return errors.New(errMsg) } - return ht.backfill(ctx, headWithChain.EarliestHeadInChain(), baseHeight) + return ht.backfill(ctx, headWithChain, latestFinalized) } func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) LatestChain() HTH { @@ -290,7 +294,13 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfillLoop() { break } { - err := ht.Backfill(ctx, head, uint(ht.config.FinalityDepth())) + latestFinalized, err := ht.calculateLatestFinalized(ctx, head.BlockNumber()) + if err != nil { + ht.log.Warnw("Failed to calculate finalized block", "err", err) + continue + } + + err = ht.Backfill(ctx, head, latestFinalized) if err != nil { ht.log.Warnw("Unexpected error while backfilling heads", "err", err) } else if ctx.Err() != nil { @@ -302,14 +312,26 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfillLoop() { } } +// calculateLatestFinalized - returns latest finalized block. It's expected that currentHeadNumber - is the head of +// canonical chain. There is no guaranties that returned block belongs to the canonical chain. Additional verification +// must be performed before usage. +func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.Context, currentHeadNumber int64) (h HTH, err error) { + if ht.config.FinalityTagEnabled() { + return ht.client.LatestFinalizedBlock(ctx) + } + finalizedBlockNumber := currentHeadNumber - int64(ht.config.FinalityDepth()) + if finalizedBlockNumber <= 0 { + finalizedBlockNumber = 0 + } + return ht.client.HeadByNumber(ctx, big.NewInt(finalizedBlockNumber)) +} + // backfill fetches all missing heads up until the base height -func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, head types.Head[BLOCK_HASH], baseHeight int64) (err error) { +func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, head, latestFinalizedHead HTH) (err error) { headBlockNumber := head.BlockNumber() - if headBlockNumber <= baseHeight { - return nil - } mark := time.Now() fetched := 0 + baseHeight := latestFinalizedHead.BlockNumber() l := ht.log.With("blockNumber", headBlockNumber, "n", headBlockNumber-baseHeight, "fromBlockHeight", baseHeight, @@ -342,6 +364,18 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea return fmt.Errorf("fetchAndSaveHead failed: %w", err) } } + + if head.BlockHash() != latestFinalizedHead.BlockHash() { + const errMsg = "expected finalized block to be present in canonical chain" + ht.log.With("finalized_block_number", latestFinalizedHead.BlockNumber(), "finalized_hash", latestFinalizedHead.BlockHash(), + "canonical_chain_block_number", head.BlockNumber(), "canonical_chain_hash", head.BlockHash()).Criticalf(errMsg) + return fmt.Errorf(errMsg) + } + + err = ht.headSaver.MarkFinalized(ctx, latestFinalizedHead.BlockHash()) + if err != nil { + return fmt.Errorf("failed to mark head as finalized: %w", err) + } return } diff --git a/common/headtracker/types/client.go b/common/headtracker/types/client.go index 906f95bbe54..246767e03fa 100644 --- a/common/headtracker/types/client.go +++ b/common/headtracker/types/client.go @@ -15,4 +15,7 @@ type Client[H types.Head[BLOCK_HASH], S types.Subscription, ID types.ID, BLOCK_H // SubscribeNewHead is the method in which the client receives new Head. // It can be implemented differently for each chain i.e websocket, polling, etc SubscribeNewHead(ctx context.Context, ch chan<- H) (S, error) + // LatestFinalizedBlock - returns the latest block that was marked as finalized. Only applicable for PoS chains + // with the finality tag support. + LatestFinalizedBlock(ctx context.Context) (head H, err error) } diff --git a/common/headtracker/types/config.go b/common/headtracker/types/config.go index ca64f7a2952..019aa9847d9 100644 --- a/common/headtracker/types/config.go +++ b/common/headtracker/types/config.go @@ -5,6 +5,7 @@ import "time" type Config interface { BlockEmissionIdleWarningThreshold() time.Duration FinalityDepth() uint32 + FinalityTagEnabled() bool } type HeadTrackerConfig interface { diff --git a/common/mocks/head_tracker.go b/common/mocks/head_tracker.go index 83ee54b1847..fea31a1d6eb 100644 --- a/common/mocks/head_tracker.go +++ b/common/mocks/head_tracker.go @@ -14,17 +14,17 @@ type HeadTracker[H types.Head[BLOCK_HASH], BLOCK_HASH types.Hashable] struct { mock.Mock } -// Backfill provides a mock function with given fields: ctx, headWithChain, depth -func (_m *HeadTracker[H, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain H, depth uint) error { - ret := _m.Called(ctx, headWithChain, depth) +// Backfill provides a mock function with given fields: ctx, headWithChain, latestFinalized +func (_m *HeadTracker[H, BLOCK_HASH]) Backfill(ctx context.Context, headWithChain H, latestFinalized H) error { + ret := _m.Called(ctx, headWithChain, latestFinalized) if len(ret) == 0 { panic("no return value specified for Backfill") } var r0 error - if rf, ok := ret.Get(0).(func(context.Context, H, uint) error); ok { - r0 = rf(ctx, headWithChain, depth) + if rf, ok := ret.Get(0).(func(context.Context, H, H) error); ok { + r0 = rf(ctx, headWithChain, latestFinalized) } else { r0 = ret.Error(0) } diff --git a/common/types/head_tracker.go b/common/types/head_tracker.go index d8fa8011783..22194084789 100644 --- a/common/types/head_tracker.go +++ b/common/types/head_tracker.go @@ -12,9 +12,8 @@ import ( //go:generate mockery --quiet --name HeadTracker --output ../mocks/ --case=underscore type HeadTracker[H Head[BLOCK_HASH], BLOCK_HASH Hashable] interface { services.Service - // Backfill given a head will fill in any missing heads up to the given depth - // (used for testing) - Backfill(ctx context.Context, headWithChain H, depth uint) (err error) + // Backfill given a head will fill in any missing heads up to latestFinalizedHead + Backfill(ctx context.Context, headWithChain, latestFinalized H) (err error) LatestChain() H } @@ -43,6 +42,9 @@ type HeadSaver[H Head[BLOCK_HASH], BLOCK_HASH Hashable] interface { LatestChain() H // Chain returns a head for the specified hash, or nil. Chain(hash BLOCK_HASH) H + // MarkFinalized - marks matching block and all it's direct ancestors as finalized. Returns error if failed to find + // finalized block in the LatestChain + MarkFinalized(ctx context.Context, finalized BLOCK_HASH) error } // HeadListener is a chain agnostic interface that manages connection of Client that receives heads from the blockchain node diff --git a/core/chains/evm/client/chain_client.go b/core/chains/evm/client/chain_client.go index 2a5a37da47c..e52d71352c3 100644 --- a/core/chains/evm/client/chain_client.go +++ b/core/chains/evm/client/chain_client.go @@ -277,3 +277,7 @@ func (c *chainClient) TransactionReceipt(ctx context.Context, txHash common.Hash //return rpc.TransactionReceipt(ctx, txHash) return rpc.TransactionReceiptGeth(ctx, txHash) } + +func (c *chainClient) LatestFinalizedBlock(ctx context.Context) (*evmtypes.Head, error) { + return c.multiNode.LatestFinalizedBlock(ctx) +} diff --git a/core/chains/evm/client/client.go b/core/chains/evm/client/client.go index 61635c59c6b..8b1bc0b624b 100644 --- a/core/chains/evm/client/client.go +++ b/core/chains/evm/client/client.go @@ -63,6 +63,7 @@ type Client interface { HeadByNumber(ctx context.Context, n *big.Int) (*evmtypes.Head, error) HeadByHash(ctx context.Context, n common.Hash) (*evmtypes.Head, error) SubscribeNewHead(ctx context.Context, ch chan<- *evmtypes.Head) (ethereum.Subscription, error) + LatestFinalizedBlock(ctx context.Context) (head *evmtypes.Head, err error) SendTransactionReturnCode(ctx context.Context, tx *types.Transaction, fromAddress common.Address) (commonclient.SendTxReturnCode, error) @@ -361,3 +362,7 @@ func (client *client) SuggestGasTipCap(ctx context.Context) (tipCap *big.Int, er func (client *client) IsL2() bool { return client.pool.ChainType().IsL2() } + +func (client *client) LatestFinalizedBlock(_ context.Context) (*evmtypes.Head, error) { + panic("not implemented. client was deprecated. New methods are added only to satisfy type constraints while we are migrating to new alternatives") +} diff --git a/core/chains/evm/client/mocks/client.go b/core/chains/evm/client/mocks/client.go index 0b45894cf28..8dd80daa5f5 100644 --- a/core/chains/evm/client/mocks/client.go +++ b/core/chains/evm/client/mocks/client.go @@ -565,6 +565,36 @@ func (_m *Client) LatestBlockHeight(ctx context.Context) (*big.Int, error) { return r0, r1 } +// LatestFinalizedBlock provides a mock function with given fields: ctx +func (_m *Client) LatestFinalizedBlock(ctx context.Context) (*evmtypes.Head, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for LatestFinalizedBlock") + } + + var r0 *evmtypes.Head + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*evmtypes.Head, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *evmtypes.Head); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*evmtypes.Head) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // NodeStates provides a mock function with given fields: func (_m *Client) NodeStates() map[string]string { ret := _m.Called() diff --git a/core/chains/evm/client/null_client.go b/core/chains/evm/client/null_client.go index e3bb1defd0d..81951200319 100644 --- a/core/chains/evm/client/null_client.go +++ b/core/chains/evm/client/null_client.go @@ -11,6 +11,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/assets" "github.com/smartcontractkit/chainlink-common/pkg/logger" + commonclient "github.com/smartcontractkit/chainlink/v2/common/client" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) @@ -221,3 +222,7 @@ func (nc *NullClient) IsL2() bool { nc.lggr.Debug("IsL2") return false } + +func (nc *NullClient) LatestFinalizedBlock(_ context.Context) (*evmtypes.Head, error) { + return nil, nil +} diff --git a/core/chains/evm/client/rpc_client.go b/core/chains/evm/client/rpc_client.go index ce3a67162ed..7a8ed47a261 100644 --- a/core/chains/evm/client/rpc_client.go +++ b/core/chains/evm/client/rpc_client.go @@ -479,9 +479,17 @@ func (r *rpcClient) HeaderByHash(ctx context.Context, hash common.Hash) (header return } +func (r *rpcClient) LatestFinalizedBlock(ctx context.Context) (head *evmtypes.Head, err error) { + return r.blockByNumber(ctx, rpc.FinalizedBlockNumber.String()) +} + func (r *rpcClient) BlockByNumber(ctx context.Context, number *big.Int) (head *evmtypes.Head, err error) { hex := ToBlockNumArg(number) - err = r.CallContext(ctx, &head, "eth_getBlockByNumber", hex, false) + return r.blockByNumber(ctx, hex) +} + +func (r *rpcClient) blockByNumber(ctx context.Context, number string) (head *evmtypes.Head, err error) { + err = r.CallContext(ctx, &head, "eth_getBlockByNumber", number, false) if err != nil { return nil, err } diff --git a/core/chains/evm/client/simulated_backend_client.go b/core/chains/evm/client/simulated_backend_client.go index bd2e959d9bc..da53e0047ab 100644 --- a/core/chains/evm/client/simulated_backend_client.go +++ b/core/chains/evm/client/simulated_backend_client.go @@ -623,6 +623,17 @@ func (c *SimulatedBackendClient) ethGetHeaderByNumber(ctx context.Context, resul return nil } +func (c *SimulatedBackendClient) LatestFinalizedBlock(ctx context.Context) (*evmtypes.Head, error) { + block := c.b.Blockchain().CurrentFinalBlock() + return &evmtypes.Head{ + EVMChainID: ubig.NewI(c.chainId.Int64()), + Hash: block.Hash(), + Number: block.Number.Int64(), + ParentHash: block.ParentHash, + Timestamp: time.Unix(int64(block.Time), 0), + }, nil +} + func toCallMsg(params map[string]interface{}) ethereum.CallMsg { var callMsg ethereum.CallMsg diff --git a/core/chains/evm/headtracker/config.go b/core/chains/evm/headtracker/config.go index 54ccb1f933f..85fe084470d 100644 --- a/core/chains/evm/headtracker/config.go +++ b/core/chains/evm/headtracker/config.go @@ -12,6 +12,7 @@ import ( type Config interface { BlockEmissionIdleWarningThreshold() time.Duration FinalityDepth() uint32 + FinalityTagEnabled() bool } type HeadTrackerConfig interface { diff --git a/core/chains/evm/headtracker/head_broadcaster_test.go b/core/chains/evm/headtracker/head_broadcaster_test.go index dcbb9bd0396..5c02fae4ad3 100644 --- a/core/chains/evm/headtracker/head_broadcaster_test.go +++ b/core/chains/evm/headtracker/head_broadcaster_test.go @@ -14,6 +14,7 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox/mailboxtest" + commonhtrk "github.com/smartcontractkit/chainlink/v2/common/headtracker" commonmocks "github.com/smartcontractkit/chainlink/v2/common/types/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" @@ -61,8 +62,8 @@ func TestHeadBroadcaster_Subscribe(t *testing.T) { chchHeaders <- args.Get(1).(chan<- *evmtypes.Head) }). Return(sub, nil) - ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(1), nil).Once() - ethClient.On("HeadByHash", mock.Anything, mock.Anything).Return(cltest.Head(1), nil) + // one for initial and 2 for backfill + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(1), nil).Times(3) sub.On("Unsubscribe").Return() sub.On("Err").Return(nil) diff --git a/core/chains/evm/headtracker/head_saver.go b/core/chains/evm/headtracker/head_saver.go index 92eedaf153e..f0c10f957c7 100644 --- a/core/chains/evm/headtracker/head_saver.go +++ b/core/chains/evm/headtracker/head_saver.go @@ -2,10 +2,12 @@ package headtracker import ( "context" + "fmt" "github.com/ethereum/go-ethereum/common" "github.com/smartcontractkit/chainlink-common/pkg/logger" + commontypes "github.com/smartcontractkit/chainlink/v2/common/types" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" @@ -72,12 +74,21 @@ func (hs *headSaver) Chain(hash common.Hash) *evmtypes.Head { return hs.heads.HeadByHash(hash) } +func (hs *headSaver) MarkFinalized(ctx context.Context, finalized common.Hash) error { + if hs.heads.MarkFinalized(finalized) { + return nil + } + + return fmt.Errorf("failed to find %s block in the latest chain to mark is as finalized", finalized) +} + var NullSaver httypes.HeadSaver = &nullSaver{} type nullSaver struct{} -func (*nullSaver) Save(ctx context.Context, head *evmtypes.Head) error { return nil } -func (*nullSaver) Load(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } -func (*nullSaver) LatestHeadFromDB(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } -func (*nullSaver) LatestChain() *evmtypes.Head { return nil } -func (*nullSaver) Chain(hash common.Hash) *evmtypes.Head { return nil } +func (*nullSaver) Save(ctx context.Context, head *evmtypes.Head) error { return nil } +func (*nullSaver) Load(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } +func (*nullSaver) LatestHeadFromDB(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } +func (*nullSaver) LatestChain() *evmtypes.Head { return nil } +func (*nullSaver) Chain(hash common.Hash) *evmtypes.Head { return nil } +func (*nullSaver) MarkFinalized(ctx context.Context, finalized common.Hash) error { return nil } diff --git a/core/chains/evm/headtracker/head_saver_test.go b/core/chains/evm/headtracker/head_saver_test.go index f541330bc98..4361ef863e3 100644 --- a/core/chains/evm/headtracker/head_saver_test.go +++ b/core/chains/evm/headtracker/head_saver_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" @@ -34,6 +35,7 @@ func (h *headTrackerConfig) MaxBufferSize() uint32 { type config struct { finalityDepth uint32 blockEmissionIdleWarningThreshold time.Duration + finalityTagEnabled bool } func (c *config) FinalityDepth() uint32 { return c.finalityDepth } @@ -41,6 +43,10 @@ func (c *config) BlockEmissionIdleWarningThreshold() time.Duration { return c.blockEmissionIdleWarningThreshold } +func (c *config) FinalityTagEnabled() bool { + return c.finalityTagEnabled +} + func configureSaver(t *testing.T) (httypes.HeadSaver, headtracker.ORM) { db := pgtest.NewSqlxDB(t) lggr := logger.Test(t) diff --git a/core/chains/evm/headtracker/head_tracker.go b/core/chains/evm/headtracker/head_tracker.go index 3cddfb71d09..1fed1aa0c51 100644 --- a/core/chains/evm/headtracker/head_tracker.go +++ b/core/chains/evm/headtracker/head_tracker.go @@ -53,7 +53,7 @@ func (*nullTracker) Ready() error { return nil } func (*nullTracker) HealthReport() map[string]error { return map[string]error{} } func (*nullTracker) Name() string { return "" } func (*nullTracker) SetLogLevel(zapcore.Level) {} -func (*nullTracker) Backfill(ctx context.Context, headWithChain *evmtypes.Head, depth uint) (err error) { +func (*nullTracker) Backfill(ctx context.Context, headWithChain, latestFinalized *evmtypes.Head) (err error) { return nil } func (*nullTracker) LatestChain() *evmtypes.Head { return nil } diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 22e931d6d0f..5f1101692d6 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -28,6 +28,7 @@ import ( commonmocks "github.com/smartcontractkit/chainlink/v2/common/types/mocks" evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" + evmclimocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" @@ -176,7 +177,9 @@ func TestHeadTracker_Start_NewHeads(t *testing.T) { chStarted := make(chan struct{}) mockEth := &evmtest.MockEth{EthClient: ethClient} sub := mockEth.NewSub(t) - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), nil) + // for initial load + ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), nil).Once() + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(0), nil).Once() ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything). Run(func(mock.Arguments) { close(chStarted) @@ -289,7 +292,7 @@ func TestHeadTracker_ReconnectOnError(t *testing.T) { func(ctx context.Context, ch chan<- *evmtypes.Head) ethereum.Subscription { return mockEth.NewSub(t) }, func(ctx context.Context, ch chan<- *evmtypes.Head) error { return nil }, ) - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), nil) + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(0), nil) checker := &cltest.MockHeadTrackable{} ht := createHeadTrackerWithChecker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm, checker) @@ -325,7 +328,7 @@ func TestHeadTracker_ResubscribeOnSubscriptionError(t *testing.T) { }, func(ctx context.Context, ch chan<- *evmtypes.Head) error { return nil }, ) - ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(0), nil).Once() + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(0), nil) ethClient.On("HeadByHash", mock.Anything, mock.Anything).Return(cltest.Head(0), nil).Maybe() checker := &cltest.MockHeadTrackable{} @@ -373,6 +376,7 @@ func TestHeadTracker_Start_LoadsLatestChain(t *testing.T) { parentHash = heads[i].Hash } ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(heads[3], nil).Maybe() + ethClient.On("HeadByNumber", mock.Anything, big.NewInt(0)).Return(heads[0], nil).Maybe() ethClient.On("HeadByHash", mock.Anything, heads[2].Hash).Return(heads[2], nil).Maybe() ethClient.On("HeadByHash", mock.Anything, heads[1].Hash).Return(heads[1], nil).Maybe() ethClient.On("HeadByHash", mock.Anything, heads[0].Hash).Return(heads[0], nil).Maybe() @@ -454,6 +458,8 @@ func TestHeadTracker_SwitchesToLongestChainWithHeadSamplingEnabled(t *testing.T) head0 := blocks.Head(0) // Initial query ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head0, nil) + // backfill query + ethClient.On("HeadByNumber", mock.Anything, big.NewInt(0)).Return(head0, nil) ht.Start(t) headSeq := cltest.NewHeadBuffer(t) @@ -582,6 +588,8 @@ func TestHeadTracker_SwitchesToLongestChainWithHeadSamplingDisabled(t *testing.T head0 := blocks.Head(0) // evmtypes.Head{Number: 0, Hash: utils.NewHash(), ParentHash: utils.NewHash(), Timestamp: time.Unix(0, 0)} // Initial query ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head0, nil) + // backfill + ethClient.On("HeadByNumber", mock.Anything, big.NewInt(0)).Return(head0, nil) headSeq := cltest.NewHeadBuffer(t) headSeq.Append(blocks.Head(0)) @@ -773,45 +781,88 @@ func TestHeadTracker_Backfill(t *testing.T) { ctx := testutils.Context(t) - t.Run("does nothing if all the heads are in database", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) + type headTrackerUniverse struct { + ethClient *evmclimocks.Client + orm headtracker.ORM + headTracker httypes.HeadTracker + headBroadcaster httypes.HeadBroadcaster + headSaver httypes.HeadSaver + mailMon *mailbox.Monitor + } + + type opts struct { + Heads []evmtypes.Head + } + newHeadTrackerUniverse := func(t *testing.T, opts opts) *headTrackerUniverse { cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - for i := range heads { + evmcfg := evmtest.NewChainScopedConfig(t, cfg) + lggr := logger.Test(t) + hb := headtracker.NewHeadBroadcaster(lggr) + db := pgtest.NewSqlxDB(t) + orm := headtracker.NewORM(db, lggr, cfg.Database(), cltest.FixtureChainID) + for i := range opts.Heads { require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) } - + hs := headtracker.NewHeadSaver(lggr, orm, evmcfg.EVM(), evmcfg.EVM().HeadTracker()) + mailMon := mailboxtest.NewMonitor(t) ethClient := evmtest.NewEthClientMock(t) ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) - - err := ht.Backfill(ctx, &h12, 2) + ht := headtracker.NewHeadTracker(lggr, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), hb, hs, mailMon) + _, err := hs.Load(testutils.Context(t)) require.NoError(t, err) + return &headTrackerUniverse{ + ethClient: ethClient, + orm: orm, + headTracker: ht, + headBroadcaster: hb, + headSaver: hs, + mailMon: mailMon, + } + } + + t.Run("returns error if latestFinalized is not valid", func(t *testing.T) { + htu := newHeadTrackerUniverse(t, opts{}) + + err := htu.headTracker.Backfill(ctx, &h12, nil) + require.EqualError(t, err, "can not perform backfill without a valid latestFinalized head") }) + t.Run("Returns error if finalized head is ahead of canonical", func(t *testing.T) { + htu := newHeadTrackerUniverse(t, opts{}) - t.Run("fetches a missing head", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - for i := range heads { - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) - } + err := htu.headTracker.Backfill(ctx, &h12, &h14Orphaned) + require.EqualError(t, err, "invariant violation: expected head of canonical chain to be ahead of the latestFinalized") + }) + t.Run("Returns error if finalizedHead is not present in the canonical chain", func(t *testing.T) { + htu := newHeadTrackerUniverse(t, opts{Heads: heads}) - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ethClient.On("HeadByHash", mock.Anything, head10.Hash). - Return(&head10, nil) + err := htu.headTracker.Backfill(ctx, &h15, &h14Orphaned) + require.EqualError(t, err, "expected finalized block to be present in canonical chain") + }) + t.Run("Marks all blocks in chain that are older than finalized", func(t *testing.T) { + htu := newHeadTrackerUniverse(t, opts{Heads: heads}) + + assertFinalized := func(expectedFinalized bool, msg string, heads ...evmtypes.Head) { + for _, h := range heads { + storedHead := htu.headSaver.Chain(h.Hash) + assert.Equal(t, expectedFinalized, storedHead != nil && storedHead.IsFinalized, msg, "block_number", h.Number) + } + } - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) + err := htu.headTracker.Backfill(ctx, &h15, &h14) + require.NoError(t, err) + assertFinalized(true, "expected heads to be marked as finalized after backfill", h14, h13, h12, h11) + assertFinalized(false, "expected heads to remain unfinalized", h15, head10) + }) - var depth uint = 3 + t.Run("fetches a missing head", func(t *testing.T) { + htu := newHeadTrackerUniverse(t, opts{Heads: heads}) + htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash). + Return(&head10, nil) - err := ht.Backfill(ctx, &h12, depth) + err := htu.headTracker.Backfill(ctx, &h12, &h9) require.NoError(t, err) - h := ht.headSaver.Chain(h12.Hash) + h := htu.headSaver.Chain(h12.Hash) assert.Equal(t, int64(12), h.Number) require.NotNil(t, h.Parent) @@ -821,37 +872,23 @@ func TestHeadTracker_Backfill(t *testing.T) { require.NotNil(t, h.Parent.Parent.Parent) assert.Equal(t, int64(9), h.Parent.Parent.Parent.Number) - writtenHead, err := orm.HeadByHash(testutils.Context(t), head10.Hash) + writtenHead, err := htu.orm.HeadByHash(testutils.Context(t), head10.Hash) require.NoError(t, err) assert.Equal(t, int64(10), writtenHead.Number) }) t.Run("fetches only heads that are missing", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - for i := range heads { - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) - } - - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) + htu := newHeadTrackerUniverse(t, opts{Heads: heads}) - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) - - ethClient.On("HeadByHash", mock.Anything, head10.Hash). + htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash). Return(&head10, nil) - ethClient.On("HeadByHash", mock.Anything, head8.Hash). + htu.ethClient.On("HeadByHash", mock.Anything, head8.Hash). Return(&head8, nil) - // Needs to be 8 because there are 8 heads in chain (15,14,13,12,11,10,9,8) - var depth uint = 8 - - err := ht.Backfill(ctx, &h15, depth) + err := htu.headTracker.Backfill(ctx, &h15, &head8) require.NoError(t, err) - h := ht.headSaver.Chain(h15.Hash) + h := htu.headSaver.Chain(h15.Hash) require.Equal(t, uint32(8), h.ChainLength()) earliestInChain := h.EarliestInChain() @@ -859,77 +896,20 @@ func TestHeadTracker_Backfill(t *testing.T) { assert.Equal(t, head8.Hash, earliestInChain.BlockHash()) }) - t.Run("does not backfill if chain length is already greater than or equal to depth", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - for i := range heads { - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) - } - - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) - - err := ht.Backfill(ctx, &h15, 3) - require.NoError(t, err) - - err = ht.Backfill(ctx, &h15, 5) - require.NoError(t, err) - }) - - t.Run("only backfills to height 0 if chain length would otherwise cause it to try and fetch a negative head", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ethClient.On("HeadByHash", mock.Anything, head0.Hash). - Return(&head0, nil) - - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &h1)) - - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) - - err := ht.Backfill(ctx, &h1, 400) - require.NoError(t, err) - - h := ht.headSaver.Chain(h1.Hash) - require.NotNil(t, h) - - require.Equal(t, uint32(2), h.ChainLength()) - require.Equal(t, int64(0), h.EarliestInChain().BlockNumber()) - }) - t.Run("abandons backfill and returns error if the eth node returns not found", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - for i := range heads { - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) - } - - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ethClient.On("HeadByHash", mock.Anything, head10.Hash). + htu := newHeadTrackerUniverse(t, opts{Heads: heads}) + htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash). Return(&head10, nil). Once() - ethClient.On("HeadByHash", mock.Anything, head8.Hash). + htu.ethClient.On("HeadByHash", mock.Anything, head8.Hash). Return(nil, ethereum.NotFound). Once() - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) - - err := ht.Backfill(ctx, &h12, 400) + err := htu.headTracker.Backfill(ctx, &h12, &head8) require.Error(t, err) require.EqualError(t, err, "fetchAndSaveHead failed: not found") - h := ht.headSaver.Chain(h12.Hash) + h := htu.headSaver.Chain(h12.Hash) // Should contain 12, 11, 10, 9 assert.Equal(t, 4, int(h.ChainLength())) @@ -937,28 +917,17 @@ func TestHeadTracker_Backfill(t *testing.T) { }) t.Run("abandons backfill and returns error if the context time budget is exceeded", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - for i := range heads { - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) - } - - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ethClient.On("HeadByHash", mock.Anything, head10.Hash). + htu := newHeadTrackerUniverse(t, opts{Heads: heads}) + htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash). Return(&head10, nil) - ethClient.On("HeadByHash", mock.Anything, head8.Hash). + htu.ethClient.On("HeadByHash", mock.Anything, head8.Hash). Return(nil, context.DeadlineExceeded) - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) - - err := ht.Backfill(ctx, &h12, 400) + err := htu.headTracker.Backfill(ctx, &h12, &head8) require.Error(t, err) require.EqualError(t, err, "fetchAndSaveHead failed: context deadline exceeded") - h := ht.headSaver.Chain(h12.Hash) + h := htu.headSaver.Chain(h12.Hash) // Should contain 12, 11, 10, 9 assert.Equal(t, 4, int(h.ChainLength())) @@ -966,24 +935,17 @@ func TestHeadTracker_Backfill(t *testing.T) { }) t.Run("abandons backfill and returns error when fetching a block by hash fails, indicating a reorg", func(t *testing.T) { - db := pgtest.NewSqlxDB(t) - cfg := configtest.NewGeneralConfig(t, nil) - logger := logger.Test(t) - orm := headtracker.NewORM(db, logger, cfg.Database(), cltest.FixtureChainID) - ethClient := evmtest.NewEthClientMock(t) - ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ethClient.On("HeadByHash", mock.Anything, h14.Hash).Return(&h14, nil).Once() - ethClient.On("HeadByHash", mock.Anything, h13.Hash).Return(&h13, nil).Once() - ethClient.On("HeadByHash", mock.Anything, h12.Hash).Return(nil, errors.New("not found")).Once() - - ht := createHeadTrackerWithNeverSleeper(t, ethClient, cfg, orm) + htu := newHeadTrackerUniverse(t, opts{}) + htu.ethClient.On("HeadByHash", mock.Anything, h14.Hash).Return(&h14, nil).Once() + htu.ethClient.On("HeadByHash", mock.Anything, h13.Hash).Return(&h13, nil).Once() + htu.ethClient.On("HeadByHash", mock.Anything, h12.Hash).Return(nil, errors.New("not found")).Once() - err := ht.Backfill(ctx, &h15, 400) + err := htu.headTracker.Backfill(ctx, &h15, &h11) require.Error(t, err) require.EqualError(t, err, "fetchAndSaveHead failed: not found") - h := ht.headSaver.Chain(h14.Hash) + h := htu.headSaver.Chain(h14.Hash) // Should contain 14, 13 (15 was never added). When trying to get the parent of h13 by hash, a reorg happened and backfill exited. assert.Equal(t, 2, int(h.ChainLength())) @@ -1048,8 +1010,8 @@ type headTrackerUniverse struct { mailMon *mailbox.Monitor } -func (u *headTrackerUniverse) Backfill(ctx context.Context, head *evmtypes.Head, depth uint) error { - return u.headTracker.Backfill(ctx, head, depth) +func (u *headTrackerUniverse) Backfill(ctx context.Context, head, finalizedHead *evmtypes.Head) error { + return u.headTracker.Backfill(ctx, head, finalizedHead) } func (u *headTrackerUniverse) Start(t *testing.T) { diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index ccb7d9b7336..5a55420c1a5 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -20,6 +20,8 @@ type Heads interface { AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) // Count returns number of heads in the collection. Count() int + // MarkFinalized - finds `finalized` in the LatestHead and marks it and all direct ancestors as finalized + MarkFinalized(finalized common.Hash) bool } type heads struct { @@ -60,6 +62,37 @@ func (h *heads) Count() int { return len(h.heads) } +func (h *heads) MarkFinalized(finalized common.Hash) bool { + h.mu.Lock() + defer h.mu.Unlock() + + if len(h.heads) == 0 { + return false + } + + // copy slice as we are going to modify it + newHeads := make([]*evmtypes.Head, len(h.heads)) + for i := range h.heads { + headCopy := *h.heads[i] + newHeads[i] = &headCopy + } + + head := newHeads[0] + foundFinalized := false + for head != nil { + if head.Hash == finalized { + foundFinalized = true + } + + // we might see finalized to move back in chain due to request to lagging RPC, + // we should not override the flag in such cases + head.IsFinalized = head.IsFinalized || foundFinalized + head = head.Parent + } + + return foundFinalized +} + func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { h.mu.Lock() defer h.mu.Unlock() @@ -75,7 +108,10 @@ func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { headCopy := *head headCopy.Parent = nil // always build it from scratch in case it points to a head too old to be included // map eliminates duplicates - headsMap[head.Hash] = &headCopy + // prefer head that was already in heads as it might have been marked as finalized on previous run + if _, ok := headsMap[head.Hash]; !ok { + headsMap[head.Hash] = &headCopy + } } heads := make([]*evmtypes.Head, len(headsMap)) diff --git a/core/chains/evm/headtracker/heads_test.go b/core/chains/evm/headtracker/heads_test.go index 9fa5ed4e548..42ad173ab36 100644 --- a/core/chains/evm/headtracker/heads_test.go +++ b/core/chains/evm/headtracker/heads_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" @@ -108,3 +109,50 @@ func TestHeads_AddHeads(t *testing.T) { require.NotNil(t, head) require.Equal(t, 2, int(head.ChainLength())) } + +func TestHeads_MarkFinalized(t *testing.T) { + t.Parallel() + + heads := headtracker.NewHeads() + + // create chain + // H0 <- H1 <- H2 <- H3 <- H4 + // \ + // H2Uncle + // + newHead := func(num int, parent common.Hash) *evmtypes.Head { + h := evmtypes.NewHead(big.NewInt(int64(num)), utils.NewHash(), parent, uint64(time.Now().Unix()), ubig.NewI(0)) + return &h + } + h0 := newHead(0, utils.NewHash()) + h1 := newHead(1, h0.Hash) + h2 := newHead(2, h1.Hash) + h3 := newHead(3, h2.Hash) + h4 := newHead(4, h3.Hash) + h5 := newHead(5, h3.Hash) + h2Uncle := newHead(2, h1.Hash) + + allHeads := []*evmtypes.Head{h0, h1, h2, h2Uncle, h3, h4, h5} + heads.AddHeads(1000, allHeads...) + // mark h3 and all ancestors as finalized + require.True(t, heads.MarkFinalized(h3.Hash), "expected MarkFinalized succeed") + for _, h := range allHeads { + assert.False(t, h.IsFinalized, "expected original heads to remain unfinalized") + } + require.False(t, heads.MarkFinalized(utils.NewHash()), "expected false if finalized hash was not found in existing LatestHead chain") + ensureProperFinalization := func(t *testing.T) { + t.Helper() + for _, head := range []*evmtypes.Head{h5, h4} { + require.False(t, heads.HeadByHash(head.Hash).IsFinalized, "expected h4-h5 not to be finalized", head.BlockNumber()) + } + for _, head := range []*evmtypes.Head{h3, h2, h1, h0} { + require.True(t, heads.HeadByHash(head.Hash).IsFinalized, "expected h3 and all ancestors to be finalized", head.BlockNumber()) + } + require.False(t, heads.HeadByHash(h2Uncle.Hash).IsFinalized, "expected uncle block not to be marked as finalized") + + } + t.Run("blocks were correctly marked as finalized", ensureProperFinalization) + heads.AddHeads(1000, h0, h1, h2, h2Uncle, h3, h4, h5) + t.Run("blocks remain finalized after re adding them to the Heads", ensureProperFinalization) + +} diff --git a/core/chains/evm/headtracker/mocks/config.go b/core/chains/evm/headtracker/mocks/config.go index 74376a71362..6cc3900ba42 100644 --- a/core/chains/evm/headtracker/mocks/config.go +++ b/core/chains/evm/headtracker/mocks/config.go @@ -49,6 +49,24 @@ func (_m *Config) FinalityDepth() uint32 { return r0 } +// FinalityTagEnabled provides a mock function with given fields: +func (_m *Config) FinalityTagEnabled() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for FinalityTagEnabled") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // NewConfig creates a new instance of Config. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewConfig(t interface { diff --git a/core/chains/evm/types/models.go b/core/chains/evm/types/models.go index 44e150b6541..0a641834d9c 100644 --- a/core/chains/evm/types/models.go +++ b/core/chains/evm/types/models.go @@ -17,6 +17,7 @@ import ( "github.com/ugorji/go/codec" "github.com/smartcontractkit/chainlink-common/pkg/utils/hex" + htrktypes "github.com/smartcontractkit/chainlink/v2/common/headtracker/types" commontypes "github.com/smartcontractkit/chainlink/v2/common/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" @@ -43,6 +44,7 @@ type Head struct { StateRoot common.Hash Difficulty *big.Int TotalDifficulty *big.Int + IsFinalized bool } var _ commontypes.Head[common.Hash] = &Head{} diff --git a/core/internal/cltest/cltest.go b/core/internal/cltest/cltest.go index 332513b28d4..1571df44c62 100644 --- a/core/internal/cltest/cltest.go +++ b/core/internal/cltest/cltest.go @@ -468,7 +468,7 @@ func NewEthMocksWithStartupAssertions(t testing.TB) *evmclimocks.Client { c.On("Dial", mock.Anything).Maybe().Return(nil) c.On("SubscribeNewHead", mock.Anything, mock.Anything).Maybe().Return(EmptyMockSubscription(t), nil) c.On("SendTransaction", mock.Anything, mock.Anything).Maybe().Return(nil) - c.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Maybe().Return(Head(0), nil) + c.On("HeadByNumber", mock.Anything, mock.Anything).Maybe().Return(Head(0), nil) c.On("ConfiguredChainID").Maybe().Return(&FixtureChainID) c.On("CallContract", mock.Anything, mock.Anything, mock.Anything).Maybe().Return([]byte{}, nil) c.On("SubscribeFilterLogs", mock.Anything, mock.Anything, mock.Anything).Maybe().Return(nil, errors.New("mocked")) @@ -496,6 +496,7 @@ func NewEthMocksWithTransactionsOnBlocksAssertions(t testing.TB) *evmclimocks.Cl h1 := HeadWithHash(1, h2.ParentHash) h0 := HeadWithHash(0, h1.ParentHash) c.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Maybe().Return(h2, nil) + c.On("HeadByNumber", mock.Anything, big.NewInt(0)).Maybe().Return(h0, nil) // finalized block c.On("HeadByHash", mock.Anything, h1.Hash).Maybe().Return(h1, nil) c.On("HeadByHash", mock.Anything, h0.Hash).Maybe().Return(h0, nil) c.On("BatchCallContext", mock.Anything, mock.Anything).Maybe().Return(nil).Run(func(args mock.Arguments) { diff --git a/core/internal/testutils/evmtest/evmtest.go b/core/internal/testutils/evmtest/evmtest.go index cc56c3c9e9b..d4075ef444e 100644 --- a/core/internal/testutils/evmtest/evmtest.go +++ b/core/internal/testutils/evmtest/evmtest.go @@ -294,7 +294,7 @@ func nodeStatus(n *evmtoml.Node, chainID string) (types.NodeStatus, error) { return s, nil } -func NewEthClientMock(t *testing.T) *evmclimocks.Client { +func NewEthClientMock(t testing.TB) *evmclimocks.Client { return evmclimocks.NewClient(t) } From 9da949b5a0dfdc250b2b00d88a7460f64cd770c3 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Mon, 19 Feb 2024 17:54:14 +0100 Subject: [PATCH 02/28] Added LatestFinalizedHead to Head --- core/chains/evm/types/head_test.go | 51 ++++++++++++++++++++++++++++++ core/chains/evm/types/models.go | 8 +++++ 2 files changed, 59 insertions(+) create mode 100644 core/chains/evm/types/head_test.go diff --git a/core/chains/evm/types/head_test.go b/core/chains/evm/types/head_test.go new file mode 100644 index 00000000000..b4f1de25c6e --- /dev/null +++ b/core/chains/evm/types/head_test.go @@ -0,0 +1,51 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHead_LatestFinalizedHead(t *testing.T) { + t.Parallel() + cases := []struct { + Name string + Head *Head + Finalized *Head + }{ + { + Name: "Empty chain returns nil on finalized", + Head: nil, + Finalized: nil, + }, + { + Name: "Chain without finalized returns nil", + Head: &Head{Parent: &Head{Parent: &Head{}}}, + Finalized: nil, + }, + { + Name: "Returns head if it's finalized", + Head: &Head{Number: 2, IsFinalized: true, Parent: &Head{Number: 1, IsFinalized: true}}, + Finalized: &Head{Number: 2}, + }, + { + Name: "Returns first block in chain if it's finalized", + Head: &Head{Number: 3, IsFinalized: false, Parent: &Head{Number: 2, IsFinalized: true, Parent: &Head{Number: 1, IsFinalized: true}}}, + Finalized: &Head{Number: 2}, + }, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + actual := tc.Head.LatestFinalizedHead() + if tc.Finalized == nil { + assert.Nil(t, actual) + } else { + require.NotNil(t, actual) + assert.Equal(t, tc.Finalized.Number, actual.BlockNumber()) + } + }) + } + +} diff --git a/core/chains/evm/types/models.go b/core/chains/evm/types/models.go index 0a641834d9c..b080feb338d 100644 --- a/core/chains/evm/types/models.go +++ b/core/chains/evm/types/models.go @@ -171,6 +171,14 @@ func (h *Head) ChainHashes() []common.Hash { return hashes } +func (h *Head) LatestFinalizedHead() commontypes.Head[common.Hash] { + for h != nil && !h.IsFinalized { + h = h.Parent + } + + return h +} + func (h *Head) ChainID() *big.Int { return h.EVMChainID.ToInt() } From 112679edd94d27a8c0d0ce973a21fe3c499f26eb Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 20 Feb 2024 12:46:49 +0100 Subject: [PATCH 03/28] remove unused func --- .../evm/headtracker/head_tracker_test.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 5f1101692d6..251147fd3df 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -967,24 +967,6 @@ func createHeadTracker(t *testing.T, ethClient evmclient.Client, config headtrac } } -func createHeadTrackerWithNeverSleeper(t *testing.T, ethClient evmclient.Client, cfg chainlink.GeneralConfig, orm headtracker.ORM) *headTrackerUniverse { - evmcfg := evmtest.NewChainScopedConfig(t, cfg) - lggr := logger.Test(t) - hb := headtracker.NewHeadBroadcaster(lggr) - hs := headtracker.NewHeadSaver(lggr, orm, evmcfg.EVM(), evmcfg.EVM().HeadTracker()) - mailMon := mailboxtest.NewMonitor(t) - ht := headtracker.NewHeadTracker(lggr, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), hb, hs, mailMon) - _, err := hs.Load(testutils.Context(t)) - require.NoError(t, err) - return &headTrackerUniverse{ - mu: new(sync.Mutex), - headTracker: ht, - headBroadcaster: hb, - headSaver: hs, - mailMon: mailMon, - } -} - func createHeadTrackerWithChecker(t *testing.T, ethClient evmclient.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM, checker httypes.HeadTrackable) *headTrackerUniverse { lggr := logger.Test(t) hb := headtracker.NewHeadBroadcaster(lggr) From 334a4d1de542793cb8b3f6d66e399705bdda9d8f Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 20 Feb 2024 13:19:32 +0100 Subject: [PATCH 04/28] fix flakey nil pointer --- common/headtracker/head_tracker.go | 2 +- core/chains/evm/headtracker/head_tracker_test.go | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 6ae38297687..b35019e83c8 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -359,7 +359,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea fetched++ if ctx.Err() != nil { ht.log.Debugw("context canceled, aborting backfill", "err", err, "ctx.Err", ctx.Err()) - break + return fmt.Errorf("fetchAndSaveHead failed: %w", ctx.Err()) } else if err != nil { return fmt.Errorf("fetchAndSaveHead failed: %w", err) } diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 251147fd3df..ba8ae35feb7 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -920,12 +920,16 @@ func TestHeadTracker_Backfill(t *testing.T) { htu := newHeadTrackerUniverse(t, opts{Heads: heads}) htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash). Return(&head10, nil) + var cancel func() + ctx, cancel = context.WithCancel(ctx) htu.ethClient.On("HeadByHash", mock.Anything, head8.Hash). - Return(nil, context.DeadlineExceeded) + Return(nil, context.DeadlineExceeded).Run(func(args mock.Arguments) { + cancel() + }) err := htu.headTracker.Backfill(ctx, &h12, &head8) require.Error(t, err) - require.EqualError(t, err, "fetchAndSaveHead failed: context deadline exceeded") + require.EqualError(t, err, "fetchAndSaveHead failed: context canceled") h := htu.headSaver.Chain(h12.Hash) From 83cf83488456024afe25a260e41af58f2a16af20 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 20 Feb 2024 14:50:08 +0100 Subject: [PATCH 05/28] improve logs & address lint issue --- common/headtracker/head_tracker.go | 11 +++++++++++ core/chains/evm/headtracker/head_tracker_test.go | 5 ++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index b35019e83c8..0f3b535d294 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -337,6 +337,13 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea "fromBlockHeight", baseHeight, "toBlockHeight", headBlockNumber-1) l.Debug("Starting backfill") + if ht.htConfig.HistoryDepth() < uint32(headBlockNumber-baseHeight) { + l.Warnw("HistoryDepth is smaller than the actual finality depth (number of blocks from the latest "+ + "finalized to the most recent bock). This might lead to making more RPC requests than necessary. "+ + "Increase HistoryDepth.", + "history_depth", ht.htConfig.HistoryDepth(), + "actual_finality_depth", headBlockNumber-baseHeight) + } defer func() { if ctx.Err() != nil { l.Warnw("Backfill context error", "err", ctx.Err()) @@ -376,6 +383,10 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea if err != nil { return fmt.Errorf("failed to mark head as finalized: %w", err) } + + l.Debugw("marked block as finalized", "block_hash", latestFinalizedHead.BlockHash(), + "block_number", latestFinalizedHead.BlockNumber()) + return } diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index ba8ae35feb7..1fc472ca052 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -920,14 +920,13 @@ func TestHeadTracker_Backfill(t *testing.T) { htu := newHeadTrackerUniverse(t, opts{Heads: heads}) htu.ethClient.On("HeadByHash", mock.Anything, head10.Hash). Return(&head10, nil) - var cancel func() - ctx, cancel = context.WithCancel(ctx) + lctx, cancel := context.WithCancel(ctx) htu.ethClient.On("HeadByHash", mock.Anything, head8.Hash). Return(nil, context.DeadlineExceeded).Run(func(args mock.Arguments) { cancel() }) - err := htu.headTracker.Backfill(ctx, &h12, &head8) + err := htu.headTracker.Backfill(lctx, &h12, &head8) require.Error(t, err) require.EqualError(t, err, "fetchAndSaveHead failed: context canceled") From 26e8d50027b25d31c5865ed564d7114ed45423ef Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 20 Feb 2024 16:17:22 +0100 Subject: [PATCH 06/28] nitpicks --- common/headtracker/head_tracker.go | 5 +++-- common/types/head_tracker.go | 2 +- core/chains/evm/headtracker/head_saver.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 0f3b535d294..6330203410d 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -339,8 +339,9 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea l.Debug("Starting backfill") if ht.htConfig.HistoryDepth() < uint32(headBlockNumber-baseHeight) { l.Warnw("HistoryDepth is smaller than the actual finality depth (number of blocks from the latest "+ - "finalized to the most recent bock). This might lead to making more RPC requests than necessary. "+ - "Increase HistoryDepth.", + "finalized to the most recent block). This might be caused by an out-of-sync RPC that returned an old "+ + "finalized block or by HistoryDepth being too small. If you see this message too often, "+ + "consider increasing HistoryDepth.", "history_depth", ht.htConfig.HistoryDepth(), "actual_finality_depth", headBlockNumber-baseHeight) } diff --git a/common/types/head_tracker.go b/common/types/head_tracker.go index 22194084789..6e92cb4a256 100644 --- a/common/types/head_tracker.go +++ b/common/types/head_tracker.go @@ -12,7 +12,7 @@ import ( //go:generate mockery --quiet --name HeadTracker --output ../mocks/ --case=underscore type HeadTracker[H Head[BLOCK_HASH], BLOCK_HASH Hashable] interface { services.Service - // Backfill given a head will fill in any missing heads up to latestFinalizedHead + // Backfill given a head will fill in any missing heads up to latestFinalized Backfill(ctx context.Context, headWithChain, latestFinalized H) (err error) LatestChain() H } diff --git a/core/chains/evm/headtracker/head_saver.go b/core/chains/evm/headtracker/head_saver.go index f0c10f957c7..cd7d0883b44 100644 --- a/core/chains/evm/headtracker/head_saver.go +++ b/core/chains/evm/headtracker/head_saver.go @@ -79,7 +79,7 @@ func (hs *headSaver) MarkFinalized(ctx context.Context, finalized common.Hash) e return nil } - return fmt.Errorf("failed to find %s block in the latest chain to mark is as finalized", finalized) + return fmt.Errorf("failed to find %s block in the canonical chain to mark it as finalized", finalized) } var NullSaver httypes.HeadSaver = &nullSaver{} From b86c872aa0acbd6e35ddb3ad6ee9264ec80f08ad Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 21 Feb 2024 14:05:18 +0100 Subject: [PATCH 07/28] fixed copy on heads on MarkFinalized --- core/chains/evm/headtracker/heads.go | 32 +++++++++++++--------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 5a55420c1a5..3099a5ca87c 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -70,14 +70,9 @@ func (h *heads) MarkFinalized(finalized common.Hash) bool { return false } - // copy slice as we are going to modify it - newHeads := make([]*evmtypes.Head, len(h.heads)) - for i := range h.heads { - headCopy := *h.heads[i] - newHeads[i] = &headCopy - } + h.heads = deepCopyUpTo(h.heads, uint(len(h.heads))) - head := newHeads[0] + head := h.heads[0] foundFinalized := false for head != nil { if head.Hash == finalized { @@ -93,12 +88,9 @@ func (h *heads) MarkFinalized(finalized common.Hash) bool { return foundFinalized } -func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { - h.mu.Lock() - defer h.mu.Unlock() - - headsMap := make(map[common.Hash]*evmtypes.Head, len(h.heads)+len(newHeads)) - for _, head := range append(h.heads, newHeads...) { +func deepCopyUpTo(oldHeads []*evmtypes.Head, depth uint) []*evmtypes.Head { + headsMap := make(map[common.Hash]*evmtypes.Head, len(oldHeads)) + for _, head := range oldHeads { if head.Hash == head.ParentHash { // shouldn't happen but it is untrusted input continue @@ -131,8 +123,8 @@ func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { }) // cut off the oldest - if uint(len(heads)) > historyDepth { - heads = heads[:historyDepth] + if uint(len(heads)) > depth { + heads = heads[:depth] } // assign parents @@ -144,6 +136,12 @@ func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { } } - // set - h.heads = heads + return heads +} + +func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { + h.mu.Lock() + defer h.mu.Unlock() + + h.heads = deepCopyUpTo(append(h.heads, newHeads...), historyDepth) } From 81774b4410314597523454f0b2b6f6e59f029a64 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 21 Feb 2024 15:59:39 +0100 Subject: [PATCH 08/28] error instead of panic --- core/chains/evm/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/client/client.go b/core/chains/evm/client/client.go index 5bfb322b7c2..a8ae83f05ab 100644 --- a/core/chains/evm/client/client.go +++ b/core/chains/evm/client/client.go @@ -369,5 +369,5 @@ func (client *client) IsL2() bool { } func (client *client) LatestFinalizedBlock(_ context.Context) (*evmtypes.Head, error) { - panic("not implemented. client was deprecated. New methods are added only to satisfy type constraints while we are migrating to new alternatives") + return nil, fmt.Errorf("not implemented. client was deprecated. New methods are added only to satisfy type constraints while we are migrating to new alternatives") } From c942663a220eabc88128f5d41e3bd232d3e25696 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 21 Feb 2024 16:03:27 +0100 Subject: [PATCH 09/28] return error instead of panic --- core/chains/evm/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chains/evm/client/client.go b/core/chains/evm/client/client.go index a8ae83f05ab..7e8b694bfd3 100644 --- a/core/chains/evm/client/client.go +++ b/core/chains/evm/client/client.go @@ -369,5 +369,5 @@ func (client *client) IsL2() bool { } func (client *client) LatestFinalizedBlock(_ context.Context) (*evmtypes.Head, error) { - return nil, fmt.Errorf("not implemented. client was deprecated. New methods are added only to satisfy type constraints while we are migrating to new alternatives") + return nil, errors.New("not implemented. client was deprecated. New methods are added only to satisfy type constraints while we are migrating to new alternatives") } From a01fb86cb6ed882d9e51f2c407ee50f66412c27a Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 21 Feb 2024 18:00:41 +0100 Subject: [PATCH 10/28] nitpicks --- common/headtracker/head_tracker.go | 10 +++++++--- common/headtracker/types/client.go | 3 +-- core/internal/testutils/evmtest/evmtest.go | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 6330203410d..44535910684 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -294,7 +294,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfillLoop() { break } { - latestFinalized, err := ht.calculateLatestFinalized(ctx, head.BlockNumber()) + latestFinalized, err := ht.calculateLatestFinalized(ctx, head) if err != nil { ht.log.Warnw("Failed to calculate finalized block", "err", err) continue @@ -315,11 +315,15 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfillLoop() { // calculateLatestFinalized - returns latest finalized block. It's expected that currentHeadNumber - is the head of // canonical chain. There is no guaranties that returned block belongs to the canonical chain. Additional verification // must be performed before usage. -func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.Context, currentHeadNumber int64) (h HTH, err error) { +func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx context.Context, currentHead HTH) (h HTH, err error) { if ht.config.FinalityTagEnabled() { return ht.client.LatestFinalizedBlock(ctx) } - finalizedBlockNumber := currentHeadNumber - int64(ht.config.FinalityDepth()) + // no need to make an additional RPC calls on chains with instant finality + if ht.config.FinalityDepth() == 0 { + return currentHead, nil + } + finalizedBlockNumber := currentHead.BlockNumber() - int64(ht.config.FinalityDepth()) if finalizedBlockNumber <= 0 { finalizedBlockNumber = 0 } diff --git a/common/headtracker/types/client.go b/common/headtracker/types/client.go index 246767e03fa..a1e419809b5 100644 --- a/common/headtracker/types/client.go +++ b/common/headtracker/types/client.go @@ -15,7 +15,6 @@ type Client[H types.Head[BLOCK_HASH], S types.Subscription, ID types.ID, BLOCK_H // SubscribeNewHead is the method in which the client receives new Head. // It can be implemented differently for each chain i.e websocket, polling, etc SubscribeNewHead(ctx context.Context, ch chan<- H) (S, error) - // LatestFinalizedBlock - returns the latest block that was marked as finalized. Only applicable for PoS chains - // with the finality tag support. + // LatestFinalizedBlock - returns the latest block that was marked as finalized LatestFinalizedBlock(ctx context.Context) (head H, err error) } diff --git a/core/internal/testutils/evmtest/evmtest.go b/core/internal/testutils/evmtest/evmtest.go index d4075ef444e..cc56c3c9e9b 100644 --- a/core/internal/testutils/evmtest/evmtest.go +++ b/core/internal/testutils/evmtest/evmtest.go @@ -294,7 +294,7 @@ func nodeStatus(n *evmtoml.Node, chainID string) (types.NodeStatus, error) { return s, nil } -func NewEthClientMock(t testing.TB) *evmclimocks.Client { +func NewEthClientMock(t *testing.T) *evmclimocks.Client { return evmclimocks.NewClient(t) } From faf61d91b178b62bf9dd961f280ea7f9605eb84b Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 23 Feb 2024 19:09:00 +0100 Subject: [PATCH 11/28] Finalized block based history depth --- common/headtracker/head_tracker.go | 86 +++++----- common/types/head_tracker.go | 9 +- .../evm/headtracker/head_broadcaster_test.go | 4 +- core/chains/evm/headtracker/head_saver.go | 42 +++-- .../chains/evm/headtracker/head_saver_test.go | 73 ++++++-- .../evm/headtracker/head_tracker_test.go | 161 ++++++++++-------- core/chains/evm/headtracker/heads.go | 53 ++++-- core/chains/evm/headtracker/heads_test.go | 49 +++--- core/chains/evm/headtracker/orm.go | 23 +-- core/chains/evm/headtracker/orm_test.go | 9 +- core/chains/evm/types/models.go | 15 +- 11 files changed, 323 insertions(+), 201 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 44535910684..78a90fd4901 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -97,18 +97,6 @@ func NewHeadTracker[ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Start(ctx context.Context) error { return ht.StartOnce("HeadTracker", func() error { ht.log.Debugw("Starting HeadTracker", "chainID", ht.chainID) - latestChain, err := ht.headSaver.Load(ctx) - if err != nil { - return err - } - if latestChain.IsValid() { - ht.log.Debugw( - fmt.Sprintf("HeadTracker: Tracking logs from last block %v with hash %s", latestChain.BlockNumber(), latestChain.BlockHash()), - "blockNumber", latestChain.BlockNumber(), - "blockHash", latestChain.BlockHash(), - ) - } - // NOTE: Always try to start the head tracker off with whatever the // latest head is, without waiting for the subscription to send us one. // @@ -116,18 +104,12 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Start(ctx context.Context) error // anyway when we connect (but we should not rely on this because it is // not specced). If it happens this is fine, and the head will be // ignored as a duplicate. - initialHead, err := ht.getInitialHead(ctx) + err := ht.loadInitialHead(ctx) if err != nil { - if errors.Is(err, ctx.Err()) { - return nil + if ctx.Err() != nil { + return ctx.Err() } ht.log.Errorw("Error getting initial head", "err", err) - } else if initialHead.IsValid() { - if err := ht.handleNewHead(ctx, initialHead); err != nil { - return fmt.Errorf("error handling initial head: %w", err) - } - } else { - ht.log.Debug("Got nil initial head") } ht.wgDone.Add(3) @@ -141,6 +123,45 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Start(ctx context.Context) error }) } +func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) loadInitialHead(ctx context.Context) error { + initialHead, err := ht.client.HeadByNumber(ctx, nil) + if err != nil { + return fmt.Errorf("failed to fetch initial head: %w", err) + } + + if !initialHead.IsValid() { + ht.log.Warnw("Got nil initial head", "head", initialHead) + return nil + } + ht.log.Debugw("Got initial head", "head", initialHead, "blockNumber", initialHead.BlockNumber(), "blockHash", initialHead.BlockHash()) + + latestFinalized, err := ht.calculateLatestFinalized(ctx, initialHead) + if err != nil { + return fmt.Errorf("failed to calculate latest finalized head: %w", err) + } + + latestChain, err := ht.headSaver.Load(ctx, latestFinalized) + if err != nil { + return fmt.Errorf("failed to initialzed headSaver: %w", err) + } + + if latestChain.IsValid() { + earliest := latestChain.EarliestHeadInChain() + ht.log.Debugw( + "Loaded chain from DB", + "latest_blockNumber", latestChain.BlockNumber(), + "latest_blockHash", latestChain.BlockHash(), + "earliest_blockNumber", earliest.BlockNumber(), + "earliest_blockHash", earliest.BlockHash(), + ) + } + if err := ht.handleNewHead(ctx, initialHead); err != nil { + return fmt.Errorf("error handling initial head: %w", err) + } + + return nil +} + // Close stops HeadTracker service. func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Close() error { return ht.StopOnce("HeadTracker", func() error { @@ -180,19 +201,6 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) LatestChain() HTH { return ht.headSaver.LatestChain() } -func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) getInitialHead(ctx context.Context) (HTH, error) { - head, err := ht.client.HeadByNumber(ctx, nil) - if err != nil { - return ht.getNilHead(), fmt.Errorf("failed to fetch initial head: %w", err) - } - loggerFields := []interface{}{"head", head} - if head.IsValid() { - loggerFields = append(loggerFields, "blockNumber", head.BlockNumber(), "blockHash", head.BlockHash()) - } - ht.log.Debugw("Got initial head", loggerFields...) - return head, nil -} - func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) handleNewHead(ctx context.Context, head HTH) error { prevHead := ht.headSaver.LatestChain() @@ -341,14 +349,6 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea "fromBlockHeight", baseHeight, "toBlockHeight", headBlockNumber-1) l.Debug("Starting backfill") - if ht.htConfig.HistoryDepth() < uint32(headBlockNumber-baseHeight) { - l.Warnw("HistoryDepth is smaller than the actual finality depth (number of blocks from the latest "+ - "finalized to the most recent block). This might be caused by an out-of-sync RPC that returned an old "+ - "finalized block or by HistoryDepth being too small. If you see this message too often, "+ - "consider increasing HistoryDepth.", - "history_depth", ht.htConfig.HistoryDepth(), - "actual_finality_depth", headBlockNumber-baseHeight) - } defer func() { if ctx.Err() != nil { l.Warnw("Backfill context error", "err", ctx.Err()) @@ -384,7 +384,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea return fmt.Errorf(errMsg) } - err = ht.headSaver.MarkFinalized(ctx, latestFinalizedHead.BlockHash()) + err = ht.headSaver.MarkFinalized(ctx, latestFinalizedHead) if err != nil { return fmt.Errorf("failed to mark head as finalized: %w", err) } diff --git a/common/types/head_tracker.go b/common/types/head_tracker.go index 6e92cb4a256..d89663a1641 100644 --- a/common/types/head_tracker.go +++ b/common/types/head_tracker.go @@ -36,15 +36,14 @@ type HeadSaver[H Head[BLOCK_HASH], BLOCK_HASH Hashable] interface { // Save updates the latest block number, if indeed the latest, and persists // this number in case of reboot. Save(ctx context.Context, head H) error - // Load loads latest EvmHeadTrackerHistoryDepth heads, returns the latest chain. - Load(ctx context.Context) (H, error) + // Load loads latest heads up to latestFinalized - historyDepth, returns the latest chain. + Load(ctx context.Context, latestFinalized H) (H, error) // LatestChain returns the block header with the highest number that has been seen, or nil. LatestChain() H // Chain returns a head for the specified hash, or nil. Chain(hash BLOCK_HASH) H - // MarkFinalized - marks matching block and all it's direct ancestors as finalized. Returns error if failed to find - // finalized block in the LatestChain - MarkFinalized(ctx context.Context, finalized BLOCK_HASH) error + // MarkFinalized - marks matching block and all it's direct ancestors as finalized + MarkFinalized(ctx context.Context, latestFinalized H) error } // HeadListener is a chain agnostic interface that manages connection of Client that receives heads from the blockchain node diff --git a/core/chains/evm/headtracker/head_broadcaster_test.go b/core/chains/evm/headtracker/head_broadcaster_test.go index 5c02fae4ad3..7c55f27c2fd 100644 --- a/core/chains/evm/headtracker/head_broadcaster_test.go +++ b/core/chains/evm/headtracker/head_broadcaster_test.go @@ -62,8 +62,8 @@ func TestHeadBroadcaster_Subscribe(t *testing.T) { chchHeaders <- args.Get(1).(chan<- *evmtypes.Head) }). Return(sub, nil) - // one for initial and 2 for backfill - ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(1), nil).Times(3) + // 2 for initial and 2 for backfill + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(1), nil).Times(4) sub.On("Unsubscribe").Return() sub.On("Err").Return(nil) diff --git a/core/chains/evm/headtracker/head_saver.go b/core/chains/evm/headtracker/head_saver.go index cd7d0883b44..f2d701763b4 100644 --- a/core/chains/evm/headtracker/head_saver.go +++ b/core/chains/evm/headtracker/head_saver.go @@ -38,23 +38,26 @@ func (hs *headSaver) Save(ctx context.Context, head *evmtypes.Head) error { return err } - historyDepth := uint(hs.htConfig.HistoryDepth()) - hs.heads.AddHeads(historyDepth, head) + hs.heads.AddHeads(head) - return hs.orm.TrimOldHeads(ctx, historyDepth) + return nil } -func (hs *headSaver) Load(ctx context.Context) (chain *evmtypes.Head, err error) { - historyDepth := uint(hs.htConfig.HistoryDepth()) - heads, err := hs.orm.LatestHeads(ctx, historyDepth) +func (hs *headSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) (chain *evmtypes.Head, err error) { + minBlockNumber := hs.calculateDeepestToKeep(latestFinalized.BlockNumber()) + heads, err := hs.orm.LatestHeads(ctx, minBlockNumber) if err != nil { return nil, err } - hs.heads.AddHeads(historyDepth, heads...) + hs.heads.AddHeads(heads...) return hs.heads.LatestHead(), nil } +func (hs *headSaver) calculateDeepestToKeep(latestFinalized int64) int64 { + return latestFinalized - int64(hs.htConfig.HistoryDepth()) +} + func (hs *headSaver) LatestHeadFromDB(ctx context.Context) (head *evmtypes.Head, err error) { return hs.orm.LatestHead(ctx) } @@ -74,21 +77,26 @@ func (hs *headSaver) Chain(hash common.Hash) *evmtypes.Head { return hs.heads.HeadByHash(hash) } -func (hs *headSaver) MarkFinalized(ctx context.Context, finalized common.Hash) error { - if hs.heads.MarkFinalized(finalized) { - return nil +func (hs *headSaver) MarkFinalized(ctx context.Context, finalized *evmtypes.Head) error { + deepestToKeep := hs.calculateDeepestToKeep(finalized.BlockNumber()) + if !hs.heads.MarkFinalized(finalized.BlockHash(), deepestToKeep) { + return fmt.Errorf("failed to find %s block in the canonical chain to mark it as finalized", finalized) } - return fmt.Errorf("failed to find %s block in the canonical chain to mark it as finalized", finalized) + return hs.orm.TrimOldHeads(ctx, deepestToKeep) } var NullSaver httypes.HeadSaver = &nullSaver{} type nullSaver struct{} -func (*nullSaver) Save(ctx context.Context, head *evmtypes.Head) error { return nil } -func (*nullSaver) Load(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } -func (*nullSaver) LatestHeadFromDB(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } -func (*nullSaver) LatestChain() *evmtypes.Head { return nil } -func (*nullSaver) Chain(hash common.Hash) *evmtypes.Head { return nil } -func (*nullSaver) MarkFinalized(ctx context.Context, finalized common.Hash) error { return nil } +func (*nullSaver) Save(ctx context.Context, head *evmtypes.Head) error { return nil } +func (*nullSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) (*evmtypes.Head, error) { + return nil, nil +} +func (*nullSaver) LatestHeadFromDB(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } +func (*nullSaver) LatestChain() *evmtypes.Head { return nil } +func (*nullSaver) Chain(hash common.Hash) *evmtypes.Head { return nil } +func (*nullSaver) MarkFinalized(ctx context.Context, latestFinalized *evmtypes.Head) error { + return nil +} diff --git a/core/chains/evm/headtracker/head_saver_test.go b/core/chains/evm/headtracker/head_saver_test.go index 4361ef863e3..4df9dfff86a 100644 --- a/core/chains/evm/headtracker/head_saver_test.go +++ b/core/chains/evm/headtracker/head_saver_test.go @@ -1,15 +1,20 @@ package headtracker_test import ( + "math/big" "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" + ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/configtest" @@ -47,20 +52,27 @@ func (c *config) FinalityTagEnabled() bool { return c.finalityTagEnabled } -func configureSaver(t *testing.T) (httypes.HeadSaver, headtracker.ORM) { +type saverOpts struct { + headTrackerConfig *headTrackerConfig +} + +func configureSaver(t *testing.T, opts saverOpts) (httypes.HeadSaver, headtracker.ORM) { + if opts.headTrackerConfig == nil { + opts.headTrackerConfig = &headTrackerConfig{historyDepth: 6} + } db := pgtest.NewSqlxDB(t) lggr := logger.Test(t) cfg := configtest.NewGeneralConfig(t, nil) htCfg := &config{finalityDepth: uint32(1)} orm := headtracker.NewORM(db, lggr, cfg.Database(), cltest.FixtureChainID) - saver := headtracker.NewHeadSaver(lggr, orm, htCfg, &headTrackerConfig{historyDepth: 6}) + saver := headtracker.NewHeadSaver(lggr, orm, htCfg, opts.headTrackerConfig) return saver, orm } func TestHeadSaver_Save(t *testing.T) { t.Parallel() - saver, _ := configureSaver(t) + saver, _ := configureSaver(t, saverOpts{}) head := cltest.Head(1) err := saver.Save(testutils.Context(t), head) @@ -82,19 +94,56 @@ func TestHeadSaver_Save(t *testing.T) { func TestHeadSaver_Load(t *testing.T) { t.Parallel() - saver, orm := configureSaver(t) - - for i := 0; i < 5; i++ { - err := orm.IdempotentInsertHead(testutils.Context(t), cltest.Head(i)) + saver, orm := configureSaver(t, saverOpts{ + headTrackerConfig: &headTrackerConfig{historyDepth: 4}, + }) + + // create chain + // H0 <- H1 <- H2 <- H3 <- H4 <- H5 + // \ + // H2Uncle + // + newHead := func(num int, parent common.Hash) *evmtypes.Head { + h := evmtypes.NewHead(big.NewInt(int64(num)), utils.NewHash(), parent, uint64(time.Now().Unix()), ubig.NewI(0)) + return &h + } + h0 := newHead(0, utils.NewHash()) + h1 := newHead(1, h0.Hash) + h2 := newHead(2, h1.Hash) + h3 := newHead(3, h2.Hash) + h4 := newHead(4, h3.Hash) + h5 := newHead(5, h4.Hash) + h2Uncle := newHead(2, h1.Hash) + + allHeads := []*evmtypes.Head{h0, h1, h2, h2Uncle, h3, h4, h5} + + for _, h := range allHeads { + err := orm.IdempotentInsertHead(testutils.Context(t), h) require.NoError(t, err) } - latestHead, err := saver.Load(testutils.Context(t)) + verifyLatestHead := func(latestHead *evmtypes.Head) { + // latest head matches h5 and chain does not include h0 + require.NotNil(t, latestHead) + require.Equal(t, int64(5), latestHead.Number) + require.Equal(t, uint32(5), latestHead.ChainLength()) + require.Nil(t, latestHead.HeadAtHeight(0)) + } + + // load all from [h5-historyDepth, h5] + latestHead, err := saver.Load(testutils.Context(t), h5) require.NoError(t, err) + // verify latest head loaded from db + verifyLatestHead(latestHead) + + //verify latest head loaded from memory store + latestHead = saver.LatestChain() require.NotNil(t, latestHead) - require.Equal(t, int64(4), latestHead.Number) + verifyLatestHead(latestHead) + + // h2Uncle was loaded and has chain up to h1 + uncleChain := saver.Chain(h2Uncle.Hash) + require.NotNil(t, uncleChain) + require.Equal(t, uint32(2), uncleChain.ChainLength()) // h2Uncle -> h1 - latestChain := saver.LatestChain() - require.NotNil(t, latestChain) - require.Equal(t, int64(4), latestChain.Number) } diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 1fc472ca052..14c5216c00e 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -16,8 +16,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "golang.org/x/exp/maps" + "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" + "github.com/jmoiron/sqlx" commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" @@ -27,7 +31,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/utils/mailbox/mailboxtest" commonmocks "github.com/smartcontractkit/chainlink/v2/common/types/mocks" - evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" evmclimocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" @@ -57,6 +60,8 @@ func TestHeadTracker_New(t *testing.T) { config := configtest.NewGeneralConfig(t, nil) ethClient := evmtest.NewEthClientMockWithDefaultChain(t) ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), nil) + // finalized + ethClient.On("HeadByNumber", mock.Anything, big.NewInt(0)).Return(cltest.Head(0), nil) orm := headtracker.NewORM(db, logger, config.Database(), cltest.FixtureChainID) assert.Nil(t, orm.IdempotentInsertHead(testutils.Context(t), cltest.Head(1))) @@ -73,12 +78,15 @@ func TestHeadTracker_New(t *testing.T) { assert.Equal(t, last.Number, latest.Number) } -func TestHeadTracker_Save_InsertsAndTrimsTable(t *testing.T) { +func TestHeadTracker_MarkFinalized_MarksAndTrimsTable(t *testing.T) { t.Parallel() db := pgtest.NewSqlxDB(t) logger := logger.Test(t) - config := cltest.NewTestChainScopedConfig(t) + gCfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, _ *chainlink.Secrets) { + c.EVM[0].HeadTracker.HistoryDepth = ptr[uint32](100) + }) + config := evmtest.NewChainScopedConfig(t, gCfg) ethClient := evmtest.NewEthClientMockWithDefaultChain(t) orm := headtracker.NewORM(db, logger, config.Database(), cltest.FixtureChainID) @@ -87,18 +95,21 @@ func TestHeadTracker_Save_InsertsAndTrimsTable(t *testing.T) { assert.Nil(t, orm.IdempotentInsertHead(testutils.Context(t), cltest.Head(idx))) } - ht := createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm) + latest := cltest.Head(201) + assert.Nil(t, orm.IdempotentInsertHead(testutils.Context(t), latest)) - h := cltest.Head(200) - require.NoError(t, ht.headSaver.Save(testutils.Context(t), h)) - assert.Equal(t, big.NewInt(200), ht.headSaver.LatestChain().ToInt()) + ht := createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm) + _, err := ht.headSaver.Load(testutils.Context(t), latest) + require.NoError(t, err) + require.NoError(t, ht.headSaver.MarkFinalized(testutils.Context(t), latest)) + assert.Equal(t, big.NewInt(201), ht.headSaver.LatestChain().ToInt()) firstHead := firstHead(t, db) assert.Equal(t, big.NewInt(101), firstHead.ToInt()) lastHead, err := orm.LatestHead(testutils.Context(t)) require.NoError(t, err) - assert.Equal(t, int64(200), lastHead.Number) + assert.Equal(t, int64(201), lastHead.Number) } func TestHeadTracker_Get(t *testing.T) { @@ -180,6 +191,8 @@ func TestHeadTracker_Start_NewHeads(t *testing.T) { // for initial load ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), nil).Once() ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(0), nil).Once() + // for backfill + ethClient.On("HeadByNumber", mock.Anything, mock.Anything).Return(cltest.Head(0), nil).Maybe() ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything). Run(func(mock.Arguments) { close(chStarted) @@ -192,43 +205,65 @@ func TestHeadTracker_Start_NewHeads(t *testing.T) { <-chStarted } -func TestHeadTracker_Start_CancelContext(t *testing.T) { +func TestHeadTracker_Start(t *testing.T) { t.Parallel() - db := pgtest.NewSqlxDB(t) - logger := logger.Test(t) - config := cltest.NewTestChainScopedConfig(t) - orm := headtracker.NewORM(db, logger, config.Database(), cltest.FixtureChainID) - ethClient := evmtest.NewEthClientMockWithDefaultChain(t) - chStarted := make(chan struct{}) - ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Run(func(args mock.Arguments) { - ctx := args.Get(0).(context.Context) - select { - case <-ctx.Done(): - return - case <-time.After(10 * time.Second): - assert.FailNow(t, "context was not cancelled within 10s") - } - }).Return(cltest.Head(0), nil) - mockEth := &evmtest.MockEth{EthClient: ethClient} - sub := mockEth.NewSub(t) - ethClient.On("SubscribeNewHead", mock.Anything, mock.Anything). - Run(func(mock.Arguments) { - close(chStarted) - }). - Return(sub, nil). - Maybe() - - ht := createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm) + const historyDepth = 100 + newHeadTracker := func(t *testing.T) *headTrackerUniverse { + db := pgtest.NewSqlxDB(t) + gCfg := configtest.NewGeneralConfig(t, func(c *chainlink.Config, _ *chainlink.Secrets) { + c.EVM[0].FinalityTagEnabled = ptr[bool](true) + c.EVM[0].HeadTracker.HistoryDepth = ptr[uint32](historyDepth) + }) + config := evmtest.NewChainScopedConfig(t, gCfg) + orm := headtracker.NewORM(db, logger.Test(t), config.Database(), cltest.FixtureChainID) + ethClient := evmtest.NewEthClientMockWithDefaultChain(t) + return createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm) + } - ctx, cancel := context.WithCancel(testutils.Context(t)) - go func() { - time.Sleep(1 * time.Second) - cancel() - }() - err := ht.headTracker.Start(ctx) - require.NoError(t, err) - require.NoError(t, ht.headTracker.Close()) + t.Run("Fail start if context was canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(testutils.Context(t)) + ht := newHeadTracker(t) + ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Run(func(args mock.Arguments) { + cancel() + }).Return(cltest.Head(0), context.Canceled) + err := ht.headTracker.Start(ctx) + require.ErrorIs(t, err, context.Canceled) + }) + t.Run("Starts even if failed to get initialHead", func(t *testing.T) { + ht := newHeadTracker(t) + ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), errors.New("failed to get init head")) + ht.Start(t) + tests.AssertLogEventually(t, ht.observer, "Error getting initial head") + }) + t.Run("Starts even if received invalid head", func(t *testing.T) { + ht := newHeadTracker(t) + ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(nil, nil) + ht.Start(t) + tests.AssertLogEventually(t, ht.observer, "Got nil initial head") + }) + t.Run("Starts even if fails to get finalizedHead", func(t *testing.T) { + ht := newHeadTracker(t) + head := cltest.Head(1000) + ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once() + ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New("failed to load latest finalized")).Once() + ht.Start(t) + tests.AssertLogEventually(t, ht.observer, "Error getting initial head") + }) + t.Run("Happy path", func(t *testing.T) { + head := cltest.Head(1000) + ht := newHeadTracker(t) + ctx := testutils.Context(t) + require.NoError(t, ht.orm.IdempotentInsertHead(ctx, cltest.Head(799))) + ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once() + finalizedHead := cltest.Head(800) + // on start + ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(finalizedHead, nil).Once() + // on backfill + ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New("backfill call to finalized failed")).Maybe() + ht.Start(t) + tests.AssertLogEventually(t, ht.observer, "Loaded chain from DB") + }) } func TestHeadTracker_CallsHeadTrackableCallbacks(t *testing.T) { @@ -781,15 +816,6 @@ func TestHeadTracker_Backfill(t *testing.T) { ctx := testutils.Context(t) - type headTrackerUniverse struct { - ethClient *evmclimocks.Client - orm headtracker.ORM - headTracker httypes.HeadTracker - headBroadcaster httypes.HeadBroadcaster - headSaver httypes.HeadSaver - mailMon *mailbox.Monitor - } - type opts struct { Heads []evmtypes.Head } @@ -797,27 +823,17 @@ func TestHeadTracker_Backfill(t *testing.T) { cfg := configtest.NewGeneralConfig(t, nil) evmcfg := evmtest.NewChainScopedConfig(t, cfg) lggr := logger.Test(t) - hb := headtracker.NewHeadBroadcaster(lggr) db := pgtest.NewSqlxDB(t) orm := headtracker.NewORM(db, lggr, cfg.Database(), cltest.FixtureChainID) for i := range opts.Heads { require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) } - hs := headtracker.NewHeadSaver(lggr, orm, evmcfg.EVM(), evmcfg.EVM().HeadTracker()) - mailMon := mailboxtest.NewMonitor(t) ethClient := evmtest.NewEthClientMock(t) ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) - ht := headtracker.NewHeadTracker(lggr, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), hb, hs, mailMon) - _, err := hs.Load(testutils.Context(t)) + ht := createHeadTracker(t, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), orm) + _, err := ht.headSaver.Load(testutils.Context(t), cltest.Head(0)) require.NoError(t, err) - return &headTrackerUniverse{ - ethClient: ethClient, - orm: orm, - headTracker: ht, - headBroadcaster: hb, - headSaver: hs, - mailMon: mailMon, - } + return ht } t.Run("returns error if latestFinalized is not valid", func(t *testing.T) { @@ -956,8 +972,8 @@ func TestHeadTracker_Backfill(t *testing.T) { }) } -func createHeadTracker(t *testing.T, ethClient evmclient.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM) *headTrackerUniverse { - lggr := logger.Test(t) +func createHeadTracker(t *testing.T, ethClient *evmclimocks.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM) *headTrackerUniverse { + lggr, ob := logger.TestObserved(t, zap.DebugLevel) hb := headtracker.NewHeadBroadcaster(lggr) hs := headtracker.NewHeadSaver(lggr, orm, config, htConfig) mailMon := mailboxtest.NewMonitor(t) @@ -967,11 +983,14 @@ func createHeadTracker(t *testing.T, ethClient evmclient.Client, config headtrac headBroadcaster: hb, headSaver: hs, mailMon: mailMon, + observer: ob, + orm: orm, + ethClient: ethClient, } } -func createHeadTrackerWithChecker(t *testing.T, ethClient evmclient.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM, checker httypes.HeadTrackable) *headTrackerUniverse { - lggr := logger.Test(t) +func createHeadTrackerWithChecker(t *testing.T, ethClient *evmclimocks.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM, checker httypes.HeadTrackable) *headTrackerUniverse { + lggr, ob := logger.TestObserved(t, zap.DebugLevel) hb := headtracker.NewHeadBroadcaster(lggr) hs := headtracker.NewHeadSaver(lggr, orm, config, htConfig) hb.Subscribe(checker) @@ -983,6 +1002,9 @@ func createHeadTrackerWithChecker(t *testing.T, ethClient evmclient.Client, conf headBroadcaster: hb, headSaver: hs, mailMon: mailMon, + observer: ob, + orm: orm, + ethClient: ethClient, } } @@ -993,6 +1015,9 @@ type headTrackerUniverse struct { headBroadcaster httypes.HeadBroadcaster headSaver httypes.HeadSaver mailMon *mailbox.Monitor + observer *observer.ObservedLogs + orm headtracker.ORM + ethClient *evmclimocks.Client } func (u *headTrackerUniverse) Backfill(ctx context.Context, head, finalizedHead *evmtypes.Head) error { diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 3099a5ca87c..9cf286de0c1 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -17,11 +17,11 @@ type Heads interface { HeadByHash(hash common.Hash) *evmtypes.Head // AddHeads adds newHeads to the collection, eliminates duplicates, // sorts by head number, fixes parents and cuts off old heads (historyDepth). - AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) + AddHeads(newHeads ...*evmtypes.Head) // Count returns number of heads in the collection. Count() int // MarkFinalized - finds `finalized` in the LatestHead and marks it and all direct ancestors as finalized - MarkFinalized(finalized common.Hash) bool + MarkFinalized(finalized common.Hash, deepestToKeep int64) bool } type heads struct { @@ -62,7 +62,9 @@ func (h *heads) Count() int { return len(h.heads) } -func (h *heads) MarkFinalized(finalized common.Hash) bool { +// MarkFinalized - marks block with has equal to finalized and all it's direct ancestors as finalized. +// Trims old blocks whose height is smaller than deepestToKeep +func (h *heads) MarkFinalized(finalized common.Hash, deepestToKeep int64) bool { h.mu.Lock() defer h.mu.Unlock() @@ -70,7 +72,8 @@ func (h *heads) MarkFinalized(finalized common.Hash) bool { return false } - h.heads = deepCopyUpTo(h.heads, uint(len(h.heads))) + // deep copy to avoid race + h.heads = deepCopy(h.heads) head := h.heads[0] foundFinalized := false @@ -85,10 +88,40 @@ func (h *heads) MarkFinalized(finalized common.Hash) bool { head = head.Parent } + // trim blocks that are too deep + h.trimRedundantBlocks(deepestToKeep) + return foundFinalized } -func deepCopyUpTo(oldHeads []*evmtypes.Head, depth uint) []*evmtypes.Head { +// trimRedundantBlocks - trims all the blocks whose blockNumber < deepestToKeep +// Not thread safe. Must be called on fresh copy of h.heads +func (h *heads) trimRedundantBlocks(deepestToKeep int64) { + if len(h.heads) == 0 { + return + } + + deepestBlock := h.heads[0].HeadAtHeight(deepestToKeep) + if deepestBlock == nil { + return + } + + for i, head := range h.heads { + // ensure that uncle chains and canonical chain do not go deeper than deepestToKeep + if deepestBlock.Parent == head.Parent { + head.Parent = nil + } + // trim slice + if head == deepestBlock { + h.heads = h.heads[:i+1] + return + } + } + + panic("invariant violation: expected deepestToKeep to present in the heads slice since we've seen it before") +} + +func deepCopy(oldHeads []*evmtypes.Head) []*evmtypes.Head { headsMap := make(map[common.Hash]*evmtypes.Head, len(oldHeads)) for _, head := range oldHeads { if head.Hash == head.ParentHash { @@ -122,11 +155,6 @@ func deepCopyUpTo(oldHeads []*evmtypes.Head, depth uint) []*evmtypes.Head { return heads[i].Number > heads[j].Number }) - // cut off the oldest - if uint(len(heads)) > depth { - heads = heads[:depth] - } - // assign parents for i := 0; i < len(heads)-1; i++ { head := heads[i] @@ -139,9 +167,10 @@ func deepCopyUpTo(oldHeads []*evmtypes.Head, depth uint) []*evmtypes.Head { return heads } -func (h *heads) AddHeads(historyDepth uint, newHeads ...*evmtypes.Head) { +func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { h.mu.Lock() defer h.mu.Unlock() - h.heads = deepCopyUpTo(append(h.heads, newHeads...), historyDepth) + // deep copy to avoid race + h.heads = deepCopy(append(h.heads, newHeads...)) } diff --git a/core/chains/evm/headtracker/heads_test.go b/core/chains/evm/headtracker/heads_test.go index 42ad173ab36..09eda2c3b34 100644 --- a/core/chains/evm/headtracker/heads_test.go +++ b/core/chains/evm/headtracker/heads_test.go @@ -20,18 +20,18 @@ func TestHeads_LatestHead(t *testing.T) { t.Parallel() heads := headtracker.NewHeads() - heads.AddHeads(3, cltest.Head(100), cltest.Head(200), cltest.Head(300)) + heads.AddHeads(cltest.Head(100), cltest.Head(200), cltest.Head(300)) latest := heads.LatestHead() require.NotNil(t, latest) require.Equal(t, int64(300), latest.Number) - heads.AddHeads(3, cltest.Head(250)) + heads.AddHeads(cltest.Head(250)) latest = heads.LatestHead() require.NotNil(t, latest) require.Equal(t, int64(300), latest.Number) - heads.AddHeads(3, cltest.Head(400)) + heads.AddHeads(cltest.Head(400)) latest = heads.LatestHead() require.NotNil(t, latest) require.Equal(t, int64(400), latest.Number) @@ -46,7 +46,7 @@ func TestHeads_HeadByHash(t *testing.T) { cltest.Head(300), } heads := headtracker.NewHeads() - heads.AddHeads(3, testHeads...) + heads.AddHeads(testHeads...) head := heads.HeadByHash(testHeads[1].Hash) require.NotNil(t, head) @@ -62,11 +62,11 @@ func TestHeads_Count(t *testing.T) { heads := headtracker.NewHeads() require.Zero(t, heads.Count()) - heads.AddHeads(3, cltest.Head(100), cltest.Head(200), cltest.Head(300)) + heads.AddHeads(cltest.Head(100), cltest.Head(200), cltest.Head(300)) require.Equal(t, 3, heads.Count()) - heads.AddHeads(1, cltest.Head(400)) - require.Equal(t, 1, heads.Count()) + heads.AddHeads(cltest.Head(400)) + require.Equal(t, 4, heads.Count()) } func TestHeads_AddHeads(t *testing.T) { @@ -89,9 +89,10 @@ func TestHeads_AddHeads(t *testing.T) { parentHash = hash } - heads.AddHeads(6, testHeads...) + heads.AddHeads(testHeads...) + require.Equal(t, 6, heads.Count()) // Add duplicates (should be ignored) - heads.AddHeads(6, testHeads[2:5]...) + heads.AddHeads(testHeads[2:5]...) require.Equal(t, 6, heads.Count()) head := heads.LatestHead() @@ -101,13 +102,6 @@ func TestHeads_AddHeads(t *testing.T) { head = heads.HeadByHash(uncleHash) require.NotNil(t, head) require.Equal(t, 3, int(head.ChainLength())) - - // Adding beyond the limit truncates - heads.AddHeads(2, testHeads...) - require.Equal(t, 2, heads.Count()) - head = heads.LatestHead() - require.NotNil(t, head) - require.Equal(t, 2, int(head.ChainLength())) } func TestHeads_MarkFinalized(t *testing.T) { @@ -116,7 +110,7 @@ func TestHeads_MarkFinalized(t *testing.T) { heads := headtracker.NewHeads() // create chain - // H0 <- H1 <- H2 <- H3 <- H4 + // H0 <- H1 <- H2 <- H3 <- H4 <- H5 // \ // H2Uncle // @@ -129,30 +123,39 @@ func TestHeads_MarkFinalized(t *testing.T) { h2 := newHead(2, h1.Hash) h3 := newHead(3, h2.Hash) h4 := newHead(4, h3.Hash) - h5 := newHead(5, h3.Hash) + h5 := newHead(5, h4.Hash) h2Uncle := newHead(2, h1.Hash) allHeads := []*evmtypes.Head{h0, h1, h2, h2Uncle, h3, h4, h5} - heads.AddHeads(1000, allHeads...) + heads.AddHeads(allHeads...) // mark h3 and all ancestors as finalized - require.True(t, heads.MarkFinalized(h3.Hash), "expected MarkFinalized succeed") + require.True(t, heads.MarkFinalized(h3.Hash, h1.BlockNumber()), "expected MarkFinalized succeed") + + // original heads remain unchanged for _, h := range allHeads { assert.False(t, h.IsFinalized, "expected original heads to remain unfinalized") } - require.False(t, heads.MarkFinalized(utils.NewHash()), "expected false if finalized hash was not found in existing LatestHead chain") + + // h0 is too old. It should not be available directly or through its children + assert.Nil(t, heads.HeadByHash(h0.Hash)) + assert.Nil(t, heads.HeadByHash(h1.Hash).Parent) + assert.Nil(t, heads.HeadByHash(h2Uncle.Hash).Parent.Parent) + + require.False(t, heads.MarkFinalized(utils.NewHash(), 0), "expected false if finalized hash was not found in existing LatestHead chain") + ensureProperFinalization := func(t *testing.T) { t.Helper() for _, head := range []*evmtypes.Head{h5, h4} { require.False(t, heads.HeadByHash(head.Hash).IsFinalized, "expected h4-h5 not to be finalized", head.BlockNumber()) } - for _, head := range []*evmtypes.Head{h3, h2, h1, h0} { + for _, head := range []*evmtypes.Head{h3, h2, h1} { require.True(t, heads.HeadByHash(head.Hash).IsFinalized, "expected h3 and all ancestors to be finalized", head.BlockNumber()) } require.False(t, heads.HeadByHash(h2Uncle.Hash).IsFinalized, "expected uncle block not to be marked as finalized") } t.Run("blocks were correctly marked as finalized", ensureProperFinalization) - heads.AddHeads(1000, h0, h1, h2, h2Uncle, h3, h4, h5) + heads.AddHeads(h0, h1, h2, h2Uncle, h3, h4, h5) t.Run("blocks remain finalized after re adding them to the Heads", ensureProperFinalization) } diff --git a/core/chains/evm/headtracker/orm.go b/core/chains/evm/headtracker/orm.go index 859f6764b63..99ecd978b31 100644 --- a/core/chains/evm/headtracker/orm.go +++ b/core/chains/evm/headtracker/orm.go @@ -11,6 +11,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/smartcontractkit/chainlink-common/pkg/logger" + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" "github.com/smartcontractkit/chainlink/v2/core/services/pg" @@ -21,11 +22,11 @@ type ORM interface { // No advisory lock required because this is thread safe. IdempotentInsertHead(ctx context.Context, head *evmtypes.Head) error // TrimOldHeads deletes heads such that only the top N block numbers remain - TrimOldHeads(ctx context.Context, n uint) (err error) + TrimOldHeads(ctx context.Context, minBlockNumber int64) (err error) // LatestHead returns the highest seen head LatestHead(ctx context.Context) (head *evmtypes.Head, err error) - // LatestHeads returns the latest heads up to given limit - LatestHeads(ctx context.Context, limit uint) (heads []*evmtypes.Head, err error) + // LatestHeads returns the latest heads with blockNumbers > minBlockNumber + LatestHeads(ctx context.Context, minBlockNumber int64) (heads []*evmtypes.Head, err error) // HeadByHash fetches the head with the given hash from the db, returns nil if none exists HeadByHash(ctx context.Context, hash common.Hash) (head *evmtypes.Head, err error) } @@ -50,19 +51,11 @@ func (orm *orm) IdempotentInsertHead(ctx context.Context, head *evmtypes.Head) e return errors.Wrap(err, "IdempotentInsertHead failed to insert head") } -func (orm *orm) TrimOldHeads(ctx context.Context, n uint) (err error) { +func (orm *orm) TrimOldHeads(ctx context.Context, minBlockNumber int64) (err error) { q := orm.q.WithOpts(pg.WithParentCtx(ctx)) return q.ExecQ(` DELETE FROM evm.heads - WHERE evm_chain_id = $1 AND number < ( - SELECT min(number) FROM ( - SELECT number - FROM evm.heads - WHERE evm_chain_id = $1 - ORDER BY number DESC - LIMIT $2 - ) numbers - )`, orm.chainID, n) + WHERE evm_chain_id = $1 AND number < $2`, orm.chainID, minBlockNumber) } func (orm *orm) LatestHead(ctx context.Context) (head *evmtypes.Head, err error) { @@ -76,9 +69,9 @@ func (orm *orm) LatestHead(ctx context.Context) (head *evmtypes.Head, err error) return } -func (orm *orm) LatestHeads(ctx context.Context, limit uint) (heads []*evmtypes.Head, err error) { +func (orm *orm) LatestHeads(ctx context.Context, minBlockNumber int64) (heads []*evmtypes.Head, err error) { q := orm.q.WithOpts(pg.WithParentCtx(ctx)) - err = q.Select(&heads, `SELECT * FROM evm.heads WHERE evm_chain_id = $1 ORDER BY number DESC, created_at DESC, id DESC LIMIT $2`, orm.chainID, limit) + err = q.Select(&heads, `SELECT * FROM evm.heads WHERE evm_chain_id = $1 AND number >= $2 ORDER BY number DESC, created_at DESC, id DESC`, orm.chainID, minBlockNumber) err = errors.Wrap(err, "LatestHeads failed") return } diff --git a/core/chains/evm/headtracker/orm_test.go b/core/chains/evm/headtracker/orm_test.go index c9a2146daf2..7f99e535093 100644 --- a/core/chains/evm/headtracker/orm_test.go +++ b/core/chains/evm/headtracker/orm_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" @@ -56,13 +57,17 @@ func TestORM_TrimOldHeads(t *testing.T) { require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), head)) } + uncleHead := cltest.Head(5) + require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), uncleHead)) + err := orm.TrimOldHeads(testutils.Context(t), 5) require.NoError(t, err) - heads, err := orm.LatestHeads(testutils.Context(t), 10) + heads, err := orm.LatestHeads(testutils.Context(t), 0) require.NoError(t, err) - require.Equal(t, 5, len(heads)) + // uncle block was loaded too + require.Equal(t, 6, len(heads)) for i := 0; i < 5; i++ { require.LessOrEqual(t, int64(5), heads[i].Number) } diff --git a/core/chains/evm/types/models.go b/core/chains/evm/types/models.go index 618e77413d5..917338a185f 100644 --- a/core/chains/evm/types/models.go +++ b/core/chains/evm/types/models.go @@ -118,16 +118,27 @@ func (h *Head) IsInChain(blockHash common.Hash) bool { // HashAtHeight returns the hash of the block at the given height, if it is in the chain. // If not in chain, returns the zero hash func (h *Head) HashAtHeight(blockNum int64) common.Hash { + head := h.HeadAtHeight(blockNum) + if head == nil { + return common.Hash{} + } + + return head.Hash +} + +// HeadAtHeight returns the head at the given height, if it is in the chain. +// If not in chain, returns nil +func (h *Head) HeadAtHeight(blockNum int64) *Head { for { if h.Number == blockNum { - return h.Hash + return h } if h.Parent == nil { break } h = h.Parent } - return common.Hash{} + return nil } // ChainLength returns the length of the chain followed by recursively looking up parents From 89a75b340e528623ef9c0afe487ff76944a9bda2 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 23 Feb 2024 21:28:20 +0100 Subject: [PATCH 12/28] simplify trimming --- core/chains/evm/headtracker/head_saver.go | 10 ++-- core/chains/evm/headtracker/heads.go | 56 ++++++++--------------- 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/core/chains/evm/headtracker/head_saver.go b/core/chains/evm/headtracker/head_saver.go index f2d701763b4..412a66d5950 100644 --- a/core/chains/evm/headtracker/head_saver.go +++ b/core/chains/evm/headtracker/head_saver.go @@ -44,7 +44,7 @@ func (hs *headSaver) Save(ctx context.Context, head *evmtypes.Head) error { } func (hs *headSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) (chain *evmtypes.Head, err error) { - minBlockNumber := hs.calculateDeepestToKeep(latestFinalized.BlockNumber()) + minBlockNumber := hs.calculateMinBlockToKeep(latestFinalized.BlockNumber()) heads, err := hs.orm.LatestHeads(ctx, minBlockNumber) if err != nil { return nil, err @@ -54,7 +54,7 @@ func (hs *headSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) ( return hs.heads.LatestHead(), nil } -func (hs *headSaver) calculateDeepestToKeep(latestFinalized int64) int64 { +func (hs *headSaver) calculateMinBlockToKeep(latestFinalized int64) int64 { return latestFinalized - int64(hs.htConfig.HistoryDepth()) } @@ -78,12 +78,12 @@ func (hs *headSaver) Chain(hash common.Hash) *evmtypes.Head { } func (hs *headSaver) MarkFinalized(ctx context.Context, finalized *evmtypes.Head) error { - deepestToKeep := hs.calculateDeepestToKeep(finalized.BlockNumber()) - if !hs.heads.MarkFinalized(finalized.BlockHash(), deepestToKeep) { + minBlockToKeep := hs.calculateMinBlockToKeep(finalized.BlockNumber()) + if !hs.heads.MarkFinalized(finalized.BlockHash(), minBlockToKeep) { return fmt.Errorf("failed to find %s block in the canonical chain to mark it as finalized", finalized) } - return hs.orm.TrimOldHeads(ctx, deepestToKeep) + return hs.orm.TrimOldHeads(ctx, minBlockToKeep) } var NullSaver httypes.HeadSaver = &nullSaver{} diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 9cf286de0c1..057ddbecf0d 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -20,8 +20,9 @@ type Heads interface { AddHeads(newHeads ...*evmtypes.Head) // Count returns number of heads in the collection. Count() int - // MarkFinalized - finds `finalized` in the LatestHead and marks it and all direct ancestors as finalized - MarkFinalized(finalized common.Hash, deepestToKeep int64) bool + // MarkFinalized - finds `finalized` in the LatestHead and marks it and all direct ancestors as finalized. + // Trims old blocks whose height is smaller than minBlockToKeep + MarkFinalized(finalized common.Hash, minBlockToKeep int64) bool } type heads struct { @@ -63,8 +64,8 @@ func (h *heads) Count() int { } // MarkFinalized - marks block with has equal to finalized and all it's direct ancestors as finalized. -// Trims old blocks whose height is smaller than deepestToKeep -func (h *heads) MarkFinalized(finalized common.Hash, deepestToKeep int64) bool { +// Trims old blocks whose height is smaller than minBlockToKeep +func (h *heads) MarkFinalized(finalized common.Hash, minBlockToKeep int64) bool { h.mu.Lock() defer h.mu.Unlock() @@ -73,7 +74,7 @@ func (h *heads) MarkFinalized(finalized common.Hash, deepestToKeep int64) bool { } // deep copy to avoid race - h.heads = deepCopy(h.heads) + h.heads = deepCopy(h.heads, minBlockToKeep) head := h.heads[0] foundFinalized := false @@ -88,40 +89,10 @@ func (h *heads) MarkFinalized(finalized common.Hash, deepestToKeep int64) bool { head = head.Parent } - // trim blocks that are too deep - h.trimRedundantBlocks(deepestToKeep) - return foundFinalized } -// trimRedundantBlocks - trims all the blocks whose blockNumber < deepestToKeep -// Not thread safe. Must be called on fresh copy of h.heads -func (h *heads) trimRedundantBlocks(deepestToKeep int64) { - if len(h.heads) == 0 { - return - } - - deepestBlock := h.heads[0].HeadAtHeight(deepestToKeep) - if deepestBlock == nil { - return - } - - for i, head := range h.heads { - // ensure that uncle chains and canonical chain do not go deeper than deepestToKeep - if deepestBlock.Parent == head.Parent { - head.Parent = nil - } - // trim slice - if head == deepestBlock { - h.heads = h.heads[:i+1] - return - } - } - - panic("invariant violation: expected deepestToKeep to present in the heads slice since we've seen it before") -} - -func deepCopy(oldHeads []*evmtypes.Head) []*evmtypes.Head { +func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head { headsMap := make(map[common.Hash]*evmtypes.Head, len(oldHeads)) for _, head := range oldHeads { if head.Hash == head.ParentHash { @@ -155,11 +126,20 @@ func deepCopy(oldHeads []*evmtypes.Head) []*evmtypes.Head { return heads[i].Number > heads[j].Number }) + // yeah, we could have used binarySearch here, but the code was much longer and more complex and did not + // solve any performance issues + for i := range heads { + if heads[i].BlockNumber() < minBlockToKeep { + heads = heads[:i] + break + } + } + // assign parents for i := 0; i < len(heads)-1; i++ { head := heads[i] parent, exists := headsMap[head.ParentHash] - if exists { + if exists && parent.BlockNumber() >= minBlockToKeep { head.Parent = parent } } @@ -172,5 +152,5 @@ func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { defer h.mu.Unlock() // deep copy to avoid race - h.heads = deepCopy(append(h.heads, newHeads...)) + h.heads = deepCopy(append(h.heads, newHeads...), 0) } From f7ab489c33974ef7b00f7e550f13b6c514be161d Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 23 Feb 2024 21:33:32 +0100 Subject: [PATCH 13/28] nit fixes --- common/headtracker/head_tracker.go | 6 +++--- core/chains/evm/headtracker/head_tracker_test.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 78a90fd4901..f2d4712b92e 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -104,12 +104,12 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Start(ctx context.Context) error // anyway when we connect (but we should not rely on this because it is // not specced). If it happens this is fine, and the head will be // ignored as a duplicate. - err := ht.loadInitialHead(ctx) + err := ht.handleInitialHead(ctx) if err != nil { if ctx.Err() != nil { return ctx.Err() } - ht.log.Errorw("Error getting initial head", "err", err) + ht.log.Errorw("Error handling initial head", "err", err) } ht.wgDone.Add(3) @@ -123,7 +123,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) Start(ctx context.Context) error }) } -func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) loadInitialHead(ctx context.Context) error { +func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) handleInitialHead(ctx context.Context) error { initialHead, err := ht.client.HeadByNumber(ctx, nil) if err != nil { return fmt.Errorf("failed to fetch initial head: %w", err) diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 14c5216c00e..91ddd0125bb 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -234,7 +234,7 @@ func TestHeadTracker_Start(t *testing.T) { ht := newHeadTracker(t) ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(cltest.Head(0), errors.New("failed to get init head")) ht.Start(t) - tests.AssertLogEventually(t, ht.observer, "Error getting initial head") + tests.AssertLogEventually(t, ht.observer, "Error handling initial head") }) t.Run("Starts even if received invalid head", func(t *testing.T) { ht := newHeadTracker(t) @@ -248,7 +248,7 @@ func TestHeadTracker_Start(t *testing.T) { ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once() ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, errors.New("failed to load latest finalized")).Once() ht.Start(t) - tests.AssertLogEventually(t, ht.observer, "Error getting initial head") + tests.AssertLogEventually(t, ht.observer, "Error handling initial head") }) t.Run("Happy path", func(t *testing.T) { head := cltest.Head(1000) From e77f529caa2e420d6de77590ac40b8952f4a26ac Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 23 Feb 2024 21:40:20 +0100 Subject: [PATCH 14/28] fix build issues caused by merge --- common/client/multi_node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/client/multi_node.go b/common/client/multi_node.go index 28e452b2db3..cc8daed599c 100644 --- a/common/client/multi_node.go +++ b/common/client/multi_node.go @@ -820,7 +820,7 @@ func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OP return n.RPC().TransactionReceipt(ctx, txHash) } -func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT]) LatestFinalizedBlock(ctx context.Context) (head HEAD, err error) { +func (c *multiNode[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, RPC_CLIENT, BATCH_ELEM]) LatestFinalizedBlock(ctx context.Context) (head HEAD, err error) { n, err := c.selectNode() if err != nil { return head, err From 908acf7396afbcfa6e2a2fd4e04684b9e37e5fd2 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 23 Feb 2024 22:06:59 +0100 Subject: [PATCH 15/28] regen --- common/client/mock_rpc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/client/mock_rpc_test.go b/common/client/mock_rpc_test.go index b15ff4c73be..54a0ae93143 100644 --- a/common/client/mock_rpc_test.go +++ b/common/client/mock_rpc_test.go @@ -427,7 +427,7 @@ func (_m *mockRPC[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS } // LatestFinalizedBlock provides a mock function with given fields: ctx -func (_m *mockRPC[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD]) LatestFinalizedBlock(ctx context.Context) (HEAD, error) { +func (_m *mockRPC[CHAIN_ID, SEQ, ADDR, BLOCK_HASH, TX, TX_HASH, EVENT, EVENT_OPS, TX_RECEIPT, FEE, HEAD, BATCH_ELEM]) LatestFinalizedBlock(ctx context.Context) (HEAD, error) { ret := _m.Called(ctx) if len(ret) == 0 { From 93b835db2697e0b68876d61dc4cf0594269814c2 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Fri, 23 Feb 2024 22:22:12 +0100 Subject: [PATCH 16/28] FIx rpc client mock generation --- core/chains/evm/client/chain_client.go | 8 +- core/chains/evm/client/chain_client_test.go | 4 +- core/chains/evm/client/helpers_test.go | 15 +-- core/chains/evm/client/mocks/rpc_client.go | 120 ++++++++++++-------- core/chains/evm/client/rpc_client.go | 8 +- core/chains/legacyevm/chain.go | 8 +- 6 files changed, 98 insertions(+), 65 deletions(-) diff --git a/core/chains/evm/client/chain_client.go b/core/chains/evm/client/chain_client.go index ec897f222b1..64e854f1de4 100644 --- a/core/chains/evm/client/chain_client.go +++ b/core/chains/evm/client/chain_client.go @@ -35,7 +35,7 @@ type chainClient struct { *evmtypes.Receipt, *assets.Wei, *evmtypes.Head, - RPCCLient, + RPCClient, rpc.BatchElem, ] logger logger.SugaredLogger @@ -46,8 +46,8 @@ func NewChainClient( selectionMode string, leaseDuration time.Duration, noNewHeadsThreshold time.Duration, - nodes []commonclient.Node[*big.Int, *evmtypes.Head, RPCCLient], - sendonlys []commonclient.SendOnlyNode[*big.Int, RPCCLient], + nodes []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient], + sendonlys []commonclient.SendOnlyNode[*big.Int, RPCClient], chainID *big.Int, chainType config.ChainType, ) Client { @@ -63,7 +63,7 @@ func NewChainClient( *evmtypes.Receipt, *assets.Wei, *evmtypes.Head, - RPCCLient, + RPCClient, ]( lggr, selectionMode, diff --git a/core/chains/evm/client/chain_client_test.go b/core/chains/evm/client/chain_client_test.go index 6af9a67ee1c..1718036641e 100644 --- a/core/chains/evm/client/chain_client_test.go +++ b/core/chains/evm/client/chain_client_test.go @@ -18,8 +18,8 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" ) -func newMockRpc(t *testing.T) *mocks.RPCCLient { - mockRpc := mocks.NewRPCCLient(t) +func newMockRpc(t *testing.T) *mocks.RPCClient { + mockRpc := mocks.NewRPCClient(t) mockRpc.On("Dial", mock.Anything).Return(nil).Once() mockRpc.On("Close").Return(nil).Once() mockRpc.On("ChainID", mock.Anything).Return(testutils.FixtureChainID, nil).Once() diff --git a/core/chains/evm/client/helpers_test.go b/core/chains/evm/client/helpers_test.go index 467195e11e8..4ff2d4a3528 100644 --- a/core/chains/evm/client/helpers_test.go +++ b/core/chains/evm/client/helpers_test.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/smartcontractkit/chainlink-common/pkg/logger" + commonclient "github.com/smartcontractkit/chainlink/v2/common/client" commonconfig "github.com/smartcontractkit/chainlink/v2/common/config" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config" @@ -89,18 +90,18 @@ func NewChainClientWithTestNode( lggr := logger.Test(t) rpc := NewRPCClient(lggr, *parsed, rpcHTTPURL, "eth-primary-rpc-0", id, chainID, commonclient.Primary) - n := commonclient.NewNode[*big.Int, *evmtypes.Head, RPCCLient]( + n := commonclient.NewNode[*big.Int, *evmtypes.Head, RPCClient]( nodeCfg, noNewHeadsThreshold, lggr, *parsed, rpcHTTPURL, "eth-primary-node-0", id, chainID, 1, rpc, "EVM") - primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCCLient]{n} + primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient]{n} - var sendonlys []commonclient.SendOnlyNode[*big.Int, RPCCLient] + var sendonlys []commonclient.SendOnlyNode[*big.Int, RPCClient] for i, u := range sendonlyRPCURLs { if u.Scheme != "http" && u.Scheme != "https" { return nil, errors.Errorf("sendonly ethereum rpc url scheme must be http(s): %s", u.String()) } var empty url.URL rpc := NewRPCClient(lggr, empty, &sendonlyRPCURLs[i], fmt.Sprintf("eth-sendonly-rpc-%d", i), id, chainID, commonclient.Secondary) - s := commonclient.NewSendOnlyNode[*big.Int, RPCCLient]( + s := commonclient.NewSendOnlyNode[*big.Int, RPCClient]( lggr, u, fmt.Sprintf("eth-sendonly-%d", i), chainID, rpc) sendonlys = append(sendonlys, s) } @@ -133,7 +134,7 @@ func NewChainClientWithMockedRpc( leaseDuration time.Duration, noNewHeadsThreshold time.Duration, chainID *big.Int, - rpc RPCCLient, + rpc RPCClient, ) Client { lggr := logger.Test(t) @@ -145,9 +146,9 @@ func NewChainClientWithMockedRpc( } parsed, _ := url.ParseRequestURI("ws://test") - n := commonclient.NewNode[*big.Int, *evmtypes.Head, RPCCLient]( + n := commonclient.NewNode[*big.Int, *evmtypes.Head, RPCClient]( cfg, noNewHeadsThreshold, lggr, *parsed, nil, "eth-primary-node-0", 1, chainID, 1, rpc, "EVM") - primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCCLient]{n} + primaries := []commonclient.Node[*big.Int, *evmtypes.Head, RPCClient]{n} c := NewChainClient(lggr, selectionMode, leaseDuration, noNewHeadsThreshold, primaries, nil, chainID, chainType) t.Cleanup(c.Close) return c diff --git a/core/chains/evm/client/mocks/rpc_client.go b/core/chains/evm/client/mocks/rpc_client.go index 186fd2534e3..562fb5786f9 100644 --- a/core/chains/evm/client/mocks/rpc_client.go +++ b/core/chains/evm/client/mocks/rpc_client.go @@ -26,13 +26,13 @@ import ( types "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" ) -// RPCCLient is an autogenerated mock type for the RPCCLient type -type RPCCLient struct { +// RPCClient is an autogenerated mock type for the RPCClient type +type RPCClient struct { mock.Mock } // BalanceAt provides a mock function with given fields: ctx, accountAddress, blockNumber -func (_m *RPCCLient) BalanceAt(ctx context.Context, accountAddress common.Address, blockNumber *big.Int) (*big.Int, error) { +func (_m *RPCClient) BalanceAt(ctx context.Context, accountAddress common.Address, blockNumber *big.Int) (*big.Int, error) { ret := _m.Called(ctx, accountAddress, blockNumber) if len(ret) == 0 { @@ -62,7 +62,7 @@ func (_m *RPCCLient) BalanceAt(ctx context.Context, accountAddress common.Addres } // BatchCallContext provides a mock function with given fields: ctx, b -func (_m *RPCCLient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { +func (_m *RPCClient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) error { ret := _m.Called(ctx, b) if len(ret) == 0 { @@ -80,7 +80,7 @@ func (_m *RPCCLient) BatchCallContext(ctx context.Context, b []rpc.BatchElem) er } // BlockByHash provides a mock function with given fields: ctx, hash -func (_m *RPCCLient) BlockByHash(ctx context.Context, hash common.Hash) (*types.Head, error) { +func (_m *RPCClient) BlockByHash(ctx context.Context, hash common.Hash) (*types.Head, error) { ret := _m.Called(ctx, hash) if len(ret) == 0 { @@ -110,7 +110,7 @@ func (_m *RPCCLient) BlockByHash(ctx context.Context, hash common.Hash) (*types. } // BlockByHashGeth provides a mock function with given fields: ctx, hash -func (_m *RPCCLient) BlockByHashGeth(ctx context.Context, hash common.Hash) (*coretypes.Block, error) { +func (_m *RPCClient) BlockByHashGeth(ctx context.Context, hash common.Hash) (*coretypes.Block, error) { ret := _m.Called(ctx, hash) if len(ret) == 0 { @@ -140,7 +140,7 @@ func (_m *RPCCLient) BlockByHashGeth(ctx context.Context, hash common.Hash) (*co } // BlockByNumber provides a mock function with given fields: ctx, number -func (_m *RPCCLient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Head, error) { +func (_m *RPCClient) BlockByNumber(ctx context.Context, number *big.Int) (*types.Head, error) { ret := _m.Called(ctx, number) if len(ret) == 0 { @@ -170,7 +170,7 @@ func (_m *RPCCLient) BlockByNumber(ctx context.Context, number *big.Int) (*types } // BlockByNumberGeth provides a mock function with given fields: ctx, number -func (_m *RPCCLient) BlockByNumberGeth(ctx context.Context, number *big.Int) (*coretypes.Block, error) { +func (_m *RPCClient) BlockByNumberGeth(ctx context.Context, number *big.Int) (*coretypes.Block, error) { ret := _m.Called(ctx, number) if len(ret) == 0 { @@ -200,7 +200,7 @@ func (_m *RPCCLient) BlockByNumberGeth(ctx context.Context, number *big.Int) (*c } // CallContext provides a mock function with given fields: ctx, result, method, args -func (_m *RPCCLient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { +func (_m *RPCClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { var _ca []interface{} _ca = append(_ca, ctx, result, method) _ca = append(_ca, args...) @@ -221,7 +221,7 @@ func (_m *RPCCLient) CallContext(ctx context.Context, result interface{}, method } // CallContract provides a mock function with given fields: ctx, msg, blockNumber -func (_m *RPCCLient) CallContract(ctx context.Context, msg interface{}, blockNumber *big.Int) ([]byte, error) { +func (_m *RPCClient) CallContract(ctx context.Context, msg interface{}, blockNumber *big.Int) ([]byte, error) { ret := _m.Called(ctx, msg, blockNumber) if len(ret) == 0 { @@ -251,7 +251,7 @@ func (_m *RPCCLient) CallContract(ctx context.Context, msg interface{}, blockNum } // ChainID provides a mock function with given fields: ctx -func (_m *RPCCLient) ChainID(ctx context.Context) (*big.Int, error) { +func (_m *RPCClient) ChainID(ctx context.Context) (*big.Int, error) { ret := _m.Called(ctx) if len(ret) == 0 { @@ -281,7 +281,7 @@ func (_m *RPCCLient) ChainID(ctx context.Context) (*big.Int, error) { } // ClientVersion provides a mock function with given fields: _a0 -func (_m *RPCCLient) ClientVersion(_a0 context.Context) (string, error) { +func (_m *RPCClient) ClientVersion(_a0 context.Context) (string, error) { ret := _m.Called(_a0) if len(ret) == 0 { @@ -309,12 +309,12 @@ func (_m *RPCCLient) ClientVersion(_a0 context.Context) (string, error) { } // Close provides a mock function with given fields: -func (_m *RPCCLient) Close() { +func (_m *RPCClient) Close() { _m.Called() } // CodeAt provides a mock function with given fields: ctx, account, blockNumber -func (_m *RPCCLient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { +func (_m *RPCClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { ret := _m.Called(ctx, account, blockNumber) if len(ret) == 0 { @@ -344,7 +344,7 @@ func (_m *RPCCLient) CodeAt(ctx context.Context, account common.Address, blockNu } // Dial provides a mock function with given fields: ctx -func (_m *RPCCLient) Dial(ctx context.Context) error { +func (_m *RPCClient) Dial(ctx context.Context) error { ret := _m.Called(ctx) if len(ret) == 0 { @@ -362,7 +362,7 @@ func (_m *RPCCLient) Dial(ctx context.Context) error { } // DialHTTP provides a mock function with given fields: -func (_m *RPCCLient) DialHTTP() error { +func (_m *RPCClient) DialHTTP() error { ret := _m.Called() if len(ret) == 0 { @@ -380,12 +380,12 @@ func (_m *RPCCLient) DialHTTP() error { } // DisconnectAll provides a mock function with given fields: -func (_m *RPCCLient) DisconnectAll() { +func (_m *RPCClient) DisconnectAll() { _m.Called() } // EstimateGas provides a mock function with given fields: ctx, call -func (_m *RPCCLient) EstimateGas(ctx context.Context, call interface{}) (uint64, error) { +func (_m *RPCClient) EstimateGas(ctx context.Context, call interface{}) (uint64, error) { ret := _m.Called(ctx, call) if len(ret) == 0 { @@ -413,7 +413,7 @@ func (_m *RPCCLient) EstimateGas(ctx context.Context, call interface{}) (uint64, } // FilterEvents provides a mock function with given fields: ctx, query -func (_m *RPCCLient) FilterEvents(ctx context.Context, query ethereum.FilterQuery) ([]coretypes.Log, error) { +func (_m *RPCClient) FilterEvents(ctx context.Context, query ethereum.FilterQuery) ([]coretypes.Log, error) { ret := _m.Called(ctx, query) if len(ret) == 0 { @@ -443,7 +443,7 @@ func (_m *RPCCLient) FilterEvents(ctx context.Context, query ethereum.FilterQuer } // HeaderByHash provides a mock function with given fields: ctx, h -func (_m *RPCCLient) HeaderByHash(ctx context.Context, h common.Hash) (*coretypes.Header, error) { +func (_m *RPCClient) HeaderByHash(ctx context.Context, h common.Hash) (*coretypes.Header, error) { ret := _m.Called(ctx, h) if len(ret) == 0 { @@ -473,7 +473,7 @@ func (_m *RPCCLient) HeaderByHash(ctx context.Context, h common.Hash) (*coretype } // HeaderByNumber provides a mock function with given fields: ctx, n -func (_m *RPCCLient) HeaderByNumber(ctx context.Context, n *big.Int) (*coretypes.Header, error) { +func (_m *RPCClient) HeaderByNumber(ctx context.Context, n *big.Int) (*coretypes.Header, error) { ret := _m.Called(ctx, n) if len(ret) == 0 { @@ -503,7 +503,7 @@ func (_m *RPCCLient) HeaderByNumber(ctx context.Context, n *big.Int) (*coretypes } // LINKBalance provides a mock function with given fields: ctx, accountAddress, linkAddress -func (_m *RPCCLient) LINKBalance(ctx context.Context, accountAddress common.Address, linkAddress common.Address) (*assets.Link, error) { +func (_m *RPCClient) LINKBalance(ctx context.Context, accountAddress common.Address, linkAddress common.Address) (*assets.Link, error) { ret := _m.Called(ctx, accountAddress, linkAddress) if len(ret) == 0 { @@ -533,7 +533,7 @@ func (_m *RPCCLient) LINKBalance(ctx context.Context, accountAddress common.Addr } // LatestBlockHeight provides a mock function with given fields: _a0 -func (_m *RPCCLient) LatestBlockHeight(_a0 context.Context) (*big.Int, error) { +func (_m *RPCClient) LatestBlockHeight(_a0 context.Context) (*big.Int, error) { ret := _m.Called(_a0) if len(ret) == 0 { @@ -562,8 +562,38 @@ func (_m *RPCCLient) LatestBlockHeight(_a0 context.Context) (*big.Int, error) { return r0, r1 } +// LatestFinalizedBlock provides a mock function with given fields: ctx +func (_m *RPCClient) LatestFinalizedBlock(ctx context.Context) (*types.Head, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for LatestFinalizedBlock") + } + + var r0 *types.Head + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (*types.Head, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) *types.Head); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Head) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // PendingCallContract provides a mock function with given fields: ctx, msg -func (_m *RPCCLient) PendingCallContract(ctx context.Context, msg interface{}) ([]byte, error) { +func (_m *RPCClient) PendingCallContract(ctx context.Context, msg interface{}) ([]byte, error) { ret := _m.Called(ctx, msg) if len(ret) == 0 { @@ -593,7 +623,7 @@ func (_m *RPCCLient) PendingCallContract(ctx context.Context, msg interface{}) ( } // PendingCodeAt provides a mock function with given fields: ctx, account -func (_m *RPCCLient) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) { +func (_m *RPCClient) PendingCodeAt(ctx context.Context, account common.Address) ([]byte, error) { ret := _m.Called(ctx, account) if len(ret) == 0 { @@ -623,7 +653,7 @@ func (_m *RPCCLient) PendingCodeAt(ctx context.Context, account common.Address) } // PendingSequenceAt provides a mock function with given fields: ctx, addr -func (_m *RPCCLient) PendingSequenceAt(ctx context.Context, addr common.Address) (types.Nonce, error) { +func (_m *RPCClient) PendingSequenceAt(ctx context.Context, addr common.Address) (types.Nonce, error) { ret := _m.Called(ctx, addr) if len(ret) == 0 { @@ -651,7 +681,7 @@ func (_m *RPCCLient) PendingSequenceAt(ctx context.Context, addr common.Address) } // SendEmptyTransaction provides a mock function with given fields: ctx, newTxAttempt, seq, gasLimit, fee, fromAddress -func (_m *RPCCLient) SendEmptyTransaction(ctx context.Context, newTxAttempt func(types.Nonce, uint32, *evmassets.Wei, common.Address) (interface{}, error), seq types.Nonce, gasLimit uint32, fee *evmassets.Wei, fromAddress common.Address) (string, error) { +func (_m *RPCClient) SendEmptyTransaction(ctx context.Context, newTxAttempt func(types.Nonce, uint32, *evmassets.Wei, common.Address) (interface{}, error), seq types.Nonce, gasLimit uint32, fee *evmassets.Wei, fromAddress common.Address) (string, error) { ret := _m.Called(ctx, newTxAttempt, seq, gasLimit, fee, fromAddress) if len(ret) == 0 { @@ -679,7 +709,7 @@ func (_m *RPCCLient) SendEmptyTransaction(ctx context.Context, newTxAttempt func } // SendTransaction provides a mock function with given fields: ctx, tx -func (_m *RPCCLient) SendTransaction(ctx context.Context, tx *coretypes.Transaction) error { +func (_m *RPCClient) SendTransaction(ctx context.Context, tx *coretypes.Transaction) error { ret := _m.Called(ctx, tx) if len(ret) == 0 { @@ -697,7 +727,7 @@ func (_m *RPCCLient) SendTransaction(ctx context.Context, tx *coretypes.Transact } // SequenceAt provides a mock function with given fields: ctx, accountAddress, blockNumber -func (_m *RPCCLient) SequenceAt(ctx context.Context, accountAddress common.Address, blockNumber *big.Int) (types.Nonce, error) { +func (_m *RPCClient) SequenceAt(ctx context.Context, accountAddress common.Address, blockNumber *big.Int) (types.Nonce, error) { ret := _m.Called(ctx, accountAddress, blockNumber) if len(ret) == 0 { @@ -725,12 +755,12 @@ func (_m *RPCCLient) SequenceAt(ctx context.Context, accountAddress common.Addre } // SetAliveLoopSub provides a mock function with given fields: _a0 -func (_m *RPCCLient) SetAliveLoopSub(_a0 commontypes.Subscription) { +func (_m *RPCClient) SetAliveLoopSub(_a0 commontypes.Subscription) { _m.Called(_a0) } // SimulateTransaction provides a mock function with given fields: ctx, tx -func (_m *RPCCLient) SimulateTransaction(ctx context.Context, tx *coretypes.Transaction) error { +func (_m *RPCClient) SimulateTransaction(ctx context.Context, tx *coretypes.Transaction) error { ret := _m.Called(ctx, tx) if len(ret) == 0 { @@ -748,7 +778,7 @@ func (_m *RPCCLient) SimulateTransaction(ctx context.Context, tx *coretypes.Tran } // Subscribe provides a mock function with given fields: ctx, channel, args -func (_m *RPCCLient) Subscribe(ctx context.Context, channel chan<- *types.Head, args ...interface{}) (commontypes.Subscription, error) { +func (_m *RPCClient) Subscribe(ctx context.Context, channel chan<- *types.Head, args ...interface{}) (commontypes.Subscription, error) { var _ca []interface{} _ca = append(_ca, ctx, channel) _ca = append(_ca, args...) @@ -781,7 +811,7 @@ func (_m *RPCCLient) Subscribe(ctx context.Context, channel chan<- *types.Head, } // SubscribeFilterLogs provides a mock function with given fields: ctx, q, ch -func (_m *RPCCLient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- coretypes.Log) (ethereum.Subscription, error) { +func (_m *RPCClient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- coretypes.Log) (ethereum.Subscription, error) { ret := _m.Called(ctx, q, ch) if len(ret) == 0 { @@ -811,7 +841,7 @@ func (_m *RPCCLient) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQ } // SubscribersCount provides a mock function with given fields: -func (_m *RPCCLient) SubscribersCount() int32 { +func (_m *RPCClient) SubscribersCount() int32 { ret := _m.Called() if len(ret) == 0 { @@ -829,7 +859,7 @@ func (_m *RPCCLient) SubscribersCount() int32 { } // SuggestGasPrice provides a mock function with given fields: ctx -func (_m *RPCCLient) SuggestGasPrice(ctx context.Context) (*big.Int, error) { +func (_m *RPCClient) SuggestGasPrice(ctx context.Context) (*big.Int, error) { ret := _m.Called(ctx) if len(ret) == 0 { @@ -859,7 +889,7 @@ func (_m *RPCCLient) SuggestGasPrice(ctx context.Context) (*big.Int, error) { } // SuggestGasTipCap provides a mock function with given fields: ctx -func (_m *RPCCLient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { +func (_m *RPCClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { ret := _m.Called(ctx) if len(ret) == 0 { @@ -889,7 +919,7 @@ func (_m *RPCCLient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { } // TokenBalance provides a mock function with given fields: ctx, accountAddress, tokenAddress -func (_m *RPCCLient) TokenBalance(ctx context.Context, accountAddress common.Address, tokenAddress common.Address) (*big.Int, error) { +func (_m *RPCClient) TokenBalance(ctx context.Context, accountAddress common.Address, tokenAddress common.Address) (*big.Int, error) { ret := _m.Called(ctx, accountAddress, tokenAddress) if len(ret) == 0 { @@ -919,7 +949,7 @@ func (_m *RPCCLient) TokenBalance(ctx context.Context, accountAddress common.Add } // TransactionByHash provides a mock function with given fields: ctx, txHash -func (_m *RPCCLient) TransactionByHash(ctx context.Context, txHash common.Hash) (*coretypes.Transaction, error) { +func (_m *RPCClient) TransactionByHash(ctx context.Context, txHash common.Hash) (*coretypes.Transaction, error) { ret := _m.Called(ctx, txHash) if len(ret) == 0 { @@ -949,7 +979,7 @@ func (_m *RPCCLient) TransactionByHash(ctx context.Context, txHash common.Hash) } // TransactionReceipt provides a mock function with given fields: ctx, txHash -func (_m *RPCCLient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { +func (_m *RPCClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { ret := _m.Called(ctx, txHash) if len(ret) == 0 { @@ -979,7 +1009,7 @@ func (_m *RPCCLient) TransactionReceipt(ctx context.Context, txHash common.Hash) } // TransactionReceiptGeth provides a mock function with given fields: ctx, txHash -func (_m *RPCCLient) TransactionReceiptGeth(ctx context.Context, txHash common.Hash) (*coretypes.Receipt, error) { +func (_m *RPCClient) TransactionReceiptGeth(ctx context.Context, txHash common.Hash) (*coretypes.Receipt, error) { ret := _m.Called(ctx, txHash) if len(ret) == 0 { @@ -1009,17 +1039,17 @@ func (_m *RPCCLient) TransactionReceiptGeth(ctx context.Context, txHash common.H } // UnsubscribeAllExceptAliveLoop provides a mock function with given fields: -func (_m *RPCCLient) UnsubscribeAllExceptAliveLoop() { +func (_m *RPCClient) UnsubscribeAllExceptAliveLoop() { _m.Called() } -// NewRPCCLient creates a new instance of RPCCLient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// NewRPCClient creates a new instance of RPCClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func NewRPCCLient(t interface { +func NewRPCClient(t interface { mock.TestingT Cleanup(func()) -}) *RPCCLient { - mock := &RPCCLient{} +}) *RPCClient { + mock := &RPCClient{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/core/chains/evm/client/rpc_client.go b/core/chains/evm/client/rpc_client.go index 5ba594a57ae..e2311d79edd 100644 --- a/core/chains/evm/client/rpc_client.go +++ b/core/chains/evm/client/rpc_client.go @@ -28,8 +28,10 @@ import ( ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" ) -// RPCCLient includes all the necessary generalized RPC methods along with any additional chain-specific methods. -type RPCCLient interface { +// RPCClient includes all the necessary generalized RPC methods along with any additional chain-specific methods. +// +//go:generate mockery --quiet --name RPCClient --output ./mocks --case=underscore +type RPCClient interface { commonclient.RPC[ *big.Int, evmtypes.Nonce, @@ -89,7 +91,7 @@ func NewRPCClient( id int32, chainID *big.Int, tier commonclient.NodeTier, -) RPCCLient { +) RPCClient { r := new(rpcClient) r.name = name r.id = id diff --git a/core/chains/legacyevm/chain.go b/core/chains/legacyevm/chain.go index 66907b8352f..4e0344281cf 100644 --- a/core/chains/legacyevm/chain.go +++ b/core/chains/legacyevm/chain.go @@ -471,19 +471,19 @@ func (c *chain) GasEstimator() gas.EvmFeeEstimator { return c.gasEstimato func newEthClientFromCfg(cfg evmconfig.NodePool, noNewHeadsThreshold time.Duration, lggr logger.Logger, chainID *big.Int, chainType commonconfig.ChainType, nodes []*toml.Node) evmclient.Client { var empty url.URL - var primaries []commonclient.Node[*big.Int, *evmtypes.Head, evmclient.RPCCLient] - var sendonlys []commonclient.SendOnlyNode[*big.Int, evmclient.RPCCLient] + var primaries []commonclient.Node[*big.Int, *evmtypes.Head, evmclient.RPCClient] + var sendonlys []commonclient.SendOnlyNode[*big.Int, evmclient.RPCClient] for i, node := range nodes { if node.SendOnly != nil && *node.SendOnly { rpc := evmclient.NewRPCClient(lggr, empty, (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, commonclient.Secondary) - sendonly := commonclient.NewSendOnlyNode[*big.Int, evmclient.RPCCLient](lggr, (url.URL)(*node.HTTPURL), + sendonly := commonclient.NewSendOnlyNode[*big.Int, evmclient.RPCClient](lggr, (url.URL)(*node.HTTPURL), *node.Name, chainID, rpc) sendonlys = append(sendonlys, sendonly) } else { rpc := evmclient.NewRPCClient(lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, commonclient.Primary) - primaryNode := commonclient.NewNode[*big.Int, *evmtypes.Head, evmclient.RPCCLient](cfg, noNewHeadsThreshold, + primaryNode := commonclient.NewNode[*big.Int, *evmtypes.Head, evmclient.RPCClient](cfg, noNewHeadsThreshold, lggr, (url.URL)(*node.WSURL), (*url.URL)(node.HTTPURL), *node.Name, int32(i), chainID, *node.Order, rpc, "EVM") primaries = append(primaries, primaryNode) From 2f554038673dadf413a0f8f4ce1321a0678d904a Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Mon, 26 Feb 2024 14:03:40 +0100 Subject: [PATCH 17/28] nit fixes --- common/headtracker/head_tracker.go | 2 +- core/chains/evm/headtracker/head_saver.go | 2 +- core/chains/evm/headtracker/heads.go | 18 +++++------------- core/chains/evm/headtracker/orm.go | 2 +- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index f2d4712b92e..20b8fe14969 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -327,7 +327,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx cont if ht.config.FinalityTagEnabled() { return ht.client.LatestFinalizedBlock(ctx) } - // no need to make an additional RPC calls on chains with instant finality + // no need to make an additional RPC call on chains with instant finality if ht.config.FinalityDepth() == 0 { return currentHead, nil } diff --git a/core/chains/evm/headtracker/head_saver.go b/core/chains/evm/headtracker/head_saver.go index 412a66d5950..7d1376072b9 100644 --- a/core/chains/evm/headtracker/head_saver.go +++ b/core/chains/evm/headtracker/head_saver.go @@ -55,7 +55,7 @@ func (hs *headSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) ( } func (hs *headSaver) calculateMinBlockToKeep(latestFinalized int64) int64 { - return latestFinalized - int64(hs.htConfig.HistoryDepth()) + return max(latestFinalized-int64(hs.htConfig.HistoryDepth()), 0) } func (hs *headSaver) LatestHeadFromDB(ctx context.Context) (head *evmtypes.Head, err error) { diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 057ddbecf0d..e7ebfc49a79 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -110,13 +110,14 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head } } - heads := make([]*evmtypes.Head, len(headsMap)) + heads := make([]*evmtypes.Head, 0, len(headsMap)) // unsorted unique heads { - var i int for _, head := range headsMap { - heads[i] = head - i++ + if head.BlockNumber() < minBlockToKeep { + continue + } + heads = append(heads, head) } } @@ -126,15 +127,6 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head return heads[i].Number > heads[j].Number }) - // yeah, we could have used binarySearch here, but the code was much longer and more complex and did not - // solve any performance issues - for i := range heads { - if heads[i].BlockNumber() < minBlockToKeep { - heads = heads[:i] - break - } - } - // assign parents for i := 0; i < len(heads)-1; i++ { head := heads[i] diff --git a/core/chains/evm/headtracker/orm.go b/core/chains/evm/headtracker/orm.go index 99ecd978b31..d1dcd702529 100644 --- a/core/chains/evm/headtracker/orm.go +++ b/core/chains/evm/headtracker/orm.go @@ -25,7 +25,7 @@ type ORM interface { TrimOldHeads(ctx context.Context, minBlockNumber int64) (err error) // LatestHead returns the highest seen head LatestHead(ctx context.Context) (head *evmtypes.Head, err error) - // LatestHeads returns the latest heads with blockNumbers > minBlockNumber + // LatestHeads returns the latest heads with blockNumbers >= minBlockNumber LatestHeads(ctx context.Context, minBlockNumber int64) (heads []*evmtypes.Head, err error) // HeadByHash fetches the head with the given hash from the db, returns nil if none exists HeadByHash(ctx context.Context, hash common.Hash) (head *evmtypes.Head, err error) From 35c3302e720154c0260f33ab6877b6a17b694750 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Mon, 26 Feb 2024 19:09:42 +0100 Subject: [PATCH 18/28] nit fixes --- core/chains/evm/headtracker/head_saver_test.go | 2 +- core/chains/evm/headtracker/heads.go | 2 +- core/chains/evm/types/models.go | 15 ++------------- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/core/chains/evm/headtracker/head_saver_test.go b/core/chains/evm/headtracker/head_saver_test.go index 4df9dfff86a..9a890c5dba6 100644 --- a/core/chains/evm/headtracker/head_saver_test.go +++ b/core/chains/evm/headtracker/head_saver_test.go @@ -127,7 +127,7 @@ func TestHeadSaver_Load(t *testing.T) { require.NotNil(t, latestHead) require.Equal(t, int64(5), latestHead.Number) require.Equal(t, uint32(5), latestHead.ChainLength()) - require.Nil(t, latestHead.HeadAtHeight(0)) + require.Greater(t, latestHead.EarliestHeadInChain().BlockNumber(), int64(0)) } // load all from [h5-historyDepth, h5] diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index e7ebfc49a79..26a019affac 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -131,7 +131,7 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head for i := 0; i < len(heads)-1; i++ { head := heads[i] parent, exists := headsMap[head.ParentHash] - if exists && parent.BlockNumber() >= minBlockToKeep { + if exists { head.Parent = parent } } diff --git a/core/chains/evm/types/models.go b/core/chains/evm/types/models.go index 917338a185f..618e77413d5 100644 --- a/core/chains/evm/types/models.go +++ b/core/chains/evm/types/models.go @@ -118,27 +118,16 @@ func (h *Head) IsInChain(blockHash common.Hash) bool { // HashAtHeight returns the hash of the block at the given height, if it is in the chain. // If not in chain, returns the zero hash func (h *Head) HashAtHeight(blockNum int64) common.Hash { - head := h.HeadAtHeight(blockNum) - if head == nil { - return common.Hash{} - } - - return head.Hash -} - -// HeadAtHeight returns the head at the given height, if it is in the chain. -// If not in chain, returns nil -func (h *Head) HeadAtHeight(blockNum int64) *Head { for { if h.Number == blockNum { - return h + return h.Hash } if h.Parent == nil { break } h = h.Parent } - return nil + return common.Hash{} } // ChainLength returns the length of the chain followed by recursively looking up parents From bd1ea1e300c0193b449b7ff647152454ca3dc50e Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 27 Feb 2024 14:55:02 +0100 Subject: [PATCH 19/28] update comments --- common/headtracker/head_tracker.go | 2 +- core/chains/evm/headtracker/heads.go | 4 ++-- core/chains/evm/headtracker/orm.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 20b8fe14969..06e38101931 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -338,7 +338,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) calculateLatestFinalized(ctx cont return ht.client.HeadByNumber(ctx, big.NewInt(finalizedBlockNumber)) } -// backfill fetches all missing heads up until the base height +// backfill fetches all missing heads up until the latestFinalizedHead func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, head, latestFinalizedHead HTH) (err error) { headBlockNumber := head.BlockNumber() mark := time.Now() diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 26a019affac..580ab8a6df3 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -73,7 +73,7 @@ func (h *heads) MarkFinalized(finalized common.Hash, minBlockToKeep int64) bool return false } - // deep copy to avoid race + // deep copy to avoid race on head.Parent h.heads = deepCopy(h.heads, minBlockToKeep) head := h.heads[0] @@ -143,6 +143,6 @@ func (h *heads) AddHeads(newHeads ...*evmtypes.Head) { h.mu.Lock() defer h.mu.Unlock() - // deep copy to avoid race + // deep copy to avoid race on head.Parent h.heads = deepCopy(append(h.heads, newHeads...), 0) } diff --git a/core/chains/evm/headtracker/orm.go b/core/chains/evm/headtracker/orm.go index d1dcd702529..9c33dd257e5 100644 --- a/core/chains/evm/headtracker/orm.go +++ b/core/chains/evm/headtracker/orm.go @@ -21,7 +21,7 @@ type ORM interface { // IdempotentInsertHead inserts a head only if the hash is new. Will do nothing if hash exists already. // No advisory lock required because this is thread safe. IdempotentInsertHead(ctx context.Context, head *evmtypes.Head) error - // TrimOldHeads deletes heads such that only the top N block numbers remain + // TrimOldHeads deletes heads such that only blocks >= minBlockNumber remain TrimOldHeads(ctx context.Context, minBlockNumber int64) (err error) // LatestHead returns the highest seen head LatestHead(ctx context.Context) (head *evmtypes.Head, err error) From 83ea5d1da613244931fb128d277106ea0026238e Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 28 Feb 2024 17:04:19 +0100 Subject: [PATCH 20/28] ensure that we trim redundant blocks both in slice and in chain in Heads handle corner case for multiple uncle blocks at the end of the slice --- core/chains/evm/headtracker/heads.go | 9 +++++---- core/chains/evm/headtracker/heads_test.go | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/chains/evm/headtracker/heads.go b/core/chains/evm/headtracker/heads.go index 580ab8a6df3..1edfb3e3788 100644 --- a/core/chains/evm/headtracker/heads.go +++ b/core/chains/evm/headtracker/heads.go @@ -99,6 +99,10 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head // shouldn't happen but it is untrusted input continue } + if head.BlockNumber() < minBlockToKeep { + // trim redundant blocks + continue + } // copy all head objects to avoid races when a previous head chain is used // elsewhere (since we mutate Parent here) headCopy := *head @@ -114,9 +118,6 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head // unsorted unique heads { for _, head := range headsMap { - if head.BlockNumber() < minBlockToKeep { - continue - } heads = append(heads, head) } } @@ -128,7 +129,7 @@ func deepCopy(oldHeads []*evmtypes.Head, minBlockToKeep int64) []*evmtypes.Head }) // assign parents - for i := 0; i < len(heads)-1; i++ { + for i := 0; i < len(heads); i++ { head := heads[i] parent, exists := headsMap[head.ParentHash] if exists { diff --git a/core/chains/evm/headtracker/heads_test.go b/core/chains/evm/headtracker/heads_test.go index 09eda2c3b34..4241b462363 100644 --- a/core/chains/evm/headtracker/heads_test.go +++ b/core/chains/evm/headtracker/heads_test.go @@ -111,8 +111,8 @@ func TestHeads_MarkFinalized(t *testing.T) { // create chain // H0 <- H1 <- H2 <- H3 <- H4 <- H5 - // \ - // H2Uncle + // \ \ + // H1Uncle H2Uncle // newHead := func(num int, parent common.Hash) *evmtypes.Head { h := evmtypes.NewHead(big.NewInt(int64(num)), utils.NewHash(), parent, uint64(time.Now().Unix()), ubig.NewI(0)) @@ -120,13 +120,14 @@ func TestHeads_MarkFinalized(t *testing.T) { } h0 := newHead(0, utils.NewHash()) h1 := newHead(1, h0.Hash) + h1Uncle := newHead(1, h0.Hash) h2 := newHead(2, h1.Hash) h3 := newHead(3, h2.Hash) h4 := newHead(4, h3.Hash) h5 := newHead(5, h4.Hash) h2Uncle := newHead(2, h1.Hash) - allHeads := []*evmtypes.Head{h0, h1, h2, h2Uncle, h3, h4, h5} + allHeads := []*evmtypes.Head{h0, h1, h1Uncle, h2, h2Uncle, h3, h4, h5} heads.AddHeads(allHeads...) // mark h3 and all ancestors as finalized require.True(t, heads.MarkFinalized(h3.Hash, h1.BlockNumber()), "expected MarkFinalized succeed") @@ -139,6 +140,7 @@ func TestHeads_MarkFinalized(t *testing.T) { // h0 is too old. It should not be available directly or through its children assert.Nil(t, heads.HeadByHash(h0.Hash)) assert.Nil(t, heads.HeadByHash(h1.Hash).Parent) + assert.Nil(t, heads.HeadByHash(h1Uncle.Hash).Parent) assert.Nil(t, heads.HeadByHash(h2Uncle.Hash).Parent.Parent) require.False(t, heads.MarkFinalized(utils.NewHash(), 0), "expected false if finalized hash was not found in existing LatestHead chain") From 2d5ae656590eb8bbb47d089f383de10d47505b7a Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 28 Feb 2024 17:26:39 +0100 Subject: [PATCH 21/28] nit fix --- common/headtracker/head_tracker.go | 2 +- common/types/head_tracker.go | 2 +- core/chains/evm/headtracker/head_saver.go | 6 +++--- core/chains/evm/headtracker/head_saver_test.go | 2 +- core/chains/evm/headtracker/head_tracker_test.go | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 06e38101931..54be84f9c3e 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -140,7 +140,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) handleInitialHead(ctx context.Con return fmt.Errorf("failed to calculate latest finalized head: %w", err) } - latestChain, err := ht.headSaver.Load(ctx, latestFinalized) + latestChain, err := ht.headSaver.Load(ctx, latestFinalized.BlockNumber()) if err != nil { return fmt.Errorf("failed to initialzed headSaver: %w", err) } diff --git a/common/types/head_tracker.go b/common/types/head_tracker.go index d89663a1641..83a2d7b8adb 100644 --- a/common/types/head_tracker.go +++ b/common/types/head_tracker.go @@ -37,7 +37,7 @@ type HeadSaver[H Head[BLOCK_HASH], BLOCK_HASH Hashable] interface { // this number in case of reboot. Save(ctx context.Context, head H) error // Load loads latest heads up to latestFinalized - historyDepth, returns the latest chain. - Load(ctx context.Context, latestFinalized H) (H, error) + Load(ctx context.Context, latestFinalized int64) (H, error) // LatestChain returns the block header with the highest number that has been seen, or nil. LatestChain() H // Chain returns a head for the specified hash, or nil. diff --git a/core/chains/evm/headtracker/head_saver.go b/core/chains/evm/headtracker/head_saver.go index 7d1376072b9..218f9d8366f 100644 --- a/core/chains/evm/headtracker/head_saver.go +++ b/core/chains/evm/headtracker/head_saver.go @@ -43,8 +43,8 @@ func (hs *headSaver) Save(ctx context.Context, head *evmtypes.Head) error { return nil } -func (hs *headSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) (chain *evmtypes.Head, err error) { - minBlockNumber := hs.calculateMinBlockToKeep(latestFinalized.BlockNumber()) +func (hs *headSaver) Load(ctx context.Context, latestFinalized int64) (chain *evmtypes.Head, err error) { + minBlockNumber := hs.calculateMinBlockToKeep(latestFinalized) heads, err := hs.orm.LatestHeads(ctx, minBlockNumber) if err != nil { return nil, err @@ -91,7 +91,7 @@ var NullSaver httypes.HeadSaver = &nullSaver{} type nullSaver struct{} func (*nullSaver) Save(ctx context.Context, head *evmtypes.Head) error { return nil } -func (*nullSaver) Load(ctx context.Context, latestFinalized *evmtypes.Head) (*evmtypes.Head, error) { +func (*nullSaver) Load(ctx context.Context, latestFinalized int64) (*evmtypes.Head, error) { return nil, nil } func (*nullSaver) LatestHeadFromDB(ctx context.Context) (*evmtypes.Head, error) { return nil, nil } diff --git a/core/chains/evm/headtracker/head_saver_test.go b/core/chains/evm/headtracker/head_saver_test.go index 9a890c5dba6..e06c36c674c 100644 --- a/core/chains/evm/headtracker/head_saver_test.go +++ b/core/chains/evm/headtracker/head_saver_test.go @@ -131,7 +131,7 @@ func TestHeadSaver_Load(t *testing.T) { } // load all from [h5-historyDepth, h5] - latestHead, err := saver.Load(testutils.Context(t), h5) + latestHead, err := saver.Load(testutils.Context(t), h5.BlockNumber()) require.NoError(t, err) // verify latest head loaded from db verifyLatestHead(latestHead) diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 91ddd0125bb..17c2965f674 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -99,7 +99,7 @@ func TestHeadTracker_MarkFinalized_MarksAndTrimsTable(t *testing.T) { assert.Nil(t, orm.IdempotentInsertHead(testutils.Context(t), latest)) ht := createHeadTracker(t, ethClient, config.EVM(), config.EVM().HeadTracker(), orm) - _, err := ht.headSaver.Load(testutils.Context(t), latest) + _, err := ht.headSaver.Load(testutils.Context(t), latest.Number) require.NoError(t, err) require.NoError(t, ht.headSaver.MarkFinalized(testutils.Context(t), latest)) assert.Equal(t, big.NewInt(201), ht.headSaver.LatestChain().ToInt()) @@ -831,7 +831,7 @@ func TestHeadTracker_Backfill(t *testing.T) { ethClient := evmtest.NewEthClientMock(t) ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) ht := createHeadTracker(t, ethClient, evmcfg.EVM(), evmcfg.EVM().HeadTracker(), orm) - _, err := ht.headSaver.Load(testutils.Context(t), cltest.Head(0)) + _, err := ht.headSaver.Load(testutils.Context(t), 0) require.NoError(t, err) return ht } From f77a8abf6dfe52aea5ffb219ce46075ae2521b0a Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko <34754799+dhaidashenko@users.noreply.github.com> Date: Wed, 28 Feb 2024 18:05:34 +0100 Subject: [PATCH 22/28] Update common/headtracker/head_tracker.go Co-authored-by: Dimitris Grigoriou --- common/headtracker/head_tracker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 54be84f9c3e..1ce23d2a094 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -142,7 +142,7 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) handleInitialHead(ctx context.Con latestChain, err := ht.headSaver.Load(ctx, latestFinalized.BlockNumber()) if err != nil { - return fmt.Errorf("failed to initialzed headSaver: %w", err) + return fmt.Errorf("failed to initialized headSaver: %w", err) } if latestChain.IsValid() { From f7c786f7e83ecbe97ff2afd676f5ceec09a4d982 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Thu, 29 Feb 2024 19:42:15 +0100 Subject: [PATCH 23/28] HeadTracker backfill test with 0 finality depth --- core/chains/evm/headtracker/head_tracker_test.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 17c2965f674..85d1fbae50a 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -826,7 +826,7 @@ func TestHeadTracker_Backfill(t *testing.T) { db := pgtest.NewSqlxDB(t) orm := headtracker.NewORM(db, lggr, cfg.Database(), cltest.FixtureChainID) for i := range opts.Heads { - require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &heads[i])) + require.NoError(t, orm.IdempotentInsertHead(testutils.Context(t), &opts.Heads[i])) } ethClient := evmtest.NewEthClientMock(t) ethClient.On("ConfiguredChainID", mock.Anything).Return(evmtest.MustGetDefaultChainID(t, cfg.EVMConfigs()), nil) @@ -970,6 +970,20 @@ func TestHeadTracker_Backfill(t *testing.T) { assert.Equal(t, 2, int(h.ChainLength())) assert.Equal(t, int64(13), h.EarliestInChain().BlockNumber()) }) + t.Run("marks head as finalized, if latestHead = finalizedHead (0 finality depth)", func(t *testing.T) { + htu := newHeadTrackerUniverse(t, opts{Heads: []evmtypes.Head{h15}}) + finalizedH15 := h15 // copy h15 to have different addresses + err := htu.headTracker.Backfill(ctx, &h15, &finalizedH15) + require.NoError(t, err) + + h := htu.headSaver.LatestChain() + + // Should contain 14, 13 (15 was never added). When trying to get the parent of h13 by hash, a reorg happened and backfill exited. + assert.Equal(t, 1, int(h.ChainLength())) + assert.True(t, h.IsFinalized) + assert.Equal(t, h15.BlockNumber(), h.BlockNumber()) + assert.Equal(t, h15.Hash, h.Hash) + }) } func createHeadTracker(t *testing.T, ethClient *evmclimocks.Client, config headtracker.Config, htConfig headtracker.HeadTrackerConfig, orm headtracker.ORM) *headTrackerUniverse { From 03b07d202918fd7e6729f23912928eccb17b3e8b Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Tue, 5 Mar 2024 15:36:22 +0100 Subject: [PATCH 24/28] docs --- core/config/docs/chains-evm.toml | 4 ++-- docs/CHANGELOG.md | 2 ++ docs/CONFIG.md | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/core/config/docs/chains-evm.toml b/core/config/docs/chains-evm.toml index f70dcd0ee45..b5c456999d7 100644 --- a/core/config/docs/chains-evm.toml +++ b/core/config/docs/chains-evm.toml @@ -282,8 +282,8 @@ TransactionPercentile = 60 # Default # # In addition to these settings, it log warnings if `EVM.NoNewHeadsThreshold` is exceeded without any new blocks being emitted. [EVM.HeadTracker] -# HistoryDepth tracks the top N block numbers to keep in the `heads` database table. -# Note that this can easily result in MORE than N records since in the case of re-orgs we keep multiple heads for a particular block height. +# HistoryDepth tracks the top N blocks on top of the latest finalized block to keep in the `heads` database table. +# Note that this can easily result in MORE than `N + finality depth` records since in the case of re-orgs we keep multiple heads for a particular block height. # This number should be at least as large as `FinalityDepth`. # There may be a small performance penalty to setting this to something very large (10,000+) HistoryDepth = 100 # Default diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ad7903a33f8..365343c4881 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Minimum required version of Postgres is now >= 12. Postgres 11 was EOL'd in November 2023. Added a new version check that will prevent Chainlink from running on EOL'd Postgres. If you are running Postgres <= 11 you should upgrade to the latest version. The check can be forcibly overridden by setting SKIP_PG_VERSION_CHECK=true. +- HeadTracker now respects the `FinalityTagEnabled` config option. If the flag is enabled, HeadTracker backfill blocks up to the latest finalized block provided by the corresponding RPC call. To address potential misconfigurations, `HistoryDepth` is now calculated from the latest finalized block instead of the head. NOTE: Consumers (e.g. TXM and LogPoller) do not fully utilize Finality Tag yet. + diff --git a/docs/CONFIG.md b/docs/CONFIG.md index e4e25e6694f..2a1e4a068f0 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -6391,8 +6391,8 @@ In addition to these settings, it log warnings if `EVM.NoNewHeadsThreshold` is e ```toml HistoryDepth = 100 # Default ``` -HistoryDepth tracks the top N block numbers to keep in the `heads` database table. -Note that this can easily result in MORE than N records since in the case of re-orgs we keep multiple heads for a particular block height. +HistoryDepth tracks the top N blocks on top of the latest finalized block to keep in the `heads` database table. +Note that this can easily result in MORE than `N + finality depth` records since in the case of re-orgs we keep multiple heads for a particular block height. This number should be at least as large as `FinalityDepth`. There may be a small performance penalty to setting this to something very large (10,000+) From 9f6e5e492563558f2c2703d0ec5c7add0b79f0be Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko <34754799+dhaidashenko@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:00:31 +0100 Subject: [PATCH 25/28] Update docs/CHANGELOG.md Co-authored-by: Dimitris Grigoriou --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 365343c4881..f82f1f880ca 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Minimum required version of Postgres is now >= 12. Postgres 11 was EOL'd in November 2023. Added a new version check that will prevent Chainlink from running on EOL'd Postgres. If you are running Postgres <= 11 you should upgrade to the latest version. The check can be forcibly overridden by setting SKIP_PG_VERSION_CHECK=true. -- HeadTracker now respects the `FinalityTagEnabled` config option. If the flag is enabled, HeadTracker backfill blocks up to the latest finalized block provided by the corresponding RPC call. To address potential misconfigurations, `HistoryDepth` is now calculated from the latest finalized block instead of the head. NOTE: Consumers (e.g. TXM and LogPoller) do not fully utilize Finality Tag yet. +- HeadTracker now respects the `FinalityTagEnabled` config option. If the flag is enabled, HeadTracker backfills blocks up to the latest finalized block provided by the corresponding RPC call. To address potential misconfigurations, `HistoryDepth` is now calculated from the latest finalized block instead of the head. NOTE: Consumers (e.g. TXM and LogPoller) do not fully utilize Finality Tag yet. From 924df1b8def2a2876230a47f5d33de305bd41fa1 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 6 Mar 2024 19:22:58 +0100 Subject: [PATCH 26/28] ensure latest finalized block is valid on startup --- common/headtracker/head_tracker.go | 4 ++++ core/chains/evm/headtracker/head_tracker_test.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 1ce23d2a094..7d196947af3 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -140,6 +140,10 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) handleInitialHead(ctx context.Con return fmt.Errorf("failed to calculate latest finalized head: %w", err) } + if !latestFinalized.IsValid() { + return fmt.Errorf("latest finalized block is not valid") + } + latestChain, err := ht.headSaver.Load(ctx, latestFinalized.BlockNumber()) if err != nil { return fmt.Errorf("failed to initialized headSaver: %w", err) diff --git a/core/chains/evm/headtracker/head_tracker_test.go b/core/chains/evm/headtracker/head_tracker_test.go index 85d1fbae50a..a2e45c59f09 100644 --- a/core/chains/evm/headtracker/head_tracker_test.go +++ b/core/chains/evm/headtracker/head_tracker_test.go @@ -250,6 +250,14 @@ func TestHeadTracker_Start(t *testing.T) { ht.Start(t) tests.AssertLogEventually(t, ht.observer, "Error handling initial head") }) + t.Run("Starts even if latest finalizedHead is nil", func(t *testing.T) { + ht := newHeadTracker(t) + head := cltest.Head(1000) + ht.ethClient.On("HeadByNumber", mock.Anything, (*big.Int)(nil)).Return(head, nil).Once() + ht.ethClient.On("LatestFinalizedBlock", mock.Anything).Return(nil, nil).Once() + ht.Start(t) + tests.AssertLogEventually(t, ht.observer, "Error handling initial head") + }) t.Run("Happy path", func(t *testing.T) { head := cltest.Head(1000) ht := newHeadTracker(t) From 4a117c4bf0dd63755451cde974e05303ba6741e4 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Mon, 11 Mar 2024 17:46:38 +0100 Subject: [PATCH 27/28] changeset --- .changeset/healthy-toes-destroy.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/healthy-toes-destroy.md diff --git a/.changeset/healthy-toes-destroy.md b/.changeset/healthy-toes-destroy.md new file mode 100644 index 00000000000..1c027fdcd01 --- /dev/null +++ b/.changeset/healthy-toes-destroy.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +HeadTracker now respects the `FinalityTagEnabled` config option. If the flag is enabled, HeadTracker backfills blocks up to the latest finalized block provided by the corresponding RPC call. To address potential misconfigurations, `HistoryDepth` is now calculated from the latest finalized block instead of the head. NOTE: Consumers (e.g. TXM and LogPoller) do not fully utilize Finality Tag yet. From 7abea66389d9ab2bc416cd65b90125b5865dbd21 Mon Sep 17 00:00:00 2001 From: Dmytro Haidashenko Date: Wed, 13 Mar 2024 18:45:45 +0100 Subject: [PATCH 28/28] switch from warn to debug level when we failed to makr block as finalized --- common/headtracker/head_tracker.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/common/headtracker/head_tracker.go b/common/headtracker/head_tracker.go index 7d196947af3..eb9d72f123c 100644 --- a/common/headtracker/head_tracker.go +++ b/common/headtracker/head_tracker.go @@ -388,13 +388,16 @@ func (ht *HeadTracker[HTH, S, ID, BLOCK_HASH]) backfill(ctx context.Context, hea return fmt.Errorf(errMsg) } + l = l.With("latest_finalized_block_hash", latestFinalizedHead.BlockHash(), + "latest_finalized_block_number", latestFinalizedHead.BlockNumber()) + err = ht.headSaver.MarkFinalized(ctx, latestFinalizedHead) if err != nil { - return fmt.Errorf("failed to mark head as finalized: %w", err) + l.Debugw("failed to mark block as finalized", "err", err) + return nil } - l.Debugw("marked block as finalized", "block_hash", latestFinalizedHead.BlockHash(), - "block_number", latestFinalizedHead.BlockNumber()) + l.Debugw("marked block as finalized") return }