From 09243e0d17bdfdff9396b3b2112e52434535016d Mon Sep 17 00:00:00 2001 From: simlecode <69969590+simlecode@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:02:49 +0800 Subject: [PATCH 1/3] feat: eth get block receipts --- app/node/rpc.go | 1 + app/submodule/eth/dummy.go | 8 + app/submodule/eth/eth_api.go | 111 ++- app/submodule/eth/eth_event_api.go | 220 +++++- app/submodule/eth/eth_utils.go | 124 ++-- fixtures/networks/forcenet.go | 2 +- go.sum | 24 - pkg/config/config.go | 5 + pkg/events/filter/event.go | 68 +- pkg/events/filter/event_test.go | 12 +- pkg/events/filter/index.go | 686 +++++++++--------- pkg/events/filter/index_migrations.go | 258 +++++++ pkg/events/filter/index_test.go | 334 ++++++--- pkg/events/filter/sqlite/sqlite.go | 169 +++++ pkg/events/filter/sqlite/sqlite_test.go | 244 +++++++ pkg/events/filter/store.go | 2 +- pkg/vf3/participation_lease.go | 4 +- venus-shared/api/chain/v1/eth.go | 24 +- venus-shared/api/chain/v1/method.md | 109 +++ .../api/chain/v1/mock/mock_fullnode.go | 30 + venus-shared/api/chain/v1/proxy_gen.go | 8 + 21 files changed, 1815 insertions(+), 628 deletions(-) create mode 100644 pkg/events/filter/index_migrations.go create mode 100644 pkg/events/filter/sqlite/sqlite.go create mode 100644 pkg/events/filter/sqlite/sqlite_test.go diff --git a/app/node/rpc.go b/app/node/rpc.go index a5dd7d9403..2cb2b96888 100644 --- a/app/node/rpc.go +++ b/app/node/rpc.go @@ -192,6 +192,7 @@ func aliasETHAPI(rpcServer *jsonrpc.RPCServer) { rpcServer.AliasMethod("eth_getMessageCidByTransactionHash", "Filecoin.EthGetMessageCidByTransactionHash") rpcServer.AliasMethod("eth_getTransactionCount", "Filecoin.EthGetTransactionCount") rpcServer.AliasMethod("eth_getTransactionReceipt", "Filecoin.EthGetTransactionReceipt") + rpcServer.AliasMethod("eth_getBlockReceipts", "Filecoin.EthGetBlockReceipts") rpcServer.AliasMethod("eth_getTransactionByBlockHashAndIndex", "Filecoin.EthGetTransactionByBlockHashAndIndex") rpcServer.AliasMethod("eth_getTransactionByBlockNumberAndIndex", "Filecoin.EthGetTransactionByBlockNumberAndIndex") diff --git a/app/submodule/eth/dummy.go b/app/submodule/eth/dummy.go index 0c9a74c9c5..4d6aa674fe 100644 --- a/app/submodule/eth/dummy.go +++ b/app/submodule/eth/dummy.go @@ -61,6 +61,14 @@ func (e *ethAPIDummy) EthGetTransactionByHash(ctx context.Context, txHash *types return nil, ErrModuleDisabled } +func (e *ethAPIDummy) EthGetBlockReceiptsLimited(ctx context.Context, blkParam types.EthBlockNumberOrHash, limit abi.ChainEpoch) ([]*types.EthTxReceipt, error) { + return nil, ErrModuleDisabled +} + +func (e *ethAPIDummy) EthGetBlockReceipts(ctx context.Context, blkParam types.EthBlockNumberOrHash) ([]*types.EthTxReceipt, error) { + return nil, ErrModuleDisabled +} + func (e *ethAPIDummy) EthGetTransactionByHashLimited(ctx context.Context, txHash *types.EthHash, limit abi.ChainEpoch) (*types.EthTx, error) { return nil, ErrModuleDisabled } diff --git a/app/submodule/eth/eth_api.go b/app/submodule/eth/eth_api.go index 9a8442158d..2bc871f763 100644 --- a/app/submodule/eth/eth_api.go +++ b/app/submodule/eth/eth_api.go @@ -35,7 +35,6 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" ) const maxEthFeeHistoryRewardPercentiles = 100 @@ -47,9 +46,10 @@ var ErrUnsupported = errors.New("unsupported method") func newEthAPI(em *EthSubModule) (*ethAPI, error) { a := ðAPI{ - em: em, - chain: em.chainModule.API(), - mpool: em.mpoolModule.API(), + em: em, + chain: em.chainModule.API(), + mpool: em.mpoolModule.API(), + EthEventHandler: em.ethEventAPI, } dbPath := filepath.Join(a.em.sqlitePath, "txhash.db") @@ -95,10 +95,12 @@ func newEthAPI(em *EthSubModule) (*ethAPI, error) { } type ethAPI struct { - em *EthSubModule - chain v1.IChain - mpool v1.IMessagePool - ethTxHashManager *ethTxHashManager + em *EthSubModule + chain v1.IChain + mpool v1.IMessagePool + ethTxHashManager *ethTxHashManager + EthEventHandler *ethEventAPI + MaxFilterHeightRange abi.ChainEpoch EthBlkCache *arc.ARCCache[cid.Cid, *types.EthBlock] // caches blocks by their CID but blocks only have the transaction hashes EthBlkTxCache *arc.ARCCache[cid.Cid, *types.EthBlock] // caches blocks along with full transaction payload by their CID @@ -279,12 +281,12 @@ func (a *ethAPI) EthGetBlockByNumber(ctx context.Context, blkParam string, fullT // Return nil for null rounds return nil, nil } - return nil, xerrors.Errorf("failed to get tipset: %w", err) + return nil, fmt.Errorf("failed to get tipset: %w", err) } // Create an Ethereum block from the Filecoin tipset block, err := newEthBlockFromFilecoinTipSet(ctx, ts, fullTxInfo, a.em.chainModule.MessageStore, a.em.chainModule.Stmgr) if err != nil { - return nil, xerrors.Errorf("failed to create Ethereum block: %w", err) + return nil, fmt.Errorf("failed to create Ethereum block: %w", err) } return &block, nil @@ -403,14 +405,14 @@ func (a *ethAPI) EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (* func (a *ethAPI) EthGetTransactionCount(ctx context.Context, sender types.EthAddress, blkParam types.EthBlockNumberOrHash) (types.EthUint64, error) { addr, err := sender.ToFilecoinAddress() if err != nil { - return types.EthUint64(0), xerrors.Errorf("invalid address: %w", err) + return types.EthUint64(0), fmt.Errorf("invalid address: %w", err) } // Handle "pending" block parameter separately if blkParam.PredefinedBlock != nil && *blkParam.PredefinedBlock == "pending" { nonce, err := a.mpool.MpoolGetNonce(ctx, addr) if err != nil { - return types.EthUint64(0), xerrors.Errorf("failed to get nonce from mpool: %w", err) + return types.EthUint64(0), fmt.Errorf("failed to get nonce from mpool: %w", err) } return types.EthUint64(nonce), nil } @@ -427,7 +429,7 @@ func (a *ethAPI) EthGetTransactionCount(ctx context.Context, sender types.EthAdd if errors.Is(err, types.ErrActorNotFound) { return 0, nil } - return 0, xerrors.Errorf("failed to lookup actor %s: %w", sender, err) + return 0, fmt.Errorf("failed to lookup actor %s: %w", sender, err) } // Handle EVM actor case @@ -464,7 +466,12 @@ func (a *ethAPI) EthGetTransactionReceiptLimited(ctx context.Context, txHash typ } msgLookup, err := a.chain.StateSearchMsg(ctx, types.EmptyTSK, c, limit, true) - if err != nil || msgLookup == nil { + if err != nil { + return nil, fmt.Errorf("failed to lookup Eth Txn %s as %s: %w", txHash, c, err) + } + if msgLookup == nil { + // This is the best we can do. In theory, we could have just not indexed this + // transaction, but there's no way to check that here. return nil, nil } @@ -473,15 +480,20 @@ func (a *ethAPI) EthGetTransactionReceiptLimited(ctx context.Context, txHash typ return nil, nil } - var events []types.Event - if rct := msgLookup.Receipt; rct.EventsRoot != nil { - events, err = a.chain.ChainGetEvents(ctx, *rct.EventsRoot) - if err != nil { - return nil, nil - } + ts, err := a.em.chainModule.ChainReader.GetTipSet(ctx, msgLookup.TipSet) + if err != nil { + return nil, fmt.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", msgLookup.TipSet, err) + } + + // The tx is located in the parent tipset + parentTs, err := a.em.chainModule.ChainReader.GetTipSet(ctx, ts.Parents()) + if err != nil { + return nil, fmt.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", ts.Parents(), err) } - receipt, err := newEthTxReceipt(ctx, tx, msgLookup, events, a.chain, a.em.chainModule.ChainReader) + baseFee := parentTs.Blocks()[0].ParentBaseFee + + receipt, err := newEthTxReceipt(ctx, tx, baseFee, msgLookup.Receipt, a.EthEventHandler) if err != nil { return nil, nil } @@ -493,6 +505,63 @@ func (a *ethAPI) EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHa return types.EthTx{}, ErrUnsupported } +func (a *ethAPI) EthGetBlockReceipts(ctx context.Context, blockParam types.EthBlockNumberOrHash) ([]*types.EthTxReceipt, error) { + return a.EthGetBlockReceiptsLimited(ctx, blockParam, constants.LookbackNoLimit) +} + +func (a *ethAPI) EthGetBlockReceiptsLimited(ctx context.Context, blockParam types.EthBlockNumberOrHash, limit abi.ChainEpoch) ([]*types.EthTxReceipt, error) { + ts, err := getTipsetByEthBlockNumberOrHash(ctx, a.em.chainModule.ChainReader, blockParam) + if err != nil { + return nil, fmt.Errorf("failed to get tipset: %w", err) + } + + tsCid, err := ts.Key().Cid() + if err != nil { + return nil, fmt.Errorf("failed to get tipset key cid: %w", err) + } + + blkHash, err := types.EthHashFromCid(tsCid) + if err != nil { + return nil, fmt.Errorf("failed to parse eth hash from cid: %w", err) + } + + // Execute the tipset to get the receipts, messages, and events + _, msgs, receipts, err := executeTipset(ctx, ts, a.em.chainModule.MessageStore, a.em.chainModule.Stmgr) + if err != nil { + return nil, fmt.Errorf("failed to execute tipset: %w", err) + } + + // Load the state tree + state, err := a.em.chainModule.ChainReader.GetTipSetState(ctx, ts) + if err != nil { + return nil, fmt.Errorf("failed to get state view: %w", err) + } + + baseFee := ts.Blocks()[0].ParentBaseFee + + ethReceipts := make([]*types.EthTxReceipt, 0, len(msgs)) + for i, msg := range msgs { + msg := msg + + tx, err := newEthTx(ctx, state, ts.Height(), tsCid, msg.Cid(), i, a.ethTxHashManager.messageStore) + if err != nil { + return nil, fmt.Errorf("failed to create EthTx: %w", err) + } + + receipt, err := newEthTxReceipt(ctx, tx, baseFee, receipts[i], a.EthEventHandler) + if err != nil { + return nil, fmt.Errorf("failed to create Eth receipt: %w", err) + } + + // Set the correct Ethereum block hash + receipt.BlockHash = blkHash + + ethReceipts = append(ethReceipts, &receipt) + } + + return ethReceipts, nil +} + func (a *ethAPI) EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum types.EthUint64, txIndex types.EthUint64) (types.EthTx, error) { return types.EthTx{}, ErrUnsupported } diff --git a/app/submodule/eth/eth_event_api.go b/app/submodule/eth/eth_event_api.go index f18731496d..d6cc46f4df 100644 --- a/app/submodule/eth/eth_event_api.go +++ b/app/submodule/eth/eth_event_api.go @@ -26,6 +26,11 @@ import ( "github.com/zyedidia/generic/queue" ) +var ( + // wait for 3 epochs + eventReadTimeout = 90 * time.Second +) + var _ v1.IETHEvent = (*ethEventAPI)(nil) func newEthEventAPI(ctx context.Context, em *EthSubModule) (*ethEventAPI, error) { @@ -147,21 +152,150 @@ func (e *ethEventAPI) Close(ctx context.Context) error { return nil } +// TODO: For now, we're fetching logs from the index for the entire block and then filtering them by the transaction hash +// This allows us to use the current schema of the event Index DB that has been optimised to use the "tipset_key_cid" index +// However, this can be replaced to filter logs in the event Index DB by the "msgCid" if we pass it down to the query generator +func (e *ethEventAPI) getEthLogsForBlockAndTransaction(ctx context.Context, blockHash *types.EthHash, txHash types.EthHash) ([]types.EthLog, error) { + ces, err := e.ethGetEventsForFilter(ctx, &types.EthFilterSpec{BlockHash: blockHash}) + if err != nil { + return nil, err + } + logs, err := ethFilterLogsFromEvents(ctx, ces, e.em.chainModule.MessageStore) + if err != nil { + return nil, err + } + var out []types.EthLog + for _, log := range logs { + if log.TransactionHash == txHash { + out = append(out, log) + } + } + return out, nil +} + func (e *ethEventAPI) EthGetLogs(ctx context.Context, filterSpec *types.EthFilterSpec) (*types.EthFilterResult, error) { + ces, err := e.ethGetEventsForFilter(ctx, filterSpec) + if err != nil { + return nil, err + } + + return ethFilterResultFromEvents(ctx, ces, e.em.chainModule.MessageStore) +} + +func (e *ethEventAPI) ethGetEventsForFilter(ctx context.Context, filterSpec *types.EthFilterSpec) ([]*filter.CollectedEvent, error) { if e.EventFilterManager == nil { return nil, api.ErrNotSupported } + if e.EventFilterManager.EventIndex == nil { + return nil, fmt.Errorf("cannot use eth_get_logs if historical event index is disabled") + } + + pf, err := e.parseEthFilterSpec(filterSpec) + if err != nil { + return nil, fmt.Errorf("failed to parse eth filter spec: %w", err) + } + + if pf.tipsetCid == cid.Undef { + maxHeight := pf.maxHeight + if maxHeight == -1 { + // heaviest tipset doesn't have events because its messages haven't been executed yet + maxHeight = e.em.chainModule.ChainReader.GetHead().Height() - 1 + } + + if maxHeight < 0 { + return nil, fmt.Errorf("maxHeight requested is less than 0") + } + + // we can't return events for the heaviest tipset as the transactions in that tipset will be executed + // in the next non null tipset (because of Filecoin's "deferred execution" model) + if maxHeight > e.em.chainModule.ChainReader.GetHead().Height()-1 { + return nil, fmt.Errorf("maxHeight requested is greater than the heaviest tipset") + } + + err := e.waitForHeightProcessed(ctx, maxHeight) + if err != nil { + return nil, err + } + // TODO: Ideally we should also check that events for the epoch at `pf.minheight` have been indexed + // However, it is currently tricky to check/guarantee this for two reasons: + // a) Event Index is not aware of null-blocks. This means that the Event Index wont be able to say whether the block at + // `pf.minheight` is a null block or whether it has no events + // b) There can be holes in the index where events at certain epoch simply haven't been indexed because of edge cases around + // node restarts while indexing. This needs a long term "auto-repair"/"automated-backfilling" implementation in the index + // So, for now, the best we can do is ensure that the event index has evenets for events at height >= `pf.maxHeight` + } else { + ts, err := e.em.chainModule.ChainReader.GetTipSetByCid(ctx, pf.tipsetCid) + if err != nil { + return nil, fmt.Errorf("failed to get tipset by cid: %w", err) + } + err = e.waitForHeightProcessed(ctx, ts.Height()) + if err != nil { + return nil, err + } + + b, err := e.EventFilterManager.EventIndex.IsTipsetProcessed(ctx, pf.tipsetCid.Bytes()) + if err != nil { + return nil, fmt.Errorf("failed to check if tipset events have been indexed: %w", err) + } + if !b { + return nil, fmt.Errorf("event index failed to index tipset %s", pf.tipsetCid.String()) + } + } + // Create a temporary filter - f, err := e.installEthFilterSpec(ctx, filterSpec) + f, err := e.EventFilterManager.Install(ctx, pf.minHeight, pf.maxHeight, pf.tipsetCid, pf.addresses, pf.keys, false) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to install event filter: %w", err) } ces := f.TakeCollectedEvents(ctx) _ = e.uninstallFilter(ctx, f) - return ethFilterResultFromEvents(ces, e.em.chainModule.MessageStore) + return ces, nil +} + +// note that we can have null blocks at the given height and the event Index is not null block aware +// so, what we do here is wait till we see the event index contain a block at a height greater than the given height +func (e *ethEventAPI) waitForHeightProcessed(ctx context.Context, height abi.ChainEpoch) error { + ei := e.EventFilterManager.EventIndex + if height > e.em.chainModule.ChainReader.GetHead().Height() { + return fmt.Errorf("height is in the future") + } + + ctx, cancel := context.WithTimeout(ctx, eventReadTimeout) + defer cancel() + + // if the height we're interested in has already been indexed -> there's nothing to do here + if b, err := ei.IsHeightPast(ctx, uint64(height)); err != nil { + return fmt.Errorf("failed to check if event index has events for given height: %w", err) + } else if b { + return nil + } + + // subscribe for updates to the event index + subCh, unSubscribeF := ei.SubscribeUpdates() + defer unSubscribeF() + + // it could be that the event index was update while the subscription was being processed -> check if index has what we need now + if b, err := ei.IsHeightPast(ctx, uint64(height)); err != nil { + return fmt.Errorf("failed to check if event index has events for given height: %w", err) + } else if b { + return nil + } + + for { + select { + case <-subCh: + if b, err := ei.IsHeightPast(ctx, uint64(height)); err != nil { + return fmt.Errorf("failed to check if event index has events for given height: %w", err) + } else if b { + return nil + } + case <-ctx.Done(): + return ctx.Err() + } + } } func (e *ethEventAPI) EthGetFilterChanges(ctx context.Context, id types.EthFilterID) (*types.EthFilterResult, error) { @@ -176,7 +310,7 @@ func (e *ethEventAPI) EthGetFilterChanges(ctx context.Context, id types.EthFilte switch fc := f.(type) { case filterEventCollector: - return ethFilterResultFromEvents(fc.TakeCollectedEvents(ctx), e.em.chainModule.MessageStore) + return ethFilterResultFromEvents(ctx, fc.TakeCollectedEvents(ctx), e.em.chainModule.MessageStore) case filterTipSetCollector: return ethFilterResultFromTipSets(fc.TakeCollectedTipSets(ctx)) case filterMessageCollector: @@ -198,7 +332,7 @@ func (e *ethEventAPI) EthGetFilterLogs(ctx context.Context, id types.EthFilterID switch fc := f.(type) { case filterEventCollector: - return ethFilterResultFromEvents(fc.TakeCollectedEvents(ctx), e.em.chainModule.MessageStore) + return ethFilterResultFromEvents(ctx, fc.TakeCollectedEvents(ctx), e.em.chainModule.MessageStore) } return nil, fmt.Errorf("wrong filter type") @@ -552,6 +686,61 @@ func (e *ethEventAPI) GC(ctx context.Context, ttl time.Duration) { } } +type parsedFilter struct { + minHeight abi.ChainEpoch + maxHeight abi.ChainEpoch + tipsetCid cid.Cid + addresses []address.Address + keys map[string][]types.ActorEventBlock +} + +func (e *ethEventAPI) parseEthFilterSpec(filterSpec *types.EthFilterSpec) (*parsedFilter, error) { + var ( + minHeight abi.ChainEpoch + maxHeight abi.ChainEpoch + tipsetCid cid.Cid + addresses []address.Address + keys = map[string][][]byte{} + ) + + if filterSpec.BlockHash != nil { + if filterSpec.FromBlock != nil || filterSpec.ToBlock != nil { + return nil, fmt.Errorf("must not specify block hash and from/to block") + } + + tipsetCid = filterSpec.BlockHash.ToCid() + } else { + var err error + head := e.em.chainModule.ChainReader.GetHead() + minHeight, maxHeight, err = parseBlockRange(head.Height(), filterSpec.FromBlock, filterSpec.ToBlock, e.MaxFilterHeightRange) + if err != nil { + return nil, err + } + } + + // Convert all addresses to filecoin f4 addresses + for _, ea := range filterSpec.Address { + a, err := ea.ToFilecoinAddress() + if err != nil { + return nil, fmt.Errorf("invalid address %x", ea) + } + addresses = append(addresses, a) + } + + keys, err := parseEthTopics(filterSpec.Topics) + if err != nil { + return nil, err + } + + return &parsedFilter{ + minHeight: minHeight, + maxHeight: maxHeight, + tipsetCid: tipsetCid, + addresses: addresses, + keys: keysToKeysWithCodec(keys), + }, nil +} + type filterEventCollector interface { TakeCollectedEvents(context.Context) []*filter.CollectedEvent } @@ -627,8 +816,9 @@ func ethLogFromEvent(entries []types.EventEntry) (data []byte, topics []types.Et return data, topics, true } -func ethFilterResultFromEvents(evs []*filter.CollectedEvent, ms *chain.MessageStore) (*types.EthFilterResult, error) { - res := &types.EthFilterResult{} +// func ethFilterResultFromEvents(evs []*filter.CollectedEvent, ms *chain.MessageStore) (*types.EthFilterResult, error) { +func ethFilterLogsFromEvents(ctx context.Context, evs []*filter.CollectedEvent, ms *chain.MessageStore) ([]types.EthLog, error) { + var logs []types.EthLog for _, ev := range evs { log := types.EthLog{ Removed: ev.Reverted, @@ -669,6 +859,20 @@ func ethFilterResultFromEvents(evs []*filter.CollectedEvent, ms *chain.MessageSt return nil, err } + logs = append(logs, log) + } + + return logs, nil +} + +func ethFilterResultFromEvents(ctx context.Context, evs []*filter.CollectedEvent, ms *chain.MessageStore) (*types.EthFilterResult, error) { + logs, err := ethFilterLogsFromEvents(ctx, evs, ms) + if err != nil { + return nil, err + } + + res := &types.EthFilterResult{} + for _, log := range logs { res.Results = append(res.Results, log) } @@ -870,7 +1074,7 @@ func (e *ethSubscription) start(ctx context.Context) { case v := <-e.in: switch vt := v.(type) { case *filter.CollectedEvent: - evs, err := ethFilterResultFromEvents([]*filter.CollectedEvent{vt}, e.messageStore) + evs, err := ethFilterResultFromEvents(ctx, []*filter.CollectedEvent{vt}, e.messageStore) if err != nil { continue } diff --git a/app/submodule/eth/eth_utils.go b/app/submodule/eth/eth_utils.go index 746b3f1e5c..f08ea2841c 100644 --- a/app/submodule/eth/eth_utils.go +++ b/app/submodule/eth/eth_utils.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/go-cid" "github.com/multiformats/go-multicodec" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -25,7 +24,6 @@ import ( "github.com/filecoin-project/venus/pkg/vm/gas" "github.com/filecoin-project/venus/venus-shared/actors" types2 "github.com/filecoin-project/venus/venus-shared/actors/types" - v1 "github.com/filecoin-project/venus/venus-shared/api/chain/v1" "github.com/filecoin-project/venus/venus-shared/types" ) @@ -259,7 +257,7 @@ func newEthBlockFromFilecoinTipSet(ctx context.Context, ts *types.TipSet, fullTx }, } default: - return types.EthBlock{}, xerrors.Errorf("failed to get signed msg %s: %w", msg.Cid(), err) + return types.EthBlock{}, fmt.Errorf("failed to get signed msg %s: %w", msg.Cid(), err) } tx, err := newEthTxFromSignedMessage(ctx, smsg, st) if err != nil { @@ -317,7 +315,7 @@ func executeTipset(ctx context.Context, ts *types.TipSet, ms *chain.MessageStore stRoot, rcptRoot, err := stmgr.RunStateTransition(ctx, ts, nil, false) if err != nil { - return cid.Undef, nil, nil, fmt.Errorf("failed to compute state: %w", err) + return cid.Undef, nil, nil, fmt.Errorf("failed to compute tipset state: %w", err) } rcpts, err := ms.LoadReceipts(ctx, rcptRoot) @@ -650,19 +648,25 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *types.MsgLookup, } } - blkHash, err := types.EthHashFromCid(parentTSCid) + state, err := cr.GetTipSetState(ctx, parentTS) if err != nil { - return types.EthTx{}, err + return types.EthTx{}, fmt.Errorf("failed to load message state tree: %w", err) } - smsg, err := getSignedMessage(ctx, ms, msgLookup.Message) - if err != nil { - return types.EthTx{}, fmt.Errorf("failed to get signed msg: %w", err) - } + return newEthTx(ctx, state, parentTS.Height(), parentTSCid, msgLookup.Message, txIdx, ms) +} - state, err := cr.GetTipSetState(ctx, parentTS) +func newEthTx(ctx context.Context, + state tree.Tree, + blockHeight abi.ChainEpoch, + msgTsCid cid.Cid, + msgCid cid.Cid, + txIdx int, + ms *chain.MessageStore, +) (types.EthTx, error) { + smsg, err := getSignedMessage(ctx, ms, msgCid) if err != nil { - return types.EthTx{}, xerrors.Errorf("failed to load message state tree: %w", err) + return types.EthTx{}, fmt.Errorf("failed to get signed msg: %w", err) } tx, err := newEthTxFromSignedMessage(ctx, smsg, state) @@ -671,10 +675,15 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *types.MsgLookup, } var ( - bn = types.EthUint64(parentTS.Height()) + bn = types.EthUint64(blockHeight) ti = types.EthUint64(txIdx) ) + blkHash, err := types.EthHashFromCid(msgTsCid) + if err != nil { + return types.EthTx{}, err + } + tx.ChainID = types.EthUint64(types2.Eip155ChainID) tx.BlockHash = &blkHash tx.BlockNumber = &bn @@ -684,10 +693,9 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *types.MsgLookup, func newEthTxReceipt(ctx context.Context, tx types.EthTx, - lookup *types.MsgLookup, - events []types.Event, - ca v1.IChain, - cr *chain.Store, + baseFee big.Int, + msgReceipt types.MessageReceipt, + ev *ethEventAPI, ) (types.EthTxReceipt, error) { var ( transactionIndex types.EthUint64 @@ -717,37 +725,18 @@ func newEthTxReceipt(ctx context.Context, LogsBloom: types.EmptyEthBloom[:], } - if lookup.Receipt.ExitCode.IsSuccess() { + if msgReceipt.ExitCode.IsSuccess() { receipt.Status = 1 } - if lookup.Receipt.ExitCode.IsError() { + if msgReceipt.ExitCode.IsError() { receipt.Status = 0 } - receipt.GasUsed = types.EthUint64(lookup.Receipt.GasUsed) + receipt.GasUsed = types.EthUint64(msgReceipt.GasUsed) // TODO: handle CumulativeGasUsed receipt.CumulativeGasUsed = types.EmptyEthInt - // TODO: avoid loading the tipset twice (once here, once when we convert the message to a txn) - ts, err := ca.ChainGetTipSet(ctx, lookup.TipSet) - if err != nil { - return types.EthTxReceipt{}, fmt.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", lookup.TipSet, err) - } - - // The tx is located in the parent tipset - parentTS, err := ca.ChainGetTipSet(ctx, ts.Parents()) - if err != nil { - return types.EthTxReceipt{}, fmt.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", ts.Parents(), err) - } - - state, err := cr.GetTipSetState(ctx, parentTS) - if err != nil { - return types.EthTxReceipt{}, fmt.Errorf("failed to load the state %s when constructing the eth txn receipt: %w", ts.ParentState(), err) - } - - baseFee := parentTS.Blocks()[0].ParentBaseFee - gasFeeCap, err := tx.GasFeeCap() if err != nil { return types.EthTxReceipt{}, fmt.Errorf("failed to get gas fee cap: %w", err) @@ -757,62 +746,41 @@ func newEthTxReceipt(ctx context.Context, return types.EthTxReceipt{}, fmt.Errorf("failed to get gas premium: %w", err) } - gasOutputs := gas.ComputeGasOutputs(lookup.Receipt.GasUsed, int64(tx.Gas), baseFee, big.Int(gasFeeCap), + gasOutputs := gas.ComputeGasOutputs(msgReceipt.GasUsed, int64(tx.Gas), baseFee, big.Int(gasFeeCap), big.Int(gasPremium), true) totalSpent := big.Sum(gasOutputs.BaseFeeBurn, gasOutputs.MinerTip, gasOutputs.OverEstimationBurn) effectiveGasPrice := big.Zero() - if lookup.Receipt.GasUsed > 0 { - effectiveGasPrice = big.Div(totalSpent, big.NewInt(lookup.Receipt.GasUsed)) + if msgReceipt.GasUsed > 0 { + effectiveGasPrice = big.Div(totalSpent, big.NewInt(msgReceipt.GasUsed)) } receipt.EffectiveGasPrice = types.EthBigInt(effectiveGasPrice) - if receipt.To == nil && lookup.Receipt.ExitCode.IsSuccess() { + if receipt.To == nil && msgReceipt.ExitCode.IsSuccess() { // Create and Create2 return the same things. var ret eam.CreateExternalReturn - if err := ret.UnmarshalCBOR(bytes.NewReader(lookup.Receipt.Return)); err != nil { + if err := ret.UnmarshalCBOR(bytes.NewReader(msgReceipt.Return)); err != nil { return types.EthTxReceipt{}, fmt.Errorf("failed to parse contract creation result: %w", err) } addr := types.EthAddress(ret.EthAddress) receipt.ContractAddress = &addr } - if len(events) > 0 { - receipt.Logs = make([]types.EthLog, 0, len(events)) - for i, evt := range events { - l := types.EthLog{ - Removed: false, - LogIndex: types.EthUint64(i), - TransactionHash: tx.Hash, - TransactionIndex: transactionIndex, - BlockHash: blockHash, - BlockNumber: blockNumber, - } - - data, topics, ok := ethLogFromEvent(evt.Entries) - if !ok { - // not an eth event. - continue - } - for _, topic := range topics { - types.EthBloomSet(receipt.LogsBloom, topic[:]) - } - l.Data = data - l.Topics = topics - - addr, err := address.NewIDAddress(uint64(evt.Emitter)) - if err != nil { - return types.EthTxReceipt{}, fmt.Errorf("failed to create ID address: %w", err) - } - - l.Address, err = lookupEthAddress(ctx, addr, state) - if err != nil { - return types.EthTxReceipt{}, fmt.Errorf("failed to resolve Ethereum address: %w", err) - } + if rct := msgReceipt; rct.EventsRoot != nil { + logs, err := ev.getEthLogsForBlockAndTransaction(ctx, &blockHash, tx.Hash) + if err != nil { + return types.EthTxReceipt{}, fmt.Errorf("failed to get eth logs for block and transaction: %w", err) + } + if len(logs) > 0 { + receipt.Logs = logs + } + } - types.EthBloomSet(receipt.LogsBloom, l.Address[:]) - receipt.Logs = append(receipt.Logs, l) + for _, log := range receipt.Logs { + for _, topic := range log.Topics { + types.EthBloomSet(receipt.LogsBloom, topic[:]) } + types.EthBloomSet(receipt.LogsBloom, log.Address[:]) } return receipt, nil diff --git a/fixtures/networks/forcenet.go b/fixtures/networks/forcenet.go index 72aac1f28a..1516aa3d4d 100644 --- a/fixtures/networks/forcenet.go +++ b/fixtures/networks/forcenet.go @@ -75,7 +75,7 @@ func ForceNet() *NetworkConf { Eip155ChainID: 31415926, ActorDebugging: true, F3Enabled: true, - F3BootstrapEpoch: 100, + F3BootstrapEpoch: 1000, ManifestServerID: "12D3KooWHcNBkqXEBrsjoveQvj6zDF3vK5S9tAfqyYaQF1LGSJwG", F3Consensus: true, F3InitialPowerTableCID: cid.Undef, diff --git a/go.sum b/go.sum index 4b1c410f18..17906efaca 100644 --- a/go.sum +++ b/go.sum @@ -1516,13 +1516,10 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1565,9 +1562,6 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1634,11 +1628,9 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1680,9 +1672,6 @@ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1796,16 +1785,12 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1815,13 +1800,10 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1838,10 +1820,7 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1914,8 +1893,6 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1924,7 +1901,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= diff --git a/pkg/config/config.go b/pkg/config/config.go index 1d7c4e2545..604d3ea424 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -503,6 +503,10 @@ type EventsConfig struct { // This will also enable the RealTimeFilterAPI and HistoricFilterAPI by default, but they can be // disabled by setting their respective Disable* options in Fevm.Event. EnableActorEventsAPI bool `json:"enableActorEventsAPI"` + + // MaxFilterHeightRange specifies the maximum range of heights that can be used in a filter (to avoid querying + // the entire chain) + MaxFilterHeightRange uint64 } func newFevmConfig() *FevmConfig { @@ -524,6 +528,7 @@ func newFevmConfig() *FevmConfig { func newEventsConfig() *EventsConfig { return &EventsConfig{ EnableActorEventsAPI: false, + MaxFilterHeightRange: 2880, // conservative limit of one day } } diff --git a/pkg/events/filter/event.go b/pkg/events/filter/event.go index 3d801bc12d..af13f832cd 100644 --- a/pkg/events/filter/event.go +++ b/pkg/events/filter/event.go @@ -10,13 +10,12 @@ import ( "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/xerrors" "github.com/filecoin-project/go-address" amt4 "github.com/filecoin-project/go-amt-ipld/v4" "github.com/filecoin-project/go-state-types/abi" blockadt "github.com/filecoin-project/specs-actors/actors/util/adt" - "github.com/filecoin-project/venus/pkg/chain" + cstore "github.com/filecoin-project/venus/pkg/chain" "github.com/filecoin-project/venus/venus-shared/actors/adt" "github.com/filecoin-project/venus/venus-shared/types" cbor "github.com/ipfs/go-ipld-cbor" @@ -59,7 +58,7 @@ var _ Filter = (*eventFilter)(nil) type CollectedEvent struct { Entries []types.EventEntry EmitterAddr address.Address // address of emitter - EventIdx int // index of the event within the list of emitted events + EventIdx int // index of the event within the list of emitted events in a given tipset Reverted bool Height abi.ChainEpoch TipSetKey types.TipSetKey // tipset that contained the message @@ -96,13 +95,16 @@ func (f *eventFilter) CollectEvents(ctx context.Context, te *TipSetEvents, rever if err != nil { return fmt.Errorf("load executed messages: %w", err) } + + eventCount := 0 + for msgIdx, em := range ems { - for evIdx, ev := range em.Events() { + for _, ev := range em.Events() { // lookup address corresponding to the actor id addr, found := addressLookups[ev.Emitter] if !found { var ok bool - addr, ok = resolver(ctx, ev.Emitter, te.rctTS) + addr, ok = resolver(ctx, ev.Emitter, te.rctTs) if !ok { // not an address we will be able to match against continue @@ -121,10 +123,10 @@ func (f *eventFilter) CollectEvents(ctx context.Context, te *TipSetEvents, rever cev := &CollectedEvent{ Entries: ev.Entries, EmitterAddr: addr, - EventIdx: evIdx, + EventIdx: eventCount, Reverted: revert, - Height: te.msgTS.Height(), - TipSetKey: te.msgTS.Key(), + Height: te.msgTs.Height(), + TipSetKey: te.msgTs.Key(), MsgCid: em.Message().Cid(), MsgIdx: msgIdx, } @@ -143,6 +145,7 @@ func (f *eventFilter) CollectEvents(ctx context.Context, te *TipSetEvents, rever } f.collected = append(f.collected, cev) f.mu.Unlock() + eventCount++ } } @@ -244,14 +247,15 @@ func (f *eventFilter) matchKeys(ees []types.EventEntry) bool { // all keys have been matched return true } + } return false } type TipSetEvents struct { - rctTS *types.TipSet // rctTs is the tipset containing the receipts of executed messages - msgTS *types.TipSet // msgTs is the tipset containing the messages that have been executed + rctTs *types.TipSet // rctTs is the tipset containing the receipts of executed messages + msgTs *types.TipSet // msgTs is the tipset containing the messages that have been executed load func(ctx context.Context, msgTs, rctTs *types.TipSet) ([]executedMessage, error) @@ -261,17 +265,17 @@ type TipSetEvents struct { } func (te *TipSetEvents) Height() abi.ChainEpoch { - return te.msgTS.Height() + return te.msgTs.Height() } func (te *TipSetEvents) Cid() (cid.Cid, error) { - return te.msgTS.Key().Cid() + return te.msgTs.Key().Cid() } func (te *TipSetEvents) messages(ctx context.Context) ([]executedMessage, error) { te.once.Do(func() { // populate executed message list - ems, err := te.load(ctx, te.msgTS, te.rctTS) + ems, err := te.load(ctx, te.msgTs, te.rctTs) if err != nil { te.err = err return @@ -301,8 +305,8 @@ func (e *executedMessage) Events() []*types.Event { } type EventFilterManager struct { - MessageStore *chain.MessageStore - ChainStore *chain.Store + ChainStore *cstore.Store + MessageStore *cstore.MessageStore AddressResolver func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool) MaxFilterResults int EventIndex *EventIndex @@ -322,8 +326,8 @@ func (m *EventFilterManager) Apply(ctx context.Context, from, to *types.TipSet) } tse := &TipSetEvents{ - msgTS: from, - rctTS: to, + msgTs: from, + rctTs: to, load: m.loadExecutedMessages, } @@ -353,8 +357,8 @@ func (m *EventFilterManager) Revert(ctx context.Context, from, to *types.TipSet) } tse := &TipSetEvents{ - msgTS: to, - rctTS: from, + msgTs: to, + rctTs: from, load: m.loadExecutedMessages, } @@ -385,12 +389,12 @@ func (m *EventFilterManager) Install(ctx context.Context, minHeight, maxHeight a m.mu.Unlock() if m.EventIndex == nil && minHeight != -1 && minHeight < currentHeight { - return nil, xerrors.Errorf("historic event index disabled") + return nil, fmt.Errorf("historic event index disabled") } id, err := newFilterID() if err != nil { - return nil, xerrors.Errorf("new filter id: %w", err) + return nil, fmt.Errorf("new filter id: %w", err) } f := &eventFilter{ @@ -430,21 +434,21 @@ func (m *EventFilterManager) Remove(ctx context.Context, id types.FilterID) erro return nil } -func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTS, rctTS *types.TipSet) ([]executedMessage, error) { - msgs, err := m.MessageStore.MessagesForTipset(msgTS) +func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTs, rctTs *types.TipSet) ([]executedMessage, error) { + msgs, err := m.MessageStore.MessagesForTipset(msgTs) if err != nil { - return nil, xerrors.Errorf("read messages: %w", err) + return nil, fmt.Errorf("read messages: %w", err) } st := adt.WrapStore(ctx, cbor.NewCborStore(m.ChainStore.Blockstore())) - arr, err := blockadt.AsArray(st, rctTS.Blocks()[0].ParentMessageReceipts) + arr, err := blockadt.AsArray(st, rctTs.Blocks()[0].ParentMessageReceipts) if err != nil { - return nil, xerrors.Errorf("load receipts amt: %w", err) + return nil, fmt.Errorf("load receipts amt: %w", err) } if uint64(len(msgs)) != arr.Length() { - return nil, xerrors.Errorf("mismatching message and receipt counts (%d msgs, %d rcts)", len(msgs), arr.Length()) + return nil, fmt.Errorf("mismatching message and receipt counts (%d msgs, %d rcts)", len(msgs), arr.Length()) } ems := make([]executedMessage, len(msgs)) @@ -455,10 +459,10 @@ func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTS, rc var rct types.MessageReceipt found, err := arr.Get(uint64(i), &rct) if err != nil { - return nil, xerrors.Errorf("load receipt: %w", err) + return nil, fmt.Errorf("load receipt: %w", err) } if !found { - return nil, xerrors.Errorf("receipt %d not found", i) + return nil, fmt.Errorf("receipt %d not found", i) } ems[i].rct = &rct @@ -468,14 +472,14 @@ func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTS, rc evtArr, err := amt4.LoadAMT(ctx, st, *rct.EventsRoot, amt4.UseTreeBitWidth(types.EventAMTBitwidth)) if err != nil { - return nil, xerrors.Errorf("load events amt: %w", err) + return nil, fmt.Errorf("load events amt: %w", err) } ems[i].evs = make([]*types.Event, evtArr.Len()) var evt types.Event err = evtArr.ForEach(ctx, func(u uint64, deferred *cbg.Deferred) error { if u > math.MaxInt { - return xerrors.Errorf("too many events") + return fmt.Errorf("too many events") } if err := evt.UnmarshalCBOR(bytes.NewReader(deferred.Raw)); err != nil { return err @@ -487,7 +491,7 @@ func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTS, rc }) if err != nil { - return nil, xerrors.Errorf("read events: %w", err) + return nil, fmt.Errorf("read events: %w", err) } } diff --git a/pkg/events/filter/event_test.go b/pkg/events/filter/event_test.go index 92601f2c79..36c6e20997 100644 --- a/pkg/events/filter/event_test.go +++ b/pkg/events/filter/event_test.go @@ -66,7 +66,7 @@ func TestEventFilterCollectEvents(t *testing.T) { } events14000 := buildTipSetEvents(t, rng, 14000, em) - cid14000, err := events14000.msgTS.Key().Cid() + cid14000, err := events14000.msgTs.Key().Cid() require.NoError(t, err, "tipset cid") noCollectedEvents := []*CollectedEvent{} @@ -77,7 +77,7 @@ func TestEventFilterCollectEvents(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTS.Key(), + TipSetKey: events14000.msgTs.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -421,12 +421,12 @@ func newStore() adt.Store { func buildTipSetEvents(tb testing.TB, rng *pseudo.Rand, h abi.ChainEpoch, em executedMessage) *TipSetEvents { tb.Helper() - msgTS := fakeTipSet(tb, rng, h, []cid.Cid{}) - rctTS := fakeTipSet(tb, rng, h+1, msgTS.Cids()) + msgTs := fakeTipSet(tb, rng, h, []cid.Cid{}) + rctTs := fakeTipSet(tb, rng, h+1, msgTs.Cids()) return &TipSetEvents{ - msgTS: msgTS, - rctTS: rctTS, + msgTs: msgTs, + rctTs: rctTs, load: func(ctx context.Context, msgTs, rctTs *types.TipSet) ([]executedMessage, error) { return []executedMessage{em}, nil }, diff --git a/pkg/events/filter/index.go b/pkg/events/filter/index.go index 2f7058d1d4..6020965390 100644 --- a/pkg/events/filter/index.go +++ b/pkg/events/filter/index.go @@ -7,29 +7,22 @@ import ( "fmt" "sort" "strings" - "time" + "sync" - "github.com/filecoin-project/venus/pkg/chain" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" _ "github.com/mattn/go-sqlite3" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/venus/pkg/chain" + "github.com/filecoin-project/venus/pkg/events/filter/sqlite" "github.com/filecoin-project/venus/venus-shared/types" ) -var pragmas = []string{ - "PRAGMA synchronous = normal", - "PRAGMA temp_store = memory", - "PRAGMA mmap_size = 30000000000", - "PRAGMA page_size = 32768", - "PRAGMA auto_vacuum = NONE", - "PRAGMA automatic_index = OFF", - "PRAGMA journal_mode = WAL", - "PRAGMA wal_autocheckpoint = 256", // checkpoint @ 256 pages - "PRAGMA journal_size_limit = 0", // always reset journal and wal files -} +const DefaultDbFilename = "events.db" + +// Any changes to this schema should be matched for the `lotus-shed indexes backfill-events` command var ddls = []string{ `CREATE TABLE IF NOT EXISTS event ( @@ -44,7 +37,8 @@ var ddls = []string{ reverted INTEGER NOT NULL )`, - `CREATE INDEX IF NOT EXISTS height_tipset_key ON event (height,tipset_key)`, + createIndexEventTipsetKeyCid, + createIndexEventHeight, `CREATE TABLE IF NOT EXISTS event_entry ( event_id INTEGER, @@ -55,299 +49,224 @@ var ddls = []string{ value BLOB NOT NULL )`, - // metadata containing version of schema - `CREATE TABLE IF NOT EXISTS _meta ( - version UINT64 NOT NULL UNIQUE - )`, + createTableEventsSeen, - // version 1. - `INSERT OR IGNORE INTO _meta (version) VALUES (1)`, - `INSERT OR IGNORE INTO _meta (version) VALUES (2)`, + createIndexEventEntryEventId, + createIndexEventsSeenHeight, + createIndexEventsSeenTipsetKeyCid, } -// const schemaVersion = 1 var ( log = logging.Logger("filter") ) const ( - schemaVersion = 2 - - eventExists = `SELECT MAX(id) FROM event WHERE height=? AND tipset_key=? AND tipset_key_cid=? AND emitter_addr=? AND event_index=? AND message_cid=? AND message_index=?` - insertEvent = `INSERT OR IGNORE INTO event(height, tipset_key, tipset_key_cid, emitter_addr, event_index, message_cid, message_index, reverted) VALUES(?, ?, ?, ?, ?, ?, ?, ?)` - insertEntry = `INSERT OR IGNORE INTO event_entry(event_id, indexed, flags, key, codec, value) VALUES(?, ?, ?, ?, ?, ?)` - revertEventsInTipset = `UPDATE event SET reverted=true WHERE height=? AND tipset_key=?` - restoreEvent = `UPDATE event SET reverted=false WHERE height=? AND tipset_key=? AND tipset_key_cid=? AND emitter_addr=? AND event_index=? AND message_cid=? AND message_index=?` + createTableEventsSeen = `CREATE TABLE IF NOT EXISTS events_seen ( + id INTEGER PRIMARY KEY, + height INTEGER NOT NULL, + tipset_key_cid BLOB NOT NULL, + reverted INTEGER NOT NULL, + UNIQUE(height, tipset_key_cid) + )` + + // When modifying indexes in this file, it is critical to test the query plan (EXPLAIN QUERY PLAN) + // of all the variations of queries built by prefillFilter to ensure that the query first hits + // an index that narrows down results to an epoch or a reasonable range of epochs. Specifically, + // event_tipset_key_cid or event_height should be the first index. Then further narrowing can take + // place within the small subset of results. + // Unfortunately SQLite has some quirks in index selection that mean that certain query types will + // bypass these indexes if alternatives are available. This has been observed specifically on + // queries with height ranges: `height>=X AND height<=Y`. + // + // e.g. we want to see that `event_height` is the first index used in this query: + // + // EXPLAIN QUERY PLAN + // SELECT + // event.height, event.tipset_key_cid, event_entry.indexed, event_entry.codec, event_entry.key, event_entry.value + // FROM event + // JOIN + // event_entry ON event.id=event_entry.event_id, + // event_entry ee2 ON event.id=ee2.event_id + // WHERE event.height>=? AND event.height<=? AND event.reverted=? AND event.emitter_addr=? AND ee2.indexed=1 AND ee2.key=? + // ORDER BY event.height DESC, event_entry._rowid_ ASC + // + // -> + // + // QUERY PLAN + // |--SEARCH event USING INDEX event_height (height>? AND height 0 FROM events_seen WHERE tipset_key_cid=?`, // QUERY PLAN: SEARCH events_seen USING COVERING INDEX events_seen_tipset_key_cid (tipset_key_cid=?) + &ps.getMaxHeightInIndex: `SELECT MAX(height) FROM events_seen`, // QUERY PLAN: SEARCH events_seen USING COVERING INDEX events_seen_height + &ps.isHeightProcessed: `SELECT COUNT(*) > 0 FROM events_seen WHERE height=?`, // QUERY PLAN: SEARCH events_seen USING COVERING INDEX events_seen_height (height=?) - stmtEventExists *sql.Stmt - stmtInsertEvent *sql.Stmt - stmtInsertEntry *sql.Stmt - stmtRevertEventsInTipset *sql.Stmt - stmtRestoreEvent *sql.Stmt -} - -func (ei *EventIndex) initStatements() (err error) { - ei.stmtEventExists, err = ei.db.Prepare(eventExists) - if err != nil { - return fmt.Errorf("prepare stmtEventExists: %w", err) - } - - ei.stmtInsertEvent, err = ei.db.Prepare(insertEvent) - if err != nil { - return fmt.Errorf("prepare stmtInsertEvent: %w", err) } +} - ei.stmtInsertEntry, err = ei.db.Prepare(insertEntry) - if err != nil { - return fmt.Errorf("prepare stmtInsertEntry: %w", err) - } +type preparedStatements struct { + insertEvent *sql.Stmt + insertEntry *sql.Stmt + revertEventsInTipset *sql.Stmt + restoreEvent *sql.Stmt + upsertEventsSeen *sql.Stmt + revertEventSeen *sql.Stmt + restoreEventSeen *sql.Stmt + eventExists *sql.Stmt + isTipsetProcessed *sql.Stmt + getMaxHeightInIndex *sql.Stmt + isHeightProcessed *sql.Stmt +} - ei.stmtRevertEventsInTipset, err = ei.db.Prepare(revertEventsInTipset) - if err != nil { - return fmt.Errorf("prepare stmtRevertEventsInTipset: %w", err) - } +type EventIndex struct { + db *sql.DB - ei.stmtRestoreEvent, err = ei.db.Prepare(restoreEvent) - if err != nil { - return fmt.Errorf("prepare stmtRestoreEvent: %w", err) - } + stmt *preparedStatements - return nil + mu sync.Mutex + subIdCounter uint64 + updateSubs map[uint64]*updateSub } -func (ei *EventIndex) migrateToVersion2(ctx context.Context, chainStore *chain.Store) error { - now := time.Now() +type updateSub struct { + ctx context.Context + ch chan EventIndexUpdated + cancel context.CancelFunc +} - tx, err := ei.db.Begin() - if err != nil { - return fmt.Errorf("begin transaction: %w", err) - } - // rollback the transaction (a no-op if the transaction was already committed) - defer tx.Rollback() //nolint:errcheck +type EventIndexUpdated struct{} - // create some temporary indices to help speed up the migration - _, err = tx.Exec("CREATE INDEX IF NOT EXISTS tmp_height_tipset_key_cid ON event (height,tipset_key_cid)") - if err != nil { - return fmt.Errorf("create index tmp_height_tipset_key_cid: %w", err) - } - _, err = tx.Exec("CREATE INDEX IF NOT EXISTS tmp_tipset_key_cid ON event (tipset_key_cid)") - if err != nil { - return fmt.Errorf("create index tmp_tipset_key_cid: %w", err) - } - - stmtDeleteOffChainEvent, err := tx.Prepare("DELETE FROM event WHERE tipset_key_cid!=? and height=?") +func NewEventIndex(ctx context.Context, path string, chainStore *chain.Store) (*EventIndex, error) { + db, _, err := sqlite.Open(path) if err != nil { - return fmt.Errorf("prepare stmtDeleteOffChainEvent: %w", err) + return nil, fmt.Errorf("failed to setup event index db: %w", err) } - stmtSelectEvent, err := tx.Prepare("SELECT id FROM event WHERE tipset_key_cid=? ORDER BY message_index ASC, event_index ASC, id DESC LIMIT 1") + err = sqlite.InitDb(ctx, "event index", db, ddls, []sqlite.MigrationFunc{ + migrationVersion2(db, chainStore), + migrationVersion3, + migrationVersion4, + migrationVersion5, + migrationVersion6, + migrationVersion7, + }) if err != nil { - return fmt.Errorf("prepare stmtSelectEvent: %w", err) + _ = db.Close() + return nil, fmt.Errorf("failed to setup event index db: %w", err) } - stmtDeleteEvent, err := tx.Prepare("DELETE FROM event WHERE tipset_key_cid=? AND id 0 说明 event 表里有数据，需要迁移数据 - if minHeight.Int64 > 0 { - for int64(currTS.Height()) >= minHeight.Int64 { - if currTS.Height()%1000 == 0 { - log.Infof("Migrating height %d (remaining %d)", currTS.Height(), int64(currTS.Height())-minHeight.Int64) - } - - tsKey := currTS.Parents() - currTS, err = chainStore.GetTipSet(ctx, tsKey) - if err != nil { - return fmt.Errorf("get tipset from key: %w", err) - } - log.Debugf("Migrating height %d", currTS.Height()) - - tsKeyCid, err := currTS.Key().Cid() - if err != nil { - return fmt.Errorf("tipset key cid: %w", err) - } - - // delete all events that are not in the canonical chain - _, err = stmtDeleteOffChainEvent.Exec(tsKeyCid.Bytes(), currTS.Height()) - if err != nil { - return fmt.Errorf("delete off chain event: %w", err) - } - - // find the first eventID from the last time the tipset was applied - var eventID sql.NullInt64 - err = stmtSelectEvent.QueryRow(tsKeyCid.Bytes()).Scan(&eventID) - if err != nil { - if err == sql.ErrNoRows { - continue - } - return fmt.Errorf("select event: %w", err) - } - - // this tipset might not have any events which is ok - if !eventID.Valid { - continue - } - log.Debugf("Deleting all events with id < %d at height %d", eventID.Int64, currTS.Height()) - - res, err := stmtDeleteEvent.Exec(tsKeyCid.Bytes(), eventID.Int64) - if err != nil { - return fmt.Errorf("delete event: %w", err) - } + return &eventIndex, nil +} - nrRowsAffected, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("rows affected: %w", err) - } - log.Debugf("deleted %d events from tipset %s", nrRowsAffected, tsKeyCid.String()) +func (ei *EventIndex) initStatements() error { + stmtMapping := preparedStatementMapping(ei.stmt) + for stmtPointer, query := range stmtMapping { + var err error + *stmtPointer, err = ei.db.Prepare(query) + if err != nil { + return fmt.Errorf("prepare statement [%s]: %w", query, err) } } - // delete all entries that have an event_id that doesn't exist (since we don't have a foreign - // key constraint that gives us cascading deletes) - res, err := tx.Exec("DELETE FROM event_entry WHERE event_id NOT IN (SELECT id FROM event)") - if err != nil { - return fmt.Errorf("delete event_entry: %w", err) - } - - nrRowsAffected, err := res.RowsAffected() - if err != nil { - return fmt.Errorf("rows affected: %w", err) - } - log.Infof("cleaned up %d entries that had deleted events", nrRowsAffected) - - // drop the temporary indices after the migration - _, err = tx.Exec("DROP INDEX IF EXISTS tmp_tipset_key_cid") - if err != nil { - return fmt.Errorf("create index tmp_tipset_key_cid: %w", err) - } - _, err = tx.Exec("DROP INDEX IF EXISTS tmp_height_tipset_key_cid") - if err != nil { - return fmt.Errorf("drop index tmp_height_tipset_key_cid: %w", err) - } - - err = tx.Commit() - if err != nil { - return fmt.Errorf("commit transaction: %w", err) - } - - // during the migration, we have likely increased the WAL size a lot, so lets do some - // simple DB administration to free up space (VACUUM followed by truncating the WAL file) - // as this would be a good time to do it when no other writes are happening - log.Infof("Performing DB vacuum and wal checkpointing to free up space after the migration") - _, err = ei.db.Exec("VACUUM") - if err != nil { - log.Warnf("error vacuuming database: %s", err) - } - _, err = ei.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)") - if err != nil { - log.Warnf("error checkpointing wal: %s", err) - } - - log.Infof("Successfully migrated events to version 2 in %s", time.Since(now)) - return nil } -func NewEventIndex(ctx context.Context, path string, chainStore *chain.Store) (*EventIndex, error) { - db, err := sql.Open("sqlite3", path+"?mode=rwc") - if err != nil { - return nil, fmt.Errorf("open sqlite3 database: %w", err) - } - - for _, pragma := range pragmas { - if _, err := db.Exec(pragma); err != nil { - _ = db.Close() - return nil, fmt.Errorf("exec pragma %q: %w", pragma, err) - } +func (ei *EventIndex) Close() error { + if ei.db == nil { + return nil } + return ei.db.Close() +} - eventIndex := EventIndex{db: db} +func (ei *EventIndex) SubscribeUpdates() (chan EventIndexUpdated, func()) { + subCtx, subCancel := context.WithCancel(context.Background()) + ch := make(chan EventIndexUpdated) - q, err := db.Query("SELECT name FROM sqlite_master WHERE type='table' AND name='_meta';") - if q != nil { - defer func() { _ = q.Close() }() + tSub := &updateSub{ + ctx: subCtx, + cancel: subCancel, + ch: ch, } - if err == sql.ErrNoRows || !q.Next() { - // empty database, create the schema - for _, ddl := range ddls { - if _, err := db.Exec(ddl); err != nil { - _ = db.Close() - return nil, fmt.Errorf("exec ddl %q: %w", ddl, err) - } - } - } else if err != nil { - _ = db.Close() - return nil, fmt.Errorf("looking for _meta table: %w", err) - } else { - // check the schema version to see if we need to upgrade the database schema - var version int - err := db.QueryRow("SELECT max(version) FROM _meta").Scan(&version) - if err != nil { - _ = db.Close() - return nil, fmt.Errorf("invalid database version: no version found") - } - - if version == 1 { - log.Infof("upgrading event index from version 1 to version 2") - err = eventIndex.migrateToVersion2(ctx, chainStore) - if err != nil { - _ = db.Close() - return nil, fmt.Errorf("could not migrate sql data to version 2: %w", err) - } - - // to upgrade to version 2 we only need to create an index on the event table - // which means we can just recreate the schema (it will not have any effect on existing data) - for _, ddl := range ddls { - if _, err := db.Exec(ddl); err != nil { - _ = db.Close() - return nil, fmt.Errorf("could not upgrade index to version 2, exec ddl %q: %w", ddl, err) - } - } + ei.mu.Lock() + subId := ei.subIdCounter + ei.subIdCounter++ + ei.updateSubs[subId] = tSub + ei.mu.Unlock() - version = 2 + unSubscribeF := func() { + ei.mu.Lock() + tSub, ok := ei.updateSubs[subId] + if !ok { + ei.mu.Unlock() + return } + delete(ei.updateSubs, subId) + ei.mu.Unlock() - if version != schemaVersion { - _ = db.Close() - return nil, fmt.Errorf("invalid database version: got %d, expected %d", version, schemaVersion) - } + // cancel the subscription + tSub.cancel() } - err = eventIndex.initStatements() - if err != nil { - _ = db.Close() - return nil, fmt.Errorf("error preparing eventIndex database statements: %w", err) - } + return tSub.ch, unSubscribeF +} - return &eventIndex, nil +func (ei *EventIndex) GetMaxHeightInIndex(ctx context.Context) (uint64, error) { + row := ei.stmt.getMaxHeightInIndex.QueryRowContext(ctx) + var maxHeight uint64 + err := row.Scan(&maxHeight) + return maxHeight, err } -func (ei *EventIndex) Close() error { - if ei.db == nil { - return nil +func (ei *EventIndex) IsHeightPast(ctx context.Context, height uint64) (bool, error) { + maxHeight, err := ei.GetMaxHeightInIndex(ctx) + if err != nil { + return false, err } - return ei.db.Close() + return height <= maxHeight, nil +} + +func (ei *EventIndex) IsTipsetProcessed(ctx context.Context, tipsetKeyCid []byte) (bool, error) { + row := ei.stmt.isTipsetProcessed.QueryRowContext(ctx, tipsetKeyCid) + var exists bool + err := row.Scan(&exists) + return exists, err } func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, revert bool, resolver func(ctx context.Context, emitter abi.ActorID, ts *types.TipSet) (address.Address, bool)) error { @@ -356,20 +275,48 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever return fmt.Errorf("begin transaction: %w", err) } // rollback the transaction (a no-op if the transaction was already committed) - defer tx.Rollback() //nolint:errcheck + defer func() { _ = tx.Rollback() }() + + tsKeyCid, err := te.msgTs.Key().Cid() + if err != nil { + return fmt.Errorf("tipset key cid: %w", err) + } // lets handle the revert case first, since its simpler and we can simply mark all events in this tipset as reverted and return if revert { - _, err = tx.Stmt(ei.stmtRevertEventsInTipset).Exec(te.msgTS.Height(), te.msgTS.Key().Bytes()) + _, err = tx.Stmt(ei.stmt.revertEventsInTipset).Exec(te.msgTs.Height(), te.msgTs.Key().Bytes()) if err != nil { return fmt.Errorf("revert event: %w", err) } + _, err = tx.Stmt(ei.stmt.revertEventSeen).Exec(te.msgTs.Height(), tsKeyCid.Bytes()) + if err != nil { + return fmt.Errorf("revert event seen: %w", err) + } + err = tx.Commit() if err != nil { return fmt.Errorf("commit transaction: %w", err) } + ei.mu.Lock() + tSubs := make([]*updateSub, 0, len(ei.updateSubs)) + for _, tSub := range ei.updateSubs { + tSubs = append(tSubs, tSub) + } + ei.mu.Unlock() + + for _, tSub := range tSubs { + tSub := tSub + select { + case tSub.ch <- EventIndexUpdated{}: + case <-tSub.ctx.Done(): + // subscription was cancelled, ignore + case <-ctx.Done(): + return ctx.Err() + } + } + return nil } @@ -381,14 +328,15 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever return fmt.Errorf("load executed messages: %w", err) } + eventCount := 0 // iterate over all executed messages in this tipset and insert them into the database if they // don't exist, otherwise mark them as not reverted for msgIdx, em := range ems { - for evIdx, ev := range em.Events() { + for _, ev := range em.Events() { addr, found := addressLookups[ev.Emitter] if !found { var ok bool - addr, ok = resolver(ctx, ev.Emitter, te.rctTS) + addr, ok = resolver(ctx, ev.Emitter, te.rctTs) if !ok { // not an address we will be able to match against continue @@ -396,19 +344,14 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever addressLookups[ev.Emitter] = addr } - tsKeyCid, err := te.msgTS.Key().Cid() - if err != nil { - return fmt.Errorf("tipset key cid: %w", err) - } - // check if this event already exists in the database var entryID sql.NullInt64 - err = tx.Stmt(ei.stmtEventExists).QueryRow( - te.msgTS.Height(), // height - te.msgTS.Key().Bytes(), // tipset_key + err = tx.Stmt(ei.stmt.eventExists).QueryRow( + te.msgTs.Height(), // height + te.msgTs.Key().Bytes(), // tipset_key tsKeyCid.Bytes(), // tipset_key_cid addr.Bytes(), // emitter_addr - evIdx, // event_index + eventCount, // event_index em.Message().Cid().Bytes(), // message_cid msgIdx, // message_index ).Scan(&entryID) @@ -418,12 +361,12 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever if !entryID.Valid { // event does not exist, lets insert it - res, err := tx.Stmt(ei.stmtInsertEvent).Exec( - te.msgTS.Height(), // height - te.msgTS.Key().Bytes(), // tipset_key + res, err := tx.Stmt(ei.stmt.insertEvent).Exec( + te.msgTs.Height(), // height + te.msgTs.Key().Bytes(), // tipset_key tsKeyCid.Bytes(), // tipset_key_cid addr.Bytes(), // emitter_addr - evIdx, // event_index + eventCount, // event_index em.Message().Cid().Bytes(), // message_cid msgIdx, // message_index false, // reverted @@ -439,7 +382,7 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever // insert all the entries for this event for _, entry := range ev.Entries { - _, err = tx.Stmt(ei.stmtInsertEntry).Exec( + _, err = tx.Stmt(ei.stmt.insertEntry).Exec( entryID.Int64, // event_id isIndexedValue(entry.Flags), // indexed []byte{entry.Flags}, // flags @@ -453,12 +396,12 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever } } else { // event already exists, lets mark it as not reverted - res, err := tx.Stmt(ei.stmtRestoreEvent).Exec( - te.msgTS.Height(), // height - te.msgTS.Key().Bytes(), // tipset_key + res, err := tx.Stmt(ei.stmt.restoreEvent).Exec( + te.msgTs.Height(), // height + te.msgTs.Key().Bytes(), // tipset_key tsKeyCid.Bytes(), // tipset_key_cid addr.Bytes(), // emitter_addr - evIdx, // event_index + eventCount, // event_index em.Message().Cid().Bytes(), // message_cid msgIdx, // message_index ) @@ -476,93 +419,51 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever log.Warnf("restored %d events but expected only one to exist", rowsAffected) } } + eventCount++ } } - err = tx.Commit() + // this statement will mark the tipset as processed and will insert a new row if it doesn't exist + // or update the reverted field to false if it does + _, err = tx.Stmt(ei.stmt.upsertEventsSeen).Exec( + te.msgTs.Height(), + tsKeyCid.Bytes(), + ) if err != nil { - return fmt.Errorf("commit transaction: %w", err) + return fmt.Errorf("exec upsert events seen: %w", err) } - return nil -} - -// prefillFilter fills a filter's collection of events from the historic index -func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, excludeReverted bool) error { - clauses := []string{} - values := []any{} - joins := []string{} - - if f.tipsetCid != cid.Undef { - clauses = append(clauses, "event.tipset_key_cid=?") - values = append(values, f.tipsetCid.Bytes()) - } else { - if f.minHeight >= 0 { - clauses = append(clauses, "event.height>=?") - values = append(values, f.minHeight) - } - if f.maxHeight >= 0 { - clauses = append(clauses, "event.height<=?") - values = append(values, f.maxHeight) - } + err = tx.Commit() + if err != nil { + return fmt.Errorf("commit transaction: %w", err) } - if len(f.addresses) > 0 { - subclauses := []string{} - for _, addr := range f.addresses { - subclauses = append(subclauses, "emitter_addr=?") - values = append(values, addr.Bytes()) - } - clauses = append(clauses, "("+strings.Join(subclauses, " OR ")+")") + ei.mu.Lock() + tSubs := make([]*updateSub, 0, len(ei.updateSubs)) + for _, tSub := range ei.updateSubs { + tSubs = append(tSubs, tSub) } + ei.mu.Unlock() - if len(f.keysWithCodec) > 0 { - join := 0 - for key, vals := range f.keysWithCodec { - if len(vals) > 0 { - join++ - joinAlias := fmt.Sprintf("ee%d", join) - joins = append(joins, fmt.Sprintf("event_entry %s on event.id=%[1]s.event_id", joinAlias)) - clauses = append(clauses, fmt.Sprintf("%s.indexed=1 AND %[1]s.key=?", joinAlias)) - values = append(values, key) - subclauses := []string{} - for _, val := range vals { - subclauses = append(subclauses, fmt.Sprintf("(%s.value=? AND %[1]s.codec=?)", joinAlias)) - values = append(values, val.Value, val.Codec) - } - clauses = append(clauses, "("+strings.Join(subclauses, " OR ")+")") - } + for _, tSub := range tSubs { + tSub := tSub + select { + case tSub.ch <- EventIndexUpdated{}: + case <-tSub.ctx.Done(): + // subscription was cancelled, ignore + case <-ctx.Done(): + return ctx.Err() } } - s := `SELECT - event.id, - event.height, - event.tipset_key, - event.tipset_key_cid, - event.emitter_addr, - event.event_index, - event.message_cid, - event.message_index, - event.reverted, - event_entry.flags, - event_entry.key, - event_entry.codec, - event_entry.value - FROM event JOIN event_entry ON event.id=event_entry.event_id` - - if len(joins) > 0 { - s = s + ", " + strings.Join(joins, ", ") - } - - if len(clauses) > 0 { - s = s + " WHERE " + strings.Join(clauses, " AND ") - } + return nil +} - // retain insertion order of event_entry rows with the implicit _rowid_ column - s += " ORDER BY event.height DESC, event_entry._rowid_ ASC" +// prefillFilter fills a filter's collection of events from the historic index +func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, excludeReverted bool) error { + values, query := makePrefillFilterQuery(f, excludeReverted) - stmt, err := ei.db.Prepare(s) + stmt, err := ei.db.Prepare(query) if err != nil { return fmt.Errorf("prepare prefill query: %w", err) } @@ -664,7 +565,6 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude Codec: row.codec, Value: row.value, }) - } if ce != nil { @@ -682,3 +582,89 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude return nil } + +func makePrefillFilterQuery(f *eventFilter, excludeReverted bool) ([]any, string) { + clauses := []string{} + values := []any{} + joins := []string{} + + if f.tipsetCid != cid.Undef { + clauses = append(clauses, "event.tipset_key_cid=?") + values = append(values, f.tipsetCid.Bytes()) + } else { + if f.minHeight >= 0 && f.minHeight == f.maxHeight { + clauses = append(clauses, "event.height=?") + values = append(values, f.minHeight) + } else { + if f.maxHeight >= 0 && f.minHeight >= 0 { + clauses = append(clauses, "event.height BETWEEN ? AND ?") + values = append(values, f.minHeight, f.maxHeight) + } else if f.minHeight >= 0 { + clauses = append(clauses, "event.height >= ?") + values = append(values, f.minHeight) + } else if f.maxHeight >= 0 { + clauses = append(clauses, "event.height <= ?") + values = append(values, f.maxHeight) + } + } + } + + if excludeReverted { + clauses = append(clauses, "event.reverted=?") + values = append(values, false) + } + + if len(f.addresses) > 0 { + for _, addr := range f.addresses { + values = append(values, addr.Bytes()) + } + clauses = append(clauses, "event.emitter_addr IN ("+strings.Repeat("?,", len(f.addresses)-1)+"?)") + } + + if len(f.keysWithCodec) > 0 { + join := 0 + for key, vals := range f.keysWithCodec { + if len(vals) > 0 { + join++ + joinAlias := fmt.Sprintf("ee%d", join) + joins = append(joins, fmt.Sprintf("event_entry %s ON event.id=%[1]s.event_id", joinAlias)) + clauses = append(clauses, fmt.Sprintf("%s.indexed=1 AND %[1]s.key=?", joinAlias)) + values = append(values, key) + subclauses := make([]string, 0, len(vals)) + for _, val := range vals { + subclauses = append(subclauses, fmt.Sprintf("(%s.value=? AND %[1]s.codec=?)", joinAlias)) + values = append(values, val.Value, val.Codec) + } + clauses = append(clauses, "("+strings.Join(subclauses, " OR ")+")") + } + } + } + + s := `SELECT + event.id, + event.height, + event.tipset_key, + event.tipset_key_cid, + event.emitter_addr, + event.event_index, + event.message_cid, + event.message_index, + event.reverted, + event_entry.flags, + event_entry.key, + event_entry.codec, + event_entry.value + FROM event JOIN event_entry ON event.id=event_entry.event_id` + + if len(joins) > 0 { + s = s + ", " + strings.Join(joins, ", ") + } + + if len(clauses) > 0 { + s = s + " WHERE " + strings.Join(clauses, " AND ") + } + + // retain insertion order of event_entry rows with the implicit _rowid_ column + s += " ORDER BY event.height DESC, event_entry._rowid_ ASC" + return values, s +} diff --git a/pkg/events/filter/index_migrations.go b/pkg/events/filter/index_migrations.go new file mode 100644 index 0000000000..dc942a639e --- /dev/null +++ b/pkg/events/filter/index_migrations.go @@ -0,0 +1,258 @@ +package filter + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/filecoin-project/venus/pkg/chain" + "github.com/filecoin-project/venus/pkg/events/filter/sqlite" +) + +func migrationVersion2(db *sql.DB, chainStore *chain.Store) sqlite.MigrationFunc { + return func(ctx context.Context, tx *sql.Tx) error { + // create some temporary indices to help speed up the migration + _, err := tx.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS tmp_height_tipset_key_cid ON event (height,tipset_key_cid)") + if err != nil { + return fmt.Errorf("create index tmp_height_tipset_key_cid: %w", err) + } + _, err = tx.ExecContext(ctx, "CREATE INDEX IF NOT EXISTS tmp_tipset_key_cid ON event (tipset_key_cid)") + if err != nil { + return fmt.Errorf("create index tmp_tipset_key_cid: %w", err) + } + + stmtDeleteOffChainEvent, err := tx.PrepareContext(ctx, "DELETE FROM event WHERE tipset_key_cid!=? and height=?") + if err != nil { + return fmt.Errorf("prepare stmtDeleteOffChainEvent: %w", err) + } + + stmtSelectEvent, err := tx.PrepareContext(ctx, "SELECT id FROM event WHERE tipset_key_cid=? ORDER BY message_index ASC, event_index ASC, id DESC LIMIT 1") + if err != nil { + return fmt.Errorf("prepare stmtSelectEvent: %w", err) + } + + stmtDeleteEvent, err := tx.PrepareContext(ctx, "DELETE FROM event WHERE tipset_key_cid=? AND id= minHeight.Int64 { + if currTs.Height()%1000 == 0 { + log.Infof("Migrating height %d (remaining %d)", currTs.Height(), int64(currTs.Height())-minHeight.Int64) + } + + tsKey := currTs.Parents() + currTs, err = chainStore.GetTipSet(ctx, tsKey) + if err != nil { + return fmt.Errorf("get tipset from key: %w", err) + } + log.Debugf("Migrating height %d", currTs.Height()) + + tsKeyCid, err := currTs.Key().Cid() + if err != nil { + return fmt.Errorf("tipset key cid: %w", err) + } + + // delete all events that are not in the canonical chain + _, err = stmtDeleteOffChainEvent.Exec(tsKeyCid.Bytes(), currTs.Height()) + if err != nil { + return fmt.Errorf("delete off chain event: %w", err) + } + + // find the first eventId from the last time the tipset was applied + var eventId sql.NullInt64 + err = stmtSelectEvent.QueryRow(tsKeyCid.Bytes()).Scan(&eventId) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + continue + } + return fmt.Errorf("select event: %w", err) + } + + // this tipset might not have any events which is ok + if !eventId.Valid { + continue + } + log.Debugf("Deleting all events with id < %d at height %d", eventId.Int64, currTs.Height()) + + res, err := stmtDeleteEvent.Exec(tsKeyCid.Bytes(), eventId.Int64) + if err != nil { + return fmt.Errorf("delete event: %w", err) + } + + nrRowsAffected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected: %w", err) + } + log.Debugf("deleted %d events from tipset %s", nrRowsAffected, tsKeyCid.String()) + } + + // delete all entries that have an event_id that doesn't exist (since we don't have a foreign + // key constraint that gives us cascading deletes) + res, err := tx.ExecContext(ctx, "DELETE FROM event_entry WHERE event_id NOT IN (SELECT id FROM event)") + if err != nil { + return fmt.Errorf("delete event_entry: %w", err) + } + + nrRowsAffected, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("rows affected: %w", err) + } + log.Infof("Cleaned up %d entries that had deleted events\n", nrRowsAffected) + + // drop the temporary indices after the migration + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS tmp_tipset_key_cid") + if err != nil { + return fmt.Errorf("drop index tmp_tipset_key_cid: %w", err) + } + _, err = tx.ExecContext(ctx, "DROP INDEX IF EXISTS tmp_height_tipset_key_cid") + if err != nil { + return fmt.Errorf("drop index tmp_height_tipset_key_cid: %w", err) + } + + // original v2 migration introduced an index: + // CREATE INDEX IF NOT EXISTS height_tipset_key ON event (height,tipset_key) + // which has subsequently been removed in v4, so it's omitted here + + return nil + } +} + +// migrationVersion3 migrates the schema from version 2 to version 3 by creating two indices: +// 1) an index on the event.emitter_addr column, and 2) an index on the event_entry.key column. +// +// As of version 7, these indices have been removed as they were found to be a performance +// hindrance. This migration is now a no-op. +func migrationVersion3(ctx context.Context, tx *sql.Tx) error { + return nil +} + +// migrationVersion4 migrates the schema from version 3 to version 4 by adjusting indexes to match +// the query patterns of the event filter. +// +// First it drops indexes introduced in previous migrations: +// 1. the index on the event.height and event.tipset_key columns +// 2. the index on the event_entry.key column +// +// And then creating the following indices: +// 1. an index on the event.tipset_key_cid column +// 2. an index on the event.height column +// 3. an index on the event.reverted column (removed in version 7) +// 4. a composite index on the event_entry.indexed and event_entry.key columns (removed in version 7) +// 5. a composite index on the event_entry.codec and event_entry.value columns (removed in version 7) +// 6. an index on the event_entry.event_id column +// +// Indexes 3, 4, and 5 were removed in version 7 as they were found to be a performance hindrance so +// are omitted here. +func migrationVersion4(ctx context.Context, tx *sql.Tx) error { + for _, create := range []struct { + desc string + query string + }{ + {"drop index height_tipset_key", "DROP INDEX IF EXISTS height_tipset_key;"}, + {"drop index event_entry_key_index", "DROP INDEX IF EXISTS event_entry_key_index;"}, + {"create index event_tipset_key_cid", createIndexEventTipsetKeyCid}, + {"create index event_height", createIndexEventHeight}, + {"create index event_entry_event_id", createIndexEventEntryEventId}, + } { + if _, err := tx.ExecContext(ctx, create.query); err != nil { + return fmt.Errorf("%s: %w", create.desc, err) + } + } + + return nil +} + +// migrationVersion5 migrates the schema from version 4 to version 5 by updating the event_index +// to be 0-indexed within a tipset. +func migrationVersion5(ctx context.Context, tx *sql.Tx) error { + stmtEventIndexUpdate, err := tx.PrepareContext(ctx, "UPDATE event SET event_index = (SELECT COUNT(*) FROM event e2 WHERE e2.tipset_key_cid = event.tipset_key_cid AND e2.id <= event.id) - 1") + if err != nil { + return fmt.Errorf("prepare stmtEventIndexUpdate: %w", err) + } + + _, err = stmtEventIndexUpdate.ExecContext(ctx) + if err != nil { + return fmt.Errorf("update event index: %w", err) + } + + return nil +} + +// migrationVersion6 migrates the schema from version 5 to version 6 by creating a new table +// events_seen that tracks the tipsets that have been seen by the event filter and populating it +// with the tipsets that have events in the event table. +func migrationVersion6(ctx context.Context, tx *sql.Tx) error { + stmtCreateTableEventsSeen, err := tx.PrepareContext(ctx, createTableEventsSeen) + if err != nil { + return fmt.Errorf("prepare stmtCreateTableEventsSeen: %w", err) + } + _, err = stmtCreateTableEventsSeen.ExecContext(ctx) + if err != nil { + return fmt.Errorf("create table events_seen: %w", err) + } + + _, err = tx.ExecContext(ctx, createIndexEventsSeenHeight) + if err != nil { + return fmt.Errorf("create index events_seen_height: %w", err) + } + _, err = tx.ExecContext(ctx, createIndexEventsSeenTipsetKeyCid) + if err != nil { + return fmt.Errorf("create index events_seen_tipset_key_cid: %w", err) + } + + // INSERT an entry in the events_seen table for all epochs we do have events for in our DB + _, err = tx.ExecContext(ctx, ` + INSERT OR IGNORE INTO events_seen (height, tipset_key_cid, reverted) + SELECT DISTINCT height, tipset_key_cid, reverted FROM event +`) + if err != nil { + return fmt.Errorf("insert events into events_seen: %w", err) + } + + return nil +} + +// migrationVersion7 migrates the schema from version 6 to version 7 by dropping the following +// indices: +// 1. the index on the event.emitter_addr column +// 2. the index on the event.reverted column +// 3. the composite index on the event_entry.indexed and event_entry.key columns +// 4. the composite index on the event_entry.codec and event_entry.value columns +// +// These indices were found to be a performance hindrance as they prevent SQLite from using the +// intended initial indexes on height or tipset_key_cid in many query variations. Without additional +// indices to fall-back on, SQLite is forced to narrow down each query via height or tipset_key_cid +// which is the desired behavior. +func migrationVersion7(ctx context.Context, tx *sql.Tx) error { + for _, drop := range []struct { + desc string + query string + }{ + {"drop index event_emitter_addr", "DROP INDEX IF EXISTS event_emitter_addr;"}, + {"drop index event_reverted", "DROP INDEX IF EXISTS event_reverted;"}, + {"drop index event_entry_indexed_key", "DROP INDEX IF EXISTS event_entry_indexed_key;"}, + {"drop index event_entry_codec_value", "DROP INDEX IF EXISTS event_entry_codec_value;"}, + } { + if _, err := tx.ExecContext(ctx, drop.query); err != nil { + return fmt.Errorf("%s: %w", drop.desc, err) + } + } + + return nil +} diff --git a/pkg/events/filter/index_test.go b/pkg/events/filter/index_test.go index e46395478e..b8b8c042ea 100644 --- a/pkg/events/filter/index_test.go +++ b/pkg/events/filter/index_test.go @@ -5,13 +5,15 @@ import ( pseudo "math/rand" "os" "path/filepath" + "regexp" + "strings" "testing" + "github.com/ipfs/go-cid" "github.com/stretchr/testify/require" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/venus/venus-shared/types" ) @@ -47,7 +49,7 @@ func TestEventIndexPrefillFilter(t *testing.T) { } events14000 := buildTipSetEvents(t, rng, 14000, em) - cid14000, err := events14000.msgTS.Key().Cid() + cid14000, err := events14000.msgTs.Key().Cid() require.NoError(t, err, "tipset cid") noCollectedEvents := []*CollectedEvent{} @@ -58,7 +60,7 @@ func TestEventIndexPrefillFilter(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTS.Key(), + TipSetKey: events14000.msgTs.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -76,10 +78,50 @@ func TestEventIndexPrefillFilter(t *testing.T) { ei, err := NewEventIndex(context.Background(), dbPath, nil) require.NoError(t, err, "create event index") + + subCh, unSubscribe := ei.SubscribeUpdates() + defer unSubscribe() + + out := make(chan EventIndexUpdated, 1) + go func() { + tu := <-subCh + out <- tu + }() + if err := ei.CollectEvents(context.Background(), events14000, false, addrMap.ResolveAddress); err != nil { require.NoError(t, err, "collect events") } + mh, err := ei.GetMaxHeightInIndex(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(14000), mh) + + b, err := ei.IsHeightPast(context.Background(), 14000) + require.NoError(t, err) + require.True(t, b) + + b, err = ei.IsHeightPast(context.Background(), 14001) + require.NoError(t, err) + require.False(t, b) + + b, err = ei.IsHeightPast(context.Background(), 13000) + require.NoError(t, err) + require.True(t, b) + + tsKey := events14000.msgTs.Key() + tsKeyCid, err := tsKey.Cid() + require.NoError(t, err, "tipset key cid") + + seen, err := ei.IsTipsetProcessed(context.Background(), tsKeyCid.Bytes()) + require.NoError(t, err) + require.True(t, seen, "tipset key should be seen") + + seen, err = ei.IsTipsetProcessed(context.Background(), []byte{1}) + require.NoError(t, err) + require.False(t, seen, "tipset key should not be seen") + + _ = <-out + testCases := []struct { name string filter *eventFilter @@ -332,9 +374,9 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { events14000 := buildTipSetEvents(t, rng, 14000, em) revertedEvents14000 := buildTipSetEvents(t, rng, 14000, revertedEm) - cid14000, err := events14000.msgTS.Key().Cid() + cid14000, err := events14000.msgTs.Key().Cid() require.NoError(t, err, "tipset cid") - reveredCID14000, err := revertedEvents14000.msgTS.Key().Cid() + reveredCID14000, err := revertedEvents14000.msgTs.Key().Cid() require.NoError(t, err, "tipset cid") noCollectedEvents := []*CollectedEvent{} @@ -345,7 +387,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTS.Key(), + TipSetKey: events14000.msgTs.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -357,7 +399,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTS.Key(), + TipSetKey: events14000.msgTs.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -367,7 +409,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: true, Height: 14000, - TipSetKey: revertedEvents14000.msgTS.Key(), + TipSetKey: revertedEvents14000.msgTs.Key(), MsgIdx: 0, MsgCid: revertedEm.msg.Cid(), }, @@ -379,7 +421,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: true, Height: 14000, - TipSetKey: revertedEvents14000.msgTS.Key(), + TipSetKey: revertedEvents14000.msgTs.Key(), MsgIdx: 0, MsgCid: revertedEm.msg.Cid(), }, @@ -397,6 +439,22 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { ei, err := NewEventIndex(context.Background(), dbPath, nil) require.NoError(t, err, "create event index") + + tCh := make(chan EventIndexUpdated, 3) + subCh, unSubscribe := ei.SubscribeUpdates() + defer unSubscribe() + go func() { + cnt := 0 + for tu := range subCh { + tCh <- tu + cnt++ + if cnt == 3 { + close(tCh) + return + } + } + }() + if err := ei.CollectEvents(context.Background(), revertedEvents14000, false, addrMap.ResolveAddress); err != nil { require.NoError(t, err, "collect reverted events") } @@ -407,6 +465,10 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { require.NoError(t, err, "collect events") } + _ = <-tCh + _ = <-tCh + _ = <-tCh + inclusiveTestCases := []struct { name string filter *eventFilter @@ -431,15 +493,15 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { te: events14000, want: noCollectedEvents, }, - // { - // name: "match tipset min height", - // filter: &eventFilter{ - // minHeight: 14000, - // maxHeight: -1, - // }, - // te: events14000, - // want: twoCollectedEvent, - // }, + { + name: "match tipset min height", + filter: &eventFilter{ + minHeight: 14000, + maxHeight: -1, + }, + te: events14000, + want: twoCollectedEvent, + }, { name: "match tipset cid", filter: &eventFilter{ @@ -671,15 +733,15 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { te: events14000, want: noCollectedEvents, }, - // { - // name: "match tipset min height", - // filter: &eventFilter{ - // minHeight: 14000, - // maxHeight: -1, - // }, - // te: events14000, - // want: oneCollectedEvent, - // }, + { + name: "match tipset min height", + filter: &eventFilter{ + minHeight: 14000, + maxHeight: -1, + }, + te: events14000, + want: oneCollectedEvent, + }, { name: "match tipset cid", filter: &eventFilter{ @@ -690,16 +752,16 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { te: events14000, want: oneCollectedEvent, }, - // { - // name: "match tipset cid but reverted", - // filter: &eventFilter{ - // minHeight: -1, - // maxHeight: -1, - // tipsetCid: reveredCID14000, - // }, - // te: revertedEvents14000, - // want: noCollectedEvents, - // }, + { + name: "match tipset cid but reverted", + filter: &eventFilter{ + minHeight: -1, + maxHeight: -1, + tipsetCid: reveredCID14000, + }, + te: revertedEvents14000, + want: noCollectedEvents, + }, { name: "nomatch address", filter: &eventFilter{ @@ -710,16 +772,16 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { te: events14000, want: noCollectedEvents, }, - // { - // name: "nomatch address 2 but reverted", - // filter: &eventFilter{ - // minHeight: -1, - // maxHeight: -1, - // addresses: []address.Address{a2}, - // }, - // te: revertedEvents14000, - // want: noCollectedEvents, - // }, + { + name: "nomatch address 2 but reverted", + filter: &eventFilter{ + minHeight: -1, + maxHeight: -1, + addresses: []address.Address{a2}, + }, + te: revertedEvents14000, + want: noCollectedEvents, + }, { name: "match address", filter: &eventFilter{ @@ -730,36 +792,36 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { te: events14000, want: oneCollectedEvent, }, - // { - // name: "match one entry", - // filter: &eventFilter{ - // minHeight: -1, - // maxHeight: -1, - // keysWithCodec: keysToKeysWithCodec(map[string][][]byte{ - // "type": { - // []byte("approval"), - // }, - // }), - // }, - // te: events14000, - // want: oneCollectedEvent, - // }, - // { - // name: "match one entry with alternate values", - // filter: &eventFilter{ - // minHeight: -1, - // maxHeight: -1, - // keysWithCodec: keysToKeysWithCodec(map[string][][]byte{ - // "type": { - // []byte("cancel"), - // []byte("propose"), - // []byte("approval"), - // }, - // }), - // }, - // te: events14000, - // want: oneCollectedEvent, - // }, + { + name: "match one entry", + filter: &eventFilter{ + minHeight: -1, + maxHeight: -1, + keysWithCodec: keysToKeysWithCodec(map[string][][]byte{ + "type": { + []byte("approval"), + }, + }), + }, + te: events14000, + want: oneCollectedEvent, + }, + { + name: "match one entry with alternate values", + filter: &eventFilter{ + minHeight: -1, + maxHeight: -1, + keysWithCodec: keysToKeysWithCodec(map[string][][]byte{ + "type": { + []byte("cancel"), + []byte("propose"), + []byte("approval"), + }, + }), + }, + te: events14000, + want: oneCollectedEvent, + }, { name: "nomatch one entry by missing value", filter: &eventFilter{ @@ -823,23 +885,23 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { te: events14000, want: noCollectedEvents, }, - // { - // name: "nomatch one entry with matching reverted value", - // filter: &eventFilter{ - // minHeight: -1, - // maxHeight: -1, - // keysWithCodec: keysToKeysWithCodec(map[string][][]byte{ - // "type": { - // []byte("approval"), - // }, - // "signer": { - // []byte("addr2"), - // }, - // }), - // }, - // te: events14000, - // want: noCollectedEvents, - // }, + { + name: "nomatch one entry with matching reverted value", + filter: &eventFilter{ + minHeight: -1, + maxHeight: -1, + keysWithCodec: keysToKeysWithCodec(map[string][][]byte{ + "type": { + []byte("approval"), + }, + "signer": { + []byte("addr2"), + }, + }), + }, + te: events14000, + want: noCollectedEvents, + }, { name: "nomatch one entry with one mismatching value", filter: &eventFilter{ @@ -897,3 +959,87 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { }) } } + +// TestQueryPlan is to ensure that future modifications to the db schema, or future upgrades to +// sqlite, do not change the query plan of the prepared statements used by the event index such that +// queries hit undesirable indexes which are likely to slow down the query. +// Changes that break this test need to be sure that the query plan is still efficient for the +// expected query patterns. +func TestQueryPlan(t *testing.T) { + ei, err := NewEventIndex(context.Background(), filepath.Join(t.TempDir(), "actorevents.db"), nil) + require.NoError(t, err, "create event index") + + verifyQueryPlan := func(stmt string) { + rows, err := ei.db.Query("EXPLAIN QUERY PLAN " + strings.Replace(stmt, "?", "1", -1)) + require.NoError(t, err, "explain query plan for query: "+stmt) + defer func() { + require.NoError(t, rows.Close()) + }() + // First response to EXPLAIN QUERY PLAN should show us the use of an index that we want to + // encounter first to narrow down the search space - either a height or tipset_key_cid index + // - sqlite_autoindex_events_seen_1 is for the UNIQUE constraint on events_seen + // - events_seen_height and events_seen_tipset_key_cid are explicit indexes on events_seen + // - event_height and event_tipset_key_cid are explicit indexes on event + rows.Next() + var id, parent, notused, detail string + require.NoError(t, rows.Scan(&id, &parent, ¬used, &detail), "scan explain query plan for query: "+stmt) + detail = strings.TrimSpace(detail) + var expectedIndexes = []string{ + "sqlite_autoindex_events_seen_1", + "events_seen_height", + "events_seen_tipset_key_cid", + "event_height", + "event_tipset_key_cid", + } + indexUsed := false + for _, index := range expectedIndexes { + if strings.Contains(detail, " INDEX "+index) { + indexUsed = true + break + } + } + require.True(t, indexUsed, "index used for query: "+stmt+" detail: "+detail) + + stmt = regexp.MustCompile(`(?m)^\s+`).ReplaceAllString(stmt, " ") // remove all leading whitespace from the statement + stmt = strings.Replace(stmt, "\n", "", -1) // remove all newlines from the statement + t.Logf("[%s] has plan start: %s", stmt, detail) + } + + // Test the hard-coded select and update queries + stmtMap := preparedStatementMapping(&preparedStatements{}) + for _, stmt := range stmtMap { + if strings.HasPrefix(strings.TrimSpace(strings.ToLower(stmt)), "insert") { + continue + } + verifyQueryPlan(stmt) + } + + // Test the dynamic prefillFilter queries + prefillCases := []*eventFilter{ + {}, + {minHeight: 14000, maxHeight: 14000}, + {minHeight: 14000, maxHeight: 15000}, + {tipsetCid: cid.MustParse("bafkqaaa")}, + {minHeight: 14000, maxHeight: 14000, addresses: []address.Address{address.TestAddress}}, + {minHeight: 14000, maxHeight: 15000, addresses: []address.Address{address.TestAddress}}, + {tipsetCid: cid.MustParse("bafkqaaa"), addresses: []address.Address{address.TestAddress}}, + {minHeight: 14000, maxHeight: 14000, addresses: []address.Address{address.TestAddress, address.TestAddress}}, + {minHeight: 14000, maxHeight: 15000, addresses: []address.Address{address.TestAddress, address.TestAddress}}, + {tipsetCid: cid.MustParse("bafkqaaa"), addresses: []address.Address{address.TestAddress, address.TestAddress}}, + {minHeight: 14000, maxHeight: 14000, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}})}, + {minHeight: 14000, maxHeight: 15000, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}})}, + {tipsetCid: cid.MustParse("bafkqaaa"), keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}})}, + {minHeight: 14000, maxHeight: 14000, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}, "signer": {[]byte("addr1")}})}, + {minHeight: 14000, maxHeight: 15000, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}, "signer": {[]byte("addr1")}})}, + {tipsetCid: cid.MustParse("bafkqaaa"), keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}, "signer": {[]byte("addr1")}})}, + {minHeight: 14000, maxHeight: 14000, addresses: []address.Address{address.TestAddress, address.TestAddress}, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}, "signer": {[]byte("addr1")}})}, + {minHeight: 14000, maxHeight: 15000, addresses: []address.Address{address.TestAddress, address.TestAddress}, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}, "signer": {[]byte("addr1")}})}, + {tipsetCid: cid.MustParse("bafkqaaa"), addresses: []address.Address{address.TestAddress, address.TestAddress}, keysWithCodec: keysToKeysWithCodec(map[string][][]byte{"type": {[]byte("approval")}, "signer": {[]byte("addr1")}})}, + } + for _, filter := range prefillCases { + _, query := makePrefillFilterQuery(filter, true) + verifyQueryPlan(query) + _, query = makePrefillFilterQuery(filter, false) + verifyQueryPlan(query) + } +} diff --git a/pkg/events/filter/sqlite/sqlite.go b/pkg/events/filter/sqlite/sqlite.go new file mode 100644 index 0000000000..cb489284c9 --- /dev/null +++ b/pkg/events/filter/sqlite/sqlite.go @@ -0,0 +1,169 @@ +package sqlite + +import ( + "context" + "database/sql" + "errors" + "io/fs" + "os" + "path/filepath" + "strconv" + "time" + + logging "github.com/ipfs/go-log/v2" + "golang.org/x/xerrors" +) + +var log = logging.Logger("sqlite") + +type MigrationFunc func(ctx context.Context, tx *sql.Tx) error + +var pragmas = []string{ + "PRAGMA synchronous = normal", + "PRAGMA temp_store = memory", + "PRAGMA mmap_size = 30000000000", + "PRAGMA page_size = 32768", + "PRAGMA auto_vacuum = NONE", + "PRAGMA automatic_index = OFF", + "PRAGMA journal_mode = WAL", + "PRAGMA wal_autocheckpoint = 256", // checkpoint @ 256 pages + "PRAGMA journal_size_limit = 0", // always reset journal and wal files +} + +const metaTableDdl = `CREATE TABLE IF NOT EXISTS _meta ( + version UINT64 NOT NULL UNIQUE +)` + +// metaDdl returns the DDL statements required to create the _meta table and add the required +// up to the given version. +func metaDdl(version uint64) []string { + var ddls []string + for i := 1; i <= int(version); i++ { + ddls = append(ddls, `INSERT OR IGNORE INTO _meta (version) VALUES (`+strconv.Itoa(i)+`)`) + } + return append([]string{metaTableDdl}, ddls...) +} + +// Open opens a database at the given path. If the database does not exist, it will be created. +func Open(path string) (*sql.DB, bool, error) { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return nil, false, xerrors.Errorf("error creating database base directory [@ %s]: %w", path, err) + } + + _, err := os.Stat(path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, false, xerrors.Errorf("error checking file status for database [@ %s]: %w", path, err) + } + exists := err == nil + + db, err := sql.Open("sqlite3", path+"?mode=rwc") + if err != nil { + return nil, false, xerrors.Errorf("error opening database [@ %s]: %w", path, err) + } + + for _, pragma := range pragmas { + if _, err := db.Exec(pragma); err != nil { + _ = db.Close() + return nil, false, xerrors.Errorf("error setting database pragma %q: %w", pragma, err) + } + } + + return db, exists, nil +} + +// InitDb initializes the database by checking whether it needs to be created or upgraded. +// The ddls are the DDL statements to create the tables in the database and their initial required +// content. The schemaVersion will be set inside the databse if it is newly created. Otherwise, the +// version is read from the databse and returned. This value should be checked against the expected +// version to determine if the database needs to be upgraded. +// It is up to the caller to close the database if an error is returned by this function. +func InitDb( + ctx context.Context, + name string, + db *sql.DB, + ddls []string, + versionMigrations []MigrationFunc, +) error { + + schemaVersion := len(versionMigrations) + 1 + + q, err := db.QueryContext(ctx, "SELECT name FROM sqlite_master WHERE type='table' AND name='_meta';") + if q != nil { + defer func() { _ = q.Close() }() + } + + if errors.Is(err, sql.ErrNoRows) || !q.Next() { + // empty database, create the schema including the _meta table and its versions + ddls := append(metaDdl(uint64(schemaVersion)), ddls...) + for _, ddl := range ddls { + if _, err := db.Exec(ddl); err != nil { + return xerrors.Errorf("failed to %s database execute ddl %q: %w", name, ddl, err) + } + } + return nil + } + + if err != nil { + return xerrors.Errorf("error looking for %s database _meta table: %w", name, err) + } + + if err := q.Close(); err != nil { + return xerrors.Errorf("error closing %s database _meta table query: %w", name, err) + } + + // check the schema version to see if we need to upgrade the database schema + var foundVersion int + if err = db.QueryRow("SELECT max(version) FROM _meta").Scan(&foundVersion); err != nil { + return xerrors.Errorf("invalid %s database version: no version found", name) + } + + if foundVersion > schemaVersion { + return xerrors.Errorf("invalid %s database version: version %d is greater than the number of migrations %d", name, foundVersion, len(versionMigrations)) + } + + runVacuum := foundVersion != schemaVersion + + // run a migration for each version that we have not yet applied, where foundVersion is what is + // currently in the database and schemaVersion is the target version. If they are the same, + // nothing is run. + for i := foundVersion + 1; i <= schemaVersion; i++ { + now := time.Now() + + log.Infof("Migrating %s database to version %d...", name, i) + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return xerrors.Errorf("failed to start %s database transaction: %w", name, err) + } + defer func() { _ = tx.Rollback() }() + // versions start at 1, but the migrations are 0-indexed where the first migration would take us to version 2 + if err := versionMigrations[i-2](ctx, tx); err != nil { + return xerrors.Errorf("failed to migrate %s database to version %d: %w", name, i, err) + } + if _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO _meta (version) VALUES (?)`, i); err != nil { + return xerrors.Errorf("failed to update %s database _meta table: %w", name, err) + } + if err := tx.Commit(); err != nil { + return xerrors.Errorf("failed to commit %s database v%d migration transaction: %w", name, i, err) + } + + log.Infof("Successfully migrated %s database from version %d to %d in %s", name, i-1, i, time.Since(now)) + } + + if runVacuum { + // During the large migrations, we have likely increased the WAL size a lot, so lets do some + // simple DB administration to free up space (VACUUM followed by truncating the WAL file) + // as this would be a good time to do it when no other writes are happening. + log.Infof("Performing %s database vacuum and wal checkpointing to free up space after the migration", name) + _, err := db.ExecContext(ctx, "VACUUM") + if err != nil { + log.Warnf("error vacuuming %s database: %s", name, err) + } + _, err = db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)") + if err != nil { + log.Warnf("error checkpointing %s database wal: %s", name, err) + } + } + + return nil +} diff --git a/pkg/events/filter/sqlite/sqlite_test.go b/pkg/events/filter/sqlite/sqlite_test.go new file mode 100644 index 0000000000..923a7580dd --- /dev/null +++ b/pkg/events/filter/sqlite/sqlite_test.go @@ -0,0 +1,244 @@ +package sqlite_test + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" + + "github.com/filecoin-project/venus/pkg/events/filter/sqlite" + tf "github.com/filecoin-project/venus/pkg/testhelpers/testflags" +) + +func TestSqlite(t *testing.T) { + tf.UnitTest(t) + req := require.New(t) + + ddl := []string{ + `CREATE TABLE IF NOT EXISTS blip ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + blip_name TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS bloop ( + blip_id INTEGER NOT NULL, + bloop_name TEXT NOT NULL, + FOREIGN KEY (blip_id) REFERENCES blip(id) + )`, + `CREATE INDEX IF NOT EXISTS blip_name_index ON blip (blip_name)`, + } + + tmpDir := t.TempDir() + dbPath := filepath.Join(tmpDir, "/test.db") + + db, exists, err := sqlite.Open(dbPath) + req.NoError(err) + req.False(exists) + req.NotNil(db) + + err = sqlite.InitDb(context.Background(), "testdb", db, ddl, nil) + req.NoError(err) + + // insert some data + + r, err := db.Exec("INSERT INTO blip (blip_name) VALUES ('blip1')") + req.NoError(err) + id, err := r.LastInsertId() + req.NoError(err) + req.Equal(int64(1), id) + _, err = db.Exec("INSERT INTO bloop (blip_id, bloop_name) VALUES (?, 'bloop1')", id) + req.NoError(err) + r, err = db.Exec("INSERT INTO blip (blip_name) VALUES ('blip2')") + req.NoError(err) + id, err = r.LastInsertId() + req.NoError(err) + req.Equal(int64(2), id) + _, err = db.Exec("INSERT INTO bloop (blip_id, bloop_name) VALUES (?, 'bloop2')", id) + req.NoError(err) + + // check that the db contains what we think it should + + expectedIndexes := []string{"blip_name_index"} + + expectedData := []tabledata{ + { + name: "_meta", + cols: []string{"version"}, + data: [][]interface{}{ + {int64(1)}, + }, + }, + { + name: "blip", + cols: []string{"id", "blip_name"}, + data: [][]interface{}{ + {int64(1), "blip1"}, + {int64(2), "blip2"}, + }, + }, + { + name: "bloop", + cols: []string{"blip_id", "bloop_name"}, + data: [][]interface{}{ + {int64(1), "bloop1"}, + {int64(2), "bloop2"}, + }, + }, + } + + actualIndexes, actualData := dumpTables(t, db) + req.Equal(expectedIndexes, actualIndexes) + req.Equal(expectedData, actualData) + + req.NoError(db.Close()) + + // open again, check contents is the same + + db, exists, err = sqlite.Open(dbPath) + req.NoError(err) + req.True(exists) + req.NotNil(db) + + err = sqlite.InitDb(context.Background(), "testdb", db, ddl, nil) + req.NoError(err) + + // database should contain the same things + + actualIndexes, actualData = dumpTables(t, db) + req.Equal(expectedIndexes, actualIndexes) + req.Equal(expectedData, actualData) + + req.NoError(db.Close()) + + // open again, with a migration + + db, exists, err = sqlite.Open(dbPath) + req.NoError(err) + req.True(exists) + req.NotNil(db) + + migration1 := func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec("ALTER TABLE blip ADD COLUMN blip_extra TEXT NOT NULL DEFAULT '!'") + return err + } + + err = sqlite.InitDb(context.Background(), "testdb", db, ddl, []sqlite.MigrationFunc{migration1}) + req.NoError(err) + + // also add something new + r, err = db.Exec("INSERT INTO blip (blip_name, blip_extra) VALUES ('blip1', '!!!')") + req.NoError(err) + id, err = r.LastInsertId() + req.NoError(err) + _, err = db.Exec("INSERT INTO bloop (blip_id, bloop_name) VALUES (?, 'bloop3')", id) + req.NoError(err) + + // database should contain new stuff + + expectedData[0].data = append(expectedData[0].data, []interface{}{int64(2)}) // _meta schema version 2 + expectedData[1] = tabledata{ + name: "blip", + cols: []string{"id", "blip_name", "blip_extra"}, + data: [][]interface{}{ + {int64(1), "blip1", "!"}, + {int64(2), "blip2", "!"}, + {int64(3), "blip1", "!!!"}, + }, + } + expectedData[2].data = append(expectedData[2].data, []interface{}{int64(3), "bloop3"}) + + actualIndexes, actualData = dumpTables(t, db) + req.Equal(expectedIndexes, actualIndexes) + req.Equal(expectedData, actualData) + + req.NoError(db.Close()) + + // open again, with another migration + + db, exists, err = sqlite.Open(dbPath) + req.NoError(err) + req.True(exists) + req.NotNil(db) + + migration2 := func(ctx context.Context, tx *sql.Tx) error { + // add an index + _, err := tx.Exec("CREATE INDEX IF NOT EXISTS blip_extra_index ON blip (blip_extra)") + return err + } + + err = sqlite.InitDb(context.Background(), "testdb", db, ddl, []sqlite.MigrationFunc{migration1, migration2}) + req.NoError(err) + + // database should contain new stuff + + expectedData[0].data = append(expectedData[0].data, []interface{}{int64(3)}) // _meta schema version 3 + expectedIndexes = append(expectedIndexes, "blip_extra_index") + + actualIndexes, actualData = dumpTables(t, db) + req.Equal(expectedIndexes, actualIndexes) + req.Equal(expectedData, actualData) + + req.NoError(db.Close()) +} + +func dumpTables(t *testing.T, db *sql.DB) ([]string, []tabledata) { + req := require.New(t) + + var indexes []string + rows, err := db.Query("SELECT name FROM sqlite_master WHERE type='index'") + req.NoError(err) + for rows.Next() { + var name string + err = rows.Scan(&name) + req.NoError(err) + if !strings.Contains(name, "sqlite_autoindex") { + indexes = append(indexes, name) + } + } + + var data []tabledata + rows, err = db.Query("SELECT name, sql FROM sqlite_master WHERE type = 'table'") + req.NoError(err) + for rows.Next() { + var name, sql string + err = rows.Scan(&name, &sql) + req.NoError(err) + if strings.HasPrefix(name, "sqlite") { + continue + } + sqla := strings.Split(sql, "\n") + cols := []string{} + for _, s := range sqla { + // alter table does funky things to the sql, hence the "," ReplaceAll: + s = strings.Split(strings.TrimSpace(strings.ReplaceAll(s, ",", "")), " ")[0] + switch s { + case "CREATE", "FOREIGN", "", ")": + default: + cols = append(cols, s) + } + } + data = append(data, tabledata{name: name, cols: cols}) + rows2, err := db.Query("SELECT * FROM " + name) + req.NoError(err) + for rows2.Next() { + vals := make([]interface{}, len(cols)) + vals2 := make([]interface{}, len(cols)) + for i := range vals { + vals[i] = &vals2[i] + } + err = rows2.Scan(vals...) + req.NoError(err) + data[len(data)-1].data = append(data[len(data)-1].data, vals2) + } + } + return indexes, data +} + +type tabledata struct { + name string + cols []string + data [][]interface{} +} diff --git a/pkg/events/filter/store.go b/pkg/events/filter/store.go index 72cffeab60..90fc6cdf93 100644 --- a/pkg/events/filter/store.go +++ b/pkg/events/filter/store.go @@ -18,7 +18,7 @@ type Filter interface { ClearSubChannel() } -type FilterStore interface { // nolint +type FilterStore interface { Add(context.Context, Filter) error Get(context.Context, types.FilterID) (Filter, error) Remove(context.Context, types.FilterID) error diff --git a/pkg/vf3/participation_lease.go b/pkg/vf3/participation_lease.go index 9ddd686c53..70fac54f21 100644 --- a/pkg/vf3/participation_lease.go +++ b/pkg/vf3/participation_lease.go @@ -3,11 +3,11 @@ package vf3 import ( "bytes" "errors" + "fmt" "sync" "github.com/libp2p/go-libp2p/core/peer" "go.uber.org/multierr" - "golang.org/x/xerrors" "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-f3/manifest" @@ -136,7 +136,7 @@ func (l *leaser) newParticipationTicket(nn gpbft.NetworkName, participant uint64 FromInstance: from, ValidityTerm: instances, }).MarshalCBOR(&buf); err != nil { - return nil, xerrors.Errorf("issuing participation ticket: %w", err) + return nil, fmt.Errorf("issuing participation ticket: %w", err) } return buf.Bytes(), nil } diff --git a/venus-shared/api/chain/v1/eth.go b/venus-shared/api/chain/v1/eth.go index f0c1a84ec1..c915666494 100644 --- a/venus-shared/api/chain/v1/eth.go +++ b/venus-shared/api/chain/v1/eth.go @@ -31,17 +31,19 @@ type IETH interface { // EthGetBlockTransactionCountByHash returns the number of messages in the TipSet EthGetBlockTransactionCountByHash(ctx context.Context, blkHash types.EthHash) (types.EthUint64, error) //perm:read - EthGetBlockByHash(ctx context.Context, blkHash types.EthHash, fullTxInfo bool) (types.EthBlock, error) //perm:read - EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (*types.EthBlock, error) //perm:read - EthGetTransactionByHash(ctx context.Context, txHash *types.EthHash) (*types.EthTx, error) //perm:read - EthGetTransactionByHashLimited(ctx context.Context, txHash *types.EthHash, limit abi.ChainEpoch) (*types.EthTx, error) //perm:read - EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*types.EthHash, error) //perm:read - EthGetMessageCidByTransactionHash(ctx context.Context, txHash *types.EthHash) (*cid.Cid, error) //perm:read - EthGetTransactionCount(ctx context.Context, sender types.EthAddress, blkParam types.EthBlockNumberOrHash) (types.EthUint64, error) //perm:read - EthGetTransactionReceipt(ctx context.Context, txHash types.EthHash) (*types.EthTxReceipt, error) //perm:read - EthGetTransactionReceiptLimited(ctx context.Context, txHash types.EthHash, limit abi.ChainEpoch) (*types.EthTxReceipt, error) //perm:read - EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash types.EthHash, txIndex types.EthUint64) (types.EthTx, error) //perm:read - EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum types.EthUint64, txIndex types.EthUint64) (types.EthTx, error) //perm:read + EthGetBlockByHash(ctx context.Context, blkHash types.EthHash, fullTxInfo bool) (types.EthBlock, error) //perm:read + EthGetBlockByNumber(ctx context.Context, blkNum string, fullTxInfo bool) (*types.EthBlock, error) //perm:read + EthGetTransactionByHash(ctx context.Context, txHash *types.EthHash) (*types.EthTx, error) //perm:read + EthGetTransactionByHashLimited(ctx context.Context, txHash *types.EthHash, limit abi.ChainEpoch) (*types.EthTx, error) //perm:read + EthGetTransactionHashByCid(ctx context.Context, cid cid.Cid) (*types.EthHash, error) //perm:read + EthGetMessageCidByTransactionHash(ctx context.Context, txHash *types.EthHash) (*cid.Cid, error) //perm:read + EthGetTransactionCount(ctx context.Context, sender types.EthAddress, blkParam types.EthBlockNumberOrHash) (types.EthUint64, error) //perm:read + EthGetBlockReceipts(ctx context.Context, blkParam types.EthBlockNumberOrHash) ([]*types.EthTxReceipt, error) //perm:read + EthGetBlockReceiptsLimited(ctx context.Context, blkParam types.EthBlockNumberOrHash, limit abi.ChainEpoch) ([]*types.EthTxReceipt, error) //perm:read + EthGetTransactionReceipt(ctx context.Context, txHash types.EthHash) (*types.EthTxReceipt, error) //perm:read + EthGetTransactionReceiptLimited(ctx context.Context, txHash types.EthHash, limit abi.ChainEpoch) (*types.EthTxReceipt, error) //perm:read + EthGetTransactionByBlockHashAndIndex(ctx context.Context, blkHash types.EthHash, txIndex types.EthUint64) (types.EthTx, error) //perm:read + EthGetTransactionByBlockNumberAndIndex(ctx context.Context, blkNum types.EthUint64, txIndex types.EthUint64) (types.EthTx, error) //perm:read EthGetCode(ctx context.Context, address types.EthAddress, blkParam types.EthBlockNumberOrHash) (types.EthBytes, error) //perm:read EthGetStorageAt(ctx context.Context, address types.EthAddress, position types.EthBytes, blkParam types.EthBlockNumberOrHash) (types.EthBytes, error) //perm:read diff --git a/venus-shared/api/chain/v1/method.md b/venus-shared/api/chain/v1/method.md index fba81e36c1..a53ad4d9e4 100644 --- a/venus-shared/api/chain/v1/method.md +++ b/venus-shared/api/chain/v1/method.md @@ -80,6 +80,8 @@ curl http://:/rpc/v1 -X POST -H "Content-Type: application/json" -H " * [EthGetBalance](#ethgetbalance) * [EthGetBlockByHash](#ethgetblockbyhash) * [EthGetBlockByNumber](#ethgetblockbynumber) + * [EthGetBlockReceipts](#ethgetblockreceipts) + * [EthGetBlockReceiptsLimited](#ethgetblockreceiptslimited) * [EthGetBlockTransactionCountByHash](#ethgetblocktransactioncountbyhash) * [EthGetBlockTransactionCountByNumber](#ethgetblocktransactioncountbynumber) * [EthGetCode](#ethgetcode) @@ -2643,6 +2645,113 @@ Response: } ``` +### EthGetBlockReceipts + + +Perms: read + +Inputs: +```json +[ + { + "blockNumber": "0x5", + "blockHash": "0x37690cfec6c1bf4c3b9288c7a5d783e98731e90b0a4c177c2a374c7a9427355e", + "requireCanonical": true + } +] +``` + +Response: +```json +[ + { + "transactionHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "transactionIndex": "0x5", + "blockHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "blockNumber": "0x5", + "from": "0x0707070707070707070707070707070707070707", + "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031", + "root": "0x0707070707070707070707070707070707070707070707070707070707070707", + "status": "0x5", + "contractAddress": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031", + "cumulativeGasUsed": "0x5", + "gasUsed": "0x5", + "effectiveGasPrice": "0x0", + "logsBloom": "0x07", + "logs": [ + { + "address": "0x0707070707070707070707070707070707070707", + "data": "0x07", + "topics": [ + "0x0707070707070707070707070707070707070707070707070707070707070707" + ], + "removed": true, + "logIndex": "0x5", + "transactionIndex": "0x5", + "transactionHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "blockHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "blockNumber": "0x5" + } + ], + "type": "0x5" + } +] +``` + +### EthGetBlockReceiptsLimited + + +Perms: read + +Inputs: +```json +[ + { + "blockNumber": "0x5", + "blockHash": "0x37690cfec6c1bf4c3b9288c7a5d783e98731e90b0a4c177c2a374c7a9427355e", + "requireCanonical": true + }, + 10101 +] +``` + +Response: +```json +[ + { + "transactionHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "transactionIndex": "0x5", + "blockHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "blockNumber": "0x5", + "from": "0x0707070707070707070707070707070707070707", + "to": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031", + "root": "0x0707070707070707070707070707070707070707070707070707070707070707", + "status": "0x5", + "contractAddress": "0x5cbeecf99d3fdb3f25e309cc264f240bb0664031", + "cumulativeGasUsed": "0x5", + "gasUsed": "0x5", + "effectiveGasPrice": "0x0", + "logsBloom": "0x07", + "logs": [ + { + "address": "0x0707070707070707070707070707070707070707", + "data": "0x07", + "topics": [ + "0x0707070707070707070707070707070707070707070707070707070707070707" + ], + "removed": true, + "logIndex": "0x5", + "transactionIndex": "0x5", + "transactionHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "blockHash": "0x0707070707070707070707070707070707070707070707070707070707070707", + "blockNumber": "0x5" + } + ], + "type": "0x5" + } +] +``` + ### EthGetBlockTransactionCountByHash EthGetBlockTransactionCountByHash returns the number of messages in the TipSet diff --git a/venus-shared/api/chain/v1/mock/mock_fullnode.go b/venus-shared/api/chain/v1/mock/mock_fullnode.go index 45e79c94cc..b19baeb43a 100644 --- a/venus-shared/api/chain/v1/mock/mock_fullnode.go +++ b/venus-shared/api/chain/v1/mock/mock_fullnode.go @@ -624,6 +624,36 @@ func (mr *MockFullNodeMockRecorder) EthGetBlockByNumber(arg0, arg1, arg2 interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EthGetBlockByNumber", reflect.TypeOf((*MockFullNode)(nil).EthGetBlockByNumber), arg0, arg1, arg2) } +// EthGetBlockReceipts mocks base method. +func (m *MockFullNode) EthGetBlockReceipts(arg0 context.Context, arg1 types.EthBlockNumberOrHash) ([]*types.EthTxReceipt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EthGetBlockReceipts", arg0, arg1) + ret0, _ := ret[0].([]*types.EthTxReceipt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EthGetBlockReceipts indicates an expected call of EthGetBlockReceipts. +func (mr *MockFullNodeMockRecorder) EthGetBlockReceipts(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EthGetBlockReceipts", reflect.TypeOf((*MockFullNode)(nil).EthGetBlockReceipts), arg0, arg1) +} + +// EthGetBlockReceiptsLimited mocks base method. +func (m *MockFullNode) EthGetBlockReceiptsLimited(arg0 context.Context, arg1 types.EthBlockNumberOrHash, arg2 abi.ChainEpoch) ([]*types.EthTxReceipt, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EthGetBlockReceiptsLimited", arg0, arg1, arg2) + ret0, _ := ret[0].([]*types.EthTxReceipt) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// EthGetBlockReceiptsLimited indicates an expected call of EthGetBlockReceiptsLimited. +func (mr *MockFullNodeMockRecorder) EthGetBlockReceiptsLimited(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EthGetBlockReceiptsLimited", reflect.TypeOf((*MockFullNode)(nil).EthGetBlockReceiptsLimited), arg0, arg1, arg2) +} + // EthGetBlockTransactionCountByHash mocks base method. func (m *MockFullNode) EthGetBlockTransactionCountByHash(arg0 context.Context, arg1 types.EthHash) (types.EthUint64, error) { m.ctrl.T.Helper() diff --git a/venus-shared/api/chain/v1/proxy_gen.go b/venus-shared/api/chain/v1/proxy_gen.go index 7f90228407..0cb3c649ad 100644 --- a/venus-shared/api/chain/v1/proxy_gen.go +++ b/venus-shared/api/chain/v1/proxy_gen.go @@ -885,6 +885,8 @@ type IETHStruct struct { EthGetBalance func(ctx context.Context, address types.EthAddress, blkParam types.EthBlockNumberOrHash) (types.EthBigInt, error) `perm:"read"` EthGetBlockByHash func(ctx context.Context, blkHash types.EthHash, fullTxInfo bool) (types.EthBlock, error) `perm:"read"` EthGetBlockByNumber func(ctx context.Context, blkNum string, fullTxInfo bool) (*types.EthBlock, error) `perm:"read"` + EthGetBlockReceipts func(ctx context.Context, blkParam types.EthBlockNumberOrHash) ([]*types.EthTxReceipt, error) `perm:"read"` + EthGetBlockReceiptsLimited func(ctx context.Context, blkParam types.EthBlockNumberOrHash, limit abi.ChainEpoch) ([]*types.EthTxReceipt, error) `perm:"read"` EthGetBlockTransactionCountByHash func(ctx context.Context, blkHash types.EthHash) (types.EthUint64, error) `perm:"read"` EthGetBlockTransactionCountByNumber func(ctx context.Context, blkNum types.EthUint64) (types.EthUint64, error) `perm:"read"` EthGetCode func(ctx context.Context, address types.EthAddress, blkParam types.EthBlockNumberOrHash) (types.EthBytes, error) `perm:"read"` @@ -945,6 +947,12 @@ func (s *IETHStruct) EthGetBlockByHash(p0 context.Context, p1 types.EthHash, p2 func (s *IETHStruct) EthGetBlockByNumber(p0 context.Context, p1 string, p2 bool) (*types.EthBlock, error) { return s.Internal.EthGetBlockByNumber(p0, p1, p2) } +func (s *IETHStruct) EthGetBlockReceipts(p0 context.Context, p1 types.EthBlockNumberOrHash) ([]*types.EthTxReceipt, error) { + return s.Internal.EthGetBlockReceipts(p0, p1) +} +func (s *IETHStruct) EthGetBlockReceiptsLimited(p0 context.Context, p1 types.EthBlockNumberOrHash, p2 abi.ChainEpoch) ([]*types.EthTxReceipt, error) { + return s.Internal.EthGetBlockReceiptsLimited(p0, p1, p2) +} func (s *IETHStruct) EthGetBlockTransactionCountByHash(p0 context.Context, p1 types.EthHash) (types.EthUint64, error) { return s.Internal.EthGetBlockTransactionCountByHash(p0, p1) } From 52fdf7c9a4554acadca3409e1d886879fda9025d Mon Sep 17 00:00:00 2001 From: simlecode <69969590+simlecode@users.noreply.github.com> Date: Wed, 16 Oct 2024 18:18:15 +0800 Subject: [PATCH 2/3] chore: fix lint --- app/submodule/eth/eth_api.go | 4 +-- app/submodule/eth/eth_event_api.go | 2 +- app/submodule/eth/eth_utils.go | 4 +-- pkg/events/filter/event.go | 26 ++++++++-------- pkg/events/filter/event_test.go | 14 ++++----- pkg/events/filter/index.go | 34 ++++++++++----------- pkg/events/filter/index_migrations.go | 32 +++++++++---------- pkg/events/filter/index_test.go | 26 ++++++++-------- venus-shared/compatible-checks/api-diff.txt | 2 -- 9 files changed, 71 insertions(+), 73 deletions(-) diff --git a/app/submodule/eth/eth_api.go b/app/submodule/eth/eth_api.go index 2bc871f763..85bbfab788 100644 --- a/app/submodule/eth/eth_api.go +++ b/app/submodule/eth/eth_api.go @@ -486,12 +486,12 @@ func (a *ethAPI) EthGetTransactionReceiptLimited(ctx context.Context, txHash typ } // The tx is located in the parent tipset - parentTs, err := a.em.chainModule.ChainReader.GetTipSet(ctx, ts.Parents()) + parentTS, err := a.em.chainModule.ChainReader.GetTipSet(ctx, ts.Parents()) if err != nil { return nil, fmt.Errorf("failed to lookup tipset %s when constructing the eth txn receipt: %w", ts.Parents(), err) } - baseFee := parentTs.Blocks()[0].ParentBaseFee + baseFee := parentTS.Blocks()[0].ParentBaseFee receipt, err := newEthTxReceipt(ctx, tx, baseFee, msgLookup.Receipt, a.EthEventHandler) if err != nil { diff --git a/app/submodule/eth/eth_event_api.go b/app/submodule/eth/eth_event_api.go index d6cc46f4df..d5d6176a39 100644 --- a/app/submodule/eth/eth_event_api.go +++ b/app/submodule/eth/eth_event_api.go @@ -817,7 +817,7 @@ func ethLogFromEvent(entries []types.EventEntry) (data []byte, topics []types.Et } // func ethFilterResultFromEvents(evs []*filter.CollectedEvent, ms *chain.MessageStore) (*types.EthFilterResult, error) { -func ethFilterLogsFromEvents(ctx context.Context, evs []*filter.CollectedEvent, ms *chain.MessageStore) ([]types.EthLog, error) { +func ethFilterLogsFromEvents(_ context.Context, evs []*filter.CollectedEvent, ms *chain.MessageStore) ([]types.EthLog, error) { var logs []types.EthLog for _, ev := range evs { log := types.EthLog{ diff --git a/app/submodule/eth/eth_utils.go b/app/submodule/eth/eth_utils.go index f08ea2841c..a95219d16a 100644 --- a/app/submodule/eth/eth_utils.go +++ b/app/submodule/eth/eth_utils.go @@ -659,7 +659,7 @@ func newEthTxFromMessageLookup(ctx context.Context, msgLookup *types.MsgLookup, func newEthTx(ctx context.Context, state tree.Tree, blockHeight abi.ChainEpoch, - msgTsCid cid.Cid, + msgTSCid cid.Cid, msgCid cid.Cid, txIdx int, ms *chain.MessageStore, @@ -679,7 +679,7 @@ func newEthTx(ctx context.Context, ti = types.EthUint64(txIdx) ) - blkHash, err := types.EthHashFromCid(msgTsCid) + blkHash, err := types.EthHashFromCid(msgTSCid) if err != nil { return types.EthTx{}, err } diff --git a/pkg/events/filter/event.go b/pkg/events/filter/event.go index af13f832cd..f80bd4fec3 100644 --- a/pkg/events/filter/event.go +++ b/pkg/events/filter/event.go @@ -104,7 +104,7 @@ func (f *eventFilter) CollectEvents(ctx context.Context, te *TipSetEvents, rever addr, found := addressLookups[ev.Emitter] if !found { var ok bool - addr, ok = resolver(ctx, ev.Emitter, te.rctTs) + addr, ok = resolver(ctx, ev.Emitter, te.rctTS) if !ok { // not an address we will be able to match against continue @@ -125,8 +125,8 @@ func (f *eventFilter) CollectEvents(ctx context.Context, te *TipSetEvents, rever EmitterAddr: addr, EventIdx: eventCount, Reverted: revert, - Height: te.msgTs.Height(), - TipSetKey: te.msgTs.Key(), + Height: te.msgTS.Height(), + TipSetKey: te.msgTS.Key(), MsgCid: em.Message().Cid(), MsgIdx: msgIdx, } @@ -254,10 +254,10 @@ func (f *eventFilter) matchKeys(ees []types.EventEntry) bool { } type TipSetEvents struct { - rctTs *types.TipSet // rctTs is the tipset containing the receipts of executed messages - msgTs *types.TipSet // msgTs is the tipset containing the messages that have been executed + rctTS *types.TipSet // rctTS is the tipset containing the receipts of executed messages + msgTS *types.TipSet // msgTS is the tipset containing the messages that have been executed - load func(ctx context.Context, msgTs, rctTs *types.TipSet) ([]executedMessage, error) + load func(ctx context.Context, msgTS, rctTS *types.TipSet) ([]executedMessage, error) once sync.Once // for lazy population of ems ems []executedMessage @@ -265,17 +265,17 @@ type TipSetEvents struct { } func (te *TipSetEvents) Height() abi.ChainEpoch { - return te.msgTs.Height() + return te.msgTS.Height() } func (te *TipSetEvents) Cid() (cid.Cid, error) { - return te.msgTs.Key().Cid() + return te.msgTS.Key().Cid() } func (te *TipSetEvents) messages(ctx context.Context) ([]executedMessage, error) { te.once.Do(func() { // populate executed message list - ems, err := te.load(ctx, te.msgTs, te.rctTs) + ems, err := te.load(ctx, te.msgTS, te.rctTS) if err != nil { te.err = err return @@ -326,8 +326,8 @@ func (m *EventFilterManager) Apply(ctx context.Context, from, to *types.TipSet) } tse := &TipSetEvents{ - msgTs: from, - rctTs: to, + msgTS: from, + rctTS: to, load: m.loadExecutedMessages, } @@ -357,8 +357,8 @@ func (m *EventFilterManager) Revert(ctx context.Context, from, to *types.TipSet) } tse := &TipSetEvents{ - msgTs: to, - rctTs: from, + msgTS: to, + rctTS: from, load: m.loadExecutedMessages, } diff --git a/pkg/events/filter/event_test.go b/pkg/events/filter/event_test.go index 36c6e20997..d23dd4c414 100644 --- a/pkg/events/filter/event_test.go +++ b/pkg/events/filter/event_test.go @@ -66,7 +66,7 @@ func TestEventFilterCollectEvents(t *testing.T) { } events14000 := buildTipSetEvents(t, rng, 14000, em) - cid14000, err := events14000.msgTs.Key().Cid() + cid14000, err := events14000.msgTS.Key().Cid() require.NoError(t, err, "tipset cid") noCollectedEvents := []*CollectedEvent{} @@ -77,7 +77,7 @@ func TestEventFilterCollectEvents(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTs.Key(), + TipSetKey: events14000.msgTS.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -421,13 +421,13 @@ func newStore() adt.Store { func buildTipSetEvents(tb testing.TB, rng *pseudo.Rand, h abi.ChainEpoch, em executedMessage) *TipSetEvents { tb.Helper() - msgTs := fakeTipSet(tb, rng, h, []cid.Cid{}) - rctTs := fakeTipSet(tb, rng, h+1, msgTs.Cids()) + msgTS := fakeTipSet(tb, rng, h, []cid.Cid{}) + rctTS := fakeTipSet(tb, rng, h+1, msgTS.Cids()) return &TipSetEvents{ - msgTs: msgTs, - rctTs: rctTs, - load: func(ctx context.Context, msgTs, rctTs *types.TipSet) ([]executedMessage, error) { + msgTS: msgTS, + rctTS: rctTS, + load: func(ctx context.Context, msgTS, rctTS *types.TipSet) ([]executedMessage, error) { return []executedMessage{em}, nil }, } diff --git a/pkg/events/filter/index.go b/pkg/events/filter/index.go index 6020965390..6bd161dbb0 100644 --- a/pkg/events/filter/index.go +++ b/pkg/events/filter/index.go @@ -20,7 +20,7 @@ import ( "github.com/filecoin-project/venus/venus-shared/types" ) -const DefaultDbFilename = "events.db" +const DefaultDBFilename = "events.db" // Any changes to this schema should be matched for the `lotus-shed indexes backfill-events` command @@ -51,7 +51,7 @@ var ddls = []string{ createTableEventsSeen, - createIndexEventEntryEventId, + createIndexEventEntryEventID, createIndexEventsSeenHeight, createIndexEventsSeenTipsetKeyCid, } @@ -101,7 +101,7 @@ const ( createIndexEventTipsetKeyCid = `CREATE INDEX IF NOT EXISTS event_tipset_key_cid ON event (tipset_key_cid);` createIndexEventHeight = `CREATE INDEX IF NOT EXISTS event_height ON event (height);` - createIndexEventEntryEventId = `CREATE INDEX IF NOT EXISTS event_entry_event_id ON event_entry(event_id);` + createIndexEventEntryEventID = `CREATE INDEX IF NOT EXISTS event_entry_event_id ON event_entry(event_id);` createIndexEventsSeenHeight = `CREATE INDEX IF NOT EXISTS events_seen_height ON events_seen (height);` createIndexEventsSeenTipsetKeyCid = `CREATE INDEX IF NOT EXISTS events_seen_tipset_key_cid ON events_seen (tipset_key_cid);` @@ -148,7 +148,7 @@ type EventIndex struct { stmt *preparedStatements mu sync.Mutex - subIdCounter uint64 + subIDCounter uint64 updateSubs map[uint64]*updateSub } @@ -225,8 +225,8 @@ func (ei *EventIndex) SubscribeUpdates() (chan EventIndexUpdated, func()) { } ei.mu.Lock() - subId := ei.subIdCounter - ei.subIdCounter++ + subId := ei.subIDCounter + ei.subIDCounter++ ei.updateSubs[subId] = tSub ei.mu.Unlock() @@ -277,19 +277,19 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever // rollback the transaction (a no-op if the transaction was already committed) defer func() { _ = tx.Rollback() }() - tsKeyCid, err := te.msgTs.Key().Cid() + tsKeyCid, err := te.msgTS.Key().Cid() if err != nil { return fmt.Errorf("tipset key cid: %w", err) } // lets handle the revert case first, since its simpler and we can simply mark all events in this tipset as reverted and return if revert { - _, err = tx.Stmt(ei.stmt.revertEventsInTipset).Exec(te.msgTs.Height(), te.msgTs.Key().Bytes()) + _, err = tx.Stmt(ei.stmt.revertEventsInTipset).Exec(te.msgTS.Height(), te.msgTS.Key().Bytes()) if err != nil { return fmt.Errorf("revert event: %w", err) } - _, err = tx.Stmt(ei.stmt.revertEventSeen).Exec(te.msgTs.Height(), tsKeyCid.Bytes()) + _, err = tx.Stmt(ei.stmt.revertEventSeen).Exec(te.msgTS.Height(), tsKeyCid.Bytes()) if err != nil { return fmt.Errorf("revert event seen: %w", err) } @@ -336,7 +336,7 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever addr, found := addressLookups[ev.Emitter] if !found { var ok bool - addr, ok = resolver(ctx, ev.Emitter, te.rctTs) + addr, ok = resolver(ctx, ev.Emitter, te.rctTS) if !ok { // not an address we will be able to match against continue @@ -347,8 +347,8 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever // check if this event already exists in the database var entryID sql.NullInt64 err = tx.Stmt(ei.stmt.eventExists).QueryRow( - te.msgTs.Height(), // height - te.msgTs.Key().Bytes(), // tipset_key + te.msgTS.Height(), // height + te.msgTS.Key().Bytes(), // tipset_key tsKeyCid.Bytes(), // tipset_key_cid addr.Bytes(), // emitter_addr eventCount, // event_index @@ -362,8 +362,8 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever if !entryID.Valid { // event does not exist, lets insert it res, err := tx.Stmt(ei.stmt.insertEvent).Exec( - te.msgTs.Height(), // height - te.msgTs.Key().Bytes(), // tipset_key + te.msgTS.Height(), // height + te.msgTS.Key().Bytes(), // tipset_key tsKeyCid.Bytes(), // tipset_key_cid addr.Bytes(), // emitter_addr eventCount, // event_index @@ -397,8 +397,8 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever } else { // event already exists, lets mark it as not reverted res, err := tx.Stmt(ei.stmt.restoreEvent).Exec( - te.msgTs.Height(), // height - te.msgTs.Key().Bytes(), // tipset_key + te.msgTS.Height(), // height + te.msgTS.Key().Bytes(), // tipset_key tsKeyCid.Bytes(), // tipset_key_cid addr.Bytes(), // emitter_addr eventCount, // event_index @@ -426,7 +426,7 @@ func (ei *EventIndex) CollectEvents(ctx context.Context, te *TipSetEvents, rever // this statement will mark the tipset as processed and will insert a new row if it doesn't exist // or update the reverted field to false if it does _, err = tx.Stmt(ei.stmt.upsertEventsSeen).Exec( - te.msgTs.Height(), + te.msgTS.Height(), tsKeyCid.Bytes(), ) if err != nil { diff --git a/pkg/events/filter/index_migrations.go b/pkg/events/filter/index_migrations.go index dc942a639e..19454f4f1a 100644 --- a/pkg/events/filter/index_migrations.go +++ b/pkg/events/filter/index_migrations.go @@ -49,34 +49,34 @@ func migrationVersion2(db *sql.DB, chainStore *chain.Store) sqlite.MigrationFunc } log.Infof("Migrating events from head to %d", minHeight.Int64) - currTs := chainStore.GetHead() + currTS := chainStore.GetHead() - for int64(currTs.Height()) >= minHeight.Int64 { - if currTs.Height()%1000 == 0 { - log.Infof("Migrating height %d (remaining %d)", currTs.Height(), int64(currTs.Height())-minHeight.Int64) + for int64(currTS.Height()) >= minHeight.Int64 { + if currTS.Height()%1000 == 0 { + log.Infof("Migrating height %d (remaining %d)", currTS.Height(), int64(currTS.Height())-minHeight.Int64) } - tsKey := currTs.Parents() - currTs, err = chainStore.GetTipSet(ctx, tsKey) + tsKey := currTS.Parents() + currTS, err = chainStore.GetTipSet(ctx, tsKey) if err != nil { return fmt.Errorf("get tipset from key: %w", err) } - log.Debugf("Migrating height %d", currTs.Height()) + log.Debugf("Migrating height %d", currTS.Height()) - tsKeyCid, err := currTs.Key().Cid() + tsKeyCid, err := currTS.Key().Cid() if err != nil { return fmt.Errorf("tipset key cid: %w", err) } // delete all events that are not in the canonical chain - _, err = stmtDeleteOffChainEvent.Exec(tsKeyCid.Bytes(), currTs.Height()) + _, err = stmtDeleteOffChainEvent.Exec(tsKeyCid.Bytes(), currTS.Height()) if err != nil { return fmt.Errorf("delete off chain event: %w", err) } - // find the first eventId from the last time the tipset was applied - var eventId sql.NullInt64 - err = stmtSelectEvent.QueryRow(tsKeyCid.Bytes()).Scan(&eventId) + // find the first eventID from the last time the tipset was applied + var eventID sql.NullInt64 + err = stmtSelectEvent.QueryRow(tsKeyCid.Bytes()).Scan(&eventID) if err != nil { if errors.Is(err, sql.ErrNoRows) { continue @@ -85,12 +85,12 @@ func migrationVersion2(db *sql.DB, chainStore *chain.Store) sqlite.MigrationFunc } // this tipset might not have any events which is ok - if !eventId.Valid { + if !eventID.Valid { continue } - log.Debugf("Deleting all events with id < %d at height %d", eventId.Int64, currTs.Height()) + log.Debugf("Deleting all events with id < %d at height %d", eventID.Int64, currTS.Height()) - res, err := stmtDeleteEvent.Exec(tsKeyCid.Bytes(), eventId.Int64) + res, err := stmtDeleteEvent.Exec(tsKeyCid.Bytes(), eventID.Int64) if err != nil { return fmt.Errorf("delete event: %w", err) } @@ -168,7 +168,7 @@ func migrationVersion4(ctx context.Context, tx *sql.Tx) error { {"drop index event_entry_key_index", "DROP INDEX IF EXISTS event_entry_key_index;"}, {"create index event_tipset_key_cid", createIndexEventTipsetKeyCid}, {"create index event_height", createIndexEventHeight}, - {"create index event_entry_event_id", createIndexEventEntryEventId}, + {"create index event_entry_event_id", createIndexEventEntryEventID}, } { if _, err := tx.ExecContext(ctx, create.query); err != nil { return fmt.Errorf("%s: %w", create.desc, err) diff --git a/pkg/events/filter/index_test.go b/pkg/events/filter/index_test.go index b8b8c042ea..0d545db257 100644 --- a/pkg/events/filter/index_test.go +++ b/pkg/events/filter/index_test.go @@ -49,7 +49,7 @@ func TestEventIndexPrefillFilter(t *testing.T) { } events14000 := buildTipSetEvents(t, rng, 14000, em) - cid14000, err := events14000.msgTs.Key().Cid() + cid14000, err := events14000.msgTS.Key().Cid() require.NoError(t, err, "tipset cid") noCollectedEvents := []*CollectedEvent{} @@ -60,7 +60,7 @@ func TestEventIndexPrefillFilter(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTs.Key(), + TipSetKey: events14000.msgTS.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -108,7 +108,7 @@ func TestEventIndexPrefillFilter(t *testing.T) { require.NoError(t, err) require.True(t, b) - tsKey := events14000.msgTs.Key() + tsKey := events14000.msgTS.Key() tsKeyCid, err := tsKey.Cid() require.NoError(t, err, "tipset key cid") @@ -120,7 +120,7 @@ func TestEventIndexPrefillFilter(t *testing.T) { require.NoError(t, err) require.False(t, seen, "tipset key should not be seen") - _ = <-out + <-out testCases := []struct { name string @@ -374,9 +374,9 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { events14000 := buildTipSetEvents(t, rng, 14000, em) revertedEvents14000 := buildTipSetEvents(t, rng, 14000, revertedEm) - cid14000, err := events14000.msgTs.Key().Cid() + cid14000, err := events14000.msgTS.Key().Cid() require.NoError(t, err, "tipset cid") - reveredCID14000, err := revertedEvents14000.msgTs.Key().Cid() + reveredCID14000, err := revertedEvents14000.msgTS.Key().Cid() require.NoError(t, err, "tipset cid") noCollectedEvents := []*CollectedEvent{} @@ -387,7 +387,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTs.Key(), + TipSetKey: events14000.msgTS.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -399,7 +399,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: false, Height: 14000, - TipSetKey: events14000.msgTs.Key(), + TipSetKey: events14000.msgTS.Key(), MsgIdx: 0, MsgCid: em.msg.Cid(), }, @@ -409,7 +409,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: true, Height: 14000, - TipSetKey: revertedEvents14000.msgTs.Key(), + TipSetKey: revertedEvents14000.msgTS.Key(), MsgIdx: 0, MsgCid: revertedEm.msg.Cid(), }, @@ -421,7 +421,7 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { EventIdx: 0, Reverted: true, Height: 14000, - TipSetKey: revertedEvents14000.msgTs.Key(), + TipSetKey: revertedEvents14000.msgTS.Key(), MsgIdx: 0, MsgCid: revertedEm.msg.Cid(), }, @@ -465,9 +465,9 @@ func TestEventIndexPrefillFilterExcludeReverted(t *testing.T) { require.NoError(t, err, "collect events") } - _ = <-tCh - _ = <-tCh - _ = <-tCh + <-tCh + <-tCh + <-tCh inclusiveTestCases := []struct { name string diff --git a/venus-shared/compatible-checks/api-diff.txt b/venus-shared/compatible-checks/api-diff.txt index 4d839cac2c..77a2e4c456 100644 --- a/venus-shared/compatible-checks/api-diff.txt +++ b/venus-shared/compatible-checks/api-diff.txt @@ -104,8 +104,6 @@ github.com/filecoin-project/venus/venus-shared/api/chain/v1.FullNode <> github.c + Concurrent - CreateBackup - Discover - - EthGetBlockReceipts - - EthGetBlockReceiptsLimited - EthSendRawTransactionUntrusted - EthTraceFilter > EthTraceReplayBlockTransactions {[func(context.Context, string, []string) ([]*types.EthTraceReplayBlockTransaction, error) <> func(context.Context, string, []string) ([]*ethtypes.EthTraceReplayBlockTransaction, error)] base=func out type: #0 input; nested={[[]*types.EthTraceReplayBlockTransaction <> []*ethtypes.EthTraceReplayBlockTransaction] base=slice element; nested={[*types.EthTraceReplayBlockTransaction <> *ethtypes.EthTraceReplayBlockTransaction] base=pointed type; nested={[types.EthTraceReplayBlockTransaction <> ethtypes.EthTraceReplayBlockTransaction] base=struct field; nested={[types.EthTraceReplayBlockTransaction <> ethtypes.EthTraceReplayBlockTransaction] base=exported field name: #4 field, VMTrace != VmTrace; nested=nil}}}}} From eac2d3c8335acd2550eb7bc45b987777c8b1e555 Mon Sep 17 00:00:00 2001 From: simlecode <69969590+simlecode@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:06:47 +0800 Subject: [PATCH 3/3] chore: fix lint --- pkg/events/filter/event.go | 6 +++--- pkg/events/filter/index.go | 8 ++++---- venus-devtool/api-gen/example.go | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/events/filter/event.go b/pkg/events/filter/event.go index f80bd4fec3..ce46a95765 100644 --- a/pkg/events/filter/event.go +++ b/pkg/events/filter/event.go @@ -434,15 +434,15 @@ func (m *EventFilterManager) Remove(ctx context.Context, id types.FilterID) erro return nil } -func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTs, rctTs *types.TipSet) ([]executedMessage, error) { - msgs, err := m.MessageStore.MessagesForTipset(msgTs) +func (m *EventFilterManager) loadExecutedMessages(ctx context.Context, msgTS, rctTS *types.TipSet) ([]executedMessage, error) { + msgs, err := m.MessageStore.MessagesForTipset(msgTS) if err != nil { return nil, fmt.Errorf("read messages: %w", err) } st := adt.WrapStore(ctx, cbor.NewCborStore(m.ChainStore.Blockstore())) - arr, err := blockadt.AsArray(st, rctTs.Blocks()[0].ParentMessageReceipts) + arr, err := blockadt.AsArray(st, rctTS.Blocks()[0].ParentMessageReceipts) if err != nil { return nil, fmt.Errorf("load receipts amt: %w", err) } diff --git a/pkg/events/filter/index.go b/pkg/events/filter/index.go index 6bd161dbb0..9fcf913da4 100644 --- a/pkg/events/filter/index.go +++ b/pkg/events/filter/index.go @@ -225,19 +225,19 @@ func (ei *EventIndex) SubscribeUpdates() (chan EventIndexUpdated, func()) { } ei.mu.Lock() - subId := ei.subIDCounter + subID := ei.subIDCounter ei.subIDCounter++ - ei.updateSubs[subId] = tSub + ei.updateSubs[subID] = tSub ei.mu.Unlock() unSubscribeF := func() { ei.mu.Lock() - tSub, ok := ei.updateSubs[subId] + tSub, ok := ei.updateSubs[subID] if !ok { ei.mu.Unlock() return } - delete(ei.updateSubs, subId) + delete(ei.updateSubs, subID) ei.mu.Unlock() // cancel the subscription diff --git a/venus-devtool/api-gen/example.go b/venus-devtool/api-gen/example.go index 8a54d6e601..5109d4a45c 100644 --- a/venus-devtool/api-gen/example.go +++ b/venus-devtool/api-gen/example.go @@ -293,9 +293,9 @@ func init() { addExample(filterid) addExample(&filterid) - subid := types.EthSubscriptionID(ethhash) - addExample(subid) - addExample(&subid) + subID := types.EthSubscriptionID(ethhash) + addExample(subID) + addExample(&subID) pstring := func(s string) *string { return &s } addExample(&types.EthFilterSpec{