From b131910c0fe4e40695d124397670e40524858546 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Thu, 21 Sep 2023 18:09:24 -0700 Subject: [PATCH 1/2] chain: refactor btcd filter rescan into maybeShouldFetchBlock We also abstract how blocks are fetched in the first place, as bitcoind uses a different name for the RPC to fetch filters. --- chain/btcd.go | 89 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 22 deletions(-) diff --git a/chain/btcd.go b/chain/btcd.go index 93735d491d..d519834dbe 100644 --- a/chain/btcd.go +++ b/chain/btcd.go @@ -197,6 +197,65 @@ func (c *RPCClient) BlockStamp() (*waddrmgr.BlockStamp, error) { } } +// fetchBlockFilter fetches the GCS filter for a block from the remote node. +func (c *RPCClient) fetchBlockFilter(blkHash chainhash.Hash, +) (*gcs.Filter, error) { + + rawFilter, err := c.GetCFilter(&blkHash, wire.GCSFilterRegular) + if err != nil { + return nil, err + } + + // Ensure the filter is large enough to be deserialized. + if len(rawFilter.Data) < 4 { + return nil, nil + } + + filter, err := gcs.FromNBytes( + builder.DefaultP, builder.DefaultM, rawFilter.Data, + ) + if err != nil { + return nil, err + } + + // Skip any empty filters. + if filter.N() == 0 { + return nil, nil + } + + return filter, nil +} + +// blockFilterFetcher is a function that fetches a block filter for a given +// block hash. A nil filter is returned if a valid filter doesn't exist for a +// given block. +type blockFilterFetcher func(blkHash chainhash.Hash) (*gcs.Filter, error) + +// maybeShouldFetchBlock returns true if the contents of a block *might* have +// some items that match our watch list. This uses the neutrino filters to do a +// quick loop up to see if the block has relevant items to avoid downloading +// the full block. +func maybeShouldFetchBlock(blockFilterFetcher blockFilterFetcher, + blk wtxmgr.BlockMeta, watchList [][]byte) (bool, error) { + + filter, err := blockFilterFetcher(blk.Hash) + if err != nil { + return false, err + } + + if filter == nil { + return false, nil + } + + key := builder.DeriveKey(&blk.Hash) + matched, err := filter.MatchAny(key, watchList) + if err != nil { + return false, err + } + + return matched, nil +} + // FilterBlocks scans the blocks contained in the FilterBlocksRequest for any // addresses of interest. For each requested block, the corresponding compact // filter will first be checked for matches, skipping those that do not report @@ -221,33 +280,19 @@ func (c *RPCClient) FilterBlocks( // the filter returns a positive match, the full block is then requested // and scanned for addresses using the block filterer. for i, blk := range req.Blocks { - rawFilter, err := c.GetCFilter(&blk.Hash, wire.GCSFilterRegular) - if err != nil { - return nil, err - } - - // Ensure the filter is large enough to be deserialized. - if len(rawFilter.Data) < 4 { - continue - } - - filter, err := gcs.FromNBytes( - builder.DefaultP, builder.DefaultM, rawFilter.Data, + shouldFetchBlock, err := maybeShouldFetchBlock( + c.fetchBlockFilter, blk, watchList, ) if err != nil { return nil, err } - // Skip any empty filters. - if filter.N() == 0 { - continue - } - - key := builder.DeriveKey(&blk.Hash) - matched, err := filter.MatchAny(key, watchList) - if err != nil { - return nil, err - } else if !matched { + // If the filter concluded that there're no matches in this + // block, then we don't need to fetch it, as there're no false + // negatives. + if !shouldFetchBlock { + log.Infof("Skipping block height=%d hash=%v, no "+ + "filter match", blk.Height, blk.Hash) continue } From 2cb1df237b52f68293c0a21a285aa8687e76af9e Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Thu, 21 Sep 2023 18:10:35 -0700 Subject: [PATCH 2/2] chain: use neutrino filters to speed up bitcoind seed recovery In this commit, we use neutrino filters to speed up bitcoind seed recovery. We use the recently created `maybeShouldFetchBlock` function to check the filters to see if we need to fetch a block at all. This saves us from fetching, decoding, then scanning the block contents if we know nothing is present in them. At this point, we can also further consolidate the `FilterBlocks` methods between the two backends, as they're now identical. --- chain/bitcoind_client.go | 119 +++++++++++++++++++++++++++++++++++++-- chain/bitcoind_conn.go | 39 +++++++++++++ 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/chain/bitcoind_client.go b/chain/bitcoind_client.go index 958375d0d7..30e6acbcaf 100644 --- a/chain/bitcoind_client.go +++ b/chain/bitcoind_client.go @@ -3,6 +3,7 @@ package chain import ( "container/list" "encoding/hex" + "encoding/json" "errors" "fmt" "sync" @@ -11,6 +12,8 @@ import ( "github.com/btcsuite/btcd/btcjson" "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/btcutil/gcs" + "github.com/btcsuite/btcd/btcutil/gcs/builder" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" @@ -956,6 +959,74 @@ func (c *BitcoindClient) reorg(currentBlock waddrmgr.BlockStamp, return nil } +// blockFilterReq is a request to fetch a neutrino filter for a block from the +// target bitocind node. +type blockFilterReq struct { + BlockHash string `json:"blockhash"` + FilterType string `json:"filtertype"` +} + +// blockFilterResp is a response containing a neutrino filter for a block from +// bitcond node. +type blockFilterResp struct { + Filter string `json:"filter"` + FilterHeader string `json:"header"` +} + +const ( + // bitcoindFilterRPC is the name of the RPC used to fetch a block + // filter from bitcoind. + bitcoindFilterRPC = "getblockfilter" + + // bitcoindFilterType is the type of filter to fetch from bitcoind. + bitcoindFilterType = "basic" +) + +// fetchBlockFilter fetches the GCS filter for a block from the remote node. +func (c *BitcoindClient) fetchBlockFilter(blkHash chainhash.Hash, +) (*gcs.Filter, error) { + + filterReq := blockFilterReq{ + BlockHash: blkHash.String(), + FilterType: bitcoindFilterType, + } + jsonFilterReq, err := json.Marshal(filterReq) + if err != nil { + return nil, err + } + + resp, err := c.chainConn.client.RawRequest( + bitcoindFilterRPC, []json.RawMessage{jsonFilterReq}, + ) + if err != nil { + return nil, err + } + + var filterResp blockFilterResp + if err := json.Unmarshal(resp, &filterResp); err != nil { + return nil, err + } + + rawFilter, err := hex.DecodeString(filterResp.Filter) + if err != nil { + return nil, err + } + + filter, err := gcs.FromNBytes( + builder.DefaultP, builder.DefaultM, rawFilter, + ) + if err != nil { + return nil, err + } + + // Skip any empty filters. + if filter.N() == 0 { + return nil, nil + } + + return filter, nil +} + // FilterBlocks scans the blocks contained in the FilterBlocksRequest for any // addresses of interest. Each block will be fetched and filtered sequentially, // returning a FilterBlocksReponse for the first block containing a matching @@ -968,12 +1039,50 @@ func (c *BitcoindClient) FilterBlocks( blockFilterer := NewBlockFilterer(c.chainConn.cfg.ChainParams, req) - // Iterate over the requested blocks, fetching each from the rpc client. - // Each block will scanned using the reverse addresses indexes generated - // above, breaking out early if any addresses are found. + // Construct the watchlist using the addresses and outpoints contained + // in the filter blocks request. + watchList, err := buildFilterBlocksWatchList(req) + if err != nil { + return nil, err + } + + // Iterate over the requested blocks, fetching each from the rpc + // client. Each block will scanned using the reverse addresses indexes + // generated above, breaking out early if any addresses are found. for i, block := range req.Blocks { - // TODO(conner): add prefetching, since we already know we'll be - // fetching *every* block + var shouldFetchBlock bool + + switch { + // If we don't have filters, then we always need to fetch the + // block. + case !c.chainConn.hasNeutrinoFilters: + shouldFetchBlock = true + + // If we have the filters, then we'll actually query the + // bitcoind node. + default: + shouldFetchBlock, err = maybeShouldFetchBlock( + c.fetchBlockFilter, block, watchList, + ) + if err != nil { + return nil, err + } + } + + // If the filter concluded that there're no matches in this + // block, then we don't need to fetch it, as there're no false + // negatives. + if !shouldFetchBlock { + log.Infof("Skipping block height=%d hash=%v, no "+ + "filter match", block.Height, block.Hash) + continue + } + + log.Infof("Fetching block height=%d hash=%v", + block.Height, block.Hash) + + // TODO(conner): add prefetching, since we already know we'll + // be fetching *every* block rawBlock, err := c.GetBlock(&block.Hash) if err != nil { return nil, err diff --git a/chain/bitcoind_conn.go b/chain/bitcoind_conn.go index 47f9a566f7..bcbaa05c5c 100644 --- a/chain/bitcoind_conn.go +++ b/chain/bitcoind_conn.go @@ -12,6 +12,8 @@ import ( "github.com/btcsuite/btcd/rpcclient" "github.com/btcsuite/btcd/wire" "github.com/lightningnetwork/lnd/ticker" + + "encoding/json" ) const ( @@ -108,6 +110,10 @@ type BitcoindConn struct { rescanClientsMtx sync.Mutex rescanClients map[uint64]*BitcoindClient + // hasNeutrinoFilters indicates if the bitcoind connection can serve + // neutrino filters over RPC. + hasNeutrinoFilters bool + quit chan struct{} wg sync.WaitGroup } @@ -116,6 +122,32 @@ type BitcoindConn struct { // running over Tor, this must support dialing peers over Tor as well. type Dialer = func(string) (net.Conn, error) +// hasNeutrinoFilters returns whether or not the bitcoind node is able to serve +// neutrino filters based on its version. +func hasNeutrinoFilters(client *rpcclient.Client) (bool, error) { + // Fetch the bitcoind version. + resp, err := client.RawRequest("getnetworkinfo", nil) + if err != nil { + return false, err + } + + info := struct { + Version int64 `json:"version"` + }{} + + if err := json.Unmarshal(resp, &info); err != nil { + return false, err + } + + // Bitcoind returns a single value representing the semantic version: + // 10000 * CLIENT_VERSION_MAJOR + 100 * CLIENT_VERSION_MINOR + 1 * + // CLIENT_VERSION_BUILD + // + // The getblockfilter call was added in version 19.0.0, so we return + // for versions >= 190000. + return info.Version >= 190000, nil +} + // NewBitcoindConn creates a client connection to the node described by the host // string. The ZMQ connections are established immediately to ensure liveness. // If the remote node does not operate on the same bitcoin network as described @@ -174,11 +206,18 @@ func NewBitcoindConn(cfg *BitcoindConfig) (*BitcoindConn, error) { } } + hasNeutrinoFilters, err := hasNeutrinoFilters(client) + if err != nil { + return nil, fmt.Errorf("unable to determine if bitcoind "+ + "has neutrino filters: %w", err) + } + bc := &BitcoindConn{ cfg: *cfg, client: client, prunedBlockDispatcher: prunedBlockDispatcher, rescanClients: make(map[uint64]*BitcoindClient), + hasNeutrinoFilters: hasNeutrinoFilters, quit: make(chan struct{}), }