From f8b7420c22b3dab966dd5f08d3a01189667973cd Mon Sep 17 00:00:00 2001 From: Charles Billette Date: Wed, 14 Aug 2024 08:42:33 -0400 Subject: [PATCH] Add batch size flag for block fetching Introduce a new "--block-fetch-batch-size" flag to set the number of blocks fetched in parallel. Adjust poller logic to use the specified batch size, improving fetch operation efficiency. --- cmd/firevara/fetcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/firevara/fetcher.go b/cmd/firevara/fetcher.go index 8c28b5c..bb4f8be 100644 --- a/cmd/firevara/fetcher.go +++ b/cmd/firevara/fetcher.go @@ -26,6 +26,7 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command { cmd.Flags().String("state-dir", "/data", "location to store the cursor.json") cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch") cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch when latest block is not available yet") + cmd.Flags().Int64("block-fetch-batch-size", 1, "number of block to fetch in parallel") return cmd } @@ -41,6 +42,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut } fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch") + batchSize := sflags.MustGetInt(cmd, "block-fetch-batch-size") logger.Info( "launching firehose-gear poller", @@ -72,7 +74,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut ) // never use batch downloading for blocks - err = poller.Run(ctx, startBlock, 1) + err = poller.Run(ctx, startBlock, batchSize) if err != nil { return fmt.Errorf("running poller: %w", err) }