Skip to content

Commit

Permalink
Add batch size flag for block fetching
Browse files Browse the repository at this point in the history
Introduce a new "--block-fetch-batch-size" flag to set the number of blocks fetched in parallel. Adjust poller logic to use the specified batch size, improving fetch operation efficiency.
  • Loading branch information
billettc committed Aug 14, 2024
1 parent f5ec321 commit f8b7420
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion cmd/firevara/fetcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ func NewFetchCmd(logger *zap.Logger, tracer logging.Tracer) *cobra.Command {
cmd.Flags().String("state-dir", "/data", "location to store the cursor.json")
cmd.Flags().Duration("interval-between-fetch", 0, "interval between fetch")
cmd.Flags().Duration("latest-block-retry-interval", time.Second, "interval between fetch when latest block is not available yet")
cmd.Flags().Int64("block-fetch-batch-size", 1, "number of block to fetch in parallel")

return cmd
}
Expand All @@ -41,6 +42,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut
}

fetchInterval := sflags.MustGetDuration(cmd, "interval-between-fetch")
batchSize := sflags.MustGetInt(cmd, "block-fetch-batch-size")

logger.Info(
"launching firehose-gear poller",
Expand Down Expand Up @@ -72,7 +74,7 @@ func fetchRunE(logger *zap.Logger, tracer logging.Tracer) firecore.CommandExecut
)

// never use batch downloading for blocks
err = poller.Run(ctx, startBlock, 1)
err = poller.Run(ctx, startBlock, batchSize)
if err != nil {
return fmt.Errorf("running poller: %w", err)
}
Expand Down

0 comments on commit f8b7420

Please sign in to comment.