Store all relays bids

cmd/service/bidcollect.go

@@ -36,6 +36,8 @@ var (
 
 	runWebserverOnly    bool // provides a SSE stream of new bids
 	WebserverListenAddr string
+
+	withDuplicates bool
 )
 
 func init() {
@@ -70,6 +72,7 @@ func init() {
 	bidCollectCmd.Flags().BoolVar(&buildWebsite, "build-website", false, "build file listing website")
 	bidCollectCmd.Flags().BoolVar(&buildWebsiteUpload, "build-website-upload", false, "upload after building")
 	bidCollectCmd.Flags().StringVar(&buildWebsiteOutDir, "build-website-out", "build", "output directory for website")
+	bidCollectCmd.Flags().BoolVar(&withDuplicates, "with-duplicates", false, "store all bids, including duplicates from different relays")
 }
 
 var bidCollectCmd = &cobra.Command{
@@ -129,6 +132,7 @@ var bidCollectCmd = &cobra.Command{
 			OutputTSV:               outputTSV,
 			RedisAddr:               redisAddr,
 			UseRedis:                useRedis,
+			WithDuplicates:          withDuplicates,
 		}
 
 		bidCollector, err := bidcollect.NewBidCollector(&opts)

docs/2024-06_bidcollect.md

@@ -74,6 +74,9 @@ Different data sources have different limitations:
 - Bids are deduplicated based on this key:
   - `fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value)`
   - this means only the first bid for a given key is stored, even if - for instance - other relays also deliver the same bid
+- To store the same bid delivered by different relays use the `--with-duplicates` flag. This will change the deduplication key to:
+  - `fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value)`
+  - this is helpful to measure builder to relay latency.
 - Bids can be published to Redis (to be consumed by whatever, i.e. a webserver). The channel is called `bidcollect/bids`.
   - Enable publishing to Redis with the `--redis` flag
   - You can start a webserver that publishes the data via a SSE stream with `--webserver`

services/bidcollect/bid-processor.go

@@ -22,12 +22,13 @@ import (
 //   - One CSV for top bids only
 
 type BidProcessorOpts struct {
-	Log       *logrus.Entry
-	UID       string
-	OutDir    string
-	OutputTSV bool
-	RedisAddr string
-	UseRedis  bool
+	Log            *logrus.Entry
+	UID            string
+	OutDir         string
+	OutputTSV      bool
+	RedisAddr      string
+	UseRedis       bool
+	WithDuplicates bool
 }
 
 type OutFiles struct {
@@ -114,10 +115,18 @@ func (c *BidProcessor) processBids(bids []*types.CommonBid) {
 			}
 		}
 
-		// process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value)
-		if _, ok := c.bidCache[bid.Slot][bid.UniqueKey()]; !ok {
+		var uniqueKey string
+		if c.opts.WithDuplicates {
+			// process regular bids only once per unique key (slot+blockhash+parenthash+relay+builderpubkey+value)
+			uniqueKey = bid.UniqueKeyWithRelay()
+		} else {
+			// process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value)
+			uniqueKey = bid.UniqueKey()
+		}
+
+		if _, ok := c.bidCache[bid.Slot][uniqueKey]; !ok {
 			// yet unknown bid, save it
-			c.bidCache[bid.Slot][bid.UniqueKey()] = bid
+			c.bidCache[bid.Slot][uniqueKey] = bid
 			isNewBid = true
 		}
 

services/bidcollect/bidcollector.go

@@ -23,6 +23,8 @@ type BidCollectorOpts struct {
 
 	RedisAddr string
 	UseRedis  bool
+
+	WithDuplicates bool
 }
 
 type BidCollector struct {
@@ -53,12 +55,13 @@ func NewBidCollector(opts *BidCollectorOpts) (c *BidCollector, err error) {
 
 	// output
 	c.processor, err = NewBidProcessor(&BidProcessorOpts{
-		Log:       opts.Log,
-		UID:       opts.UID,
-		OutDir:    opts.OutDir,
-		OutputTSV: opts.OutputTSV,
-		RedisAddr: opts.RedisAddr,
-		UseRedis:  opts.UseRedis,
+		Log:            opts.Log,
+		UID:            opts.UID,
+		OutDir:         opts.OutDir,
+		OutputTSV:      opts.OutputTSV,
+		RedisAddr:      opts.RedisAddr,
+		UseRedis:       opts.UseRedis,
+		WithDuplicates: opts.WithDuplicates,
 	})
 	return c, err
 }

services/bidcollect/types/types.go

@@ -60,6 +60,10 @@ func (bid *CommonBid) UniqueKey() string {
 	return fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value)
 }
 
+func (bid *CommonBid) UniqueKeyWithRelay() string {
+	return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value)
+}
+
 func (bid *CommonBid) ValueAsBigInt() *big.Int {
 	value := new(big.Int)
 	value.SetString(bid.Value, 10)