From e675f30b03c500f26f91a5289910b54b2367680b Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Thu, 1 Aug 2024 12:25:44 -0300 Subject: [PATCH 1/7] change unique key --- services/bidcollect/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/bidcollect/types.go b/services/bidcollect/types.go index 25ecffa..a3ba883 100644 --- a/services/bidcollect/types.go +++ b/services/bidcollect/types.go @@ -57,7 +57,7 @@ type CommonBid struct { } func (bid *CommonBid) UniqueKey() string { - return fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value) + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) } func (bid *CommonBid) ValueAsBigInt() *big.Int { From b7865ee7d37050cf4d4a97e4fa9c7f0f6d723ad6 Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Thu, 1 Aug 2024 12:25:44 -0300 Subject: [PATCH 2/7] change unique key --- services/bidcollect/types/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/bidcollect/types/types.go b/services/bidcollect/types/types.go index 0efccd4..671159d 100644 --- a/services/bidcollect/types/types.go +++ b/services/bidcollect/types/types.go @@ -57,7 +57,7 @@ type CommonBid struct { } func (bid *CommonBid) UniqueKey() string { - return fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value) + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) } func (bid *CommonBid) ValueAsBigInt() *big.Int { From b4c895a61770ebfd204694ba0aebd6ba7618586c Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Tue, 10 Sep 2024 13:02:43 -0300 Subject: [PATCH 3/7] add with-duplicates flag --- cmd/service/bidcollect.go | 4 ++++ services/bidcollect/bid-processor.go | 15 ++++++++++++--- services/bidcollect/bidcollector.go | 3 +++ services/bidcollect/types/types.go | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cmd/service/bidcollect.go b/cmd/service/bidcollect.go index 41d7b1c..5f5399d 100644 --- a/cmd/service/bidcollect.go +++ b/cmd/service/bidcollect.go @@ -36,6 +36,8 @@ var ( runWebserverOnly bool // provides a SSE stream of new bids WebserverListenAddr string + + withDuplicates bool ) func init() { @@ -70,6 +72,7 @@ func init() { bidCollectCmd.Flags().BoolVar(&buildWebsite, "build-website", false, "build file listing website") bidCollectCmd.Flags().BoolVar(&buildWebsiteUpload, "build-website-upload", false, "upload after building") bidCollectCmd.Flags().StringVar(&buildWebsiteOutDir, "build-website-out", "build", "output directory for website") + bidCollectCmd.Flags().BoolVar(&withDuplicates, "with-duplicates", false, "store all bids, including duplicates from different relays") } var bidCollectCmd = &cobra.Command{ @@ -128,6 +131,7 @@ var bidCollectCmd = &cobra.Command{ OutDir: outDir, OutputTSV: outputTSV, RedisAddr: redisAddr, + WithDuplicates: withDuplicates, } bidCollector, err := bidcollect.NewBidCollector(&opts) diff --git a/services/bidcollect/bid-processor.go b/services/bidcollect/bid-processor.go index 9093d37..8f0480d 100644 --- a/services/bidcollect/bid-processor.go +++ b/services/bidcollect/bid-processor.go @@ -27,6 +27,7 @@ type BidProcessorOpts struct { OutDir string OutputTSV bool RedisAddr string + WithDuplicates bool } type OutFiles struct { @@ -113,10 +114,18 @@ func (c *BidProcessor) processBids(bids []*types.CommonBid) { } } - // process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value) - if _, ok := c.bidCache[bid.Slot][bid.UniqueKey()]; !ok { + var uniqueKey string + if c.opts.WithDuplicates { + // process regular bids only once per unique key (slot+blockhash+parenthash+relay+builderpubkey+value) + uniqueKey = bid.UniqueKeyWithRelay() + } else { + // process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value) + uniqueKey = bid.UniqueKey() + } + + if _, ok := c.bidCache[bid.Slot][uniqueKey]; !ok { // yet unknown bid, save it - c.bidCache[bid.Slot][bid.UniqueKey()] = bid + c.bidCache[bid.Slot][uniqueKey] = bid isNewBid = true } diff --git a/services/bidcollect/bidcollector.go b/services/bidcollect/bidcollector.go index 44aef66..8e194ab 100644 --- a/services/bidcollect/bidcollector.go +++ b/services/bidcollect/bidcollector.go @@ -22,6 +22,8 @@ type BidCollectorOpts struct { OutputTSV bool RedisAddr string + + WithDuplicates bool } type BidCollector struct { @@ -57,6 +59,7 @@ func NewBidCollector(opts *BidCollectorOpts) (c *BidCollector, err error) { OutDir: opts.OutDir, OutputTSV: opts.OutputTSV, RedisAddr: opts.RedisAddr, + WithDuplicates: opts.WithDuplicates, }) return c, err } diff --git a/services/bidcollect/types/types.go b/services/bidcollect/types/types.go index 671159d..5cced26 100644 --- a/services/bidcollect/types/types.go +++ b/services/bidcollect/types/types.go @@ -60,6 +60,10 @@ func (bid *CommonBid) UniqueKey() string { return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) } +func (bid *CommonBid) UniqueKeyWithRelay() string { + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) +} + func (bid *CommonBid) ValueAsBigInt() *big.Int { value := new(big.Int) value.SetString(bid.Value, 10) From c6316e1ea7140411b6fa747abeddb9ee1754fe57 Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Thu, 1 Aug 2024 12:25:44 -0300 Subject: [PATCH 4/7] change unique key --- services/bidcollect/types/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/bidcollect/types/types.go b/services/bidcollect/types/types.go index 0efccd4..671159d 100644 --- a/services/bidcollect/types/types.go +++ b/services/bidcollect/types/types.go @@ -57,7 +57,7 @@ type CommonBid struct { } func (bid *CommonBid) UniqueKey() string { - return fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value) + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) } func (bid *CommonBid) ValueAsBigInt() *big.Int { From c44d8717b27db5b70dc9c7d652dc6821b3c3a1b3 Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Tue, 10 Sep 2024 13:02:43 -0300 Subject: [PATCH 5/7] add with-duplicates flag --- cmd/service/bidcollect.go | 4 ++++ services/bidcollect/bid-processor.go | 27 ++++++++++++++++++--------- services/bidcollect/bidcollector.go | 15 +++++++++------ services/bidcollect/types/types.go | 4 ++++ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/cmd/service/bidcollect.go b/cmd/service/bidcollect.go index d9117ab..56e3b60 100644 --- a/cmd/service/bidcollect.go +++ b/cmd/service/bidcollect.go @@ -36,6 +36,8 @@ var ( runWebserverOnly bool // provides a SSE stream of new bids WebserverListenAddr string + + withDuplicates bool ) func init() { @@ -70,6 +72,7 @@ func init() { bidCollectCmd.Flags().BoolVar(&buildWebsite, "build-website", false, "build file listing website") bidCollectCmd.Flags().BoolVar(&buildWebsiteUpload, "build-website-upload", false, "upload after building") bidCollectCmd.Flags().StringVar(&buildWebsiteOutDir, "build-website-out", "build", "output directory for website") + bidCollectCmd.Flags().BoolVar(&withDuplicates, "with-duplicates", false, "store all bids, including duplicates from different relays") } var bidCollectCmd = &cobra.Command{ @@ -129,6 +132,7 @@ var bidCollectCmd = &cobra.Command{ OutputTSV: outputTSV, RedisAddr: redisAddr, UseRedis: useRedis, + WithDuplicates: withDuplicates, } bidCollector, err := bidcollect.NewBidCollector(&opts) diff --git a/services/bidcollect/bid-processor.go b/services/bidcollect/bid-processor.go index c55517c..56ca67a 100644 --- a/services/bidcollect/bid-processor.go +++ b/services/bidcollect/bid-processor.go @@ -22,12 +22,13 @@ import ( // - One CSV for top bids only type BidProcessorOpts struct { - Log *logrus.Entry - UID string - OutDir string - OutputTSV bool - RedisAddr string - UseRedis bool + Log *logrus.Entry + UID string + OutDir string + OutputTSV bool + RedisAddr string + UseRedis bool + WithDuplicates bool } type OutFiles struct { @@ -114,10 +115,18 @@ func (c *BidProcessor) processBids(bids []*types.CommonBid) { } } - // process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value) - if _, ok := c.bidCache[bid.Slot][bid.UniqueKey()]; !ok { + var uniqueKey string + if c.opts.WithDuplicates { + // process regular bids only once per unique key (slot+blockhash+parenthash+relay+builderpubkey+value) + uniqueKey = bid.UniqueKeyWithRelay() + } else { + // process regular bids only once per unique key (slot+blockhash+parenthash+builderpubkey+value) + uniqueKey = bid.UniqueKey() + } + + if _, ok := c.bidCache[bid.Slot][uniqueKey]; !ok { // yet unknown bid, save it - c.bidCache[bid.Slot][bid.UniqueKey()] = bid + c.bidCache[bid.Slot][uniqueKey] = bid isNewBid = true } diff --git a/services/bidcollect/bidcollector.go b/services/bidcollect/bidcollector.go index 4efac2b..f59e839 100644 --- a/services/bidcollect/bidcollector.go +++ b/services/bidcollect/bidcollector.go @@ -23,6 +23,8 @@ type BidCollectorOpts struct { RedisAddr string UseRedis bool + + WithDuplicates bool } type BidCollector struct { @@ -53,12 +55,13 @@ func NewBidCollector(opts *BidCollectorOpts) (c *BidCollector, err error) { // output c.processor, err = NewBidProcessor(&BidProcessorOpts{ - Log: opts.Log, - UID: opts.UID, - OutDir: opts.OutDir, - OutputTSV: opts.OutputTSV, - RedisAddr: opts.RedisAddr, - UseRedis: opts.UseRedis, + Log: opts.Log, + UID: opts.UID, + OutDir: opts.OutDir, + OutputTSV: opts.OutputTSV, + RedisAddr: opts.RedisAddr, + UseRedis: opts.UseRedis, + WithDuplicates: opts.WithDuplicates, }) return c, err } diff --git a/services/bidcollect/types/types.go b/services/bidcollect/types/types.go index 671159d..5cced26 100644 --- a/services/bidcollect/types/types.go +++ b/services/bidcollect/types/types.go @@ -60,6 +60,10 @@ func (bid *CommonBid) UniqueKey() string { return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) } +func (bid *CommonBid) UniqueKeyWithRelay() string { + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) +} + func (bid *CommonBid) ValueAsBigInt() *big.Int { value := new(big.Int) value.SetString(bid.Value, 10) From f32ba900cc52aaf45e6d88f419fd2d056773a7c3 Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Wed, 11 Sep 2024 14:36:26 -0300 Subject: [PATCH 6/7] add docs --- docs/2024-06_bidcollect.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/2024-06_bidcollect.md b/docs/2024-06_bidcollect.md index bd43462..5f72d65 100644 --- a/docs/2024-06_bidcollect.md +++ b/docs/2024-06_bidcollect.md @@ -74,6 +74,9 @@ Different data sources have different limitations: - Bids are deduplicated based on this key: - `fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value)` - this means only the first bid for a given key is stored, even if - for instance - other relays also deliver the same bid +- To store the same bid delivered by different relays use the `--with-duplicates` flag. This will change the deduplication key to: + - `fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value)` + - this is helpful to measure builder to relay latency. - Bids can be published to Redis (to be consumed by whatever, i.e. a webserver). The channel is called `bidcollect/bids`. - Enable publishing to Redis with the `--redis` flag - You can start a webserver that publishes the data via a SSE stream with `--webserver` From 504a191e6c01060f102d4a0b0ce592fff3b4db17 Mon Sep 17 00:00:00 2001 From: Andres Babino Date: Wed, 11 Sep 2024 14:39:10 -0300 Subject: [PATCH 7/7] fix old unique key --- services/bidcollect/types/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/bidcollect/types/types.go b/services/bidcollect/types/types.go index 5cced26..bba9c5d 100644 --- a/services/bidcollect/types/types.go +++ b/services/bidcollect/types/types.go @@ -57,11 +57,11 @@ type CommonBid struct { } func (bid *CommonBid) UniqueKey() string { - return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) + return fmt.Sprintf("%d-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.BuilderPubkey, bid.Value) } func (bid *CommonBid) UniqueKeyWithRelay() string { - return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) + return fmt.Sprintf("%d-%s-%s-%s-%s-%s", bid.Slot, bid.BlockHash, bid.ParentHash, bid.Relay, bid.BuilderPubkey, bid.Value) } func (bid *CommonBid) ValueAsBigInt() *big.Int {