From c5177440d56bd1fc462d364f3be7afc9febf1d92 Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Tue, 10 Dec 2024 11:50:30 +0000 Subject: [PATCH] Revert changes related to batching and dedup --- cmd/conformance/aws/main.go | 3 +-- cmd/conformance/gcp/main.go | 3 +-- cmd/conformance/mysql/main.go | 2 +- cmd/conformance/posix/main.go | 5 ++--- log.go | 3 +-- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/conformance/aws/main.go b/cmd/conformance/aws/main.go index 7f2769e5..f86c6043 100644 --- a/cmd/conformance/aws/main.go +++ b/cmd/conformance/aws/main.go @@ -25,7 +25,6 @@ import ( "time" tessera "github.com/transparency-dev/trillian-tessera" - "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/storage/aws" "golang.org/x/mod/sumdb/note" "golang.org/x/net/http2" @@ -73,7 +72,7 @@ func main() { if err != nil { klog.Exitf("Failed to create new AWS storage: %v", err) } - dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth) + dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256) // Expose a HTTP handler for the conformance test writes. // This should accept arbitrary bytes POSTed to /add, and return an ascii diff --git a/cmd/conformance/gcp/main.go b/cmd/conformance/gcp/main.go index 61f36d8c..e34b08ed 100644 --- a/cmd/conformance/gcp/main.go +++ b/cmd/conformance/gcp/main.go @@ -25,7 +25,6 @@ import ( "time" tessera "github.com/transparency-dev/trillian-tessera" - "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/storage/gcp" "golang.org/x/mod/sumdb/note" "golang.org/x/net/http2" @@ -78,7 +77,7 @@ func main() { klog.Exitf("Failed to create new GCP dedupe: %v", err) } } - dedupeAdd := tessera.InMemoryDedupe(addDelegate, layout.TileWidth) + dedupeAdd := tessera.InMemoryDedupe(addDelegate, 256) // Expose a HTTP handler for the conformance test writes. // This should accept arbitrary bytes POSTed to /add, and return an ascii diff --git a/cmd/conformance/mysql/main.go b/cmd/conformance/mysql/main.go index 0ac782e8..16eb7959 100644 --- a/cmd/conformance/mysql/main.go +++ b/cmd/conformance/mysql/main.go @@ -69,7 +69,7 @@ func main() { if err != nil { klog.Exitf("Failed to create new MySQL storage: %v", err) } - dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth) + dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256) // Set up the handlers for the tlog-tiles GET methods, and a custom handler for HTTP POSTs to /add configureTilesReadAPI(http.DefaultServeMux, storage) diff --git a/cmd/conformance/posix/main.go b/cmd/conformance/posix/main.go index b1604d1e..cc1fcf3b 100644 --- a/cmd/conformance/posix/main.go +++ b/cmd/conformance/posix/main.go @@ -30,7 +30,6 @@ import ( "golang.org/x/mod/sumdb/note" tessera "github.com/transparency-dev/trillian-tessera" - "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/storage/posix" "k8s.io/klog/v2" ) @@ -59,11 +58,11 @@ func main() { s, a := getSignersOrDie() // Create the Tessera POSIX storage, using the directory from the --storage_dir flag - storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(layout.TileWidth, time.Second)) + storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(256, time.Second)) if err != nil { klog.Exitf("Failed to construct storage: %v", err) } - dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth) + dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256) // Define a handler for /add that accepts POST requests and adds the POST body to the log http.HandleFunc("POST /add", func(w http.ResponseWriter, r *http.Request) { diff --git a/log.go b/log.go index c7a3e9b0..10265a00 100644 --- a/log.go +++ b/log.go @@ -21,7 +21,6 @@ import ( "time" f_log "github.com/transparency-dev/formats/log" - "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/internal/options" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" @@ -29,7 +28,7 @@ import ( const ( // DefaultBatchMaxSize is used by storage implementations if no WithBatching option is provided when instantiating it. - DefaultBatchMaxSize = layout.TileWidth + DefaultBatchMaxSize = 256 // DefaultBatchMaxAge is used by storage implementations if no WithBatching option is provided when instantiating it. DefaultBatchMaxAge = 250 * time.Millisecond // DefaultCheckpointInterval is used by storage implementations if no WithCheckpointInterval option is provided when instantiating it.