From d8e27548bee9c8f80ebbfb54944708503837364b Mon Sep 17 00:00:00 2001 From: Roger Ng Date: Mon, 9 Dec 2024 19:57:27 +0000 Subject: [PATCH] Replace `256` with `layout.TileWidth`/`layout.EntryBundleWidth` --- client/client_test.go | 2 +- cmd/conformance/aws/main.go | 3 ++- cmd/conformance/gcp/main.go | 3 ++- cmd/conformance/mysql/main.go | 2 +- cmd/conformance/posix/main.go | 5 +++-- integration/integration_test.go | 5 +++-- internal/hammer/workers.go | 5 +++-- log.go | 3 ++- storage/aws/aws.go | 1 - storage/aws/aws_test.go | 6 +++--- storage/gcp/gcp_test.go | 8 ++++---- storage/internal/integrate.go | 4 ++-- storage/internal/integrate_test.go | 8 ++++---- storage/mysql/mysql.go | 11 +++++------ storage/mysql/mysql_test.go | 6 +++--- 15 files changed, 38 insertions(+), 34 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 10e34b1b..d0e495b0 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -359,7 +359,7 @@ func TestGetEntryBundleAddressing(t *testing.T) { { name: "works - full tile", idx: 1, - logSize: 256*2 + 45, + logSize: layout.TileWidth*2 + 45, wantPartialTileSize: 0, }, } { diff --git a/cmd/conformance/aws/main.go b/cmd/conformance/aws/main.go index f86c6043..7f2769e5 100644 --- a/cmd/conformance/aws/main.go +++ b/cmd/conformance/aws/main.go @@ -25,6 +25,7 @@ import ( "time" tessera "github.com/transparency-dev/trillian-tessera" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/storage/aws" "golang.org/x/mod/sumdb/note" "golang.org/x/net/http2" @@ -72,7 +73,7 @@ func main() { if err != nil { klog.Exitf("Failed to create new AWS storage: %v", err) } - dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256) + dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth) // Expose a HTTP handler for the conformance test writes. // This should accept arbitrary bytes POSTed to /add, and return an ascii diff --git a/cmd/conformance/gcp/main.go b/cmd/conformance/gcp/main.go index e34b08ed..61f36d8c 100644 --- a/cmd/conformance/gcp/main.go +++ b/cmd/conformance/gcp/main.go @@ -25,6 +25,7 @@ import ( "time" tessera "github.com/transparency-dev/trillian-tessera" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/storage/gcp" "golang.org/x/mod/sumdb/note" "golang.org/x/net/http2" @@ -77,7 +78,7 @@ func main() { klog.Exitf("Failed to create new GCP dedupe: %v", err) } } - dedupeAdd := tessera.InMemoryDedupe(addDelegate, 256) + dedupeAdd := tessera.InMemoryDedupe(addDelegate, layout.TileWidth) // Expose a HTTP handler for the conformance test writes. // This should accept arbitrary bytes POSTed to /add, and return an ascii diff --git a/cmd/conformance/mysql/main.go b/cmd/conformance/mysql/main.go index 16eb7959..0ac782e8 100644 --- a/cmd/conformance/mysql/main.go +++ b/cmd/conformance/mysql/main.go @@ -69,7 +69,7 @@ func main() { if err != nil { klog.Exitf("Failed to create new MySQL storage: %v", err) } - dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256) + dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth) // Set up the handlers for the tlog-tiles GET methods, and a custom handler for HTTP POSTs to /add configureTilesReadAPI(http.DefaultServeMux, storage) diff --git a/cmd/conformance/posix/main.go b/cmd/conformance/posix/main.go index cc1fcf3b..b1604d1e 100644 --- a/cmd/conformance/posix/main.go +++ b/cmd/conformance/posix/main.go @@ -30,6 +30,7 @@ import ( "golang.org/x/mod/sumdb/note" tessera "github.com/transparency-dev/trillian-tessera" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/storage/posix" "k8s.io/klog/v2" ) @@ -58,11 +59,11 @@ func main() { s, a := getSignersOrDie() // Create the Tessera POSIX storage, using the directory from the --storage_dir flag - storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(256, time.Second)) + storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(layout.TileWidth, time.Second)) if err != nil { klog.Exitf("Failed to construct storage: %v", err) } - dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256) + dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth) // Define a handler for /add that accepts POST requests and adds the POST body to the log http.HandleFunc("POST /add", func(w http.ResponseWriter, r *http.Request) { diff --git a/integration/integration_test.go b/integration/integration_test.go index b2bb8e38..a19b7173 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -35,6 +35,7 @@ import ( "github.com/transparency-dev/formats/log" "github.com/transparency-dev/merkle/proof" "github.com/transparency-dev/merkle/rfc6962" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/client" "golang.org/x/mod/sumdb/note" "golang.org/x/sync/errgroup" @@ -180,12 +181,12 @@ func TestLiveLogIntegration(t *testing.T) { index := v.(uint64) // Step 4.1 - Get entry bundles to read back what was written, check leaves are correct. - entryBundle, err := client.GetEntryBundle(ctx, logReadEntryBundle, index/256, checkpoint.Size) + entryBundle, err := client.GetEntryBundle(ctx, logReadEntryBundle, index/layout.EntryBundleWidth, checkpoint.Size) if err != nil { t.Fatalf("client.GetEntryBundle: %v", err) } - got, want := entryBundle.Entries[index%256], []byte(fmt.Sprintf("%d", data)) + got, want := entryBundle.Entries[index%layout.EntryBundleWidth], []byte(fmt.Sprintf("%d", data)) if !bytes.Equal(got, want) { t.Errorf("Entry bundle (index: %d) got %v want %v", index, got, want) } diff --git a/internal/hammer/workers.go b/internal/hammer/workers.go index e153be07..2f17605d 100644 --- a/internal/hammer/workers.go +++ b/internal/hammer/workers.go @@ -22,6 +22,7 @@ import ( "math/rand/v2" "time" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/client" "k8s.io/klog/v2" ) @@ -94,11 +95,11 @@ func (r *LeafReader) getLeaf(ctx context.Context, i uint64, logSize uint64) ([]b return cached, nil } - bundle, err := client.GetEntryBundle(ctx, r.f, i/256, logSize) + bundle, err := client.GetEntryBundle(ctx, r.f, i/layout.EntryBundleWidth, logSize) if err != nil { return nil, fmt.Errorf("failed to get entry bundle: %v", err) } - ti := i % 256 + ti := i % layout.EntryBundleWidth r.c = leafBundleCache{ start: i - ti, leaves: bundle.Entries, diff --git a/log.go b/log.go index 10265a00..c7a3e9b0 100644 --- a/log.go +++ b/log.go @@ -21,6 +21,7 @@ import ( "time" f_log "github.com/transparency-dev/formats/log" + "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/internal/options" "golang.org/x/mod/sumdb/note" "k8s.io/klog/v2" @@ -28,7 +29,7 @@ import ( const ( // DefaultBatchMaxSize is used by storage implementations if no WithBatching option is provided when instantiating it. - DefaultBatchMaxSize = 256 + DefaultBatchMaxSize = layout.TileWidth // DefaultBatchMaxAge is used by storage implementations if no WithBatching option is provided when instantiating it. DefaultBatchMaxAge = 250 * time.Millisecond // DefaultCheckpointInterval is used by storage implementations if no WithCheckpointInterval option is provided when instantiating it. diff --git a/storage/aws/aws.go b/storage/aws/aws.go index 81cb402f..aeda7e4e 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -59,7 +59,6 @@ import ( ) const ( - entryBundleSize = 256 logContType = "application/octet-stream" ckptContType = "text/plain; charset=utf-8" minCheckpointInterval = time.Second diff --git a/storage/aws/aws_test.go b/storage/aws/aws_test.go index 7c866657..a2f2c540 100644 --- a/storage/aws/aws_test.go +++ b/storage/aws/aws_test.go @@ -283,8 +283,8 @@ func TestTileRoundtrip(t *testing.T) { { name: "ok", level: 0, - index: 3 * 256, - logSize: 3*256 + 20, + index: 3 * layout.TileWidth, + logSize: 3*layout.TileWidth + 20, tileSize: 20, }, } { @@ -339,7 +339,7 @@ func TestBundleRoundtrip(t *testing.T) { }{ { name: "ok", - index: 3 * 256, + index: 3 * layout.EntryBundleWidth, p: 20, bundleSize: 20, }, diff --git a/storage/gcp/gcp_test.go b/storage/gcp/gcp_test.go index ed44df86..fcda9a3e 100644 --- a/storage/gcp/gcp_test.go +++ b/storage/gcp/gcp_test.go @@ -221,8 +221,8 @@ func TestTileRoundtrip(t *testing.T) { { name: "ok", level: 0, - index: 3 * 256, - logSize: 3*256 + 20, + index: 3 * layout.TileWidth, + logSize: 3*layout.TileWidth + 20, tileSize: 20, }, } { @@ -277,8 +277,8 @@ func TestBundleRoundtrip(t *testing.T) { }{ { name: "ok", - index: 3 * 256, - logSize: 3*256 + 20, + index: 3 * layout.EntryBundleWidth, + logSize: 3*layout.EntryBundleWidth + 20, bundleSize: 20, }, } { diff --git a/storage/internal/integrate.go b/storage/internal/integrate.go index 275016c8..0f83de68 100644 --- a/storage/internal/integrate.go +++ b/storage/internal/integrate.go @@ -251,7 +251,7 @@ func (tc *tileWriteCache) Err() error { // minImpliedTreeSize returns the smallest possible tree size implied by the existence of a tile // with the given ID. func minImpliedTreeSize(id TileID) uint64 { - return (id.Index * 256) << (id.Level * 8) + return (id.Index * layout.TileWidth) << (id.Level * 8) } // Visitor returns a function suitable for use with the compact.Range visitor pattern. @@ -312,7 +312,7 @@ type populatedTile struct { func newPopulatedTile(h *api.HashTile) (*populatedTile, error) { ft := &populatedTile{ inner: make(map[compact.NodeID][]byte), - leaves: make([][]byte, 0, 256), + leaves: make([][]byte, 0, layout.TileWidth), } if h != nil { diff --git a/storage/internal/integrate_test.go b/storage/internal/integrate_test.go index 07a6c737..458b0e1b 100644 --- a/storage/internal/integrate_test.go +++ b/storage/internal/integrate_test.go @@ -43,7 +43,7 @@ func TestNewRangeFetchesTiles(t *testing.T) { } for _, id := range wantIDs { - if err := m.setTile(ctx, id, treeSize, zeroTile(256)); err != nil { + if err := m.setTile(ctx, id, treeSize, zeroTile(layout.TileWidth)); err != nil { t.Fatalf("setTile: %v", err) } } @@ -78,9 +78,9 @@ func TestTileVisit(t *testing.T) { { name: "ok - multiple tiles", visits: map[compact.NodeID][]byte{ - {Level: 0, Index: 0}: {0}, - {Level: 0, Index: 1 * 256}: {1}, - {Level: 8, Index: 2 * 256}: {2}, + {Level: 0, Index: 0}: {0}, + {Level: 0, Index: 1 * layout.TileWidth}: {1}, + {Level: 8, Index: 2 * layout.TileWidth}: {2}, }, wantTiles: map[TileID]*api.HashTile{ {Level: 0, Index: 0}: {Nodes: [][]byte{{0}}}, diff --git a/storage/mysql/mysql.go b/storage/mysql/mysql.go index 537db2db..992b81aa 100644 --- a/storage/mysql/mysql.go +++ b/storage/mysql/mysql.go @@ -49,9 +49,8 @@ const ( selectTiledLeavesSQL = "SELECT `size`, `data` FROM `TiledLeaves` WHERE `tile_index` = ?" replaceTiledLeavesSQL = "REPLACE INTO `TiledLeaves` (`tile_index`, `size`, `data`) VALUES (?, ?, ?)" - checkpointID = 0 - treeStateID = 0 - entryBundleSize = 256 + checkpointID = 0 + treeStateID = 0 minCheckpointInterval = time.Second ) @@ -268,7 +267,7 @@ func (s *Storage) ReadTile(ctx context.Context, level, index uint64, p uint8) ([ numEntries := uint64(len(tile) / sha256.Size) requestedEntries := uint64(p) if requestedEntries == 0 { - requestedEntries = 256 + requestedEntries = layout.TileWidth } if requestedEntries > numEntries { // If the user has requested a size larger than we have, they can't have it @@ -450,7 +449,7 @@ func (s *Storage) integrate(ctx context.Context, tx *sql.Tx, fromSeq uint64, ent } // Add sequenced entries to entry bundles. - bundleIndex, entriesInBundle := fromSeq/entryBundleSize, fromSeq%entryBundleSize + bundleIndex, entriesInBundle := fromSeq/layout.EntryBundleWidth, fromSeq%layout.EntryBundleWidth bundleWriter := &bytes.Buffer{} // If the latest bundle is partial, we need to read the data it contains in for our newer, larger, bundle. @@ -482,7 +481,7 @@ func (s *Storage) integrate(ctx context.Context, tx *sql.Tx, fromSeq uint64, ent entriesInBundle++ // This bundle is full, so we need to write it out. - if entriesInBundle == entryBundleSize { + if entriesInBundle == layout.EntryBundleWidth { if err := s.writeEntryBundle(ctx, tx, bundleIndex, uint32(entriesInBundle), bundleWriter.Bytes()); err != nil { return fmt.Errorf("writeEntryBundle: %w", err) } diff --git a/storage/mysql/mysql_test.go b/storage/mysql/mysql_test.go index 60c0aae4..93e28fd6 100644 --- a/storage/mysql/mysql_test.go +++ b/storage/mysql/mysql_test.go @@ -200,7 +200,7 @@ func TestGetTile(t *testing.T) { { name: "requested partial tile for a complete tile", level: 0, index: 0, p: 10, - wantEntries: 256, + wantEntries: layout.TileWidth, wantNotFound: false, }, { @@ -438,7 +438,7 @@ func TestEntryBundleRoundTrip(t *testing.T) { if err != nil { t.Errorf("Add got err: %v", err) } - entryBundleRaw, err := s.ReadEntryBundle(ctx, entryIndex/256, layout.PartialTileSize(0, entryIndex, entryIndex+1)) + entryBundleRaw, err := s.ReadEntryBundle(ctx, entryIndex/layout.EntryBundleWidth, layout.PartialTileSize(0, entryIndex, entryIndex+1)) if err != nil { t.Fatalf("ReadEntryBundle got err: %v", err) } @@ -451,7 +451,7 @@ func TestEntryBundleRoundTrip(t *testing.T) { if len(gotEntries) == 0 { t.Error("no entry found") } else { - if !bytes.Equal(bundle.Entries[entryIndex%256], test.entry) { + if !bytes.Equal(bundle.Entries[entryIndex%layout.EntryBundleWidth], test.entry) { t.Errorf("got entry %v want %v", bundle.Entries[0], test.entry) } }