Skip to content

Commit

Permalink
Replace 256 with layout.TileWidth/layout.EntryBundleWidth
Browse files Browse the repository at this point in the history
  • Loading branch information
roger2hk committed Dec 9, 2024
1 parent 6039e35 commit d8e2754
Show file tree
Hide file tree
Showing 15 changed files with 38 additions and 34 deletions.
2 changes: 1 addition & 1 deletion client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -359,7 +359,7 @@ func TestGetEntryBundleAddressing(t *testing.T) {
{
name: "works - full tile",
idx: 1,
logSize: 256*2 + 45,
logSize: layout.TileWidth*2 + 45,
wantPartialTileSize: 0,
},
} {
Expand Down
3 changes: 2 additions & 1 deletion cmd/conformance/aws/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

tessera "github.com/transparency-dev/trillian-tessera"
"github.com/transparency-dev/trillian-tessera/api/layout"
"github.com/transparency-dev/trillian-tessera/storage/aws"
"golang.org/x/mod/sumdb/note"
"golang.org/x/net/http2"
Expand Down Expand Up @@ -72,7 +73,7 @@ func main() {
if err != nil {
klog.Exitf("Failed to create new AWS storage: %v", err)
}
dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256)
dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth)

// Expose a HTTP handler for the conformance test writes.
// This should accept arbitrary bytes POSTed to /add, and return an ascii
Expand Down
3 changes: 2 additions & 1 deletion cmd/conformance/gcp/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

tessera "github.com/transparency-dev/trillian-tessera"
"github.com/transparency-dev/trillian-tessera/api/layout"
"github.com/transparency-dev/trillian-tessera/storage/gcp"
"golang.org/x/mod/sumdb/note"
"golang.org/x/net/http2"
Expand Down Expand Up @@ -77,7 +78,7 @@ func main() {
klog.Exitf("Failed to create new GCP dedupe: %v", err)
}
}
dedupeAdd := tessera.InMemoryDedupe(addDelegate, 256)
dedupeAdd := tessera.InMemoryDedupe(addDelegate, layout.TileWidth)

// Expose a HTTP handler for the conformance test writes.
// This should accept arbitrary bytes POSTed to /add, and return an ascii
Expand Down
2 changes: 1 addition & 1 deletion cmd/conformance/mysql/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func main() {
if err != nil {
klog.Exitf("Failed to create new MySQL storage: %v", err)
}
dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256)
dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth)

// Set up the handlers for the tlog-tiles GET methods, and a custom handler for HTTP POSTs to /add
configureTilesReadAPI(http.DefaultServeMux, storage)
Expand Down
5 changes: 3 additions & 2 deletions cmd/conformance/posix/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"golang.org/x/mod/sumdb/note"

tessera "github.com/transparency-dev/trillian-tessera"
"github.com/transparency-dev/trillian-tessera/api/layout"
"github.com/transparency-dev/trillian-tessera/storage/posix"
"k8s.io/klog/v2"
)
Expand Down Expand Up @@ -58,11 +59,11 @@ func main() {
s, a := getSignersOrDie()

// Create the Tessera POSIX storage, using the directory from the --storage_dir flag
storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(256, time.Second))
storage, err := posix.New(ctx, *storageDir, *initialise, tessera.WithCheckpointSigner(s, a...), tessera.WithBatching(layout.TileWidth, time.Second))
if err != nil {
klog.Exitf("Failed to construct storage: %v", err)
}
dedupeAdd := tessera.InMemoryDedupe(storage.Add, 256)
dedupeAdd := tessera.InMemoryDedupe(storage.Add, layout.TileWidth)

// Define a handler for /add that accepts POST requests and adds the POST body to the log
http.HandleFunc("POST /add", func(w http.ResponseWriter, r *http.Request) {
Expand Down
5 changes: 3 additions & 2 deletions integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"github.com/transparency-dev/formats/log"
"github.com/transparency-dev/merkle/proof"
"github.com/transparency-dev/merkle/rfc6962"
"github.com/transparency-dev/trillian-tessera/api/layout"
"github.com/transparency-dev/trillian-tessera/client"
"golang.org/x/mod/sumdb/note"
"golang.org/x/sync/errgroup"
Expand Down Expand Up @@ -180,12 +181,12 @@ func TestLiveLogIntegration(t *testing.T) {
index := v.(uint64)

// Step 4.1 - Get entry bundles to read back what was written, check leaves are correct.
entryBundle, err := client.GetEntryBundle(ctx, logReadEntryBundle, index/256, checkpoint.Size)
entryBundle, err := client.GetEntryBundle(ctx, logReadEntryBundle, index/layout.EntryBundleWidth, checkpoint.Size)
if err != nil {
t.Fatalf("client.GetEntryBundle: %v", err)
}

got, want := entryBundle.Entries[index%256], []byte(fmt.Sprintf("%d", data))
got, want := entryBundle.Entries[index%layout.EntryBundleWidth], []byte(fmt.Sprintf("%d", data))
if !bytes.Equal(got, want) {
t.Errorf("Entry bundle (index: %d) got %v want %v", index, got, want)
}
Expand Down
5 changes: 3 additions & 2 deletions internal/hammer/workers.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"math/rand/v2"
"time"

"github.com/transparency-dev/trillian-tessera/api/layout"
"github.com/transparency-dev/trillian-tessera/client"
"k8s.io/klog/v2"
)
Expand Down Expand Up @@ -94,11 +95,11 @@ func (r *LeafReader) getLeaf(ctx context.Context, i uint64, logSize uint64) ([]b
return cached, nil
}

bundle, err := client.GetEntryBundle(ctx, r.f, i/256, logSize)
bundle, err := client.GetEntryBundle(ctx, r.f, i/layout.EntryBundleWidth, logSize)
if err != nil {
return nil, fmt.Errorf("failed to get entry bundle: %v", err)
}
ti := i % 256
ti := i % layout.EntryBundleWidth
r.c = leafBundleCache{
start: i - ti,
leaves: bundle.Entries,
Expand Down
3 changes: 2 additions & 1 deletion log.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,15 @@ import (
"time"

f_log "github.com/transparency-dev/formats/log"
"github.com/transparency-dev/trillian-tessera/api/layout"
"github.com/transparency-dev/trillian-tessera/internal/options"
"golang.org/x/mod/sumdb/note"
"k8s.io/klog/v2"
)

const (
// DefaultBatchMaxSize is used by storage implementations if no WithBatching option is provided when instantiating it.
DefaultBatchMaxSize = 256
DefaultBatchMaxSize = layout.TileWidth
// DefaultBatchMaxAge is used by storage implementations if no WithBatching option is provided when instantiating it.
DefaultBatchMaxAge = 250 * time.Millisecond
// DefaultCheckpointInterval is used by storage implementations if no WithCheckpointInterval option is provided when instantiating it.
Expand Down
1 change: 0 additions & 1 deletion storage/aws/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ import (
)

const (
entryBundleSize = 256
logContType = "application/octet-stream"
ckptContType = "text/plain; charset=utf-8"
minCheckpointInterval = time.Second
Expand Down
6 changes: 3 additions & 3 deletions storage/aws/aws_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,8 @@ func TestTileRoundtrip(t *testing.T) {
{
name: "ok",
level: 0,
index: 3 * 256,
logSize: 3*256 + 20,
index: 3 * layout.TileWidth,
logSize: 3*layout.TileWidth + 20,
tileSize: 20,
},
} {
Expand Down Expand Up @@ -339,7 +339,7 @@ func TestBundleRoundtrip(t *testing.T) {
}{
{
name: "ok",
index: 3 * 256,
index: 3 * layout.EntryBundleWidth,
p: 20,
bundleSize: 20,
},
Expand Down
8 changes: 4 additions & 4 deletions storage/gcp/gcp_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,8 @@ func TestTileRoundtrip(t *testing.T) {
{
name: "ok",
level: 0,
index: 3 * 256,
logSize: 3*256 + 20,
index: 3 * layout.TileWidth,
logSize: 3*layout.TileWidth + 20,
tileSize: 20,
},
} {
Expand Down Expand Up @@ -277,8 +277,8 @@ func TestBundleRoundtrip(t *testing.T) {
}{
{
name: "ok",
index: 3 * 256,
logSize: 3*256 + 20,
index: 3 * layout.EntryBundleWidth,
logSize: 3*layout.EntryBundleWidth + 20,
bundleSize: 20,
},
} {
Expand Down
4 changes: 2 additions & 2 deletions storage/internal/integrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ func (tc *tileWriteCache) Err() error {
// minImpliedTreeSize returns the smallest possible tree size implied by the existence of a tile
// with the given ID.
func minImpliedTreeSize(id TileID) uint64 {
return (id.Index * 256) << (id.Level * 8)
return (id.Index * layout.TileWidth) << (id.Level * 8)
}

// Visitor returns a function suitable for use with the compact.Range visitor pattern.
Expand Down Expand Up @@ -312,7 +312,7 @@ type populatedTile struct {
func newPopulatedTile(h *api.HashTile) (*populatedTile, error) {
ft := &populatedTile{
inner: make(map[compact.NodeID][]byte),
leaves: make([][]byte, 0, 256),
leaves: make([][]byte, 0, layout.TileWidth),
}

if h != nil {
Expand Down
8 changes: 4 additions & 4 deletions storage/internal/integrate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func TestNewRangeFetchesTiles(t *testing.T) {
}

for _, id := range wantIDs {
if err := m.setTile(ctx, id, treeSize, zeroTile(256)); err != nil {
if err := m.setTile(ctx, id, treeSize, zeroTile(layout.TileWidth)); err != nil {
t.Fatalf("setTile: %v", err)
}
}
Expand Down Expand Up @@ -78,9 +78,9 @@ func TestTileVisit(t *testing.T) {
{
name: "ok - multiple tiles",
visits: map[compact.NodeID][]byte{
{Level: 0, Index: 0}: {0},
{Level: 0, Index: 1 * 256}: {1},
{Level: 8, Index: 2 * 256}: {2},
{Level: 0, Index: 0}: {0},
{Level: 0, Index: 1 * layout.TileWidth}: {1},
{Level: 8, Index: 2 * layout.TileWidth}: {2},
},
wantTiles: map[TileID]*api.HashTile{
{Level: 0, Index: 0}: {Nodes: [][]byte{{0}}},
Expand Down
11 changes: 5 additions & 6 deletions storage/mysql/mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,8 @@ const (
selectTiledLeavesSQL = "SELECT `size`, `data` FROM `TiledLeaves` WHERE `tile_index` = ?"
replaceTiledLeavesSQL = "REPLACE INTO `TiledLeaves` (`tile_index`, `size`, `data`) VALUES (?, ?, ?)"

checkpointID = 0
treeStateID = 0
entryBundleSize = 256
checkpointID = 0
treeStateID = 0

minCheckpointInterval = time.Second
)
Expand Down Expand Up @@ -268,7 +267,7 @@ func (s *Storage) ReadTile(ctx context.Context, level, index uint64, p uint8) ([
numEntries := uint64(len(tile) / sha256.Size)
requestedEntries := uint64(p)
if requestedEntries == 0 {
requestedEntries = 256
requestedEntries = layout.TileWidth
}
if requestedEntries > numEntries {
// If the user has requested a size larger than we have, they can't have it
Expand Down Expand Up @@ -450,7 +449,7 @@ func (s *Storage) integrate(ctx context.Context, tx *sql.Tx, fromSeq uint64, ent
}

// Add sequenced entries to entry bundles.
bundleIndex, entriesInBundle := fromSeq/entryBundleSize, fromSeq%entryBundleSize
bundleIndex, entriesInBundle := fromSeq/layout.EntryBundleWidth, fromSeq%layout.EntryBundleWidth
bundleWriter := &bytes.Buffer{}

// If the latest bundle is partial, we need to read the data it contains in for our newer, larger, bundle.
Expand Down Expand Up @@ -482,7 +481,7 @@ func (s *Storage) integrate(ctx context.Context, tx *sql.Tx, fromSeq uint64, ent
entriesInBundle++

// This bundle is full, so we need to write it out.
if entriesInBundle == entryBundleSize {
if entriesInBundle == layout.EntryBundleWidth {
if err := s.writeEntryBundle(ctx, tx, bundleIndex, uint32(entriesInBundle), bundleWriter.Bytes()); err != nil {
return fmt.Errorf("writeEntryBundle: %w", err)
}
Expand Down
6 changes: 3 additions & 3 deletions storage/mysql/mysql_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ func TestGetTile(t *testing.T) {
{
name: "requested partial tile for a complete tile",
level: 0, index: 0, p: 10,
wantEntries: 256,
wantEntries: layout.TileWidth,
wantNotFound: false,
},
{
Expand Down Expand Up @@ -438,7 +438,7 @@ func TestEntryBundleRoundTrip(t *testing.T) {
if err != nil {
t.Errorf("Add got err: %v", err)
}
entryBundleRaw, err := s.ReadEntryBundle(ctx, entryIndex/256, layout.PartialTileSize(0, entryIndex, entryIndex+1))
entryBundleRaw, err := s.ReadEntryBundle(ctx, entryIndex/layout.EntryBundleWidth, layout.PartialTileSize(0, entryIndex, entryIndex+1))
if err != nil {
t.Fatalf("ReadEntryBundle got err: %v", err)
}
Expand All @@ -451,7 +451,7 @@ func TestEntryBundleRoundTrip(t *testing.T) {
if len(gotEntries) == 0 {
t.Error("no entry found")
} else {
if !bytes.Equal(bundle.Entries[entryIndex%256], test.entry) {
if !bytes.Equal(bundle.Entries[entryIndex%layout.EntryBundleWidth], test.entry) {
t.Errorf("got entry %v want %v", bundle.Entries[0], test.entry)
}
}
Expand Down

0 comments on commit d8e2754

Please sign in to comment.