From 73942bf5b00d89dfeb77ea26881764f639a8d3db Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Fri, 27 Oct 2023 13:07:53 -0500 Subject: [PATCH] refactor!: move all blob share commitment code to the inclusion package (#2770) ## Overview We currently have code relating to creating and proving commitment across the inclusion, x/blob/types, and shares package. This api breaking refactor moves all of the commitment code to the inclusion package. The reasoning behind this is that it will eventually allow for users to import commitment creation and proving code while not having to import the blob module, tendermint, or the sdk. The reasoning behind keeping it in the inclusion instead of shares is that entire reason for making a commitment is to prove inclusion. So the name seems to fit better. The reasoning behind note keeping it in the blob package was that it was difficult to not have an import cycle since this logic requires the share splitting logic for blobs. ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- pkg/da/data_availability_header.go | 28 ++++- .../blob_share_commitment_rules.go | 7 +- .../blob_share_commitment_rules_test.go | 5 +- pkg/inclusion/commitment.go | 114 ++++++++++++++++++ pkg/inclusion/commitment_test.go | 108 +++++++++++++++++ pkg/inclusion/paths.go | 6 +- pkg/square/builder.go | 7 +- pkg/square/square.go | 10 +- test/util/malicious/out_of_order_builder.go | 5 +- x/blob/types/blob_tx.go | 3 +- x/blob/types/blob_tx_test.go | 3 +- x/blob/types/payforblob.go | 108 +---------------- x/blob/types/payforblob_test.go | 100 +-------------- 13 files changed, 279 insertions(+), 225 deletions(-) rename pkg/{shares => inclusion}/blob_share_commitment_rules.go (95%) rename pkg/{shares => inclusion}/blob_share_commitment_rules_test.go (98%) create mode 100644 pkg/inclusion/commitment.go create mode 100644 pkg/inclusion/commitment_test.go diff --git a/pkg/da/data_availability_header.go b/pkg/da/data_availability_header.go index bff5b3cf27..7bc7c944ce 100644 --- a/pkg/da/data_availability_header.go +++ b/pkg/da/data_availability_header.go @@ -4,14 +4,15 @@ import ( "bytes" "errors" "fmt" + "math" "github.com/celestiaorg/rsmt2d" "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/types" + "golang.org/x/exp/constraints" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/celestia-app/pkg/square" "github.com/celestiaorg/celestia-app/pkg/wrapper" daproto "github.com/celestiaorg/celestia-app/proto/celestia/core/v1/da" ) @@ -66,7 +67,7 @@ func ExtendShares(s [][]byte) (*rsmt2d.ExtendedDataSquare, error) { if !shares.IsPowerOfTwo(len(s)) { return nil, fmt.Errorf("number of shares is not a power of 2: got %d", len(s)) } - squareSize := square.Size(len(s)) + squareSize := SquareSize(len(s)) // here we construct a tree // Note: uses the nmt wrapper to construct the tree. @@ -190,5 +191,26 @@ func MinDataAvailabilityHeader() DataAvailabilityHeader { // MinShares returns one tail-padded share. func MinShares() [][]byte { - return shares.ToBytes(square.EmptySquare()) + return shares.ToBytes(EmptySquareShares()) +} + +// EmptySquare is a copy of the function defined in the square package to avoid +// a circular dependency. TODO deduplicate +func EmptySquareShares() []shares.Share { + return shares.TailPaddingShares(appconsts.MinShareCount) +} + +// SquareSize is a copy of the function defined in the square package to avoid +// a circular dependency. TODO deduplicate +func SquareSize(len int) int { + return RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(len))))) +} + +// RoundUpPowerOfTwo returns the next power of two greater than or equal to input. +func RoundUpPowerOfTwo[I constraints.Integer](input I) I { + var result I = 1 + for result < input { + result = result << 1 + } + return result } diff --git a/pkg/shares/blob_share_commitment_rules.go b/pkg/inclusion/blob_share_commitment_rules.go similarity index 95% rename from pkg/shares/blob_share_commitment_rules.go rename to pkg/inclusion/blob_share_commitment_rules.go index 8c9ed6a3ee..d7b1809791 100644 --- a/pkg/shares/blob_share_commitment_rules.go +++ b/pkg/inclusion/blob_share_commitment_rules.go @@ -1,8 +1,9 @@ -package shares +package inclusion import ( "math" + "github.com/celestiaorg/celestia-app/pkg/da" "golang.org/x/exp/constraints" ) @@ -73,7 +74,7 @@ func roundUpByMultipleOf(cursor, v int) int { // BlobMinSquareSize returns the minimum square size that can contain shareCount // number of shares. func BlobMinSquareSize(shareCount int) int { - return RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(shareCount))))) + return da.RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(shareCount))))) } // SubTreeWidth determines the maximum number of leaves per subtree in the share @@ -93,7 +94,7 @@ func SubTreeWidth(shareCount, subtreeRootThreshold int) int { // use a power of two equal to or larger than the multiple of the subtree // root threshold - s = RoundUpPowerOfTwo(s) + s = da.RoundUpPowerOfTwo(s) // use the minimum of the subtree width and the min square size, this // gurarantees that a valid value is returned diff --git a/pkg/shares/blob_share_commitment_rules_test.go b/pkg/inclusion/blob_share_commitment_rules_test.go similarity index 98% rename from pkg/shares/blob_share_commitment_rules_test.go rename to pkg/inclusion/blob_share_commitment_rules_test.go index 466aade1cd..8b6f40f237 100644 --- a/pkg/shares/blob_share_commitment_rules_test.go +++ b/pkg/inclusion/blob_share_commitment_rules_test.go @@ -1,10 +1,11 @@ -package shares +package inclusion import ( "fmt" "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/stretchr/testify/assert" ) @@ -232,7 +233,7 @@ func TestNextShareIndex(t *testing.T) { name: "at threshold", cursor: 11, blobLen: appconsts.DefaultSubtreeRootThreshold, - squareSize: RoundUpPowerOfTwo(appconsts.DefaultSubtreeRootThreshold), + squareSize: da.RoundUpPowerOfTwo(appconsts.DefaultSubtreeRootThreshold), expectedIndex: 11, }, { diff --git a/pkg/inclusion/commitment.go b/pkg/inclusion/commitment.go new file mode 100644 index 0000000000..8492e340bd --- /dev/null +++ b/pkg/inclusion/commitment.go @@ -0,0 +1,114 @@ +package inclusion + +import ( + "crypto/sha256" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + appshares "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/nmt" + "github.com/tendermint/tendermint/crypto/merkle" +) + +// CreateCommitment generates the share commitment for a given blob. +// See [data square layout rationale] and [blob share commitment rules]. +// +// [data square layout rationale]: ../../specs/src/specs/data_square_layout.md +// [blob share commitment rules]: ../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules +func CreateCommitment(blob *blob.Blob) ([]byte, error) { + if err := blob.Validate(); err != nil { + return nil, err + } + namespace := blob.Namespace() + + shares, err := appshares.SplitBlobs(blob) + if err != nil { + return nil, err + } + + // the commitment is the root of a merkle mountain range with max tree size + // determined by the number of roots required to create a share commitment + // over that blob. The size of the tree is only increased if the number of + // subtree roots surpasses a constant threshold. + subTreeWidth := SubTreeWidth(len(shares), appconsts.DefaultSubtreeRootThreshold) + treeSizes, err := MerkleMountainRangeSizes(uint64(len(shares)), uint64(subTreeWidth)) + if err != nil { + return nil, err + } + leafSets := make([][][]byte, len(treeSizes)) + cursor := uint64(0) + for i, treeSize := range treeSizes { + leafSets[i] = appshares.ToBytes(shares[cursor : cursor+treeSize]) + cursor = cursor + treeSize + } + + // create the commitments by pushing each leaf set onto an nmt + subTreeRoots := make([][]byte, len(leafSets)) + for i, set := range leafSets { + // create the nmt todo(evan) use nmt wrapper + tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(appns.NamespaceSize), nmt.IgnoreMaxNamespace(true)) + for _, leaf := range set { + // the namespace must be added again here even though it is already + // included in the leaf to ensure that the hash will match that of + // the nmt wrapper (pkg/wrapper). Each namespace is added to keep + // the namespace in the share, and therefore the parity data, while + // also allowing for the manual addition of the parity namespace to + // the parity data. + nsLeaf := make([]byte, 0) + nsLeaf = append(nsLeaf, namespace.Bytes()...) + nsLeaf = append(nsLeaf, leaf...) + + err = tree.Push(nsLeaf) + if err != nil { + return nil, err + } + } + // add the root + root, err := tree.Root() + if err != nil { + return nil, err + } + subTreeRoots[i] = root + } + return merkle.HashFromByteSlices(subTreeRoots), nil +} + +func CreateCommitments(blobs []*blob.Blob) ([][]byte, error) { + commitments := make([][]byte, len(blobs)) + for i, blob := range blobs { + commitment, err := CreateCommitment(blob) + if err != nil { + return nil, err + } + commitments[i] = commitment + } + return commitments, nil +} + +// MerkleMountainRangeSizes returns the sizes (number of leaf nodes) of the +// trees in a merkle mountain range constructed for a given totalSize and +// maxTreeSize. +// +// https://docs.grin.mw/wiki/chain-state/merkle-mountain-range/ +// https://github.com/opentimestamps/opentimestamps-server/blob/master/doc/merkle-mountain-range.md +func MerkleMountainRangeSizes(totalSize, maxTreeSize uint64) ([]uint64, error) { + var treeSizes []uint64 + + for totalSize != 0 { + switch { + case totalSize >= maxTreeSize: + treeSizes = append(treeSizes, maxTreeSize) + totalSize = totalSize - maxTreeSize + case totalSize < maxTreeSize: + treeSize, err := appshares.RoundDownPowerOfTwo(totalSize) + if err != nil { + return treeSizes, err + } + treeSizes = append(treeSizes, treeSize) + totalSize = totalSize - treeSize + } + } + + return treeSizes, nil +} diff --git a/pkg/inclusion/commitment_test.go b/pkg/inclusion/commitment_test.go new file mode 100644 index 0000000000..68e087e1a1 --- /dev/null +++ b/pkg/inclusion/commitment_test.go @@ -0,0 +1,108 @@ +package inclusion_test + +import ( + "bytes" + "testing" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_MerkleMountainRangeHeights(t *testing.T) { + type test struct { + totalSize uint64 + squareSize uint64 + expected []uint64 + } + tests := []test{ + { + totalSize: 11, + squareSize: 4, + expected: []uint64{4, 4, 2, 1}, + }, + { + totalSize: 2, + squareSize: 64, + expected: []uint64{2}, + }, + { + totalSize: 64, + squareSize: 8, + expected: []uint64{8, 8, 8, 8, 8, 8, 8, 8}, + }, + // Height + // 3 x x + // / \ / \ + // / \ / \ + // / \ / \ + // / \ / \ + // 2 x x x x + // / \ / \ / \ / \ + // 1 x x x x x x x x x + // / \ / \ / \ / \ / \ / \ / \ / \ / \ + // 0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 + { + totalSize: 19, + squareSize: 8, + expected: []uint64{8, 8, 2, 1}, + }, + } + for _, tt := range tests { + res, err := inclusion.MerkleMountainRangeSizes(tt.totalSize, tt.squareSize) + require.NoError(t, err) + assert.Equal(t, tt.expected, res) + } +} + +// TestCreateCommitment will fail if a change is made to share encoding or how +// the commitment is calculated. If this is the case, the expected commitment +// bytes will need to be updated. +func TestCreateCommitment(t *testing.T) { + ns1 := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) + + type test struct { + name string + namespace appns.Namespace + blob []byte + expected []byte + expectErr bool + shareVersion uint8 + } + tests := []test{ + { + name: "blob of 3 shares succeeds", + namespace: ns1, + blob: bytes.Repeat([]byte{0xFF}, 3*appconsts.ShareSize), + expected: []byte{0x3b, 0x9e, 0x78, 0xb6, 0x64, 0x8e, 0xc1, 0xa2, 0x41, 0x92, 0x5b, 0x31, 0xda, 0x2e, 0xcb, 0x50, 0xbf, 0xc6, 0xf4, 0xad, 0x55, 0x2d, 0x32, 0x79, 0x92, 0x8c, 0xa1, 0x3e, 0xbe, 0xba, 0x8c, 0x2b}, + shareVersion: appconsts.ShareVersionZero, + }, + { + name: "blob with unsupported share version should return error", + namespace: ns1, + blob: bytes.Repeat([]byte{0xFF}, 12*appconsts.ShareSize), + expectErr: true, + shareVersion: uint8(1), // unsupported share version + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + blob := &blob.Blob{ + NamespaceId: tt.namespace.ID, + Data: tt.blob, + ShareVersion: uint32(tt.shareVersion), + NamespaceVersion: uint32(tt.namespace.Version), + } + res, err := inclusion.CreateCommitment(blob) + if tt.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expected, res) + }) + } +} diff --git a/pkg/inclusion/paths.go b/pkg/inclusion/paths.go index 0e787d63a8..2119e17e78 100644 --- a/pkg/inclusion/paths.go +++ b/pkg/inclusion/paths.go @@ -2,8 +2,6 @@ package inclusion import ( "math" - - "github.com/celestiaorg/celestia-app/pkg/shares" ) type path struct { @@ -14,7 +12,7 @@ type path struct { // calculateCommitmentPaths calculates all of the paths to subtree roots needed to // create the commitment for a given blob. func calculateCommitmentPaths(squareSize, start, blobShareLen, subtreeRootThreshold int) []path { - start = shares.NextShareIndex(start, blobShareLen, subtreeRootThreshold) + start = NextShareIndex(start, blobShareLen, subtreeRootThreshold) startRow, endRow := start/squareSize, (start+blobShareLen-1)/squareSize normalizedStartIndex := start % squareSize normalizedEndIndex := (start + blobShareLen) - endRow*squareSize @@ -32,7 +30,7 @@ func calculateCommitmentPaths(squareSize, start, blobShareLen, subtreeRootThresh // subTreeRootMaxDepth is the maximum depth of a subtree root that was // used to generate the commitment. The height is based on the // SubtreeRootThreshold. See ADR-013 for more details. - subTreeRootMaxDepth := int(math.Log2(float64(shares.SubTreeWidth(blobShareLen, subtreeRootThreshold)))) + subTreeRootMaxDepth := int(math.Log2(float64(SubTreeWidth(blobShareLen, subtreeRootThreshold)))) minDepth := maxDepth - subTreeRootMaxDepth coords := calculateSubTreeRootCoordinates(maxDepth, minDepth, start, end) for _, c := range coords { diff --git a/pkg/square/builder.go b/pkg/square/builder.go index 02c912bb1b..57ba035544 100644 --- a/pkg/square/builder.go +++ b/pkg/square/builder.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/tendermint/tendermint/pkg/consts" @@ -125,7 +126,7 @@ func (b *Builder) Export() (Square, error) { // calculate the square size. // NOTE: A future optimization could be to recalculate the currentSize based on the actual // interblob padding used when the blobs are correctly ordered instead of using worst case padding. - ss := shares.BlobMinSquareSize(b.currentSize) + ss := inclusion.BlobMinSquareSize(b.currentSize) // Sort the blobs by namespace. This uses SliceStable to preserve the order // of blobs within a namespace because b.Blobs are already ordered by tx @@ -150,7 +151,7 @@ func (b *Builder) Export() (Square, error) { for i, element := range b.Blobs { // NextShareIndex returned where the next blob should start so as to comply with the share commitment rules // We fill out the remaining - cursor = shares.NextShareIndex(cursor, element.NumShares, b.subtreeRootThreshold) + cursor = inclusion.NextShareIndex(cursor, element.NumShares, b.subtreeRootThreshold) if i == 0 { nonReservedStart = cursor } @@ -400,7 +401,7 @@ func newElement(blob *blob.Blob, pfbIndex, blobIndex, subtreeRootThreshold int) // // Note that the padding would actually belong to the namespace of the transaction before it, but // this makes no difference to the total share size. - MaxPadding: shares.SubTreeWidth(numShares, subtreeRootThreshold) - 1, + MaxPadding: inclusion.SubTreeWidth(numShares, subtreeRootThreshold) - 1, } } diff --git a/pkg/square/square.go b/pkg/square/square.go index ccdba287dd..85a4e47359 100644 --- a/pkg/square/square.go +++ b/pkg/square/square.go @@ -3,10 +3,10 @@ package square import ( "bytes" "fmt" - "math" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" @@ -195,8 +195,12 @@ func (s Square) Size() int { return Size(len(s)) } +// Size returns the size of the row or column in shares of a square. This +// function is currently a wrapper around the da packages equivalent function to +// avoid breaking the api. In future versions there will not be a copy of this +// code here. func Size(len int) int { - return shares.RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(len))))) + return da.SquareSize(len) } // Equals returns true if two squares are equal @@ -227,7 +231,7 @@ func (s Square) IsEmpty() bool { // EmptySquare returns a 1x1 square with a single tail padding share func EmptySquare() Square { - return shares.TailPaddingShares(appconsts.MinShareCount) + return da.EmptySquareShares() } func WriteSquare( diff --git a/test/util/malicious/out_of_order_builder.go b/test/util/malicious/out_of_order_builder.go index 0fb22988f0..9c3e8c6428 100644 --- a/test/util/malicious/out_of_order_builder.go +++ b/test/util/malicious/out_of_order_builder.go @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/square" @@ -67,7 +68,7 @@ func OutOfOrderExport(b *square.Builder) (square.Square, error) { // calculate the square size. // NOTE: A future optimization could be to recalculate the currentSize based on the actual // interblob padding used when the blobs are correctly ordered instead of using worst case padding. - ss := shares.BlobMinSquareSize(b.CurrentSize()) + ss := inclusion.BlobMinSquareSize(b.CurrentSize()) // sort the blobs in order of namespace. We use slice stable here to respect the // order of multiple blobs within a namespace as per the priority of the PFB @@ -102,7 +103,7 @@ func OutOfOrderExport(b *square.Builder) (square.Square, error) { for i, element := range b.Blobs { // NextShareIndex returned where the next blob should start so as to comply with the share commitment rules // We fill out the remaining - cursor = shares.NextShareIndex(cursor, element.NumShares, b.SubtreeRootThreshold()) + cursor = inclusion.NextShareIndex(cursor, element.NumShares, b.SubtreeRootThreshold()) if i == 0 { nonReservedStart = cursor } diff --git a/x/blob/types/blob_tx.go b/x/blob/types/blob_tx.go index 1ebf9b15fc..b6a433304e 100644 --- a/x/blob/types/blob_tx.go +++ b/x/blob/types/blob_tx.go @@ -4,6 +4,7 @@ import ( "bytes" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" appns "github.com/celestiaorg/celestia-app/pkg/namespace" shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/cosmos/cosmos-sdk/client" @@ -89,7 +90,7 @@ func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx blob.BlobTx) error { // verify that the commitment of the blob matches that of the msgPFB for i, commitment := range msgPFB.ShareCommitments { - calculatedCommit, err := CreateCommitment(bTx.Blobs[i]) + calculatedCommit, err := inclusion.CreateCommitment(bTx.Blobs[i]) if err != nil { return ErrCalculateCommitment } diff --git a/x/blob/types/blob_tx_test.go b/x/blob/types/blob_tx_test.go index de43cbf53f..ec066c3392 100644 --- a/x/blob/types/blob_tx_test.go +++ b/x/blob/types/blob_tx_test.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/test/util/blobfactory" "github.com/celestiaorg/celestia-app/test/util/testnode" @@ -110,7 +111,7 @@ func TestValidateBlobTx(t *testing.T) { ) require.NoError(t, err) - badCommit, err := types.CreateCommitment( + badCommit, err := inclusion.CreateCommitment( &blob.Blob{ NamespaceVersion: uint32(namespace.RandomBlobNamespace().Version), NamespaceId: namespace.RandomBlobNamespace().ID, diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 44727da70b..edd971f207 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -1,20 +1,18 @@ package types import ( - "crypto/sha256" fmt "fmt" "cosmossdk.io/errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" appns "github.com/celestiaorg/celestia-app/pkg/namespace" appshares "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/nmt" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx" auth "github.com/cosmos/cosmos-sdk/x/auth/types" - "github.com/tendermint/tendermint/crypto/merkle" "golang.org/x/exp/slices" ) @@ -51,7 +49,7 @@ func NewMsgPayForBlobs(signer string, blobs ...*blob.Blob) (*MsgPayForBlobs, err if err != nil { return nil, err } - commitments, err := CreateCommitments(blobs) + commitments, err := inclusion.CreateCommitments(blobs) if err != nil { return nil, err } @@ -209,81 +207,6 @@ func (msg *MsgPayForBlobs) GetSigners() []sdk.AccAddress { return []sdk.AccAddress{address} } -// CreateCommitment generates the share commitment for a given blob. -// See [data square layout rationale] and [blob share commitment rules]. -// -// [data square layout rationale]: ../../specs/src/specs/data_square_layout.md -// [blob share commitment rules]: ../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules -func CreateCommitment(blob *blob.Blob) ([]byte, error) { - if err := blob.Validate(); err != nil { - return nil, err - } - namespace := blob.Namespace() - - shares, err := appshares.SplitBlobs(blob) - if err != nil { - return nil, err - } - - // the commitment is the root of a merkle mountain range with max tree size - // determined by the number of roots required to create a share commitment - // over that blob. The size of the tree is only increased if the number of - // subtree roots surpasses a constant threshold. - subTreeWidth := appshares.SubTreeWidth(len(shares), appconsts.DefaultSubtreeRootThreshold) - treeSizes, err := MerkleMountainRangeSizes(uint64(len(shares)), uint64(subTreeWidth)) - if err != nil { - return nil, err - } - leafSets := make([][][]byte, len(treeSizes)) - cursor := uint64(0) - for i, treeSize := range treeSizes { - leafSets[i] = appshares.ToBytes(shares[cursor : cursor+treeSize]) - cursor = cursor + treeSize - } - - // create the commitments by pushing each leaf set onto an nmt - subTreeRoots := make([][]byte, len(leafSets)) - for i, set := range leafSets { - // create the nmt todo(evan) use nmt wrapper - tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(appns.NamespaceSize), nmt.IgnoreMaxNamespace(true)) - for _, leaf := range set { - // the namespace must be added again here even though it is already - // included in the leaf to ensure that the hash will match that of - // the nmt wrapper (pkg/wrapper). Each namespace is added to keep - // the namespace in the share, and therefore the parity data, while - // also allowing for the manual addition of the parity namespace to - // the parity data. - nsLeaf := make([]byte, 0) - nsLeaf = append(nsLeaf, namespace.Bytes()...) - nsLeaf = append(nsLeaf, leaf...) - - err = tree.Push(nsLeaf) - if err != nil { - return nil, err - } - } - // add the root - root, err := tree.Root() - if err != nil { - return nil, err - } - subTreeRoots[i] = root - } - return merkle.HashFromByteSlices(subTreeRoots), nil -} - -func CreateCommitments(blobs []*blob.Blob) ([][]byte, error) { - commitments := make([][]byte, len(blobs)) - for i, blob := range blobs { - commitment, err := CreateCommitment(blob) - if err != nil { - return nil, err - } - commitments[i] = commitment - } - return commitments, nil -} - // ValidateBlobs performs basic checks over the components of one or more PFBs. func ValidateBlobs(blobs ...*blob.Blob) error { if len(blobs) == 0 { @@ -332,30 +255,3 @@ func ExtractBlobComponents(pblobs []*blob.Blob) (namespaceVersions []uint32, nam return namespaceVersions, namespaceIds, sizes, shareVersions } - -// MerkleMountainRangeSizes returns the sizes (number of leaf nodes) of the -// trees in a merkle mountain range constructed for a given totalSize and -// maxTreeSize. -// -// https://docs.grin.mw/wiki/chain-state/merkle-mountain-range/ -// https://github.com/opentimestamps/opentimestamps-server/blob/master/doc/merkle-mountain-range.md -func MerkleMountainRangeSizes(totalSize, maxTreeSize uint64) ([]uint64, error) { - var treeSizes []uint64 - - for totalSize != 0 { - switch { - case totalSize >= maxTreeSize: - treeSizes = append(treeSizes, maxTreeSize) - totalSize = totalSize - maxTreeSize - case totalSize < maxTreeSize: - treeSize, err := appshares.RoundDownPowerOfTwo(totalSize) - if err != nil { - return treeSizes, err - } - treeSizes = append(treeSizes, treeSize) - totalSize = totalSize - treeSize - } - } - - return treeSizes, nil -} diff --git a/x/blob/types/payforblob_test.go b/x/blob/types/payforblob_test.go index a367ead573..5709a4a316 100644 --- a/x/blob/types/payforblob_test.go +++ b/x/blob/types/payforblob_test.go @@ -7,6 +7,7 @@ import ( sdkerrors "cosmossdk.io/errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" appns "github.com/celestiaorg/celestia-app/pkg/namespace" shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/test/util/testfactory" @@ -18,101 +19,6 @@ import ( tmrand "github.com/tendermint/tendermint/libs/rand" ) -func Test_MerkleMountainRangeHeights(t *testing.T) { - type test struct { - totalSize uint64 - squareSize uint64 - expected []uint64 - } - tests := []test{ - { - totalSize: 11, - squareSize: 4, - expected: []uint64{4, 4, 2, 1}, - }, - { - totalSize: 2, - squareSize: 64, - expected: []uint64{2}, - }, - { - totalSize: 64, - squareSize: 8, - expected: []uint64{8, 8, 8, 8, 8, 8, 8, 8}, - }, - // Height - // 3 x x - // / \ / \ - // / \ / \ - // / \ / \ - // / \ / \ - // 2 x x x x - // / \ / \ / \ / \ - // 1 x x x x x x x x x - // / \ / \ / \ / \ / \ / \ / \ / \ / \ - // 0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 - { - totalSize: 19, - squareSize: 8, - expected: []uint64{8, 8, 2, 1}, - }, - } - for _, tt := range tests { - res, err := types.MerkleMountainRangeSizes(tt.totalSize, tt.squareSize) - require.NoError(t, err) - assert.Equal(t, tt.expected, res) - } -} - -// TestCreateCommitment will fail if a change is made to share encoding or how -// the commitment is calculated. If this is the case, the expected commitment -// bytes will need to be updated. -func TestCreateCommitment(t *testing.T) { - ns1 := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) - - type test struct { - name string - namespace appns.Namespace - blob []byte - expected []byte - expectErr bool - shareVersion uint8 - } - tests := []test{ - { - name: "blob of 3 shares succeeds", - namespace: ns1, - blob: bytes.Repeat([]byte{0xFF}, 3*appconsts.ShareSize), - expected: []byte{0x3b, 0x9e, 0x78, 0xb6, 0x64, 0x8e, 0xc1, 0xa2, 0x41, 0x92, 0x5b, 0x31, 0xda, 0x2e, 0xcb, 0x50, 0xbf, 0xc6, 0xf4, 0xad, 0x55, 0x2d, 0x32, 0x79, 0x92, 0x8c, 0xa1, 0x3e, 0xbe, 0xba, 0x8c, 0x2b}, - shareVersion: appconsts.ShareVersionZero, - }, - { - name: "blob with unsupported share version should return error", - namespace: ns1, - blob: bytes.Repeat([]byte{0xFF}, 12*appconsts.ShareSize), - expectErr: true, - shareVersion: uint8(1), // unsupported share version - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - blob := &blob.Blob{ - NamespaceId: tt.namespace.ID, - Data: tt.blob, - ShareVersion: uint32(tt.shareVersion), - NamespaceVersion: uint32(tt.namespace.Version), - } - res, err := types.CreateCommitment(blob) - if tt.expectErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, tt.expected, res) - }) - } -} - func TestMsgTypeURLParity(t *testing.T) { require.Equal(t, sdk.MsgTypeURL(&types.MsgPayForBlobs{}), types.URLMsgPayForBlobs) } @@ -294,7 +200,7 @@ func invalidNamespaceVersionMsgPayForBlobs(t *testing.T) *types.MsgPayForBlobs { blobs := []*blob.Blob{pblob} - commitments, err := types.CreateCommitments(blobs) + commitments, err := inclusion.CreateCommitments(blobs) require.NoError(t, err) namespaceVersions, namespaceIds, sizes, shareVersions := types.ExtractBlobComponents(blobs) @@ -432,7 +338,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { assert.Equal(t, ns.ID, blob.NamespaceId) assert.Equal(t, uint32(ns.Version), blob.NamespaceVersion) - expectedCommitment, err := types.CreateCommitment(blob) + expectedCommitment, err := inclusion.CreateCommitment(blob) require.NoError(t, err) assert.Equal(t, expectedCommitment, msgPFB.ShareCommitments[i]) }