diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 797472a250..64ffbe921f 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -47,10 +47,21 @@ jobs: version: latest args: check + # branch_name trims ref/heads/ from github.ref to access a clean branch name + branch_name: + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.trim_ref.outputs.branch }} + steps: + - name: Trim branch name + id: trim_ref + run: | + echo "branch=$(${${{ github.ref }}:11})" >> $GITHUB_OUTPUT + # If this was a workflow dispatch event, we need to generate and push a tag # for goreleaser to grab version_bump: - needs: [lint, test] + needs: [lint, test, branch_name, goreleaser-check] runs-on: ubuntu-latest permissions: "write-all" steps: @@ -66,6 +77,9 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.version }} + # Setting the branch name so that release branch other than + # master/main doesn't impact tag name + release_branches: ${{ needs.branch_name.outputs.branch }} # Generate the release with goreleaser to include pre-built binaries goreleaser: diff --git a/pkg/da/data_availability_header.go b/pkg/da/data_availability_header.go index bff5b3cf27..7bc7c944ce 100644 --- a/pkg/da/data_availability_header.go +++ b/pkg/da/data_availability_header.go @@ -4,14 +4,15 @@ import ( "bytes" "errors" "fmt" + "math" "github.com/celestiaorg/rsmt2d" "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/types" + "golang.org/x/exp/constraints" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/celestia-app/pkg/square" "github.com/celestiaorg/celestia-app/pkg/wrapper" daproto "github.com/celestiaorg/celestia-app/proto/celestia/core/v1/da" ) @@ -66,7 +67,7 @@ func ExtendShares(s [][]byte) (*rsmt2d.ExtendedDataSquare, error) { if !shares.IsPowerOfTwo(len(s)) { return nil, fmt.Errorf("number of shares is not a power of 2: got %d", len(s)) } - squareSize := square.Size(len(s)) + squareSize := SquareSize(len(s)) // here we construct a tree // Note: uses the nmt wrapper to construct the tree. @@ -190,5 +191,26 @@ func MinDataAvailabilityHeader() DataAvailabilityHeader { // MinShares returns one tail-padded share. func MinShares() [][]byte { - return shares.ToBytes(square.EmptySquare()) + return shares.ToBytes(EmptySquareShares()) +} + +// EmptySquare is a copy of the function defined in the square package to avoid +// a circular dependency. TODO deduplicate +func EmptySquareShares() []shares.Share { + return shares.TailPaddingShares(appconsts.MinShareCount) +} + +// SquareSize is a copy of the function defined in the square package to avoid +// a circular dependency. TODO deduplicate +func SquareSize(len int) int { + return RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(len))))) +} + +// RoundUpPowerOfTwo returns the next power of two greater than or equal to input. +func RoundUpPowerOfTwo[I constraints.Integer](input I) I { + var result I = 1 + for result < input { + result = result << 1 + } + return result } diff --git a/pkg/shares/blob_share_commitment_rules.go b/pkg/inclusion/blob_share_commitment_rules.go similarity index 95% rename from pkg/shares/blob_share_commitment_rules.go rename to pkg/inclusion/blob_share_commitment_rules.go index 8c9ed6a3ee..d7b1809791 100644 --- a/pkg/shares/blob_share_commitment_rules.go +++ b/pkg/inclusion/blob_share_commitment_rules.go @@ -1,8 +1,9 @@ -package shares +package inclusion import ( "math" + "github.com/celestiaorg/celestia-app/pkg/da" "golang.org/x/exp/constraints" ) @@ -73,7 +74,7 @@ func roundUpByMultipleOf(cursor, v int) int { // BlobMinSquareSize returns the minimum square size that can contain shareCount // number of shares. func BlobMinSquareSize(shareCount int) int { - return RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(shareCount))))) + return da.RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(shareCount))))) } // SubTreeWidth determines the maximum number of leaves per subtree in the share @@ -93,7 +94,7 @@ func SubTreeWidth(shareCount, subtreeRootThreshold int) int { // use a power of two equal to or larger than the multiple of the subtree // root threshold - s = RoundUpPowerOfTwo(s) + s = da.RoundUpPowerOfTwo(s) // use the minimum of the subtree width and the min square size, this // gurarantees that a valid value is returned diff --git a/pkg/shares/blob_share_commitment_rules_test.go b/pkg/inclusion/blob_share_commitment_rules_test.go similarity index 98% rename from pkg/shares/blob_share_commitment_rules_test.go rename to pkg/inclusion/blob_share_commitment_rules_test.go index 466aade1cd..8b6f40f237 100644 --- a/pkg/shares/blob_share_commitment_rules_test.go +++ b/pkg/inclusion/blob_share_commitment_rules_test.go @@ -1,10 +1,11 @@ -package shares +package inclusion import ( "fmt" "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/stretchr/testify/assert" ) @@ -232,7 +233,7 @@ func TestNextShareIndex(t *testing.T) { name: "at threshold", cursor: 11, blobLen: appconsts.DefaultSubtreeRootThreshold, - squareSize: RoundUpPowerOfTwo(appconsts.DefaultSubtreeRootThreshold), + squareSize: da.RoundUpPowerOfTwo(appconsts.DefaultSubtreeRootThreshold), expectedIndex: 11, }, { diff --git a/pkg/inclusion/commitment.go b/pkg/inclusion/commitment.go new file mode 100644 index 0000000000..8492e340bd --- /dev/null +++ b/pkg/inclusion/commitment.go @@ -0,0 +1,114 @@ +package inclusion + +import ( + "crypto/sha256" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + appshares "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/nmt" + "github.com/tendermint/tendermint/crypto/merkle" +) + +// CreateCommitment generates the share commitment for a given blob. +// See [data square layout rationale] and [blob share commitment rules]. +// +// [data square layout rationale]: ../../specs/src/specs/data_square_layout.md +// [blob share commitment rules]: ../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules +func CreateCommitment(blob *blob.Blob) ([]byte, error) { + if err := blob.Validate(); err != nil { + return nil, err + } + namespace := blob.Namespace() + + shares, err := appshares.SplitBlobs(blob) + if err != nil { + return nil, err + } + + // the commitment is the root of a merkle mountain range with max tree size + // determined by the number of roots required to create a share commitment + // over that blob. The size of the tree is only increased if the number of + // subtree roots surpasses a constant threshold. + subTreeWidth := SubTreeWidth(len(shares), appconsts.DefaultSubtreeRootThreshold) + treeSizes, err := MerkleMountainRangeSizes(uint64(len(shares)), uint64(subTreeWidth)) + if err != nil { + return nil, err + } + leafSets := make([][][]byte, len(treeSizes)) + cursor := uint64(0) + for i, treeSize := range treeSizes { + leafSets[i] = appshares.ToBytes(shares[cursor : cursor+treeSize]) + cursor = cursor + treeSize + } + + // create the commitments by pushing each leaf set onto an nmt + subTreeRoots := make([][]byte, len(leafSets)) + for i, set := range leafSets { + // create the nmt todo(evan) use nmt wrapper + tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(appns.NamespaceSize), nmt.IgnoreMaxNamespace(true)) + for _, leaf := range set { + // the namespace must be added again here even though it is already + // included in the leaf to ensure that the hash will match that of + // the nmt wrapper (pkg/wrapper). Each namespace is added to keep + // the namespace in the share, and therefore the parity data, while + // also allowing for the manual addition of the parity namespace to + // the parity data. + nsLeaf := make([]byte, 0) + nsLeaf = append(nsLeaf, namespace.Bytes()...) + nsLeaf = append(nsLeaf, leaf...) + + err = tree.Push(nsLeaf) + if err != nil { + return nil, err + } + } + // add the root + root, err := tree.Root() + if err != nil { + return nil, err + } + subTreeRoots[i] = root + } + return merkle.HashFromByteSlices(subTreeRoots), nil +} + +func CreateCommitments(blobs []*blob.Blob) ([][]byte, error) { + commitments := make([][]byte, len(blobs)) + for i, blob := range blobs { + commitment, err := CreateCommitment(blob) + if err != nil { + return nil, err + } + commitments[i] = commitment + } + return commitments, nil +} + +// MerkleMountainRangeSizes returns the sizes (number of leaf nodes) of the +// trees in a merkle mountain range constructed for a given totalSize and +// maxTreeSize. +// +// https://docs.grin.mw/wiki/chain-state/merkle-mountain-range/ +// https://github.com/opentimestamps/opentimestamps-server/blob/master/doc/merkle-mountain-range.md +func MerkleMountainRangeSizes(totalSize, maxTreeSize uint64) ([]uint64, error) { + var treeSizes []uint64 + + for totalSize != 0 { + switch { + case totalSize >= maxTreeSize: + treeSizes = append(treeSizes, maxTreeSize) + totalSize = totalSize - maxTreeSize + case totalSize < maxTreeSize: + treeSize, err := appshares.RoundDownPowerOfTwo(totalSize) + if err != nil { + return treeSizes, err + } + treeSizes = append(treeSizes, treeSize) + totalSize = totalSize - treeSize + } + } + + return treeSizes, nil +} diff --git a/pkg/inclusion/commitment_test.go b/pkg/inclusion/commitment_test.go new file mode 100644 index 0000000000..68e087e1a1 --- /dev/null +++ b/pkg/inclusion/commitment_test.go @@ -0,0 +1,108 @@ +package inclusion_test + +import ( + "bytes" + "testing" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_MerkleMountainRangeHeights(t *testing.T) { + type test struct { + totalSize uint64 + squareSize uint64 + expected []uint64 + } + tests := []test{ + { + totalSize: 11, + squareSize: 4, + expected: []uint64{4, 4, 2, 1}, + }, + { + totalSize: 2, + squareSize: 64, + expected: []uint64{2}, + }, + { + totalSize: 64, + squareSize: 8, + expected: []uint64{8, 8, 8, 8, 8, 8, 8, 8}, + }, + // Height + // 3 x x + // / \ / \ + // / \ / \ + // / \ / \ + // / \ / \ + // 2 x x x x + // / \ / \ / \ / \ + // 1 x x x x x x x x x + // / \ / \ / \ / \ / \ / \ / \ / \ / \ + // 0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 + { + totalSize: 19, + squareSize: 8, + expected: []uint64{8, 8, 2, 1}, + }, + } + for _, tt := range tests { + res, err := inclusion.MerkleMountainRangeSizes(tt.totalSize, tt.squareSize) + require.NoError(t, err) + assert.Equal(t, tt.expected, res) + } +} + +// TestCreateCommitment will fail if a change is made to share encoding or how +// the commitment is calculated. If this is the case, the expected commitment +// bytes will need to be updated. +func TestCreateCommitment(t *testing.T) { + ns1 := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) + + type test struct { + name string + namespace appns.Namespace + blob []byte + expected []byte + expectErr bool + shareVersion uint8 + } + tests := []test{ + { + name: "blob of 3 shares succeeds", + namespace: ns1, + blob: bytes.Repeat([]byte{0xFF}, 3*appconsts.ShareSize), + expected: []byte{0x3b, 0x9e, 0x78, 0xb6, 0x64, 0x8e, 0xc1, 0xa2, 0x41, 0x92, 0x5b, 0x31, 0xda, 0x2e, 0xcb, 0x50, 0xbf, 0xc6, 0xf4, 0xad, 0x55, 0x2d, 0x32, 0x79, 0x92, 0x8c, 0xa1, 0x3e, 0xbe, 0xba, 0x8c, 0x2b}, + shareVersion: appconsts.ShareVersionZero, + }, + { + name: "blob with unsupported share version should return error", + namespace: ns1, + blob: bytes.Repeat([]byte{0xFF}, 12*appconsts.ShareSize), + expectErr: true, + shareVersion: uint8(1), // unsupported share version + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + blob := &blob.Blob{ + NamespaceId: tt.namespace.ID, + Data: tt.blob, + ShareVersion: uint32(tt.shareVersion), + NamespaceVersion: uint32(tt.namespace.Version), + } + res, err := inclusion.CreateCommitment(blob) + if tt.expectErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expected, res) + }) + } +} diff --git a/pkg/inclusion/paths.go b/pkg/inclusion/paths.go index 0e787d63a8..2119e17e78 100644 --- a/pkg/inclusion/paths.go +++ b/pkg/inclusion/paths.go @@ -2,8 +2,6 @@ package inclusion import ( "math" - - "github.com/celestiaorg/celestia-app/pkg/shares" ) type path struct { @@ -14,7 +12,7 @@ type path struct { // calculateCommitmentPaths calculates all of the paths to subtree roots needed to // create the commitment for a given blob. func calculateCommitmentPaths(squareSize, start, blobShareLen, subtreeRootThreshold int) []path { - start = shares.NextShareIndex(start, blobShareLen, subtreeRootThreshold) + start = NextShareIndex(start, blobShareLen, subtreeRootThreshold) startRow, endRow := start/squareSize, (start+blobShareLen-1)/squareSize normalizedStartIndex := start % squareSize normalizedEndIndex := (start + blobShareLen) - endRow*squareSize @@ -32,7 +30,7 @@ func calculateCommitmentPaths(squareSize, start, blobShareLen, subtreeRootThresh // subTreeRootMaxDepth is the maximum depth of a subtree root that was // used to generate the commitment. The height is based on the // SubtreeRootThreshold. See ADR-013 for more details. - subTreeRootMaxDepth := int(math.Log2(float64(shares.SubTreeWidth(blobShareLen, subtreeRootThreshold)))) + subTreeRootMaxDepth := int(math.Log2(float64(SubTreeWidth(blobShareLen, subtreeRootThreshold)))) minDepth := maxDepth - subTreeRootMaxDepth coords := calculateSubTreeRootCoordinates(maxDepth, minDepth, start, end) for _, c := range coords { diff --git a/pkg/square/builder.go b/pkg/square/builder.go index 02c912bb1b..57ba035544 100644 --- a/pkg/square/builder.go +++ b/pkg/square/builder.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/tendermint/tendermint/pkg/consts" @@ -125,7 +126,7 @@ func (b *Builder) Export() (Square, error) { // calculate the square size. // NOTE: A future optimization could be to recalculate the currentSize based on the actual // interblob padding used when the blobs are correctly ordered instead of using worst case padding. - ss := shares.BlobMinSquareSize(b.currentSize) + ss := inclusion.BlobMinSquareSize(b.currentSize) // Sort the blobs by namespace. This uses SliceStable to preserve the order // of blobs within a namespace because b.Blobs are already ordered by tx @@ -150,7 +151,7 @@ func (b *Builder) Export() (Square, error) { for i, element := range b.Blobs { // NextShareIndex returned where the next blob should start so as to comply with the share commitment rules // We fill out the remaining - cursor = shares.NextShareIndex(cursor, element.NumShares, b.subtreeRootThreshold) + cursor = inclusion.NextShareIndex(cursor, element.NumShares, b.subtreeRootThreshold) if i == 0 { nonReservedStart = cursor } @@ -400,7 +401,7 @@ func newElement(blob *blob.Blob, pfbIndex, blobIndex, subtreeRootThreshold int) // // Note that the padding would actually belong to the namespace of the transaction before it, but // this makes no difference to the total share size. - MaxPadding: shares.SubTreeWidth(numShares, subtreeRootThreshold) - 1, + MaxPadding: inclusion.SubTreeWidth(numShares, subtreeRootThreshold) - 1, } } diff --git a/pkg/square/square.go b/pkg/square/square.go index ccdba287dd..85a4e47359 100644 --- a/pkg/square/square.go +++ b/pkg/square/square.go @@ -3,10 +3,10 @@ package square import ( "bytes" "fmt" - "math" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" @@ -195,8 +195,12 @@ func (s Square) Size() int { return Size(len(s)) } +// Size returns the size of the row or column in shares of a square. This +// function is currently a wrapper around the da packages equivalent function to +// avoid breaking the api. In future versions there will not be a copy of this +// code here. func Size(len int) int { - return shares.RoundUpPowerOfTwo(int(math.Ceil(math.Sqrt(float64(len))))) + return da.SquareSize(len) } // Equals returns true if two squares are equal @@ -227,7 +231,7 @@ func (s Square) IsEmpty() bool { // EmptySquare returns a 1x1 square with a single tail padding share func EmptySquare() Square { - return shares.TailPaddingShares(appconsts.MinShareCount) + return da.EmptySquareShares() } func WriteSquare( diff --git a/test/util/malicious/out_of_order_builder.go b/test/util/malicious/out_of_order_builder.go index 0fb22988f0..9c3e8c6428 100644 --- a/test/util/malicious/out_of_order_builder.go +++ b/test/util/malicious/out_of_order_builder.go @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/pkg/square" @@ -67,7 +68,7 @@ func OutOfOrderExport(b *square.Builder) (square.Square, error) { // calculate the square size. // NOTE: A future optimization could be to recalculate the currentSize based on the actual // interblob padding used when the blobs are correctly ordered instead of using worst case padding. - ss := shares.BlobMinSquareSize(b.CurrentSize()) + ss := inclusion.BlobMinSquareSize(b.CurrentSize()) // sort the blobs in order of namespace. We use slice stable here to respect the // order of multiple blobs within a namespace as per the priority of the PFB @@ -102,7 +103,7 @@ func OutOfOrderExport(b *square.Builder) (square.Square, error) { for i, element := range b.Blobs { // NextShareIndex returned where the next blob should start so as to comply with the share commitment rules // We fill out the remaining - cursor = shares.NextShareIndex(cursor, element.NumShares, b.SubtreeRootThreshold()) + cursor = inclusion.NextShareIndex(cursor, element.NumShares, b.SubtreeRootThreshold()) if i == 0 { nonReservedStart = cursor } diff --git a/x/blob/types/blob_tx.go b/x/blob/types/blob_tx.go index 1ebf9b15fc..b6a433304e 100644 --- a/x/blob/types/blob_tx.go +++ b/x/blob/types/blob_tx.go @@ -4,6 +4,7 @@ import ( "bytes" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" appns "github.com/celestiaorg/celestia-app/pkg/namespace" shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/cosmos/cosmos-sdk/client" @@ -89,7 +90,7 @@ func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx blob.BlobTx) error { // verify that the commitment of the blob matches that of the msgPFB for i, commitment := range msgPFB.ShareCommitments { - calculatedCommit, err := CreateCommitment(bTx.Blobs[i]) + calculatedCommit, err := inclusion.CreateCommitment(bTx.Blobs[i]) if err != nil { return ErrCalculateCommitment } diff --git a/x/blob/types/blob_tx_test.go b/x/blob/types/blob_tx_test.go index de43cbf53f..ec066c3392 100644 --- a/x/blob/types/blob_tx_test.go +++ b/x/blob/types/blob_tx_test.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/app/encoding" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/test/util/blobfactory" "github.com/celestiaorg/celestia-app/test/util/testnode" @@ -110,7 +111,7 @@ func TestValidateBlobTx(t *testing.T) { ) require.NoError(t, err) - badCommit, err := types.CreateCommitment( + badCommit, err := inclusion.CreateCommitment( &blob.Blob{ NamespaceVersion: uint32(namespace.RandomBlobNamespace().Version), NamespaceId: namespace.RandomBlobNamespace().ID, diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 44727da70b..edd971f207 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -1,20 +1,18 @@ package types import ( - "crypto/sha256" fmt "fmt" "cosmossdk.io/errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" appns "github.com/celestiaorg/celestia-app/pkg/namespace" appshares "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/nmt" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx" auth "github.com/cosmos/cosmos-sdk/x/auth/types" - "github.com/tendermint/tendermint/crypto/merkle" "golang.org/x/exp/slices" ) @@ -51,7 +49,7 @@ func NewMsgPayForBlobs(signer string, blobs ...*blob.Blob) (*MsgPayForBlobs, err if err != nil { return nil, err } - commitments, err := CreateCommitments(blobs) + commitments, err := inclusion.CreateCommitments(blobs) if err != nil { return nil, err } @@ -209,81 +207,6 @@ func (msg *MsgPayForBlobs) GetSigners() []sdk.AccAddress { return []sdk.AccAddress{address} } -// CreateCommitment generates the share commitment for a given blob. -// See [data square layout rationale] and [blob share commitment rules]. -// -// [data square layout rationale]: ../../specs/src/specs/data_square_layout.md -// [blob share commitment rules]: ../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules -func CreateCommitment(blob *blob.Blob) ([]byte, error) { - if err := blob.Validate(); err != nil { - return nil, err - } - namespace := blob.Namespace() - - shares, err := appshares.SplitBlobs(blob) - if err != nil { - return nil, err - } - - // the commitment is the root of a merkle mountain range with max tree size - // determined by the number of roots required to create a share commitment - // over that blob. The size of the tree is only increased if the number of - // subtree roots surpasses a constant threshold. - subTreeWidth := appshares.SubTreeWidth(len(shares), appconsts.DefaultSubtreeRootThreshold) - treeSizes, err := MerkleMountainRangeSizes(uint64(len(shares)), uint64(subTreeWidth)) - if err != nil { - return nil, err - } - leafSets := make([][][]byte, len(treeSizes)) - cursor := uint64(0) - for i, treeSize := range treeSizes { - leafSets[i] = appshares.ToBytes(shares[cursor : cursor+treeSize]) - cursor = cursor + treeSize - } - - // create the commitments by pushing each leaf set onto an nmt - subTreeRoots := make([][]byte, len(leafSets)) - for i, set := range leafSets { - // create the nmt todo(evan) use nmt wrapper - tree := nmt.New(sha256.New(), nmt.NamespaceIDSize(appns.NamespaceSize), nmt.IgnoreMaxNamespace(true)) - for _, leaf := range set { - // the namespace must be added again here even though it is already - // included in the leaf to ensure that the hash will match that of - // the nmt wrapper (pkg/wrapper). Each namespace is added to keep - // the namespace in the share, and therefore the parity data, while - // also allowing for the manual addition of the parity namespace to - // the parity data. - nsLeaf := make([]byte, 0) - nsLeaf = append(nsLeaf, namespace.Bytes()...) - nsLeaf = append(nsLeaf, leaf...) - - err = tree.Push(nsLeaf) - if err != nil { - return nil, err - } - } - // add the root - root, err := tree.Root() - if err != nil { - return nil, err - } - subTreeRoots[i] = root - } - return merkle.HashFromByteSlices(subTreeRoots), nil -} - -func CreateCommitments(blobs []*blob.Blob) ([][]byte, error) { - commitments := make([][]byte, len(blobs)) - for i, blob := range blobs { - commitment, err := CreateCommitment(blob) - if err != nil { - return nil, err - } - commitments[i] = commitment - } - return commitments, nil -} - // ValidateBlobs performs basic checks over the components of one or more PFBs. func ValidateBlobs(blobs ...*blob.Blob) error { if len(blobs) == 0 { @@ -332,30 +255,3 @@ func ExtractBlobComponents(pblobs []*blob.Blob) (namespaceVersions []uint32, nam return namespaceVersions, namespaceIds, sizes, shareVersions } - -// MerkleMountainRangeSizes returns the sizes (number of leaf nodes) of the -// trees in a merkle mountain range constructed for a given totalSize and -// maxTreeSize. -// -// https://docs.grin.mw/wiki/chain-state/merkle-mountain-range/ -// https://github.com/opentimestamps/opentimestamps-server/blob/master/doc/merkle-mountain-range.md -func MerkleMountainRangeSizes(totalSize, maxTreeSize uint64) ([]uint64, error) { - var treeSizes []uint64 - - for totalSize != 0 { - switch { - case totalSize >= maxTreeSize: - treeSizes = append(treeSizes, maxTreeSize) - totalSize = totalSize - maxTreeSize - case totalSize < maxTreeSize: - treeSize, err := appshares.RoundDownPowerOfTwo(totalSize) - if err != nil { - return treeSizes, err - } - treeSizes = append(treeSizes, treeSize) - totalSize = totalSize - treeSize - } - } - - return treeSizes, nil -} diff --git a/x/blob/types/payforblob_test.go b/x/blob/types/payforblob_test.go index a367ead573..5709a4a316 100644 --- a/x/blob/types/payforblob_test.go +++ b/x/blob/types/payforblob_test.go @@ -7,6 +7,7 @@ import ( sdkerrors "cosmossdk.io/errors" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/blob" + "github.com/celestiaorg/celestia-app/pkg/inclusion" appns "github.com/celestiaorg/celestia-app/pkg/namespace" shares "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/test/util/testfactory" @@ -18,101 +19,6 @@ import ( tmrand "github.com/tendermint/tendermint/libs/rand" ) -func Test_MerkleMountainRangeHeights(t *testing.T) { - type test struct { - totalSize uint64 - squareSize uint64 - expected []uint64 - } - tests := []test{ - { - totalSize: 11, - squareSize: 4, - expected: []uint64{4, 4, 2, 1}, - }, - { - totalSize: 2, - squareSize: 64, - expected: []uint64{2}, - }, - { - totalSize: 64, - squareSize: 8, - expected: []uint64{8, 8, 8, 8, 8, 8, 8, 8}, - }, - // Height - // 3 x x - // / \ / \ - // / \ / \ - // / \ / \ - // / \ / \ - // 2 x x x x - // / \ / \ / \ / \ - // 1 x x x x x x x x x - // / \ / \ / \ / \ / \ / \ / \ / \ / \ - // 0 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 - { - totalSize: 19, - squareSize: 8, - expected: []uint64{8, 8, 2, 1}, - }, - } - for _, tt := range tests { - res, err := types.MerkleMountainRangeSizes(tt.totalSize, tt.squareSize) - require.NoError(t, err) - assert.Equal(t, tt.expected, res) - } -} - -// TestCreateCommitment will fail if a change is made to share encoding or how -// the commitment is calculated. If this is the case, the expected commitment -// bytes will need to be updated. -func TestCreateCommitment(t *testing.T) { - ns1 := appns.MustNewV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) - - type test struct { - name string - namespace appns.Namespace - blob []byte - expected []byte - expectErr bool - shareVersion uint8 - } - tests := []test{ - { - name: "blob of 3 shares succeeds", - namespace: ns1, - blob: bytes.Repeat([]byte{0xFF}, 3*appconsts.ShareSize), - expected: []byte{0x3b, 0x9e, 0x78, 0xb6, 0x64, 0x8e, 0xc1, 0xa2, 0x41, 0x92, 0x5b, 0x31, 0xda, 0x2e, 0xcb, 0x50, 0xbf, 0xc6, 0xf4, 0xad, 0x55, 0x2d, 0x32, 0x79, 0x92, 0x8c, 0xa1, 0x3e, 0xbe, 0xba, 0x8c, 0x2b}, - shareVersion: appconsts.ShareVersionZero, - }, - { - name: "blob with unsupported share version should return error", - namespace: ns1, - blob: bytes.Repeat([]byte{0xFF}, 12*appconsts.ShareSize), - expectErr: true, - shareVersion: uint8(1), // unsupported share version - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - blob := &blob.Blob{ - NamespaceId: tt.namespace.ID, - Data: tt.blob, - ShareVersion: uint32(tt.shareVersion), - NamespaceVersion: uint32(tt.namespace.Version), - } - res, err := types.CreateCommitment(blob) - if tt.expectErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, tt.expected, res) - }) - } -} - func TestMsgTypeURLParity(t *testing.T) { require.Equal(t, sdk.MsgTypeURL(&types.MsgPayForBlobs{}), types.URLMsgPayForBlobs) } @@ -294,7 +200,7 @@ func invalidNamespaceVersionMsgPayForBlobs(t *testing.T) *types.MsgPayForBlobs { blobs := []*blob.Blob{pblob} - commitments, err := types.CreateCommitments(blobs) + commitments, err := inclusion.CreateCommitments(blobs) require.NoError(t, err) namespaceVersions, namespaceIds, sizes, shareVersions := types.ExtractBlobComponents(blobs) @@ -432,7 +338,7 @@ func TestNewMsgPayForBlobs(t *testing.T) { assert.Equal(t, ns.ID, blob.NamespaceId) assert.Equal(t, uint32(ns.Version), blob.NamespaceVersion) - expectedCommitment, err := types.CreateCommitment(blob) + expectedCommitment, err := inclusion.CreateCommitment(blob) require.NoError(t, err) assert.Equal(t, expectedCommitment, msgPFB.ShareCommitments[i]) }