From 0b447adf776689cc8f14823c6ef925b8f8249b5a Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 3 Feb 2023 07:49:15 -0700 Subject: [PATCH] chore: handle PayForBlobNamespaceID in `merge` (#1298) Closes https://github.com/celestiaorg/celestia-app/issues/1285 ## Description 1. Handle PayForBlobNamespaceID in `merge` 1. Extract `merge` to a new file 1. Extract `merge` to a new file 1. Add a unit test that uses sample block data --- pkg/shares/merge.go | 72 +++++++++ pkg/shares/merge_test.go | 110 ++++++++++++++ pkg/shares/parse.go | 78 ++++++++++ .../{share_merging_test.go => parse_test.go} | 127 ---------------- .../{share_merging.go => share_sequence.go} | 143 +----------------- pkg/shares/share_sequence_test.go | 137 +++++++++++++++++ pkg/shares/shares_test.go | 90 ----------- pkg/shares/sparse_shares_test.go | 15 ++ pkg/shares/testdata/sample-block.json | 1 + 9 files changed, 421 insertions(+), 352 deletions(-) create mode 100644 pkg/shares/merge.go create mode 100644 pkg/shares/merge_test.go create mode 100644 pkg/shares/parse.go rename pkg/shares/{share_merging_test.go => parse_test.go} (60%) rename pkg/shares/{share_merging.go => share_sequence.go} (51%) create mode 100644 pkg/shares/share_sequence_test.go create mode 100755 pkg/shares/testdata/sample-block.json diff --git a/pkg/shares/merge.go b/pkg/shares/merge.go new file mode 100644 index 0000000000..950e9dddab --- /dev/null +++ b/pkg/shares/merge.go @@ -0,0 +1,72 @@ +package shares + +import ( + "bytes" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/rsmt2d" + coretypes "github.com/tendermint/tendermint/types" +) + +// merge extracts block data from an extended data square. +// TODO: export this function +func merge(eds *rsmt2d.ExtendedDataSquare) (coretypes.Data, error) { + squareSize := eds.Width() / 2 + + // sort block data shares by namespace + var ( + sortedTxShares [][]byte + sortedPfbTxShares [][]byte + sortedBlobShares [][]byte + ) + + // iterate over each row index + for x := uint(0); x < squareSize; x++ { + // iterate over each share in the original data square + row := eds.Row(x) + + for _, share := range row[:squareSize] { + // sort the data of that share types via namespace + nid := share[:appconsts.NamespaceSize] + switch { + case bytes.Equal(appconsts.TxNamespaceID, nid): + sortedTxShares = append(sortedTxShares, share) + case bytes.Equal(appconsts.PayForBlobNamespaceID, nid): + sortedPfbTxShares = append(sortedPfbTxShares, share) + case bytes.Equal(appconsts.TailPaddingNamespaceID, nid): + continue + + // ignore unused but reserved namespaces + case bytes.Compare(nid, appconsts.MaxReservedNamespace) < 1: + continue + + // every other namespaceID should be a blob + default: + sortedBlobShares = append(sortedBlobShares, share) + } + } + } + + // pass the raw share data to their respective parsers + ordinaryTxs, err := ParseTxs(sortedTxShares) + if err != nil { + return coretypes.Data{}, err + } + pfbTxs, err := ParseTxs(sortedPfbTxShares) + if err != nil { + return coretypes.Data{}, err + } + txs := append(ordinaryTxs, pfbTxs...) + + blobs, err := ParseBlobs(sortedBlobShares) + if err != nil { + return coretypes.Data{}, err + } + + // TODO the Data returned below does not have the correct data.hash populated. + return coretypes.Data{ + Txs: txs, + Blobs: blobs, + SquareSize: uint64(squareSize), + }, nil +} diff --git a/pkg/shares/merge_test.go b/pkg/shares/merge_test.go new file mode 100644 index 0000000000..46cdf57a2e --- /dev/null +++ b/pkg/shares/merge_test.go @@ -0,0 +1,110 @@ +package shares + +import ( + "context" + _ "embed" + "encoding/json" + "testing" + "time" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/testutil/testfactory" + "github.com/celestiaorg/rsmt2d" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + coretypes "github.com/tendermint/tendermint/types" +) + +func TestFuzz_merge(t *testing.T) { + t.Skip() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + for { + select { + case <-ctx.Done(): + return + default: + Test_merge_randomData(t) + } + } +} + +func Test_merge_randomData(t *testing.T) { + type test struct { + name string + txCount int + blobCount int + maxSize int // max size of each tx or blob + } + + tests := []test{ + {"one of each random small size", 1, 1, 40}, + {"one of each random large size", 1, 1, 400}, + {"many of each random large size", 10, 10, 40}, + {"many of each random large size", 10, 10, 400}, + {"only transactions", 10, 0, 400}, + {"only blobs", 0, 10, 400}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + data := generateRandomBlockData( + tc.txCount, + tc.blobCount, + tc.maxSize, + ) + + shares, err := Split(data, false) + require.NoError(t, err) + + eds, err := rsmt2d.ComputeExtendedDataSquare(ToBytes(shares), appconsts.DefaultCodec(), rsmt2d.NewDefaultTree) + assert.NoError(t, err) + + got, err := merge(eds) + assert.NoError(t, err) + assert.Equal(t, data, got) + }) + } +} + +func Test_merge_sampleBlock(t *testing.T) { + var pb tmproto.Block + err := json.Unmarshal([]byte(sampleBlock), &pb) + require.NoError(t, err) + + b, err := coretypes.BlockFromProto(&pb) + require.NoError(t, err) + + shares, err := Split(b.Data, false) + require.NoError(t, err) + + eds, err := rsmt2d.ComputeExtendedDataSquare(ToBytes(shares), appconsts.DefaultCodec(), rsmt2d.NewDefaultTree) + assert.NoError(t, err) + + got, err := merge(eds) + assert.NoError(t, err) + + // TODO: the assertions below are a hack because the data returned by merge does + // contain the same hash as the original block. Ideally this test would verify: + // + // assert.Equal(t, b.Data, got) + // + // Instead this test verifies all public fields of Data are identical. + assert.Equal(t, b.Data.Txs, got.Txs) + assert.Equal(t, b.Data.Blobs, got.Blobs) + assert.Equal(t, b.Data.SquareSize, got.SquareSize) +} + +// generateRandomBlockData returns randomly generated block data for testing purposes +func generateRandomBlockData(txCount, blobCount, maxSize int) (data coretypes.Data) { + data.Txs = testfactory.GenerateRandomlySizedTxs(txCount, maxSize) + data.Blobs = testfactory.GenerateRandomlySizedBlobs(blobCount, maxSize) + data.SquareSize = appconsts.DefaultMaxSquareSize + return data +} + +// this is a sample block +// +//go:embed "testdata/sample-block.json" +var sampleBlock string diff --git a/pkg/shares/parse.go b/pkg/shares/parse.go new file mode 100644 index 0000000000..0d88f22bc3 --- /dev/null +++ b/pkg/shares/parse.go @@ -0,0 +1,78 @@ +package shares + +import ( + "bytes" + "fmt" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + coretypes "github.com/tendermint/tendermint/types" +) + +// ParseTxs collects all of the transactions from the shares provided +func ParseTxs(shares [][]byte) (coretypes.Txs, error) { + // parse the sharse + rawTxs, err := parseCompactShares(shares, appconsts.SupportedShareVersions) + if err != nil { + return nil, err + } + + // convert to the Tx type + txs := make(coretypes.Txs, len(rawTxs)) + for i := 0; i < len(txs); i++ { + txs[i] = coretypes.Tx(rawTxs[i]) + } + + return txs, nil +} + +// ParseBlobs collects all blobs from the shares provided +func ParseBlobs(shares [][]byte) ([]coretypes.Blob, error) { + blobList, err := parseSparseShares(shares, appconsts.SupportedShareVersions) + if err != nil { + return []coretypes.Blob{}, err + } + + return blobList, nil +} + +func ParseShares(rawShares [][]byte) ([]ShareSequence, error) { + sequences := []ShareSequence{} + currentSequence := ShareSequence{} + + for _, rawShare := range rawShares { + share, err := NewShare(rawShare) + if err != nil { + return sequences, err + } + isStart, err := share.IsSequenceStart() + if err != nil { + return sequences, err + } + if isStart { + if len(currentSequence.Shares) > 0 { + sequences = append(sequences, currentSequence) + } + currentSequence = ShareSequence{ + Shares: []Share{share}, + NamespaceID: share.NamespaceID(), + } + } else { + if !bytes.Equal(currentSequence.NamespaceID, share.NamespaceID()) { + return sequences, fmt.Errorf("share sequence %v has inconsistent namespace IDs with share %v", currentSequence, share) + } + currentSequence.Shares = append(currentSequence.Shares, share) + } + } + + if len(currentSequence.Shares) > 0 { + sequences = append(sequences, currentSequence) + } + + for _, sequence := range sequences { + if err := sequence.validSequenceLen(); err != nil { + return sequences, err + } + } + + return sequences, nil +} diff --git a/pkg/shares/share_merging_test.go b/pkg/shares/parse_test.go similarity index 60% rename from pkg/shares/share_merging_test.go rename to pkg/shares/parse_test.go index ba94f9853e..71904fce73 100644 --- a/pkg/shares/share_merging_test.go +++ b/pkg/shares/parse_test.go @@ -1,14 +1,12 @@ package shares import ( - "bytes" "encoding/binary" "math/rand" "reflect" "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" - testns "github.com/celestiaorg/celestia-app/testutil/namespace" "github.com/celestiaorg/nmt/namespace" "github.com/stretchr/testify/assert" tmrand "github.com/tendermint/tendermint/libs/rand" @@ -152,117 +150,6 @@ func TestParseShares(t *testing.T) { } } -func TestShareSequenceRawData(t *testing.T) { - type testCase struct { - name string - shareSequence ShareSequence - want []byte - wantErr bool - } - blobNamespace := testns.RandomBlobNamespace() - - testCases := []testCase{ - { - name: "empty share sequence", - shareSequence: ShareSequence{ - NamespaceID: appconsts.TxNamespaceID, - Shares: []Share{}, - }, - want: []byte{}, - wantErr: false, - }, - { - name: "one empty share", - shareSequence: ShareSequence{ - NamespaceID: appconsts.TxNamespaceID, - Shares: []Share{ - shareWithData(blobNamespace, true, 0, []byte{}), - }, - }, - want: []byte{}, - wantErr: false, - }, - { - name: "one share with one byte", - shareSequence: ShareSequence{ - NamespaceID: appconsts.TxNamespaceID, - Shares: []Share{ - shareWithData(blobNamespace, true, 1, []byte{0x0f}), - }, - }, - want: []byte{0xf}, - wantErr: false, - }, - { - name: "removes padding from last share", - shareSequence: ShareSequence{ - NamespaceID: appconsts.TxNamespaceID, - Shares: []Share{ - shareWithData(blobNamespace, true, appconsts.FirstSparseShareContentSize+1, bytes.Repeat([]byte{0xf}, appconsts.FirstSparseShareContentSize)), - shareWithData(blobNamespace, false, 0, []byte{0x0f}), - }, - }, - want: bytes.Repeat([]byte{0xf}, appconsts.FirstSparseShareContentSize+1), - wantErr: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := tc.shareSequence.RawData() - if tc.wantErr { - assert.Error(t, err) - return - } - assert.Equal(t, tc.want, got) - }) - } -} - -func Test_compactSharesNeeded(t *testing.T) { - type testCase struct { - sequenceLen int - want int - } - testCases := []testCase{ - {0, 0}, - {1, 1}, - {2, 1}, - {appconsts.FirstCompactShareContentSize, 1}, - {appconsts.FirstCompactShareContentSize + 1, 2}, - {appconsts.FirstCompactShareContentSize + appconsts.ContinuationCompactShareContentSize, 2}, - {appconsts.FirstCompactShareContentSize + appconsts.ContinuationCompactShareContentSize*100, 101}, - } - for _, tc := range testCases { - got := CompactSharesNeeded(tc.sequenceLen) - assert.Equal(t, tc.want, got) - } -} - -func Test_sparseSharesNeeded(t *testing.T) { - type testCase struct { - sequenceLen uint32 - want int - } - testCases := []testCase{ - {0, 0}, - {1, 1}, - {2, 1}, - {appconsts.FirstSparseShareContentSize, 1}, - {appconsts.FirstSparseShareContentSize + 1, 2}, - {appconsts.FirstSparseShareContentSize + appconsts.ContinuationSparseShareContentSize, 2}, - {appconsts.FirstSparseShareContentSize + appconsts.ContinuationCompactShareContentSize*2, 3}, - {appconsts.FirstSparseShareContentSize + appconsts.ContinuationCompactShareContentSize*99, 100}, - {1000, 2}, - {10000, 20}, - {100000, 199}, - } - for _, tc := range testCases { - got := SparseSharesNeeded(tc.sequenceLen) - assert.Equal(t, tc.want, got) - } -} - func generateRawShare(namespace namespace.ID, isSequenceStart bool, sequenceLen uint32) (rawShare []byte) { infoByte, _ := NewInfoByte(appconsts.ShareVersionZero, isSequenceStart) @@ -276,20 +163,6 @@ func generateRawShare(namespace namespace.ID, isSequenceStart bool, sequenceLen return padWithRandomBytes(rawShare) } -func shareWithData(namespace namespace.ID, isSequenceStart bool, sequenceLen uint32, data []byte) (rawShare []byte) { - infoByte, _ := NewInfoByte(appconsts.ShareVersionZero, isSequenceStart) - rawShare = append(rawShare, namespace...) - rawShare = append(rawShare, byte(infoByte)) - if isSequenceStart { - sequenceLenBuf := make([]byte, appconsts.SequenceLenBytes) - binary.BigEndian.PutUint32(sequenceLenBuf, sequenceLen) - rawShare = append(rawShare, sequenceLenBuf...) - } - rawShare = append(rawShare, data...) - - return padShare(rawShare) -} - func padWithRandomBytes(partialShare Share) (paddedShare Share) { paddedShare = make([]byte, appconsts.ShareSize) copy(paddedShare, partialShare) diff --git a/pkg/shares/share_merging.go b/pkg/shares/share_sequence.go similarity index 51% rename from pkg/shares/share_merging.go rename to pkg/shares/share_sequence.go index af9cea4ec9..a8e7d4c2db 100644 --- a/pkg/shares/share_merging.go +++ b/pkg/shares/share_sequence.go @@ -1,147 +1,20 @@ package shares import ( - "bytes" "fmt" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/rsmt2d" - coretypes "github.com/tendermint/tendermint/types" ) -// merge extracts block data from an extended data square. -func merge(eds *rsmt2d.ExtendedDataSquare) (coretypes.Data, error) { - squareSize := eds.Width() / 2 - - // sort block data shares by namespace - var ( - sortedTxShares [][]byte - sortedBlobShares [][]byte - ) - - // iterate over each row index - for x := uint(0); x < squareSize; x++ { - // iterate over each share in the original data square - row := eds.Row(x) - - for _, share := range row[:squareSize] { - // sort the data of that share types via namespace - nid := share[:appconsts.NamespaceSize] - switch { - case bytes.Equal(appconsts.TxNamespaceID, nid): - sortedTxShares = append(sortedTxShares, share) - - case bytes.Equal(appconsts.TailPaddingNamespaceID, nid): - continue - - // ignore unused but reserved namespaces - case bytes.Compare(nid, appconsts.MaxReservedNamespace) < 1: - continue - - // every other namespaceID should be a blob - default: - sortedBlobShares = append(sortedBlobShares, share) - } - } - } - - // pass the raw share data to their respective parsers - txs, err := ParseTxs(sortedTxShares) - if err != nil { - return coretypes.Data{}, err - } - - blobs, err := ParseBlobs(sortedBlobShares) - if err != nil { - return coretypes.Data{}, err - } - - return coretypes.Data{ - Txs: txs, - Blobs: blobs, - SquareSize: uint64(squareSize), - }, nil -} - -// ParseTxs collects all of the transactions from the shares provided -func ParseTxs(shares [][]byte) (coretypes.Txs, error) { - // parse the sharse - rawTxs, err := parseCompactShares(shares, appconsts.SupportedShareVersions) - if err != nil { - return nil, err - } - - // convert to the Tx type - txs := make(coretypes.Txs, len(rawTxs)) - for i := 0; i < len(txs); i++ { - txs[i] = coretypes.Tx(rawTxs[i]) - } - - return txs, nil -} - -// ParseBlobs collects all blobs from the shares provided -func ParseBlobs(shares [][]byte) ([]coretypes.Blob, error) { - blobList, err := parseSparseShares(shares, appconsts.SupportedShareVersions) - if err != nil { - return []coretypes.Blob{}, err - } - - return blobList, nil -} - // ShareSequence represents a contiguous sequence of shares that are part of the // same namespace and blob. For compact shares, one share sequence exists per // reserved namespace. For sparse shares, one share sequence exists per blob. -// TODO consider extracting the ShareSequence struct to a new file. type ShareSequence struct { NamespaceID namespace.ID Shares []Share } -func ParseShares(rawShares [][]byte) ([]ShareSequence, error) { - sequences := []ShareSequence{} - currentSequence := ShareSequence{} - - for _, rawShare := range rawShares { - share, err := NewShare(rawShare) - if err != nil { - return sequences, err - } - isStart, err := share.IsSequenceStart() - if err != nil { - return sequences, err - } - if isStart { - if len(currentSequence.Shares) > 0 { - sequences = append(sequences, currentSequence) - } - currentSequence = ShareSequence{ - Shares: []Share{share}, - NamespaceID: share.NamespaceID(), - } - } else { - if !bytes.Equal(currentSequence.NamespaceID, share.NamespaceID()) { - return sequences, fmt.Errorf("share sequence %v has inconsistent namespace IDs with share %v", currentSequence, share) - } - currentSequence.Shares = append(currentSequence.Shares, share) - } - } - - if len(currentSequence.Shares) > 0 { - sequences = append(sequences, currentSequence) - } - - for _, sequence := range sequences { - if err := sequence.validSequenceLen(); err != nil { - return sequences, err - } - } - - return sequences, nil -} - // RawData returns the raw share data of this share sequence. The raw data does // not contain the namespace ID, info byte, sequence length, or reserved bytes. func (s ShareSequence) RawData() (data []byte, err error) { @@ -161,6 +34,14 @@ func (s ShareSequence) RawData() (data []byte, err error) { return data[:sequenceLen], nil } +func (s ShareSequence) SequenceLen() (uint32, error) { + if len(s.Shares) == 0 { + return 0, fmt.Errorf("invalid sequence length because share sequence %v has no shares", s) + } + firstShare := s.Shares[0] + return firstShare.SequenceLen() +} + // validSequenceLen extracts the sequenceLen written to the first share // and returns an error if the number of shares needed to store a sequence of // length sequenceLen doesn't match the number of shares in this share @@ -181,14 +62,6 @@ func (s ShareSequence) validSequenceLen() error { return nil } -func (s ShareSequence) SequenceLen() (uint32, error) { - if len(s.Shares) == 0 { - return 0, fmt.Errorf("invalid sequence length because share sequence %v has no shares", s) - } - firstShare := s.Shares[0] - return firstShare.SequenceLen() -} - // numberOfSharesNeeded extracts the sequenceLen written to the share // firstShare and returns the number of shares needed to store a sequence of // that length. diff --git a/pkg/shares/share_sequence_test.go b/pkg/shares/share_sequence_test.go new file mode 100644 index 0000000000..40213a9b65 --- /dev/null +++ b/pkg/shares/share_sequence_test.go @@ -0,0 +1,137 @@ +package shares + +import ( + "bytes" + "encoding/binary" + "testing" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + testns "github.com/celestiaorg/celestia-app/testutil/namespace" + "github.com/celestiaorg/nmt/namespace" + "github.com/stretchr/testify/assert" +) + +func TestShareSequenceRawData(t *testing.T) { + type testCase struct { + name string + shareSequence ShareSequence + want []byte + wantErr bool + } + blobNamespace := testns.RandomBlobNamespace() + + testCases := []testCase{ + { + name: "empty share sequence", + shareSequence: ShareSequence{ + NamespaceID: appconsts.TxNamespaceID, + Shares: []Share{}, + }, + want: []byte{}, + wantErr: false, + }, + { + name: "one empty share", + shareSequence: ShareSequence{ + NamespaceID: appconsts.TxNamespaceID, + Shares: []Share{ + shareWithData(blobNamespace, true, 0, []byte{}), + }, + }, + want: []byte{}, + wantErr: false, + }, + { + name: "one share with one byte", + shareSequence: ShareSequence{ + NamespaceID: appconsts.TxNamespaceID, + Shares: []Share{ + shareWithData(blobNamespace, true, 1, []byte{0x0f}), + }, + }, + want: []byte{0xf}, + wantErr: false, + }, + { + name: "removes padding from last share", + shareSequence: ShareSequence{ + NamespaceID: appconsts.TxNamespaceID, + Shares: []Share{ + shareWithData(blobNamespace, true, appconsts.FirstSparseShareContentSize+1, bytes.Repeat([]byte{0xf}, appconsts.FirstSparseShareContentSize)), + shareWithData(blobNamespace, false, 0, []byte{0x0f}), + }, + }, + want: bytes.Repeat([]byte{0xf}, appconsts.FirstSparseShareContentSize+1), + wantErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.shareSequence.RawData() + if tc.wantErr { + assert.Error(t, err) + return + } + assert.Equal(t, tc.want, got) + }) + } +} + +func Test_compactSharesNeeded(t *testing.T) { + type testCase struct { + sequenceLen int + want int + } + testCases := []testCase{ + {0, 0}, + {1, 1}, + {2, 1}, + {appconsts.FirstCompactShareContentSize, 1}, + {appconsts.FirstCompactShareContentSize + 1, 2}, + {appconsts.FirstCompactShareContentSize + appconsts.ContinuationCompactShareContentSize, 2}, + {appconsts.FirstCompactShareContentSize + appconsts.ContinuationCompactShareContentSize*100, 101}, + } + for _, tc := range testCases { + got := CompactSharesNeeded(tc.sequenceLen) + assert.Equal(t, tc.want, got) + } +} + +func Test_sparseSharesNeeded(t *testing.T) { + type testCase struct { + sequenceLen uint32 + want int + } + testCases := []testCase{ + {0, 0}, + {1, 1}, + {2, 1}, + {appconsts.FirstSparseShareContentSize, 1}, + {appconsts.FirstSparseShareContentSize + 1, 2}, + {appconsts.FirstSparseShareContentSize + appconsts.ContinuationSparseShareContentSize, 2}, + {appconsts.FirstSparseShareContentSize + appconsts.ContinuationCompactShareContentSize*2, 3}, + {appconsts.FirstSparseShareContentSize + appconsts.ContinuationCompactShareContentSize*99, 100}, + {1000, 2}, + {10000, 20}, + {100000, 199}, + } + for _, tc := range testCases { + got := SparseSharesNeeded(tc.sequenceLen) + assert.Equal(t, tc.want, got) + } +} + +func shareWithData(namespace namespace.ID, isSequenceStart bool, sequenceLen uint32, data []byte) (rawShare []byte) { + infoByte, _ := NewInfoByte(appconsts.ShareVersionZero, isSequenceStart) + rawShare = append(rawShare, namespace...) + rawShare = append(rawShare, byte(infoByte)) + if isSequenceStart { + sequenceLenBuf := make([]byte, appconsts.SequenceLenBytes) + binary.BigEndian.PutUint32(sequenceLenBuf, sequenceLen) + rawShare = append(rawShare, sequenceLenBuf...) + } + rawShare = append(rawShare, data...) + + return padShare(rawShare) +} diff --git a/pkg/shares/shares_test.go b/pkg/shares/shares_test.go index 6e88ec6d43..752160d4c2 100644 --- a/pkg/shares/shares_test.go +++ b/pkg/shares/shares_test.go @@ -2,68 +2,15 @@ package shares import ( "bytes" - "context" "testing" - "time" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/testutil/testfactory" - "github.com/celestiaorg/rsmt2d" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" coretypes "github.com/tendermint/tendermint/types" ) -func TestMerge(t *testing.T) { - type test struct { - name string - txCount int - blobCount int - maxSize int // max size of each tx or blob - } - - tests := []test{ - {"one of each random small size", 1, 1, 40}, - {"one of each random large size", 1, 1, 400}, - {"many of each random large size", 10, 10, 40}, - {"many of each random large size", 10, 10, 400}, - {"only transactions", 10, 0, 400}, - {"only blobs", 0, 10, 400}, - } - - for _, tc := range tests { - tc := tc - - t.Run(tc.name, func(t *testing.T) { - // generate random data - data := generateRandomBlockData( - tc.txCount, - tc.blobCount, - tc.maxSize, - ) - - shares, err := Split(data, false) - require.NoError(t, err) - rawShares := ToBytes(shares) - - eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, appconsts.DefaultCodec(), rsmt2d.NewDefaultTree) - if err != nil { - t.Error(err) - } - - res, err := merge(eds) - if err != nil { - t.Fatal(err) - } - - res.SquareSize = data.SquareSize - - assert.Equal(t, data, res) - }) - } -} - // TestPadFirstIndexedBlob ensures that we are adding padding to the first share // instead of calculating the value. func TestPadFirstIndexedBlob(t *testing.T) { @@ -90,43 +37,6 @@ func TestPadFirstIndexedBlob(t *testing.T) { require.True(t, bytes.Contains(resShare, blob)) } -func TestFuzz_Merge(t *testing.T) { - t.Skip() - // run random shares through processCompactShares for a minute - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - for { - select { - case <-ctx.Done(): - return - default: - TestMerge(t) - } - } -} - -// generateRandomBlockData returns randomly generated block data for testing purposes -func generateRandomBlockData(txCount, blobCount, maxSize int) (data coretypes.Data) { - data.Txs = testfactory.GenerateRandomlySizedTxs(txCount, maxSize) - data.Blobs = testfactory.GenerateRandomlySizedBlobs(blobCount, maxSize) - data.SquareSize = appconsts.DefaultMaxSquareSize - return data -} - -// generateRandomBlobOfShareCount returns a blob that spans the given -// number of shares -func generateRandomBlobOfShareCount(count int) coretypes.Blob { - size := rawBlobSize(appconsts.FirstSparseShareContentSize * count) - return testfactory.GenerateRandomBlob(size) -} - -// rawBlobSize returns the raw blob size that can be used to construct a -// blob of totalSize bytes. This function is useful in tests to account for -// the delimiter length that is prefixed to a blob's data. -func rawBlobSize(totalSize int) int { - return totalSize - DelimLen(uint64(totalSize)) -} - func TestSequenceLen(t *testing.T) { type testCase struct { name string diff --git a/pkg/shares/sparse_shares_test.go b/pkg/shares/sparse_shares_test.go index b05a92b7bf..f922673b8f 100644 --- a/pkg/shares/sparse_shares_test.go +++ b/pkg/shares/sparse_shares_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/testutil/testfactory" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" coretypes "github.com/tendermint/tendermint/types" @@ -83,3 +84,17 @@ func TestSparseShareSplitterCount(t *testing.T) { }) } } + +// generateRandomBlobOfShareCount returns a blob that spans the given +// number of shares +func generateRandomBlobOfShareCount(count int) coretypes.Blob { + size := rawBlobSize(appconsts.FirstSparseShareContentSize * count) + return testfactory.GenerateRandomBlob(size) +} + +// rawBlobSize returns the raw blob size that can be used to construct a +// blob of totalSize bytes. This function is useful in tests to account for +// the delimiter length that is prefixed to a blob's data. +func rawBlobSize(totalSize int) int { + return totalSize - DelimLen(uint64(totalSize)) +} diff --git a/pkg/shares/testdata/sample-block.json b/pkg/shares/testdata/sample-block.json new file mode 100755 index 0000000000..ac37dac573 --- /dev/null +++ b/pkg/shares/testdata/sample-block.json @@ -0,0 +1 @@ +{"header":{"version":{"block":11},"chain_id":"hVcdds","height":4,"time":"2023-01-30T18:55:27.534092Z","last_block_id":{"hash":"CnTXMWDvJXaHOjecWXb2nGvLsJ23GmgUbAYIZmteP2M=","part_set_header":{"total":1,"hash":"PgAxB6DYr8huhDZJWMnFSgYDCXwuF3jI1GNbnOskv9U="}},"last_commit_hash":"sd0o4vJyCO1W3BxOcke3/Frtu46hx+DQlDb1NjcDaTk=","data_hash":"uPWH3TVoZNb3NcyXN/vbX5N1mDZ8dfXMh9bhne04CpM=","validators_hash":"p4Iap0aX3N436a//6jkEvJV61B8xpfp82lP0P1Ji23g=","next_validators_hash":"p4Iap0aX3N436a//6jkEvJV61B8xpfp82lP0P1Ji23g=","consensus_hash":"BICRvH3cKD93v7+R1zxE2ljD34qcvIZ0Bdi389qtoi8=","app_hash":"N28hOwD/a26Nwnp3R8UteUU6TcM28zIb3+Undtt3YCw=","last_results_hash":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","evidence_hash":"47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=","proposer_address":"XbENo47oRQwd9YSP9vr+10SttTc="},"data":{"txs":["Cp0CCn0KewoTL2Jsb2IuTXNnUGF5Rm9yQmxvYhJkCi9jZWxlc3RpYTEyc2U4MGR3aHIzOWxxNm45dXR3eXV2NDJlZnhyZXkycGV2YXludBII6P2EbA1mM94aAsEbIiDTB7FJ5XbuiAUwwKj3yEybf4eTcfZS81uO74ySllRJm0IBABJaCk4KRgofL2Nvc21vcy5jcnlwdG8uc2VjcDI1NmsxLlB1YktleRIjCiED5ZlClAcCdabMf1ivEUWgN5bJpluGNNl9XdZqlN0bNF8SBAoCCAESCBCAgOmDsd4WGkCE4EMZYENqU97tLHhxWnb5zQ64JdjqlVR2iUHxH54mrGtaThLGSKEh5ohdFQDXSHuLi+zRH7xN8XQISAhH/WLEEgEEGgRJTkRY"],"blobs":[{"namespace_id":"6P2EbA1mM94=","data":"+xg8TFSwDVXT1rILXjOb1fzQNJ6MeyO6rtOiUQCN6PIG58JsN2yFenuGREACJ31AU/TFEMzWEyj3IDLZO9NmPgRS8JCV5azfz26SmowYQtRtFcV+nICKR1HLvzuKV7Rfowey9duLvi2WST/NVaety6tRKmRbMkbljnRqlc1tFWYDPa8uOA/fSfYnDA3zH70MREnJb0GsIJ5kBlPVOVtD2o3gtD89Qwy07BF/3gAQzht7B1A0FZmV/rAjWpQ2xumLK9k2TFvgqMcltZuSy8cvxb7YHihlEPindGDch+BdZCxvYUom3bLBEcKDFnp0Cu7MeyH+299XPDzqYt7V2CLZpyV/bJUdOCPOGbNqK0mGvj1RhHz9ORrB71mGBBOAArDXUyR/GmplrhlqFQ90kyMhJR8SmPfbVgVYj/q2TyODvOagr7qb364lZcSJg1UAlwbxCJXG1xAhRnlKIkxAd3mgJ4XP6wuWOzFvyKXyc6JuusRcmm/KzyLqa7wPRAI9mRe0A9jqmY4kuJYSV7H6XqCmS6RMNeEZ/5GvpR1dWnHkLaJtCpQxQxqZdFzd8KBQqBjEsdwyZ8Wun37+ajQ6QYwcZQNjTlc8U4JQm2wWXNO3i+Z7F9fXRoy95yRS84OlBcKDxokWi+cqRsPGR2kcwF4iH8P0nQDrp7sR/sHKt01Qiw/smyCP44LHxSctU79ySmsTMm0jJSdAMfi3loC1M4S66wq/g6H2YHSOIjrAZAVqtuTt1f3C9iGfY55u3mMhwKue/4cQ/GSdAYyF25R03IPdZBG7HyCGKWkNYkj1kLszdQEJROEahx12GU/8LapzEk6c97ltd4a59d9CavbyAIpgwKseBqY2VJxl8QBPRim62WqoyOi2b6FMr7ZEMOzNxN6qnZTcysbF3mAdN7fpcrO4mrcyiDCxzWHqPxsYevcEB7vLY9EN8oPbNcc9N4nh7lfHti3HqYmCOBDEBQHjyegdjiMqRQ/mJdWzDjBf5qJiEQbMj/hXNwfIqskZHKJ3Aziv/3cz2kP3W6cifZ/KxxtrBhY8zzUoeIB36IHBiYVb1ZbsaLl+Bc8HVq1SKEt8Ykl4wyjn6LHuB+ju/sGgwWmWJUU3+9p0tzJh5rToUm0PjCa//WLvU6DZ6+5jsGyWtSoC431QPFip94LAzHM+FnlwxoWFQRo0SAzPn1MPrawj6aHAdtPX3DTSX/RR3okVIa9/0kMV4fw7rCtfp09yyeWgeRliY8dklAYQODmZMIPTteY/Y2vs6ww+KIJpdy5iEto9aaI9HmviPCmPMY666LHIY1U6LaAv3O6XRAs/dCL4olqfoMZVzUEYSmd1/1+DbzPB2lgLXg531HrQa4rQys8EfhBgwDtx0Hskh7m5bEqPx+mLMd74iOJ1WRFWt+BW44BuaWSTC4iqyBvj/t2UQBBBUK3DP/8KwrYrqKDVzVA+5Q8B2bPOEvexzS9XGoH+44hZGeI35oEvyDAisGs1ovgeeMHfjEVxGmUAGEEiIrYByFxvBqJ3fImhrhhAgabC0GR6hQgmN3qo1i4JX7cUF2IH1na9OMn4e5dwzdK0/0nAv64G4j95fv3HMQbzGyv1pkmxNDiC6fDNoWhYLclUY/jUviogffIJYSWNJK8T6kCwrtH21hnD12DBfpOFI45ESHFzgVvgY/TLQd+Xk1FoLvtiSiKHVjX2GfDOjpdhzCNhGdBVtiLRI+zX8bGHVPyDgmm7sJiQcQjSuro/nzOwcFFKkkxcPtB7Yxc8lkjfmM6tqjabMA0cjcC6jCDk4aZ8XZ5kevLSJSTfdcRbB4Z3L2FEsZ75MOrV2qF65kv5kYQ2kofQTLx63dzW6q3BejJQDf40tJAUtL3xkU2B2cqnfYuXtMQohKdGg99OAECL/QgLSCeibPDXJYGQxCzK/3h9rYdMDXGXrIAq3H0ktSQhDtXmfqto0ZIZV+i1lLZfseAfqCwuhHTakudPYTTLa9/N075ZQ2s6sToVMMTgM1y/DSlRbkfJrfX3qb+1z3EOOK/zx0O2ljzcnqD2oIbmBRpNznR7rIVOusn8KSgHJ8fkn9hHWDPAog3dYRRZWii1tg5dv9Jd7bb7U0T7SevNWRthOLSfzHn8baxp+ltH1bwBkHAHSdZhhdpdpQohAR9DZBWvHni8hVDgnPhZRS/l11BGM99UEV40obpQ/igzn7dDIEsfHbiuRERcUjXnQGSh0BXQe2eYR5ucXoUMdEtebQq3vttEdwMnjmn8rofu1Q3bO+0aSQBVOgLO1k6/G0ZL/VsFBqAuRFcY20ha/i2P6sYZLE2nXTetQqx+VwD2ttgVsqh7FzLQ+/BjM5VnobVxLa8xTUNhaaK0WRBDOSOXR3MhFP/HK3+SpXCOGYyHg3BX+G6hzSX1D7rZZdH8k6ER+sCWf6izLm4+h2pkgVXQeyrcc5ZxeLCyq24C7jgIbRL1Zrwutah2sI1bFMKOzg5bnWPSqzX6rNzwOO1Ike6YPer88rop0CyWe/NNc5i4YutmxS5/cy7eVnPMc+kKzIPhWtKv7o/soc88c3QKPcdmTyeUat1refZ+v6dzjssvH/xvfb8GFpelYyhcGBEsWCwE6WdVIlU2SShLbd7ACY50TDw8f564QjA/D9xvUDQKFNfPM+2M1uzqvjT0sTIHv6fBOtMXYAE3cetqMz3L2NnAXVWgSVw/P7Z3QXNCgZWnLLJ5OPdY769RQMc16dRvAyIPxCCoYAw4wisSz0wLPUGrpH4s3uxMNfaHKBHBGmWy8lLxGWjCrSwnCAAl1suYTMCvSP2azNncsmHswJ7GzmtkLB8etyp1flVrpzYayZH/3EYKkCoVYcs2iHqQLo3Od/OyirhwbWauHuZfdxx1er9/Roo+DIayM9DrM9TgzEMygTnvt5GBS3yinCKAbr31CiCBsF9ICankdWMqKKHO1aDrIIzfhAjcdxx/wM1mORJLm5AZmIXN3tqGKcRDxiPv7mXEkm8b068WdtH5X9qXg9PBV/caUgpEhDAuGoYpmHJrYHfWCk5++EjSDUnvP7EWNkbKFBN7jdNCZkTgPSHJb2q0B+1mUkBYROlTDp2DqO+1Th9qySZIjlCP2gZaKUo4zCtxvap5hUmD4jgzpmjBEBuKBwSZlH/0fScXazO6PWxNve1eeUrdLUJ+Zrr8A9fpZbLXTTMdflX0EPrbNU+mUBRvMz54Mfc6VtrwYd9hKiBviO1yIDcYIJQcXG3J253KSjqpIv3Uff5i5XtZNcIY1LBEdp+r5UYpwSwv9zZC/IsDZb3kggp9A1XAZZsXcDK8bd3VHH6ZrdgIDZdyDUD+8XQ2AVVHF/j7vDD59Tcs0O9yz7+WQV2qv1DlskRtaRde6SLo62H3vYM6Au/VjRT0x5Fnz/SVEGQVnav6aWEAawlCOi8xaTXQwxxqsbG+/SpX47bwQ2wECoKIg/gvfe23sgOMHVJXVfw0eA7DzreMMAnfjWVE1Ap6vLpuTGtwcUCa0Hjfp031076q9GiKrbRP5Jj/NiVV0TmMfoJFm6PQctoLdHb39ZMpthcfIsknQIzn+7jtXxxkfSww3QIBIvnG3BLBeQba4uRER5wcgUOUc+mK+YKmvMmtWbDD0Tv3+5CDONxjlFbVWsR+LNNlEf/T+2dJb00Ol2tIOd069cmkPqmvH2eNjOv/W/bubA6Ogrq9cLrAP2HNySvLp2PN1QNvAnCrvnYnwqgmYewHMrIqXiF+atJHM2qgofidLBd2W+mUsrHzrnid7NsxHGvHz5oStqTSU8e2NhOqSUUu9Hfh7Bi8p0hfXj+zIPhZfFcJFHCKc1qDN6RhmYBkpihJ90/+m2lWY2+SUgLCjse3O6ohh3bgmozyItrIFSTu1KWrdiO5mJ/snHeVKX110ZDiHku/Jygq3j8dPSF5RfElfPJxeFgSbHWX2KGtAA5oEKhp4wnhZhnFEppZ1mFrXgdPO4AAwKZ3V/vwmfbv1xZ9zT7O5PU1DymWsdQqTtR4Bwi0t0/napqH534F0cEiKT4ZVvPErTGckYfEpAuTd1JbNekrTAT3Gdy9E97LKxBGAJiP7dcfRuZKaVtt62lSkOM9dK61YYY7TlRRsG31YV04Q4eoF1Zxh81xRyV19OjIjngHzYuG7KUnmt6uMc/aQjplIEfQXud7PYm8CP4DBaqd6cAgclmsW0XkLmHk04JTe7Vd6xaPAjhK+H/2Z59tiDAYOxUqybAPy/Un8p3kf9WfkqLVzQZviOJYoIEXsQBtAA3P1f3Cs0V8NZQZcZmHCL/cCCJ6V+V3sEmhUl1nblrTgBmpfdlZRcASB20XgFXT6LQDimSYv9ycrMwpJXRxHLEgnbCo72mrBh4p2opBbfZBLkhAnkqfV/mjS/sxAZ4kKmASHM/Wbqqia2mo6FIvk9OeyVwU1v15v5iGwDd77uGB95MfcA/7qXqgPrUxaZUl9fQ8R6kcqvWIXOu8+KbSm1IB58VRxnXYFFlcRoBr4lTWHPrBivtArOSIZ6fZ/HY9Dw9oj0lACaXxZEtkLl/vDNxPEx5B8MN+onhOzgLccK5q8RBGf8hA9ebClWvRmLvYT/+yTcvOmwPWgSs6nOYprckWc47tia2350CXU6EPCnzECSE/X3AcNnwgli2Z1T+vUbYrBr3jrUjmQpujFOsuJZV9oaAfXXPlN+Lsz4I7bil4kyomQ/w3q3gTQfWmmkU="}],"square_size":4,"hash":"uPWH3TVoZNb3NcyXN/vbX5N1mDZ8dfXMh9bhne04CpM="},"evidence":{"evidence":[]},"last_commit":{"height":3,"block_id":{"hash":"CnTXMWDvJXaHOjecWXb2nGvLsJ23GmgUbAYIZmteP2M=","part_set_header":{"total":1,"hash":"PgAxB6DYr8huhDZJWMnFSgYDCXwuF3jI1GNbnOskv9U="}},"signatures":[{"block_id_flag":2,"validator_address":"XbENo47oRQwd9YSP9vr+10SttTc=","timestamp":"2023-01-30T18:55:27.534092Z","signature":"CPafxD85F5yNxKP/9hm2ew4WLPRIo472AcVNMgOZh239V4AnKsRpL8FUwS14X48DpsZuzVCqLgUKY2+agupxBQ=="}]}}