From 42a804a67661064c615868bfe8c85c91d0120fe2 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 30 Sep 2022 13:57:01 -0600 Subject: [PATCH] chore!: remove `NamespacedShares` (#822) Closes https://github.com/celestiaorg/celestia-app/issues/721 --- app/prepare_proposal.go | 2 +- app/process_proposal.go | 2 +- pkg/prove/proof.go | 29 +++++---- pkg/prove/proof_test.go | 2 +- pkg/shares/compact_shares_test.go | 19 +++--- pkg/shares/share_splitting.go | 42 +++++++++---- pkg/shares/share_splitting_test.go | 14 ++--- pkg/shares/shares.go | 65 ++++++++++++++------ pkg/shares/shares_test.go | 3 +- pkg/shares/sparse_shares_test.go | 17 +++-- pkg/shares/split_compact_shares.go | 99 ++++++++---------------------- pkg/shares/split_sparse_shares.go | 55 ++++++++++------- x/payment/types/payfordata.go | 6 +- 13 files changed, 188 insertions(+), 167 deletions(-) diff --git a/app/prepare_proposal.go b/app/prepare_proposal.go index fe7038f942..61b4dc92e2 100644 --- a/app/prepare_proposal.go +++ b/app/prepare_proposal.go @@ -59,7 +59,7 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr } // erasure the data square which we use to create the data root. - eds, err := da.ExtendShares(squareSize, dataSquare) + eds, err := da.ExtendShares(squareSize, shares.ToBytes(dataSquare)) if err != nil { app.Logger().Error( "failure to erasure the data square while creating a proposal block", diff --git a/app/process_proposal.go b/app/process_proposal.go index 1a0f38c1e6..9722ed5b09 100644 --- a/app/process_proposal.go +++ b/app/process_proposal.go @@ -50,7 +50,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) abci.ResponsePr } cacher := inclusion.NewSubtreeCacher(data.OriginalSquareSize) - eds, err := rsmt2d.ComputeExtendedDataSquare(dataSquare, appconsts.DefaultCodec(), cacher.Constructor) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares.ToBytes(dataSquare), appconsts.DefaultCodec(), cacher.Constructor) if err != nil { logInvalidPropBlockError(app.Logger(), req.Header, "failure to erasure the data square", err) return abci.ResponseProcessProposal{ diff --git a/pkg/prove/proof.go b/pkg/prove/proof.go index 1c7d9b3e5e..2f46697862 100644 --- a/pkg/prove/proof.go +++ b/pkg/prove/proof.go @@ -37,7 +37,7 @@ func TxInclusion(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.TxP } var proofs []*tmproto.NMTProof //nolint:prealloc // rarely will this contain more than a single proof - var shares [][]byte //nolint:prealloc // rarely will this contain more than a single share + var rawShares [][]byte //nolint:prealloc // rarely will this contain more than a single share var rowRoots []tmbytes.HexBytes //nolint:prealloc // rarely will this contain more than a single root for i, row := range rowShares { // create an nmt to use to generate a proof @@ -64,8 +64,7 @@ func TxInclusion(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.TxP endLeafPos = data.OriginalSquareSize - 1 } - shares = append(shares, row[startLeafPos:endLeafPos+1]...) - + rawShares = append(rawShares, shares.ToBytes(row[startLeafPos:endLeafPos+1])...) proof, err := tree.Tree().ProveRange(int(startLeafPos), int(endLeafPos+1)) if err != nil { return types.TxProof{}, err @@ -85,7 +84,7 @@ func TxInclusion(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.TxP return types.TxProof{ RowRoots: rowRoots, - Data: shares, + Data: rawShares, Proofs: proofs, }, nil } @@ -132,7 +131,7 @@ func txShareIndex(totalTxLen int) (index uint64) { } // genRowShares progessively generates data square rows from block data -func genRowShares(codec rsmt2d.Codec, data types.Data, startRow, endRow uint64) ([][][]byte, error) { +func genRowShares(codec rsmt2d.Codec, data types.Data, startRow, endRow uint64) ([][]shares.Share, error) { if endRow > data.OriginalSquareSize { return nil, errors.New("cannot generate row shares past the original square size") } @@ -141,17 +140,17 @@ func genRowShares(codec rsmt2d.Codec, data types.Data, startRow, endRow uint64) genOrigRowShares(data, startRow, endRow), ) - encodedRowShares := make([][][]byte, len(origRowShares)) + encodedRowShares := make([][]shares.Share, len(origRowShares)) for i, row := range origRowShares { - encRow, err := codec.Encode(row) + encRow, err := codec.Encode(shares.ToBytes(row)) if err != nil { panic(err) } encodedRowShares[i] = append( append( - make([][]byte, 0, len(row)+len(encRow)), + make([]shares.Share, 0, len(row)+len(encRow)), row..., - ), encRow..., + ), shares.FromBytes(encRow)..., ) } @@ -161,7 +160,7 @@ func genRowShares(codec rsmt2d.Codec, data types.Data, startRow, endRow uint64) // genOrigRowShares progressively generates data square rows for the original // data square, meaning the rows only half the full square length, as there is // not erasure data -func genOrigRowShares(data types.Data, startRow, endRow uint64) [][]byte { +func genOrigRowShares(data types.Data, startRow, endRow uint64) []shares.Share { wantLen := (endRow + 1) * data.OriginalSquareSize startPos := startRow * data.OriginalSquareSize @@ -196,17 +195,17 @@ func genOrigRowShares(data types.Data, startRow, endRow uint64) [][]byte { } tailShares := shares.TailPaddingShares(int(wantLen) - len(rawShares)) - rawShares = append(rawShares, tailShares.RawShares()...) + rawShares = append(rawShares, tailShares...) return rawShares[startPos:wantLen] } // splitIntoRows splits shares into rows of a particular square size -func splitIntoRows(squareSize uint64, shares [][]byte) [][][]byte { - rowCount := uint64(len(shares)) / squareSize - rows := make([][][]byte, rowCount) +func splitIntoRows(squareSize uint64, s []shares.Share) [][]shares.Share { + rowCount := uint64(len(s)) / squareSize + rows := make([][]shares.Share, rowCount) for i := uint64(0); i < rowCount; i++ { - rows[i] = shares[i*squareSize : (i+1)*squareSize] + rows[i] = s[i*squareSize : (i+1)*squareSize] } return rows } diff --git a/pkg/prove/proof_test.go b/pkg/prove/proof_test.go index 07417a3d7e..f12be08862 100644 --- a/pkg/prove/proof_test.go +++ b/pkg/prove/proof_test.go @@ -226,7 +226,7 @@ func TestTxShareIndex(t *testing.T) { // stripCompactShares strips the universal prefix (namespace, info byte, data length) and // reserved byte from a list of compact shares and joins them into a single byte // slice. -func stripCompactShares(compactShares [][]byte, start uint64, end uint64) (result []byte) { +func stripCompactShares(compactShares []shares.Share, start uint64, end uint64) (result []byte) { for i := start; i <= end; i++ { if i == 0 { // the first compact share includes a total data length varint diff --git a/pkg/shares/compact_shares_test.go b/pkg/shares/compact_shares_test.go index 4b6c9336a7..2b0664864d 100644 --- a/pkg/shares/compact_shares_test.go +++ b/pkg/shares/compact_shares_test.go @@ -21,8 +21,9 @@ func TestCompactShareWriter(t *testing.T) { rawTx, _ := MarshalDelimitedTx(tx) w.WriteBytes(rawTx) } - resShares := w.Export() - rawResTxs, err := parseCompactShares(resShares.RawShares()) + shares := w.Export() + rawShares := ToBytes(shares) + rawResTxs, err := parseCompactShares(rawShares) resTxs := coretypes.ToTxs(rawResTxs) require.NoError(t, err) @@ -92,8 +93,9 @@ func Test_processCompactShares(t *testing.T) { txs := generateRandomTransaction(tc.txCount, tc.txSize) shares := SplitTxs(txs) + rawShares := ToBytes(shares) - parsedTxs, err := parseCompactShares(shares) + parsedTxs, err := parseCompactShares(rawShares) if err != nil { t.Error(err) } @@ -109,8 +111,9 @@ func Test_processCompactShares(t *testing.T) { txs := generateRandomlySizedTransactions(tc.txCount, tc.txSize) shares := SplitTxs(txs) + rawShares := ToBytes(shares) - parsedTxs, err := parseCompactShares(shares) + parsedTxs, err := parseCompactShares(rawShares) if err != nil { t.Error(err) } @@ -131,7 +134,7 @@ func TestCompactShareContainsInfoByte(t *testing.T) { css.WriteTx(tx) } - shares := css.Export().RawShares() + shares := css.Export() assert.Condition(t, func() bool { return len(shares) == 1 }) infoByte := shares[0][appconsts.NamespaceSize : appconsts.NamespaceSize+appconsts.ShareInfoBytes][0] @@ -151,7 +154,7 @@ func TestContiguousCompactShareContainsInfoByte(t *testing.T) { css.WriteTx(tx) } - shares := css.Export().RawShares() + shares := css.Export() assert.Condition(t, func() bool { return len(shares) > 1 }) infoByte := shares[1][appconsts.NamespaceSize : appconsts.NamespaceSize+appconsts.ShareInfoBytes][0] @@ -166,6 +169,8 @@ func TestContiguousCompactShareContainsInfoByte(t *testing.T) { func Test_parseCompactSharesReturnsErrForShareWithStartIndicatorFalse(t *testing.T) { txs := generateRandomTransaction(2, appconsts.ContinuationCompactShareContentSize*4) shares := SplitTxs(txs) - _, err := parseCompactShares(shares[1:]) // the second share has the message start indicator set to false + rawShares := ToBytes(shares) + + _, err := parseCompactShares(rawShares[1:]) // the second share has the message start indicator set to false assert.Error(t, err) } diff --git a/pkg/shares/share_splitting.go b/pkg/shares/share_splitting.go index 1d37b18e43..0a1a35e4c8 100644 --- a/pkg/shares/share_splitting.go +++ b/pkg/shares/share_splitting.go @@ -1,6 +1,7 @@ package shares import ( + "bytes" "errors" "fmt" "sort" @@ -21,7 +22,7 @@ var ( // Split converts block data into encoded shares, optionally using share indexes // that are encoded as wrapped transactions. Most use cases out of this package // should use these share indexes and therefore set useShareIndexes to true. -func Split(data coretypes.Data, useShareIndexes bool) ([][]byte, error) { +func Split(data coretypes.Data, useShareIndexes bool) ([]Share, error) { if data.OriginalSquareSize == 0 || !isPowerOf2(data.OriginalSquareSize) { return nil, fmt.Errorf("square size is not a power of two: %d", data.OriginalSquareSize) } @@ -43,7 +44,7 @@ func Split(data coretypes.Data, useShareIndexes bool) ([][]byte, error) { msgIndexes := ExtractShareIndexes(data.Txs) sort.Slice(msgIndexes, func(i, j int) bool { return msgIndexes[i] < msgIndexes[j] }) - var padding [][]byte + var padding []Share if len(data.Messages.MessagesList) > 0 { msgShareStart, _ := NextAlignedPowerOfTwo( currentShareCount, @@ -54,11 +55,11 @@ func Split(data coretypes.Data, useShareIndexes bool) ([][]byte, error) { if len(evdShares) > 0 { ns = appconsts.EvidenceNamespaceID } - padding = namespacedPaddedShares(ns, msgShareStart-currentShareCount).RawShares() + padding = namespacedPaddedShares(ns, msgShareStart-currentShareCount) } currentShareCount += len(padding) - var msgShares [][]byte + var msgShares []Share if msgIndexes != nil && int(msgIndexes[0]) < currentShareCount { return nil, ErrUnexpectedFirstMessageShareIndex } @@ -68,7 +69,7 @@ func Split(data coretypes.Data, useShareIndexes bool) ([][]byte, error) { return nil, err } currentShareCount += len(msgShares) - tailShares := TailPaddingShares(wantShareCount - currentShareCount).RawShares() + tailShares := TailPaddingShares(wantShareCount - currentShareCount) // todo: optimize using a predefined slice shares := append(append(append(append( @@ -105,15 +106,15 @@ func ExtractShareIndexes(txs coretypes.Txs) []uint32 { return msgIndexes } -func SplitTxs(txs coretypes.Txs) [][]byte { +func SplitTxs(txs coretypes.Txs) []Share { writer := NewCompactShareSplitter(appconsts.TxNamespaceID, appconsts.ShareVersion) for _, tx := range txs { writer.WriteTx(tx) } - return writer.Export().RawShares() + return writer.Export() } -func SplitEvidence(evd coretypes.EvidenceList) ([][]byte, error) { +func SplitEvidence(evd coretypes.EvidenceList) ([]Share, error) { writer := NewCompactShareSplitter(appconsts.EvidenceNamespaceID, appconsts.ShareVersion) for _, ev := range evd { err := writer.WriteEvidence(ev) @@ -121,10 +122,10 @@ func SplitEvidence(evd coretypes.EvidenceList) ([][]byte, error) { return nil, err } } - return writer.Export().RawShares(), nil + return writer.Export(), nil } -func SplitMessages(cursor int, indexes []uint32, msgs []coretypes.Message, useShareIndexes bool) ([][]byte, error) { +func SplitMessages(cursor int, indexes []uint32, msgs []coretypes.Message, useShareIndexes bool) ([]Share, error) { if useShareIndexes && len(indexes) != len(msgs) { return nil, ErrIncorrectNumberOfIndexes } @@ -136,5 +137,24 @@ func SplitMessages(cursor int, indexes []uint32, msgs []coretypes.Message, useSh writer.WriteNamespacedPaddedShares(paddedShareCount) } } - return writer.Export().RawShares(), nil + return writer.Export(), nil +} + +var tailPaddingInfo, _ = NewInfoByte(appconsts.ShareVersion, false) + +// tail is filler for all tail padded shares +// it is allocated once and used everywhere +var tailPaddingShare = append(append( + append(make([]byte, 0, appconsts.ShareSize), appconsts.TailPaddingNamespaceID...), + byte(tailPaddingInfo)), + bytes.Repeat([]byte{0}, appconsts.ShareSize-appconsts.NamespaceSize-appconsts.ShareInfoBytes)..., +) + +// TailPaddingShares creates n tail padding shares. +func TailPaddingShares(n int) []Share { + shares := make([]Share, n) + for i := 0; i < n; i++ { + shares[i] = tailPaddingShare + } + return shares } diff --git a/pkg/shares/share_splitting_test.go b/pkg/shares/share_splitting_test.go index 3f826ffdf7..699acb0518 100644 --- a/pkg/shares/share_splitting_test.go +++ b/pkg/shares/share_splitting_test.go @@ -12,18 +12,18 @@ func TestSplitTxs(t *testing.T) { type testCase struct { name string txs coretypes.Txs - want [][]byte + want []Share } testCases := []testCase{ { name: "empty txs", txs: coretypes.Txs{}, - want: [][]byte{}, + want: []Share{}, }, { name: "one small tx", txs: coretypes.Txs{coretypes.Tx{0xa}}, - want: [][]uint8{ + want: []Share{ append([]uint8{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id 0x1, // info byte @@ -37,7 +37,7 @@ func TestSplitTxs(t *testing.T) { { name: "two small txs", txs: coretypes.Txs{coretypes.Tx{0xa}, coretypes.Tx{0xb}}, - want: [][]uint8{ + want: []Share{ append([]uint8{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id 0x1, // info byte @@ -53,7 +53,7 @@ func TestSplitTxs(t *testing.T) { { name: "one large tx that spans two shares", txs: coretypes.Txs{bytes.Repeat([]byte{0xC}, 241)}, - want: [][]uint8{ + want: []Share{ append([]uint8{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id 0x1, // info byte @@ -72,7 +72,7 @@ func TestSplitTxs(t *testing.T) { { name: "one small tx then one large tx that spans two shares", txs: coretypes.Txs{coretypes.Tx{0xd}, bytes.Repeat([]byte{0xe}, 241)}, - want: [][]uint8{ + want: []Share{ append([]uint8{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id 0x1, // info byte @@ -93,7 +93,7 @@ func TestSplitTxs(t *testing.T) { { name: "one large tx that spans two shares then one small tx", txs: coretypes.Txs{bytes.Repeat([]byte{0xe}, 241), coretypes.Tx{0xd}}, - want: [][]uint8{ + want: []Share{ append([]uint8{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id 0x1, // info byte diff --git a/pkg/shares/shares.go b/pkg/shares/shares.go index f3c6fd64fe..810c75a491 100644 --- a/pkg/shares/shares.go +++ b/pkg/shares/shares.go @@ -1,36 +1,65 @@ package shares import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/nmt/namespace" ) // Share contains the raw share data (including namespace ID). type Share []byte -// NamespacedShare extends a Share with the corresponding namespace. -type NamespacedShare struct { - Share - ID namespace.ID +func (s Share) NamespaceID() namespace.ID { + if len(s) < appconsts.NamespaceSize { + panic(fmt.Sprintf("share %s is too short to contain a namespace ID", s)) + } + return namespace.ID(s[:appconsts.NamespaceSize]) +} + +func (s Share) InfoByte() (InfoByte, error) { + if len(s) < appconsts.NamespaceSize+appconsts.ShareInfoBytes { + panic(fmt.Sprintf("share %s is too short to contain an info byte", s)) + } + // the info byte is the first byte after the namespace ID + unparsed := s[appconsts.NamespaceSize] + return ParseInfoByte(unparsed) } -func (n NamespacedShare) NamespaceID() namespace.ID { - return n.ID +func (s Share) MessageLength() (uint64, error) { + infoByte, err := s.InfoByte() + if err != nil { + return 0, err + } + if !infoByte.IsMessageStart() { + return 0, nil + } + if s.isCompactShare() || len(s) < appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes { + panic(fmt.Sprintf("compact share %s is too short to contain message length", s)) + } + reader := bytes.NewReader(s[appconsts.NamespaceSize+appconsts.ShareInfoBytes:]) + return binary.ReadUvarint(reader) } -func (n NamespacedShare) Data() []byte { - return n.Share +// isCompactShare returns true if this share is a compact share. +func (s Share) isCompactShare() bool { + return s.NamespaceID().Equal(appconsts.TxNamespaceID) || s.NamespaceID().Equal(appconsts.EvidenceNamespaceID) } -// NamespacedShares is just a list of NamespacedShare elements. -// It can be used to extract the raw shares. -type NamespacedShares []NamespacedShare +func ToBytes(shares []Share) (bytes [][]byte) { + bytes = make([][]byte, len(shares)) + for i, share := range shares { + bytes[i] = []byte(share) + } + return bytes +} -// RawShares returns the raw shares that can be fed into the erasure coding -// library (e.g. rsmt2d). -func (ns NamespacedShares) RawShares() [][]byte { - res := make([][]byte, len(ns)) - for i, nsh := range ns { - res[i] = nsh.Share +func FromBytes(bytes [][]byte) (shares []Share) { + shares = make([]Share, len(bytes)) + for i, b := range bytes { + shares[i] = Share(b) } - return res + return shares } diff --git a/pkg/shares/shares_test.go b/pkg/shares/shares_test.go index b72d8883e7..0f726d10b1 100644 --- a/pkg/shares/shares_test.go +++ b/pkg/shares/shares_test.go @@ -200,8 +200,9 @@ func TestMerge(t *testing.T) { tc.maxSize, ) - rawShares, err := Split(data, false) + shares, err := Split(data, false) require.NoError(t, err) + rawShares := ToBytes(shares) eds, err := rsmt2d.ComputeExtendedDataSquare(rawShares, appconsts.DefaultCodec(), rsmt2d.NewDefaultTree) if err != nil { diff --git a/pkg/shares/sparse_shares_test.go b/pkg/shares/sparse_shares_test.go index 1b14975bf6..f3cc2659aa 100644 --- a/pkg/shares/sparse_shares_test.go +++ b/pkg/shares/sparse_shares_test.go @@ -47,8 +47,9 @@ func Test_parseSparseShares(t *testing.T) { msgs.SortMessages() shares, _ := SplitMessages(0, nil, msgs.MessagesList, false) + rawShares := ToBytes(shares) - parsedMsgs, err := parseSparseShares(shares) + parsedMsgs, err := parseSparseShares(rawShares) if err != nil { t.Error(err) } @@ -64,8 +65,12 @@ func Test_parseSparseShares(t *testing.T) { t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) { msgs := generateRandomlySizedMessages(tc.msgCount, tc.msgSize) shares, _ := SplitMessages(0, nil, msgs.MessagesList, false) + rawShares := make([][]byte, len(shares)) + for i, share := range shares { + rawShares[i] = []byte(share) + } - parsedMsgs, err := parseSparseShares(shares) + parsedMsgs, err := parseSparseShares(rawShares) if err != nil { t.Error(err) } @@ -94,7 +99,9 @@ func TestParsePaddedMsg(t *testing.T) { msgWr.WriteNamespacedPaddedShares(4) msgWr.Write(msgs.MessagesList[1]) msgWr.WriteNamespacedPaddedShares(10) - pmsgs, err := parseSparseShares(msgWr.Export().RawShares()) + shares := msgWr.Export() + rawShares := ToBytes(shares) + pmsgs, err := parseSparseShares(rawShares) require.NoError(t, err) require.Equal(t, msgs.MessagesList, pmsgs) } @@ -104,7 +111,7 @@ func TestMsgShareContainsInfoByte(t *testing.T) { smallMsg := generateRandomMessage(appconsts.SparseShareContentSize / 2) sss.Write(smallMsg) - shares := sss.Export().RawShares() + shares := sss.Export() got := shares[0][appconsts.NamespaceSize : appconsts.NamespaceSize+appconsts.ShareInfoBytes][0] @@ -120,7 +127,7 @@ func TestContiguousMsgShareContainsInfoByte(t *testing.T) { longMsg := generateRandomMessage(appconsts.SparseShareContentSize * 4) sss.Write(longMsg) - shares := sss.Export().RawShares() + shares := sss.Export() // we expect longMsg to occupy more than one share assert.Condition(t, func() bool { return len(shares) > 1 }) diff --git a/pkg/shares/split_compact_shares.go b/pkg/shares/split_compact_shares.go index 7cb6f9ed09..72211985ec 100644 --- a/pkg/shares/split_compact_shares.go +++ b/pkg/shares/split_compact_shares.go @@ -1,7 +1,6 @@ package shares import ( - "bytes" "encoding/binary" "fmt" @@ -15,8 +14,8 @@ import ( // increasing set of shares. It is used to lazily split block data such as // transactions, intermediate state roots, and evidence into shares. type CompactShareSplitter struct { - shares []NamespacedShare - pendingShare NamespacedShare + shares []Share + pendingShare Share namespace namespace.ID version uint8 } @@ -24,7 +23,7 @@ type CompactShareSplitter struct { // NewCompactShareSplitter returns a CompactShareSplitter using the provided // namespace. func NewCompactShareSplitter(ns namespace.ID, version uint8) *CompactShareSplitter { - pendingShare := NamespacedShare{ID: ns, Share: make([]byte, 0, appconsts.ShareSize)} + pendingShare := make([]byte, 0, appconsts.ShareSize) infoByte, err := NewInfoByte(version, true) if err != nil { panic(err) @@ -32,10 +31,10 @@ func NewCompactShareSplitter(ns namespace.ID, version uint8) *CompactShareSplitt placeholderDataLength := make([]byte, appconsts.FirstCompactShareDataLengthBytes) placeholderReservedBytes := make([]byte, appconsts.CompactShareReservedBytes) - pendingShare.Share = append(pendingShare.Share, ns...) - pendingShare.Share = append(pendingShare.Share, byte(infoByte)) - pendingShare.Share = append(pendingShare.Share, placeholderDataLength...) - pendingShare.Share = append(pendingShare.Share, placeholderReservedBytes...) + pendingShare = append(pendingShare, ns...) + pendingShare = append(pendingShare, byte(infoByte)) + pendingShare = append(pendingShare, placeholderDataLength...) + pendingShare = append(pendingShare, placeholderReservedBytes...) return &CompactShareSplitter{pendingShare: pendingShare, namespace: ns} } @@ -67,19 +66,19 @@ func (css *CompactShareSplitter) WriteBytes(rawData []byte) { txCursor := len(rawData) for txCursor != 0 { // find the len left in the pending share - pendingLeft := appconsts.ShareSize - len(css.pendingShare.Share) + pendingLeft := appconsts.ShareSize - len(css.pendingShare) // if we can simply add the tx to the share without creating a new // pending share, do so and return if len(rawData) <= pendingLeft { - css.pendingShare.Share = append(css.pendingShare.Share, rawData...) + css.pendingShare = append(css.pendingShare, rawData...) break } // if we can only add a portion of the transaction to the pending share, // then we add it and add the pending share to the finalized shares. chunk := rawData[:pendingLeft] - css.pendingShare.Share = append(css.pendingShare.Share, chunk...) + css.pendingShare = append(css.pendingShare, chunk...) css.stackPending() // update the cursor @@ -88,14 +87,14 @@ func (css *CompactShareSplitter) WriteBytes(rawData []byte) { } // if the share is exactly the correct size, then append to shares - if len(css.pendingShare.Share) == appconsts.ShareSize { + if len(css.pendingShare) == appconsts.ShareSize { css.stackPending() } } // stackPending will add the pending share to accumlated shares provided that it is long enough func (css *CompactShareSplitter) stackPending() { - if len(css.pendingShare.Share) < appconsts.ShareSize { + if len(css.pendingShare) < appconsts.ShareSize { return } css.shares = append(css.shares, css.pendingShare) @@ -108,22 +107,19 @@ func (css *CompactShareSplitter) stackPending() { placeholderReservedBytes := make([]byte, appconsts.CompactShareReservedBytes) newPendingShare = append(newPendingShare, byte(infoByte)) newPendingShare = append(newPendingShare, placeholderReservedBytes...) - css.pendingShare = NamespacedShare{ - Share: newPendingShare, - ID: css.namespace, - } + css.pendingShare = newPendingShare } // Export finalizes and returns the underlying compact shares. -func (css *CompactShareSplitter) Export() NamespacedShares { +func (css *CompactShareSplitter) Export() []Share { if css.isEmpty() { - return []NamespacedShare{} + return []Share{} } var bytesOfPadding int // add the pending share to the current shares before returning if !css.isEmptyPendingShare() { - css.pendingShare.Share, bytesOfPadding = zeroPadIfNecessary(css.pendingShare.Share, appconsts.ShareSize) + css.pendingShare, bytesOfPadding = zeroPadIfNecessary(css.pendingShare, appconsts.ShareSize) css.shares = append(css.shares, css.pendingShare) } @@ -154,17 +150,12 @@ func (css *CompactShareSplitter) writeDataLengthVarintToFirstShare(dataLengthVar // write the data length varint to the first share firstShare := css.shares[0] - rawFirstShare := firstShare.Data() for i := 0; i < appconsts.FirstCompactShareDataLengthBytes; i++ { - rawFirstShare[appconsts.NamespaceSize+appconsts.ShareInfoBytes+i] = dataLengthVarint[i] + firstShare[appconsts.NamespaceSize+appconsts.ShareInfoBytes+i] = dataLengthVarint[i] } // replace existing first share with new first share - newFirstShare := NamespacedShare{ - Share: rawFirstShare, - ID: firstShare.NamespaceID(), - } - css.shares[0] = newFirstShare + css.shares[0] = firstShare } // maybeWriteReservedByteToPendingShare will be a no-op if the reserved byte has @@ -175,16 +166,16 @@ func (css *CompactShareSplitter) maybeWriteReservedByteToPendingShare() { return } - locationOfNextUnit := len(css.pendingShare.Share) + locationOfNextUnit := len(css.pendingShare) if locationOfNextUnit >= appconsts.ShareSize { panic(fmt.Sprintf("location of next unit %v is greater than or equal to the share size %v", locationOfNextUnit, appconsts.ShareSize)) } // write the location of next unit to the reserved byte of the pending share if css.isPendingShareTheFirstShare() { - css.pendingShare.Share[appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareContentSize+appconsts.CompactShareReservedBytes][0] = byte(locationOfNextUnit) + css.pendingShare[appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareContentSize+appconsts.CompactShareReservedBytes][0] = byte(locationOfNextUnit) } else { - css.pendingShare.Share[appconsts.NamespaceSize+appconsts.ShareInfoBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.CompactShareReservedBytes][0] = byte(locationOfNextUnit) + css.pendingShare[appconsts.NamespaceSize+appconsts.ShareInfoBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.CompactShareReservedBytes][0] = byte(locationOfNextUnit) } } @@ -193,9 +184,9 @@ func (css *CompactShareSplitter) isEmptyReservedByte() bool { var reservedByte byte if css.isPendingShareTheFirstShare() { - reservedByte = css.pendingShare.Share[appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareContentSize+appconsts.CompactShareReservedBytes][0] + reservedByte = css.pendingShare[appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareContentSize+appconsts.CompactShareReservedBytes][0] } else { - reservedByte = css.pendingShare.Share[appconsts.NamespaceSize+appconsts.ShareInfoBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.CompactShareReservedBytes][0] + reservedByte = css.pendingShare[appconsts.NamespaceSize+appconsts.ShareInfoBytes : appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.CompactShareReservedBytes][0] } return reservedByte == 0 @@ -222,9 +213,9 @@ func (css *CompactShareSplitter) dataLength(bytesOfPadding int) uint64 { // isEmptyPendingShare returns true if the pending share is empty, false otherwise. func (css *CompactShareSplitter) isEmptyPendingShare() bool { if css.isPendingShareTheFirstShare() { - return len(css.pendingShare.Share) == appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes+appconsts.CompactShareReservedBytes + return len(css.pendingShare) == appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.FirstCompactShareDataLengthBytes+appconsts.CompactShareReservedBytes } - return len(css.pendingShare.Share) == appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.CompactShareReservedBytes + return len(css.pendingShare) == appconsts.NamespaceSize+appconsts.ShareInfoBytes+appconsts.CompactShareReservedBytes } // isPendingShareTheFirstShare returns true if the pending share is the first @@ -248,28 +239,6 @@ func (css *CompactShareSplitter) Count() (shareCount int) { return len(css.shares) } -var tailPaddingInfo, _ = NewInfoByte(appconsts.ShareVersion, false) - -// tail is filler for all tail padded shares -// it is allocated once and used everywhere -var tailPaddingShare = append(append( - append(make([]byte, 0, appconsts.ShareSize), appconsts.TailPaddingNamespaceID...), - byte(tailPaddingInfo)), - bytes.Repeat([]byte{0}, appconsts.ShareSize-appconsts.NamespaceSize-appconsts.ShareInfoBytes)..., -) - -// TailPaddingShares creates n tail padding shares. -func TailPaddingShares(n int) NamespacedShares { - shares := make([]NamespacedShare, n) - for i := 0; i < n; i++ { - shares[i] = NamespacedShare{ - Share: tailPaddingShare, - ID: appconsts.TailPaddingNamespaceID, - } - } - return shares -} - // MarshalDelimitedTx prefixes a transaction with the length of the transaction // encoded as a varint. func MarshalDelimitedTx(tx coretypes.Tx) ([]byte, error) { @@ -279,24 +248,6 @@ func MarshalDelimitedTx(tx coretypes.Tx) ([]byte, error) { return append(lenBuf[:n], tx...), nil } -func namespacedPaddedShares(ns []byte, count int) NamespacedShares { - infoByte, err := NewInfoByte(appconsts.ShareVersion, true) - if err != nil { - panic(err) - } - shares := make([]NamespacedShare, count) - for i := 0; i < count; i++ { - shares[i] = NamespacedShare{ - Share: append(append(append( - make([]byte, 0, appconsts.ShareSize), ns...), - byte(infoByte)), - make([]byte, appconsts.SparseShareContentSize)...), - ID: ns, - } - } - return shares -} - func min(a, b int) int { if a <= b { return a diff --git a/pkg/shares/split_sparse_shares.go b/pkg/shares/split_sparse_shares.go index 8bcc906245..91c61fef88 100644 --- a/pkg/shares/split_sparse_shares.go +++ b/pkg/shares/split_sparse_shares.go @@ -13,7 +13,7 @@ import ( // included in a data square. It also has methods to help progressively count // how many shares the messages written take up. type SparseShareSplitter struct { - shares [][]NamespacedShare + shares []Share count int } @@ -27,9 +27,9 @@ func (sss *SparseShareSplitter) Write(msg coretypes.Message) { if err != nil { panic(fmt.Sprintf("app accepted a Message that can not be encoded %#v", msg)) } - newShares := make([]NamespacedShare, 0) + newShares := make([]Share, 0) newShares = AppendToShares(newShares, msg.NamespaceID, rawMsg) - sss.shares = append(sss.shares, newShares) + sss.shares = append(sss.shares, newShares...) sss.count += len(newShares) } @@ -39,7 +39,7 @@ func (sss *SparseShareSplitter) RemoveMessage(i int) (int, error) { j := 1 initialCount := sss.count if len(sss.shares) > i+1 { - _, msgLen, err := ParseDelimiter(sss.shares[i+1][0].Share[appconsts.NamespaceSize:]) + msgLen, err := sss.shares[i+1].MessageLength() if err != nil { return 0, err } @@ -68,21 +68,13 @@ func (sss *SparseShareSplitter) WriteNamespacedPaddedShares(count int) { return } lastMessage := sss.shares[len(sss.shares)-1] - sss.shares = append(sss.shares, namespacedPaddedShares(lastMessage[0].ID, count)) + sss.shares = append(sss.shares, namespacedPaddedShares(lastMessage.NamespaceID(), count)...) sss.count += count } // Export finalizes and returns the underlying shares. -func (sss *SparseShareSplitter) Export() NamespacedShares { - shares := make([]NamespacedShare, sss.count) - cursor := 0 - for _, namespacedShares := range sss.shares { - for _, share := range namespacedShares { - shares[cursor] = share - cursor++ - } - } - return shares +func (sss *SparseShareSplitter) Export() []Share { + return sss.shares } // Count returns the current number of shares that will be made if exporting. @@ -92,7 +84,7 @@ func (sss *SparseShareSplitter) Count() int { // AppendToShares appends raw data as shares. // Used for messages. -func AppendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) []NamespacedShare { +func AppendToShares(shares []Share, nid namespace.ID, rawData []byte) []Share { if len(rawData) <= appconsts.SparseShareContentSize { infoByte, err := NewInfoByte(appconsts.ShareVersion, true) if err != nil { @@ -105,8 +97,7 @@ func AppendToShares(shares []NamespacedShare, nid namespace.ID, rawData []byte) rawData..., ) paddedShare, _ := zeroPadIfNecessary(rawShare, appconsts.ShareSize) - share := NamespacedShare{paddedShare, nid} - shares = append(shares, share) + shares = append(shares, paddedShare) } else { // len(rawData) > MsgShareSize shares = append(shares, splitMessage(rawData, nid)...) } @@ -125,8 +116,7 @@ func MarshalDelimitedMessage(msg coretypes.Message) ([]byte, error) { // splitMessage breaks the data in a message into the minimum number of // namespaced shares -func splitMessage(rawData []byte, nid namespace.ID) NamespacedShares { - shares := make([]NamespacedShare, 0) +func splitMessage(rawData []byte, nid namespace.ID) (shares []Share) { infoByte, err := NewInfoByte(appconsts.ShareVersion, true) if err != nil { panic(err) @@ -137,7 +127,7 @@ func splitMessage(rawData []byte, nid namespace.ID) NamespacedShares { byte(infoByte)), rawData[:appconsts.SparseShareContentSize]..., ) - shares = append(shares, NamespacedShare{firstRawShare, nid}) + shares = append(shares, firstRawShare) rawData = rawData[appconsts.SparseShareContentSize:] for len(rawData) > 0 { shareSizeOrLen := min(appconsts.SparseShareContentSize, len(rawData)) @@ -152,9 +142,28 @@ func splitMessage(rawData []byte, nid namespace.ID) NamespacedShares { rawData[:shareSizeOrLen]..., ) paddedShare, _ := zeroPadIfNecessary(rawShare, appconsts.ShareSize) - share := NamespacedShare{paddedShare, nid} - shares = append(shares, share) + shares = append(shares, paddedShare) rawData = rawData[shareSizeOrLen:] } return shares } + +func namespacedPaddedShares(ns namespace.ID, count int) []Share { + shares := make([]Share, count) + for i := 0; i < count; i++ { + shares[i] = namespacedPaddedShare(ns) + } + return shares +} + +func namespacedPaddedShare(ns namespace.ID) Share { + infoByte, err := NewInfoByte(appconsts.ShareVersion, true) + if err != nil { + panic(err) + } + share := make([]byte, 0, appconsts.ShareSize) + share = append(share, ns...) + share = append(share, byte(infoByte)) + share = append(share, appconsts.NameSpacedPaddedShareBytes...) + return share +} diff --git a/x/payment/types/payfordata.go b/x/payment/types/payfordata.go index f0da3ae5d1..80c8d8e512 100644 --- a/x/payment/types/payfordata.go +++ b/x/payment/types/payfordata.go @@ -13,7 +13,7 @@ import ( "github.com/tendermint/tendermint/crypto/merkle" coretypes "github.com/tendermint/tendermint/types" - shares "github.com/celestiaorg/celestia-app/pkg/shares" + appshares "github.com/celestiaorg/celestia-app/pkg/shares" ) const ( @@ -124,7 +124,7 @@ func CreateCommitment(squareSize uint64, namespace, message []byte) ([]byte, err // split into shares that are length delimited and include the namespace in // each share - shares, err := shares.SplitMessages(0, nil, msg.MessagesList, false) + shares, err := appshares.SplitMessages(0, nil, msg.MessagesList, false) if err != nil { return nil, err } @@ -142,7 +142,7 @@ func CreateCommitment(squareSize uint64, namespace, message []byte) ([]byte, err leafSets := make([][][]byte, len(heights)) cursor := uint64(0) for i, height := range heights { - leafSets[i] = shares[cursor : cursor+height] + leafSets[i] = appshares.ToBytes(shares[cursor : cursor+height]) cursor = cursor + height }