chore!: break out BlobTx and IndexWrapper types into separate tx pack…

…age (#97)

While updating `celestia-app`, it felt out of place that BlobTx and
IndexWrapper should be part of the `shares` package. This moves them out

README.md

@@ -7,8 +7,10 @@
 Package   | Description
 ----------|---------------------------------------------------------------------------------------------------------------------
 inclusion | Package inclusion contains functions to generate the blob share commitment from a given blob.
+proto     | Package contains proto definitions and go generated code
 share     | Package share contains encoding and decoding logic from blobs to shares.
 square    | Package square implements the logic to construct the original data square based on a list of transactions.
+tx        | Package tx contains BlobTx and IndexWrapper types
 
 ## Installation
 

builder.go

@@ -9,6 +9,7 @@ import (
 	"github.com/celestiaorg/go-square/v2/inclusion"
 	v1 "github.com/celestiaorg/go-square/v2/proto/blob/v1"
 	"github.com/celestiaorg/go-square/v2/share"
+	"github.com/celestiaorg/go-square/v2/tx"
 	"golang.org/x/exp/constraints"
 	"google.golang.org/protobuf/proto"
 )
@@ -88,11 +89,7 @@ func (b *Builder) AppendTx(tx []byte) bool {
 // AppendBlobTx attempts to allocate the blob transaction to the square. It returns false if there is not
 // enough space in the square to fit the transaction.
 func (b *Builder) AppendBlobTx(blobTx *share.BlobTx) bool {
-	iw := &v1.IndexWrapper{
-		Tx:           blobTx.Tx,
-		TypeId:       share.ProtoIndexWrapperTypeID,
-		ShareIndexes: worstCaseShareIndexes(len(blobTx.Blobs)),
-	}
+	iw := tx.NewIndexWrapper(blobTx.Tx, worstCaseShareIndexes(len(blobTx.Blobs))...)
 	size := proto.Size(iw)
 	pfbShareDiff := b.PfbCounter.Add(size)
 

builder_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/celestiaorg/go-square/v2"
 	"github.com/celestiaorg/go-square/v2/internal/test"
 	"github.com/celestiaorg/go-square/v2/share"
+	"github.com/celestiaorg/go-square/v2/tx"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -303,8 +304,8 @@ func TestSquareBlobPostions(t *testing.T) {
 			require.NoError(t, err)
 			txs, err := share.ParseTxs(square)
 			require.NoError(t, err)
-			for j, tx := range txs {
-				wrappedPFB, isWrappedPFB := share.UnmarshalIndexWrapper(tx)
+			for j, rawTx := range txs {
+				wrappedPFB, isWrappedPFB := tx.UnmarshalIndexWrapper(rawTx)
 				assert.True(t, isWrappedPFB)
 				assert.Equal(t, tt.expectedIndexes[j], wrappedPFB.ShareIndexes, j)
 			}

inclusion/commitment.go

@@ -15,7 +15,7 @@ type MerkleRootFn func([][]byte) []byte
 // [data square layout rationale]: ../../specs/src/specs/data_square_layout.md
 // [blob share commitment rules]: ../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules
 func CreateCommitment(blob *sh.Blob, merkleRootFn MerkleRootFn, subtreeRootThreshold int) ([]byte, error) {
-	shares, err := sh.SplitBlobs(blob)
+	shares, err := splitBlobs(blob)
 	if err != nil {
 		return nil, err
 	}
@@ -106,3 +106,14 @@ func MerkleMountainRangeSizes(totalSize, maxTreeSize uint64) ([]uint64, error) {
 
 	return treeSizes, nil
 }
+
+// SplitBlobs splits the provided blobs into shares.
+func splitBlobs(blobs ...*sh.Blob) ([]sh.Share, error) {
+	writer := sh.NewSparseShareSplitter()
+	for _, blob := range blobs {
+		if err := writer.Write(blob); err != nil {
+			return nil, err
+		}
+	}
+	return writer.Export(), nil
+}

share/compact_shares_test.go

@@ -77,7 +77,7 @@ func Test_processCompactShares(t *testing.T) {
 		t.Run(fmt.Sprintf("%s idendically sized", tc.name), func(t *testing.T) {
 			txs := generateRandomTxs(tc.txCount, tc.txSize)
 
-			shares, _, _, err := SplitTxs(txs)
+			shares, _, err := splitTxs(txs)
 			require.NoError(t, err)
 
 			parsedTxs, err := parseCompactShares(shares, SupportedShareVersions)
@@ -95,7 +95,7 @@ func Test_processCompactShares(t *testing.T) {
 		t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
 			txs := generateRandomlySizedTxs(tc.txCount, tc.txSize)
 
-			txShares, _, _, err := SplitTxs(txs)
+			txShares, _, err := splitTxs(txs)
 			require.NoError(t, err)
 			parsedTxs, err := parseCompactShares(txShares, SupportedShareVersions)
 			if err != nil {
@@ -110,18 +110,9 @@ func Test_processCompactShares(t *testing.T) {
 	}
 }
 
-func TestAllSplit(t *testing.T) {
-	txs := generateRandomlySizedTxs(1000, 150)
-	txShares, _, _, err := SplitTxs(txs)
-	require.NoError(t, err)
-	resTxs, err := ParseTxs(txShares)
-	require.NoError(t, err)
-	assert.Equal(t, resTxs, txs)
-}
-
 func TestParseRandomOutOfContextShares(t *testing.T) {
 	txs := generateRandomlySizedTxs(1000, 150)
-	txShares, _, _, err := SplitTxs(txs)
+	txShares, _, err := splitTxs(txs)
 	require.NoError(t, err)
 
 	for i := 0; i < 1000; i++ {
@@ -160,7 +151,7 @@ func checkSubArray(txList [][]byte, subTxList [][]byte) bool {
 
 func TestParseOutOfContextSharesUsingShareRanges(t *testing.T) {
 	txs := generateRandomlySizedTxs(1000, 150)
-	txShares, _, shareRanges, err := SplitTxs(txs)
+	txShares, shareRanges, err := splitTxs(txs)
 	require.NoError(t, err)
 
 	for key, r := range shareRanges {
@@ -228,7 +219,7 @@ func Test_parseCompactSharesErrors(t *testing.T) {
 	}
 
 	txs := generateRandomTxs(2, ContinuationCompactShareContentSize*4)
-	txShares, _, _, err := SplitTxs(txs)
+	txShares, _, err := splitTxs(txs)
 	require.NoError(t, err)
 	rawShares := ToBytes(txShares)
 
@@ -268,3 +259,18 @@ func generateRandomlySizedTxs(count, maxSize int) [][]byte {
 	}
 	return txs
 }
+
+func splitTxs(txs [][]byte) ([]Share, map[[sha256.Size]byte]Range, error) {
+	txWriter := NewCompactShareSplitter(TxNamespace, ShareVersionZero)
+	for _, tx := range txs {
+		err := txWriter.WriteTx(tx)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	shares, err := txWriter.Export()
+	if err != nil {
+		return nil, nil, err
+	}
+	return shares, txWriter.ShareRanges(0), nil
+}

share/parse_sparse_shares_test.go

@@ -58,7 +58,7 @@ func Test_parseSparseShares(t *testing.T) {
 
 			SortBlobs(blobs)
 
-			shares, err := SplitBlobs(blobs...)
+			shares, err := splitBlobs(blobs...)
 			require.NoError(t, err)
 			parsedBlobs, err := parseSparseShares(shares, SupportedShareVersions)
 			if err != nil {
@@ -74,8 +74,8 @@ func Test_parseSparseShares(t *testing.T) {
 
 		// run the same tests using randomly sized blobs with caps of tc.blobSize
 		t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
-			blobs := GenerateRandomlySizedBlobs(tc.blobCount, tc.blobSize)
-			shares, err := SplitBlobs(blobs...)
+			blobs := generateRandomlySizedBlobs(tc.blobCount, tc.blobSize)
+			shares, err := splitBlobs(blobs...)
 			require.NoError(t, err)
 			parsedBlobs, err := parseSparseShares(shares, SupportedShareVersions)
 			if err != nil {
@@ -154,7 +154,7 @@ func Test_parseSparseSharesWithNamespacedPadding(t *testing.T) {
 func Test_parseShareVersionOne(t *testing.T) {
 	v1blob, err := NewV1Blob(MustNewV0Namespace(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize)), []byte("data"), bytes.Repeat([]byte{1}, SignerSize))
 	require.NoError(t, err)
-	v1shares, err := SplitBlobs(v1blob)
+	v1shares, err := splitBlobs(v1blob)
 	require.NoError(t, err)
 
 	parsedBlobs, err := parseSparseShares(v1shares, SupportedShareVersions)
@@ -181,7 +181,7 @@ func generateRandomBlob(dataSize int) *Blob {
 	return generateRandomBlobWithNamespace(ns, dataSize)
 }
 
-func GenerateRandomlySizedBlobs(count, maxBlobSize int) []*Blob {
+func generateRandomlySizedBlobs(count, maxBlobSize int) []*Blob {
 	blobs := make([]*Blob, count)
 	for i := 0; i < count; i++ {
 		blobs[i] = generateRandomBlob(rand.Intn(maxBlobSize-1) + 1)
@@ -198,3 +198,13 @@ func GenerateRandomlySizedBlobs(count, maxBlobSize int) []*Blob {
 	SortBlobs(blobs)
 	return blobs
 }
+
+func splitBlobs(blobs ...*Blob) ([]Share, error) {
+	writer := NewSparseShareSplitter()
+	for _, blob := range blobs {
+		if err := writer.Write(blob); err != nil {
+			return nil, err
+		}
+	}
+	return writer.Export(), nil
+}

share/parse_test.go

@@ -15,17 +15,17 @@ func TestParseShares(t *testing.T) {
 	ns1 := MustNewV0Namespace(bytes.Repeat([]byte{1}, NamespaceVersionZeroIDSize))
 	ns2 := MustNewV0Namespace(bytes.Repeat([]byte{2}, NamespaceVersionZeroIDSize))
 
-	txShares, _, _, err := SplitTxs(generateRandomTxs(2, 1000))
+	txShares, _, err := splitTxs(generateRandomTxs(2, 1000))
 	require.NoError(t, err)
 	txShareStart := txShares[0]
 	txShareContinuation := txShares[1]
 
-	blobOneShares, err := SplitBlobs(generateRandomBlobWithNamespace(ns1, 1000))
+	blobOneShares, err := splitBlobs(generateRandomBlobWithNamespace(ns1, 1000))
 	require.NoError(t, err)
 	blobOneStart := blobOneShares[0]
 	blobOneContinuation := blobOneShares[1]
 
-	blobTwoShares, err := SplitBlobs(generateRandomBlobWithNamespace(ns2, 1000))
+	blobTwoShares, err := splitBlobs(generateRandomBlobWithNamespace(ns2, 1000))
 	require.NoError(t, err)
 	blobTwoStart := blobTwoShares[0]
 	blobTwoContinuation := blobTwoShares[1]

share/share_benchmark_test.go

@@ -16,14 +16,16 @@ func BenchmarkBlobsToShares(b *testing.B) {
 			b.Run(fmt.Sprintf("ShareEncoding%dBlobs%dBytes", numBlobs, size), func(b *testing.B) {
 				b.ReportAllocs()
 				blobs := test.GenerateBlobs(test.Repeat(size, numBlobs)...)
-
 				b.ResetTimer()
 				for i := 0; i < b.N; i++ {
 					// Convert blob to shares
-					_, err := share.SplitBlobs(blobs...)
-					if err != nil {
-						b.Fatal("Failed to split blob into shares:", err)
+					writer := share.NewSparseShareSplitter()
+					for _, blob := range blobs {
+						if err := writer.Write(blob); err != nil {
+							b.Fatal("Failed to write blob into shares:", err)
+						}
 					}
+					_ = writer.Export()
 				}
 			})
 		}
@@ -38,10 +40,13 @@ func BenchmarkSharesToBlobs(b *testing.B) {
 			b.Run(fmt.Sprintf("ShareDecoding%dBlobs%dBytes", numBlobs, size), func(b *testing.B) {
 				b.ReportAllocs()
 				blobs := test.GenerateBlobs(test.Repeat(size, numBlobs)...)
-				s, err := share.SplitBlobs(blobs...)
-				if err != nil {
-					b.Fatal("Failed to split blob into shares:", err)
+				writer := share.NewSparseShareSplitter()
+				for _, blob := range blobs {
+					if err := writer.Write(blob); err != nil {
+						b.Fatal("Failed to write blob into shares:", err)
+					}
 				}
+				s := writer.Export()
 
 				b.ResetTimer()
 				for i := 0; i < b.N; i++ {

share/share_sequence_test.go

@@ -250,3 +250,8 @@ func FuzzValidSequenceLen(f *testing.F) {
 		assert.NoError(t, err)
 	})
 }
+
+// padShare returns a share padded with trailing zeros.
+func padShare(share Share) (paddedShare Share) {
+	return fillShare(share, 0)
+}