chore!: add native blob tx (#94)

Closes: #77,
#33

.golangci.yml

@@ -12,6 +12,7 @@ linters:
     - prealloc
     - stylecheck
     - gocritic
+    - prealloc
 
 linters-settings:
   nakedret:

README.md

@@ -2,12 +2,12 @@
 
 [![Go Reference](https://pkg.go.dev/badge/github.com/celestiaorg/go-square.svg)](https://pkg.go.dev/github.com/celestiaorg/go-square)
 
-`go-square` is a Go module that provides data structures and utilities for interacting with data squares in the Celestia network. The data square is a special form of block serialization in the Celestia blockchain. This repo deals with the original data square which is distinct from the extended data square. Operations on the extended data square are handled by [rsmt2d](https://github.com/celestiaorg/rsmt2d).
+`go-square` is a Go module that provides data structures and utilities for interacting with data squares in the Celestia network. The data square is a special form of block serialization in the Celestia blockchain designed for sampling. This repo deals with the original data square which is distinct from the extended data square. Operations on the extended data square are handled by [rsmt2d](https://github.com/celestiaorg/rsmt2d).
 
 Package   | Description
 ----------|---------------------------------------------------------------------------------------------------------------------
 inclusion | Package inclusion contains functions to generate the blob share commitment from a given blob.
-share    | Package share contains the Share data structure.
+share     | Package share contains encoding and decoding logic from blobs to shares.
 square    | Package square implements the logic to construct the original data square based on a list of transactions.
 
 ## Installation

builder.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 
 	"github.com/celestiaorg/go-square/inclusion"
+	v1 "github.com/celestiaorg/go-square/proto/blob/v1"
 	"github.com/celestiaorg/go-square/share"
 	"golang.org/x/exp/constraints"
 	"google.golang.org/protobuf/proto"
@@ -20,7 +21,7 @@ type Builder struct {
 
 	// here we keep track of the pending data to go in a square
 	Txs   [][]byte
-	Pfbs  []*share.IndexWrapper
+	Pfbs  []*v1.IndexWrapper
 	Blobs []*Element
 
 	// for compact shares we use a counter to track the amount of shares needed
@@ -42,14 +43,17 @@ func NewBuilder(maxSquareSize int, subtreeRootThreshold int, txs ...[]byte) (*Bu
 		maxSquareSize:        maxSquareSize,
 		subtreeRootThreshold: subtreeRootThreshold,
 		Blobs:                make([]*Element, 0),
-		Pfbs:                 make([]*share.IndexWrapper, 0),
+		Pfbs:                 make([]*v1.IndexWrapper, 0),
 		Txs:                  make([][]byte, 0),
 		TxCounter:            share.NewCompactShareCounter(),
 		PfbCounter:           share.NewCompactShareCounter(),
 	}
 	seenFirstBlobTx := false
 	for idx, tx := range txs {
-		blobTx, isBlobTx := share.UnmarshalBlobTx(tx)
+		blobTx, isBlobTx, err := share.UnmarshalBlobTx(tx)
+		if err != nil && isBlobTx {
+			return nil, fmt.Errorf("unmarshalling blob tx at index %d: %w", idx, err)
+		}
 		if isBlobTx {
 			seenFirstBlobTx = true
 			if !builder.AppendBlobTx(blobTx) {
@@ -84,7 +88,7 @@ func (b *Builder) AppendTx(tx []byte) bool {
 // AppendBlobTx attempts to allocate the blob transaction to the square. It returns false if there is not
 // enough space in the square to fit the transaction.
 func (b *Builder) AppendBlobTx(blobTx *share.BlobTx) bool {
-	iw := &share.IndexWrapper{
+	iw := &v1.IndexWrapper{
 		Tx:           blobTx.Tx,
 		TypeId:       share.ProtoIndexWrapperTypeID,
 		ShareIndexes: worstCaseShareIndexes(len(blobTx.Blobs)),
@@ -95,13 +99,7 @@ func (b *Builder) AppendBlobTx(blobTx *share.BlobTx) bool {
 	// create a new blob element for each blob and track the worst-case share count
 	blobElements := make([]*Element, len(blobTx.Blobs))
 	maxBlobShareCount := 0
-	for idx, protoBlob := range blobTx.Blobs {
-		blob, err := share.NewBlobFromProto(protoBlob)
-		if err != nil {
-			// TODO: we should look at having a go native BlobTx type
-			// that we have already verified instead of doing it twice here
-			panic(fmt.Sprintf("invalid blob %d: %v", idx, err))
-		}
+	for idx, blob := range blobTx.Blobs {
 		blobElements[idx] = newElement(blob, len(b.Pfbs), idx, b.subtreeRootThreshold)
 		maxBlobShareCount += blobElements[idx].maxShareOffset()
 	}
@@ -321,7 +319,7 @@ func (b *Builder) FindTxShareRange(txIndex int) (share.Range, error) {
 	return share.NewRange(start, end), nil
 }
 
-func (b *Builder) GetWrappedPFB(txIndex int) (*share.IndexWrapper, error) {
+func (b *Builder) GetWrappedPFB(txIndex int) (*v1.IndexWrapper, error) {
 	if txIndex < 0 {
 		return nil, fmt.Errorf("txIndex %d must not be negative", txIndex)
 	}

builder_test.go

@@ -85,7 +85,8 @@ func TestBuilderRejectsBlobTransactions(t *testing.T) {
 			require.NoError(t, err)
 			txs := generateBlobTxsWithNamespaces(ns1.Repeat(len(tc.blobSize)), [][]int{tc.blobSize})
 			require.Len(t, txs, 1)
-			blobTx, isBlobTx := share.UnmarshalBlobTx(txs[0])
+			blobTx, isBlobTx, err := share.UnmarshalBlobTx(txs[0])
+			require.NoError(t, err)
 			require.True(t, isBlobTx)
 			require.Equal(t, tc.added, builder.AppendBlobTx(blobTx))
 		})
@@ -102,7 +103,7 @@ func TestBuilderInvalidConstructor(t *testing.T) {
 }
 
 func newTx(len int) []byte {
-	return bytes.Repeat([]byte{0}, share.RawTxSize(len))
+	return bytes.Repeat([]byte{0}, len-test.DelimLen(uint64(len)))
 }
 
 func TestBuilderFindTxShareRange(t *testing.T) {
@@ -118,8 +119,9 @@ func TestBuilderFindTxShareRange(t *testing.T) {
 
 	var lastEnd int
 	for idx, tx := range blockTxs {
-		blobTx, isBlobTx := share.UnmarshalBlobTx(tx)
+		blobTx, isBlobTx, err := share.UnmarshalBlobTx(tx)
 		if isBlobTx {
+			require.NoError(t, err)
 			tx = blobTx.Tx
 		}
 		shareRange, err := builder.FindTxShareRange(idx)
@@ -292,7 +294,8 @@ func TestSquareBlobPostions(t *testing.T) {
 			builder, err := square.NewBuilder(tt.squareSize, defaultSubtreeRootThreshold)
 			require.NoError(t, err)
 			for _, tx := range tt.blobTxs {
-				blobTx, isBlobTx := share.UnmarshalBlobTx(tx)
+				blobTx, isBlobTx, err := share.UnmarshalBlobTx(tx)
+				require.NoError(t, err)
 				require.True(t, isBlobTx)
 				_ = builder.AppendBlobTx(blobTx)
 			}

internal/test/factory.go

@@ -128,3 +128,9 @@ func Repeat[T any](s T, count int) []T {
 	}
 	return ss
 }
+
+// DelimLen calculates the length of the delimiter for a given unit size
+func DelimLen(size uint64) int {
+	lenBuf := make([]byte, binary.MaxVarintLen64)
+	return binary.PutUvarint(lenBuf, size)
+}