chore: refactor share splitting and merging

app/split_shares.go

@@ -191,6 +191,8 @@ func (sqwr *shareSplitter) writeMalleatedTx(
 		return false, nil, nil, err
 	}
 
+	// we use a share index of 0 here because this implementation doesn't
+	// support non-interactive defaults or the usuage of wrapped txs
 	wrappedTx, err := coretypes.WrapMalleatedTx(parentHash[:], 0, rawProcessedTx)
 	if err != nil {
 		return false, nil, nil, err

pkg/shares/contiguous_shares_test.go

@@ -0,0 +1,123 @@
+package shares
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/pkg/consts"
+	coretypes "github.com/tendermint/tendermint/types"
+)
+
+func TestContigShareWriter(t *testing.T) {
+	// note that this test is mainly for debugging purposes, the main round trip
+	// tests occur in TestMerge and Test_processContiguousShares
+	w := NewContiguousShareSplitter(consts.TxNamespaceID)
+	txs := generateRandomContiguousShares(33, 200)
+	for _, tx := range txs {
+		rawTx, _ := tx.MarshalDelimited()
+		w.WriteBytes(rawTx)
+	}
+	resShares := w.Export()
+	rawResTxs, err := processContiguousShares(resShares.RawShares())
+	resTxs := coretypes.ToTxs(rawResTxs)
+	require.NoError(t, err)
+
+	assert.Equal(t, txs, resTxs)
+}
+
+func Test_parseDelimiter(t *testing.T) {
+	for i := uint64(0); i < 100; i++ {
+		tx := generateRandomContiguousShares(1, int(i))[0]
+		input, err := tx.MarshalDelimited()
+		if err != nil {
+			panic(err)
+		}
+		res, txLen, err := ParseDelimiter(input)
+		if err != nil {
+			panic(err)
+		}
+		assert.Equal(t, i, txLen)
+		assert.Equal(t, []byte(tx), res)
+	}
+}
+
+func TestFuzz_processContiguousShares(t *testing.T) {
+	t.Skip()
+	// run random shares through processContiguousShares for a minute
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			Test_processContiguousShares(t)
+		}
+	}
+}
+
+func Test_processContiguousShares(t *testing.T) {
+	// exactTxShareSize is the length of tx that will fit exactly into a single
+	// share, accounting for namespace id and the length delimiter prepended to
+	// each tx
+	const exactTxShareSize = consts.TxShareSize - 1
+
+	type test struct {
+		name    string
+		txSize  int
+		txCount int
+	}
+
+	// each test is ran twice, once using txSize as an exact size, and again
+	// using it as a cap for randomly sized txs
+	tests := []test{
+		{"single small tx", 10, 1},
+		{"many small txs", 10, 10},
+		{"single big tx", 1000, 1},
+		{"many big txs", 1000, 10},
+		{"single exact size tx", exactTxShareSize, 1},
+		{"many exact size txs", exactTxShareSize, 10},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+
+		// run the tests with identically sized txs
+		t.Run(fmt.Sprintf("%s idendically sized ", tc.name), func(t *testing.T) {
+			txs := generateRandomContiguousShares(tc.txCount, tc.txSize)
+
+			shares := SplitTxs(txs)
+
+			parsedTxs, err := processContiguousShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the data parsed is identical
+			for i := 0; i < len(txs); i++ {
+				assert.Equal(t, []byte(txs[i]), parsedTxs[i])
+			}
+		})
+
+		// run the same tests using randomly sized txs with caps of tc.txSize
+		t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
+			txs := generateRandomlySizedContiguousShares(tc.txCount, tc.txSize)
+
+			shares := SplitTxs(txs)
+
+			parsedTxs, err := processContiguousShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the data parsed is identical to the original
+			for i := 0; i < len(txs); i++ {
+				assert.Equal(t, []byte(txs[i]), parsedTxs[i])
+			}
+		})
+	}
+}

pkg/shares/message_shares_test.go

@@ -0,0 +1,99 @@
+package shares
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/pkg/consts"
+	coretypes "github.com/tendermint/tendermint/types"
+)
+
+func Test_parseMsgShares(t *testing.T) {
+	// exactMsgShareSize is the length of message that will fit exactly into a single
+	// share, accounting for namespace id and the length delimiter prepended to
+	// each message
+	const exactMsgShareSize = consts.MsgShareSize - 2
+
+	type test struct {
+		name     string
+		msgSize  int
+		msgCount int
+	}
+
+	// each test is ran twice, once using msgSize as an exact size, and again
+	// using it as a cap for randomly sized leaves
+	tests := []test{
+		{"single small msg", 100, 1},
+		{"many small msgs", 100, 10},
+		{"single big msg", 1000, 1},
+		{"many big msgs", 1000, 10},
+		{"single exact size msg", exactMsgShareSize, 1},
+		{"many exact size msgs", consts.MsgShareSize, 10},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		// run the tests with identically sized messages
+		t.Run(fmt.Sprintf("%s identically sized ", tc.name), func(t *testing.T) {
+			rawmsgs := make([]coretypes.Message, tc.msgCount)
+			for i := 0; i < tc.msgCount; i++ {
+				rawmsgs[i] = generateRandomMessage(tc.msgSize)
+			}
+
+			msgs := coretypes.Messages{MessagesList: rawmsgs}
+			msgs.SortMessages()
+
+			shares, _ := SplitMessages(nil, msgs.MessagesList)
+
+			parsedMsgs, err := parseMsgShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the namespaces and data are the same
+			for i := 0; i < len(msgs.MessagesList); i++ {
+				assert.Equal(t, msgs.MessagesList[i].NamespaceID, parsedMsgs[i].NamespaceID)
+				assert.Equal(t, msgs.MessagesList[i].Data, parsedMsgs[i].Data)
+			}
+		})
+
+		// run the same tests using randomly sized messages with caps of tc.msgSize
+		t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
+			msgs := generateRandomlySizedMessages(tc.msgCount, tc.msgSize)
+			shares, _ := SplitMessages(nil, msgs.MessagesList)
+
+			parsedMsgs, err := parseMsgShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the namesapces and data are the same
+			for i := 0; i < len(msgs.MessagesList); i++ {
+				assert.Equal(t, msgs.MessagesList[i].NamespaceID, parsedMsgs[i].NamespaceID)
+				assert.Equal(t, msgs.MessagesList[i].Data, parsedMsgs[i].Data)
+			}
+		})
+	}
+}
+
+func TestParsePaddedMsg(t *testing.T) {
+	msgWr := NewMessageShareSplitter()
+	randomSmallMsg := generateRandomMessage(100)
+	randomLargeMsg := generateRandomMessage(10000)
+	msgs := coretypes.Messages{
+		MessagesList: []coretypes.Message{
+			randomSmallMsg,
+			randomLargeMsg,
+		},
+	}
+	msgs.SortMessages()
+	msgWr.Write(msgs.MessagesList[0])
+	msgWr.WriteNamespacedPaddedShares(4)
+	msgWr.Write(msgs.MessagesList[1])
+	msgWr.WriteNamespacedPaddedShares(10)
+	pmsgs, err := parseMsgShares(msgWr.Export().RawShares())
+	require.NoError(t, err)
+	require.Equal(t, msgs.MessagesList, pmsgs)
+}

pkg/shares/merge_contiguous_shares.go → pkg/shares/parse_contiguous_shares.go

@@ -8,7 +8,7 @@ import (
 )
 
 // processContiguousShares takes raw shares and extracts out transactions,
-// intermediate state roots, or evidence. The returned [][]byte do have
+// intermediate state roots, or evidence. The returned [][]byte do not have
 // namespaces or length delimiters and are ready to be unmarshalled
 func processContiguousShares(shares [][]byte) (txs [][]byte, err error) {
 	if len(shares) == 0 {
@@ -19,11 +19,13 @@ func processContiguousShares(shares [][]byte) (txs [][]byte, err error) {
 	return ss.resolve()
 }
 
-// shareStack hold variables for peel
+// shareStack holds variables for peel
 type shareStack struct {
-	shares [][]byte
-	txLen  uint64
-	txs    [][]byte
+	shares  [][]byte
+	dataLen uint64
+	// data may be transactions, intermediate state roots, or evidence depending
+	// on the namespace ID for this share
+	data   [][]byte
 	cursor int
 }
 
@@ -36,7 +38,7 @@ func (ss *shareStack) resolve() ([][]byte, error) {
 		return nil, nil
 	}
 	err := ss.peel(ss.shares[0][consts.NamespaceSize+consts.ShareReservedBytes:], true)
-	return ss.txs, err
+	return ss.data, err
 }
 
 // peel recursively parses each chunk of data (either a transaction,
@@ -51,7 +53,7 @@ func (ss *shareStack) peel(share []byte, delimited bool) (err error) {
 		if txLen == 0 {
 			return nil
 		}
-		ss.txLen = txLen
+		ss.dataLen = txLen
 	}
 	// safeLen describes the point in the share where it can be safely split. If
 	// split beyond this point, it is possible to break apart a length
@@ -60,9 +62,9 @@ func (ss *shareStack) peel(share []byte, delimited bool) (err error) {
 	if safeLen < 0 {
 		safeLen = 0
 	}
-	if ss.txLen <= uint64(safeLen) {
-		ss.txs = append(ss.txs, share[:ss.txLen])
-		share = share[ss.txLen:]
+	if ss.dataLen <= uint64(safeLen) {
+		ss.data = append(ss.data, share[:ss.dataLen])
+		share = share[ss.dataLen:]
 		return ss.peel(share, true)
 	}
 	// add the next share to the current share to continue merging if possible
@@ -72,9 +74,9 @@ func (ss *shareStack) peel(share []byte, delimited bool) (err error) {
 		return ss.peel(share, false)
 	}
 	// collect any remaining data
-	if ss.txLen <= uint64(len(share)) {
-		ss.txs = append(ss.txs, share[:ss.txLen])
-		share = share[ss.txLen:]
+	if ss.dataLen <= uint64(len(share)) {
+		ss.data = append(ss.data, share[:ss.dataLen])
+		share = share[ss.dataLen:]
 		return ss.peel(share, true)
 	}
 	return errors.New("failure to parse block data: transaction length exceeded data length")

pkg/shares/merge_message_shares.go → pkg/shares/parse_message_shares.go

@@ -102,12 +102,13 @@ func ParseDelimiter(input []byte) ([]byte, uint64, error) {
 
 func zeroPadIfNecessary(share []byte, width int) []byte {
 	oldLen := len(share)
-	if oldLen < width {
-		missingBytes := width - oldLen
-		padByte := []byte{0}
-		padding := bytes.Repeat(padByte, missingBytes)
-		share = append(share, padding...)
+	if oldLen >= width {
 		return share
 	}
+
+	missingBytes := width - oldLen
+	padByte := []byte{0}
+	padding := bytes.Repeat(padByte, missingBytes)
+	share = append(share, padding...)
 	return share
 }

pkg/shares/share_splitting.go

@@ -18,7 +18,7 @@ var (
 )
 
 func Split(data coretypes.Data) ([][]byte, error) {
-	if data.OriginalSquareSize == 0 || !powerOf2(data.OriginalSquareSize) {
+	if data.OriginalSquareSize == 0 || !isPowerOf2(data.OriginalSquareSize) {
 		return nil, fmt.Errorf("square size is not a power of two: %d", data.OriginalSquareSize)
 	}
 	wantShareCount := int(data.OriginalSquareSize * data.OriginalSquareSize)