review feedback

pkg/shares/contiguous_shares_test.go

@@ -0,0 +1,123 @@
+package shares
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/pkg/consts"
+	coretypes "github.com/tendermint/tendermint/types"
+)
+
+func TestContigShareWriter(t *testing.T) {
+	// note that this test is mainly for debugging purposes, the main round trip
+	// tests occur in TestMerge and Test_processContiguousShares
+	w := NewContiguousShareSplitter(consts.TxNamespaceID)
+	txs := generateRandomContiguousShares(33, 200)
+	for _, tx := range txs {
+		rawTx, _ := tx.MarshalDelimited()
+		w.WriteBytes(rawTx)
+	}
+	resShares := w.Export()
+	rawResTxs, err := processContiguousShares(resShares.RawShares())
+	resTxs := coretypes.ToTxs(rawResTxs)
+	require.NoError(t, err)
+
+	assert.Equal(t, txs, resTxs)
+}
+
+func Test_parseDelimiter(t *testing.T) {
+	for i := uint64(0); i < 100; i++ {
+		tx := generateRandomContiguousShares(1, int(i))[0]
+		input, err := tx.MarshalDelimited()
+		if err != nil {
+			panic(err)
+		}
+		res, txLen, err := ParseDelimiter(input)
+		if err != nil {
+			panic(err)
+		}
+		assert.Equal(t, i, txLen)
+		assert.Equal(t, []byte(tx), res)
+	}
+}
+
+func TestFuzz_processContiguousShares(t *testing.T) {
+	t.Skip()
+	// run random shares through processContiguousShares for a minute
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			Test_processContiguousShares(t)
+		}
+	}
+}
+
+func Test_processContiguousShares(t *testing.T) {
+	// exactTxShareSize is the length of tx that will fit exactly into a single
+	// share, accounting for namespace id and the length delimiter prepended to
+	// each tx
+	const exactTxShareSize = consts.TxShareSize - 1
+
+	type test struct {
+		name    string
+		txSize  int
+		txCount int
+	}
+
+	// each test is ran twice, once using txSize as an exact size, and again
+	// using it as a cap for randomly sized txs
+	tests := []test{
+		{"single small tx", 10, 1},
+		{"many small txs", 10, 10},
+		{"single big tx", 1000, 1},
+		{"many big txs", 1000, 10},
+		{"single exact size tx", exactTxShareSize, 1},
+		{"many exact size txs", exactTxShareSize, 10},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+
+		// run the tests with identically sized txs
+		t.Run(fmt.Sprintf("%s idendically sized ", tc.name), func(t *testing.T) {
+			txs := generateRandomContiguousShares(tc.txCount, tc.txSize)
+
+			shares := SplitTxs(txs)
+
+			parsedTxs, err := processContiguousShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the data parsed is identical
+			for i := 0; i < len(txs); i++ {
+				assert.Equal(t, []byte(txs[i]), parsedTxs[i])
+			}
+		})
+
+		// run the same tests using randomly sized txs with caps of tc.txSize
+		t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
+			txs := generateRandomlySizedContiguousShares(tc.txCount, tc.txSize)
+
+			shares := SplitTxs(txs)
+
+			parsedTxs, err := processContiguousShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the data parsed is identical to the original
+			for i := 0; i < len(txs); i++ {
+				assert.Equal(t, []byte(txs[i]), parsedTxs[i])
+			}
+		})
+	}
+}

pkg/shares/message_shares_test.go

@@ -0,0 +1,99 @@
+package shares
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/pkg/consts"
+	coretypes "github.com/tendermint/tendermint/types"
+)
+
+func Test_parseMsgShares(t *testing.T) {
+	// exactMsgShareSize is the length of message that will fit exactly into a single
+	// share, accounting for namespace id and the length delimiter prepended to
+	// each message
+	const exactMsgShareSize = consts.MsgShareSize - 2
+
+	type test struct {
+		name     string
+		msgSize  int
+		msgCount int
+	}
+
+	// each test is ran twice, once using msgSize as an exact size, and again
+	// using it as a cap for randomly sized leaves
+	tests := []test{
+		{"single small msg", 100, 1},
+		{"many small msgs", 100, 10},
+		{"single big msg", 1000, 1},
+		{"many big msgs", 1000, 10},
+		{"single exact size msg", exactMsgShareSize, 1},
+		{"many exact size msgs", consts.MsgShareSize, 10},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		// run the tests with identically sized messagses
+		t.Run(fmt.Sprintf("%s idendically sized ", tc.name), func(t *testing.T) {
+			rawmsgs := make([]coretypes.Message, tc.msgCount)
+			for i := 0; i < tc.msgCount; i++ {
+				rawmsgs[i] = generateRandomMessage(tc.msgSize)
+			}
+
+			msgs := coretypes.Messages{MessagesList: rawmsgs}
+			msgs.SortMessages()
+
+			shares, _ := SplitMessages(nil, msgs.MessagesList)
+
+			parsedMsgs, err := parseMsgShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the namesapces and data are the same
+			for i := 0; i < len(msgs.MessagesList); i++ {
+				assert.Equal(t, msgs.MessagesList[i].NamespaceID, parsedMsgs[i].NamespaceID)
+				assert.Equal(t, msgs.MessagesList[i].Data, parsedMsgs[i].Data)
+			}
+		})
+
+		// run the same tests using randomly sized messages with caps of tc.msgSize
+		t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
+			msgs := generateRandomlySizedMessages(tc.msgCount, tc.msgSize)
+			shares, _ := SplitMessages(nil, msgs.MessagesList)
+
+			parsedMsgs, err := parseMsgShares(shares)
+			if err != nil {
+				t.Error(err)
+			}
+
+			// check that the namesapces and data are the same
+			for i := 0; i < len(msgs.MessagesList); i++ {
+				assert.Equal(t, msgs.MessagesList[i].NamespaceID, parsedMsgs[i].NamespaceID)
+				assert.Equal(t, msgs.MessagesList[i].Data, parsedMsgs[i].Data)
+			}
+		})
+	}
+}
+
+func TestParsePaddedMsg(t *testing.T) {
+	msgWr := NewMessageShareSplitter()
+	randomSmallMsg := generateRandomMessage(100)
+	randomLargeMsg := generateRandomMessage(10000)
+	msgs := coretypes.Messages{
+		MessagesList: []coretypes.Message{
+			randomSmallMsg,
+			randomLargeMsg,
+		},
+	}
+	msgs.SortMessages()
+	msgWr.Write(msgs.MessagesList[0])
+	msgWr.WriteNamespacedPaddedShares(4)
+	msgWr.Write(msgs.MessagesList[1])
+	msgWr.WriteNamespacedPaddedShares(10)
+	pmsgs, err := parseMsgShares(msgWr.Export().RawShares())
+	require.NoError(t, err)
+	require.Equal(t, msgs.MessagesList, pmsgs)
+}

pkg/shares/merge_message_shares.go → pkg/shares/parse_message_shares.go

@@ -102,12 +102,13 @@ func ParseDelimiter(input []byte) ([]byte, uint64, error) {
 
 func zeroPadIfNecessary(share []byte, width int) []byte {
 	oldLen := len(share)
-	if oldLen < width {
-		missingBytes := width - oldLen
-		padByte := []byte{0}
-		padding := bytes.Repeat(padByte, missingBytes)
-		share = append(share, padding...)
+	if oldLen >= width {
 		return share
 	}
+
+	missingBytes := width - oldLen
+	padByte := []byte{0}
+	padding := bytes.Repeat(padByte, missingBytes)
+	share = append(share, padding...)
 	return share
 }