Skip to content

Commit

Permalink
feat!: adapt tx inclusion proofs to use share inclusion proofs (#1276)
Browse files Browse the repository at this point in the history 
Closes celestiaorg/celestia-core#920
Closes celestiaorg/celestia-app#1226 b/c
deletes that code
Closes celestiaorg/celestia-app#1049 b/c
deletes that code

## Description

Previously the logic to generate tx proofs was flawed because it did not
accurately identify which shares a transaction belongs to. Additionally
the struct `TxProof` (originally defined by Tendermint but later
modified in celestia-core) was incomplete because it did not contain
proofs for the inclusion of a set of rows to the data square root. This
PR modifies the TxInclusion logic to return a share inclusion proof
instead of a transaction inclusion proof. The share inclusion proof does
contain proofs for the rows to the data square root. In order for txs to
use share inclusion proofs, `SplitTxs` was updated to keep track of the
range of shares that a tx occupies.

---------

Co-authored-by: CHAMI Rachid <[email protected]>
Co-authored-by: Sanaz Taheri <[email protected]>
  • Loading branch information
@rootulp @rach-id @staheri14
3 people authored Jan 30, 2023
1 parent 2afaa65 commit 2e13319
Show file tree
Hide file tree
Showing 6 changed files with 406 additions and 41 deletions.
23 changes: 11 additions & 12 deletions pkg/shares/compact_shares_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,15 @@ import (
coretypes "github.com/tendermint/tendermint/types"
)

func TestCompactShareWriter(t *testing.T) {
func TestCompactShareSplitter(t *testing.T) {
// note that this test is mainly for debugging purposes, the main round trip
// tests occur in TestMerge and Test_processCompactShares
w := NewCompactShareSplitter(appconsts.TxNamespaceID, appconsts.ShareVersionZero)
css := NewCompactShareSplitter(appconsts.TxNamespaceID, appconsts.ShareVersionZero)
txs := testfactory.GenerateRandomTxs(33, 200)
for _, tx := range txs {
rawTx, _ := MarshalDelimitedTx(tx)
w.WriteBytes(rawTx)
css.WriteTx(tx)
}
shares := w.Export()
shares, _ := css.Export(0)
rawShares := ToBytes(shares)
rawResTxs, err := parseCompactShares(rawShares, appconsts.SupportedShareVersions)
resTxs := coretypes.ToTxs(rawResTxs)
Expand Down Expand Up @@ -77,7 +76,7 @@ func Test_processCompactShares(t *testing.T) {
t.Run(fmt.Sprintf("%s idendically sized", tc.name), func(t *testing.T) {
txs := testfactory.GenerateRandomTxs(tc.txCount, tc.txSize)

shares, _ := SplitTxs(txs)
shares, _, _ := SplitTxs(txs)
rawShares := ToBytes(shares)

parsedTxs, err := parseCompactShares(rawShares, appconsts.SupportedShareVersions)
Expand All @@ -95,8 +94,8 @@ func Test_processCompactShares(t *testing.T) {
t.Run(fmt.Sprintf("%s randomly sized", tc.name), func(t *testing.T) {
txs := testfactory.GenerateRandomlySizedTxs(tc.txCount, tc.txSize)

shares, _ := SplitTxs(txs)
rawShares := ToBytes(shares)
txShares, _, _ := SplitTxs(txs)
rawShares := ToBytes(txShares)

parsedTxs, err := parseCompactShares(rawShares, appconsts.SupportedShareVersions)
if err != nil {
Expand All @@ -119,7 +118,7 @@ func TestCompactShareContainsInfoByte(t *testing.T) {
css.WriteTx(tx)
}

shares := css.Export()
shares, _ := css.Export(0)
assert.Condition(t, func() bool { return len(shares) == 1 })

infoByte := shares[0][appconsts.NamespaceSize : appconsts.NamespaceSize+appconsts.ShareInfoBytes][0]
Expand All @@ -139,7 +138,7 @@ func TestContiguousCompactShareContainsInfoByte(t *testing.T) {
css.WriteTx(tx)
}

shares := css.Export()
shares, _ := css.Export(0)
assert.Condition(t, func() bool { return len(shares) > 1 })

infoByte := shares[1][appconsts.NamespaceSize : appconsts.NamespaceSize+appconsts.ShareInfoBytes][0]
Expand All @@ -158,8 +157,8 @@ func Test_parseCompactSharesErrors(t *testing.T) {
}

txs := testfactory.GenerateRandomTxs(2, appconsts.ContinuationCompactShareContentSize*4)
shares, _ := SplitTxs(txs)
rawShares := ToBytes(shares)
txShares, _, _ := SplitTxs(txs)
rawShares := ToBytes(txShares)

unsupportedShareVersion := 5
infoByte, _ := NewInfoByte(uint8(unsupportedShareVersion), true)
Expand Down
22 changes: 11 additions & 11 deletions pkg/shares/share_merging_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ func TestParseShares(t *testing.T) {
blobOneNamespace := namespace.ID{1, 1, 1, 1, 1, 1, 1, 1}
blobTwoNamespace := namespace.ID{2, 2, 2, 2, 2, 2, 2, 2}

transactionShares, _ := SplitTxs(generateRandomTxs(2, 1000))
transactionShareStart := transactionShares[0]
transactionShareContinuation := transactionShares[1]
txShares, _, _ := SplitTxs(generateRandomTxs(2, 1000))
txShareStart := txShares[0]
txShareContinuation := txShares[1]

blobOneShares, err := SplitBlobs(0, []uint32{}, []types.Blob{generateRandomBlobWithNamespace(blobOneNamespace, 1000)}, false)
if err != nil {
Expand Down Expand Up @@ -63,14 +63,14 @@ func TestParseShares(t *testing.T) {
},
{
"one transaction share",
[][]byte{transactionShareStart},
[]ShareSequence{{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{transactionShareStart}}},
[][]byte{txShareStart},
[]ShareSequence{{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{txShareStart}}},
false,
},
{
"two transaction shares",
[][]byte{transactionShareStart, transactionShareContinuation},
[]ShareSequence{{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{transactionShareStart, transactionShareContinuation}}},
[][]byte{txShareStart, txShareContinuation},
[]ShareSequence{{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{txShareStart, txShareContinuation}}},
false,
},
{
Expand All @@ -96,18 +96,18 @@ func TestParseShares(t *testing.T) {
},
{
"one transaction, one blob",
[][]byte{transactionShareStart, blobOneStart},
[][]byte{txShareStart, blobOneStart},
[]ShareSequence{
{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{transactionShareStart}},
{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{txShareStart}},
{NamespaceID: blobOneNamespace, Shares: []Share{blobOneStart}},
},
false,
},
{
"one transaction, two blobs",
[][]byte{transactionShareStart, blobOneStart, blobTwoStart},
[][]byte{txShareStart, blobOneStart, blobTwoStart},
[]ShareSequence{
{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{transactionShareStart}},
{NamespaceID: appconsts.TxNamespaceID, Shares: []Share{txShareStart}},
{NamespaceID: blobOneNamespace, Shares: []Share{blobOneStart}},
{NamespaceID: blobTwoNamespace, Shares: []Share{blobTwoStart}},
},
Expand Down
21 changes: 18 additions & 3 deletions pkg/shares/share_splitting.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/celestiaorg/celestia-app/pkg/appconsts"
coretypes "github.com/tendermint/tendermint/types"
"golang.org/x/exp/maps"
)

var (
Expand All @@ -28,7 +29,7 @@ func Split(data coretypes.Data, useShareIndexes bool) ([]Share, error) {
wantShareCount := int(data.SquareSize * data.SquareSize)
currentShareCount := 0

txShares, pfbTxShares := SplitTxs(data.Txs)
txShares, pfbTxShares, _ := SplitTxs(data.Txs)
currentShareCount += len(txShares) + len(pfbTxShares)
// blobIndexes will be nil if we are working with a list of txs that do not
// have a blob index. This preserves backwards compatibility with old blocks
Expand Down Expand Up @@ -97,17 +98,22 @@ func ExtractShareIndexes(txs coretypes.Txs) []uint32 {
return shareIndexes
}

func SplitTxs(txs coretypes.Txs) ([]Share, []Share) {
func SplitTxs(txs coretypes.Txs) (txShares []Share, pfbShares []Share, shareRanges map[coretypes.TxKey]ShareRange) {
txWriter := NewCompactShareSplitter(appconsts.TxNamespaceID, appconsts.ShareVersionZero)
pfbTxWriter := NewCompactShareSplitter(appconsts.PayForBlobNamespaceID, appconsts.ShareVersionZero)

for _, tx := range txs {
if _, isIndexWrapper := coretypes.UnmarshalIndexWrapper(tx); isIndexWrapper {
pfbTxWriter.WriteTx(tx)
} else {
txWriter.WriteTx(tx)
}
}
return txWriter.Export(), pfbTxWriter.Export()

txShares, txMap := txWriter.Export(0)
pfbShares, pfbMap := pfbTxWriter.Export(len(txShares))

return txShares, pfbShares, mergeMaps(txMap, pfbMap)
}

func SplitBlobs(cursor int, indexes []uint32, blobs []coretypes.Blob, useShareIndexes bool) ([]Share, error) {
Expand All @@ -126,3 +132,12 @@ func SplitBlobs(cursor int, indexes []uint32, blobs []coretypes.Blob, useShareIn
}
return writer.Export(), nil
}

// mergeMaps merges two maps into a new map. If there are any duplicate keys,
// the value in the second map takes precedence.
func mergeMaps(mapOne, mapTwo map[coretypes.TxKey]ShareRange) map[coretypes.TxKey]ShareRange {
merged := make(map[coretypes.TxKey]ShareRange, len(mapOne)+len(mapTwo))
maps.Copy(merged, mapOne)
maps.Copy(merged, mapTwo)
return merged
}
179 changes: 177 additions & 2 deletions pkg/shares/share_splitting_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@ import (
"testing"

"github.com/celestiaorg/celestia-app/pkg/appconsts"
"github.com/stretchr/testify/assert"
coretypes "github.com/tendermint/tendermint/types"
)

func TestSplitTxs(t *testing.T) {
func TestSplitTxs_forTxShares(t *testing.T) {
smallTransactionA := coretypes.Tx{0xa}
smallTransactionB := coretypes.Tx{0xb}
largeTransaction := bytes.Repeat([]byte{0xc}, 512)
Expand Down Expand Up @@ -120,14 +121,122 @@ func TestSplitTxs(t *testing.T) {
}
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
got, _ := SplitTxs(tt.txs)
got, _, _ := SplitTxs(tt.txs)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("SplitTxs()\n got %#v\n want %#v", got, tt.want)
}
})
}
}

func TestSplitTxs(t *testing.T) {
type testCase struct {
name string
txs coretypes.Txs
wantTxShares []Share
wantPfbShares []Share
wantMap map[coretypes.TxKey]ShareRange
}

smallTx := coretypes.Tx{0xa} // spans one share
smallTxShares := []Share{
padShare([]uint8{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id
0x1, // info byte
0x0, 0x0, 0x0, 0x2, // 1 byte (unit) + 1 byte (unit length) = 2 bytes sequence length
0x0, 0x0, 0x0, 17, // reserved bytes
0x1, // unit length of first transaction
0xa, // data of first transaction
}),
}

pfbTx, err := coretypes.MarshalIndexWrapper(coretypes.Tx{0xb}, 10) // spans one share
assert.NoError(t, err)
pfbTxShares := []Share{
padShare([]uint8{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, // namespace id
0x1, // info byte
0x0, 0x0, 0x0, 13, // 1 byte (unit) + 1 byte (unit length) = 2 bytes sequence length
0x0, 0x0, 0x0, 17, // reserved bytes
12, // unit length of first transaction
0xa, 0x1, 0xb, 0x12, 0x1, 0xa, 0x1a, 0x4, 0x49, 0x4e, 0x44, 0x58, // data of first transaction
}),
}

largeTx := coretypes.Tx(bytes.Repeat([]byte{0xc}, 512)) // spans two shares
largeTxShares := []Share{
fillShare([]uint8{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id
0x1, // info byte
0x0, 0x0, 0x2, 0x2, // 512 (unit) + 2 (unit length) = 514 sequence length
0x0, 0x0, 0x0, 17, // reserved bytes
128, 4, // unit length of transaction is 512
}, 0xc), // data of transaction
padShare(append([]uint8{
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id
0x0, // info byte
0x0, 0x0, 0x0, 0x0, // reserved bytes
}, bytes.Repeat([]byte{0xc}, 19)..., // continuation data of transaction
)),
}

testCases := []testCase{
{
name: "empty",
txs: coretypes.Txs{},
wantTxShares: []Share{},
wantPfbShares: []Share{},
wantMap: map[coretypes.TxKey]ShareRange{},
},
{
name: "smallTx",
txs: coretypes.Txs{smallTx},
wantTxShares: smallTxShares,
wantPfbShares: []Share{},
wantMap: map[coretypes.TxKey]ShareRange{
smallTx.Key(): {0, 0},
},
},
{
name: "largeTx",
txs: coretypes.Txs{largeTx},
wantTxShares: largeTxShares,
wantPfbShares: []Share{},
wantMap: map[coretypes.TxKey]ShareRange{
largeTx.Key(): {0, 1},
},
},
{
name: "pfbTx",
txs: coretypes.Txs{pfbTx},
wantTxShares: []Share{},
wantPfbShares: pfbTxShares,
wantMap: map[coretypes.TxKey]ShareRange{
pfbTx.Key(): {0, 0},
},
},
{
name: "largeTx then pfbTx",
txs: coretypes.Txs{largeTx, pfbTx},
wantTxShares: largeTxShares,
wantPfbShares: pfbTxShares,
wantMap: map[coretypes.TxKey]ShareRange{
largeTx.Key(): {0, 1},
pfbTx.Key(): {2, 2},
},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
txShares, pfbTxShares, gotMap := SplitTxs(tc.txs)
assert.Equal(t, tc.wantTxShares, txShares)
assert.Equal(t, tc.wantPfbShares, pfbTxShares)
assert.Equal(t, tc.wantMap, gotMap)
})
}
}

// padShare returns a share padded with trailing zeros.
func padShare(share []byte) (paddedShare []byte) {
return fillShare(share, 0)
Expand All @@ -138,3 +247,69 @@ func padShare(share []byte) (paddedShare []byte) {
func fillShare(share []byte, filler byte) (paddedShare []byte) {
return append(share, bytes.Repeat([]byte{filler}, appconsts.ShareSize-len(share))...)
}

func Test_mergeMaps(t *testing.T) {
type testCase struct {
name string
mapOne map[coretypes.TxKey]ShareRange
mapTwo map[coretypes.TxKey]ShareRange
want map[coretypes.TxKey]ShareRange
}
testCases := []testCase{
{
name: "empty maps",
mapOne: map[coretypes.TxKey]ShareRange{},
mapTwo: map[coretypes.TxKey]ShareRange{},
want: map[coretypes.TxKey]ShareRange{},
},
{
name: "merges maps with one key each",
mapOne: map[coretypes.TxKey]ShareRange{
{0x1}: {0, 1},
},
mapTwo: map[coretypes.TxKey]ShareRange{
{0x2}: {2, 3},
},
want: map[coretypes.TxKey]ShareRange{
{0x1}: {0, 1},
{0x2}: {2, 3},
},
},
{
name: "merges maps with multiple keys each",
mapOne: map[coretypes.TxKey]ShareRange{
{0x1}: {0, 1},
{0x2}: {2, 3},
},
mapTwo: map[coretypes.TxKey]ShareRange{
{0x3}: {3, 3},
{0x4}: {4, 4},
},
want: map[coretypes.TxKey]ShareRange{
{0x1}: {0, 1},
{0x2}: {2, 3},
{0x3}: {3, 3},
{0x4}: {4, 4},
},
},
{
name: "merges maps with a duplicate key and the second map's value takes precedence",
mapOne: map[coretypes.TxKey]ShareRange{
{0x1}: {0, 0},
},
mapTwo: map[coretypes.TxKey]ShareRange{
{0x1}: {1, 1},
},
want: map[coretypes.TxKey]ShareRange{
{0x1}: {1, 1},
},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
got := mergeMaps(tc.mapOne, tc.mapTwo)
assert.Equal(t, tc.want, got)
})
}
}
Loading

0 comments on commit 2e13319

Please sign in to comment.