feat!: adapt tx inclusion proofs to use share inclusion proofs (#1276)

Closes celestiaorg/celestia-core#920
Closes #1226 b/c
deletes that code
Closes #1049 b/c
deletes that code

## Description

Previously the logic to generate tx proofs was flawed because it did not
accurately identify which shares a transaction belongs to. Additionally
the struct `TxProof` (originally defined by Tendermint but later
modified in celestia-core) was incomplete because it did not contain
proofs for the inclusion of a set of rows to the data square root. This
PR modifies the TxInclusion logic to return a share inclusion proof
instead of a transaction inclusion proof. The share inclusion proof does
contain proofs for the rows to the data square root. In order for txs to
use share inclusion proofs, `SplitTxs` was updated to keep track of the
range of shares that a tx occupies.

---------

Co-authored-by: CHAMI Rachid <[email protected]>
Co-authored-by: Sanaz Taheri <[email protected]>

app/estimate_square_size_test.go

@@ -138,7 +138,7 @@ func Test_estimatePFBTxSharesUsed(t *testing.T) {
 				require.NoError(t, err)
 				txs[i] = wPFBTx
 			}
-			_, pfbTxShares := shares.SplitTxs(txs)
+			_, pfbTxShares, _ := shares.SplitTxs(txs)
 			assert.LessOrEqual(t, len(pfbTxShares), got)
 		})
 	}

app/test/integration_test.go

@@ -177,8 +177,7 @@ func (s *IntegrationTestSuite) TestMaxBlockSize() {
 
 			heights := make(map[int64]int)
 			for _, hash := range hashes {
-				// TODO: reenable fetching and verifying proofs
-				resp, err := queryTx(val.ClientCtx, hash, false)
+				resp, err := queryTx(val.ClientCtx, hash, true)
 				assert.NoError(err)
 				assert.NotNil(resp)
 				if resp == nil {
@@ -349,7 +348,7 @@ func (s *IntegrationTestSuite) TestShareInclusionProof() {
 	}
 
 	for _, hash := range hashes {
-		txResp, err := queryTx(val.ClientCtx, hash, false)
+		txResp, err := queryTx(val.ClientCtx, hash, true)
 		require.NoError(err)
 		require.Equal(abci.CodeTypeOK, txResp.TxResult.Code)
 

app/test/std_sdk_test.go

@@ -208,7 +208,7 @@ func (s *StandardSDKIntegrationTestSuite) TestStandardSDK() {
 	require.NoError(s.T(), s.cctx.WaitForNextBlock())
 
 	for _, tt := range tests {
-		res, err := queryTx(s.cctx.Context, tt.hash, false)
+		res, err := queryTx(s.cctx.Context, tt.hash, true)
 		require.NoError(t, err)
 		assert.Equal(t, abci.CodeTypeOK, res.TxResult.Code, tt.name)
 	}

go.mod

@@ -186,5 +186,5 @@ require (
 replace (
 	github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.6.0-sdk-v0.46.7
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
-	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.13.0-tm-v0.34.23
+	github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.14.0-tm-v0.34.23
 )

go.sum

@@ -186,8 +186,8 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f
 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
-github.com/celestiaorg/celestia-core v1.13.0-tm-v0.34.23 h1:nMg928RIanyA2ICN3PHpIE6jgg+008BtD5XmxeObEhM=
-github.com/celestiaorg/celestia-core v1.13.0-tm-v0.34.23/go.mod h1:Iu2XuSzF+QG3rzd60/sGhhk3n4wOAm8YyadU80KlcMs=
+github.com/celestiaorg/celestia-core v1.14.0-tm-v0.34.23 h1:8zE523TUe5W33/nheJ9umHF2d1q6iHQlqJfMXMTPe3k=
+github.com/celestiaorg/celestia-core v1.14.0-tm-v0.34.23/go.mod h1:fGDSg7aw2OH/Uze1zymop0x0y1kAPEO9OII2A2cb99Q=
 github.com/celestiaorg/cosmos-sdk v1.6.0-sdk-v0.46.7 h1:wP4RRK9zbVBlQXqRqJXji09e9UUFpNpwL2zorHwbJhQ=
 github.com/celestiaorg/cosmos-sdk v1.6.0-sdk-v0.46.7/go.mod h1:MkOIxmKiICA4d8yPWo/590S4fqpaLXfocz0xn1fxBp4=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc=

pkg/proof/proof.go

@@ -22,99 +22,43 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-// TxInclusion uses the provided block data to progressively generate rows
-// of a data square, and then using those shares to creates nmt inclusion proofs.
-// It is possible that a transaction spans more than one row. In that case, we
-// have to return more than one proof.
-func TxInclusion(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.TxProof, error) {
-	// calculate the index of the shares that contain the tx
-	startPos, endPos, err := TxSharePosition(data.Txs, txIndex)
+// NewTxInclusionProof returns a new share inclusion proof for the given
+// transaction index.
+func NewTxInclusionProof(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.ShareProof, error) {
+	rawShares, err := shares.Split(data, true)
 	if err != nil {
-		return types.TxProof{}, err
+		return types.ShareProof{}, err
 	}
 
-	// use the index of the shares and the square size to determine the row that
-	// contains the tx we need to prove
-	startRow := startPos / data.SquareSize
-	endRow := endPos / data.SquareSize
-	startLeaf := startPos % data.SquareSize
-	endLeaf := endPos % data.SquareSize
-
-	rowShares, err := genRowShares(codec, data, startRow, endRow)
+	startShare, endShare, err := TxSharePosition(data, txIndex)
 	if err != nil {
-		return types.TxProof{}, err
+		return types.ShareProof{}, err
 	}
 
-	var proofs []*tmproto.NMTProof  //nolint:prealloc // rarely will this contain more than a single proof
-	var rawShares [][]byte          //nolint:prealloc // rarely will this contain more than a single share
-	var rowRoots []tmbytes.HexBytes //nolint:prealloc // rarely will this contain more than a single root
-	for i, row := range rowShares {
-		// create an nmt to use to generate a proof
-		tree := wrapper.NewErasuredNamespacedMerkleTree(data.SquareSize, uint(i))
-		for _, share := range row {
-			tree.Push(
-				share,
-			)
-		}
-
-		startLeafPos := startLeaf
-		endLeafPos := endLeaf
-
-		// if this is not the first row, then start with the first leaf
-		if i > 0 {
-			startLeafPos = 0
-		}
-		// if this is not the last row, then select for the rest of the row
-		if i != (len(rowShares) - 1) {
-			endLeafPos = data.SquareSize - 1
-		}
-
-		rawShares = append(rawShares, shares.ToBytes(row[startLeafPos:endLeafPos+1])...)
-		proof, err := tree.Tree().ProveRange(int(startLeafPos), int(endLeafPos+1))
-		if err != nil {
-			return types.TxProof{}, err
-		}
-
-		proofs = append(proofs, &tmproto.NMTProof{
-			Start:    int32(proof.Start()),
-			End:      int32(proof.End()),
-			Nodes:    proof.Nodes(),
-			LeafHash: proof.LeafHash(),
-		})
+	namespace := getTxNamespace(data.Txs[txIndex])
+	return NewShareInclusionProof(rawShares, data.SquareSize, namespace, startShare, endShare)
+}
 
-		// we don't store the data availability header anywhere, so we
-		// regenerate the roots to each row
-		rowRoots = append(rowRoots, tree.Root())
+func getTxNamespace(tx types.Tx) (ns namespace.ID) {
+	_, isIndexWrapper := types.UnmarshalIndexWrapper(tx)
+	if isIndexWrapper {
+		return appconsts.PayForBlobNamespaceID
 	}
-
-	return types.TxProof{
-		RowRoots: rowRoots,
-		Data:     rawShares,
-		Proofs:   proofs,
-	}, nil
+	return appconsts.TxNamespaceID
 }
 
 // TxSharePosition returns the start and end positions for the shares that
 // include a given txIndex. Returns an error if index is greater than the length
 // of txs.
-func TxSharePosition(txs types.Txs, txIndex uint64) (startSharePos, endSharePos uint64, err error) {
-	if txIndex >= uint64(len(txs)) {
-		return startSharePos, endSharePos, errors.New("transaction index is greater than the number of txs")
-	}
-
-	prevTxTotalLen := 0
-	for i := uint64(0); i < txIndex; i++ {
-		txLen := len(txs[i])
-		prevTxTotalLen += (shares.DelimLen(uint64(txLen)) + txLen)
+func TxSharePosition(data types.Data, txIndex uint64) (startShare uint64, endShare uint64, err error) {
+	if int(txIndex) >= len(data.Txs) {
+		return 0, 0, errors.New("transaction index is greater than the number of txs")
 	}
 
-	currentTxLen := len(txs[txIndex])
-	currentTxTotalLen := shares.DelimLen(uint64(currentTxLen)) + currentTxLen
-	endOfCurrentTxLen := prevTxTotalLen + currentTxTotalLen
+	_, _, shareRanges := shares.SplitTxs(data.Txs)
+	shareRange := shareRanges[data.Txs[txIndex].Key()]
 
-	startSharePos = txShareIndex(prevTxTotalLen)
-	endSharePos = txShareIndex(endOfCurrentTxLen)
-	return startSharePos, endSharePos, nil
+	return uint64(shareRange.Start), uint64(shareRange.End), nil
 }
 
 // BlobShareRange returns the start and end positions for the shares
@@ -158,126 +102,35 @@ func BlobShareRange(tx types.Tx) (beginShare uint64, endShare uint64, err error)
 	return beginShare, beginShare + uint64(sharesUsed) - 1, nil
 }
 
-// txShareIndex returns the index of the compact share that would contain
-// transactions with totalTxLen
-func txShareIndex(totalTxLen int) (index uint64) {
-	if totalTxLen <= appconsts.FirstCompactShareContentSize {
-		return 0
-	}
-
-	index++
-	totalTxLen -= appconsts.FirstCompactShareContentSize
-
-	for totalTxLen > appconsts.ContinuationCompactShareContentSize {
-		index++
-		totalTxLen -= appconsts.ContinuationCompactShareContentSize
-	}
-	return index
-}
-
-// genRowShares progessively generates data square rows from block data
-func genRowShares(codec rsmt2d.Codec, data types.Data, startRow, endRow uint64) ([][]shares.Share, error) {
-	if endRow > data.SquareSize {
-		return nil, errors.New("cannot generate row shares past the original square size")
-	}
-	origRowShares := splitIntoRows(
-		data.SquareSize,
-		genOrigRowShares(data, startRow, endRow),
-	)
-
-	encodedRowShares := make([][]shares.Share, len(origRowShares))
-	for i, row := range origRowShares {
-		encRow, err := codec.Encode(shares.ToBytes(row))
-		if err != nil {
-			panic(err)
-		}
-		encodedRowShares[i] = append(
-			append(
-				make([]shares.Share, 0, len(row)+len(encRow)),
-				row...,
-			), shares.FromBytes(encRow)...,
-		)
-	}
-
-	return encodedRowShares, nil
-}
-
-// genOrigRowShares progressively generates data square rows for the original
-// data square, meaning the rows only half the full square length, as there is
-// not erasure data
-func genOrigRowShares(data types.Data, startRow, endRow uint64) []shares.Share {
-	wantLen := (endRow + 1) * data.SquareSize
-	startPos := startRow * data.SquareSize
-
-	rawTxShares, pfbTxShares := shares.SplitTxs(data.Txs)
-	rawShares := append(rawTxShares, pfbTxShares...)
-	// return if we have enough shares
-	if uint64(len(rawShares)) >= wantLen {
-		return rawShares[startPos:wantLen]
-	}
-
-	for _, blob := range data.Blobs {
-		blobShares, err := shares.SplitBlobs(0, nil, []types.Blob{blob}, false)
-		if err != nil {
-			panic(err)
-		}
-
-		// TODO: does this need to account for padding between compact shares
-		// and the first blob?
-		// https://github.com/celestiaorg/celestia-app/issues/1226
-		rawShares = append(rawShares, blobShares...)
-
-		// return if we have enough shares
-		if uint64(len(rawShares)) >= wantLen {
-			return rawShares[startPos:wantLen]
-		}
-	}
-
-	tailShares := shares.TailPaddingShares(int(wantLen) - len(rawShares))
-	rawShares = append(rawShares, tailShares...)
-
-	return rawShares[startPos:wantLen]
-}
-
-// splitIntoRows splits shares into rows of a particular square size
-func splitIntoRows(squareSize uint64, s []shares.Share) [][]shares.Share {
-	rowCount := uint64(len(s)) / squareSize
-	rows := make([][]shares.Share, rowCount)
-	for i := uint64(0); i < rowCount; i++ {
-		rows[i] = s[i*squareSize : (i+1)*squareSize]
-	}
-	return rows
-}
-
-// GenerateSharesInclusionProof generates an nmt inclusion proof for a set of shares to the data root.
+// NewShareInclusionProof returns an NMT inclusion proof for a set of shares to the data root.
 // Expects the share range to be pre-validated.
 // Note: only supports inclusion proofs for shares belonging to the same namespace.
-func GenerateSharesInclusionProof(
+func NewShareInclusionProof(
 	allRawShares []shares.Share,
 	squareSize uint64,
 	namespaceID namespace.ID,
 	startShare uint64,
 	endShare uint64,
-) (types.SharesProof, error) {
+) (types.ShareProof, error) {
 	startRow := startShare / squareSize
 	endRow := endShare / squareSize
 	startLeaf := startShare % squareSize
 	endLeaf := endShare % squareSize
 
 	eds, err := da.ExtendShares(squareSize, shares.ToBytes(allRawShares))
 	if err != nil {
-		return types.SharesProof{}, err
+		return types.ShareProof{}, err
 	}
 
 	edsRowRoots := eds.RowRoots()
 
 	// create the binary merkle inclusion proof for all the square rows to the data root
 	_, allProofs := merkle.ProofsFromByteSlices(append(edsRowRoots, eds.ColRoots()...))
-	rowsProofs := make([]*merkle.Proof, endRow-startRow+1)
-	rowsRoots := make([]tmbytes.HexBytes, endRow-startRow+1)
+	rowProofs := make([]*merkle.Proof, endRow-startRow+1)
+	rowRoots := make([]tmbytes.HexBytes, endRow-startRow+1)
 	for i := startRow; i <= endRow; i++ {
-		rowsProofs[i-startRow] = allProofs[i]
-		rowsRoots[i-startRow] = edsRowRoots[i]
+		rowProofs[i-startRow] = allProofs[i]
+		rowRoots[i-startRow] = edsRowRoots[i]
 	}
 
 	// get the extended rows containing the shares.
@@ -286,7 +139,7 @@ func GenerateSharesInclusionProof(
 		rows[i-startRow] = shares.FromBytes(eds.Row(uint(i)))
 	}
 
-	var sharesProofs []*tmproto.NMTProof //nolint:prealloc
+	var shareProofs []*tmproto.NMTProof //nolint:prealloc
 	var rawShares [][]byte
 	for i, row := range rows {
 		// create an nmt to generate a proof.
@@ -313,31 +166,31 @@ func GenerateSharesInclusionProof(
 		rawShares = append(rawShares, shares.ToBytes(row[startLeafPos:endLeafPos+1])...)
 		proof, err := tree.Tree().ProveRange(int(startLeafPos), int(endLeafPos+1))
 		if err != nil {
-			return types.SharesProof{}, err
+			return types.ShareProof{}, err
 		}
 
-		sharesProofs = append(sharesProofs, &tmproto.NMTProof{
+		shareProofs = append(shareProofs, &tmproto.NMTProof{
 			Start:    int32(proof.Start()),
 			End:      int32(proof.End()),
 			Nodes:    proof.Nodes(),
 			LeafHash: proof.LeafHash(),
 		})
 
 		// make sure that the generated root is the same as the eds row root.
-		if !bytes.Equal(rowsRoots[i].Bytes(), tree.Root()) {
-			return types.SharesProof{}, errors.New("eds row root is different than tree root")
+		if !bytes.Equal(rowRoots[i].Bytes(), tree.Root()) {
+			return types.ShareProof{}, errors.New("eds row root is different than tree root")
 		}
 	}
 
-	return types.SharesProof{
-		RowsProof: types.RowsProof{
-			RowsRoots: rowsRoots,
-			Proofs:    rowsProofs,
-			StartRow:  uint32(startRow),
-			EndRow:    uint32(endRow),
+	return types.ShareProof{
+		RowProof: types.RowProof{
+			RowRoots: rowRoots,
+			Proofs:   rowProofs,
+			StartRow: uint32(startRow),
+			EndRow:   uint32(endRow),
 		},
-		Data:         rawShares,
-		SharesProofs: sharesProofs,
-		NamespaceID:  namespaceID,
+		Data:        rawShares,
+		ShareProofs: shareProofs,
+		NamespaceID: namespaceID,
 	}, nil
 }