Index malleated transactions properly (celestiaorg#612)

* index malleated transactions properly

* linter

* fix malleated transaction indexing and event tracking

* fix malleated event test

* fix test

* fix other test

* update proto building and formatting to use the latest tendermint specs style generation and formatting

* fix app tests

linter

fix max size tests

Makefile

@@ -4,6 +4,9 @@ BUILDDIR ?= $(CURDIR)/build
 
 BUILD_TAGS?=tendermint
 
+IMAGE := ghcr.io/tendermint/docker-build-proto:latest
+DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(IMAGE)
+
 # If building a release, please checkout the version tag to get the correct version setting
 ifneq ($(shell git symbolic-ref -q --short HEAD),)
 VERSION := unreleased-$(shell git symbolic-ref -q --short HEAD)-$(shell git rev-parse HEAD)
@@ -80,7 +83,7 @@ $(BUILDDIR)/:
 ###                                Protobuf                                 ###
 ###############################################################################
 
-proto-all: proto-gen proto-lint proto-check-breaking
+proto-all: proto-lint proto-check-breaking
 .PHONY: proto-all
 
 proto-gen:

internal/mempool/v0/clist_mempool.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"crypto/sha256"
+
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/libs/clist"
@@ -624,10 +626,13 @@ func (mem *CListMempool) Update(
 			mem.removeTx(tx, e.(*clist.CElement), false)
 			// see if the transaction is a child transaction of a some parent
 			// transaction that exists in the mempool
-		} else if parentHash, _, isChild := types.DecodeChildTx(tx); isChild {
-			var parentKey [TxKeySize]byte
-			copy(parentKey[:], parentHash)
-			mem.RemoveTxByKey(parentKey, false)
+		} else if originalHash, _, isMalleated := types.UnwrapMalleatedTx(tx); isMalleated {
+			var origianlKey [sha256.Size]byte
+			copy(origianlKey[:], originalHash)
+			err := mem.RemoveTxByKey(origianlKey)
+			if err != nil {
+				return err
+			}
 		}
 	}
 

internal/mempool/v0/clist_mempool_test.go

@@ -3,6 +3,7 @@ package v0
 import (
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/binary"
 	"fmt"
 	mrand "math/rand"
@@ -226,24 +227,24 @@ func TestMempoolUpdate(t *testing.T) {
 
 	// 4. Removes a parent transaction after receiving a child transaction in the update
 	{
-		mempool.Flush()
-		parentTx := []byte{1, 2, 3, 4}
-		childTx := []byte{1, 2}
-		parentHash := sha256.Sum256(parentTx)
+		mp.Flush()
+		originalTx := []byte{1, 2, 3, 4}
+		malleatedTx := []byte{1, 2}
+		originalHash := sha256.Sum256(originalTx)
 
 		// create the wrapped child transaction
-		wTx, err := types.WrapChildTx(parentHash[:], childTx)
+		wTx, err := types.WrapMalleatedTx(originalHash[:], malleatedTx)
 		require.NoError(t, err)
 
 		// add the parent transaction to the mempool
-		err = mempool.CheckTx(parentTx, nil, TxInfo{})
+		err = mp.CheckTx(context.Background(), originalTx, nil, mempool.TxInfo{})
 		require.NoError(t, err)
 
 		// remove the parent from the mempool using the wrapped child tx
-		err = mempool.Update(1, []types.Tx{wTx}, abciResponses(1, abci.CodeTypeOK), nil, nil)
+		err = mp.Update(1, []types.Tx{wTx}, abciResponses(1, abci.CodeTypeOK), nil, nil)
 		require.NoError(t, err)
 
-		assert.Zero(t, mempool.Size())
+		assert.Zero(t, mp.Size())
 
 	}
 }

internal/state/execution.go

@@ -544,11 +544,22 @@ func fireEvents(
 	}
 
 	for i, tx := range block.Data.Txs {
+		var txHash []byte
+		var rawTx []byte
+		if originalHash, malleatedTx, ismalleated := types.UnwrapMalleatedTx(tx); ismalleated {
+			txHash = originalHash
+			rawTx = malleatedTx
+		} else {
+			txHash = tx.Hash()
+			rawTx = tx
+		}
+
 		if err := eventBus.PublishEventTx(types.EventDataTx{TxResult: abci.TxResult{
-			Height: block.Height,
-			Index:  uint32(i),
-			Tx:     tx,
-			Result: *(abciResponses.DeliverTxs[i]),
+			Height:       block.Height,
+			Index:        uint32(i),
+			Tx:           rawTx,
+			Result:       *(abciResponses.DeliverTxs[i]),
+			OriginalHash: txHash,
 		}}); err != nil {
 			logger.Error("failed publishing event TX", "err", err)
 		}

internal/state/indexer/tx/kv/kv.go

@@ -12,6 +12,7 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/tmhash"
 	indexer "github.com/tendermint/tendermint/internal/state/indexer"
 	"github.com/tendermint/tendermint/libs/pubsub/query"
 	"github.com/tendermint/tendermint/libs/pubsub/query/syntax"
@@ -68,7 +69,12 @@ func (txi *TxIndex) Index(results []*abci.TxResult) error {
 	defer b.Close()
 
 	for _, result := range results {
-		hash := types.Tx(result.Tx).Hash()
+		var hash []byte
+		if len(result.OriginalHash) == tmhash.Size {
+			hash = result.OriginalHash
+		} else {
+			hash = types.Tx(result.Tx).Hash()
+		}
 
 		// index tx by events
 		err := txi.indexEvents(result, hash, b)

internal/state/indexer/tx/kv/kv_test.go

@@ -65,6 +65,49 @@ func TestTxIndex(t *testing.T) {
 	assert.True(t, proto.Equal(txResult2, loadedTxResult2))
 }
 
+func TestMalleatedTxIndex(t *testing.T) {
+	type test struct {
+		tx           types.Tx
+		originalHash []byte
+		expectedTx   []byte
+	}
+	originalTx1 := types.Tx([]byte("ORIGINAL_TX"))
+	malleatedTx1 := types.Tx([]byte("MALLEATED_TX"))
+
+	tests := []test{
+		// we expect to get the malleated tx returned when searching using the original hash
+		{
+			tx:           malleatedTx1,
+			originalHash: originalTx1.Hash(),
+			expectedTx:   malleatedTx1,
+		},
+	}
+
+	indexer := NewTxIndex(dbm.NewMemDB())
+
+	for i, tt := range tests {
+
+		txResult := &abci.TxResult{
+			Height: int64(i),
+			Index:  0,
+			Tx:     tt.tx,
+			Result: abci.ResponseDeliverTx{
+				Data: []byte{0},
+				Code: abci.CodeTypeOK, Log: "", Events: nil,
+			},
+			OriginalHash: tt.originalHash,
+		}
+
+		err := indexer.Index([]*abci.TxResult{txResult})
+		require.NoError(t, err)
+
+		loadedTxResult, err := indexer.Get(tt.originalHash)
+		require.NoError(t, err)
+		require.NotNil(t, loadedTxResult)
+		assert.Equal(t, tt.expectedTx, loadedTxResult.Tx)
+	}
+}
+
 func TestTxSearch(t *testing.T) {
 	indexer := NewTxIndex(dbm.NewMemDB())
 

node/node_test.go

@@ -406,7 +406,7 @@ func TestMaxProposalBlockSize(t *testing.T) {
 
 	// fill the mempool with one txs just below the maximum size
 	txLength := int(types.MaxDataBytesNoEvidence(maxBytes, types.MaxVotesCount))
-	tx := tmrand.Bytes(txLength - 6 - 4) // to account for the varint
+	tx := tmrand.Bytes(txLength - 9) // to account for the varint
 	err = mp.CheckTx(context.Background(), tx, nil, mempool.TxInfo{})
 	assert.NoError(t, err)
 	// now produce more txs than what a normal block can hold with 10 smaller txs
@@ -483,7 +483,7 @@ func TestMaxProposalBlockSize(t *testing.T) {
 	require.Equal(t, int64(pb.Header.Size()), types.MaxHeaderBytes)
 	require.Equal(t, int64(pb.LastCommit.Size()), types.MaxCommitBytes(types.MaxVotesCount))
 	// make sure that the block is less than the max possible size
-	assert.Equal(t, int64(pb.Size()), maxBytes)
+	assert.LessOrEqual(t, int64(pb.Size()), maxBytes)
 	// because of the proto overhead we expect the part set bytes to be equal or
 	// less than the pb block size
 	assert.LessOrEqual(t, partSet.ByteSize(), int64(pb.Size()))

proto/tendermint/abci/types.proto

@@ -289,10 +289,11 @@ message EventAttribute {
 //
 // One usage is indexing transaction results.
 message TxResult {
-  int64             height = 1;
-  uint32            index  = 2;
-  bytes             tx     = 3;
-  ResponseDeliverTx result = 4 [(gogoproto.nullable) = false];
+  int64             height        = 1;
+  uint32            index         = 2;
+  bytes             tx            = 3;
+  ResponseDeliverTx result        = 4 [(gogoproto.nullable) = false];
+  bytes             original_hash = 5;
 }
 
 //----------------------------------------

proto/tendermint/types/block.proto

@@ -7,7 +7,7 @@ import "gogoproto/gogo.proto";
 import "tendermint/types/types.proto";
 
 message Block {
-  Header                        header      = 1 [(gogoproto.nullable) = false];
-  Data                          data        = 2 [(gogoproto.nullable) = false];
-  Commit                        last_commit = 4;
+  Header header      = 1 [(gogoproto.nullable) = false];
+  Data   data        = 2 [(gogoproto.nullable) = false];
+  Commit last_commit = 4;
 }