Index malleated transactions properly (#612)

* index malleated transactions properly

* linter

* fix malleated transaction indexing and event tracking

* fix malleated event test

* fix test

* fix other test

* update proto building and formatting to use the latest tendermint specs style generation and formatting

* fix app tests

Makefile

@@ -3,6 +3,9 @@ OUTPUT?=build/tendermint
 
 BUILD_TAGS?=tendermint
 
+IMAGE := ghcr.io/tendermint/docker-build-proto:latest
+DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(IMAGE)
+
 # If building a release, please checkout the version tag to get the correct version setting
 ifneq ($(shell git symbolic-ref -q --short HEAD),)
 VERSION := unreleased-$(shell git symbolic-ref -q --short HEAD)-$(shell git rev-parse HEAD)
@@ -13,7 +16,6 @@ endif
 LD_FLAGS = -X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION)
 BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)"
 HTTPS_GIT := https://github.com/tendermint/tendermint.git
-DOCKER_BUF := docker run -v $(shell pwd):/workspace --workdir /workspace bufbuild/buf
 CGO_ENABLED ?= 0
 
 # handle nostrip
@@ -74,30 +76,29 @@ install:
 ###                                Protobuf                                 ###
 ###############################################################################
 
-proto-all: proto-gen proto-lint proto-check-breaking
+proto-all: proto-lint proto-check-breaking
 .PHONY: proto-all
 
 proto-gen:
-	@docker pull -q tendermintdev/docker-build-proto
 	@echo "Generating Protobuf files"
-	@docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto sh ./scripts/protocgen.sh
+	@$(DOCKER_PROTO_BUILDER) buf generate --template=./buf.gen.yaml --config ./buf.yaml
 .PHONY: proto-gen
 
 proto-lint:
-	@$(DOCKER_BUF) check lint --error-format=json
+	@$(DOCKER_PROTO_BUILDER) buf lint --error-format=json --config ./buf.yaml
 .PHONY: proto-lint
 
 proto-format:
 	@echo "Formatting Protobuf files"
-	docker run -v $(shell pwd):/workspace --workdir /workspace tendermintdev/docker-build-proto find ./ -not -path "./third_party/*" -name *.proto -exec clang-format -i {} \;
+	@$(DOCKER_PROTO_BUILDER) find . -name '*.proto' -path "./proto/*" -exec clang-format -i {} \;
 .PHONY: proto-format
 
 proto-check-breaking:
-	@$(DOCKER_BUF) check breaking --against-input .git#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against .git --config ./buf.yaml
 .PHONY: proto-check-breaking
 
 proto-check-breaking-ci:
-	@$(DOCKER_BUF) check breaking --against-input $(HTTPS_GIT)#branch=master
+	@$(DOCKER_PROTO_BUILDER) buf breaking --against $(HTTPS_GIT) --config ./buf.yaml
 .PHONY: proto-check-breaking-ci
 
 ###############################################################################

abci/types/application.go

@@ -98,7 +98,7 @@ func (BaseApplication) ApplySnapshotChunk(req RequestApplySnapshotChunk) Respons
 }
 
 func (BaseApplication) PreprocessTxs(req RequestPreprocessTxs) ResponsePreprocessTxs {
-	return ResponsePreprocessTxs{}
+	return ResponsePreprocessTxs{Txs: req.Txs}
 }
 
 //-------------------------------------------------------

proto/tendermint/abci/types.proto

@@ -228,7 +228,7 @@ message ResponseDeliverTx {
   int64          gas_wanted = 5 [json_name = "gas_wanted"];
   int64          gas_used   = 6 [json_name = "gas_used"];
   repeated Event events     = 7
-      [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; // nondeterministic
+      [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"];  // nondeterministic
   string codespace = 8;
 }
 
@@ -334,10 +334,11 @@ message EventAttribute {
 //
 // One usage is indexing transaction results.
 message TxResult {
-  int64             height = 1;
-  uint32            index  = 2;
-  bytes             tx     = 3;
-  ResponseDeliverTx result = 4 [(gogoproto.nullable) = false];
+  int64             height        = 1;
+  uint32            index         = 2;
+  bytes             tx            = 3;
+  ResponseDeliverTx result        = 4 [(gogoproto.nullable) = false];
+  bytes             original_hash = 5;
 }
 
 //----------------------------------------

proto/tendermint/da/data_availability_header.proto

@@ -14,8 +14,8 @@ option go_package = "github.com/tendermint/tendermint/proto/tendermint/da";
 // Note that currently we list row and column roots in separate fields
 // (different from the spec).
 message DataAvailabilityHeader {
-    // RowRoot_j 	= root((M_{j,1} || M_{j,2} || ... || M_{j,2k} ))
-    repeated bytes row_roots = 1;
-    // ColumnRoot_j = root((M_{1,j} || M_{2,j} || ... || M_{2k,j} ))
-    repeated bytes column_roots = 2;
-  }
+  // RowRoot_j 	= root((M_{j,1} || M_{j,2} || ... || M_{j,2k} ))
+  repeated bytes row_roots = 1;
+  // ColumnRoot_j = root((M_{1,j} || M_{2,j} || ... || M_{2k,j} ))
+  repeated bytes column_roots = 2;
+}

proto/tendermint/types/block.proto

@@ -7,7 +7,7 @@ import "gogoproto/gogo.proto";
 import "tendermint/types/types.proto";
 
 message Block {
-  Header                        header      = 1 [(gogoproto.nullable) = false];
-  Data                          data        = 2 [(gogoproto.nullable) = false];
-  Commit                        last_commit = 4;
+  Header header      = 1 [(gogoproto.nullable) = false];
+  Data   data        = 2 [(gogoproto.nullable) = false];
+  Commit last_commit = 4;
 }

proto/tendermint/types/types.proto

@@ -89,8 +89,8 @@ message Data {
   repeated bytes txs = 1;
 
   IntermediateStateRoots intermediate_state_roots = 2 [(gogoproto.nullable) = false];
-  EvidenceList evidence    = 3 [(gogoproto.nullable) = false];
-  Messages messages        = 4 [(gogoproto.nullable) = false];
+  EvidenceList           evidence                 = 3 [(gogoproto.nullable) = false];
+  Messages               messages                 = 4 [(gogoproto.nullable) = false];
 }
 
 // DuplicateVoteEvidence contains evidence of a validator signed two conflicting votes.
@@ -170,10 +170,10 @@ message Vote {
 
 // Commit contains the evidence that a block was committed by a set of validators.
 message Commit {
-  int64                         height     = 1;
-  int32                         round      = 2;
-  BlockID                       block_id   = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"];
-  repeated CommitSig            signatures = 4 [(gogoproto.nullable) = false];
+  int64              height     = 1;
+  int32              round      = 2;
+  BlockID            block_id   = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"];
+  repeated CommitSig signatures = 4 [(gogoproto.nullable) = false];
 }
 
 // CommitSig is a part of the Vote included in a Commit.
@@ -225,5 +225,5 @@ message TxProof {
 // transactions
 message MalleatedTx {
   bytes original_tx_hash = 1;
-  bytes tx = 2;
+  bytes tx               = 2;
 }

state/execution.go

@@ -548,11 +548,22 @@ func fireEvents(
 	}
 
 	for i, tx := range block.Data.Txs {
+		var txHash []byte
+		var rawTx []byte
+		if originalHash, malleatedTx, ismalleated := types.UnwrapMalleatedTx(tx); ismalleated {
+			txHash = originalHash
+			rawTx = malleatedTx
+		} else {
+			txHash = tx.Hash()
+			rawTx = tx
+		}
+
 		if err := eventBus.PublishEventTx(types.EventDataTx{TxResult: abci.TxResult{
-			Height: block.Height,
-			Index:  uint32(i),
-			Tx:     tx,
-			Result: *(abciResponses.DeliverTxs[i]),
+			Height:       block.Height,
+			Index:        uint32(i),
+			Tx:           rawTx,
+			Result:       *(abciResponses.DeliverTxs[i]),
+			OriginalHash: txHash,
 		}}); err != nil {
 			logger.Error("failed publishing event TX", "err", err)
 		}

state/txindex/kv/kv.go

@@ -12,6 +12,7 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/tmhash"
 	"github.com/tendermint/tendermint/libs/pubsub/query"
 	"github.com/tendermint/tendermint/state/indexer"
 	"github.com/tendermint/tendermint/state/txindex"
@@ -69,26 +70,7 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error {
 	defer storeBatch.Close()
 
 	for _, result := range b.Ops {
-		hash := types.Tx(result.Tx).Hash()
-
-		// index tx by events
-		err := txi.indexEvents(result, hash, storeBatch)
-		if err != nil {
-			return err
-		}
-
-		// index by height (always)
-		err = storeBatch.Set(keyForHeight(result), hash)
-		if err != nil {
-			return err
-		}
-
-		rawBytes, err := proto.Marshal(result)
-		if err != nil {
-			return err
-		}
-		// index by hash (always)
-		err = storeBatch.Set(hash, rawBytes)
+		err := txi.indexResult(storeBatch, result)
 		if err != nil {
 			return err
 		}
@@ -105,26 +87,7 @@ func (txi *TxIndex) Index(result *abci.TxResult) error {
 	b := txi.store.NewBatch()
 	defer b.Close()
 
-	hash := types.Tx(result.Tx).Hash()
-
-	// index tx by events
-	err := txi.indexEvents(result, hash, b)
-	if err != nil {
-		return err
-	}
-
-	// index by height (always)
-	err = b.Set(keyForHeight(result), hash)
-	if err != nil {
-		return err
-	}
-
-	rawBytes, err := proto.Marshal(result)
-	if err != nil {
-		return err
-	}
-	// index by hash (always)
-	err = b.Set(hash, rawBytes)
+	err := txi.indexResult(b, result)
 	if err != nil {
 		return err
 	}
@@ -158,6 +121,39 @@ func (txi *TxIndex) indexEvents(result *abci.TxResult, hash []byte, store dbm.Ba
 	return nil
 }
 
+func (txi *TxIndex) indexResult(batch dbm.Batch, result *abci.TxResult) error {
+	var hash []byte
+	if len(result.OriginalHash) == tmhash.Size {
+		hash = result.OriginalHash
+	} else {
+		hash = types.Tx(result.Tx).Hash()
+	}
+
+	rawBytes, err := proto.Marshal(result)
+	if err != nil {
+		return err
+	}
+
+	// index tx by events
+	err = txi.indexEvents(result, hash, batch)
+	if err != nil {
+		return err
+	}
+
+	// index by height (always)
+	err = batch.Set(keyForHeight(result), hash)
+	if err != nil {
+		return err
+	}
+
+	// index by hash (always)
+	err = batch.Set(hash, rawBytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // Search performs a search using the given query.
 //
 // It breaks the query into conditions (like "tx.height > 5"). For each

state/txindex/kv/kv_test.go

@@ -66,6 +66,51 @@ func TestTxIndex(t *testing.T) {
 	assert.True(t, proto.Equal(txResult2, loadedTxResult2))
 }
 
+func TestMalleatedTxIndex(t *testing.T) {
+	type test struct {
+		tx           types.Tx
+		originalHash []byte
+		expectedTx   []byte
+	}
+	originalTx1 := types.Tx([]byte("ORIGINAL_TX"))
+	malleatedTx1 := types.Tx([]byte("MALLEATED_TX"))
+
+	tests := []test{
+		// we expect to get the malleated tx returned when searching using the original hash
+		{
+			tx:           malleatedTx1,
+			originalHash: originalTx1.Hash(),
+			expectedTx:   malleatedTx1,
+		},
+	}
+
+	indexer := NewTxIndex(db.NewMemDB())
+
+	for i, tt := range tests {
+
+		txResult := &abci.TxResult{
+			Height: int64(i),
+			Index:  0,
+			Tx:     tt.tx,
+			Result: abci.ResponseDeliverTx{
+				Data: []byte{0},
+				Code: abci.CodeTypeOK, Log: "", Events: nil,
+			},
+			OriginalHash: tt.originalHash,
+		}
+		batch := txindex.NewBatch(1)
+		if err := batch.Add(txResult); err != nil {
+			t.Error(err)
+		}
+		err := indexer.AddBatch(batch)
+		require.NoError(t, err)
+
+		loadedTxResult, err := indexer.Get(tt.originalHash)
+		require.NoError(t, err)
+		assert.Equal(t, tt.expectedTx, loadedTxResult.Tx)
+	}
+}
+
 func TestTxSearch(t *testing.T) {
 	indexer := NewTxIndex(db.NewMemDB())
 

types/event_bus.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/tmhash"
 	"github.com/tendermint/tendermint/libs/log"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 	"github.com/tendermint/tendermint/libs/service"
@@ -178,9 +179,8 @@ func (b *EventBus) PublishEventTx(data EventDataTx) error {
 	ctx := context.Background()
 
 	var txHash []byte
-	if originalHash, malleated, ismalleated := UnwrapMalleatedTx(data.Tx); ismalleated {
-		txHash = originalHash
-		data.Tx = malleated
+	if len(data.OriginalHash) == tmhash.Size {
+		txHash = data.OriginalHash
 	} else {
 		txHash = Tx(data.Tx).Hash()
 	}

types/event_bus_test.go

@@ -77,8 +77,6 @@ func TestEventBusPublishEventMalleatedTx(t *testing.T) {
 
 	tx := Tx("foo")
 	malleatedTx := Tx("foo-malleated")
-	wrappedMalleatedTx, err := WrapMalleatedTx(tx.Hash(), malleatedTx)
-	require.NoError(t, err)
 
 	result := abci.ResponseDeliverTx{
 		Data: []byte("bar"),
@@ -104,10 +102,11 @@ func TestEventBusPublishEventMalleatedTx(t *testing.T) {
 	}()
 
 	err = eventBus.PublishEventTx(EventDataTx{abci.TxResult{
-		Height: 1,
-		Index:  0,
-		Tx:     wrappedMalleatedTx,
-		Result: result,
+		Height:       1,
+		Index:        0,
+		Tx:           malleatedTx,
+		Result:       result,
+		OriginalHash: tx.Hash(),
 	}})
 	assert.NoError(t, err)