From 678f324889e6325567597bcba58d9eea86015bfd Mon Sep 17 00:00:00 2001 From: Nina Barbakadze Date: Tue, 30 Jan 2024 16:31:24 +0100 Subject: [PATCH 1/6] initial commit --- node/node.go | 9 ++++++++- state/txindex/kv/kv.go | 20 +++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/node/node.go b/node/node.go index 0feeb3edc4..968f255dd6 100644 --- a/node/node.go +++ b/node/node.go @@ -295,6 +295,14 @@ func createAndStartIndexerService( txIndexer = kv.NewTxIndex(store) blockIndexer = blockidxkv.New(dbm.NewPrefixDB(store, []byte("block_events"))) + case "kv_lite": + store, err := dbProvider(&DBContext{"tx_index", config}) + if err != nil { + return nil, nil, nil, err + } + txIndexer = kv.NewTxIndexWithConfig(store, config) + blockIndexer = blockidxkv.New(dbm.NewPrefixDB(store, []byte("block_events"))) + case "psql": if config.TxIndex.PsqlConn == "" { return nil, nil, nil, errors.New(`no psql-conn is set for the "psql" indexer`) @@ -1162,7 +1170,6 @@ func (n *Node) OnStop() { n.Logger.Error("Pyroscope tracer Shutdown", "err", err) } } - } // ConfigureRPC makes sure RPC has all the objects it needs to operate. diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 6843a510b6..69ca0d8b57 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -12,6 +12,7 @@ import ( "github.com/gogo/protobuf/proto" abci "github.com/cometbft/cometbft/abci/types" + cfg "github.com/cometbft/cometbft/config" "github.com/cometbft/cometbft/libs/pubsub/query" "github.com/cometbft/cometbft/state/indexer" "github.com/cometbft/cometbft/state/txindex" @@ -29,6 +30,7 @@ var _ txindex.TxIndexer = (*TxIndex)(nil) type TxIndex struct { store dbm.DB // Number the events in the event list + config *cfg.Config eventSeq int64 } @@ -39,6 +41,13 @@ func NewTxIndex(store dbm.DB) *TxIndex { } } +func NewTxIndexWithConfig(store dbm.DB, config *cfg.Config) *TxIndex { + return &TxIndex{ + store: store, + config: config, + } +} + // Get gets transaction from the TxIndex storage and returns it or nil if the // transaction is not found. func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) { @@ -121,6 +130,12 @@ func (txi *TxIndex) Index(result *abci.TxResult) error { return err } + if txi.config.TxIndex.Indexer == "kv-lite" { + // set the transaction bytes to empty + // this is to avoid the transaction bytes being set + // in the kv_lite indexer + result.Tx = []byte{} + } rawBytes, err := proto.Marshal(result) if err != nil { return err @@ -282,7 +297,7 @@ func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResul // extract ranges // if both upper and lower bounds exist, it's better to get them in order not // no iterate over kvs that are not within range. - //If we have a query range over height and want to still look for + // If we have a query range over height and want to still look for // specific event values we do not want to simply return all // transactios in this height range. We remember the height range info // and pass it on to match() to take into account when processing events. @@ -381,6 +396,7 @@ func lookForHeight(conditions []query.Condition) (height int64, heightIdx int) { } return 0, -1 } + func (txi *TxIndex) setTmpHashes(tmpHeights map[string][]byte, it dbm.Iterator, matchEvents bool) { if matchEvents { eventSeq := extractEventSeqFromKey(it.Key()) @@ -663,6 +679,7 @@ func extractHeightFromKey(key []byte) (int64, error) { parts := strings.SplitN(string(key), tagKeySeparator, -1) return strconv.ParseInt(parts[len(parts)-2], 10, 64) } + func extractValueFromKey(key []byte) string { keyString := string(key) parts := strings.SplitN(keyString, tagKeySeparator, -1) @@ -688,6 +705,7 @@ func extractEventSeqFromKey(key []byte) string { } return "0" } + func keyForEvent(key string, value string, result *abci.TxResult, eventSeq int64) []byte { return []byte(fmt.Sprintf("%s/%s/%d/%d%s", key, From 06914763927c49eb362e9475703b053b6224a0a0 Mon Sep 17 00:00:00 2001 From: Nina Barbakadze Date: Tue, 30 Jan 2024 16:41:58 +0100 Subject: [PATCH 2/6] fix: try fix nil pointer error --- state/txindex/kv/kv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 69ca0d8b57..bc91d11456 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -130,7 +130,7 @@ func (txi *TxIndex) Index(result *abci.TxResult) error { return err } - if txi.config.TxIndex.Indexer == "kv-lite" { + if txi.config != nil && txi.config.TxIndex.Indexer == "kv-lite" { // set the transaction bytes to empty // this is to avoid the transaction bytes being set // in the kv_lite indexer From 3933a496b3e5873e732814f55d31dc14f91593c4 Mon Sep 17 00:00:00 2001 From: Nina Barbakadze Date: Wed, 31 Jan 2024 18:44:42 +0100 Subject: [PATCH 3/6] test: add unit test for indexing with empty TX fields --- state/txindex/kv/kv.go | 9 ++++++- state/txindex/kv/kv_test.go | 53 +++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index bc91d11456..ecf7f6a14b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -130,7 +130,7 @@ func (txi *TxIndex) Index(result *abci.TxResult) error { return err } - if txi.config != nil && txi.config.TxIndex.Indexer == "kv-lite" { + if txi.config != nil && txi.config.TxIndex.Indexer == "kv_lite" { // set the transaction bytes to empty // this is to avoid the transaction bytes being set // in the kv_lite indexer @@ -183,6 +183,13 @@ func (txi *TxIndex) indexEvents(result *abci.TxResult, hash []byte, store dbm.Ba func (txi *TxIndex) indexResult(batch dbm.Batch, result *abci.TxResult) error { hash := types.Tx(result.Tx).Hash() + if txi.config != nil && txi.config.TxIndex.Indexer == "kv_lite" { + // set the transaction bytes to empty + // this is to avoid the transaction bytes being set + // in the kv_lite indexer + result.Tx = []byte{} + } + rawBytes, err := proto.Marshal(result) if err != nil { return err diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 6311eaca41..e9330480a7 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "context" "fmt" "os" @@ -13,6 +14,7 @@ import ( db "github.com/cometbft/cometbft-db" abci "github.com/cometbft/cometbft/abci/types" + "github.com/cometbft/cometbft/config" "github.com/cometbft/cometbft/libs/pubsub/query" cmtrand "github.com/cometbft/cometbft/libs/rand" "github.com/cometbft/cometbft/state/txindex" @@ -65,6 +67,57 @@ func TestTxIndex(t *testing.T) { assert.True(t, proto.Equal(txResult2, loadedTxResult2)) } +func TestKvLiteTxIndex(t *testing.T) { + // create kv_lite indexer + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + tx := types.Tx("TX SHOULD BE EMPTY AFTER INDEXING") + txResult := &abci.TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, Log: "", Events: nil, + }, + } + hash := tx.Hash() + + batch := txindex.NewBatch(1) + if err := batch.Add(txResult); err != nil { + t.Error(err) + } + err := indexer.AddBatch(batch) + require.NoError(t, err) + + loadedTxResult, err := indexer.Get(hash) + require.NoError(t, err) + assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) + + tx2 := types.Tx("BYE BYE WORLD") + txResult2 := &abci.TxResult{ + Height: 1, + Index: 0, + Tx: tx2, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, Log: "", Events: nil, + }, + } + hash2 := tx2.Hash() + + err = indexer.Index(txResult2) + require.NoError(t, err) + + loadedTxResult2, err := indexer.Get(hash2) + require.NoError(t, err) + assert.True(t, bytes.Equal(loadedTxResult2.Tx, []byte{})) +} + func TestWrappedTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) From 326536dba6cd72646055c33a81a90dff236ee9a0 Mon Sep 17 00:00:00 2001 From: Nina Barbakadze Date: Thu, 1 Feb 2024 12:28:06 +0100 Subject: [PATCH 4/6] add kv_lite test and add emtpy tx batching logic for indexResult --- state/txindex/kv/kv.go | 4 +- state/txindex/kv/kv_lite_test.go | 750 +++++++++++++++++++++++++++++++ state/txindex/kv/kv_test.go | 88 +++- 3 files changed, 834 insertions(+), 8 deletions(-) create mode 100644 state/txindex/kv/kv_lite_test.go diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index ecf7f6a14b..69b10bebe7 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -68,7 +68,8 @@ func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) { if err != nil { return nil, fmt.Errorf("error reading TxResult: %v", err) } - + + fmt.Println("txResult is nil", txResult.Tx == nil) return txResult, nil } @@ -136,6 +137,7 @@ func (txi *TxIndex) Index(result *abci.TxResult) error { // in the kv_lite indexer result.Tx = []byte{} } + fmt.Println("txResult is nil after indexing", result.Tx == nil) rawBytes, err := proto.Marshal(result) if err != nil { return err diff --git a/state/txindex/kv/kv_lite_test.go b/state/txindex/kv/kv_lite_test.go new file mode 100644 index 0000000000..4a8a1987b1 --- /dev/null +++ b/state/txindex/kv/kv_lite_test.go @@ -0,0 +1,750 @@ +package kv + +import ( + "bytes" + "context" + "fmt" + "os" + "testing" + + "github.com/gogo/protobuf/proto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + db "github.com/cometbft/cometbft-db" + + abci "github.com/cometbft/cometbft/abci/types" + "github.com/cometbft/cometbft/config" + "github.com/cometbft/cometbft/libs/pubsub/query" + cmtrand "github.com/cometbft/cometbft/libs/rand" + "github.com/cometbft/cometbft/state/txindex" + "github.com/cometbft/cometbft/types" +) + +func TestTxIndex_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + tx := types.Tx("HELLO WORLD") + txResult := &abci.TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, Log: "", Events: nil, + }, + } + hash := tx.Hash() + + batch := txindex.NewBatch(1) + if err := batch.Add(txResult); err != nil { + t.Error(err) + } + err := indexer.AddBatch(batch) + require.NoError(t, err) + + loadedTxResult, err := indexer.Get(hash) + require.NoError(t, err) + assert.True(t, proto.Equal(txResult, loadedTxResult)) + // --------------------- check tx is empty ----------------------- + assert.True(t, bytes.Equal(txResult.Tx, []byte{})) + + tx2 := types.Tx("BYE BYE WORLD") + txResult2 := &abci.TxResult{ + Height: 1, + Index: 0, + Tx: tx2, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, Log: "", Events: nil, + }, + } + hash2 := tx2.Hash() + + err = indexer.Index(txResult2) + require.NoError(t, err) + + loadedTxResult2, err := indexer.Get(hash2) + require.NoError(t, err) + assert.True(t, proto.Equal(txResult2, loadedTxResult2)) + // --------------------- check tx is empty ----------------------- + assert.True(t, bytes.Equal(loadedTxResult2.Tx, []byte{})) +} + +func TestWrappedTxIndex_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + + tx := types.Tx("HELLO WORLD") + wrappedTx, err := types.MarshalIndexWrapper(tx, 11) + require.NoError(t, err) + txResult := &abci.TxResult{ + Height: 1, + Index: 0, + Tx: wrappedTx, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, Log: "", Events: nil, + }, + } + hash := tx.Hash() + + batch := txindex.NewBatch(1) + if err := batch.Add(txResult); err != nil { + t.Error(err) + } + err = indexer.AddBatch(batch) + require.NoError(t, err) + + loadedTxResult, err := indexer.Get(hash) + require.NoError(t, err) + assert.True(t, proto.Equal(txResult, loadedTxResult)) + // --------------------- check tx is empty ----------------------- + fmt.Println(loadedTxResult.Tx, "LOADED TX RESULT") + assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) +} + +func TestTxSearch_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + txResult := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "owner", Value: "Ivan", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "owner", Value: "/Ivan/", Index: true}, {Key: "number", Value: "10", Index: true}}}, + {Type: "", Attributes: []abci.EventAttribute{{Key: "not_allowed", Value: "Vlad", Index: true}}}, + }) + fmt.Println(txResult, "TXRESULt") + hash := types.Tx(txResult.Tx).Hash() + + err := indexer.Index(txResult) + require.NoError(t, err) + + testCases := []struct { + q string + resultsLength int + }{ + // search by hash + {fmt.Sprintf("tx.hash = '%X'", hash), 1}, + // search by hash (lower) + {fmt.Sprintf("tx.hash = '%x'", hash), 1}, + // search by exact match (one key) + {"account.number = 1", 1}, + // search by exact match (two keys) + {"account.number = 1 AND account.owner = 'Ivan'", 1}, + {"account.owner = 'Ivan' AND account.number = 1", 1}, + // search by exact match (two keys) + {"account.number = 1 AND account.owner = 'Vlad'", 0}, + {"account.owner = 'Vlad' AND account.number = 1", 0}, + {"account.number >= 1 AND account.owner = 'Vlad'", 0}, + {"account.owner = 'Vlad' AND account.number >= 1", 0}, + {"account.number <= 0", 0}, + {"account.number <= 0 AND account.owner = 'Ivan'", 0}, + {"account.number < 10000 AND account.owner = 'Ivan'", 1}, + // search using a prefix of the stored value + {"account.owner = 'Iv'", 0}, + // search for owner with slash in name + {"account.owner = '/Ivan/'", 1}, + // search for owner with slash in name and match events + {"match.events = 1 AND account.owner = '/Ivan/' AND account.number = 10", 1}, + // search for owner with slash in name and match events with no match + {"match.events = 1 AND account.owner = '/Ivan/' AND account.number = 1", 0}, + // search for owner with slash in name with CONTAINS + {"account.owner CONTAINS 'an'", 1}, + // search for owner with slash in name with CONTAINS and match events + {"match.events = 1 AND account.owner CONTAINS 'an'", 1}, + // search by range + {"account.number >= 1 AND account.number <= 5", 1}, + // search by range and another key + {"account.number >= 1 AND account.owner = 'Ivan' AND account.number <= 5", 1}, + // search by range (lower bound) + {"account.number >= 1", 1}, + // search by range (upper bound) + {"account.number <= 5", 1}, + {"account.number <= 1", 1}, + // search using not allowed key + {"not_allowed = 'boom'", 0}, + {"not_allowed = 'Vlad'", 0}, + // search for not existing tx result + {"account.number >= 2 AND account.number <= 5", 0}, + // search using not existing key + {"account.date >= TIME 2013-05-03T14:45:00Z", 0}, + // search using CONTAINS + {"account.owner CONTAINS 'an'", 1}, + // search for non existing value using CONTAINS + {"account.owner CONTAINS 'Vlad'", 0}, + {"account.owner CONTAINS 'Ivann'", 0}, + {"account.owner CONTAINS 'IIvan'", 0}, + {"account.owner CONTAINS 'Iva n'", 0}, + {"account.owner CONTAINS ' Ivan'", 0}, + {"account.owner CONTAINS 'Ivan '", 0}, + // search using the wrong key (of numeric type) using CONTAINS + {"account.number CONTAINS 'Iv'", 0}, + // search using EXISTS + {"account.number EXISTS", 1}, + // search using EXISTS for non existing key + {"account.date EXISTS", 0}, + {"not_allowed EXISTS", 0}, + } + + ctx := context.Background() + + for _, tc := range testCases { + tc := tc + t.Run(tc.q, func(t *testing.T) { + results, err := indexer.Search(ctx, query.MustParse(tc.q)) + + assert.NoError(t, err) + + assert.Len(t, results, tc.resultsLength) + if tc.resultsLength > 0 { + for _, txr := range results { + fmt.Println(txr, "TXR") + fmt.Println(txResult, "TXRESULT") + assert.True(t, proto.Equal(txResult, txr)) + } + } + }) + } +} + +func TestTxSearchEventMatch_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + txResult := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}, {Key: "owner", Value: "Ana", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "2", Index: true}, {Key: "owner", Value: "Ivan", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "3", Index: false}, {Key: "owner", Value: "Mickey", Index: false}}}, + {Type: "", Attributes: []abci.EventAttribute{{Key: "not_allowed", Value: "Vlad", Index: true}}}, + }) + + err := indexer.Index(txResult) + require.NoError(t, err) + + testCases := map[string]struct { + q string + resultsLength int + }{ + "Don't match non-indexed events": { + q: "match.events = 1 AND account.number = 3 AND account.owner = 'Mickey'", + resultsLength: 0, + }, + "Return all events from a height with range": { + q: "match.events = 1 AND tx.height > 0", + resultsLength: 1, + }, + "Return all events from a height with range 2": { + q: "match.events = 1 AND tx.height <= 1", + resultsLength: 1, + }, + "Return all events from a height": { + q: "match.events = 1 AND tx.height = 1", + resultsLength: 1, + }, + "Return all events from a height (deduplicate height)": { + q: "match.events = 1 AND tx.height = 1 AND tx.height = 1", + resultsLength: 1, + }, + "Match attributes with height range and event": { + q: "match.events = 1 AND tx.height < 2 AND tx.height > 0 AND account.number = 1 AND account.owner CONTAINS 'Ana' AND account.owner CONTAINS 'An'", + resultsLength: 1, + }, + "Match attributes with height range and event - no match": { + q: "match.events = 1 AND tx.height < 2 AND tx.height > 0 AND account.number = 2 AND account.owner = 'Ana'", + resultsLength: 0, + }, + "Deduplucation test - match events only at the beginning": { + q: "tx.height < 2 AND tx.height > 0 AND account.number = 2 AND account.owner = 'Ana' AND match.events = 1", + resultsLength: 1, + }, + "Deduplucation test - should return nothing if attribute repeats multiple times": { + q: "match.events = 0 AND tx.height < 2 AND account.number = 3 AND account.number = 2 AND account.number = 5", + resultsLength: 0, + }, + "Deduplucation test - should return nothing if attribute repeats multiple times with match events": { + q: "match.events = 1 AND tx.height < 2 AND account.number = 3 AND account.number = 2 AND account.number = 5", + resultsLength: 0, + }, + "Deduplucation test - match events multiple": { + q: "match.events = 1 AND tx.height < 2 AND tx.height > 0 AND account.number = 2 AND account.owner = 'Ana' AND match.events = 1", + resultsLength: 0, + }, + "Match attributes with event": { + q: "account.number = 2 AND account.owner = 'Ana' AND tx.height = 1", + resultsLength: 1, + }, + "Match range w/o match events": { + q: "account.number < 2 AND account.owner = 'Ivan'", + resultsLength: 1, + }, + " Match range with match events set to 0": { + q: "match.events = 0 AND account.number < 2 AND account.owner = 'Ivan' AND tx.height > 0", + resultsLength: 1, + }, + " Match range with match events": { + q: "match.events = 1 AND account.number < 2 AND account.owner = 'Ivan' AND tx.height > 0", + resultsLength: 0, + }, + " Match range with match events 2": { + q: "match.events = 1 AND account.number <= 2 AND account.owner = 'Ivan' AND tx.height > 0", + resultsLength: 1, + }, + } + + ctx := context.Background() + + for _, tc := range testCases { + tc := tc + t.Run(tc.q, func(t *testing.T) { + results, err := indexer.Search(ctx, query.MustParse(tc.q)) + assert.NoError(t, err) + + assert.Len(t, results, tc.resultsLength) + if tc.resultsLength > 0 { + for _, txr := range results { + assert.True(t, proto.Equal(txResult, txr)) + } + } + }) + } +} + +func TestTxSearchWithCancelation_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + txResult := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "owner", Value: "Ivan", Index: true}}}, + {Type: "", Attributes: []abci.EventAttribute{{Key: "not_allowed", Value: "Vlad", Index: true}}}, + }) + err := indexer.Index(txResult) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + results, err := indexer.Search(ctx, query.MustParse("account.number = 1")) + assert.NoError(t, err) + assert.Empty(t, results) +} + +func TestTxSearchDeprecatedIndexing_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + // index tx using events indexing (composite key) + txResult1 := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, + }) + hash1 := types.Tx(txResult1.Tx).Hash() + + err := indexer.Index(txResult1) + require.NoError(t, err) + + // index tx also using deprecated indexing (event as key) + txResult2 := txResultWithEvents(nil) + txResult2.Tx = types.Tx("HELLO WORLD 2") + + hash2 := types.Tx(txResult2.Tx).Hash() + b := indexer.store.NewBatch() + + rawBytes, err := proto.Marshal(txResult2) + require.NoError(t, err) + + depKey := []byte(fmt.Sprintf("%s/%s/%d/%d", + "sender", + "addr1", + txResult2.Height, + txResult2.Index, + )) + + err = b.Set(depKey, hash2) + require.NoError(t, err) + err = b.Set(keyForHeight(txResult2), hash2) + require.NoError(t, err) + err = b.Set(hash2, rawBytes) + require.NoError(t, err) + err = b.Write() + require.NoError(t, err) + + testCases := []struct { + q string + results []*abci.TxResult + }{ + // search by hash + {fmt.Sprintf("tx.hash = '%X'", hash1), []*abci.TxResult{txResult1}}, + // search by hash + {fmt.Sprintf("tx.hash = '%X'", hash2), []*abci.TxResult{txResult2}}, + // search by exact match (one key) + {"account.number = 1", []*abci.TxResult{txResult1}}, + {"account.number >= 1 AND account.number <= 5", []*abci.TxResult{txResult1}}, + // search by range (lower bound) + {"account.number >= 1", []*abci.TxResult{txResult1}}, + // search by range (upper bound) + {"account.number <= 5", []*abci.TxResult{txResult1}}, + // search using not allowed key + {"not_allowed = 'boom'", []*abci.TxResult{}}, + // search for not existing tx result + {"account.number >= 2 AND account.number <= 5", []*abci.TxResult{}}, + // search using not existing key + {"account.date >= TIME 2013-05-03T14:45:00Z", []*abci.TxResult{}}, + // search by deprecated key + {"sender = 'addr1'", []*abci.TxResult{txResult2}}, + } + + ctx := context.Background() + + for _, tc := range testCases { + tc := tc + t.Run(tc.q, func(t *testing.T) { + results, err := indexer.Search(ctx, query.MustParse(tc.q)) + require.NoError(t, err) + for _, txr := range results { + for _, tr := range tc.results { + assert.True(t, proto.Equal(tr, txr)) + } + } + }) + } +} + +func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + txResult := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "2", Index: true}}}, + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "3", Index: false}}}, + }) + + err := indexer.Index(txResult) + require.NoError(t, err) + + testCases := []struct { + q string + found bool + }{ + { + q: "match.events = 1 AND account.number >= 1", + found: true, + }, + { + q: "match.events = 1 AND account.number > 2", + found: false, + }, + { + q: "match.events = 1 AND account.number >= 1 AND tx.height = 3 AND tx.height > 0", + found: true, + }, + { + q: "match.events = 1 AND account.number >= 1 AND tx.height > 0 AND tx.height = 3", + found: true, + }, + + // { + // q: "match.events = 1 AND account.number >= 1 AND tx.height = 3 AND tx.height = 2 AND tx.height = 1", + // found: true, + // }, + + { + q: "match.events = 1 AND account.number >= 1 AND tx.height = 1 AND tx.height = 2 AND tx.height = 1", + found: true, + }, + { + q: "match.events = 1 AND account.number >= 1 AND tx.height = 3", + found: false, + }, + { + q: "match.events = 1 AND account.number > 1 AND tx.height < 2", + found: true, + }, + { + q: "match.events = 1 AND account.number >= 2", + found: true, + }, + { + q: "match.events = 1 AND account.number <= 1", + found: true, + }, + { + q: "match.events = 1 AND account.number = 'something'", + found: false, + }, + { + q: "match.events = 1 AND account.number CONTAINS 'bla'", + found: false, + }, + } + + ctx := context.Background() + + for _, tc := range testCases { + results, err := indexer.Search(ctx, query.MustParse(tc.q)) + assert.NoError(t, err) + + len := 0 + if tc.found { + len = 1 + } + assert.Len(t, results, len) + assert.True(t, !tc.found || proto.Equal(txResult, results[0])) + } +} + +func TestTxIndexDuplicatePreviouslySuccessful_kv_lite(t *testing.T) { + mockTx := types.Tx("MOCK_TX_HASH") + + testCases := []struct { + name string + tx1 *abci.TxResult + tx2 *abci.TxResult + expOverwrite bool // do we expect the second tx to overwrite the first tx + }{ + { + "don't overwrite as a non-zero code was returned and the previous tx was successful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + false, + }, + { + "overwrite as the previous tx was also unsuccessful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + true, + }, + { + "overwrite as the most recent tx was successful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + true, + }, + } + + hash := mockTx.Hash() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fmt.Println(tc.tx2, "TX2 BEFORE") + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + // index the first tx + err := indexer.Index(tc.tx1) + require.NoError(t, err) + + // index the same tx with different results + err = indexer.Index(tc.tx2) + require.NoError(t, err) + + res, err := indexer.Get(hash) + require.NoError(t, err) + if tc.expOverwrite { + require.Equal(t, tc.tx2.Height, res.Height) + require.Equal(t, tc.tx2.Index, res.Index) + // require.Equal(t, tc.tx2.Tx, res.Tx) + require.Equal(t, tc.tx2.Result, res.Result) + // pointer was changed here + bytes.Equal(tc.tx2.Tx, []byte{}) + // here check that res.tx is empty + bytes.Equal(res.Tx, nil) + + } else { + require.Equal(t, tc.tx1.Height, res.Height) + require.Equal(t, tc.tx1.Index, res.Index) + // require.Equal(t, tc.tx2.Tx, res.Tx) + require.Equal(t, tc.tx1.Result, res.Result) + // pointer was changed here + bytes.Equal(tc.tx1.Tx, []byte{}) + // here check that res.tx is empty + bytes.Equal(res.Tx, nil) + } + }) + } +} + +func TestTxSearchMultipleTxs_kv_lite(t *testing.T) { + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + // indexed first, but bigger height (to test the order of transactions) + txResult := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, + }) + + txResult.Tx = types.Tx("Bob's account") + txResult.Height = 2 + txResult.Index = 1 + err := indexer.Index(txResult) + require.NoError(t, err) + + // indexed second, but smaller height (to test the order of transactions) + txResult2 := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "2", Index: true}}}, + }) + txResult2.Tx = types.Tx("Alice's account") + txResult2.Height = 1 + txResult2.Index = 2 + + err = indexer.Index(txResult2) + require.NoError(t, err) + + // indexed third (to test the order of transactions) + txResult3 := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "3", Index: true}}}, + }) + txResult3.Tx = types.Tx("Jack's account") + txResult3.Height = 1 + txResult3.Index = 1 + err = indexer.Index(txResult3) + require.NoError(t, err) + + // indexed fourth (to test we don't include txs with similar events) + // https://github.com/cometbft/cometbft/issues/2908 + txResult4 := txResultWithEvents([]abci.Event{ + {Type: "account", Attributes: []abci.EventAttribute{{Key: "number.id", Value: "1", Index: true}}}, + }) + txResult4.Tx = types.Tx("Mike's account") + txResult4.Height = 2 + txResult4.Index = 2 + err = indexer.Index(txResult4) + require.NoError(t, err) + + ctx := context.Background() + + results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + assert.NoError(t, err) + + require.Len(t, results, 3) + + // hereeee check that all results txs are empty + for _, txr := range results { + fmt.Println(txr.Tx, "TXR TX HERE") + fmt.Println(txr, "TXRRR") + fmt.Println(txr.Tx == nil, "RES TX NIL") + assert.True(t, bytes.Equal(txr.Tx, []byte{})) + } +} + +func benchmarkTxIndex_kv_lite(txsCount int64, b *testing.B) { + dir, err := os.MkdirTemp("", "tx_index_db") + require.NoError(b, err) + defer os.RemoveAll(dir) + + store, err := db.NewDB("tx_index", "goleveldb", dir) + require.NoError(b, err) + // changed here + indexer := NewTxIndexWithConfig(store, &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) + + batch := txindex.NewBatch(txsCount) + txIndex := uint32(0) + for i := int64(0); i < txsCount; i++ { + tx := cmtrand.Bytes(250) + txResult := &abci.TxResult{ + Height: 1, + Index: txIndex, + Tx: tx, + Result: abci.ResponseDeliverTx{ + Data: []byte{0}, + Code: abci.CodeTypeOK, + Log: "", + Events: []abci.Event{}, + }, + } + if err := batch.Add(txResult); err != nil { + b.Fatal(err) + } + txIndex++ + } + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + err = indexer.AddBatch(batch) + } + if err != nil { + b.Fatal(err) + } +} + +func BenchmarkTxIndex1_kv_lite(b *testing.B) { benchmarkTxIndex(1, b) } +func BenchmarkTxIndex500_kv_lite(b *testing.B) { benchmarkTxIndex(500, b) } +func BenchmarkTxIndex1000_kv_lite(b *testing.B) { benchmarkTxIndex(1000, b) } +func BenchmarkTxIndex2000_kv_lite(b *testing.B) { benchmarkTxIndex(2000, b) } +func BenchmarkTxIndex10000_kv_lite(b *testing.B) { benchmarkTxIndex(10000, b) } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index e9330480a7..b93c9d266e 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -22,7 +22,12 @@ import ( ) func TestTxIndex(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) tx := types.Tx("HELLO WORLD") txResult := &abci.TxResult{ @@ -98,7 +103,7 @@ func TestKvLiteTxIndex(t *testing.T) { require.NoError(t, err) assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) - tx2 := types.Tx("BYE BYE WORLD") + tx2 := types.Tx("This should be empty after indexing too") txResult2 := &abci.TxResult{ Height: 1, Index: 0, @@ -116,8 +121,42 @@ func TestKvLiteTxIndex(t *testing.T) { loadedTxResult2, err := indexer.Get(hash2) require.NoError(t, err) assert.True(t, bytes.Equal(loadedTxResult2.Tx, []byte{})) + } +// func TestWrappedTxIndex_kv_lite(t *testing.T) { +// indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ +// TxIndex: &config.TxIndexConfig{ +// Indexer: "kv_lite", +// }, +// }) + +// tx := types.Tx("HELLO WORLD") +// wrappedTx, err := types.MarshalIndexWrapper(tx, 11) +// require.NoError(t, err) +// txResult := &abci.TxResult{ +// Height: 1, +// Index: 0, +// Tx: wrappedTx, +// Result: abci.ResponseDeliverTx{ +// Data: []byte{0}, +// Code: abci.CodeTypeOK, Log: "", Events: nil, +// }, +// } +// hash := tx.Hash() + +// batch := txindex.NewBatch(1) +// if err := batch.Add(txResult); err != nil { +// t.Error(err) +// } +// err = indexer.AddBatch(batch) +// require.NoError(t, err) + +// loadedTxResult, err := indexer.Get(hash) +// require.NoError(t, err) +// assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) +// } + func TestWrappedTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) @@ -148,7 +187,12 @@ func TestWrappedTxIndex(t *testing.T) { } func TestTxSearch(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, @@ -156,6 +200,7 @@ func TestTxSearch(t *testing.T) { {Type: "account", Attributes: []abci.EventAttribute{{Key: "owner", Value: "/Ivan/", Index: true}, {Key: "number", Value: "10", Index: true}}}, {Type: "", Attributes: []abci.EventAttribute{{Key: "not_allowed", Value: "Vlad", Index: true}}}, }) + fmt.Println(txResult, "TXRESULt") hash := types.Tx(txResult.Tx).Hash() err := indexer.Index(txResult) @@ -234,11 +279,14 @@ func TestTxSearch(t *testing.T) { tc := tc t.Run(tc.q, func(t *testing.T) { results, err := indexer.Search(ctx, query.MustParse(tc.q)) + assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) if tc.resultsLength > 0 { for _, txr := range results { + fmt.Println(txr, "TXR") + fmt.Println(txResult, "TXRESULT") assert.True(t, proto.Equal(txResult, txr)) } } @@ -248,7 +296,13 @@ func TestTxSearch(t *testing.T) { func TestTxSearchEventMatch(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndex(db.NewMemDB()) + + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}, {Key: "owner", Value: "Ana", Index: true}}}, @@ -348,7 +402,12 @@ func TestTxSearchEventMatch(t *testing.T) { } } func TestTxSearchWithCancelation(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, @@ -366,7 +425,12 @@ func TestTxSearchWithCancelation(t *testing.T) { } func TestTxSearchDeprecatedIndexing(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) // index tx using events indexing (composite key) txResult1 := txResultWithEvents([]abci.Event{ @@ -445,7 +509,12 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { } func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { - indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndex(db.NewMemDB()) + indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + TxIndex: &config.TxIndexConfig{ + Indexer: "kv_lite", + }, + }) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, @@ -626,6 +695,11 @@ func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { func TestTxSearchMultipleTxs(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) + // indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ + // TxIndex: &config.TxIndexConfig{ + // Indexer: "kv_lite", + // }, + // }) // indexed first, but bigger height (to test the order of transactions) txResult := txResultWithEvents([]abci.Event{ From 0b88b702c24e873ccf26be4d22211a38ea1e9d63 Mon Sep 17 00:00:00 2001 From: Nina Barbakadze Date: Mon, 5 Feb 2024 12:44:45 +0100 Subject: [PATCH 5/6] refactor: tests --- state/txindex/kv/kv.go | 1 - state/txindex/kv/kv_lite_test.go | 40 ++++----- state/txindex/kv/kv_test.go | 139 ++----------------------------- 3 files changed, 26 insertions(+), 154 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 69b10bebe7..12de4d0b35 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -137,7 +137,6 @@ func (txi *TxIndex) Index(result *abci.TxResult) error { // in the kv_lite indexer result.Tx = []byte{} } - fmt.Println("txResult is nil after indexing", result.Tx == nil) rawBytes, err := proto.Marshal(result) if err != nil { return err diff --git a/state/txindex/kv/kv_lite_test.go b/state/txindex/kv/kv_lite_test.go index 4a8a1987b1..ab43b01084 100644 --- a/state/txindex/kv/kv_lite_test.go +++ b/state/txindex/kv/kv_lite_test.go @@ -22,6 +22,7 @@ import ( ) func TestTxIndex_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -50,7 +51,7 @@ func TestTxIndex_kv_lite(t *testing.T) { loadedTxResult, err := indexer.Get(hash) require.NoError(t, err) assert.True(t, proto.Equal(txResult, loadedTxResult)) - // --------------------- check tx is empty ----------------------- + // --------------------- check TX field is empty ----------------------- assert.True(t, bytes.Equal(txResult.Tx, []byte{})) tx2 := types.Tx("BYE BYE WORLD") @@ -71,18 +72,18 @@ func TestTxIndex_kv_lite(t *testing.T) { loadedTxResult2, err := indexer.Get(hash2) require.NoError(t, err) assert.True(t, proto.Equal(txResult2, loadedTxResult2)) - // --------------------- check tx is empty ----------------------- + // --------------------- check TX field is empty ----------------------- assert.True(t, bytes.Equal(loadedTxResult2.Tx, []byte{})) } func TestWrappedTxIndex_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", }, }) - tx := types.Tx("HELLO WORLD") wrappedTx, err := types.MarshalIndexWrapper(tx, 11) require.NoError(t, err) @@ -107,12 +108,12 @@ func TestWrappedTxIndex_kv_lite(t *testing.T) { loadedTxResult, err := indexer.Get(hash) require.NoError(t, err) assert.True(t, proto.Equal(txResult, loadedTxResult)) - // --------------------- check tx is empty ----------------------- - fmt.Println(loadedTxResult.Tx, "LOADED TX RESULT") + // --------------------- check TX field is empty ----------------------- assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) } func TestTxSearch_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -125,7 +126,6 @@ func TestTxSearch_kv_lite(t *testing.T) { {Type: "account", Attributes: []abci.EventAttribute{{Key: "owner", Value: "/Ivan/", Index: true}, {Key: "number", Value: "10", Index: true}}}, {Type: "", Attributes: []abci.EventAttribute{{Key: "not_allowed", Value: "Vlad", Index: true}}}, }) - fmt.Println(txResult, "TXRESULt") hash := types.Tx(txResult.Tx).Hash() err := indexer.Index(txResult) @@ -210,8 +210,6 @@ func TestTxSearch_kv_lite(t *testing.T) { assert.Len(t, results, tc.resultsLength) if tc.resultsLength > 0 { for _, txr := range results { - fmt.Println(txr, "TXR") - fmt.Println(txResult, "TXRESULT") assert.True(t, proto.Equal(txResult, txr)) } } @@ -220,6 +218,7 @@ func TestTxSearch_kv_lite(t *testing.T) { } func TestTxSearchEventMatch_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -325,6 +324,7 @@ func TestTxSearchEventMatch_kv_lite(t *testing.T) { } func TestTxSearchWithCancelation_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -347,6 +347,7 @@ func TestTxSearchWithCancelation_kv_lite(t *testing.T) { } func TestTxSearchDeprecatedIndexing_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -430,6 +431,7 @@ func TestTxSearchDeprecatedIndexing_kv_lite(t *testing.T) { } func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -591,7 +593,7 @@ func TestTxIndexDuplicatePreviouslySuccessful_kv_lite(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fmt.Println(tc.tx2, "TX2 BEFORE") + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -609,23 +611,23 @@ func TestTxIndexDuplicatePreviouslySuccessful_kv_lite(t *testing.T) { res, err := indexer.Get(hash) require.NoError(t, err) if tc.expOverwrite { + // --------------------- verify transaction details ----------------------- require.Equal(t, tc.tx2.Height, res.Height) require.Equal(t, tc.tx2.Index, res.Index) - // require.Equal(t, tc.tx2.Tx, res.Tx) require.Equal(t, tc.tx2.Result, res.Result) - // pointer was changed here + // during the marshall process, the Tx field is set to nil + // we can't have strict equality bytes.Equal(tc.tx2.Tx, []byte{}) - // here check that res.tx is empty bytes.Equal(res.Tx, nil) } else { + // --------------------- verify transaction details ----------------------- require.Equal(t, tc.tx1.Height, res.Height) require.Equal(t, tc.tx1.Index, res.Index) - // require.Equal(t, tc.tx2.Tx, res.Tx) require.Equal(t, tc.tx1.Result, res.Result) - // pointer was changed here + // during the marshall process, the Tx field is set to nil + // we can't have strict equality bytes.Equal(tc.tx1.Tx, []byte{}) - // here check that res.tx is empty bytes.Equal(res.Tx, nil) } }) @@ -633,6 +635,7 @@ func TestTxIndexDuplicatePreviouslySuccessful_kv_lite(t *testing.T) { } func TestTxSearchMultipleTxs_kv_lite(t *testing.T) { + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", @@ -689,11 +692,8 @@ func TestTxSearchMultipleTxs_kv_lite(t *testing.T) { require.Len(t, results, 3) - // hereeee check that all results txs are empty + // --------------------- verify all TX fields are empty ----------------------- for _, txr := range results { - fmt.Println(txr.Tx, "TXR TX HERE") - fmt.Println(txr, "TXRRR") - fmt.Println(txr.Tx == nil, "RES TX NIL") assert.True(t, bytes.Equal(txr.Tx, []byte{})) } } @@ -705,7 +705,7 @@ func benchmarkTxIndex_kv_lite(txsCount int64, b *testing.B) { store, err := db.NewDB("tx_index", "goleveldb", dir) require.NoError(b, err) - // changed here + // --------------------- kv_lite indexer ----------------------- indexer := NewTxIndexWithConfig(store, &config.Config{ TxIndex: &config.TxIndexConfig{ Indexer: "kv_lite", diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index b93c9d266e..6311eaca41 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -1,7 +1,6 @@ package kv import ( - "bytes" "context" "fmt" "os" @@ -14,7 +13,6 @@ import ( db "github.com/cometbft/cometbft-db" abci "github.com/cometbft/cometbft/abci/types" - "github.com/cometbft/cometbft/config" "github.com/cometbft/cometbft/libs/pubsub/query" cmtrand "github.com/cometbft/cometbft/libs/rand" "github.com/cometbft/cometbft/state/txindex" @@ -22,12 +20,7 @@ import ( ) func TestTxIndex(t *testing.T) { - // indexer := NewTxIndex(db.NewMemDB()) - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) + indexer := NewTxIndex(db.NewMemDB()) tx := types.Tx("HELLO WORLD") txResult := &abci.TxResult{ @@ -72,91 +65,6 @@ func TestTxIndex(t *testing.T) { assert.True(t, proto.Equal(txResult2, loadedTxResult2)) } -func TestKvLiteTxIndex(t *testing.T) { - // create kv_lite indexer - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) - - tx := types.Tx("TX SHOULD BE EMPTY AFTER INDEXING") - txResult := &abci.TxResult{ - Height: 1, - Index: 0, - Tx: tx, - Result: abci.ResponseDeliverTx{ - Data: []byte{0}, - Code: abci.CodeTypeOK, Log: "", Events: nil, - }, - } - hash := tx.Hash() - - batch := txindex.NewBatch(1) - if err := batch.Add(txResult); err != nil { - t.Error(err) - } - err := indexer.AddBatch(batch) - require.NoError(t, err) - - loadedTxResult, err := indexer.Get(hash) - require.NoError(t, err) - assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) - - tx2 := types.Tx("This should be empty after indexing too") - txResult2 := &abci.TxResult{ - Height: 1, - Index: 0, - Tx: tx2, - Result: abci.ResponseDeliverTx{ - Data: []byte{0}, - Code: abci.CodeTypeOK, Log: "", Events: nil, - }, - } - hash2 := tx2.Hash() - - err = indexer.Index(txResult2) - require.NoError(t, err) - - loadedTxResult2, err := indexer.Get(hash2) - require.NoError(t, err) - assert.True(t, bytes.Equal(loadedTxResult2.Tx, []byte{})) - -} - -// func TestWrappedTxIndex_kv_lite(t *testing.T) { -// indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ -// TxIndex: &config.TxIndexConfig{ -// Indexer: "kv_lite", -// }, -// }) - -// tx := types.Tx("HELLO WORLD") -// wrappedTx, err := types.MarshalIndexWrapper(tx, 11) -// require.NoError(t, err) -// txResult := &abci.TxResult{ -// Height: 1, -// Index: 0, -// Tx: wrappedTx, -// Result: abci.ResponseDeliverTx{ -// Data: []byte{0}, -// Code: abci.CodeTypeOK, Log: "", Events: nil, -// }, -// } -// hash := tx.Hash() - -// batch := txindex.NewBatch(1) -// if err := batch.Add(txResult); err != nil { -// t.Error(err) -// } -// err = indexer.AddBatch(batch) -// require.NoError(t, err) - -// loadedTxResult, err := indexer.Get(hash) -// require.NoError(t, err) -// assert.True(t, bytes.Equal(loadedTxResult.Tx, []byte{})) -// } - func TestWrappedTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) @@ -187,12 +95,7 @@ func TestWrappedTxIndex(t *testing.T) { } func TestTxSearch(t *testing.T) { - // indexer := NewTxIndex(db.NewMemDB()) - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) + indexer := NewTxIndex(db.NewMemDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, @@ -200,7 +103,6 @@ func TestTxSearch(t *testing.T) { {Type: "account", Attributes: []abci.EventAttribute{{Key: "owner", Value: "/Ivan/", Index: true}, {Key: "number", Value: "10", Index: true}}}, {Type: "", Attributes: []abci.EventAttribute{{Key: "not_allowed", Value: "Vlad", Index: true}}}, }) - fmt.Println(txResult, "TXRESULt") hash := types.Tx(txResult.Tx).Hash() err := indexer.Index(txResult) @@ -279,14 +181,11 @@ func TestTxSearch(t *testing.T) { tc := tc t.Run(tc.q, func(t *testing.T) { results, err := indexer.Search(ctx, query.MustParse(tc.q)) - assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) if tc.resultsLength > 0 { for _, txr := range results { - fmt.Println(txr, "TXR") - fmt.Println(txResult, "TXRESULT") assert.True(t, proto.Equal(txResult, txr)) } } @@ -296,13 +195,7 @@ func TestTxSearch(t *testing.T) { func TestTxSearchEventMatch(t *testing.T) { - // indexer := NewTxIndex(db.NewMemDB()) - - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) + indexer := NewTxIndex(db.NewMemDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}, {Key: "owner", Value: "Ana", Index: true}}}, @@ -402,12 +295,7 @@ func TestTxSearchEventMatch(t *testing.T) { } } func TestTxSearchWithCancelation(t *testing.T) { - // indexer := NewTxIndex(db.NewMemDB()) - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) + indexer := NewTxIndex(db.NewMemDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, @@ -425,12 +313,7 @@ func TestTxSearchWithCancelation(t *testing.T) { } func TestTxSearchDeprecatedIndexing(t *testing.T) { - // indexer := NewTxIndex(db.NewMemDB()) - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) + indexer := NewTxIndex(db.NewMemDB()) // index tx using events indexing (composite key) txResult1 := txResultWithEvents([]abci.Event{ @@ -509,12 +392,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { } func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { - // indexer := NewTxIndex(db.NewMemDB()) - indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - TxIndex: &config.TxIndexConfig{ - Indexer: "kv_lite", - }, - }) + indexer := NewTxIndex(db.NewMemDB()) txResult := txResultWithEvents([]abci.Event{ {Type: "account", Attributes: []abci.EventAttribute{{Key: "number", Value: "1", Index: true}}}, @@ -695,11 +573,6 @@ func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { func TestTxSearchMultipleTxs(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) - // indexer := NewTxIndexWithConfig(db.NewMemDB(), &config.Config{ - // TxIndex: &config.TxIndexConfig{ - // Indexer: "kv_lite", - // }, - // }) // indexed first, but bigger height (to test the order of transactions) txResult := txResultWithEvents([]abci.Event{ From bedac34583250e7b626bc875f917324891cea631 Mon Sep 17 00:00:00 2001 From: Nina Barbakadze Date: Mon, 5 Feb 2024 13:29:11 +0100 Subject: [PATCH 6/6] style: lint --- state/txindex/kv/kv.go | 2 +- state/txindex/kv/kv_lite_test.go | 14 ++++++++------ state/txindex/kv/kv_test.go | 2 ++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 12de4d0b35..18ac18ae7c 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -68,7 +68,7 @@ func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) { if err != nil { return nil, fmt.Errorf("error reading TxResult: %v", err) } - + fmt.Println("txResult is nil", txResult.Tx == nil) return txResult, nil } diff --git a/state/txindex/kv/kv_lite_test.go b/state/txindex/kv/kv_lite_test.go index ab43b01084..67b028e627 100644 --- a/state/txindex/kv/kv_lite_test.go +++ b/state/txindex/kv/kv_lite_test.go @@ -235,6 +235,7 @@ func TestTxSearchEventMatch_kv_lite(t *testing.T) { err := indexer.Index(txResult) require.NoError(t, err) + //nolint:dupl testCases := map[string]struct { q string resultsLength int @@ -521,6 +522,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues_kv_lite(t *testing. func TestTxIndexDuplicatePreviouslySuccessful_kv_lite(t *testing.T) { mockTx := types.Tx("MOCK_TX_HASH") + //nolint:dupl testCases := []struct { name string tx1 *abci.TxResult @@ -698,7 +700,7 @@ func TestTxSearchMultipleTxs_kv_lite(t *testing.T) { } } -func benchmarkTxIndex_kv_lite(txsCount int64, b *testing.B) { +func benchmarkTxIndexKvLite(txsCount int64, b *testing.B) { dir, err := os.MkdirTemp("", "tx_index_db") require.NoError(b, err) defer os.RemoveAll(dir) @@ -743,8 +745,8 @@ func benchmarkTxIndex_kv_lite(txsCount int64, b *testing.B) { } } -func BenchmarkTxIndex1_kv_lite(b *testing.B) { benchmarkTxIndex(1, b) } -func BenchmarkTxIndex500_kv_lite(b *testing.B) { benchmarkTxIndex(500, b) } -func BenchmarkTxIndex1000_kv_lite(b *testing.B) { benchmarkTxIndex(1000, b) } -func BenchmarkTxIndex2000_kv_lite(b *testing.B) { benchmarkTxIndex(2000, b) } -func BenchmarkTxIndex10000_kv_lite(b *testing.B) { benchmarkTxIndex(10000, b) } +func BenchmarkTxIndex1_kv_lite(b *testing.B) { benchmarkTxIndexKvLite(1, b) } +func BenchmarkTxIndex500_kv_lite(b *testing.B) { benchmarkTxIndexKvLite(500, b) } +func BenchmarkTxIndex1000_kv_lite(b *testing.B) { benchmarkTxIndexKvLite(1000, b) } +func BenchmarkTxIndex2000_kv_lite(b *testing.B) { benchmarkTxIndexKvLite(2000, b) } +func BenchmarkTxIndex10000_kv_lite(b *testing.B) { benchmarkTxIndexKvLite(10000, b) } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 6311eaca41..9f9341a981 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -207,6 +207,7 @@ func TestTxSearchEventMatch(t *testing.T) { err := indexer.Index(txResult) require.NoError(t, err) + //nolint:dupl testCases := map[string]struct { q string resultsLength int @@ -477,6 +478,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { mockTx := types.Tx("MOCK_TX_HASH") + //nolint:dupl testCases := []struct { name string tx1 *abci.TxResult