From 1bd62229a2010abd48182c96d4cb3f1d7d58ce29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 4 Mar 2024 16:51:43 +0100 Subject: [PATCH 1/6] store incoming transactions in a database --- integration/obscurogateway/tengateway_test.go | 1 + tools/walletextension/config/config.go | 1 + tools/walletextension/main/cli.go | 6 +++++ .../mariadb/002_store_incoming_txs.sql | 10 +++++++ .../storage/database/mariadb/mariadb.go | 15 +++++++++++ .../storage/database/sqlite/sqlite.go | 27 +++++++++++++++++++ tools/walletextension/storage/storage.go | 1 + tools/walletextension/storage/storage_test.go | 11 ++++++++ tools/walletextension/wallet_extension.go | 14 ++++++++++ 9 files changed, 86 insertions(+) create mode 100644 tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql diff --git a/integration/obscurogateway/tengateway_test.go b/integration/obscurogateway/tengateway_test.go index a06e199289..380e598be7 100644 --- a/integration/obscurogateway/tengateway_test.go +++ b/integration/obscurogateway/tengateway_test.go @@ -67,6 +67,7 @@ func TestTenGateway(t *testing.T) { VerboseFlag: false, DBType: "sqlite", TenChainID: 443, + StoreIncomingTxs: true, } tenGwContainer := container.NewWalletExtensionContainerFromConfig(tenGatewayConf, testlog.Logger()) diff --git a/tools/walletextension/config/config.go b/tools/walletextension/config/config.go index 0914a93fe4..9e423ca476 100644 --- a/tools/walletextension/config/config.go +++ b/tools/walletextension/config/config.go @@ -13,4 +13,5 @@ type Config struct { DBType string DBConnectionURL string TenChainID int + StoreIncomingTxs bool } diff --git a/tools/walletextension/main/cli.go b/tools/walletextension/main/cli.go index e69ae73708..b84b854d90 100644 --- a/tools/walletextension/main/cli.go +++ b/tools/walletextension/main/cli.go @@ -55,6 +55,10 @@ const ( tenChainIDName = "tenChainID" tenChainIDDefault = 443 tenChainIDFlagUsage = "ChainID of Ten network that the gateway is communicating with" + + storeIncomingTxs = "storeIncomingTxs" + storeIncomingTxsDefault = true + storeIncomingTxsUsage = "Flag to enable storing incoming transactions in the database for debugging purposes. Default: true" ) func parseCLIArgs() config.Config { @@ -70,6 +74,7 @@ func parseCLIArgs() config.Config { dbType := flag.String(dbTypeFlagName, dbTypeFlagDefault, dbTypeFlagUsage) dbConnectionURL := flag.String(dbConnectionURLFlagName, dbConnectionURLFlagDefault, dbConnectionURLFlagUsage) tenChainID := flag.Int(tenChainIDName, tenChainIDDefault, tenChainIDFlagUsage) + storeIncomingTransactions := flag.Bool(storeIncomingTxs, storeIncomingTxsDefault, storeIncomingTxsUsage) flag.Parse() return config.Config{ @@ -84,5 +89,6 @@ func parseCLIArgs() config.Config { DBType: *dbType, DBConnectionURL: *dbConnectionURL, TenChainID: *tenChainID, + StoreIncomingTxs: *storeIncomingTransactions, } } diff --git a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql new file mode 100644 index 0000000000..abfd6e9f7d --- /dev/null +++ b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql @@ -0,0 +1,10 @@ +/* + This is a migration file for MariaDB that creates transactions table for storing incoming transactions +*/ + +CREATE TABLE IF NOT EXISTS ogdb.transactions ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id varbinary(20), + tx TEXT + tx_time DATETIME DEFAULT CURRENT_TIMESTAMP +); \ No newline at end of file diff --git a/tools/walletextension/storage/database/mariadb/mariadb.go b/tools/walletextension/storage/database/mariadb/mariadb.go index e68cc009a2..aff477769a 100644 --- a/tools/walletextension/storage/database/mariadb/mariadb.go +++ b/tools/walletextension/storage/database/mariadb/mariadb.go @@ -142,3 +142,18 @@ func (m *MariaDB) GetAllUsers() ([]common.UserDB, error) { return users, nil } + +func (m *MariaDB) StoreTransaction(rawTx string, userID []byte) error { + stmt, err := m.db.Prepare("INSERT INTO transactions(user_id, tx) VALUES (?, ?)") + if err != nil { + return err + } + defer stmt.Close() + + _, err = stmt.Exec(userID, rawTx) + if err != nil { + return err + } + + return nil +} diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index 5ef0d92b5f..da2da43127 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -58,6 +58,18 @@ func NewSqliteDatabase(dbPath string) (*Database, error) { return nil, err } + // create transactions table + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS ogdb.transactions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id binary(20), + tx TEXT, + tx_time TEXT DEFAULT (datetime('now')) +) ;`) + + if err != nil { + return nil, err + } + return &Database{db: db}, nil } @@ -187,3 +199,18 @@ func createOrLoad(dbPath string) (string, error) { return dbPath, nil } + +func (s *Database) StoreTransaction(rawTx string, userID []byte) error { + stmt, err := s.db.Prepare("INSERT INTO transactions(user_id, tx) VALUES (?, ?)") + if err != nil { + return err + } + defer stmt.Close() + + _, err = stmt.Exec(userID, rawTx) + if err != nil { + return err + } + + return nil +} diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index 57e751b02b..5cbc969ae1 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -16,6 +16,7 @@ type Storage interface { AddAccount(userID []byte, accountAddress []byte, signature []byte) error GetAccounts(userID []byte) ([]common.AccountDB, error) GetAllUsers() ([]common.UserDB, error) + StoreTransaction(rawTx string, userID []byte) error } func New(dbType string, dbConnectionURL, dbPath string) (Storage, error) { diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index fda81cdc8c..fc71dd41fd 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -14,6 +14,7 @@ var tests = map[string]func(storage Storage, t *testing.T){ "testAddAndGetAccounts": testAddAndGetAccounts, "testDeleteUser": testDeleteUser, "testGetAllUsers": testGetAllUsers, + "testStoringNewTx": testStoringNewTx, } func TestSQLiteGatewayDB(t *testing.T) { @@ -144,3 +145,13 @@ func testGetAllUsers(storage Storage, t *testing.T) { t.Errorf("Expected user count to increase by 1. Got %d initially and %d after insert", len(initialUsers), len(afterInsertUsers)) } } + +func testStoringNewTx(storage Storage, t *testing.T) { + userID := []byte("userID") + rawTransaction := "0x0123456789" + + err := storage.StoreTransaction(rawTransaction, userID) + if err != nil { + t.Fatal(err) + } +} diff --git a/tools/walletextension/wallet_extension.go b/tools/walletextension/wallet_extension.go index 242b60ea5a..e1ccc1fc34 100644 --- a/tools/walletextension/wallet_extension.go +++ b/tools/walletextension/wallet_extension.go @@ -142,6 +142,20 @@ func (w *WalletExtension) ProxyEthRequest(request *common.RPCRequest, conn userc } } + // check if user is sending a new transaction and if we should store it in the database for debugging purposes + if request.Method == rpc.SendRawTransaction && w.config.StoreIncomingTxs { + userIDBytes, err := common.GetUserIDbyte(hexUserID) + if err != nil { + w.Logger().Error(fmt.Errorf("error decoding string (%s), %w", hexUserID[2:], err).Error()) + return nil, errors.New("error decoding userID. It should be in hex format") + } + err = w.storage.StoreTransaction(request.Params[0].(string), userIDBytes) + if err != nil { + w.Logger().Error(fmt.Errorf("error storing transaction in the database: %w", err).Error()) + return nil, err + } + } + // get account manager for current user (if there is no users in the query parameters - use defaultUser for WE endpoints) selectedAccountManager, err := w.userAccountManager.GetUserAccountManager(hexUserID) if err != nil { From 2daca9d8b1dfd1c56a549ad920c210887bb4ecb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 4 Mar 2024 17:06:26 +0100 Subject: [PATCH 2/6] fix --- tools/walletextension/storage/database/sqlite/sqlite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index da2da43127..a77dcd1d3a 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -59,7 +59,7 @@ func NewSqliteDatabase(dbPath string) (*Database, error) { } // create transactions table - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS ogdb.transactions ( + _, err = db.Exec(`CREATE TABLE IF NOT EXISTS transactions ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id binary(20), tx TEXT, From 856c1f38ac9ccf0ba2a413174142355df4412071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 5 Mar 2024 08:44:15 +0100 Subject: [PATCH 3/6] fix --- .../storage/database/mariadb/002_store_incoming_txs.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql index abfd6e9f7d..92515d58f0 100644 --- a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql +++ b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql @@ -5,6 +5,6 @@ CREATE TABLE IF NOT EXISTS ogdb.transactions ( id INT AUTO_INCREMENT PRIMARY KEY, user_id varbinary(20), - tx TEXT + tx TEXT, tx_time DATETIME DEFAULT CURRENT_TIMESTAMP ); \ No newline at end of file From d9bc42e1e0ecfab3265035491744c5571f28cab5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 5 Mar 2024 14:31:27 +0100 Subject: [PATCH 4/6] add tx_hash --- .../database/mariadb/002_store_incoming_txs.sql | 1 + .../storage/database/mariadb/mariadb.go | 13 +++++++++++-- .../storage/database/sqlite/sqlite.go | 14 ++++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql index 92515d58f0..c5dbc2af0e 100644 --- a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql +++ b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql @@ -5,6 +5,7 @@ CREATE TABLE IF NOT EXISTS ogdb.transactions ( id INT AUTO_INCREMENT PRIMARY KEY, user_id varbinary(20), + tx_hash TEXT, tx TEXT, tx_time DATETIME DEFAULT CURRENT_TIMESTAMP ); \ No newline at end of file diff --git a/tools/walletextension/storage/database/mariadb/mariadb.go b/tools/walletextension/storage/database/mariadb/mariadb.go index aff477769a..526dfd419a 100644 --- a/tools/walletextension/storage/database/mariadb/mariadb.go +++ b/tools/walletextension/storage/database/mariadb/mariadb.go @@ -2,7 +2,9 @@ package mariadb import ( "database/sql" + "encoding/hex" "fmt" + "github.com/ethereum/go-ethereum/crypto" "path/filepath" "runtime" @@ -144,13 +146,20 @@ func (m *MariaDB) GetAllUsers() ([]common.UserDB, error) { } func (m *MariaDB) StoreTransaction(rawTx string, userID []byte) error { - stmt, err := m.db.Prepare("INSERT INTO transactions(user_id, tx) VALUES (?, ?)") + stmt, err := m.db.Prepare("INSERT INTO transactions(user_id, tx_hash, tx) VALUES (?, ?, ?)") if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(userID, rawTx) + // Calculate tx hash + rawTxBytes, err := hex.DecodeString(rawTx[2:]) + if err != nil { + panic(err) + } + txHash := crypto.Keccak256Hash(rawTxBytes) + + _, err = stmt.Exec(userID, txHash, rawTx) if err != nil { return err } diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index a77dcd1d3a..d711e57f13 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -2,7 +2,9 @@ package sqlite import ( "database/sql" + "encoding/hex" "fmt" + "github.com/ethereum/go-ethereum/crypto" "os" "path/filepath" @@ -62,6 +64,7 @@ func NewSqliteDatabase(dbPath string) (*Database, error) { _, err = db.Exec(`CREATE TABLE IF NOT EXISTS transactions ( id INTEGER PRIMARY KEY AUTOINCREMENT, user_id binary(20), + tx_hash TEXT, tx TEXT, tx_time TEXT DEFAULT (datetime('now')) ) ;`) @@ -201,13 +204,20 @@ func createOrLoad(dbPath string) (string, error) { } func (s *Database) StoreTransaction(rawTx string, userID []byte) error { - stmt, err := s.db.Prepare("INSERT INTO transactions(user_id, tx) VALUES (?, ?)") + stmt, err := s.db.Prepare("INSERT INTO transactions(user_id, tx_hash, tx) VALUES (?, ?, ?)") if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(userID, rawTx) + // Calculate tx hash + rawTxBytes, err := hex.DecodeString(rawTx[2:]) + if err != nil { + panic(err) + } + txHash := crypto.Keccak256Hash(rawTxBytes) + + _, err = stmt.Exec(userID, txHash, rawTx) if err != nil { return err } From 371719de97794261e7a91aeecade5d8277c68559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 5 Mar 2024 17:10:18 +0100 Subject: [PATCH 5/6] fix lint --- tools/walletextension/api/routes.go | 5 +++++ .../walletextension/storage/database/mariadb/mariadb.go | 9 +++++++-- tools/walletextension/storage/database/sqlite/sqlite.go | 9 +++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tools/walletextension/api/routes.go b/tools/walletextension/api/routes.go index 44fb2f7bd2..eabb7e1845 100644 --- a/tools/walletextension/api/routes.go +++ b/tools/walletextension/api/routes.go @@ -171,6 +171,11 @@ func ethRequestHandler(walletExt *walletextension.WalletExtension, conn userconn hexUserID = hex.EncodeToString([]byte(common.DefaultUser)) // todo (@ziga) - this can be removed once old WE endpoints are removed } + if len(hexUserID) < 3 { + handleError(conn, walletExt.Logger(), fmt.Errorf("encryption token length is incorrect")) + return + } + // todo (@pedro) remove this conn dependency response, err := walletExt.ProxyEthRequest(request, conn, hexUserID) if err != nil { diff --git a/tools/walletextension/storage/database/mariadb/mariadb.go b/tools/walletextension/storage/database/mariadb/mariadb.go index 526dfd419a..4af4290693 100644 --- a/tools/walletextension/storage/database/mariadb/mariadb.go +++ b/tools/walletextension/storage/database/mariadb/mariadb.go @@ -4,10 +4,11 @@ import ( "database/sql" "encoding/hex" "fmt" - "github.com/ethereum/go-ethereum/crypto" "path/filepath" "runtime" + "github.com/ethereum/go-ethereum/crypto" + _ "github.com/go-sql-driver/mysql" // Importing MariaDB driver "github.com/ten-protocol/go-ten/go/common/errutil" "github.com/ten-protocol/go-ten/tools/walletextension/common" @@ -153,9 +154,13 @@ func (m *MariaDB) StoreTransaction(rawTx string, userID []byte) error { defer stmt.Close() // Calculate tx hash + if len(rawTx) < 3 { + fmt.Println("Invalid rawTx: ", rawTx) + return nil + } rawTxBytes, err := hex.DecodeString(rawTx[2:]) if err != nil { - panic(err) + fmt.Println("Error decoding rawTx: ", rawTx) } txHash := crypto.Keccak256Hash(rawTxBytes) diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index d711e57f13..68e504072f 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -4,10 +4,11 @@ import ( "database/sql" "encoding/hex" "fmt" - "github.com/ethereum/go-ethereum/crypto" "os" "path/filepath" + "github.com/ethereum/go-ethereum/crypto" + _ "github.com/mattn/go-sqlite3" // sqlite driver for sql.Open() obscurocommon "github.com/ten-protocol/go-ten/go/common" "github.com/ten-protocol/go-ten/go/common/errutil" @@ -211,9 +212,13 @@ func (s *Database) StoreTransaction(rawTx string, userID []byte) error { defer stmt.Close() // Calculate tx hash + if len(rawTx) < 3 { + fmt.Println("Invalid rawTx: ", rawTx) + return nil + } rawTxBytes, err := hex.DecodeString(rawTx[2:]) if err != nil { - panic(err) + fmt.Println("Error decoding rawTx: ", rawTx) } txHash := crypto.Keccak256Hash(rawTxBytes) From 476f29877b8c3930eed2a415db5a4bc683b7a29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 5 Mar 2024 17:36:43 +0100 Subject: [PATCH 6/6] refactor --- .../storage/database/mariadb/mariadb.go | 18 +++++++++++------- .../storage/database/sqlite/sqlite.go | 17 ++++++++++------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tools/walletextension/storage/database/mariadb/mariadb.go b/tools/walletextension/storage/database/mariadb/mariadb.go index 4af4290693..23030f168f 100644 --- a/tools/walletextension/storage/database/mariadb/mariadb.go +++ b/tools/walletextension/storage/database/mariadb/mariadb.go @@ -153,16 +153,20 @@ func (m *MariaDB) StoreTransaction(rawTx string, userID []byte) error { } defer stmt.Close() - // Calculate tx hash + // Validate rawTx length and get the txHash + txHash := "" if len(rawTx) < 3 { fmt.Println("Invalid rawTx: ", rawTx) - return nil - } - rawTxBytes, err := hex.DecodeString(rawTx[2:]) - if err != nil { - fmt.Println("Error decoding rawTx: ", rawTx) + } else { + // Decode the hex string to bytes, excluding the '0x' prefix + rawTxBytes, err := hex.DecodeString(rawTx[2:]) + if err != nil { + fmt.Println("Error decoding rawTx: ", err) + } else { + // Compute Keccak-256 hash + txHash = crypto.Keccak256Hash(rawTxBytes).Hex() + } } - txHash := crypto.Keccak256Hash(rawTxBytes) _, err = stmt.Exec(userID, txHash, rawTx) if err != nil { diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index 68e504072f..e1ee97d3fb 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -211,16 +211,19 @@ func (s *Database) StoreTransaction(rawTx string, userID []byte) error { } defer stmt.Close() - // Calculate tx hash + txHash := "" if len(rawTx) < 3 { fmt.Println("Invalid rawTx: ", rawTx) - return nil - } - rawTxBytes, err := hex.DecodeString(rawTx[2:]) - if err != nil { - fmt.Println("Error decoding rawTx: ", rawTx) + } else { + // Decode the hex string to bytes, excluding the '0x' prefix + rawTxBytes, err := hex.DecodeString(rawTx[2:]) + if err != nil { + fmt.Println("Error decoding rawTx: ", err) + } else { + // Compute Keccak-256 hash + txHash = crypto.Keccak256Hash(rawTxBytes).Hex() + } } - txHash := crypto.Keccak256Hash(rawTxBytes) _, err = stmt.Exec(userID, txHash, rawTx) if err != nil {