From a6ca87112f504054d3295b51107040bb642a536f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 22 Oct 2024 13:08:28 +0200 Subject: [PATCH 01/18] use cosmosDB as key value storage --- go.mod | 4 + go.sum | 8 + .../walletextension/rpcapi/transaction_api.go | 22 -- .../storage/database/cosmosdb/cosmosdb.go | 242 ++++++++++++++++++ .../storage/database/mariadb/001_init.sql | 15 -- .../mariadb/002_store_incoming_txs.sql | 11 - .../mariadb/003_add_signature_type.sql | 1 - .../storage/database/mariadb/mariadb.go | 179 ------------- tools/walletextension/storage/storage.go | 17 +- tools/walletextension/storage/storage_test.go | 116 ++++++--- 10 files changed, 339 insertions(+), 276 deletions(-) create mode 100644 tools/walletextension/storage/database/cosmosdb/cosmosdb.go delete mode 100644 tools/walletextension/storage/database/mariadb/001_init.sql delete mode 100644 tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql delete mode 100644 tools/walletextension/storage/database/mariadb/003_add_signature_type.sql delete mode 100644 tools/walletextension/storage/database/mariadb/mariadb.go diff --git a/go.mod b/go.mod index e7c7715b16..d97a2e34d8 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,10 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/VictoriaMetrics/fastcache v1.12.2 // indirect diff --git a/go.sum b/go.sum index 552abd901c..ae9bc01dd4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,13 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0 h1:c726lgbwpwFBuj+Fyrwuh/vUilqFo+hUAOUNjsKj5DI= +github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0/go.mod h1:WzFGxuepAtZIZtQbz8/WviJycLMKJHpaEAqcXONxlag= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= diff --git a/tools/walletextension/rpcapi/transaction_api.go b/tools/walletextension/rpcapi/transaction_api.go index a0a8baedd4..ffa978145e 100644 --- a/tools/walletextension/rpcapi/transaction_api.go +++ b/tools/walletextension/rpcapi/transaction_api.go @@ -2,8 +2,6 @@ package rpcapi import ( "context" - "encoding/json" - "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -105,19 +103,6 @@ func (s *TransactionAPI) SendTransaction(ctx context.Context, args gethapi.Trans if err != nil { return common.Hash{}, err } - userIDBytes, _ := extractUserID(ctx, s.we) - if s.we.Config.StoreIncomingTxs && len(userIDBytes) > 10 { - tx, err := json.Marshal(args) - if err != nil { - s.we.Logger().Error("error marshalling transaction: %s", err) - return *txRec, nil - } - err = s.we.Storage.StoreTransaction(string(tx), userIDBytes) - if err != nil { - s.we.Logger().Error("error storing transaction in the database: %s", err) - return *txRec, nil - } - } return *txRec, err } @@ -135,13 +120,6 @@ func (s *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil.B if err != nil { return common.Hash{}, err } - userIDBytes, err := extractUserID(ctx, s.we) - if s.we.Config.StoreIncomingTxs && len(userIDBytes) > 10 { - err = s.we.Storage.StoreTransaction(input.String(), userIDBytes) - if err != nil { - s.we.Logger().Error(fmt.Errorf("error storing transaction in the database: %w", err).Error()) - } - } return *txRec, err } diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go new file mode 100644 index 0000000000..3d0cc71841 --- /dev/null +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -0,0 +1,242 @@ +package cosmosdb + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" + "github.com/ten-protocol/go-ten/go/common/errutil" + "github.com/ten-protocol/go-ten/go/common/viewingkey" + "github.com/ten-protocol/go-ten/tools/walletextension/common" +) + +/* +This is a CosmosDB implementation of the Storage interface. + +We need to make sure we have a CosmosDB account and a database created before using this. + +Quick summary of the CosmosDB setup: +- Create a CosmosDB account (Azure Cosmos DB for NoSQL) +- account name should follow the format: -gateway-cosmosdb +- use Serverless capacity mode for testnets +- go to "Data Explorer" in the CosmosDB account and create new database named "gatewayDB" +- inside the database create a container named "users" with partition key of "/id" +- to get your connection string go to settings -> keys -> primary connection string + +*/ + +type CosmosDB struct { + client *azcosmos.Client + usersContainer *azcosmos.ContainerClient +} + +const ( + DATABASE_NAME = "gatewayDB" + USERS_CONTAINER_NAME = "users" + PARTITION_KEY = "/id" +) + +// GWUser is the user struct for the gateway +// both ID and UserID are the same for now, but we will use different values with encryption +type GWUser struct { + ID string `json:"id"` // Required by CosmosDB + UserId []byte `json:"userId"` + PrivateKey []byte `json:"privateKey"` + Accounts []GWAccount `json:"accounts"` // List of Accounts +} + +type GWAccount struct { + AccountAddress []byte `json:"accountAddress"` + Signature []byte `json:"signature"` + SignatureType int `json:"signatureType"` +} + +func NewCosmosDB(connectionString string) (*CosmosDB, error) { + client, err := azcosmos.NewClientFromConnectionString(connectionString, nil) + if err != nil { + return nil, err + } + + // Create database if it doesn't exist + ctx := context.Background() + _, err = client.CreateDatabase(ctx, azcosmos.DatabaseProperties{ID: DATABASE_NAME}, nil) + if err != nil && !strings.Contains(err.Error(), "Conflict") { + return nil, fmt.Errorf("failed to create database: %w", err) + } + + // Create container client for users container + usersContainer, err := client.NewContainer(DATABASE_NAME, USERS_CONTAINER_NAME) + if err != nil { + return nil, fmt.Errorf("failed to create users container: %w", err) + } + + return &CosmosDB{ + client: client, + usersContainer: usersContainer, + }, nil +} + +func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { + user := GWUser{ + ID: hex.EncodeToString(userID), + UserId: userID, + PrivateKey: privateKey, + Accounts: []GWAccount{}, + } + userJSON, err := json.Marshal(user) + if err != nil { + return err + } + + // add to cosmosdb + partitionKey := azcosmos.NewPartitionKeyString(user.ID) + + ctx := context.Background() + _, err = c.usersContainer.CreateItem(ctx, partitionKey, userJSON, nil) + if err != nil { + return err + } + return nil +} + +func (c *CosmosDB) DeleteUser(userID []byte) error { + // Convert userID to hex string for use as partition key + userIDHex := hex.EncodeToString(userID) + partitionKey := azcosmos.NewPartitionKeyString(userIDHex) + + ctx := context.Background() + + // Delete the item from the container + _, err := c.usersContainer.DeleteItem(ctx, partitionKey, userIDHex, nil) + if err != nil { + return fmt.Errorf("failed to delete user: %w", err) + } + return nil +} + +func (c *CosmosDB) GetUserPrivateKey(userID []byte) ([]byte, error) { + // Convert userID to hex string for use as partition key + userIDHex := hex.EncodeToString(userID) + partitionKey := azcosmos.NewPartitionKeyString(userIDHex) + + ctx := context.Background() + + // Read the item from the container + itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) + if err != nil { + return nil, errutil.ErrNotFound + } + + var user GWUser + err = json.Unmarshal(itemResponse.Value, &user) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal user data: %w", err) + } + + return user.PrivateKey, nil +} + +func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { + // Convert userID to hex string for use as partition key + userIDHex := hex.EncodeToString(userID) + partitionKey := azcosmos.NewPartitionKeyString(userIDHex) + + ctx := context.Background() + + // Read the existing user + itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + + var user GWUser + err = json.Unmarshal(itemResponse.Value, &user) + if err != nil { + return fmt.Errorf("failed to unmarshal user data: %w", err) + } + + // Create new account + newAccount := GWAccount{ + AccountAddress: accountAddress, + Signature: signature, + SignatureType: int(signatureType), + } + + // Add new account to user's accounts + user.Accounts = append(user.Accounts, newAccount) + + // Marshal updated user back to JSON + updatedUserJSON, err := json.Marshal(user) + if err != nil { + return fmt.Errorf("error marshaling updated user: %w", err) + } + + // Update the item in the container + _, err = c.usersContainer.ReplaceItem(ctx, partitionKey, userIDHex, updatedUserJSON, nil) + if err != nil { + return fmt.Errorf("failed to update user with new account: %w", err) + } + return nil +} + +func (c *CosmosDB) GetAccounts(userID []byte) ([]common.AccountDB, error) { + // Convert userID to hex string for use as partition key + userIDHex := hex.EncodeToString(userID) + partitionKey := azcosmos.NewPartitionKeyString(userIDHex) + + ctx := context.Background() + + // Read the existing user + itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) + if err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + + var user GWUser + err = json.Unmarshal(itemResponse.Value, &user) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal user data: %w", err) + } + + // Convert GWAccount to common.AccountDB + accounts := make([]common.AccountDB, len(user.Accounts)) + for i, acc := range user.Accounts { + accounts[i] = common.AccountDB{ + AccountAddress: acc.AccountAddress, + Signature: acc.Signature, + SignatureType: acc.SignatureType, + } + } + + return accounts, nil +} + +func (c *CosmosDB) GetUser(userID []byte) (common.UserDB, error) { + // Convert userID to hex string for use as partition key + userIDHex := hex.EncodeToString(userID) + partitionKey := azcosmos.NewPartitionKeyString(userIDHex) + + ctx := context.Background() + + // Read the existing user + itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) + if err != nil { + return common.UserDB{}, fmt.Errorf("failed to get user: %w", err) + } + + var user GWUser + err = json.Unmarshal(itemResponse.Value, &user) + if err != nil { + return common.UserDB{}, fmt.Errorf("failed to unmarshal user data: %w", err) + } + + // TODO: @ziga - check if I can use user directly instead of GWUser/UserDB since they are mostly the same... + // Convert GWUser to common.UserDB + return common.UserDB{ + UserID: userID, + PrivateKey: user.PrivateKey, + }, nil +} diff --git a/tools/walletextension/storage/database/mariadb/001_init.sql b/tools/walletextension/storage/database/mariadb/001_init.sql deleted file mode 100644 index 95a4e61513..0000000000 --- a/tools/walletextension/storage/database/mariadb/001_init.sql +++ /dev/null @@ -1,15 +0,0 @@ -/* - This is a migration file for MariaDB and is executed when the Gateway is started to make sure the database schema is up to date. -*/ - -CREATE TABLE IF NOT EXISTS ogdb.users ( - user_id varbinary(20) PRIMARY KEY, - private_key varbinary(32) -); - -CREATE TABLE IF NOT EXISTS ogdb.accounts ( - user_id varbinary(20), - account_address varbinary(20), - signature varbinary(65), - FOREIGN KEY(user_id) REFERENCES ogdb.users(user_id) ON DELETE CASCADE -); diff --git a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql b/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql deleted file mode 100644 index c5dbc2af0e..0000000000 --- a/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql +++ /dev/null @@ -1,11 +0,0 @@ -/* - This is a migration file for MariaDB that creates transactions table for storing incoming transactions -*/ - -CREATE TABLE IF NOT EXISTS ogdb.transactions ( - id INT AUTO_INCREMENT PRIMARY KEY, - user_id varbinary(20), - tx_hash TEXT, - tx TEXT, - tx_time DATETIME DEFAULT CURRENT_TIMESTAMP -); \ No newline at end of file diff --git a/tools/walletextension/storage/database/mariadb/003_add_signature_type.sql b/tools/walletextension/storage/database/mariadb/003_add_signature_type.sql deleted file mode 100644 index f04ffb6f23..0000000000 --- a/tools/walletextension/storage/database/mariadb/003_add_signature_type.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE ogdb.accounts ADD COLUMN IF NOT EXISTS signature_type INT DEFAULT 0; \ No newline at end of file diff --git a/tools/walletextension/storage/database/mariadb/mariadb.go b/tools/walletextension/storage/database/mariadb/mariadb.go deleted file mode 100644 index a377970ba2..0000000000 --- a/tools/walletextension/storage/database/mariadb/mariadb.go +++ /dev/null @@ -1,179 +0,0 @@ -package mariadb - -import ( - "database/sql" - "encoding/hex" - "fmt" - "path/filepath" - "runtime" - - "github.com/ten-protocol/go-ten/go/common/storage" - - "github.com/ten-protocol/go-ten/go/common/viewingkey" - - "github.com/ethereum/go-ethereum/crypto" - - _ "github.com/go-sql-driver/mysql" // Importing MariaDB driver - "github.com/ten-protocol/go-ten/go/common/errutil" - "github.com/ten-protocol/go-ten/tools/walletextension/common" -) - -type MariaDB struct { - db *sql.DB -} - -// NewMariaDB creates a new MariaDB connection instance -func NewMariaDB(dbURL string) (*MariaDB, error) { - db, err := sql.Open("mysql", dbURL+"?multiStatements=true") - if err != nil { - return nil, fmt.Errorf("failed to connect to database: %w", err) - } - - // get the path to the migrations (they are always in the same directory as file containing connection function) - _, filename, _, ok := runtime.Caller(0) - if !ok { - return nil, fmt.Errorf("failed to get current directory") - } - migrationsDir := filepath.Dir(filename) - - if err = storage.ApplyMigrations(db, migrationsDir); err != nil { - return nil, err - } - - return &MariaDB{db: db}, nil -} - -func (m *MariaDB) AddUser(userID []byte, privateKey []byte) error { - stmt, err := m.db.Prepare("REPLACE INTO users(user_id, private_key) VALUES (?, ?)") - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec(userID, privateKey) - if err != nil { - return err - } - - return nil -} - -func (m *MariaDB) DeleteUser(userID []byte) error { - stmt, err := m.db.Prepare("DELETE FROM users WHERE user_id = ?") - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec(userID) - if err != nil { - return err - } - - return nil -} - -func (m *MariaDB) GetUserPrivateKey(userID []byte) ([]byte, error) { - var privateKey []byte - err := m.db.QueryRow("SELECT private_key FROM users WHERE user_id = ?", userID).Scan(&privateKey) - if err != nil { - if err == sql.ErrNoRows { - // No rows found for the given userID - return nil, errutil.ErrNotFound - } - return nil, err - } - - return privateKey, nil -} - -func (m *MariaDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { - stmt, err := m.db.Prepare("INSERT INTO accounts(user_id, account_address, signature, signature_type) VALUES (?, ?, ?, ?)") - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec(userID, accountAddress, signature, int(signatureType)) - if err != nil { - return err - } - - return nil -} - -func (m *MariaDB) GetAccounts(userID []byte) ([]common.AccountDB, error) { - rows, err := m.db.Query("SELECT account_address, signature, signature_type FROM accounts WHERE user_id = ?", userID) - if err != nil { - return nil, err - } - defer rows.Close() - - var accounts []common.AccountDB - for rows.Next() { - var account common.AccountDB - if err := rows.Scan(&account.AccountAddress, &account.Signature, &account.SignatureType); err != nil { - return nil, err - } - accounts = append(accounts, account) - } - if err := rows.Err(); err != nil { - return nil, err - } - - return accounts, nil -} - -func (m *MariaDB) GetAllUsers() ([]common.UserDB, error) { - rows, err := m.db.Query("SELECT user_id, private_key FROM users") - if err != nil { - return nil, err - } - defer rows.Close() - - var users []common.UserDB - for rows.Next() { - var user common.UserDB - err = rows.Scan(&user.UserID, &user.PrivateKey) - if err != nil { - return nil, err - } - users = append(users, user) - } - - if err = rows.Err(); err != nil { - return nil, err - } - - return users, nil -} - -func (m *MariaDB) StoreTransaction(rawTx string, userID []byte) error { - stmt, err := m.db.Prepare("INSERT INTO transactions(user_id, tx_hash, tx) VALUES (?, ?, ?)") - if err != nil { - return err - } - defer stmt.Close() - - // Validate rawTx length and get the txHash - txHash := "" - if len(rawTx) < 3 { - fmt.Println("Invalid rawTx: ", rawTx) - } else { - // Decode the hex string to bytes, excluding the '0x' prefix - rawTxBytes, err := hex.DecodeString(rawTx[2:]) - if err != nil { - fmt.Println("Error decoding rawTx: ", err) - } else { - // Compute Keccak-256 hash - txHash = crypto.Keccak256Hash(rawTxBytes).Hex() - } - } - - _, err = stmt.Exec(userID, txHash, rawTx) - if err != nil { - return err - } - - return nil -} diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index 78efcd1a39..3db0744b87 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -5,10 +5,8 @@ import ( "github.com/ten-protocol/go-ten/go/common/viewingkey" - "github.com/ten-protocol/go-ten/tools/walletextension/storage/database/mariadb" - "github.com/ten-protocol/go-ten/tools/walletextension/storage/database/sqlite" - "github.com/ten-protocol/go-ten/tools/walletextension/common" + "github.com/ten-protocol/go-ten/tools/walletextension/storage/database/cosmosdb" ) type Storage interface { @@ -17,16 +15,17 @@ type Storage interface { GetUserPrivateKey(userID []byte) ([]byte, error) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error GetAccounts(userID []byte) ([]common.AccountDB, error) - GetAllUsers() ([]common.UserDB, error) - StoreTransaction(rawTx string, userID []byte) error + GetUser(userID []byte) (common.UserDB, error) } func New(dbType string, dbConnectionURL, dbPath string) (Storage, error) { switch dbType { - case "mariaDB": - return mariadb.NewMariaDB(dbConnectionURL) - case "sqlite": - return sqlite.NewSqliteDatabase(dbPath) + // TODO: @ziga - fix sqlite for new Storage interface to allow local testing and running it locally without cosmosdb + // case "sqlite": + // return sqlite.NewSqliteDatabase(dbPath) + case "cosmosDB": + return cosmosdb.NewCosmosDB(dbConnectionURL) } + return nil, fmt.Errorf("unknown db %s", dbType) } diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index 4bdc6e1cee..af4246f813 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -2,6 +2,7 @@ package storage import ( "bytes" + "crypto/rand" "errors" "testing" @@ -15,15 +16,14 @@ var tests = map[string]func(storage Storage, t *testing.T){ "testAddAndGetUser": testAddAndGetUser, "testAddAndGetAccounts": testAddAndGetAccounts, "testDeleteUser": testDeleteUser, - "testGetAllUsers": testGetAllUsers, - "testStoringNewTx": testStoringNewTx, + "testGetUser": testGetUser, } -func TestSQLiteGatewayDB(t *testing.T) { +func TestGatewayStorage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - // storage, err := New("mariaDB", "obscurouser:password@tcp(127.0.0.1:3306)/ogdb", "") allows to run tests against a local instance of MariaDB - storage, err := New("sqlite", "", "") + // storage, err := New("sqlite", "", "") + storage, err := New("cosmosDB", "", "") require.NoError(t, err) test(storage, t) @@ -32,60 +32,87 @@ func TestSQLiteGatewayDB(t *testing.T) { } func testAddAndGetUser(storage Storage, t *testing.T) { - userID := []byte("userID") - privateKey := []byte("privateKey") + // Generate random user ID and private key + userID := make([]byte, 20) + _, err := rand.Read(userID) + if err != nil { + t.Fatal(err) + } + privateKey := make([]byte, 32) + _, err = rand.Read(privateKey) + if err != nil { + t.Fatal(err) + } - err := storage.AddUser(userID, privateKey) + // Add user to storage + err = storage.AddUser(userID, privateKey) if err != nil { t.Fatal(err) } + // Retrieve user's private key from storage returnedPrivateKey, err := storage.GetUserPrivateKey(userID) if err != nil { t.Fatal(err) } + // Check if retrieved private key matches the original if !bytes.Equal(returnedPrivateKey, privateKey) { t.Errorf("privateKey mismatch: got %v, want %v", returnedPrivateKey, privateKey) } } func testAddAndGetAccounts(storage Storage, t *testing.T) { - userID := []byte("userID") - privateKey := []byte("privateKey") - accountAddress1 := []byte("accountAddress1") - signature1 := []byte("signature1") - + // Generate random user ID, private key, and account details + userID := make([]byte, 20) + rand.Read(userID) + privateKey := make([]byte, 32) + rand.Read(privateKey) + accountAddress1 := make([]byte, 20) + rand.Read(accountAddress1) + signature1 := make([]byte, 65) + rand.Read(signature1) + + // Add a new user to the storage err := storage.AddUser(userID, privateKey) if err != nil { t.Fatal(err) } + // Add the first account for the user err = storage.AddAccount(userID, accountAddress1, signature1, viewingkey.EIP712Signature) if err != nil { t.Fatal(err) } - accountAddress2 := []byte("accountAddress2") - signature2 := []byte("signature2") + // Generate details for a second account + accountAddress2 := make([]byte, 20) + rand.Read(accountAddress2) + signature2 := make([]byte, 65) + rand.Read(signature2) + // Add the second account for the user err = storage.AddAccount(userID, accountAddress2, signature2, viewingkey.EIP712Signature) if err != nil { t.Fatal(err) } + // Retrieve all accounts for the user accounts, err := storage.GetAccounts(userID) if err != nil { t.Fatal(err) } + // Check if the correct number of accounts were retrieved if len(accounts) != 2 { t.Errorf("Expected 2 accounts, got %d", len(accounts)) } + // Flags to check if both accounts are found foundAccount1 := false foundAccount2 := false + // Iterate through retrieved accounts and check if they match the added accounts for _, account := range accounts { if bytes.Equal(account.AccountAddress, accountAddress1) && bytes.Equal(account.Signature, signature1) { foundAccount1 = true @@ -95,6 +122,7 @@ func testAddAndGetAccounts(storage Storage, t *testing.T) { } } + // Verify that both accounts were found if !foundAccount1 { t.Errorf("Account 1 was not found in the result") } @@ -105,55 +133,65 @@ func testAddAndGetAccounts(storage Storage, t *testing.T) { } func testDeleteUser(storage Storage, t *testing.T) { - userID := []byte("testDeleteUserID") - privateKey := []byte("testDeleteUserPrivateKey") + // Generate random user ID and private key + userID := make([]byte, 20) + rand.Read(userID) + privateKey := make([]byte, 32) + rand.Read(privateKey) + // Add user to storage err := storage.AddUser(userID, privateKey) if err != nil { t.Fatal(err) } + // Delete the user err = storage.DeleteUser(userID) if err != nil { t.Fatal(err) } + // Attempt to retrieve the deleted user's private key + // This should fail with a "not found" error _, err = storage.GetUserPrivateKey(userID) if err == nil || !errors.Is(err, errutil.ErrNotFound) { - t.Fatal("Expected error when getting deleted user, but got none") + t.Fatal("Expected 'not found' error when getting deleted user, but got none or different error") } } -func testGetAllUsers(storage Storage, t *testing.T) { - initialUsers, err := storage.GetAllUsers() - if err != nil { - t.Fatal(err) - } - - userID := []byte("getAllUsersTestID") - privateKey := []byte("getAllUsersTestPrivateKey") +func testGetUser(storage Storage, t *testing.T) { + // Generate random user ID and private key + userID := make([]byte, 20) + rand.Read(userID) + privateKey := make([]byte, 32) + rand.Read(privateKey) - err = storage.AddUser(userID, privateKey) + // Add user to storage + err := storage.AddUser(userID, privateKey) if err != nil { - t.Fatal(err) + t.Fatalf("Failed to add user: %v", err) } - afterInsertUsers, err := storage.GetAllUsers() + // Get user from storage + user, err := storage.GetUser(userID) if err != nil { - t.Fatal(err) + t.Fatalf("Failed to get user: %v", err) } - if len(afterInsertUsers) != len(initialUsers)+1 { - t.Errorf("Expected user count to increase by 1. Got %d initially and %d after insert", len(initialUsers), len(afterInsertUsers)) + // Check if retrieved user matches the added user + if !bytes.Equal(user.UserID, userID) { + t.Errorf("Retrieved user ID does not match. Expected %x, got %x", userID, user.UserID) } -} -func testStoringNewTx(storage Storage, t *testing.T) { - userID := []byte("userID") - rawTransaction := "0x0123456789" + if !bytes.Equal(user.PrivateKey, privateKey) { + t.Errorf("Retrieved private key does not match. Expected %x, got %x", privateKey, user.PrivateKey) + } - err := storage.StoreTransaction(rawTransaction, userID) - if err != nil { - t.Fatal(err) + // Try to get a non-existent user + nonExistentUserID := make([]byte, 20) + rand.Read(nonExistentUserID) + _, err = storage.GetUser(nonExistentUserID) + if err == nil { + t.Error("Expected error when getting non-existent user, but got none") } } From da5361bb37f1e46ae556bd08a86d9a39b19c5112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 22 Oct 2024 14:11:34 +0200 Subject: [PATCH 02/18] change Storage interface, fix sqlite to be able to run locally, refactoring --- go.mod | 2 +- go.sum | 8 + tools/walletextension/common/db_types.go | 16 +- .../storage/database/cosmosdb/cosmosdb.go | 57 ++---- .../storage/database/sqlite/sqlite.go | 187 ++++++++---------- tools/walletextension/storage/storage.go | 10 +- tools/walletextension/storage/storage_test.go | 8 +- 7 files changed, 125 insertions(+), 163 deletions(-) diff --git a/go.mod b/go.mod index d97a2e34d8..135a350186 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21.11 replace github.com/docker/docker => github.com/docker/docker v20.10.3-0.20220224222438-c78f6963a1c0+incompatible require ( + github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0 github.com/FantasyJony/openzeppelin-merkle-tree-go v1.1.2 github.com/Microsoft/go-winio v0.6.2 github.com/andybalholm/brotli v1.1.0 @@ -57,7 +58,6 @@ require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/DataDog/zstd v1.5.5 // indirect diff --git a/go.sum b/go.sum index ae9bc01dd4..ed4cf945c0 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,16 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0 h1:c726lgbwpwFBuj+Fyrwuh/vUilqFo+hUAOUNjsKj5DI= github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0/go.mod h1:WzFGxuepAtZIZtQbz8/WviJycLMKJHpaEAqcXONxlag= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/FantasyJony/openzeppelin-merkle-tree-go v1.1.2 h1:oZixv5U6eReqc1COEtng6/bVdAOAAWgtf38Ngn9Squc= @@ -171,6 +175,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keL github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= @@ -302,6 +308,8 @@ github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOS github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/tools/walletextension/common/db_types.go b/tools/walletextension/common/db_types.go index cde690813f..709662c37e 100644 --- a/tools/walletextension/common/db_types.go +++ b/tools/walletextension/common/db_types.go @@ -1,12 +1,14 @@ package common -type AccountDB struct { - AccountAddress []byte - Signature []byte - SignatureType int +type GWUserDB struct { + ID string `json:"id"` // Required by CosmosDB + UserId []byte `json:"userId"` + PrivateKey []byte `json:"privateKey"` + Accounts []GWAccountDB `json:"accounts"` // List of Accounts } -type UserDB struct { - UserID []byte - PrivateKey []byte +type GWAccountDB struct { + AccountAddress []byte `json:"accountAddress"` + Signature []byte `json:"signature"` + SignatureType int `json:"signatureType"` } diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index 3d0cc71841..2277ff2ef9 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -39,21 +39,6 @@ const ( PARTITION_KEY = "/id" ) -// GWUser is the user struct for the gateway -// both ID and UserID are the same for now, but we will use different values with encryption -type GWUser struct { - ID string `json:"id"` // Required by CosmosDB - UserId []byte `json:"userId"` - PrivateKey []byte `json:"privateKey"` - Accounts []GWAccount `json:"accounts"` // List of Accounts -} - -type GWAccount struct { - AccountAddress []byte `json:"accountAddress"` - Signature []byte `json:"signature"` - SignatureType int `json:"signatureType"` -} - func NewCosmosDB(connectionString string) (*CosmosDB, error) { client, err := azcosmos.NewClientFromConnectionString(connectionString, nil) if err != nil { @@ -80,11 +65,11 @@ func NewCosmosDB(connectionString string) (*CosmosDB, error) { } func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { - user := GWUser{ + user := common.GWUserDB{ ID: hex.EncodeToString(userID), UserId: userID, PrivateKey: privateKey, - Accounts: []GWAccount{}, + Accounts: []common.GWAccountDB{}, } userJSON, err := json.Marshal(user) if err != nil { @@ -130,7 +115,7 @@ func (c *CosmosDB) GetUserPrivateKey(userID []byte) ([]byte, error) { return nil, errutil.ErrNotFound } - var user GWUser + var user common.GWUserDB err = json.Unmarshal(itemResponse.Value, &user) if err != nil { return nil, fmt.Errorf("failed to unmarshal user data: %w", err) @@ -152,14 +137,14 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] return fmt.Errorf("failed to get user: %w", err) } - var user GWUser + var user common.GWUserDB err = json.Unmarshal(itemResponse.Value, &user) if err != nil { return fmt.Errorf("failed to unmarshal user data: %w", err) } // Create new account - newAccount := GWAccount{ + newAccount := common.GWAccountDB{ AccountAddress: accountAddress, Signature: signature, SignatureType: int(signatureType), @@ -182,7 +167,7 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] return nil } -func (c *CosmosDB) GetAccounts(userID []byte) ([]common.AccountDB, error) { +func (c *CosmosDB) GetAccounts(userID []byte) ([]common.GWAccountDB, error) { // Convert userID to hex string for use as partition key userIDHex := hex.EncodeToString(userID) partitionKey := azcosmos.NewPartitionKeyString(userIDHex) @@ -195,26 +180,15 @@ func (c *CosmosDB) GetAccounts(userID []byte) ([]common.AccountDB, error) { return nil, fmt.Errorf("failed to get user: %w", err) } - var user GWUser + var user common.GWUserDB err = json.Unmarshal(itemResponse.Value, &user) if err != nil { return nil, fmt.Errorf("failed to unmarshal user data: %w", err) } - - // Convert GWAccount to common.AccountDB - accounts := make([]common.AccountDB, len(user.Accounts)) - for i, acc := range user.Accounts { - accounts[i] = common.AccountDB{ - AccountAddress: acc.AccountAddress, - Signature: acc.Signature, - SignatureType: acc.SignatureType, - } - } - - return accounts, nil + return user.Accounts, nil } -func (c *CosmosDB) GetUser(userID []byte) (common.UserDB, error) { +func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { // Convert userID to hex string for use as partition key userIDHex := hex.EncodeToString(userID) partitionKey := azcosmos.NewPartitionKeyString(userIDHex) @@ -224,19 +198,14 @@ func (c *CosmosDB) GetUser(userID []byte) (common.UserDB, error) { // Read the existing user itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) if err != nil { - return common.UserDB{}, fmt.Errorf("failed to get user: %w", err) + return common.GWUserDB{}, fmt.Errorf("failed to get user: %w", err) } - var user GWUser + var user common.GWUserDB err = json.Unmarshal(itemResponse.Value, &user) if err != nil { - return common.UserDB{}, fmt.Errorf("failed to unmarshal user data: %w", err) + return common.GWUserDB{}, fmt.Errorf("failed to unmarshal user data: %w", err) } - // TODO: @ziga - check if I can use user directly instead of GWUser/UserDB since they are mostly the same... - // Convert GWUser to common.UserDB - return common.UserDB{ - UserID: userID, - PrivateKey: user.PrivateKey, - }, nil + return user, nil } diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index a39abfd65c..7b1612e5bb 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -1,20 +1,24 @@ package sqlite +/* + SQLite database implementation of the Storage interface + + SQLite is used for local deployments and testing without the need for a cloud database. + To make sure to see similar behaviour as in production using CosmosDB we use SQLite database in a similar way as comosDB (as key-value database). +*/ import ( "database/sql" - "encoding/hex" + "encoding/json" "fmt" "os" "path/filepath" "github.com/ten-protocol/go-ten/go/common/viewingkey" - "github.com/ethereum/go-ethereum/crypto" - _ "github.com/mattn/go-sqlite3" // sqlite driver for sql.Open() obscurocommon "github.com/ten-protocol/go-ten/go/common" "github.com/ten-protocol/go-ten/go/common/errutil" - common "github.com/ten-protocol/go-ten/tools/walletextension/common" + "github.com/ten-protocol/go-ten/tools/walletextension/common" ) type Database struct { @@ -41,50 +45,39 @@ func NewSqliteDatabase(dbPath string) (*Database, error) { return nil, err } - // create users table + // Modify the users table to store the entire GWUserDB as JSON _, err = db.Exec(`CREATE TABLE IF NOT EXISTS users ( - user_id binary(20) PRIMARY KEY, - private_key binary(32) - );`) - if err != nil { - return nil, err - } - - // create accounts table - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS accounts ( - user_id binary(20), - account_address binary(20), - signature binary(65), - signature_type int, - FOREIGN KEY(user_id) REFERENCES users(user_id) ON DELETE CASCADE + id TEXT PRIMARY KEY, + user_data TEXT );`) if err != nil { return nil, err } - // create transactions table - _, err = db.Exec(`CREATE TABLE IF NOT EXISTS transactions ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - user_id binary(20), - tx_hash TEXT, - tx TEXT, - tx_time TEXT DEFAULT (datetime('now')) -) ;`) - if err != nil { - return nil, err - } + // Remove the accounts table as it will be stored within the user_data JSON return &Database{db: db}, nil } func (s *Database) AddUser(userID []byte, privateKey []byte) error { - stmt, err := s.db.Prepare("INSERT OR REPLACE INTO users(user_id, private_key) VALUES (?, ?)") + user := common.GWUserDB{ + ID: string(userID), + UserId: userID, + PrivateKey: privateKey, + Accounts: []common.GWAccountDB{}, + } + userJSON, err := json.Marshal(user) + if err != nil { + return err + } + + stmt, err := s.db.Prepare("INSERT OR REPLACE INTO users(id, user_data) VALUES (?, ?)") if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(userID, privateKey) + _, err = stmt.Exec(user.ID, string(userJSON)) if err != nil { return err } @@ -93,93 +86,112 @@ func (s *Database) AddUser(userID []byte, privateKey []byte) error { } func (s *Database) DeleteUser(userID []byte) error { - stmt, err := s.db.Prepare("DELETE FROM users WHERE user_id = ?") + stmt, err := s.db.Prepare("DELETE FROM users WHERE id = ?") if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(userID) + _, err = stmt.Exec(string(userID)) if err != nil { - return err + return fmt.Errorf("failed to delete user: %w", err) } return nil } func (s *Database) GetUserPrivateKey(userID []byte) ([]byte, error) { - var privateKey []byte - err := s.db.QueryRow("SELECT private_key FROM users WHERE user_id = ?", userID).Scan(&privateKey) + var userDataJSON string + err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) if err != nil { if err == sql.ErrNoRows { - // No rows found for the given userID return nil, errutil.ErrNotFound } return nil, err } - return privateKey, nil + var user common.GWUserDB + err = json.Unmarshal([]byte(userDataJSON), &user) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal user data: %w", err) + } + + return user.PrivateKey, nil } func (s *Database) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { - stmt, err := s.db.Prepare("INSERT INTO accounts(user_id, account_address, signature, signature_type) VALUES (?, ?, ?, ?)") + var userDataJSON string + err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) + if err != nil { + return fmt.Errorf("failed to get user: %w", err) + } + + var user common.GWUserDB + err = json.Unmarshal([]byte(userDataJSON), &user) + if err != nil { + return fmt.Errorf("failed to unmarshal user data: %w", err) + } + + newAccount := common.GWAccountDB{ + AccountAddress: accountAddress, + Signature: signature, + SignatureType: int(signatureType), + } + + user.Accounts = append(user.Accounts, newAccount) + + updatedUserJSON, err := json.Marshal(user) + if err != nil { + return fmt.Errorf("error marshaling updated user: %w", err) + } + + stmt, err := s.db.Prepare("UPDATE users SET user_data = ? WHERE id = ?") if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(userID, accountAddress, signature, int(signatureType)) + _, err = stmt.Exec(string(updatedUserJSON), string(userID)) if err != nil { - return err + return fmt.Errorf("failed to update user with new account: %w", err) } return nil } -func (s *Database) GetAccounts(userID []byte) ([]common.AccountDB, error) { - rows, err := s.db.Query("SELECT account_address, signature, signature_type FROM accounts WHERE user_id = ?", userID) +func (s *Database) GetAccounts(userID []byte) ([]common.GWAccountDB, error) { + var userDataJSON string + err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get user: %w", err) } - defer rows.Close() - var accounts []common.AccountDB - for rows.Next() { - var account common.AccountDB - if err := rows.Scan(&account.AccountAddress, &account.Signature, &account.SignatureType); err != nil { - return nil, err - } - accounts = append(accounts, account) - } - if err := rows.Err(); err != nil { - return nil, err + var user common.GWUserDB + err = json.Unmarshal([]byte(userDataJSON), &user) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal user data: %w", err) } - return accounts, nil + return user.Accounts, nil } -func (s *Database) GetAllUsers() ([]common.UserDB, error) { - rows, err := s.db.Query("SELECT user_id, private_key FROM users") +func (s *Database) GetUser(userID []byte) (common.GWUserDB, error) { + var userDataJSON string + err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) if err != nil { - return nil, err - } - defer rows.Close() - - var users []common.UserDB - for rows.Next() { - var user common.UserDB - err = rows.Scan(&user.UserID, &user.PrivateKey) - if err != nil { - return nil, err + if err == sql.ErrNoRows { + return common.GWUserDB{}, fmt.Errorf("failed to get user: %w", errutil.ErrNotFound) } - users = append(users, user) + return common.GWUserDB{}, fmt.Errorf("failed to get user: %w", err) } - if err = rows.Err(); err != nil { - return nil, err + var user common.GWUserDB + err = json.Unmarshal([]byte(userDataJSON), &user) + if err != nil { + return common.GWUserDB{}, fmt.Errorf("failed to unmarshal user data: %w", err) } - return users, nil + return user, nil } func createOrLoad(dbPath string) (string, error) { @@ -203,32 +215,3 @@ func createOrLoad(dbPath string) (string, error) { return dbPath, nil } - -func (s *Database) StoreTransaction(rawTx string, userID []byte) error { - stmt, err := s.db.Prepare("INSERT INTO transactions(user_id, tx_hash, tx) VALUES (?, ?, ?)") - if err != nil { - return err - } - defer stmt.Close() - - txHash := "" - if len(rawTx) < 3 { - fmt.Println("Invalid rawTx: ", rawTx) - } else { - // Decode the hex string to bytes, excluding the '0x' prefix - rawTxBytes, err := hex.DecodeString(rawTx[2:]) - if err != nil { - fmt.Println("Error decoding rawTx: ", err) - } else { - // Compute Keccak-256 hash - txHash = crypto.Keccak256Hash(rawTxBytes).Hex() - } - } - - _, err = stmt.Exec(userID, txHash, rawTx) - if err != nil { - return err - } - - return nil -} diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index 3db0744b87..1f9ca83ae4 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -7,6 +7,7 @@ import ( "github.com/ten-protocol/go-ten/tools/walletextension/common" "github.com/ten-protocol/go-ten/tools/walletextension/storage/database/cosmosdb" + "github.com/ten-protocol/go-ten/tools/walletextension/storage/database/sqlite" ) type Storage interface { @@ -14,15 +15,14 @@ type Storage interface { DeleteUser(userID []byte) error GetUserPrivateKey(userID []byte) ([]byte, error) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error - GetAccounts(userID []byte) ([]common.AccountDB, error) - GetUser(userID []byte) (common.UserDB, error) + GetAccounts(userID []byte) ([]common.GWAccountDB, error) + GetUser(userID []byte) (common.GWUserDB, error) } func New(dbType string, dbConnectionURL, dbPath string) (Storage, error) { switch dbType { - // TODO: @ziga - fix sqlite for new Storage interface to allow local testing and running it locally without cosmosdb - // case "sqlite": - // return sqlite.NewSqliteDatabase(dbPath) + case "sqlite": + return sqlite.NewSqliteDatabase(dbPath) case "cosmosDB": return cosmosdb.NewCosmosDB(dbConnectionURL) } diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index af4246f813..e1975a8d34 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -22,8 +22,8 @@ var tests = map[string]func(storage Storage, t *testing.T){ func TestGatewayStorage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - // storage, err := New("sqlite", "", "") - storage, err := New("cosmosDB", "", "") + //storage, err := New("sqlite", "", "") + storage, err := New("cosmosDB", "", "") require.NoError(t, err) test(storage, t) @@ -179,8 +179,8 @@ func testGetUser(storage Storage, t *testing.T) { } // Check if retrieved user matches the added user - if !bytes.Equal(user.UserID, userID) { - t.Errorf("Retrieved user ID does not match. Expected %x, got %x", userID, user.UserID) + if !bytes.Equal(user.UserId, userID) { + t.Errorf("Retrieved user ID does not match. Expected %x, got %x", userID, user.UserId) } if !bytes.Equal(user.PrivateKey, privateKey) { From e5c796c9db979bfd597be3bbb1316ba465f1129b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 22 Oct 2024 14:34:02 +0200 Subject: [PATCH 03/18] remove not needed getPrivateKey --- tools/walletextension/rpcapi/gw_user.go | 39 ++++++++++++------- .../rpcapi/wallet_extension.go | 4 +- .../storage/database/cosmosdb/cosmosdb.go | 24 +----------- .../storage/database/sqlite/sqlite.go | 21 +--------- tools/walletextension/storage/storage.go | 1 - tools/walletextension/storage/storage_test.go | 12 +++--- 6 files changed, 36 insertions(+), 65 deletions(-) diff --git a/tools/walletextension/rpcapi/gw_user.go b/tools/walletextension/rpcapi/gw_user.go index 302a488557..73117eb1f4 100644 --- a/tools/walletextension/rpcapi/gw_user.go +++ b/tools/walletextension/rpcapi/gw_user.go @@ -7,6 +7,7 @@ import ( "github.com/ten-protocol/go-ten/go/common/viewingkey" "github.com/ethereum/go-ethereum/common" + wecommon "github.com/ten-protocol/go-ten/tools/walletextension/common" ) var userCacheKeyPrefix = []byte{0x0, 0x1, 0x2, 0x3} @@ -33,6 +34,28 @@ func (u GWUser) GetAllAddresses() []*common.Address { return accts } +func gwUserFromDB(userDB wecommon.GWUserDB, s *Services) (*GWUser, error) { + result := &GWUser{ + userID: userDB.UserId, + services: s, + accounts: make(map[common.Address]*GWAccount), + userKey: userDB.PrivateKey, + } + + for _, accountDB := range userDB.Accounts { + address := common.BytesToAddress(accountDB.AccountAddress) + gwAccount := &GWAccount{ + user: result, + address: &address, + signature: accountDB.Signature, + signatureType: viewingkey.SignatureType(accountDB.SignatureType), + } + result.accounts[address] = gwAccount + } + + return result, nil +} + func userCacheKey(userID []byte) []byte { var key []byte key = append(key, userCacheKeyPrefix...) @@ -42,21 +65,11 @@ func userCacheKey(userID []byte) []byte { func getUser(userID []byte, s *Services) (*GWUser, error) { return withCache(s.Cache, &CacheCfg{CacheType: LongLiving}, userCacheKey(userID), func() (*GWUser, error) { - result := GWUser{userID: userID, services: s, accounts: map[common.Address]*GWAccount{}} - userPrivateKey, err := s.Storage.GetUserPrivateKey(userID) + user, err := s.Storage.GetUser(userID) if err != nil { return nil, fmt.Errorf("user %s not found. %w", hexutils.BytesToHex(userID), err) } - result.userKey = userPrivateKey - allAccounts, err := s.Storage.GetAccounts(userID) - if err != nil { - return nil, err - } - - for _, account := range allAccounts { - address := common.BytesToAddress(account.AccountAddress) - result.accounts[address] = &GWAccount{user: &result, address: &address, signature: account.Signature, signatureType: viewingkey.SignatureType(uint8(account.SignatureType))} - } - return &result, nil + result, err := gwUserFromDB(user, s) + return result, err }) } diff --git a/tools/walletextension/rpcapi/wallet_extension.go b/tools/walletextension/rpcapi/wallet_extension.go index 060ea5370f..cc2f7ba284 100644 --- a/tools/walletextension/rpcapi/wallet_extension.go +++ b/tools/walletextension/rpcapi/wallet_extension.go @@ -262,12 +262,12 @@ func (w *Services) UserExists(userID []byte) bool { // Check if user exists and don't log error if user doesn't exist, because we expect this to happen in case of // user revoking encryption token or using different testnet. // todo add a counter here in the future - key, err := w.Storage.GetUserPrivateKey(userID) + users, err := w.Storage.GetUser(userID) if err != nil { return false } - return len(key) > 0 + return len(users.PrivateKey) > 0 } func (w *Services) Version() string { diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index 2277ff2ef9..15cb68c12d 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -102,28 +102,6 @@ func (c *CosmosDB) DeleteUser(userID []byte) error { return nil } -func (c *CosmosDB) GetUserPrivateKey(userID []byte) ([]byte, error) { - // Convert userID to hex string for use as partition key - userIDHex := hex.EncodeToString(userID) - partitionKey := azcosmos.NewPartitionKeyString(userIDHex) - - ctx := context.Background() - - // Read the item from the container - itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) - if err != nil { - return nil, errutil.ErrNotFound - } - - var user common.GWUserDB - err = json.Unmarshal(itemResponse.Value, &user) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal user data: %w", err) - } - - return user.PrivateKey, nil -} - func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { // Convert userID to hex string for use as partition key userIDHex := hex.EncodeToString(userID) @@ -198,7 +176,7 @@ func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { // Read the existing user itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) if err != nil { - return common.GWUserDB{}, fmt.Errorf("failed to get user: %w", err) + return common.GWUserDB{}, errutil.ErrNotFound } var user common.GWUserDB diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index 7b1612e5bb..fadd1df6af 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -14,11 +14,11 @@ import ( "path/filepath" "github.com/ten-protocol/go-ten/go/common/viewingkey" + "github.com/ten-protocol/go-ten/tools/walletextension/common" _ "github.com/mattn/go-sqlite3" // sqlite driver for sql.Open() obscurocommon "github.com/ten-protocol/go-ten/go/common" "github.com/ten-protocol/go-ten/go/common/errutil" - "github.com/ten-protocol/go-ten/tools/walletextension/common" ) type Database struct { @@ -100,25 +100,6 @@ func (s *Database) DeleteUser(userID []byte) error { return nil } -func (s *Database) GetUserPrivateKey(userID []byte) ([]byte, error) { - var userDataJSON string - err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) - if err != nil { - if err == sql.ErrNoRows { - return nil, errutil.ErrNotFound - } - return nil, err - } - - var user common.GWUserDB - err = json.Unmarshal([]byte(userDataJSON), &user) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal user data: %w", err) - } - - return user.PrivateKey, nil -} - func (s *Database) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { var userDataJSON string err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index 1f9ca83ae4..f5f063d7f2 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -13,7 +13,6 @@ import ( type Storage interface { AddUser(userID []byte, privateKey []byte) error DeleteUser(userID []byte) error - GetUserPrivateKey(userID []byte) ([]byte, error) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error GetAccounts(userID []byte) ([]common.GWAccountDB, error) GetUser(userID []byte) (common.GWUserDB, error) diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index e1975a8d34..01079fba99 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -22,8 +22,8 @@ var tests = map[string]func(storage Storage, t *testing.T){ func TestGatewayStorage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - //storage, err := New("sqlite", "", "") - storage, err := New("cosmosDB", "", "") + storage, err := New("sqlite", "", "") + // storage, err := New("cosmosDB", "", "") require.NoError(t, err) test(storage, t) @@ -51,14 +51,14 @@ func testAddAndGetUser(storage Storage, t *testing.T) { } // Retrieve user's private key from storage - returnedPrivateKey, err := storage.GetUserPrivateKey(userID) + user, err := storage.GetUser(userID) if err != nil { t.Fatal(err) } // Check if retrieved private key matches the original - if !bytes.Equal(returnedPrivateKey, privateKey) { - t.Errorf("privateKey mismatch: got %v, want %v", returnedPrivateKey, privateKey) + if !bytes.Equal(user.PrivateKey, privateKey) { + t.Errorf("privateKey mismatch: got %v, want %v", user.PrivateKey, privateKey) } } @@ -153,7 +153,7 @@ func testDeleteUser(storage Storage, t *testing.T) { // Attempt to retrieve the deleted user's private key // This should fail with a "not found" error - _, err = storage.GetUserPrivateKey(userID) + _, err = storage.GetUser(userID) if err == nil || !errors.Is(err, errutil.ErrNotFound) { t.Fatal("Expected 'not found' error when getting deleted user, but got none or different error") } From e9edd4715e9d3c481a00b42e6a8fbd678a42ad87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 22 Oct 2024 14:43:06 +0200 Subject: [PATCH 04/18] remove getAccounts as it is no longer needed --- .../rpcapi/wallet_extension.go | 3 ++- .../storage/database/cosmosdb/cosmosdb.go | 21 ------------------- .../storage/database/sqlite/sqlite.go | 16 -------------- tools/walletextension/storage/storage.go | 1 - tools/walletextension/storage/storage_test.go | 18 ++++++++-------- 5 files changed, 11 insertions(+), 48 deletions(-) diff --git a/tools/walletextension/rpcapi/wallet_extension.go b/tools/walletextension/rpcapi/wallet_extension.go index cc2f7ba284..9e93885369 100644 --- a/tools/walletextension/rpcapi/wallet_extension.go +++ b/tools/walletextension/rpcapi/wallet_extension.go @@ -228,11 +228,12 @@ func (w *Services) UserHasAccount(userID []byte, address string) (bool, error) { // todo - this can be optimised and done in the database if we will have users with large number of accounts // get all the accounts for the selected user - accounts, err := w.Storage.GetAccounts(userID) + user, err := w.Storage.GetUser(userID) if err != nil { w.Logger().Error(fmt.Errorf("error getting accounts for user (%s), %w", userID, err).Error()) return false, err } + accounts := user.Accounts // check if any of the account matches given account found := false diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index 15cb68c12d..d685b4b674 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -145,27 +145,6 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] return nil } -func (c *CosmosDB) GetAccounts(userID []byte) ([]common.GWAccountDB, error) { - // Convert userID to hex string for use as partition key - userIDHex := hex.EncodeToString(userID) - partitionKey := azcosmos.NewPartitionKeyString(userIDHex) - - ctx := context.Background() - - // Read the existing user - itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) - if err != nil { - return nil, fmt.Errorf("failed to get user: %w", err) - } - - var user common.GWUserDB - err = json.Unmarshal(itemResponse.Value, &user) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal user data: %w", err) - } - return user.Accounts, nil -} - func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { // Convert userID to hex string for use as partition key userIDHex := hex.EncodeToString(userID) diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index fadd1df6af..93b0519f0a 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -140,22 +140,6 @@ func (s *Database) AddAccount(userID []byte, accountAddress []byte, signature [] return nil } -func (s *Database) GetAccounts(userID []byte) ([]common.GWAccountDB, error) { - var userDataJSON string - err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) - if err != nil { - return nil, fmt.Errorf("failed to get user: %w", err) - } - - var user common.GWUserDB - err = json.Unmarshal([]byte(userDataJSON), &user) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal user data: %w", err) - } - - return user.Accounts, nil -} - func (s *Database) GetUser(userID []byte) (common.GWUserDB, error) { var userDataJSON string err := s.db.QueryRow("SELECT user_data FROM users WHERE id = ?", string(userID)).Scan(&userDataJSON) diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index f5f063d7f2..e28d9bfbe9 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -14,7 +14,6 @@ type Storage interface { AddUser(userID []byte, privateKey []byte) error DeleteUser(userID []byte) error AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error - GetAccounts(userID []byte) ([]common.GWAccountDB, error) GetUser(userID []byte) (common.GWUserDB, error) } diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index 01079fba99..1fd37cec6f 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -13,10 +13,10 @@ import ( ) var tests = map[string]func(storage Storage, t *testing.T){ - "testAddAndGetUser": testAddAndGetUser, - "testAddAndGetAccounts": testAddAndGetAccounts, - "testDeleteUser": testDeleteUser, - "testGetUser": testGetUser, + "testAddAndGetUser": testAddAndGetUser, + "testAddAccounts": testAddAccounts, + "testDeleteUser": testDeleteUser, + "testGetUser": testGetUser, } func TestGatewayStorage(t *testing.T) { @@ -62,7 +62,7 @@ func testAddAndGetUser(storage Storage, t *testing.T) { } } -func testAddAndGetAccounts(storage Storage, t *testing.T) { +func testAddAccounts(storage Storage, t *testing.T) { // Generate random user ID, private key, and account details userID := make([]byte, 20) rand.Read(userID) @@ -98,14 +98,14 @@ func testAddAndGetAccounts(storage Storage, t *testing.T) { } // Retrieve all accounts for the user - accounts, err := storage.GetAccounts(userID) + user, err := storage.GetUser(userID) if err != nil { t.Fatal(err) } // Check if the correct number of accounts were retrieved - if len(accounts) != 2 { - t.Errorf("Expected 2 accounts, got %d", len(accounts)) + if len(user.Accounts) != 2 { + t.Errorf("Expected 2 accounts, got %d", len(user.Accounts)) } // Flags to check if both accounts are found @@ -113,7 +113,7 @@ func testAddAndGetAccounts(storage Storage, t *testing.T) { foundAccount2 := false // Iterate through retrieved accounts and check if they match the added accounts - for _, account := range accounts { + for _, account := range user.Accounts { if bytes.Equal(account.AccountAddress, accountAddress1) && bytes.Equal(account.Signature, signature1) { foundAccount1 = true } From dde7135e06dceda3a9cb3b2f7eb37ee137013ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 22 Oct 2024 17:16:32 +0200 Subject: [PATCH 05/18] tests passing with cosmosDB encrypted --- tools/walletextension/common/common.go | 10 ++ .../walletextension/encryption/encryption.go | 52 ++++++ .../encryption/encryption_test.go | 115 ++++++++++++ .../storage/database/cosmosdb/cosmosdb.go | 167 ++++++++++++++---- tools/walletextension/storage/storage.go | 8 +- tools/walletextension/storage/storage_test.go | 2 +- 6 files changed, 318 insertions(+), 36 deletions(-) create mode 100644 tools/walletextension/encryption/encryption.go create mode 100644 tools/walletextension/encryption/encryption_test.go diff --git a/tools/walletextension/common/common.go b/tools/walletextension/common/common.go index 94ac8be729..59ed43a57b 100644 --- a/tools/walletextension/common/common.go +++ b/tools/walletextension/common/common.go @@ -1,6 +1,7 @@ package common import ( + "crypto/rand" "encoding/json" "fmt" @@ -76,3 +77,12 @@ func (r *RPCRequest) Clone() *RPCRequest { Params: r.Params, } } + +func GenerateRandomKey(keySize int) ([]byte, error) { + key := make([]byte, keySize) + _, err := rand.Read(key) + if err != nil { + return nil, err + } + return key, nil +} diff --git a/tools/walletextension/encryption/encryption.go b/tools/walletextension/encryption/encryption.go new file mode 100644 index 0000000000..fe8da56bc2 --- /dev/null +++ b/tools/walletextension/encryption/encryption.go @@ -0,0 +1,52 @@ +package encryption + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "errors" + "io" +) + +type Encryptor struct { + gcm cipher.AEAD + key []byte +} + +func NewEncryptor(key []byte) (*Encryptor, error) { + // TODO: @ziga Check key length! + + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + return &Encryptor{gcm: gcm, key: key}, nil +} + +func (e *Encryptor) Encrypt(plaintext []byte) ([]byte, error) { + nonce := make([]byte, e.gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, err + } + return e.gcm.Seal(nonce, nonce, plaintext, nil), nil +} + +func (e *Encryptor) Decrypt(ciphertext []byte) ([]byte, error) { + if len(ciphertext) < e.gcm.NonceSize() { + return nil, errors.New("ciphertext too short") + } + nonce, ciphertext := ciphertext[:e.gcm.NonceSize()], ciphertext[e.gcm.NonceSize():] + return e.gcm.Open(nil, nonce, ciphertext, nil) +} + +func (e *Encryptor) HashWithHMAC(data []byte) []byte { + h := hmac.New(sha256.New, e.key) + h.Write(data) + return h.Sum(nil) +} diff --git a/tools/walletextension/encryption/encryption_test.go b/tools/walletextension/encryption/encryption_test.go new file mode 100644 index 0000000000..8395fd6da9 --- /dev/null +++ b/tools/walletextension/encryption/encryption_test.go @@ -0,0 +1,115 @@ +package encryption + +import ( + "bytes" + "crypto/rand" + "testing" +) + +func TestNewEncryptor(t *testing.T) { + key := make([]byte, 32) // 256-bit key + _, err := rand.Read(key) + if err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + + encryptor, err := NewEncryptor(key) + if err != nil { + t.Fatalf("NewEncryptor failed: %v", err) + } + + if encryptor == nil { + t.Fatal("NewEncryptor returned nil") + } +} + +func TestEncryptDecrypt(t *testing.T) { + key := make([]byte, 32) // 256-bit key + _, err := rand.Read(key) + if err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + + encryptor, err := NewEncryptor(key) + if err != nil { + t.Fatalf("NewEncryptor failed: %v", err) + } + + plaintext := []byte("Hello, World!") + + ciphertext, err := encryptor.Encrypt(plaintext) + if err != nil { + t.Fatalf("Encryption failed: %v", err) + } + + decrypted, err := encryptor.Decrypt(ciphertext) + if err != nil { + t.Fatalf("Decryption failed: %v", err) + } + + if !bytes.Equal(plaintext, decrypted) { + t.Fatalf("Decrypted text does not match original plaintext") + } +} + +func TestEncryptDecryptEmptyString(t *testing.T) { + key := make([]byte, 32) // 256-bit key + _, err := rand.Read(key) + if err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + + encryptor, err := NewEncryptor(key) + if err != nil { + t.Fatalf("NewEncryptor failed: %v", err) + } + + plaintext := []byte("") + + ciphertext, err := encryptor.Encrypt(plaintext) + if err != nil { + t.Fatalf("Encryption of empty string failed: %v", err) + } + + decrypted, err := encryptor.Decrypt(ciphertext) + if err != nil { + t.Fatalf("Decryption of empty string failed: %v", err) + } + + if !bytes.Equal(plaintext, decrypted) { + t.Fatalf("Decrypted empty string does not match original") + } +} + +func TestDecryptInvalidCiphertext(t *testing.T) { + key := make([]byte, 32) // 256-bit key + _, err := rand.Read(key) + if err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + + encryptor, err := NewEncryptor(key) + if err != nil { + t.Fatalf("NewEncryptor failed: %v", err) + } + + invalidCiphertext := []byte("This is not a valid ciphertext") + + _, err = encryptor.Decrypt(invalidCiphertext) + if err == nil { + t.Fatal("Decryption of invalid ciphertext should have failed, but didn't") + } +} + +func TestNewEncryptorInvalidKeySize(t *testing.T) { + invalidKey := make([]byte, 31) // Invalid key size (not 16, 24, or 32 bytes) + _, err := rand.Read(invalidKey) + if err != nil { + t.Fatalf("Failed to generate random key: %v", err) + } + + _, err = NewEncryptor(invalidKey) + if err == nil { + t.Fatal("NewEncryptor should have failed with invalid key size, but didn't") + } +} diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index d685b4b674..5a4ad9d491 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -2,15 +2,18 @@ package cosmosdb import ( "context" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" "strings" + "github.com/ten-protocol/go-ten/go/common/viewingkey" + "github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos" "github.com/ten-protocol/go-ten/go/common/errutil" - "github.com/ten-protocol/go-ten/go/common/viewingkey" "github.com/ten-protocol/go-ten/tools/walletextension/common" + "github.com/ten-protocol/go-ten/tools/walletextension/encryption" ) /* @@ -31,6 +34,12 @@ Quick summary of the CosmosDB setup: type CosmosDB struct { client *azcosmos.Client usersContainer *azcosmos.ContainerClient + encryptor *encryption.Encryptor +} + +type EncryptedDocument struct { + ID string `json:"id"` + Data string `json:"data"` } const ( @@ -39,7 +48,13 @@ const ( PARTITION_KEY = "/id" ) -func NewCosmosDB(connectionString string) (*CosmosDB, error) { +func NewCosmosDB(connectionString string, encryptionKey []byte) (*CosmosDB, error) { + // create encryptor + encryptor, err := encryption.NewEncryptor(encryptionKey) + if err != nil { + return nil, err + } + client, err := azcosmos.NewClientFromConnectionString(connectionString, nil) if err != nil { return nil, err @@ -61,6 +76,7 @@ func NewCosmosDB(connectionString string) (*CosmosDB, error) { return &CosmosDB{ client: client, usersContainer: usersContainer, + encryptor: encryptor, }, nil } @@ -76,26 +92,54 @@ func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { return err } - // add to cosmosdb - partitionKey := azcosmos.NewPartitionKeyString(user.ID) + var encryptedData string + if c.encryptor != nil { + ciphertext, err := c.encryptor.Encrypt(userJSON) + if err != nil { + return err + } + encryptedData = base64.StdEncoding.EncodeToString(ciphertext) + } else { + encryptedData = base64.StdEncoding.EncodeToString(userJSON) + } + + // Hash the userID to use as the key + key := userID + if c.encryptor != nil { + key = c.encryptor.HashWithHMAC(userID) + } + keyString := hex.EncodeToString(key) - ctx := context.Background() - _, err = c.usersContainer.CreateItem(ctx, partitionKey, userJSON, nil) + // Create the encrypted document + doc := EncryptedDocument{ + ID: keyString, + Data: encryptedData, + } + + docJSON, err := json.Marshal(doc) if err != nil { return err } + + partitionKey := azcosmos.NewPartitionKeyString(keyString) + ctx := context.Background() + _, err = c.usersContainer.CreateItem(ctx, partitionKey, docJSON, nil) + if err != nil { + return fmt.Errorf("failed to create item: %w", err) + } return nil } func (c *CosmosDB) DeleteUser(userID []byte) error { - // Convert userID to hex string for use as partition key - userIDHex := hex.EncodeToString(userID) - partitionKey := azcosmos.NewPartitionKeyString(userIDHex) - + key := userID + if c.encryptor != nil { + key = c.encryptor.HashWithHMAC(userID) + } + keyString := hex.EncodeToString(key) + partitionKey := azcosmos.NewPartitionKeyString(keyString) ctx := context.Background() - // Delete the item from the container - _, err := c.usersContainer.DeleteItem(ctx, partitionKey, userIDHex, nil) + _, err := c.usersContainer.DeleteItem(ctx, partitionKey, keyString, nil) if err != nil { return fmt.Errorf("failed to delete user: %w", err) } @@ -103,42 +147,78 @@ func (c *CosmosDB) DeleteUser(userID []byte) error { } func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { - // Convert userID to hex string for use as partition key - userIDHex := hex.EncodeToString(userID) - partitionKey := azcosmos.NewPartitionKeyString(userIDHex) - + key := userID + if c.encryptor != nil { + key = c.encryptor.HashWithHMAC(userID) + } + keyString := hex.EncodeToString(key) + partitionKey := azcosmos.NewPartitionKeyString(keyString) ctx := context.Background() - // Read the existing user - itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) + itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, keyString, nil) if err != nil { return fmt.Errorf("failed to get user: %w", err) } + var doc EncryptedDocument + err = json.Unmarshal(itemResponse.Value, &doc) + if err != nil { + return fmt.Errorf("failed to unmarshal document: %w", err) + } + + encryptedData, err := base64.StdEncoding.DecodeString(doc.Data) + if err != nil { + return fmt.Errorf("failed to decode base64 data: %w", err) + } + + data := encryptedData + if c.encryptor != nil { + data, err = c.encryptor.Decrypt(encryptedData) + if err != nil { + return fmt.Errorf("failed to decrypt data: %w", err) + } + } + var user common.GWUserDB - err = json.Unmarshal(itemResponse.Value, &user) + err = json.Unmarshal(data, &user) if err != nil { return fmt.Errorf("failed to unmarshal user data: %w", err) } - // Create new account + // Add the new account newAccount := common.GWAccountDB{ AccountAddress: accountAddress, Signature: signature, SignatureType: int(signatureType), } - - // Add new account to user's accounts user.Accounts = append(user.Accounts, newAccount) - // Marshal updated user back to JSON - updatedUserJSON, err := json.Marshal(user) + userJSON, err := json.Marshal(user) if err != nil { return fmt.Errorf("error marshaling updated user: %w", err) } - // Update the item in the container - _, err = c.usersContainer.ReplaceItem(ctx, partitionKey, userIDHex, updatedUserJSON, nil) + var encryptedDataStr string + if c.encryptor != nil { + ciphertext, err := c.encryptor.Encrypt(userJSON) + if err != nil { + return fmt.Errorf("failed to encrypt updated user data: %w", err) + } + encryptedDataStr = base64.StdEncoding.EncodeToString(ciphertext) + } else { + encryptedDataStr = base64.StdEncoding.EncodeToString(userJSON) + } + + // Update the document + doc.Data = encryptedDataStr + + docJSON, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("failed to marshal updated document: %w", err) + } + + // Replace the item in the container + _, err = c.usersContainer.ReplaceItem(ctx, partitionKey, keyString, docJSON, nil) if err != nil { return fmt.Errorf("failed to update user with new account: %w", err) } @@ -146,23 +226,42 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] } func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { - // Convert userID to hex string for use as partition key - userIDHex := hex.EncodeToString(userID) - partitionKey := azcosmos.NewPartitionKeyString(userIDHex) - + key := userID + if c.encryptor != nil { + key = c.encryptor.HashWithHMAC(userID) + } + keyString := hex.EncodeToString(key) + partitionKey := azcosmos.NewPartitionKeyString(keyString) ctx := context.Background() - // Read the existing user - itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, userIDHex, nil) + itemResponse, err := c.usersContainer.ReadItem(ctx, partitionKey, keyString, nil) if err != nil { return common.GWUserDB{}, errutil.ErrNotFound } + var doc EncryptedDocument + err = json.Unmarshal(itemResponse.Value, &doc) + if err != nil { + return common.GWUserDB{}, fmt.Errorf("failed to unmarshal document: %w", err) + } + + encryptedData, err := base64.StdEncoding.DecodeString(doc.Data) + if err != nil { + return common.GWUserDB{}, fmt.Errorf("failed to decode base64 data: %w", err) + } + + data := encryptedData + if c.encryptor != nil { + data, err = c.encryptor.Decrypt(encryptedData) + if err != nil { + return common.GWUserDB{}, fmt.Errorf("failed to decrypt data: %w", err) + } + } + var user common.GWUserDB - err = json.Unmarshal(itemResponse.Value, &user) + err = json.Unmarshal(data, &user) if err != nil { return common.GWUserDB{}, fmt.Errorf("failed to unmarshal user data: %w", err) } - return user, nil } diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index e28d9bfbe9..4bdf75d433 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -18,11 +18,17 @@ type Storage interface { } func New(dbType string, dbConnectionURL, dbPath string) (Storage, error) { + // TODO @ziga: Generate random key in a different part of the code! + randomKey, err := common.GenerateRandomKey(32) + if err != nil { + return nil, err + } + switch dbType { case "sqlite": return sqlite.NewSqliteDatabase(dbPath) case "cosmosDB": - return cosmosdb.NewCosmosDB(dbConnectionURL) + return cosmosdb.NewCosmosDB(dbConnectionURL, randomKey) } return nil, fmt.Errorf("unknown db %s", dbType) diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index 1fd37cec6f..9d6eae2e1d 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -23,7 +23,7 @@ func TestGatewayStorage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { storage, err := New("sqlite", "", "") - // storage, err := New("cosmosDB", "", "") + //storage, err := New("cosmosDB", "", "") require.NoError(t, err) test(storage, t) From f0caa6d64fb454742d49c2cfff25027efc384148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Tue, 22 Oct 2024 17:30:38 +0200 Subject: [PATCH 06/18] refactored & added comments --- .../storage/database/cosmosdb/cosmosdb.go | 87 +++++++------------ 1 file changed, 32 insertions(+), 55 deletions(-) diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index 5a4ad9d491..4ce633ac27 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -31,17 +31,23 @@ Quick summary of the CosmosDB setup: */ +// CosmosDB struct represents the CosmosDB storage implementation type CosmosDB struct { client *azcosmos.Client usersContainer *azcosmos.ContainerClient - encryptor *encryption.Encryptor + encryptor encryption.Encryptor } +// EncryptedDocument struct is used to store encrypted user data in CosmosDB +// We use this structure to add an extra layer of security by encrypting the actual user data +// The 'ID' field is used as the document ID and partition key in CosmosDB +// The 'Data' field contains the base64-encoded encrypted user data type EncryptedDocument struct { ID string `json:"id"` Data string `json:"data"` } +// Constants for the CosmosDB database and container names const ( DATABASE_NAME = "gatewayDB" USERS_CONTAINER_NAME = "users" @@ -49,15 +55,15 @@ const ( ) func NewCosmosDB(connectionString string, encryptionKey []byte) (*CosmosDB, error) { - // create encryptor + // Create encryptor encryptor, err := encryption.NewEncryptor(encryptionKey) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create encryptor: %w", err) } client, err := azcosmos.NewClientFromConnectionString(connectionString, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create CosmosDB client: %w", err) } // Create database if it doesn't exist @@ -76,7 +82,7 @@ func NewCosmosDB(connectionString string, encryptionKey []byte) (*CosmosDB, erro return &CosmosDB{ client: client, usersContainer: usersContainer, - encryptor: encryptor, + encryptor: *encryptor, }, nil } @@ -89,28 +95,19 @@ func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { } userJSON, err := json.Marshal(user) if err != nil { - return err + return fmt.Errorf("failed to marshal user: %w", err) } - var encryptedData string - if c.encryptor != nil { - ciphertext, err := c.encryptor.Encrypt(userJSON) - if err != nil { - return err - } - encryptedData = base64.StdEncoding.EncodeToString(ciphertext) - } else { - encryptedData = base64.StdEncoding.EncodeToString(userJSON) + ciphertext, err := c.encryptor.Encrypt(userJSON) + if err != nil { + return fmt.Errorf("failed to encrypt user data: %w", err) } + encryptedData := base64.StdEncoding.EncodeToString(ciphertext) - // Hash the userID to use as the key - key := userID - if c.encryptor != nil { - key = c.encryptor.HashWithHMAC(userID) - } + key := c.encryptor.HashWithHMAC(userID) keyString := hex.EncodeToString(key) - // Create the encrypted document + // Create an EncryptedDocument struct to store in CosmosDB doc := EncryptedDocument{ ID: keyString, Data: encryptedData, @@ -118,7 +115,7 @@ func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { docJSON, err := json.Marshal(doc) if err != nil { - return err + return fmt.Errorf("failed to marshal document: %w", err) } partitionKey := azcosmos.NewPartitionKeyString(keyString) @@ -131,10 +128,7 @@ func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { } func (c *CosmosDB) DeleteUser(userID []byte) error { - key := userID - if c.encryptor != nil { - key = c.encryptor.HashWithHMAC(userID) - } + key := c.encryptor.HashWithHMAC(userID) keyString := hex.EncodeToString(key) partitionKey := azcosmos.NewPartitionKeyString(keyString) ctx := context.Background() @@ -147,10 +141,7 @@ func (c *CosmosDB) DeleteUser(userID []byte) error { } func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { - key := userID - if c.encryptor != nil { - key = c.encryptor.HashWithHMAC(userID) - } + key := c.encryptor.HashWithHMAC(userID) keyString := hex.EncodeToString(key) partitionKey := azcosmos.NewPartitionKeyString(keyString) ctx := context.Background() @@ -171,12 +162,9 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] return fmt.Errorf("failed to decode base64 data: %w", err) } - data := encryptedData - if c.encryptor != nil { - data, err = c.encryptor.Decrypt(encryptedData) - if err != nil { - return fmt.Errorf("failed to decrypt data: %w", err) - } + data, err := c.encryptor.Decrypt(encryptedData) + if err != nil { + return fmt.Errorf("failed to decrypt data: %w", err) } var user common.GWUserDB @@ -198,16 +186,11 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] return fmt.Errorf("error marshaling updated user: %w", err) } - var encryptedDataStr string - if c.encryptor != nil { - ciphertext, err := c.encryptor.Encrypt(userJSON) - if err != nil { - return fmt.Errorf("failed to encrypt updated user data: %w", err) - } - encryptedDataStr = base64.StdEncoding.EncodeToString(ciphertext) - } else { - encryptedDataStr = base64.StdEncoding.EncodeToString(userJSON) + ciphertext, err := c.encryptor.Encrypt(userJSON) + if err != nil { + return fmt.Errorf("failed to encrypt updated user data: %w", err) } + encryptedDataStr := base64.StdEncoding.EncodeToString(ciphertext) // Update the document doc.Data = encryptedDataStr @@ -226,10 +209,7 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] } func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { - key := userID - if c.encryptor != nil { - key = c.encryptor.HashWithHMAC(userID) - } + key := c.encryptor.HashWithHMAC(userID) keyString := hex.EncodeToString(key) partitionKey := azcosmos.NewPartitionKeyString(keyString) ctx := context.Background() @@ -250,12 +230,9 @@ func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { return common.GWUserDB{}, fmt.Errorf("failed to decode base64 data: %w", err) } - data := encryptedData - if c.encryptor != nil { - data, err = c.encryptor.Decrypt(encryptedData) - if err != nil { - return common.GWUserDB{}, fmt.Errorf("failed to decrypt data: %w", err) - } + data, err := c.encryptor.Decrypt(encryptedData) + if err != nil { + return common.GWUserDB{}, fmt.Errorf("failed to decrypt data: %w", err) } var user common.GWUserDB From 0196b8a4617547052ce6257d4202cd39b54e1f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Wed, 23 Oct 2024 08:40:40 +0200 Subject: [PATCH 07/18] use random key generated at startup --- tools/walletextension/common/common.go | 6 ++++-- tools/walletextension/encryption/encryption.go | 5 ++++- tools/walletextension/storage/storage.go | 8 +------- tools/walletextension/storage/storage_test.go | 8 ++++++-- tools/walletextension/walletextension_container.go | 9 ++++++++- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/tools/walletextension/common/common.go b/tools/walletextension/common/common.go index 59ed43a57b..838d2a37d5 100644 --- a/tools/walletextension/common/common.go +++ b/tools/walletextension/common/common.go @@ -16,6 +16,8 @@ import ( gethlog "github.com/ethereum/go-ethereum/log" ) +const EncryptionKeySize = 32 + // PrivateKeyToCompressedPubKey converts *ecies.PrivateKey to compressed PubKey ([]byte with length 33) func PrivateKeyToCompressedPubKey(prvKey *ecies.PrivateKey) []byte { ecdsaPublicKey := prvKey.PublicKey.ExportECDSA() @@ -78,8 +80,8 @@ func (r *RPCRequest) Clone() *RPCRequest { } } -func GenerateRandomKey(keySize int) ([]byte, error) { - key := make([]byte, keySize) +func GenerateRandomKey() ([]byte, error) { + key := make([]byte, EncryptionKeySize) _, err := rand.Read(key) if err != nil { return nil, err diff --git a/tools/walletextension/encryption/encryption.go b/tools/walletextension/encryption/encryption.go index fe8da56bc2..31a41fd1fd 100644 --- a/tools/walletextension/encryption/encryption.go +++ b/tools/walletextension/encryption/encryption.go @@ -7,6 +7,7 @@ import ( "crypto/rand" "crypto/sha256" "errors" + "fmt" "io" ) @@ -16,7 +17,9 @@ type Encryptor struct { } func NewEncryptor(key []byte) (*Encryptor, error) { - // TODO: @ziga Check key length! + if len(key) != 32 { + return nil, fmt.Errorf("key must be 32 bytes long") + } block, err := aes.NewCipher(key) if err != nil { diff --git a/tools/walletextension/storage/storage.go b/tools/walletextension/storage/storage.go index 4bdf75d433..cb1fb6304b 100644 --- a/tools/walletextension/storage/storage.go +++ b/tools/walletextension/storage/storage.go @@ -17,13 +17,7 @@ type Storage interface { GetUser(userID []byte) (common.GWUserDB, error) } -func New(dbType string, dbConnectionURL, dbPath string) (Storage, error) { - // TODO @ziga: Generate random key in a different part of the code! - randomKey, err := common.GenerateRandomKey(32) - if err != nil { - return nil, err - } - +func New(dbType string, dbConnectionURL, dbPath string, randomKey []byte) (Storage, error) { switch dbType { case "sqlite": return sqlite.NewSqliteDatabase(dbPath) diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index 9d6eae2e1d..058a1894ff 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/ten-protocol/go-ten/go/common/errutil" + wecommon "github.com/ten-protocol/go-ten/tools/walletextension/common" ) var tests = map[string]func(storage Storage, t *testing.T){ @@ -20,10 +21,13 @@ var tests = map[string]func(storage Storage, t *testing.T){ } func TestGatewayStorage(t *testing.T) { + randomKey, err := wecommon.GenerateRandomKey() + require.NoError(t, err) + for name, test := range tests { t.Run(name, func(t *testing.T) { - storage, err := New("sqlite", "", "") - //storage, err := New("cosmosDB", "", "") + //storage, err := New("sqlite", "", "", randomKey) + storage, err := New("cosmosDB", "", "", randomKey) require.NoError(t, err) test(storage, t) diff --git a/tools/walletextension/walletextension_container.go b/tools/walletextension/walletextension_container.go index f6499ec6c7..d21832bc8f 100644 --- a/tools/walletextension/walletextension_container.go +++ b/tools/walletextension/walletextension_container.go @@ -32,8 +32,15 @@ func NewContainerFromConfig(config wecommon.Config, logger gethlog.Logger) *Cont // create the account manager with a single unauthenticated connection hostRPCBindAddrWS := wecommon.WSProtocol + config.NodeRPCWebsocketAddress hostRPCBindAddrHTTP := wecommon.HTTPProtocol + config.NodeRPCHTTPAddress + + // TODO fix passing this random key to next enclave and not generating it here every time we start the wallet extension + randomKey, err := wecommon.GenerateRandomKey() + if err != nil { + logger.Crit("unable to generate random encryption key", log.ErrKey, err) + os.Exit(1) + } // start the database - databaseStorage, err := storage.New(config.DBType, config.DBConnectionURL, config.DBPathOverride) + databaseStorage, err := storage.New(config.DBType, config.DBConnectionURL, config.DBPathOverride, randomKey) if err != nil { logger.Crit("unable to create database to store viewing keys ", log.ErrKey, err) os.Exit(1) From b8437af15de4af04402c647a48bb37184032afbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Wed, 23 Oct 2024 08:43:01 +0200 Subject: [PATCH 08/18] fix deployment script --- .github/workflows/manual-deploy-obscuro-gateway.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual-deploy-obscuro-gateway.yml b/.github/workflows/manual-deploy-obscuro-gateway.yml index 4fee72c3c5..b04124670a 100644 --- a/.github/workflows/manual-deploy-obscuro-gateway.yml +++ b/.github/workflows/manual-deploy-obscuro-gateway.yml @@ -296,5 +296,5 @@ jobs: "${{ env.DOCKER_BUILD_TAG_GATEWAY }}" \ ego run /home/ten/go-ten/tools/walletextension/main/main \ -host=0.0.0.0 -port=80 -portWS=81 -nodeHost="${{ env.L2_RPC_URL_VALIDATOR }}" -verbose=true \ - -logPath=sys_out -dbType=mariaDB -dbConnectionURL="obscurouser:${{ secrets.OBSCURO_GATEWAY_MARIADB_USER_PWD }}@tcp(obscurogateway-mariadb-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com:3306)/ogdb" \ + -logPath=sys_out -dbType=cosmosDB -dbConnectionURL="${{ secrets.COSMOS_DB_CONNECTION_STRING }}" \ -rateLimitUserComputeTime="${{ env.GATEWAY_RATE_LIMIT_USER_COMPUTE_TIME }}" -rateLimitWindow="${{ env.GATEWAY_RATE_LIMIT_WINDOW }}" -maxConcurrentRequestsPerUser="${{ env.GATEWAY_MAX_CONCURRENT_REQUESTS_PER_USER }}" ' From 147046736375a482cd027763e41c72aa5277494b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Wed, 23 Oct 2024 09:25:13 +0200 Subject: [PATCH 09/18] remove migration files from the enclave files --- tools/walletextension/main/enclave.json | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tools/walletextension/main/enclave.json b/tools/walletextension/main/enclave.json index 24304b82c5..7ce2a41b55 100644 --- a/tools/walletextension/main/enclave.json +++ b/tools/walletextension/main/enclave.json @@ -12,18 +12,5 @@ "value": "false" } ], - "files": [ - { - "source": "../storage/database/mariadb/001_init.sql", - "target": "/home/ten/go-ten/tools/walletextension/storage/database/mariadb/001_init.sql" - }, - { - "source": "../storage/database/mariadb/002_store_incoming_txs.sql", - "target": "/home/ten/go-ten/tools/walletextension/storage/database/mariadb/002_store_incoming_txs.sql" - }, - { - "source": "../storage/database/mariadb/003_add_signature_type.sql", - "target": "/home/ten/go-ten/tools/walletextension/storage/database/mariadb/003_add_signature_type.sql" - } - ] + "files": [] } \ No newline at end of file From 18c034f615dfc247095dd6bef3ec7629ed9988d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Wed, 23 Oct 2024 09:50:24 +0200 Subject: [PATCH 10/18] add certificates to the enclave to be able to check cosmosDB cert --- tools/walletextension/enclave.Dockerfile | 6 ++++++ tools/walletextension/main/enclave.json | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/walletextension/enclave.Dockerfile b/tools/walletextension/enclave.Dockerfile index e0a8b77504..33673df659 100644 --- a/tools/walletextension/enclave.Dockerfile +++ b/tools/walletextension/enclave.Dockerfile @@ -10,6 +10,12 @@ FROM ghcr.io/edgelesssys/ego-dev:v1.5.3 AS build-base +# Install ca-certificates package and update it +RUN apt-get update && apt-get install -y \ + ca-certificates \ + && update-ca-certificates + + # setup container data structure RUN mkdir -p /home/ten/go-ten diff --git a/tools/walletextension/main/enclave.json b/tools/walletextension/main/enclave.json index 7ce2a41b55..c4bf5eb33b 100644 --- a/tools/walletextension/main/enclave.json +++ b/tools/walletextension/main/enclave.json @@ -12,5 +12,10 @@ "value": "false" } ], - "files": [] + "files": [ + { + "source": "/etc/ssl/certs/ca-certificates.crt", + "target": "/etc/ssl/certs/ca-certificates.crt" + } + ] } \ No newline at end of file From a2f97d3d8f147357b796f295705b8a67d8ce3e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Wed, 23 Oct 2024 10:51:12 +0200 Subject: [PATCH 11/18] run storage tests on sqlite by default --- tools/walletextension/storage/storage_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index 058a1894ff..4c61a065a3 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -26,8 +26,8 @@ func TestGatewayStorage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - //storage, err := New("sqlite", "", "", randomKey) - storage, err := New("cosmosDB", "", "", randomKey) + storage, err := New("sqlite", "", "", randomKey) + //storage, err := New("cosmosDB", "", "", randomKey) require.NoError(t, err) test(storage, t) From e0a07c9739d3495a2215164cba52b6f910df4b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Fri, 25 Oct 2024 18:06:35 +0200 Subject: [PATCH 12/18] don't encode the data --- .../storage/database/cosmosdb/cosmosdb.go | 24 ++++--------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index 4ce633ac27..8c5ba3c175 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -2,7 +2,6 @@ package cosmosdb import ( "context" - "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -44,14 +43,13 @@ type CosmosDB struct { // The 'Data' field contains the base64-encoded encrypted user data type EncryptedDocument struct { ID string `json:"id"` - Data string `json:"data"` + Data []byte `json:"data"` } // Constants for the CosmosDB database and container names const ( DATABASE_NAME = "gatewayDB" USERS_CONTAINER_NAME = "users" - PARTITION_KEY = "/id" ) func NewCosmosDB(connectionString string, encryptionKey []byte) (*CosmosDB, error) { @@ -102,7 +100,6 @@ func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { if err != nil { return fmt.Errorf("failed to encrypt user data: %w", err) } - encryptedData := base64.StdEncoding.EncodeToString(ciphertext) key := c.encryptor.HashWithHMAC(userID) keyString := hex.EncodeToString(key) @@ -110,7 +107,7 @@ func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { // Create an EncryptedDocument struct to store in CosmosDB doc := EncryptedDocument{ ID: keyString, - Data: encryptedData, + Data: ciphertext, } docJSON, err := json.Marshal(doc) @@ -157,12 +154,7 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] return fmt.Errorf("failed to unmarshal document: %w", err) } - encryptedData, err := base64.StdEncoding.DecodeString(doc.Data) - if err != nil { - return fmt.Errorf("failed to decode base64 data: %w", err) - } - - data, err := c.encryptor.Decrypt(encryptedData) + data, err := c.encryptor.Decrypt(doc.Data) if err != nil { return fmt.Errorf("failed to decrypt data: %w", err) } @@ -190,10 +182,9 @@ func (c *CosmosDB) AddAccount(userID []byte, accountAddress []byte, signature [] if err != nil { return fmt.Errorf("failed to encrypt updated user data: %w", err) } - encryptedDataStr := base64.StdEncoding.EncodeToString(ciphertext) // Update the document - doc.Data = encryptedDataStr + doc.Data = ciphertext docJSON, err := json.Marshal(doc) if err != nil { @@ -225,12 +216,7 @@ func (c *CosmosDB) GetUser(userID []byte) (common.GWUserDB, error) { return common.GWUserDB{}, fmt.Errorf("failed to unmarshal document: %w", err) } - encryptedData, err := base64.StdEncoding.DecodeString(doc.Data) - if err != nil { - return common.GWUserDB{}, fmt.Errorf("failed to decode base64 data: %w", err) - } - - data, err := c.encryptor.Decrypt(encryptedData) + data, err := c.encryptor.Decrypt(doc.Data) if err != nil { return common.GWUserDB{}, fmt.Errorf("failed to decrypt data: %w", err) } From 421be9051d8df49af1a06ba5bb156c3f60c02f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Sat, 26 Oct 2024 08:13:15 +0200 Subject: [PATCH 13/18] remove ID field, because it is no neeed --- tools/walletextension/common/db_types.go | 3 +-- tools/walletextension/storage/database/cosmosdb/cosmosdb.go | 1 - tools/walletextension/storage/database/sqlite/sqlite.go | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tools/walletextension/common/db_types.go b/tools/walletextension/common/db_types.go index 709662c37e..04993bbd75 100644 --- a/tools/walletextension/common/db_types.go +++ b/tools/walletextension/common/db_types.go @@ -1,10 +1,9 @@ package common type GWUserDB struct { - ID string `json:"id"` // Required by CosmosDB UserId []byte `json:"userId"` PrivateKey []byte `json:"privateKey"` - Accounts []GWAccountDB `json:"accounts"` // List of Accounts + Accounts []GWAccountDB `json:"accounts"` } type GWAccountDB struct { diff --git a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go index 8c5ba3c175..e8296f2523 100644 --- a/tools/walletextension/storage/database/cosmosdb/cosmosdb.go +++ b/tools/walletextension/storage/database/cosmosdb/cosmosdb.go @@ -86,7 +86,6 @@ func NewCosmosDB(connectionString string, encryptionKey []byte) (*CosmosDB, erro func (c *CosmosDB) AddUser(userID []byte, privateKey []byte) error { user := common.GWUserDB{ - ID: hex.EncodeToString(userID), UserId: userID, PrivateKey: privateKey, Accounts: []common.GWAccountDB{}, diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index 93b0519f0a..f08086bf5b 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -61,7 +61,6 @@ func NewSqliteDatabase(dbPath string) (*Database, error) { func (s *Database) AddUser(userID []byte, privateKey []byte) error { user := common.GWUserDB{ - ID: string(userID), UserId: userID, PrivateKey: privateKey, Accounts: []common.GWAccountDB{}, @@ -77,7 +76,7 @@ func (s *Database) AddUser(userID []byte, privateKey []byte) error { } defer stmt.Close() - _, err = stmt.Exec(user.ID, string(userJSON)) + _, err = stmt.Exec(string(user.UserId), string(userJSON)) if err != nil { return err } From 9f8911a26216b9faa5171121d765e3c2e6bc4bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 28 Oct 2024 10:06:41 +0100 Subject: [PATCH 14/18] wip --- tools/walletextension/walletextension_container.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/walletextension/walletextension_container.go b/tools/walletextension/walletextension_container.go index d21832bc8f..f838114b85 100644 --- a/tools/walletextension/walletextension_container.go +++ b/tools/walletextension/walletextension_container.go @@ -2,6 +2,7 @@ package walletextension import ( "os" + "path/filepath" "time" "github.com/ten-protocol/go-ten/go/common/subscription" @@ -15,11 +16,16 @@ import ( gethlog "github.com/ethereum/go-ethereum/log" "github.com/ten-protocol/go-ten/go/common/log" "github.com/ten-protocol/go-ten/go/common/stopcontrol" + "github.com/ten-protocol/go-ten/go/enclave/egoutils" gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc" wecommon "github.com/ten-protocol/go-ten/tools/walletextension/common" "github.com/ten-protocol/go-ten/tools/walletextension/storage" ) +var ( + encryptionKeyFilepath = filepath.Join("/data", "encryption_key.json") +) + type Container struct { stopControl *stopcontrol.StopControl logger gethlog.Logger @@ -39,6 +45,14 @@ func NewContainerFromConfig(config wecommon.Config, logger gethlog.Logger) *Cont logger.Crit("unable to generate random encryption key", log.ErrKey, err) os.Exit(1) } + + // seal the encryption key + err = egoutils.SealAndPersist(string(randomKey), encryptionKeyFilepath, true) + if err != nil { + logger.Error("unable to seal and persist encryption key", log.ErrKey, err) + // os.Exit(1) + } + // start the database databaseStorage, err := storage.New(config.DBType, config.DBConnectionURL, config.DBPathOverride, randomKey) if err != nil { From 211b87c57d9290543d5d58576d9d3d0c5cba8543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 28 Oct 2024 12:07:11 +0100 Subject: [PATCH 15/18] wip --- .../walletextension_container.go | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tools/walletextension/walletextension_container.go b/tools/walletextension/walletextension_container.go index f838114b85..0f95ebab17 100644 --- a/tools/walletextension/walletextension_container.go +++ b/tools/walletextension/walletextension_container.go @@ -6,6 +6,7 @@ import ( "time" "github.com/ten-protocol/go-ten/go/common/subscription" + "github.com/ten-protocol/go-ten/go/enclave/core/egoutils" "github.com/ten-protocol/go-ten/tools/walletextension/httpapi" @@ -16,16 +17,11 @@ import ( gethlog "github.com/ethereum/go-ethereum/log" "github.com/ten-protocol/go-ten/go/common/log" "github.com/ten-protocol/go-ten/go/common/stopcontrol" - "github.com/ten-protocol/go-ten/go/enclave/egoutils" gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc" wecommon "github.com/ten-protocol/go-ten/tools/walletextension/common" "github.com/ten-protocol/go-ten/tools/walletextension/storage" ) -var ( - encryptionKeyFilepath = filepath.Join("/data", "encryption_key.json") -) - type Container struct { stopControl *stopcontrol.StopControl logger gethlog.Logger @@ -46,11 +42,17 @@ func NewContainerFromConfig(config wecommon.Config, logger gethlog.Logger) *Cont os.Exit(1) } - // seal the encryption key - err = egoutils.SealAndPersist(string(randomKey), encryptionKeyFilepath, true) - if err != nil { - logger.Error("unable to seal and persist encryption key", log.ErrKey, err) - // os.Exit(1) + encryptionKeyFilepath := filepath.Join(".", "encryption_key.json") + + debug := true + + if !debug { + // seal the encryption key + err = egoutils.SealAndPersist(string(randomKey), encryptionKeyFilepath, debug) + if err != nil { + logger.Error("unable to seal and persist encryption key", log.ErrKey, err) + // os.Exit(1) + } } // start the database From a82f8d4863f40dc876682292f9921594ac1473ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 28 Oct 2024 14:33:11 +0100 Subject: [PATCH 16/18] wip commit --- tools/walletextension/common/config.go | 1 + tools/walletextension/main/cli.go | 6 ++++ .../walletextension_container.go | 36 ++++++++++++------- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/tools/walletextension/common/config.go b/tools/walletextension/common/config.go index 26f43cf459..799d92f36b 100644 --- a/tools/walletextension/common/config.go +++ b/tools/walletextension/common/config.go @@ -19,4 +19,5 @@ type Config struct { RateLimitUserComputeTime time.Duration RateLimitWindow time.Duration RateLimitMaxConcurrentRequests int + Debug bool } diff --git a/tools/walletextension/main/cli.go b/tools/walletextension/main/cli.go index e1c4befd4d..1d604af99f 100644 --- a/tools/walletextension/main/cli.go +++ b/tools/walletextension/main/cli.go @@ -72,6 +72,10 @@ const ( rateLimitMaxConcurrentRequestsName = "maxConcurrentRequestsPerUser" rateLimitMaxConcurrentRequestsDefault = 3 rateLimitMaxConcurrentRequestsUsage = "Number of concurrent requests allowed per user. Default: 3" + + debugFlagName = "debug" + debugFlagDefault = false + debugFlagUsage = "Flag to enable debug mode" ) func parseCLIArgs() wecommon.Config { @@ -91,6 +95,7 @@ func parseCLIArgs() wecommon.Config { rateLimitUserComputeTime := flag.Duration(rateLimitUserComputeTimeName, rateLimitUserComputeTimeDefault, rateLimitUserComputeTimeUsage) rateLimitWindow := flag.Duration(rateLimitWindowName, rateLimitWindowDefault, rateLimitWindowUsage) rateLimitMaxConcurrentRequests := flag.Int(rateLimitMaxConcurrentRequestsName, rateLimitMaxConcurrentRequestsDefault, rateLimitMaxConcurrentRequestsUsage) + debugFlag := flag.Bool(debugFlagName, debugFlagDefault, debugFlagUsage) flag.Parse() return wecommon.Config{ @@ -109,5 +114,6 @@ func parseCLIArgs() wecommon.Config { RateLimitUserComputeTime: *rateLimitUserComputeTime, RateLimitWindow: *rateLimitWindow, RateLimitMaxConcurrentRequests: *rateLimitMaxConcurrentRequests, + Debug: *debugFlag, } } diff --git a/tools/walletextension/walletextension_container.go b/tools/walletextension/walletextension_container.go index 0f95ebab17..aa7318da22 100644 --- a/tools/walletextension/walletextension_container.go +++ b/tools/walletextension/walletextension_container.go @@ -35,28 +35,38 @@ func NewContainerFromConfig(config wecommon.Config, logger gethlog.Logger) *Cont hostRPCBindAddrWS := wecommon.WSProtocol + config.NodeRPCWebsocketAddress hostRPCBindAddrHTTP := wecommon.HTTPProtocol + config.NodeRPCHTTPAddress - // TODO fix passing this random key to next enclave and not generating it here every time we start the wallet extension - randomKey, err := wecommon.GenerateRandomKey() - if err != nil { - logger.Crit("unable to generate random encryption key", log.ErrKey, err) - os.Exit(1) - } - + // Database encryption key handling + // First we try to unseal the encryption key from the file + // If we fail to unseal the key, we generate a new one and seal it to the file + // TODO: We should have a mechanism to get the key from an enclave that already runs (part of the next PR) + // TODO: Move this to a separate file along with key exchange logic (part of the next PR) encryptionKeyFilepath := filepath.Join(".", "encryption_key.json") - debug := true + // try to read and unseal the encryption key + encryptionKey, err := egoutils.ReadAndUnseal(encryptionKeyFilepath) + if err != nil { + // we were not able to unseal the key, generate a new one + logger.Info("unable to read and unseal encryption key", log.ErrKey, err) + encryptionKey, err = wecommon.GenerateRandomKey() + if err != nil { + logger.Crit("unable to generate random encryption key", log.ErrKey, err) + os.Exit(1) + } + logger.Info("generated new encryption key", log.ErrKey, err) + } - if !debug { - // seal the encryption key - err = egoutils.SealAndPersist(string(randomKey), encryptionKeyFilepath, debug) + // try to seal the encryption key to the file + // debug mode is used for testing purposes when we don't run inside an enclave, but we still want to test gateway functionality + if !config.Debug { + err = egoutils.SealAndPersist(string(encryptionKey), encryptionKeyFilepath, true) if err != nil { logger.Error("unable to seal and persist encryption key", log.ErrKey, err) // os.Exit(1) } } - // start the database - databaseStorage, err := storage.New(config.DBType, config.DBConnectionURL, config.DBPathOverride, randomKey) + // start the database with the encryption key + databaseStorage, err := storage.New(config.DBType, config.DBConnectionURL, config.DBPathOverride, encryptionKey) if err != nil { logger.Crit("unable to create database to store viewing keys ", log.ErrKey, err) os.Exit(1) From 23aa123cbf04e5350921dbaec7f1ae50a396bff2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 4 Nov 2024 11:20:50 +0100 Subject: [PATCH 17/18] PR review fixes --- .../walletextension/encryption/encryption.go | 10 +++++++ tools/walletextension/storage/storage_test.go | 2 +- .../walletextension_container.go | 29 +++---------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/tools/walletextension/encryption/encryption.go b/tools/walletextension/encryption/encryption.go index 31a41fd1fd..c4e8de7c22 100644 --- a/tools/walletextension/encryption/encryption.go +++ b/tools/walletextension/encryption/encryption.go @@ -11,6 +11,16 @@ import ( "io" ) +// Encryptor provides AES-GCM encryption/decryption with the following characteristics: +// - Uses AES-256-GCM (Galois/Counter Mode) with a 32-byte key +// - Generates a random 12-byte nonce for each encryption operation using crypto/rand +// - The nonce is prepended to the ciphertext output from Encrypt() and is generated +// using crypto/rand.Reader for cryptographically secure random values +// +// Additionally provides HMAC-SHA256 hashing functionality: +// - Uses the same 32-byte key as the encryption operations +// - Generates a 32-byte (256-bit) message authentication code +// - Suitable for creating secure message digests and verifying data integrity type Encryptor struct { gcm cipher.AEAD key []byte diff --git a/tools/walletextension/storage/storage_test.go b/tools/walletextension/storage/storage_test.go index 4c61a065a3..2053fc37c5 100644 --- a/tools/walletextension/storage/storage_test.go +++ b/tools/walletextension/storage/storage_test.go @@ -27,7 +27,7 @@ func TestGatewayStorage(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { storage, err := New("sqlite", "", "", randomKey) - //storage, err := New("cosmosDB", "", "", randomKey) + // storage, err := New("cosmosDB", "", "", randomKey) require.NoError(t, err) test(storage, t) diff --git a/tools/walletextension/walletextension_container.go b/tools/walletextension/walletextension_container.go index aa7318da22..a306e3a99a 100644 --- a/tools/walletextension/walletextension_container.go +++ b/tools/walletextension/walletextension_container.go @@ -2,11 +2,9 @@ package walletextension import ( "os" - "path/filepath" "time" "github.com/ten-protocol/go-ten/go/common/subscription" - "github.com/ten-protocol/go-ten/go/enclave/core/egoutils" "github.com/ten-protocol/go-ten/tools/walletextension/httpapi" @@ -36,33 +34,14 @@ func NewContainerFromConfig(config wecommon.Config, logger gethlog.Logger) *Cont hostRPCBindAddrHTTP := wecommon.HTTPProtocol + config.NodeRPCHTTPAddress // Database encryption key handling - // First we try to unseal the encryption key from the file - // If we fail to unseal the key, we generate a new one and seal it to the file + // TODO: Check if encryption key is already sealed and unseal it and generate new one if not (part of the next PR) // TODO: We should have a mechanism to get the key from an enclave that already runs (part of the next PR) // TODO: Move this to a separate file along with key exchange logic (part of the next PR) - encryptionKeyFilepath := filepath.Join(".", "encryption_key.json") - // try to read and unseal the encryption key - encryptionKey, err := egoutils.ReadAndUnseal(encryptionKeyFilepath) + encryptionKey, err := wecommon.GenerateRandomKey() if err != nil { - // we were not able to unseal the key, generate a new one - logger.Info("unable to read and unseal encryption key", log.ErrKey, err) - encryptionKey, err = wecommon.GenerateRandomKey() - if err != nil { - logger.Crit("unable to generate random encryption key", log.ErrKey, err) - os.Exit(1) - } - logger.Info("generated new encryption key", log.ErrKey, err) - } - - // try to seal the encryption key to the file - // debug mode is used for testing purposes when we don't run inside an enclave, but we still want to test gateway functionality - if !config.Debug { - err = egoutils.SealAndPersist(string(encryptionKey), encryptionKeyFilepath, true) - if err != nil { - logger.Error("unable to seal and persist encryption key", log.ErrKey, err) - // os.Exit(1) - } + logger.Crit("unable to generate random encryption key", log.ErrKey, err) + os.Exit(1) } // start the database with the encryption key From fda0871d1e6c161cc491f51b238c88d82c32a5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=BDiga=20Kokelj?= Date: Mon, 4 Nov 2024 11:31:02 +0100 Subject: [PATCH 18/18] storage with cache --- .../storage/storage_with_cache.go | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tools/walletextension/storage/storage_with_cache.go diff --git a/tools/walletextension/storage/storage_with_cache.go b/tools/walletextension/storage/storage_with_cache.go new file mode 100644 index 0000000000..7c83d75423 --- /dev/null +++ b/tools/walletextension/storage/storage_with_cache.go @@ -0,0 +1,90 @@ +package storage + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ten-protocol/go-ten/go/common/viewingkey" + "github.com/ten-protocol/go-ten/tools/walletextension/cache" + wecommon "github.com/ten-protocol/go-ten/tools/walletextension/common" +) + +// StorageWithCache implements the Storage interface with caching +type StorageWithCache struct { + storage Storage + cache cache.Cache + mu sync.RWMutex +} + +// NewStorageWithCache creates a new StorageWithCache instance +func NewStorageWithCache(storage Storage, logger log.Logger) (*StorageWithCache, error) { + c, err := cache.NewCache(logger) + if err != nil { + return nil, err + } + return &StorageWithCache{ + storage: storage, + cache: c, + }, nil +} + +// AddUser adds a new user and invalidates the cache for the userID +func (s *StorageWithCache) AddUser(userID []byte, privateKey []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + err := s.storage.AddUser(userID, privateKey) + if err != nil { + return err + } + s.cache.Remove(userID) + return nil +} + +// DeleteUser deletes a user and invalidates the cache for the userID +func (s *StorageWithCache) DeleteUser(userID []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + err := s.storage.DeleteUser(userID) + if err != nil { + return err + } + s.cache.Remove(userID) + return nil +} + +// AddAccount adds an account to a user and invalidates the cache for the userID +func (s *StorageWithCache) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { + s.mu.Lock() + defer s.mu.Unlock() + err := s.storage.AddAccount(userID, accountAddress, signature, signatureType) + if err != nil { + return err + } + s.cache.Remove(userID) + return nil +} + +// GetUser retrieves a user from the cache or underlying storage +func (s *StorageWithCache) GetUser(userID []byte) (wecommon.GWUserDB, error) { + s.mu.RLock() + // Check if the user is in the cache + if cachedUser, found := s.cache.Get(userID); found { + s.mu.RUnlock() + return cachedUser.(wecommon.GWUserDB), nil + } + s.mu.RUnlock() + + // If not in cache, retrieve from storage + user, err := s.storage.GetUser(userID) + if err != nil { + return wecommon.GWUserDB{}, err + } + + // Store the retrieved user in the cache + s.mu.Lock() + s.cache.Set(userID, user, 5*time.Minute) + s.mu.Unlock() + + return user, nil +}