Fix personal transactions RPC request (#1923)

go/common/custom_query_types.go

@@ -0,0 +1,29 @@
+package common
+
+import "github.com/ethereum/go-ethereum/common"
+
+// CustomQueries are Ten-specific queries that are not supported by the Ethereum RPC API but that we wish to support
+// through the same interface.
+//
+// We currently use the eth_getStorageAt method to route these queries through the Ethereum RPC API.
+//
+// In order to match the eth_getStorageAt method signature, we require that all custom queries use an incrementing "address"
+// to specify the method we are calling (e.g. 0x000...001 is getUserID, 0x000...002 is listPrivateTransactions).
+//
+// The signature is: eth_getStorageAt(method, params, nil) where:
+// - method is the address of the custom query as an address (e.g. 0x000...001)
+// - params is a JSON string with the parameters for the query (this complies with the eth_getStorageAt method signature since position gets encoded as a hex string)
+//
+// NOTE: Private custom queries must also include "address" as a top-level field in the params json object to indicate
+// the account the query is being made for.
+
+// CustomQuery methods
+const (
+	UserIDRequestCQMethod           = "0x0000000000000000000000000000000000000001"
+	ListPrivateTransactionsCQMethod = "0x0000000000000000000000000000000000000002"
+)
+
+type ListPrivateTransactionsQueryParams struct {
+	Address    common.Address  `json:"address"`
+	Pagination QueryPagination `json:"pagination"`
+}

go/common/gethencoding/geth_encoding.go

@@ -2,6 +2,7 @@ package gethencoding
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"math/big"
@@ -344,18 +345,41 @@ func (enc *gethEncodingServiceImpl) CreateEthBlockFromBatch(ctx context.Context,
 	return (*types.Block)(unsafe.Pointer(&lb)), nil
 }
 
-func ExtractPrivateCustomQuery(_ interface{}, query interface{}) (*common.PrivateCustomQueryListTransactions, error) {
-	// Convert the map to a JSON string
-	jsonData, err := json.Marshal(query)
-	if err != nil {
-		return nil, err
+// ExtractPrivateCustomQuery is designed to support a wide range of custom Ten queries.
+// The first parameter here is the method name, which is used to determine the query type.
+// The second parameter is the query parameters.
+func ExtractPrivateCustomQuery(methodName any, queryParams any) (*common.ListPrivateTransactionsQueryParams, error) {
+	// we expect the first parameter to be a string
+	methodNameStr, ok := methodName.(string)
+	if !ok {
+		return nil, fmt.Errorf("expected methodName as string but was type %T", methodName)
+	}
+	// currently we only have to support this custom query method in the enclave
+	if methodNameStr != common.ListPrivateTransactionsCQMethod {
+		return nil, fmt.Errorf("unsupported method %s", methodNameStr)
 	}
 
-	var result common.PrivateCustomQueryListTransactions
-	err = json.Unmarshal(jsonData, &result)
+	// we expect second param to be a json string
+	queryParamsStr, ok := queryParams.(string)
+	if !ok {
+		return nil, fmt.Errorf("expected queryParams as string but was type %T", queryParams)
+	}
+
+	var privateQueryParams common.ListPrivateTransactionsQueryParams
+	err := json.Unmarshal([]byte(queryParamsStr), &privateQueryParams)
 	if err != nil {
-		return nil, err
+		// if it fails, check if the string was base64 encoded
+		bytesStr, err64 := base64.StdEncoding.DecodeString(queryParamsStr)
+		if err64 != nil {
+			// was not base64 encoded, give up
+			return nil, fmt.Errorf("unable to unmarshal params string: %w", err)
+		}
+		// was base64 encoded, try to unmarshal
+		err = json.Unmarshal(bytesStr, &privateQueryParams)
+		if err != nil {
+			return nil, fmt.Errorf("unable to unmarshal params string: %w", err)
+		}
 	}
 
-	return &result, nil
+	return &privateQueryParams, nil
 }

go/common/query_types.go

@@ -10,7 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-type PrivateQueryResponse struct {
+type PrivateTransactionsQueryResponse struct {
 	Receipts types.Receipts
 	Total    uint64
 }
@@ -110,11 +110,6 @@ func (p *QueryPagination) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-type PrivateCustomQueryListTransactions struct {
-	Address    common.Address  `json:"address"`
-	Pagination QueryPagination `json:"pagination"`
-}
-
 type ObscuroNetworkInfo struct {
 	ManagementContractAddress common.Address
 	L1StartHash               common.Hash

go/enclave/enclave.go

@@ -855,10 +855,12 @@ func (e *enclaveImpl) GetTotalContractCount(ctx context.Context) (*big.Int, comm
 	return e.storage.GetContractCount(ctx)
 }
 
+// GetCustomQuery is a generic query method for queries that don't match the eth API.
+// todo: get rid of this method and use specific methods for each query (e.g. GetPersonalTransactions)
 func (e *enclaveImpl) GetCustomQuery(ctx context.Context, encryptedParams common.EncryptedParamsGetStorageAt) (*responses.PrivateQueryResponse, common.SystemError) {
 	// ensure the enclave is running
 	if e.stopControl.IsStopping() {
-		return nil, responses.ToInternalError(fmt.Errorf("requested GetReceiptsByAddress with the enclave stopping"))
+		return nil, responses.ToInternalError(fmt.Errorf("requested GetPrivateTransactions with the enclave stopping"))
 	}
 
 	return rpc.WithVKEncryption(ctx, e.rpcEncryptionManager, encryptedParams, rpc.GetCustomQueryValidate, rpc.GetCustomQueryExecute)

go/enclave/rpc/GetCustomQuery.go

@@ -7,10 +7,10 @@ import (
 	"github.com/ten-protocol/go-ten/go/common/gethencoding"
 )
 
-func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.PrivateCustomQueryListTransactions, common.PrivateQueryResponse], _ *EncryptionManager) error {
+func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], _ *EncryptionManager) error {
 	// Parameters are [PrivateCustomQueryHeader, PrivateCustomQueryArgs, null]
 	if len(reqParams) != 3 {
-		builder.Err = fmt.Errorf("unexpected number of parameters")
+		builder.Err = fmt.Errorf("unexpected number of parameters (expected %d, got %d)", 3, len(reqParams))
 		return nil
 	}
 
@@ -19,29 +19,30 @@ func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.Private
 		builder.Err = fmt.Errorf("unable to extract query - %w", err)
 		return nil
 	}
-	builder.From = &privateCustomQuery.Address
+	addr := privateCustomQuery.Address
+	builder.From = &addr
 	builder.Param = privateCustomQuery
 	return nil
 }
 
-func GetCustomQueryExecute(builder *CallBuilder[common.PrivateCustomQueryListTransactions, common.PrivateQueryResponse], rpc *EncryptionManager) error {
+func GetCustomQueryExecute(builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], rpc *EncryptionManager) error {
 	err := authenticateFrom(builder.VK, builder.From)
 	if err != nil {
 		builder.Err = err
 		return nil //nolint:nilerr
 	}
-
-	encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &builder.Param.Address, &builder.Param.Pagination)
+	addr := builder.Param.Address
+	encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &addr, &builder.Param.Pagination)
 	if err != nil {
 		return fmt.Errorf("GetTransactionsPerAddress - %w", err)
 	}
 
-	receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &builder.Param.Address)
+	receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &addr)
 	if err != nil {
 		return fmt.Errorf("CountTransactionsPerAddress - %w", err)
 	}
 
-	builder.ReturnValue = &common.PrivateQueryResponse{
+	builder.ReturnValue = &common.PrivateTransactionsQueryResponse{
 		Receipts: encryptReceipts,
 		Total:    receiptsCount,
 	}

go/enclave/storage/enclavedb/batch.go

@@ -428,11 +428,14 @@ func BatchWasExecuted(ctx context.Context, db *sql.DB, hash common.L2BatchHash)
 }
 
 func GetTransactionsPerAddress(ctx context.Context, db *sql.DB, config *params.ChainConfig, address *gethcommon.Address, pagination *common.QueryPagination) (types.Receipts, error) {
-	return selectReceipts(ctx, db, config, "where tx.sender_address = ? ORDER BY height DESC LIMIT ? OFFSET ? ", address.Bytes(), pagination.Size, pagination.Offset)
+	return selectReceipts(ctx, db, config, "join externally_owned_account eoa on tx.sender_address = eoa.id where eoa.address = ? ORDER BY height DESC LIMIT ? OFFSET ? ", address.Bytes(), pagination.Size, pagination.Offset)
 }
 
 func CountTransactionsPerAddress(ctx context.Context, db *sql.DB, address *gethcommon.Address) (uint64, error) {
-	row := db.QueryRowContext(ctx, "select count(1) from receipt join tx on tx.id=receipt.tx join batch on batch.sequence=receipt.batch "+" where tx.sender_address = ?", address.Bytes())
+	row := db.QueryRowContext(ctx, "select count(1) from receipt "+
+		"join tx on tx.id=receipt.tx "+
+		"join externally_owned_account eoa on eoa.id = tx.sender_address "+
+		"where eoa.address = ?", address.Bytes())
 
 	var count uint64
 	err := row.Scan(&count)

go/obsclient/authclient.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
@@ -249,14 +250,21 @@ func (ac *AuthObsClient) EstimateGasAndGasPrice(txData types.TxData) types.TxDat
 	}
 }
 
-// GetReceiptsByAddress retrieves the receipts for the account registered on this client (due to obscuro privacy restrictions,
-// balance cannot be requested for other accounts)
-func (ac *AuthObsClient) GetReceiptsByAddress(ctx context.Context, address *gethcommon.Address) (types.Receipts, error) {
-	var result types.Receipts
-	err := ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, address, nil, nil)
+// GetPrivateTransactions retrieves the receipts for the specified account (must be registered on this client), returns requested range of receipts and the total number of receipts for that acc
+func (ac *AuthObsClient) GetPrivateTransactions(ctx context.Context, address *gethcommon.Address, pagination common.QueryPagination) (types.Receipts, uint64, error) {
+	queryParam := &common.ListPrivateTransactionsQueryParams{
+		Address:    *address,
+		Pagination: pagination,
+	}
+	queryParamStr, err := json.Marshal(queryParam)
 	if err != nil {
-		return nil, err
+		return nil, 0, fmt.Errorf("unable to marshal query params - %w", err)
+	}
+	var result common.PrivateTransactionsQueryResponse
+	err = ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, common.ListPrivateTransactionsCQMethod, string(queryParamStr), nil)
+	if err != nil {
+		return nil, 0, err
 	}
 
-	return result, nil
+	return result.Receipts, result.Total, nil
 }

go/rpc/encrypted_client.go

@@ -166,6 +166,12 @@ func (c *EncRPCClient) executeSensitiveCall(ctx context.Context, result interfac
 	// and never error.
 	resultBytes, _ := decodedResult.MarshalJSON()
 
+	// if expected result type is bytes, we return the bytes
+	if _, ok := result.(*[]byte); ok {
+		*result.(*[]byte) = resultBytes
+		return nil
+	}
+
 	// We put the raw json in the passed result object.
 	// This works for structs, strings, integers and interface types.
 	err = json.Unmarshal(resultBytes, result)

integration/networktest/env/network_setup.go

@@ -57,8 +57,8 @@ func DevTestnet(opts ...TestnetEnvOption) networktest.Environment {
 // LongRunningLocalNetwork is a local network, the l1WSURL is optional (can be empty string), only required if testing L1 interactions
 func LongRunningLocalNetwork(l1WSURL string) networktest.Environment {
 	connector := newTestnetConnectorWithFaucetAccount(
-		"ws://127.0.0.1:26900",
-		[]string{"ws://127.0.0.1:26901"},
+		"ws://127.0.0.1:17900",
+		[]string{"ws://127.0.0.1:17901"},
 		genesis.TestnetPrefundedPK,
 		l1WSURL,
 		"",

integration/networktest/tests/helpful/spin_up_local_network_test.go

@@ -29,6 +29,8 @@ const (
 	_sepoliaValidator1PK = "<pk>" // account 0x<acc>
 )
 
+// Spins up a local network with a gateway, with all processes debuggable. The network will run until the test is stopped.
+// Note: If you want to access the gateway frontend you need to `npm run build` its frontend with NEXT_PUBLIC_API_GATEWAY_URL=http://localhost:11180
 func TestRunLocalNetwork(t *testing.T) {
 	networktest.TestOnlyRunsInIDE(t)
 	networktest.EnsureTestLogsSetUp("local-geth-network")

integration/networktest/tests/tenscan/tenscan_rpc_test.go

@@ -1,14 +1,21 @@
 package tenscan
 
 import (
+	"context"
+	"fmt"
+	"math/big"
 	"testing"
 
+	"github.com/ten-protocol/go-ten/go/common"
 	"github.com/ten-protocol/go-ten/integration/networktest"
 	"github.com/ten-protocol/go-ten/integration/networktest/actions"
 	"github.com/ten-protocol/go-ten/integration/networktest/actions/publicdata"
 	"github.com/ten-protocol/go-ten/integration/networktest/env"
+	"github.com/ten-protocol/go-ten/integration/simulation/devnetwork"
 )
 
+var _transferAmount = big.NewInt(100_000_000)
+
 // Verify and debug the RPC endpoints that Tenscan relies on for data in various environments
 
 func TestRPC(t *testing.T) {
@@ -22,3 +29,62 @@ func TestRPC(t *testing.T) {
 		),
 	)
 }
+
+// Test the personal transactions endpoint in various environments (it uses getStorageAt so it can run through MM etc.)
+// 1. create user
+// 2. send some transactions
+// 3. verify transactions are returned by the personal transactions endpoint that tenscan uses
+func TestPersonalTransactions(t *testing.T) {
+	networktest.TestOnlyRunsInIDE(t)
+	networktest.Run(
+		"tenscan-personal-transactions",
+		t,
+		env.LocalDevNetwork(devnetwork.WithGateway()),
+		actions.Series(
+			// create 3 users
+			&actions.CreateTestUser{UserID: 0, UseGateway: true}, // <-- this user makes the PersonalTransactions request, choose gateway or not here
+			&actions.CreateTestUser{UserID: 1},
+			&actions.CreateTestUser{UserID: 2},
+			actions.SetContextValue(actions.KeyNumberOfTestUsers, 3),
+
+			&actions.AllocateFaucetFunds{UserID: 0},
+			actions.SnapshotUserBalances(actions.SnapAfterAllocation), // record user balances (we have no guarantee on how much the network faucet allocates)
+
+			// user 0 sends funds to users 1 and 2
+			&actions.SendNativeFunds{FromUser: 0, ToUser: 1, Amount: _transferAmount},
+			&actions.SendNativeFunds{FromUser: 0, ToUser: 2, Amount: _transferAmount},
+
+			// after the test we will verify the other users received them
+			&actions.VerifyBalanceAfterTest{UserID: 1, ExpectedBalance: _transferAmount},
+			&actions.VerifyBalanceAfterTest{UserID: 2, ExpectedBalance: _transferAmount},
+
+			// verify the personal transactions endpoint returns the two txs
+			actions.VerifyOnlyAction(func(ctx context.Context, network networktest.NetworkConnector) error {
+				user, err := actions.FetchTestUser(ctx, 0)
+				if err != nil {
+					return err
+				}
+
+				pagination := common.QueryPagination{
+					Offset: 0,
+					Size:   20,
+				}
+				personalTxs, total, err := user.GetPersonalTransactions(ctx, pagination)
+				if err != nil {
+					return fmt.Errorf("unable to get personal transactions - %w", err)
+				}
+
+				// verify the transactions
+				if len(personalTxs) != 2 {
+					return fmt.Errorf("expected 2 transactions, got %d", len(personalTxs))
+				}
+
+				// verify total set
+				if total != 2 {
+					return fmt.Errorf("expected total receipts to be at least 2, got %d", total)
+				}
+				return nil
+			}),
+		),
+	)
+}