Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix personal transactions RPC request #1923

Merged
merged 11 commits into from
Jul 4, 2024
29 changes: 29 additions & 0 deletions go/common/custom_query_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package common

import "github.com/ethereum/go-ethereum/common"

// CustomQueries are Ten-specific queries that are not supported by the Ethereum RPC API but that we wish to support
// through the same interface.
//
// We currently use the eth_getStorageAt method to route these queries through the Ethereum RPC API.
//
// In order to match the eth_getStorageAt method signature, we require that all custom queries use an incrementing "address"
// to specify the method we are calling (e.g. 0x000...001 is getUserID, 0x000...002 is listPrivateTransactions).
//
// The signature is: eth_getStorageAt(method, params, nil) where:
// - method is the address of the custom query as an address (e.g. 0x000...001)
// - params is a JSON string with the parameters for the query (this complies with the eth_getStorageAt method signature since position gets encoded as a hex string)
//
// NOTE: Private custom queries must also include "address" as a top-level field in the params json object to indicate
// the account the query is being made for.

// CustomQuery methods
const (
UserIDRequestCQMethod = "0x0000000000000000000000000000000000000001"
ListPrivateTransactionsCQMethod = "0x0000000000000000000000000000000000000002"
)

type ListPrivateTransactionsQueryParams struct {
Address common.Address `json:"address"`
Pagination QueryPagination `json:"pagination"`
}
42 changes: 33 additions & 9 deletions go/common/gethencoding/geth_encoding.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package gethencoding

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"math/big"
Expand Down Expand Up @@ -344,18 +345,41 @@ func (enc *gethEncodingServiceImpl) CreateEthBlockFromBatch(ctx context.Context,
return (*types.Block)(unsafe.Pointer(&lb)), nil
}

func ExtractPrivateCustomQuery(_ interface{}, query interface{}) (*common.PrivateCustomQueryListTransactions, error) {
// Convert the map to a JSON string
jsonData, err := json.Marshal(query)
if err != nil {
return nil, err
// ExtractPrivateCustomQuery is designed to support a wide range of custom Ten queries.
// The first parameter here is the method name, which is used to determine the query type.
// The second parameter is the query parameters.
func ExtractPrivateCustomQuery(methodName any, queryParams any) (*common.ListPrivateTransactionsQueryParams, error) {
// we expect the first parameter to be a string
methodNameStr, ok := methodName.(string)
if !ok {
return nil, fmt.Errorf("expected methodName as string but was type %T", methodName)
}
// currently we only have to support this custom query method in the enclave
if methodNameStr != common.ListPrivateTransactionsCQMethod {
return nil, fmt.Errorf("unsupported method %s", methodNameStr)
}

var result common.PrivateCustomQueryListTransactions
err = json.Unmarshal(jsonData, &result)
// we expect second param to be a json string
queryParamsStr, ok := queryParams.(string)
if !ok {
return nil, fmt.Errorf("expected queryParams as string but was type %T", queryParams)
}

var privateQueryParams common.ListPrivateTransactionsQueryParams
err := json.Unmarshal([]byte(queryParamsStr), &privateQueryParams)
if err != nil {
return nil, err
// if it fails, check if the string was base64 encoded
bytesStr, err64 := base64.StdEncoding.DecodeString(queryParamsStr)
if err64 != nil {
// was not base64 encoded, give up
return nil, fmt.Errorf("unable to unmarshal params string: %w", err)
}
// was base64 encoded, try to unmarshal
err = json.Unmarshal(bytesStr, &privateQueryParams)
if err != nil {
return nil, fmt.Errorf("unable to unmarshal params string: %w", err)
}
}

return &result, nil
return &privateQueryParams, nil
}
7 changes: 1 addition & 6 deletions go/common/query_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"github.com/ethereum/go-ethereum/common"
)

type PrivateQueryResponse struct {
type PrivateTransactionsQueryResponse struct {
Receipts types.Receipts
Total uint64
}
Expand Down Expand Up @@ -110,11 +110,6 @@ func (p *QueryPagination) UnmarshalJSON(data []byte) error {
return nil
}

type PrivateCustomQueryListTransactions struct {
Address common.Address `json:"address"`
Pagination QueryPagination `json:"pagination"`
}

type ObscuroNetworkInfo struct {
ManagementContractAddress common.Address
L1StartHash common.Hash
Expand Down
4 changes: 2 additions & 2 deletions go/common/rpc/generated/enclave.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions go/common/rpc/generated/enclave_grpc.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion go/enclave/enclave.go
Original file line number Diff line number Diff line change
Expand Up @@ -858,7 +858,7 @@ func (e *enclaveImpl) GetTotalContractCount(ctx context.Context) (*big.Int, comm
func (e *enclaveImpl) GetCustomQuery(ctx context.Context, encryptedParams common.EncryptedParamsGetStorageAt) (*responses.PrivateQueryResponse, common.SystemError) {
// ensure the enclave is running
if e.stopControl.IsStopping() {
return nil, responses.ToInternalError(fmt.Errorf("requested GetReceiptsByAddress with the enclave stopping"))
return nil, responses.ToInternalError(fmt.Errorf("requested GetPrivateTransactions with the enclave stopping"))
}

return rpc.WithVKEncryption(ctx, e.rpcEncryptionManager, encryptedParams, rpc.GetCustomQueryValidate, rpc.GetCustomQueryExecute)
Expand Down
17 changes: 9 additions & 8 deletions go/enclave/rpc/GetCustomQuery.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (
"github.com/ten-protocol/go-ten/go/common/gethencoding"
)

func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.PrivateCustomQueryListTransactions, common.PrivateQueryResponse], _ *EncryptionManager) error {
func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], _ *EncryptionManager) error {
// Parameters are [PrivateCustomQueryHeader, PrivateCustomQueryArgs, null]
if len(reqParams) != 3 {
builder.Err = fmt.Errorf("unexpected number of parameters")
builder.Err = fmt.Errorf("unexpected number of parameters (expected %d, got %d)", 3, len(reqParams))
return nil
}

Expand All @@ -19,29 +19,30 @@ func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.Private
builder.Err = fmt.Errorf("unable to extract query - %w", err)
return nil
}
builder.From = &privateCustomQuery.Address
addr := privateCustomQuery.Address
builder.From = &addr
builder.Param = privateCustomQuery
return nil
}

func GetCustomQueryExecute(builder *CallBuilder[common.PrivateCustomQueryListTransactions, common.PrivateQueryResponse], rpc *EncryptionManager) error {
func GetCustomQueryExecute(builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], rpc *EncryptionManager) error {
err := authenticateFrom(builder.VK, builder.From)
if err != nil {
builder.Err = err
return nil //nolint:nilerr
}

encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &builder.Param.Address, &builder.Param.Pagination)
addr := builder.Param.Address
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the difference between this and privateCustomQuery.Address?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing, but we don't have a privateCustomQuery in this Execute method, privateCustomQuery got set on the builder as builder.Param in the Validate method.

Yeah this feels a little weird, but will tidy it up when we get rid of 'CustomQuery' from enclave, it will just be a 'GetPersonalTransactions' method and won't be trying to do anything generic.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

then let's add a check that they are equal. To avoid any security hole

encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &addr, &builder.Param.Pagination)
if err != nil {
return fmt.Errorf("GetTransactionsPerAddress - %w", err)
}

receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &builder.Param.Address)
receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &addr)
if err != nil {
return fmt.Errorf("CountTransactionsPerAddress - %w", err)
}

builder.ReturnValue = &common.PrivateQueryResponse{
builder.ReturnValue = &common.PrivateTransactionsQueryResponse{
Receipts: encryptReceipts,
Total: receiptsCount,
}
Expand Down
7 changes: 5 additions & 2 deletions go/enclave/storage/enclavedb/batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,11 +428,14 @@ func BatchWasExecuted(ctx context.Context, db *sql.DB, hash common.L2BatchHash)
}

func GetTransactionsPerAddress(ctx context.Context, db *sql.DB, config *params.ChainConfig, address *gethcommon.Address, pagination *common.QueryPagination) (types.Receipts, error) {
return selectReceipts(ctx, db, config, "where tx.sender_address = ? ORDER BY height DESC LIMIT ? OFFSET ? ", address.Bytes(), pagination.Size, pagination.Offset)
return selectReceipts(ctx, db, config, "join externally_owned_account eoa on tx.sender_address = eoa.id where eoa.address = ? ORDER BY height DESC LIMIT ? OFFSET ? ", address.Bytes(), pagination.Size, pagination.Offset)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oops. I forgot to change these

}

func CountTransactionsPerAddress(ctx context.Context, db *sql.DB, address *gethcommon.Address) (uint64, error) {
row := db.QueryRowContext(ctx, "select count(1) from receipt join tx on tx.id=receipt.tx join batch on batch.sequence=receipt.batch "+" where tx.sender_address = ?", address.Bytes())
row := db.QueryRowContext(ctx, "select count(1) from receipt "+
"join tx on tx.id=receipt.tx "+
"join externally_owned_account eoa on eoa.id = tx.sender_address "+
"where eoa.address = ?", address.Bytes())

var count uint64
err := row.Scan(&count)
Expand Down
22 changes: 15 additions & 7 deletions go/obsclient/authclient.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"math/big"

"github.com/ethereum/go-ethereum"
Expand Down Expand Up @@ -249,14 +250,21 @@ func (ac *AuthObsClient) EstimateGasAndGasPrice(txData types.TxData) types.TxDat
}
}

// GetReceiptsByAddress retrieves the receipts for the account registered on this client (due to obscuro privacy restrictions,
// balance cannot be requested for other accounts)
func (ac *AuthObsClient) GetReceiptsByAddress(ctx context.Context, address *gethcommon.Address) (types.Receipts, error) {
var result types.Receipts
err := ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, address, nil, nil)
// GetPrivateTransactions retrieves the receipts for the specified account (must be registered on this client), returns requested range of receipts and the total number of receipts for that acc
func (ac *AuthObsClient) GetPrivateTransactions(ctx context.Context, address *gethcommon.Address, pagination common.QueryPagination) (types.Receipts, uint64, error) {
queryParam := &common.ListPrivateTransactionsQueryParams{
Address: *address,
Pagination: pagination,
}
queryParamStr, err := json.Marshal(queryParam)
if err != nil {
return nil, err
return nil, 0, fmt.Errorf("unable to marshal query params - %w", err)
}
var result common.PrivateTransactionsQueryResponse
err = ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, common.ListPrivateTransactionsCQMethod, string(queryParamStr), nil)
if err != nil {
return nil, 0, err
}

return result, nil
return result.Receipts, result.Total, nil
}
6 changes: 6 additions & 0 deletions go/rpc/encrypted_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,12 @@ func (c *EncRPCClient) executeSensitiveCall(ctx context.Context, result interfac
// and never error.
resultBytes, _ := decodedResult.MarshalJSON()

// if expected result type is bytes, we return the bytes
if _, ok := result.(*[]byte); ok {
*result.(*[]byte) = resultBytes
return nil
}

// We put the raw json in the passed result object.
// This works for structs, strings, integers and interface types.
err = json.Unmarshal(resultBytes, result)
Expand Down
4 changes: 2 additions & 2 deletions integration/networktest/env/network_setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ func DevTestnet(opts ...TestnetEnvOption) networktest.Environment {
// LongRunningLocalNetwork is a local network, the l1WSURL is optional (can be empty string), only required if testing L1 interactions
func LongRunningLocalNetwork(l1WSURL string) networktest.Environment {
connector := newTestnetConnectorWithFaucetAccount(
"ws://127.0.0.1:26900",
[]string{"ws://127.0.0.1:26901"},
"ws://127.0.0.1:17900",
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These are the correct ports now.

[]string{"ws://127.0.0.1:17901"},
genesis.TestnetPrefundedPK,
l1WSURL,
"",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const (
_sepoliaValidator1PK = "<pk>" // account 0x<acc>
)

// Spins up a local network with a gateway, with all processes debuggable. The network will run until the test is stopped.
// Note: If you want to access the gateway frontend you need to `npm run build` its frontend with NEXT_PUBLIC_API_GATEWAY_URL=http://localhost:11180
func TestRunLocalNetwork(t *testing.T) {
networktest.TestOnlyRunsInIDE(t)
networktest.EnsureTestLogsSetUp("local-geth-network")
Expand Down
66 changes: 66 additions & 0 deletions integration/networktest/tests/tenscan/tenscan_rpc_test.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
package tenscan

import (
"context"
"fmt"
"math/big"
"testing"

"github.com/ten-protocol/go-ten/go/common"
"github.com/ten-protocol/go-ten/integration/networktest"
"github.com/ten-protocol/go-ten/integration/networktest/actions"
"github.com/ten-protocol/go-ten/integration/networktest/actions/publicdata"
"github.com/ten-protocol/go-ten/integration/networktest/env"
"github.com/ten-protocol/go-ten/integration/simulation/devnetwork"
)

var _transferAmount = big.NewInt(100_000_000)

// Verify and debug the RPC endpoints that Tenscan relies on for data in various environments

func TestRPC(t *testing.T) {
Expand All @@ -22,3 +29,62 @@ func TestRPC(t *testing.T) {
),
)
}

// Test the personal transactions endpoint in various environments (it uses getStorageAt so it can run through MM etc.)
// 1. create user
// 2. send some transactions
// 3. verify transactions are returned by the personal transactions endpoint that tenscan uses
func TestPersonalTransactions(t *testing.T) {
networktest.TestOnlyRunsInIDE(t)
networktest.Run(
"tenscan-personal-transactions",
t,
env.LocalDevNetwork(devnetwork.WithGateway()),
actions.Series(
// create 3 users
&actions.CreateTestUser{UserID: 0, UseGateway: true}, // <-- this user makes the PersonalTransactions request, choose gateway or not here
&actions.CreateTestUser{UserID: 1},
&actions.CreateTestUser{UserID: 2},
actions.SetContextValue(actions.KeyNumberOfTestUsers, 3),

&actions.AllocateFaucetFunds{UserID: 0},
actions.SnapshotUserBalances(actions.SnapAfterAllocation), // record user balances (we have no guarantee on how much the network faucet allocates)

// user 0 sends funds to users 1 and 2
&actions.SendNativeFunds{FromUser: 0, ToUser: 1, Amount: _transferAmount},
&actions.SendNativeFunds{FromUser: 0, ToUser: 2, Amount: _transferAmount},

// after the test we will verify the other users received them
&actions.VerifyBalanceAfterTest{UserID: 1, ExpectedBalance: _transferAmount},
&actions.VerifyBalanceAfterTest{UserID: 2, ExpectedBalance: _transferAmount},

// verify the personal transactions endpoint returns the two txs
actions.VerifyOnlyAction(func(ctx context.Context, network networktest.NetworkConnector) error {
user, err := actions.FetchTestUser(ctx, 0)
if err != nil {
return err
}

pagination := common.QueryPagination{
Offset: 0,
Size: 20,
}
personalTxs, total, err := user.GetPersonalTransactions(ctx, pagination)
if err != nil {
return fmt.Errorf("unable to get personal transactions - %w", err)
}

// verify the transactions
if len(personalTxs) != 2 {
return fmt.Errorf("expected 2 transactions, got %d", len(personalTxs))
}

// verify total set
if total != 2 {
return fmt.Errorf("expected total receipts to be at least 2, got %d", total)
}
return nil
}),
),
)
}
Loading