WIP marshalling issue

go/common/custom_query_types.go

@@ -5,20 +5,22 @@ import "github.com/ethereum/go-ethereum/common"
 // CustomQueries are Ten-specific queries that are not supported by the Ethereum RPC API but that we wish to support
 // through the same interface.
 //
-// We currently use the eth_getStorageAt method to route these queries through the Ethereum RPC API, since it will not
-// be supported by the Ten network.
+// We currently use the eth_getStorageAt method to route these queries through the Ethereum RPC API.
 //
-// A custom query has a name (string), an address (if private request) and a params field (generic json object).
+// In order to match the eth_getStorageAt method signature, we require that all custom queries use an incrementing "address"
+// to specify the method we are calling (e.g. 0x000...001 is getUserID, 0x000...002 is listPrivateTransactions).
 //
-// NOTE: Private custom queries must include "address" as a top-level field in the params json object.
+// The signature is: eth_getStorageAt(method, params, nil) where:
+// - method is the address of the custom query as an address (e.g. 0x000...001)
+// - params is a JSON string with the parameters for the query (this complies with the eth_getStorageAt method signature since position gets encoded as a hex string)
+//
+// NOTE: Private custom queries must also include "address" as a top-level field in the params json object to indicate
+// the account the query is being made for.
 
 // CustomQuery methods
 const (
-	UserIDRequestCQMethod           = "getUserID"
-	ListPrivateTransactionsCQMethod = "listPersonalTransactions"
-
-	// DeprecatedUserIDRequestCQMethod is still supported for backwards compatibility for User ID requests
-	DeprecatedUserIDRequestCQMethod = "0x0000000000000000000000000000000000000000"
+	UserIDRequestCQMethod           = "0x0000000000000000000000000000000000000001"
+	ListPrivateTransactionsCQMethod = "0x0000000000000000000000000000000000000002"
 )
 
 type ListPrivateTransactionsQueryParams struct {

go/common/gethencoding/geth_encoding.go

@@ -397,27 +397,28 @@ func (enc *gethEncodingServiceImpl) CreateEthBlockFromBatch(ctx context.Context,
 // ExtractPrivateCustomQuery is designed to support a wide range of custom Ten queries.
 // The first parameter here is the method name, which is used to determine the query type.
 // The second parameter is the query parameters.
-func ExtractPrivateCustomQuery(methodName interface{}, queryParams interface{}) (*common.ListPrivateTransactionsQueryParams, error) {
+func ExtractPrivateCustomQuery(methodName any, queryParams any) (*common.ListPrivateTransactionsQueryParams, error) {
+	// we expect the first parameter to be a string
 	methodNameStr, ok := methodName.(string)
 	if !ok {
-		return nil, fmt.Errorf("unsupported method type %T", methodName)
+		return nil, fmt.Errorf("expected methodName as string but was type %T", methodName)
 	}
 	// currently we only have to support this custom query method in the enclave
 	if methodNameStr != common.ListPrivateTransactionsCQMethod {
-		return nil, fmt.Errorf("unsupported method %s", methodName)
+		return nil, fmt.Errorf("unsupported method %s", methodNameStr)
 	}
 
-	// Convert the map to a JSON string
-	jsonData, err := json.Marshal(queryParams)
-	if err != nil {
-		return nil, err
+	// we expect second param to be a json string
+	queryParamsStr, ok := queryParams.(string)
+	if !ok {
+		return nil, fmt.Errorf("expected queryParams as string but was type %T", queryParams)
 	}
 
-	var result common.ListPrivateTransactionsQueryParams
-	err = json.Unmarshal(jsonData, &result)
+	var privateQueryParams common.ListPrivateTransactionsQueryParams
+	err := json.Unmarshal([]byte(queryParamsStr), &privateQueryParams)
 	if err != nil {
 		return nil, err
 	}
 
-	return &result, nil
+	return &privateQueryParams, nil
 }

go/common/query_types.go

@@ -10,7 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-type PrivateQueryResponse struct {
+type PrivateTransactionsQueryResponse struct {
 	Receipts types.Receipts
 	Total    uint64
 }

go/enclave/enclave.go

@@ -858,7 +858,7 @@ func (e *enclaveImpl) GetTotalContractCount(ctx context.Context) (*big.Int, comm
 func (e *enclaveImpl) GetCustomQuery(ctx context.Context, encryptedParams common.EncryptedParamsGetStorageAt) (*responses.PrivateQueryResponse, common.SystemError) {
 	// ensure the enclave is running
 	if e.stopControl.IsStopping() {
-		return nil, responses.ToInternalError(fmt.Errorf("requested GetReceiptsByAddress with the enclave stopping"))
+		return nil, responses.ToInternalError(fmt.Errorf("requested GetPrivateTransactions with the enclave stopping"))
 	}
 
 	return rpc.WithVKEncryption(ctx, e.rpcEncryptionManager, encryptedParams, rpc.GetCustomQueryValidate, rpc.GetCustomQueryExecute)

go/enclave/rpc/GetCustomQuery.go

@@ -3,11 +3,12 @@ package rpc
 import (
 	"fmt"
 
+	gethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ten-protocol/go-ten/go/common"
 	"github.com/ten-protocol/go-ten/go/common/gethencoding"
 )
 
-func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateQueryResponse], _ *EncryptionManager) error {
+func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], _ *EncryptionManager) error {
 	// Parameters are [PrivateCustomQueryHeader, PrivateCustomQueryArgs, null]
 	if len(reqParams) != 3 {
 		builder.Err = fmt.Errorf("unexpected number of parameters (expected %d, got %d)", 3, len(reqParams))
@@ -19,29 +20,30 @@ func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPri
 		builder.Err = fmt.Errorf("unable to extract query - %w", err)
 		return nil
 	}
-	builder.From = &privateCustomQuery.Address
+	addr := gethcommon.Address(privateCustomQuery.Address)
+	builder.From = &addr
 	builder.Param = privateCustomQuery
 	return nil
 }
 
-func GetCustomQueryExecute(builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateQueryResponse], rpc *EncryptionManager) error {
+func GetCustomQueryExecute(builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], rpc *EncryptionManager) error {
 	err := authenticateFrom(builder.VK, builder.From)
 	if err != nil {
 		builder.Err = err
 		return nil //nolint:nilerr
 	}
-
-	encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &builder.Param.Address, &builder.Param.Pagination)
+	addr := gethcommon.Address(builder.Param.Address)
+	encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &addr, &builder.Param.Pagination)
 	if err != nil {
 		return fmt.Errorf("GetTransactionsPerAddress - %w", err)
 	}
 
-	receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &builder.Param.Address)
+	receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &addr)
 	if err != nil {
 		return fmt.Errorf("CountTransactionsPerAddress - %w", err)
 	}
 
-	builder.ReturnValue = &common.PrivateQueryResponse{
+	builder.ReturnValue = &common.PrivateTransactionsQueryResponse{
 		Receipts: encryptReceipts,
 		Total:    receiptsCount,
 	}

go/obsclient/authclient.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
@@ -249,15 +250,18 @@ func (ac *AuthObsClient) EstimateGasAndGasPrice(txData types.TxData) types.TxDat
 	}
 }
 
-// GetReceiptsByAddress retrieves the receipts for the account registered on this client (due to obscuro privacy restrictions,
-// balance cannot be requested for other accounts)
-func (ac *AuthObsClient) GetReceiptsByAddress(ctx context.Context, address *gethcommon.Address, pagination common.QueryPagination) (types.Receipts, error) {
+// GetPrivateTransactions retrieves the receipts for the specified account (must be registered on this client)
+func (ac *AuthObsClient) GetPrivateTransactions(ctx context.Context, address *gethcommon.Address, pagination common.QueryPagination) (types.Receipts, error) {
 	queryParam := &common.ListPrivateTransactionsQueryParams{
 		Address:    *address,
 		Pagination: pagination,
 	}
-	var result common.PrivateQueryResponse
-	err := ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, "listPersonalTransactions", queryParam, nil)
+	queryParamStr, err := json.Marshal(queryParam)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal query params - %w", err)
+	}
+	var result common.PrivateTransactionsQueryResponse
+	err = ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, common.ListPrivateTransactionsCQMethod, string(queryParamStr), nil)
 	if err != nil {
 		return nil, err
 	}

go/rpc/encrypted_client.go

@@ -166,6 +166,12 @@ func (c *EncRPCClient) executeSensitiveCall(ctx context.Context, result interfac
 	// and never error.
 	resultBytes, _ := decodedResult.MarshalJSON()
 
+	// if expected result type is bytes, we return the bytes
+	if _, ok := result.(*[]byte); ok {
+		*result.(*[]byte) = resultBytes
+		return nil
+	}
+
 	// We put the raw json in the passed result object.
 	// This works for structs, strings, integers and interface types.
 	err = json.Unmarshal(resultBytes, result)

integration/networktest/userwallet/authclient.go

@@ -134,5 +134,5 @@ func (s *AuthClientUser) Wallet() wallet.Wallet {
 
 func (s *AuthClientUser) GetPersonalTransactions(ctx context.Context, pagination common.QueryPagination) (types.Receipts, error) {
 	address := s.wal.Address()
-	return s.client.GetReceiptsByAddress(ctx, &address, pagination)
+	return s.client.GetPrivateTransactions(ctx, &address, pagination)
 }

integration/networktest/userwallet/gateway.go

@@ -117,11 +117,17 @@ func (g *GatewayUser) GetPersonalTransactions(ctx context.Context, pagination co
 		Address:    g.wal.Address(),
 		Pagination: pagination,
 	}
-	var result common.PrivateQueryResponse
+	var result common.PrivateTransactionsQueryResponse
 	err := rpcClient.CallContext(ctx, &result, "eth_getStorageAt", "listPersonalTransactions", queryParams, nil)
 	if err != nil {
 		return nil, fmt.Errorf("rpc call failed - %w", err)
 	}
+	//var result common.PrivateTransactionsQueryResponse
+	//err = json.Unmarshal([]byte(resultStr), &result)
+	//if err != nil {
+	//	return nil, fmt.Errorf("failed to unmarshal result - %w", err)
+	//
+	//}
 	return result.Receipts, nil
 }
 

tools/tenscan/frontend/src/routes/index.ts

@@ -31,6 +31,11 @@ export const apiRoutes = {
 export const ethMethods = {
   getStorageAt: "eth_getStorageAt",
 };
+// to send TEN Custom Queries (CQ) through the provider we call eth_getStorageAt and use these addresses to identify the TEN CQ method
+export const tenCustomQueryMethods = {
+  getUserID: "0x0000000000000000000000000000000000000001",
+  listPersonalTransactions: "0x0000000000000000000000000000000000000002",
+};
 
 export const NavLinks: NavLink[] = [
   {

tools/tenscan/frontend/src/services/useTransactionsService.ts

@@ -15,7 +15,7 @@ import { PersonalTransactionsResponse } from "../types/interfaces/TransactionInt
 import { useRouter } from "next/router";
 import { showToast } from "../components/ui/use-toast";
 import { ToastType } from "../types/interfaces";
-import { ethMethods } from "../routes";
+import {ethMethods, tenCustomQueryMethods} from "../routes";
 
 export const useTransactionsService = () => {
   const { query } = useRouter();
@@ -62,8 +62,8 @@ export const useTransactionsService = () => {
           },
         };
         const personalTxData = await provider.send(ethMethods.getStorageAt, [
-          "listPersonalTransactions",
-          requestPayload,
+          tenCustomQueryMethods.listPersonalTransactions,
+          JSON.stringify(requestPayload),
           null,
         ]);
         setPersonalTxns(personalTxData);

tools/walletextension/rpcapi/blockchain_api.go

@@ -136,18 +136,19 @@ func (api *BlockChainAPI) GetCode(ctx context.Context, address gethcommon.Addres
 	return *resp, err
 }
 
-// GetStorageAt is not compatible with ETH RPC tooling. Ten network will never support getStorageAt because it would
+// GetStorageAt is not compatible with ETH RPC tooling. Ten network does not getStorageAt because it would
 // violate the privacy guarantees of the network.
 //
 // However, we can repurpose this method to be able to route Ten-specific requests through from an ETH RPC client.
 // We call these requests Custom Queries.
 //
-// If this method is called using the standard ETH API parameters it will error, the correct params for this method are:
-// [ customMethodName string, customMethodParams any, nil ]
-// the final nil is to support the same number of params that getStorageAt sends, it is unused.
-func (api *BlockChainAPI) GetStorageAt(ctx context.Context, customMethod string, customParams any, _ any) (hexutil.Bytes, error) {
-	// GetStorageAt is repurposed to return the userID
-	if customMethod == common.UserIDRequestCQMethod {
+// This method signature matches eth_getStorageAt, but we use the address field to specify the custom query method,
+// the hex-encoded position field to specify the parameters json, and nil for the block number.
+//
+// In future, we can support both CustomQueries and some debug version of eth_getStorageAt if needed.
+func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address gethcommon.Address, params string, _ rpc.BlockNumberOrHash) (hexutil.Bytes, error) {
+	switch address.Hex() {
+	case common.UserIDRequestCQMethod:
 		userID, err := extractUserID(ctx, api.we)
 		if err != nil {
 			return nil, err
@@ -158,18 +159,25 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, customMethod string,
 			return nil, err
 		}
 		return userID, nil
+	case common.ListPrivateTransactionsCQMethod:
+		// sensitive CustomQuery methods use the convention of having "address" at the top level of the params json
+		userAddr, err := extractCustomQueryAddress(params)
+		if err != nil {
+			return nil, fmt.Errorf("unable to extract address from custom query params: %w", err)
+		}
+		resp, err := ExecAuthRPC[common.PrivateTransactionsQueryResponse](ctx, api.we, &ExecCfg{account: userAddr}, "eth_getStorageAt", address.Hex(), params, nil)
+		if err != nil {
+			return nil, fmt.Errorf("unable to execute custom query: %w", err)
+		}
+		// turn resp object into hexutil.Bytes
+		serialised, err := json.Marshal(resp)
+		if err != nil {
+			return nil, fmt.Errorf("unable to marshal response object: %w", err)
+		}
+		return serialised, nil
+	default: // address was not a recognised custom query method address
+		return nil, fmt.Errorf("eth_getStorageAt is not supported on TEN")
 	}
-
-	// sensitive CustomQuery methods use the convention of having "address" at the top level of the params json
-	address, err := extractCustomQueryAddress(customParams)
-	if err != nil {
-		return nil, fmt.Errorf("unable to extract address from custom query params: %w", err)
-	}
-	resp, err := ExecAuthRPC[hexutil.Bytes](ctx, api.we, &ExecCfg{account: address}, "eth_getStorageAt", customMethod, customParams, nil)
-	if resp == nil {
-		return nil, err
-	}
-	return *resp, err
 }
 
 func (s *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) {