Rework to comply strictly with getStorageAt signature

go/common/custom_query_types.go

@@ -5,20 +5,22 @@ import "github.com/ethereum/go-ethereum/common"
 // CustomQueries are Ten-specific queries that are not supported by the Ethereum RPC API but that we wish to support
 // through the same interface.
 //
-// We currently use the eth_getStorageAt method to route these queries through the Ethereum RPC API, since it will not
-// be supported by the Ten network.
+// We currently use the eth_getStorageAt method to route these queries through the Ethereum RPC API.
 //
-// A custom query has a name (string), an address (if private request) and a params field (generic json object).
+// In order to match the eth_getStorageAt method signature, we require that all custom queries use an incrementing "address"
+// to specify the method we are calling (e.g. 0x000...001 is getUserID, 0x000...002 is listPrivateTransactions).
 //
-// NOTE: Private custom queries must include "address" as a top-level field in the params json object.
+// The signature is: eth_getStorageAt(method, params, nil) where:
+// - method is the address of the custom query as an address (e.g. 0x000...001)
+// - params is a JSON string with the parameters for the query (this complies with the eth_getStorageAt method signature since position gets encoded as a hex string)
+//
+// NOTE: Private custom queries must also include "address" as a top-level field in the params json object to indicate
+// the account the query is being made for.
 
 // CustomQuery methods
 const (
-	UserIDRequestCQMethod           = "getUserID"
-	ListPrivateTransactionsCQMethod = "listPersonalTransactions"
-
-	// DeprecatedUserIDRequestCQMethod is still supported for backwards compatibility for User ID requests
-	DeprecatedUserIDRequestCQMethod = "0x0000000000000000000000000000000000000000"
+	UserIDRequestCQMethod           = "0x0000000000000000000000000000000000000001"
+	ListPrivateTransactionsCQMethod = "0x0000000000000000000000000000000000000002"
 )
 
 type ListPrivateTransactionsQueryParams struct {

go/common/gethencoding/geth_encoding.go

@@ -2,6 +2,7 @@ package gethencoding
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"math/big"
@@ -397,27 +398,38 @@ func (enc *gethEncodingServiceImpl) CreateEthBlockFromBatch(ctx context.Context,
 // ExtractPrivateCustomQuery is designed to support a wide range of custom Ten queries.
 // The first parameter here is the method name, which is used to determine the query type.
 // The second parameter is the query parameters.
-func ExtractPrivateCustomQuery(methodName interface{}, queryParams interface{}) (*common.ListPrivateTransactionsQueryParams, error) {
+func ExtractPrivateCustomQuery(methodName any, queryParams any) (*common.ListPrivateTransactionsQueryParams, error) {
+	// we expect the first parameter to be a string
 	methodNameStr, ok := methodName.(string)
 	if !ok {
-		return nil, fmt.Errorf("unsupported method type %T", methodName)
+		return nil, fmt.Errorf("expected methodName as string but was type %T", methodName)
 	}
 	// currently we only have to support this custom query method in the enclave
 	if methodNameStr != common.ListPrivateTransactionsCQMethod {
-		return nil, fmt.Errorf("unsupported method %s", methodName)
+		return nil, fmt.Errorf("unsupported method %s", methodNameStr)
 	}
 
-	// Convert the map to a JSON string
-	jsonData, err := json.Marshal(queryParams)
-	if err != nil {
-		return nil, err
+	// we expect second param to be a json string
+	queryParamsStr, ok := queryParams.(string)
+	if !ok {
+		return nil, fmt.Errorf("expected queryParams as string but was type %T", queryParams)
 	}
 
-	var result common.ListPrivateTransactionsQueryParams
-	err = json.Unmarshal(jsonData, &result)
+	var privateQueryParams common.ListPrivateTransactionsQueryParams
+	err := json.Unmarshal([]byte(queryParamsStr), &privateQueryParams)
 	if err != nil {
-		return nil, err
+		// if it fails, check if the string was base64 encoded
+		bytesStr, err64 := base64.StdEncoding.DecodeString(queryParamsStr)
+		if err64 != nil {
+			// was not base64 encoded, give up
+			return nil, fmt.Errorf("unable to unmarshal params string: %w", err)
+		}
+		// was base64 encoded, try to unmarshal
+		err = json.Unmarshal(bytesStr, &privateQueryParams)
+		if err != nil {
+			return nil, fmt.Errorf("unable to unmarshal params string: %w", err)
+		}
 	}
 
-	return &result, nil
+	return &privateQueryParams, nil
 }

go/common/query_types.go

@@ -10,7 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
-type PrivateQueryResponse struct {
+type PrivateTransactionsQueryResponse struct {
 	Receipts types.Receipts
 	Total    uint64
 }

go/enclave/components/batch_executor.go

@@ -379,7 +379,7 @@ func (executor *batchExecutor) populateOutboundCrossChainData(ctx context.Contex
 
 		encodedTree, err := json.Marshal(xchainTree)
 		if err != nil {
-			panic(err) //todo: figure out what to do
+			panic(err) // todo: figure out what to do
 		}
 
 		batch.Header.CrossChainTree = encodedTree

go/enclave/crosschain/common.go

@@ -234,7 +234,7 @@ func (ms MessageStructs) HashPacked(index int) gethcommon.Hash {
 		},
 	}
 
-	//todo @siliev: err
+	// todo @siliev: err
 	packed, _ := args.Pack(messageStruct.Sender, messageStruct.Sequence, messageStruct.Nonce, messageStruct.Topic, messageStruct.Payload, messageStruct.ConsistencyLevel)
 	hash := crypto.Keccak256Hash(packed)
 	return hash

go/enclave/enclave.go

@@ -858,7 +858,7 @@ func (e *enclaveImpl) GetTotalContractCount(ctx context.Context) (*big.Int, comm
 func (e *enclaveImpl) GetCustomQuery(ctx context.Context, encryptedParams common.EncryptedParamsGetStorageAt) (*responses.PrivateQueryResponse, common.SystemError) {
 	// ensure the enclave is running
 	if e.stopControl.IsStopping() {
-		return nil, responses.ToInternalError(fmt.Errorf("requested GetReceiptsByAddress with the enclave stopping"))
+		return nil, responses.ToInternalError(fmt.Errorf("requested GetPrivateTransactions with the enclave stopping"))
 	}
 
 	return rpc.WithVKEncryption(ctx, e.rpcEncryptionManager, encryptedParams, rpc.GetCustomQueryValidate, rpc.GetCustomQueryExecute)

go/enclave/genesis/testnet_genesis.go

@@ -9,8 +9,10 @@ import (
 )
 
 const TestnetPrefundedPK = "8dfb8083da6275ae3e4f41e3e8a8c19d028d32c9247e24530933782f2a05035b" // The genesis main account private key.
-var GasBridgingKeys, _ = crypto.GenerateKey()                                                 // todo - make static
-var GasWithdrawalKeys, _ = crypto.GenerateKey()                                               // todo - make static
+var (
+	GasBridgingKeys, _   = crypto.GenerateKey() // todo - make static
+	GasWithdrawalKeys, _ = crypto.GenerateKey() // todo - make static
+)
 
 var TestnetGenesis = Genesis{
 	Accounts: []Account{

go/enclave/nodetype/common.go

@@ -40,6 +40,6 @@ func ExportCrossChainData(ctx context.Context, storage storage.Storage, fromSeqN
 		L1BlockHash:          block.Hash(),
 		L1BlockNum:           big.NewInt(0).Set(block.Header().Number),
 		CrossChainRootHashes: crossChainHashes,
-	} //todo: check fromSeqNo
+	} // todo: check fromSeqNo
 	return bundle, nil
 }

go/enclave/nodetype/sequencer.go

@@ -468,6 +468,7 @@ func (s *sequencer) signCrossChainBundle(bundle *common.ExtCrossChainBundle) err
 	}
 	return nil
 }
+
 func (s *sequencer) OnL1Block(ctx context.Context, block *types.Block, result *components.BlockIngestionType) error {
 	// nothing to do
 	return nil

go/enclave/rpc/GetCustomQuery.go

@@ -3,11 +3,12 @@ package rpc
 import (
 	"fmt"
 
+	gethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ten-protocol/go-ten/go/common"
 	"github.com/ten-protocol/go-ten/go/common/gethencoding"
 )
 
-func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateQueryResponse], _ *EncryptionManager) error {
+func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], _ *EncryptionManager) error {
 	// Parameters are [PrivateCustomQueryHeader, PrivateCustomQueryArgs, null]
 	if len(reqParams) != 3 {
 		builder.Err = fmt.Errorf("unexpected number of parameters (expected %d, got %d)", 3, len(reqParams))
@@ -19,29 +20,30 @@ func GetCustomQueryValidate(reqParams []any, builder *CallBuilder[common.ListPri
 		builder.Err = fmt.Errorf("unable to extract query - %w", err)
 		return nil
 	}
-	builder.From = &privateCustomQuery.Address
+	addr := privateCustomQuery.Address
+	builder.From = &addr
 	builder.Param = privateCustomQuery
 	return nil
 }
 
-func GetCustomQueryExecute(builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateQueryResponse], rpc *EncryptionManager) error {
+func GetCustomQueryExecute(builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], rpc *EncryptionManager) error {
 	err := authenticateFrom(builder.VK, builder.From)
 	if err != nil {
 		builder.Err = err
 		return nil //nolint:nilerr
 	}
-
-	encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &builder.Param.Address, &builder.Param.Pagination)
+	addr := gethcommon.Address(builder.Param.Address)
+	encryptReceipts, err := rpc.storage.GetTransactionsPerAddress(builder.ctx, &addr, &builder.Param.Pagination)
 	if err != nil {
 		return fmt.Errorf("GetTransactionsPerAddress - %w", err)
 	}
 
-	receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &builder.Param.Address)
+	receiptsCount, err := rpc.storage.CountTransactionsPerAddress(builder.ctx, &addr)
 	if err != nil {
 		return fmt.Errorf("CountTransactionsPerAddress - %w", err)
 	}
 
-	builder.ReturnValue = &common.PrivateQueryResponse{
+	builder.ReturnValue = &common.PrivateTransactionsQueryResponse{
 		Receipts: encryptReceipts,
 		Total:    receiptsCount,
 	}

go/obsclient/authclient.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum"
@@ -249,15 +250,18 @@ func (ac *AuthObsClient) EstimateGasAndGasPrice(txData types.TxData) types.TxDat
 	}
 }
 
-// GetReceiptsByAddress retrieves the receipts for the account registered on this client (due to obscuro privacy restrictions,
-// balance cannot be requested for other accounts)
-func (ac *AuthObsClient) GetReceiptsByAddress(ctx context.Context, address *gethcommon.Address, pagination common.QueryPagination) (types.Receipts, error) {
+// GetPrivateTransactions retrieves the receipts for the specified account (must be registered on this client)
+func (ac *AuthObsClient) GetPrivateTransactions(ctx context.Context, address *gethcommon.Address, pagination common.QueryPagination) (types.Receipts, error) {
 	queryParam := &common.ListPrivateTransactionsQueryParams{
 		Address:    *address,
 		Pagination: pagination,
 	}
-	var result common.PrivateQueryResponse
-	err := ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, "listPersonalTransactions", queryParam, nil)
+	queryParamStr, err := json.Marshal(queryParam)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal query params - %w", err)
+	}
+	var result common.PrivateTransactionsQueryResponse
+	err = ac.rpcClient.CallContext(ctx, &result, rpc.GetStorageAt, common.ListPrivateTransactionsCQMethod, string(queryParamStr), nil)
 	if err != nil {
 		return nil, err
 	}

go/rpc/encrypted_client.go

@@ -166,6 +166,12 @@ func (c *EncRPCClient) executeSensitiveCall(ctx context.Context, result interfac
 	// and never error.
 	resultBytes, _ := decodedResult.MarshalJSON()
 
+	// if expected result type is bytes, we return the bytes
+	if _, ok := result.(*[]byte); ok {
+		*result.(*[]byte) = resultBytes
+		return nil
+	}
+
 	// We put the raw json in the passed result object.
 	// This works for structs, strings, integers and interface types.
 	err = json.Unmarshal(resultBytes, result)

integration/networktest/userwallet/authclient.go

@@ -134,5 +134,5 @@ func (s *AuthClientUser) Wallet() wallet.Wallet {
 
 func (s *AuthClientUser) GetPersonalTransactions(ctx context.Context, pagination common.QueryPagination) (types.Receipts, error) {
 	address := s.wal.Address()
-	return s.client.GetReceiptsByAddress(ctx, &address, pagination)
+	return s.client.GetPrivateTransactions(ctx, &address, pagination)
 }

integration/networktest/userwallet/gateway.go

@@ -2,13 +2,15 @@ package userwallet
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
 	gethcommon "github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	gethlog "github.com/ethereum/go-ethereum/log"
@@ -117,11 +119,20 @@ func (g *GatewayUser) GetPersonalTransactions(ctx context.Context, pagination co
 		Address:    g.wal.Address(),
 		Pagination: pagination,
 	}
-	var result common.PrivateQueryResponse
-	err := rpcClient.CallContext(ctx, &result, "eth_getStorageAt", "listPersonalTransactions", queryParams, nil)
+	queryParamsStr, err := json.Marshal(queryParams)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal query params - %w", err)
+	}
+	var resultBytes hexutil.Bytes
+	err = rpcClient.CallContext(ctx, &resultBytes, "eth_getStorageAt", common.ListPrivateTransactionsCQMethod, queryParamsStr, nil)
 	if err != nil {
 		return nil, fmt.Errorf("rpc call failed - %w", err)
 	}
+	var result common.PrivateTransactionsQueryResponse
+	err = json.Unmarshal(resultBytes, &result)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal result - %w", err)
+	}
 	return result.Receipts, nil
 }
 

tools/tenscan/frontend/src/routes/index.ts

@@ -31,6 +31,11 @@ export const apiRoutes = {
 export const ethMethods = {
   getStorageAt: "eth_getStorageAt",
 };
+// to send TEN Custom Queries (CQ) through the provider we call eth_getStorageAt and use these addresses to identify the TEN CQ method
+export const tenCustomQueryMethods = {
+  getUserID: "0x0000000000000000000000000000000000000001",
+  listPersonalTransactions: "0x0000000000000000000000000000000000000002",
+};
 
 export const NavLinks: NavLink[] = [
   {

tools/tenscan/frontend/src/services/useTransactionsService.ts

@@ -15,7 +15,7 @@ import { PersonalTransactionsResponse } from "../types/interfaces/TransactionInt
 import { useRouter } from "next/router";
 import { showToast } from "../components/ui/use-toast";
 import { ToastType } from "../types/interfaces";
-import { ethMethods } from "../routes";
+import {ethMethods, tenCustomQueryMethods} from "../routes";
 
 export const useTransactionsService = () => {
   const { query } = useRouter();
@@ -62,8 +62,8 @@ export const useTransactionsService = () => {
           },
         };
         const personalTxData = await provider.send(ethMethods.getStorageAt, [
-          "listPersonalTransactions",
-          requestPayload,
+          tenCustomQueryMethods.listPersonalTransactions,
+          JSON.stringify(requestPayload),
           null,
         ]);
         setPersonalTxns(personalTxData);