New config flag to store executed transactions (#2206)

* add new config flag to store executed transactions. Set gas limit per batch to real value.

* better comment

* wire the new flag in the launcher

* lint typo

* reduce gas to reasonable values

* tweaks

go/config/defaults/0-base-config.yaml

@@ -8,7 +8,7 @@ network:
   batch:
     interval: 1s
     maxInterval: 1s # if this is greater than batch.interval then we make batches more slowly when there are no transactions
-    maxSize: 56320 # 55kb
+    maxSize: 45000 # around 45kb - around 200 transactions / batch
   rollup:
     interval: 5s
     maxInterval: 10m # rollups will be produced after this time even if the data blob is not full
@@ -17,7 +17,7 @@ network:
     baseFee: 1000000000 # using geth's initial base fee for EIP-1559 blocks.
     minGasPrice: 1000000000 # using geth's initial base fee for EIP-1559 blocks.
     paymentAddress: 0xd6C9230053f45F873Cb66D8A02439380a37A4fbF
-    batchExecutionLimit: 300000000000 # 300 gwei
+    batchExecutionLimit: 30000000 # same as Ethereum blocks
     localExecutionCap: 300000000000 # 300 gwei
   l1:
     chainId: 1337
@@ -74,6 +74,7 @@ host:
 
 enclave:
   enableAttestation: false
+  storeExecutedTransactions: true
   db:
     useInMemory: true
     edgelessDBHost: "" # host address for postgres db when used

go/config/enclave.go

@@ -7,7 +7,8 @@ import "time"
 //	yaml: `enclave`
 type EnclaveConfig struct {
 	// EnableAttestation specifies whether the enclave will produce verified attestation report.
-	EnableAttestation bool `mapstructure:"enableAttestation"`
+	EnableAttestation         bool `mapstructure:"enableAttestation"`
+	StoreExecutedTransactions bool `mapstructure:"storeExecutedTransactions"`
 
 	DB    *EnclaveDB    `mapstructure:"db"`
 	Debug *EnclaveDebug `mapstructure:"debug"`

go/enclave/config/config.go

@@ -74,14 +74,18 @@ type EnclaveConfig struct {
 	// RPCTimeout - calls that are longer than this will be cancelled, to prevent resource starvation
 	// normally, the context is propagated from the host, but in some cases ( like the evm, we have to create a context)
 	RPCTimeout time.Duration
+
+	// StoreExecutedTransactions is a flag that instructs the current enclave to store data required to answer RPC queries.
+	StoreExecutedTransactions bool
 }
 
 func EnclaveConfigFromTenConfig(tenCfg *config.TenConfig) *EnclaveConfig {
 	return &EnclaveConfig{
-		HostID:      tenCfg.Node.ID,
-		HostAddress: tenCfg.Node.HostAddress,
-		NodeType:    tenCfg.Node.NodeType,
-		WillAttest:  tenCfg.Enclave.EnableAttestation,
+		HostID:                    tenCfg.Node.ID,
+		HostAddress:               tenCfg.Node.HostAddress,
+		NodeType:                  tenCfg.Node.NodeType,
+		WillAttest:                tenCfg.Enclave.EnableAttestation,
+		StoreExecutedTransactions: tenCfg.Enclave.StoreExecutedTransactions,
 
 		ObscuroChainID:      tenCfg.Network.ChainID,
 		SequencerP2PAddress: tenCfg.Network.Sequencer.P2PAddress,

go/enclave/crosschain/message_bus_manager.go

@@ -94,7 +94,7 @@ func (m *MessageBusManager) GenerateMessageBusDeployTx() (*common.L2Tx, error) {
 	tx := &types.LegacyTx{
 		Nonce:    0, // The first transaction of the owner identity should always be deploying the contract
 		Value:    gethcommon.Big0,
-		Gas:      500_000_000,     // It's quite the expensive contract.
+		Gas:      5_000_000,       // It's quite the expensive contract.
 		GasPrice: gethcommon.Big0, // Synthetic transactions are on the house. Or the house.
 		Data:     gethcommon.FromHex(MessageBus.MessageBusMetaData.Bin),
 		To:       nil, // Geth requires nil instead of gethcommon.Address{} which equates to zero address in order to return receipt.

go/enclave/enclave_rpc_service.go

@@ -88,6 +88,9 @@ func (e *enclaveRPCService) GetCode(ctx context.Context, address gethcommon.Addr
 }
 
 func (e *enclaveRPCService) Subscribe(ctx context.Context, id gethrpc.ID, encryptedSubscription common.EncryptedParamsLogSubscription) common.SystemError {
+	if !e.config.StoreExecutedTransactions {
+		return fmt.Errorf("the current TEN enclave does not support log subscriptions")
+	}
 	encodedSubscription, err := e.rpcEncryptionManager.DecryptBytes(encryptedSubscription)
 	if err != nil {
 		return fmt.Errorf("could not decrypt params in eth_subscribe logs request. Cause: %w", err)

go/enclave/evm/evm_facade.go

@@ -269,8 +269,8 @@ func receiptToString(r *types.Receipt) string {
 	return fmt.Sprintf("Successfully executed. Receipt: %s", string(receiptJSON))
 }
 */
-// ExecuteObsCall - executes the eth_call call
-func ExecuteObsCall(
+// ExecuteCall - executes the eth_call call
+func ExecuteCall(
 	ctx context.Context,
 	msg *gethcore.Message,
 	s *state.StateDB,
@@ -294,7 +294,7 @@ func ExecuteObsCall(
 
 	snapshot := s.Snapshot()
 	defer s.RevertToSnapshot(snapshot) // Always revert after simulation
-	defer core.LogMethodDuration(logger, measure.NewStopwatch(), "evm_facade.go:ObsCall()")
+	defer core.LogMethodDuration(logger, measure.NewStopwatch(), "evm_facade.go:Call()")
 
 	gp := gethcore.GasPool(gasEstimationCap)
 	gp.SetGas(gasEstimationCap)

go/enclave/genesis/genesis_test.go

@@ -42,7 +42,7 @@ func TestDefaultGenesis(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable to create temp db: %s", err)
 	}
-	storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, gethlog.New())
+	storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, nil, gethlog.New())
 	stateDB, err := gen.applyAllocations(storageDB)
 	if err != nil {
 		t.Fatalf("unable to apply genesis allocations")
@@ -85,7 +85,7 @@ func TestCustomGenesis(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable to create temp db: %s", err)
 	}
-	storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, gethlog.New())
+	storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, nil, gethlog.New())
 	stateDB, err := gen.applyAllocations(storageDB)
 	if err != nil {
 		t.Fatalf("unable to apply genesis allocations")

go/enclave/l2chain/interfaces.go

@@ -20,7 +20,7 @@ type ObscuroChain interface {
 	GetBalanceAtBlock(ctx context.Context, accountAddr gethcommon.Address, blockNumber *gethrpc.BlockNumber) (*hexutil.Big, error)
 
 	// ObsCall - The interface for executing eth_call RPC commands against obscuro.
-	ObsCall(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error)
+	Call(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error)
 
 	// ObsCallAtBlock - Execute eth_call RPC against obscuro for a specific block (batch) number.
 	ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error)

go/enclave/l2chain/l2_chain.go

@@ -28,7 +28,7 @@ import (
 	gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc"
 )
 
-type obscuroChain struct {
+type tenChain struct {
 	chainConfig         *params.ChainConfig
 	config              enclaveconfig.EnclaveConfig
 	storage             storage.Storage
@@ -51,7 +51,7 @@ func NewChain(
 	registry components.BatchRegistry,
 	gasEstimationCap uint64,
 ) ObscuroChain {
-	return &obscuroChain{
+	return &tenChain{
 		storage:             storage,
 		config:              config,
 		gethEncodingService: gethEncodingService,
@@ -63,7 +63,7 @@ func NewChain(
 	}
 }
 
-func (oc *obscuroChain) GetBalanceAtBlock(ctx context.Context, accountAddr gethcommon.Address, blockNumber *gethrpc.BlockNumber) (*hexutil.Big, error) {
+func (oc *tenChain) GetBalanceAtBlock(ctx context.Context, accountAddr gethcommon.Address, blockNumber *gethrpc.BlockNumber) (*hexutil.Big, error) {
 	chainState, err := oc.Registry.GetBatchStateAtHeight(ctx, blockNumber)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get blockchain state - %w", err)
@@ -72,7 +72,7 @@ func (oc *obscuroChain) GetBalanceAtBlock(ctx context.Context, accountAddr gethc
 	return (*hexutil.Big)(chainState.GetBalance(accountAddr).ToBig()), nil
 }
 
-func (oc *obscuroChain) ObsCall(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) {
+func (oc *tenChain) Call(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) {
 	result, err := oc.ObsCallAtBlock(ctx, apiArgs, blockNumber)
 	if err != nil {
 		oc.logger.Debug(fmt.Sprintf("Obs_Call: failed to execute contract %s.", apiArgs.To), log.CtrErrKey, err.Error())
@@ -91,7 +91,7 @@ func (oc *obscuroChain) ObsCall(ctx context.Context, apiArgs *gethapi.Transactio
 	return result, nil
 }
 
-func (oc *obscuroChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) {
+func (oc *tenChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) {
 	// fetch the chain state at given batch
 	blockState, err := oc.Registry.GetBatchStateAtHeight(ctx, blockNumber)
 	if err != nil {
@@ -117,7 +117,7 @@ func (oc *obscuroChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.Tra
 			batch.Header.Root.Hex()))
 	}
 
-	result, err := evm.ExecuteObsCall(ctx, callMsg, blockState, batch.Header, oc.storage, oc.gethEncodingService, oc.chainConfig, oc.gasEstimationCap, oc.config, oc.logger)
+	result, err := evm.ExecuteCall(ctx, callMsg, blockState, batch.Header, oc.storage, oc.gethEncodingService, oc.chainConfig, oc.gasEstimationCap, oc.config, oc.logger)
 	if err != nil {
 		// also return the result as the result can be evaluated on some errors like ErrIntrinsicGas
 		return result, err
@@ -128,7 +128,7 @@ func (oc *obscuroChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.Tra
 
 // GetChainStateAtTransaction Returns the state of the chain at certain block height after executing transactions up to the selected transaction
 // TODO make this cacheable - why isn't this in the evm_facade?
-func (oc *obscuroChain) GetChainStateAtTransaction(ctx context.Context, batch *core.Batch, txIndex int, _ uint64) (*gethcore.Message, vm.BlockContext, *state.StateDB, error) {
+func (oc *tenChain) GetChainStateAtTransaction(ctx context.Context, batch *core.Batch, txIndex int, _ uint64) (*gethcore.Message, vm.BlockContext, *state.StateDB, error) {
 	// Short circuit if it's genesis batch.
 	if batch.NumberU64() == 0 {
 		return nil, vm.BlockContext{}, nil, errors.New("no transaction in genesis")

go/enclave/main/enclave.json

@@ -25,6 +25,7 @@
     { "fromHost": true, "name": "ENCLAVE_DEBUG_ENABLEDEBUGNAMESPACE" },
     { "fromHost": true, "name": "ENCLAVE_DEBUG_ENABLEPROFILER" },
     { "fromHost": true, "name": "ENCLAVE_ENABLEATTESTATION" },
+    { "fromHost": true, "name": "ENCLAVE_STOREEXECUTEDTRANSACTIONS" },
     { "fromHost": true, "name": "ENCLAVE_L1_ENABLEBLOCKVALIDATION" },
     { "fromHost": true, "name": "ENCLAVE_L1_GENESISJSON" },
     { "fromHost": true, "name": "ENCLAVE_LOG_LEVEL" },

go/enclave/rpc/GetLogs.go

@@ -14,7 +14,10 @@ import (
 	"github.com/ten-protocol/go-ten/go/common/syserr"
 )
 
-func GetLogsValidate(reqParams []any, builder *CallBuilder[filters.FilterCriteria, []*types.Log], _ *EncryptionManager) error {
+func GetLogsValidate(reqParams []any, builder *CallBuilder[filters.FilterCriteria, []*types.Log], rpc *EncryptionManager) error {
+	if !storeTxEnabled(rpc, builder) {
+		return nil
+	}
 	// Parameters are [Filter]
 	if len(reqParams) != 1 {
 		builder.Err = fmt.Errorf("unexpected number of parameters")

go/enclave/rpc/GetPersonalTransactions.go

@@ -9,7 +9,11 @@ import (
 	"github.com/ten-protocol/go-ten/go/common/gethencoding"
 )
 
-func GetPersonalTransactionsValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], _ *EncryptionManager) error {
+func GetPersonalTransactionsValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], rpc *EncryptionManager) error {
+	if !storeTxEnabled(rpc, builder) {
+		return nil
+	}
+
 	// Parameters are [PrivateTransactionListParams]
 	if len(reqParams) != 1 {
 		builder.Err = fmt.Errorf("unexpected number of parameters (expected %d, got %d)", 1, len(reqParams))

go/enclave/rpc/GetTransaction.go

@@ -14,7 +14,10 @@ import (
 	"github.com/ten-protocol/go-ten/go/common/errutil"
 )
 
-func GetTransactionValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, RpcTransaction], _ *EncryptionManager) error {
+func GetTransactionValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, RpcTransaction], rpc *EncryptionManager) error {
+	if !storeTxEnabled(rpc, builder) {
+		return nil
+	}
 	// Parameters are [Hash]
 	if len(reqParams) != 1 {
 		builder.Err = fmt.Errorf("wrong parameters")

go/enclave/rpc/GetTransactionReceipt.go

@@ -16,7 +16,10 @@ import (
 	"github.com/ten-protocol/go-ten/go/common/log"
 )
 
-func GetTransactionReceiptValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, map[string]interface{}], _ *EncryptionManager) error {
+func GetTransactionReceiptValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, map[string]interface{}], rpc *EncryptionManager) error {
+	if !storeTxEnabled(rpc, builder) {
+		return nil
+	}
 	// Parameters are [Hash]
 	if len(reqParams) < 1 {
 		builder.Err = fmt.Errorf("unexpected number of parameters")

go/enclave/rpc/TenEthCall.go

@@ -49,7 +49,7 @@ func TenCallExecute(builder *CallBuilder[CallParamsWithBlock, string], rpc *Encr
 
 	apiArgs := builder.Param.callParams
 	blkNumber := builder.Param.block
-	execResult, err := rpc.chain.ObsCall(builder.ctx, apiArgs, blkNumber)
+	execResult, err := rpc.chain.Call(builder.ctx, apiArgs, blkNumber)
 	if err != nil {
 		rpc.logger.Debug("Failed eth_call.", log.ErrKey, err)
 

go/enclave/rpc/rpc_utils.go

@@ -28,3 +28,11 @@ type CallParamsWithBlock struct {
 	callParams *gethapi.TransactionArgs
 	block      *gethrpc.BlockNumber
 }
+
+func storeTxEnabled[P any, R any](rpc *EncryptionManager, builder *CallBuilder[P, R]) bool {
+	if !rpc.config.StoreExecutedTransactions {
+		builder.Err = fmt.Errorf("the current TEN enclave is not configured to respond to this query")
+		return false
+	}
+	return true
+}

go/enclave/storage/storage.go

@@ -58,6 +58,7 @@ type storageImpl struct {
 
 	stateCache  state.Database
 	chainConfig *params.ChainConfig
+	config      *enclaveconfig.EnclaveConfig
 	logger      gethlog.Logger
 }
 
@@ -66,7 +67,7 @@ func NewStorageFromConfig(config *enclaveconfig.EnclaveConfig, cachingService *C
 	if err != nil {
 		logger.Crit("Failed to connect to backing database", log.ErrKey, err)
 	}
-	return NewStorage(backingDB, cachingService, chainConfig, logger)
+	return NewStorage(backingDB, cachingService, config, chainConfig, logger)
 }
 
 var defaultCacheConfig = &gethcore.CacheConfig{
@@ -86,7 +87,7 @@ var trieDBConfig = &triedb.Config{
 	},
 }
 
-func NewStorage(backingDB enclavedb.EnclaveDB, cachingService *CacheService, chainConfig *params.ChainConfig, logger gethlog.Logger) Storage {
+func NewStorage(backingDB enclavedb.EnclaveDB, cachingService *CacheService, config *enclaveconfig.EnclaveConfig, chainConfig *params.ChainConfig, logger gethlog.Logger) Storage {
 	// Open trie database with provided config
 	triedb := triedb.NewDatabase(backingDB, trieDBConfig)
 
@@ -96,6 +97,7 @@ func NewStorage(backingDB enclavedb.EnclaveDB, cachingService *CacheService, cha
 		db:             backingDB,
 		stateCache:     stateDB,
 		chainConfig:    chainConfig,
+		config:         config,
 		cachingService: cachingService,
 		eventsStorage:  newEventsStorage(cachingService, backingDB, logger),
 		logger:         logger,
@@ -668,12 +670,15 @@ func (s *storageImpl) StoreExecutedBatch(ctx context.Context, batch *core.Batch,
 		return fmt.Errorf("could not write synthetic txs. Cause: %w", err)
 	}
 
-	for _, txExecResult := range results {
-		err = s.eventsStorage.storeReceiptAndEventLogs(ctx, dbTx, batch.Header, txExecResult)
-		if err != nil {
-			return fmt.Errorf("could not store receipt. Cause: %w", err)
+	if s.config.StoreExecutedTransactions {
+		for _, txExecResult := range results {
+			err = s.eventsStorage.storeReceiptAndEventLogs(ctx, dbTx, batch.Header, txExecResult)
+			if err != nil {
+				return fmt.Errorf("could not store receipt. Cause: %w", err)
+			}
 		}
 	}
+
 	if err = dbTx.Commit(); err != nil {
 		return fmt.Errorf("could not commit batch %w", err)
 	}

go/enclave/system/contracts.go

@@ -15,7 +15,7 @@ func GenerateDeploymentTransaction(initCode []byte, logger gethlog.Logger) (*com
 	tx := &types.LegacyTx{
 		Nonce:    0, // The first transaction of the owner identity should always be deploying the contract
 		Value:    gethcommon.Big0,
-		Gas:      500_000_000,     // It's quite the expensive contract.
+		Gas:      10_000_000,      // It's quite the expensive contract.
 		GasPrice: gethcommon.Big0, // Synthetic transactions are on the house. Or the house.
 		Data:     initCode,        // gethcommon.FromHex(SystemDeployer.SystemDeployerMetaData.Bin),
 		To:       nil,             // Geth requires nil instead of gethcommon.Address{} which equates to zero address in order to return receipt.

go/node/cmd/cli.go

@@ -216,5 +216,11 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig {
 	tenCfg.Enclave.RPC.BindAddress = fmt.Sprintf("0.0.0.0:%d", cliCfg.enclaveWSPort)
 	tenCfg.Enclave.Log.Level = cliCfg.logLevel
 
+	// the sequencer does not store the executed transactions
+	// todo - once we replace this launcher we'll configure this flag explicitly via an environment variable
+	if nodeType == common.ActiveSequencer {
+		tenCfg.Enclave.StoreExecutedTransactions = false
+	}
+
 	return tenCfg
 }

integration/common/constants.go

@@ -88,8 +88,9 @@ func DefaultEnclaveConfig() *enclaveconfig.EnclaveConfig {
 		// Due to hiding L1 costs in the gas quantity, the gas limit needs to be huge
 		// Arbitrum with the same approach has gas limit of 1,125,899,906,842,624,
 		// whilst the usage is small. Should be ok since execution is paid for anyway.
-		GasLocalExecutionCapFlag: 300_000_000_000,
-		GasBatchExecutionLimit:   300_000_000_000,
-		RPCTimeout:               5 * time.Second,
+		GasLocalExecutionCapFlag:  300_000_000_000,
+		GasBatchExecutionLimit:    30_000_000,
+		RPCTimeout:                5 * time.Second,
+		StoreExecutedTransactions: true,
 	}
 }

integration/simulation/devnetwork/node.go

@@ -223,6 +223,7 @@ func (n *InMemNodeOperator) createEnclaveContainer(idx int) *enclavecontainer.En
 		GasPaymentAddress:         defaultCfg.GasPaymentAddress,
 		RPCTimeout:                5 * time.Second,
 		SystemContractOwner:       gethcommon.HexToAddress("0xA58C60cc047592DE97BF1E8d2f225Fc5D959De77"),
+		StoreExecutedTransactions: true,
 	}
 	return enclavecontainer.NewEnclaveContainerWithLogger(enclaveConfig, enclaveLogger)
 }

integration/simulation/network/network_utils.go

@@ -103,6 +103,7 @@ func createInMemTenNode(
 		GasLocalExecutionCapFlag:  params.MaxGasLimit / 2,
 		GasBatchExecutionLimit:    params.MaxGasLimit / 2,
 		RPCTimeout:                5 * time.Second,
+		StoreExecutedTransactions: true,
 	}
 
 	enclaveLogger := testlog.Logger().New(log.NodeIDKey, id, log.CmpKey, log.EnclaveCmp)

integration/simulation/transaction_injector.go

@@ -447,7 +447,7 @@ func (ti *TransactionInjector) issueRandomWithdrawals() {
 		tx := &types.LegacyTx{
 			Nonce:    fromWallet.GetNonceAndIncrement(),
 			Value:    gethcommon.Big1,
-			Gas:      uint64(1_000_000_000),
+			Gas:      uint64(1_000_000),
 			GasPrice: price,
 			Data:     nil,
 			To:       &msgBusAddr,
@@ -530,7 +530,7 @@ func (ti *TransactionInjector) newTx(data []byte, nonce uint64, ercType testcomm
 	return &types.LegacyTx{
 		Nonce:    nonce,
 		Value:    gethcommon.Big0,
-		Gas:      uint64(1_000_000_000),
+		Gas:      uint64(1_000_000),
 		GasPrice: gethcommon.Big1,
 		Data:     data,
 		To:       ti.wallets.Tokens[ercType].L2ContractAddress,