[Gas Mechanics] Implement Gas history; Migrate to Arbitrum Gas model (#…

…1714)

* Fixes.

* More fixes.

* Fixes?.

* log change.

* More fixes for sim tests.

* Fix for native transfers.

* gas limit change.

* Something is working

* Comments.

* Add somehow deleted favicon?

* Fix for merge error.

* Fix for build.

* Disable test again.

* Changes to gateway.

* Fixes.

* Fixes for linter.

* Test fixes.

* Fix for xchain failures.

* Deployment fix.

* More deployment fixes.

* Maybe fix.

* Fix for faucet.

* Added comments.

* Linter and other unpushed stuff.

* Stupid linter.

* Fix for new test.

---------

Co-authored-by: StefanIliev545 <[email protected]>

contracts/deployment_scripts/testnet/layer2/001_whitelist_tokens.ts

@@ -97,12 +97,12 @@ const func: DeployFunction = async function (hre: HardhatRuntimeEnvironment) {
         setTimeout(fail, 30_000)
         const messageBusContract = (await hre.ethers.getContractAt('MessageBus', '0x526c84529b2b8c11f57d93d3f5537aca3aecef9b'));
         const gasLimit = await messageBusContract.estimateGas.verifyMessageFinalized(messages[1], {
-            maxFeePerGas: 2,
+            maxFeePerGas: 1000000001,
         })
         try {
             while (await messageBusContract.callStatic.verifyMessageFinalized(messages[1], {
-                maxFeePerGas: 2,
-                gasLimit: gasLimit.mul(2),
+                maxFeePerGas: 1000000001,
+                gasLimit: gasLimit.add(gasLimit.div(2)),
                 from: l2Accounts.deployer
             }) != true) {
                 console.log(`Messages not stored on L2 yet, retrying...`);

go/common/types.go

@@ -44,6 +44,12 @@ type (
 	L2Receipt      = types.Receipt
 	L2Receipts     = types.Receipts
 
+	L2PricedTransaction struct {
+		Tx             *L2Tx
+		PublishingCost *big.Int
+	}
+	L2PricedTransactions []L2PricedTransaction
+
 	CrossChainMessage  = MessageBus.StructsCrossChainMessage
 	CrossChainMessages = []CrossChainMessage
 	ValueTransferEvent struct {
@@ -72,6 +78,14 @@ type (
 	EncodedBatchRequest []byte
 )
 
+func (txs L2PricedTransactions) ToTransactions() types.Transactions {
+	ret := make(types.Transactions, 0)
+	for _, tx := range txs {
+		ret = append(ret, tx.Tx)
+	}
+	return ret
+}
+
 const (
 	L2GenesisHeight = uint64(0)
 	L1GenesisHeight = uint64(0)

go/config/enclave_cli_flags.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"github.com/ethereum/go-ethereum/params"
 	"github.com/ten-protocol/go-ten/go/common"
 	"github.com/ten-protocol/go-ten/go/common/flag"
 )
@@ -53,16 +54,16 @@ var EnclaveFlags = map[string]*flag.TenFlag{
 	SequencerIDFlag:               flag.NewStringFlag(SequencerIDFlag, "", "The 20 bytes of the address of the sequencer for this network"),
 	MaxBatchSizeFlag:              flag.NewUint64Flag(MaxBatchSizeFlag, 1024*25, "The maximum size a batch is allowed to reach uncompressed"),
 	MaxRollupSizeFlag:             flag.NewUint64Flag(MaxRollupSizeFlag, 1024*64, "The maximum size a rollup is allowed to reach"),
-	L2BaseFeeFlag:                 flag.NewUint64Flag(L2BaseFeeFlag, 1, ""),
+	L2BaseFeeFlag:                 flag.NewUint64Flag(L2BaseFeeFlag, params.InitialBaseFee, ""),
 	L2CoinbaseFlag:                flag.NewStringFlag(L2CoinbaseFlag, "0xd6C9230053f45F873Cb66D8A02439380a37A4fbF", ""),
-	GasBatchExecutionLimit:        flag.NewUint64Flag(GasBatchExecutionLimit, 30_000_000, "Max gas that can be executed in a single batch"),
+	GasBatchExecutionLimit:        flag.NewUint64Flag(GasBatchExecutionLimit, 3_000_000_000, "Max gas that can be executed in a single batch"),
 	ObscuroGenesisFlag:            flag.NewStringFlag(ObscuroGenesisFlag, "", "The json string with the obscuro genesis"),
 	L1ChainIDFlag:                 flag.NewInt64Flag(L1ChainIDFlag, 1337, "An integer representing the unique chain id of the Ethereum chain used as an L1 (default 1337)"),
 	ObscuroChainIDFlag:            flag.NewInt64Flag(ObscuroChainIDFlag, 443, "An integer representing the unique chain id of the Obscuro chain (default 443)"),
 	UseInMemoryDBFlag:             flag.NewBoolFlag(UseInMemoryDBFlag, true, "Whether the enclave will use an in-memory DB rather than persist data"),
 	ProfilerEnabledFlag:           flag.NewBoolFlag(ProfilerEnabledFlag, false, "Runs a profiler instance (Defaults to false)"),
 	DebugNamespaceEnabledFlag:     flag.NewBoolFlag(DebugNamespaceEnabledFlag, false, "Whether the debug namespace is enabled"),
-	GasLocalExecutionCapFlag:      flag.NewUint64Flag(GasLocalExecutionCapFlag, 40_000_000, "Max gas usage when executing local transactions"),
+	GasLocalExecutionCapFlag:      flag.NewUint64Flag(GasLocalExecutionCapFlag, 4_000_000_000, "Max gas usage when executing local transactions"),
 }
 
 // enclaveRestrictedFlags are the flags that the enclave can receive ONLY over the Ego signed enclave.json

go/enclave/components/batch_executor.go

@@ -72,12 +72,12 @@ func NewBatchExecutor(
 	}
 }
 
-// payL1Fees - this function modifies the state db according to the transactions contained within the batch context
-// in order to subtract gas fees from the balance. It returns a list of the transactions that have prepaid for their L1
-// publishing costs.
-func (executor *batchExecutor) payL1Fees(stateDB *state.StateDB, context *BatchExecutionContext) (common.L2Transactions, common.L2Transactions) {
-	transactions := make(common.L2Transactions, 0)
-	freeTransactions := make(common.L2Transactions, 0)
+// filterTransactionsWithSufficientFunds - this function estimates hte l1 fees for the transaction in a given batch execution context. It does so by taking the price of the
+// pinned L1 block and using it as the cost per gas for the estimated gas of the calldata encoding of a transaction. It filters out any transactions that cannot afford to pay for their L1
+// publishing cost.
+func (executor *batchExecutor) filterTransactionsWithSufficientFunds(stateDB *state.StateDB, context *BatchExecutionContext) (common.L2PricedTransactions, common.L2PricedTransactions) {
+	transactions := make(common.L2PricedTransactions, 0)
+	freeTransactions := make(common.L2PricedTransactions, 0)
 	block, _ := executor.storage.FetchBlock(context.BlockPtr)
 
 	for _, tx := range context.Transactions {
@@ -102,43 +102,25 @@ func (executor *batchExecutor) payL1Fees(stateDB *state.StateDB, context *BatchE
 		isFreeTransaction = isFreeTransaction && tx.GasPrice().Cmp(gethcommon.Big0) == 0
 
 		if isFreeTransaction {
-			freeTransactions = append(freeTransactions, tx)
+			freeTransactions = append(freeTransactions, common.L2PricedTransaction{
+				Tx:             tx,
+				PublishingCost: big.NewInt(0),
+			})
 			continue
 		}
 		if accBalance.Cmp(cost) == -1 {
 			executor.logger.Info(fmt.Sprintf("insufficient account balance for tx - want: %d have: %d", cost, accBalance), log.TxKey, tx.Hash(), "addr", sender.Hex())
 			continue
 		}
-		stateDB.SubBalance(*sender, cost)
-		stateDB.AddBalance(context.Creator, cost)
-		// todo - add refund logic.
 
-		transactions = append(transactions, tx)
+		transactions = append(transactions, common.L2PricedTransaction{
+			Tx:             tx,
+			PublishingCost: big.NewInt(0).Set(cost),
+		})
 	}
 	return transactions, freeTransactions
 }
 
-func (executor *batchExecutor) refundL1Fees(stateDB *state.StateDB, context *BatchExecutionContext, transactions []*common.L2Tx) {
-	block, _ := executor.storage.FetchBlock(context.BlockPtr)
-	for _, tx := range transactions {
-		cost, err := executor.gasOracle.EstimateL1StorageGasCost(tx, block)
-		if err != nil {
-			executor.logger.Warn("Unable to get gas cost for tx", log.TxKey, tx.Hash(), log.ErrKey, err)
-			continue
-		}
-
-		sender, err := core.GetAuthenticatedSender(context.ChainConfig.ChainID.Int64(), tx)
-		if err != nil {
-			// todo @siliev - is this critical? Potential desync spot
-			executor.logger.Warn("Unable to extract sender for tx", log.TxKey, tx.Hash())
-			continue
-		}
-
-		stateDB.AddBalance(*sender, cost)
-		stateDB.SubBalance(context.Creator, cost)
-	}
-}
-
 func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext, failForEmptyBatch bool) (*ComputedBatch, error) { //nolint:gocognit
 	defer core.LogMethodDuration(executor.logger, measure.NewStopwatch(), "Batch context processed")
 
@@ -188,23 +170,29 @@ func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext, fail
 	crossChainTransactions := executor.crossChainProcessors.Local.CreateSyntheticTransactions(messages, stateDB)
 	executor.crossChainProcessors.Local.ExecuteValueTransfers(transfers, stateDB)
 
-	transactionsToProcess, freeTransactions := executor.payL1Fees(stateDB, context)
+	transactionsToProcess, freeTransactions := executor.filterTransactionsWithSufficientFunds(stateDB, context)
+
+	xchainTxs := make(common.L2PricedTransactions, 0)
+	for _, xTx := range crossChainTransactions {
+		xchainTxs = append(xchainTxs, common.L2PricedTransaction{
+			Tx:             xTx,
+			PublishingCost: big.NewInt(0),
+		})
+	}
 
-	crossChainTransactions = append(crossChainTransactions, freeTransactions...)
+	syntheticTransactions := append(xchainTxs, freeTransactions...)
 
 	successfulTxs, excludedTxs, txReceipts, err := executor.processTransactions(batch, 0, transactionsToProcess, stateDB, context.ChainConfig, false)
 	if err != nil {
 		return nil, fmt.Errorf("could not process transactions. Cause: %w", err)
 	}
 
-	executor.refundL1Fees(stateDB, context, excludedTxs)
-
-	ccSuccessfulTxs, _, ccReceipts, err := executor.processTransactions(batch, len(successfulTxs), crossChainTransactions, stateDB, context.ChainConfig, true)
+	ccSuccessfulTxs, _, ccReceipts, err := executor.processTransactions(batch, len(successfulTxs), syntheticTransactions, stateDB, context.ChainConfig, true)
 	if err != nil {
 		return nil, err
 	}
 
-	if err = executor.verifyInboundCrossChainTransactions(crossChainTransactions, ccSuccessfulTxs, ccReceipts); err != nil {
+	if err = executor.verifyInboundCrossChainTransactions(syntheticTransactions, ccSuccessfulTxs, ccReceipts); err != nil {
 		return nil, fmt.Errorf("batch computation failed due to cross chain messages. Cause: %w", err)
 	}
 
@@ -225,7 +213,7 @@ func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext, fail
 	// we need to copy the batch to reset the internal hash cache
 	copyBatch := *batch
 	copyBatch.Header.Root = stateDB.IntermediateRoot(false)
-	copyBatch.Transactions = append(successfulTxs, freeTransactions...)
+	copyBatch.Transactions = append(successfulTxs, freeTransactions.ToTransactions()...)
 	copyBatch.ResetHash()
 
 	if err = executor.populateOutboundCrossChainData(&copyBatch, block, txReceipts); err != nil {
@@ -392,8 +380,8 @@ func (executor *batchExecutor) populateHeader(batch *core.Batch, receipts types.
 	}
 }
 
-func (executor *batchExecutor) verifyInboundCrossChainTransactions(transactions types.Transactions, executedTxs types.Transactions, receipts types.Receipts) error {
-	if transactions.Len() != executedTxs.Len() {
+func (executor *batchExecutor) verifyInboundCrossChainTransactions(transactions common.L2PricedTransactions, executedTxs types.Transactions, receipts types.Receipts) error {
+	if len(transactions) != executedTxs.Len() {
 		return fmt.Errorf("some synthetic transactions have not been executed")
 	}
 
@@ -409,7 +397,7 @@ func (executor *batchExecutor) verifyInboundCrossChainTransactions(transactions
 func (executor *batchExecutor) processTransactions(
 	batch *core.Batch,
 	tCount int,
-	txs []*common.L2Tx,
+	txs common.L2PricedTransactions,
 	stateDB *state.StateDB,
 	cc *params.ChainConfig,
 	noBaseFee bool,
@@ -430,18 +418,18 @@ func (executor *batchExecutor) processTransactions(
 		executor.logger,
 	)
 	for _, tx := range txs {
-		result, f := txResults[tx.Hash()]
+		result, f := txResults[tx.Tx.Hash()]
 		if !f {
 			return nil, nil, nil, fmt.Errorf("there should be an entry for each transaction")
 		}
 		rec, foundReceipt := result.(*types.Receipt)
 		if foundReceipt {
-			executedTransactions = append(executedTransactions, tx)
+			executedTransactions = append(executedTransactions, tx.Tx)
 			txReceipts = append(txReceipts, rec)
 		} else {
 			// Exclude all errors
-			excludedTransactions = append(excludedTransactions, tx)
-			executor.logger.Info("Excluding transaction from batch", log.TxKey, tx.Hash(), log.BatchHashKey, batch.Hash(), "cause", result)
+			excludedTransactions = append(excludedTransactions, tx.Tx)
+			executor.logger.Info("Excluding transaction from batch", log.TxKey, tx.Tx.Hash(), log.BatchHashKey, batch.Hash(), "cause", result)
 		}
 	}
 	sort.Sort(sortByTxIndex(txReceipts))

go/enclave/crosschain/message_bus_manager.go

@@ -86,7 +86,7 @@ func (m *MessageBusManager) GenerateMessageBusDeployTx() (*common.L2Tx, error) {
 	tx := &types.LegacyTx{
 		Nonce:    0, // The first transaction of the owner identity should always be deploying the contract
 		Value:    gethcommon.Big0,
-		Gas:      5_000_000,       // It's quite the expensive contract.
+		Gas:      500_000_000,     // It's quite the expensive contract.
 		GasPrice: gethcommon.Big0, // Synthetic transactions are on the house. Or the house.
 		Data:     gethcommon.FromHex(MessageBus.MessageBusMetaData.Bin),
 		To:       nil, // Geth requires nil instead of gethcommon.Address{} which equates to zero address in order to return receipt.

go/enclave/enclave.go

@@ -77,9 +77,10 @@ type enclaveImpl struct {
 	crossChainProcessors  *crosschain.Processors
 	sharedSecretProcessor *components.SharedSecretProcessor
 
-	chain    l2chain.ObscuroChain
-	service  nodetype.NodeType
-	registry components.BatchRegistry
+	chain     l2chain.ObscuroChain
+	service   nodetype.NodeType
+	registry  components.BatchRegistry
+	gasOracle gas.Oracle
 
 	mgmtContractLib     mgmtcontractlib.MgmtContractLib
 	attestationProvider components.AttestationProvider // interface for producing attestation reports and verifying them
@@ -273,9 +274,10 @@ func NewEnclave(
 		debugger:               debug,
 		stopControl:            stopcontrol.New(),
 
-		chain:    chain,
-		registry: registry,
-		service:  service,
+		chain:     chain,
+		registry:  registry,
+		service:   service,
+		gasOracle: gasOracle,
 
 		mainMutex: sync.Mutex{},
 	}
@@ -689,25 +691,36 @@ func (e *enclaveImpl) GetTransactionCount(encryptedParams common.EncryptedParams
 
 	address := gethcommon.HexToAddress(addressStr)
 
+	seqNo := e.registry.HeadBatchSeq().Uint64()
+	if len(paramList) == 3 {
+		tag, err := gethencoding.ExtractBlockNumber(paramList[2])
+		if err != nil {
+			return responses.AsPlaintextError(fmt.Errorf("unexpected tag parameter. Cause: %w", err)), nil
+		}
+
+		b, err := e.registry.GetBatchAtHeight(*tag)
+		if err != nil {
+			return responses.AsPlaintextError(fmt.Errorf("cant retrieve batch for tag. Cause: %w", err)), nil
+		}
+		seqNo = b.SeqNo().Uint64()
+	}
+
 	// extract, create and validate the VK encryption handler
 	vkHandler, err := createVKHandler(&address, paramList[0], e.config.ObscuroChainID)
 	if err != nil {
 		return responses.AsPlaintextError(fmt.Errorf("unable to create VK encryptor - %w", err)), nil
 	}
 
 	var nonce uint64
-	headBatch := e.registry.HeadBatchSeq()
-	if headBatch != nil {
-		l2Head, err := e.storage.FetchBatchBySeqNo(headBatch.Uint64())
-		if err == nil {
-			// todo - we should return an error when head state is not available, but for current test situations with race
-			//  conditions we allow it to return zero while head state is uninitialized
-			s, err := e.storage.CreateStateDB(l2Head.Hash())
-			if err != nil {
-				return nil, responses.ToInternalError(err)
-			}
-			nonce = s.GetNonce(address)
+	l2Head, err := e.storage.FetchBatchBySeqNo(seqNo)
+	if err == nil {
+		// todo - we should return an error when head state is not available, but for current test situations with race
+		//  conditions we allow it to return zero while head state is uninitialized
+		s, err := e.storage.CreateStateDB(l2Head.Hash())
+		if err != nil {
+			return nil, responses.ToInternalError(err)
 		}
+		nonce = s.GetNonce(address)
 	}
 
 	encoded := hexutil.EncodeUint64(nonce)
@@ -1036,6 +1049,32 @@ func (e *enclaveImpl) EstimateGas(encryptedParams common.EncryptedParamsEstimate
 		return responses.AsEncryptedError(err, vkHandler), nil
 	}
 
+	block, err := e.blockResolver.FetchHeadBlock()
+	if err != nil {
+		return responses.AsPlaintextError(fmt.Errorf("internal server error")), err
+	}
+
+	// The message is ran through the l1 publishing cost estimation for the current
+	// known head block.
+	l1Cost, err := e.gasOracle.EstimateL1CostForMsg(callMsg, block)
+	if err != nil {
+		return responses.AsPlaintextError(fmt.Errorf("internal server error")), err
+	}
+
+	batch, err := e.storage.FetchHeadBatch()
+	if err != nil {
+		return responses.AsPlaintextError(fmt.Errorf("internal server error")), err
+	}
+
+	// We divide the total estimated l1 cost by the l2 fee per gas in order to convert
+	// the expected cost into l2 gas based on current pricing.
+	// todo @siliev - add overhead when the base fee becomes dynamic.
+	publishingGas := big.NewInt(0).Div(l1Cost, batch.Header.BaseFee)
+
+	// The one additional gas captures the modulo leftover in some edge cases
+	// where BaseFee is bigger than the l1cost.
+	publishingGas = big.NewInt(0).Add(publishingGas, gethcommon.Big1)
+
 	executionGasEstimate, err := e.DoEstimateGas(callMsg, blockNumber, e.config.GasLocalExecutionCapFlag)
 	if err != nil {
 		err = fmt.Errorf("unable to estimate transaction - %w", err)
@@ -1053,7 +1092,9 @@ func (e *enclaveImpl) EstimateGas(encryptedParams common.EncryptedParamsEstimate
 		return responses.AsEncryptedError(err, vkHandler), nil
 	}
 
-	return responses.AsEncryptedResponse(&executionGasEstimate, vkHandler), nil
+	totalGasEstimate := hexutil.Uint64(publishingGas.Uint64() + uint64(executionGasEstimate))
+
+	return responses.AsEncryptedResponse(&totalGasEstimate, vkHandler), nil
 }
 
 func (e *enclaveImpl) GetLogs(encryptedParams common.EncryptedParamsGetLogs) (*responses.Logs, common.SystemError) { //nolint