Mempool Perf fix

go.mod

@@ -7,11 +7,13 @@ replace github.com/docker/docker => github.com/docker/docker v20.10.3-0.20220224
 require (
 	github.com/allegro/bigcache/v3 v3.1.0
 	github.com/andybalholm/brotli v1.0.5
+	github.com/dgraph-io/ristretto v0.1.1
 	github.com/docker/docker v1.6.2
 	github.com/docker/go-connections v0.4.0
 	github.com/edgelesssys/ego v1.1.0
-	github.com/eko/gocache/lib/v4 v4.1.2
-	github.com/eko/gocache/store/bigcache/v4 v4.1.2
+	github.com/eko/gocache/lib/v4 v4.1.5
+	github.com/eko/gocache/store/bigcache/v4 v4.2.1
+	github.com/eko/gocache/store/ristretto/v4 v4.2.1
 	github.com/ethereum/go-ethereum v1.12.2
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-gonic/gin v1.9.1
@@ -65,6 +67,7 @@ require (
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/ethereum/c-kzg-4844 v0.3.1 // indirect
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
@@ -82,6 +85,7 @@ require (
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/mock v1.6.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect

go.sum

@@ -129,7 +129,10 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etly
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
 github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
 github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
 github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -140,6 +143,7 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
@@ -148,10 +152,12 @@ github.com/edgelesssys/ego v1.1.0 h1:UcDiGGJ8PF8YStlticxi2hMgPqRTtP42FzaGaAxm9Ys
 github.com/edgelesssys/ego v1.1.0/go.mod h1:ex4cDvgi0l6wxDm5xBaQzJqi547FMPDsxv+3ERLJfLI=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
-github.com/eko/gocache/lib/v4 v4.1.2 h1:cX54GhJJsfc5jvCEaPW8595h9Pq6bbNfkv0o/669Tw4=
-github.com/eko/gocache/lib/v4 v4.1.2/go.mod h1:FqyrANKct257VFHVVs11m6V2syGobOmHycQCyRSMwu0=
-github.com/eko/gocache/store/bigcache/v4 v4.1.2 h1:8uMDpgxTG7BvyLHBFqL3Ao809bVrXfrWqo7v6ALiwTw=
-github.com/eko/gocache/store/bigcache/v4 v4.1.2/go.mod h1:Lut5Sk/yI835w02tmwx4ecezYQo445L5sdICsk1zgho=
+github.com/eko/gocache/lib/v4 v4.1.5 h1:CeMQmdIzwBKKLRjk3FCDXzNFsQTyqJ01JLI7Ib0C9r8=
+github.com/eko/gocache/lib/v4 v4.1.5/go.mod h1:XaNfCwW8KYW1bRZ/KoHA1TugnnkMz0/gT51NDIu7LSY=
+github.com/eko/gocache/store/bigcache/v4 v4.2.1 h1:xf9R5HZqmrfT4+NzlJPQJQUWftfWW06FHbjz4IEjE08=
+github.com/eko/gocache/store/bigcache/v4 v4.2.1/go.mod h1:Q9+hxUE+XUVGSRGP1tqW8sPHcZ50PfyBVh9VKh0OjrA=
+github.com/eko/gocache/store/ristretto/v4 v4.2.1 h1:xB5E1LP1gh8yUV1G3KVRSL4T0OTnxp4OixuTljn2848=
+github.com/eko/gocache/store/ristretto/v4 v4.2.1/go.mod h1:KyshDyWQqfSVrg2rH06fFQZTj6vG2fxlY7oAW9oxNHY=
 github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -245,6 +251,8 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
 github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog=
 github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -799,6 +807,7 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

go/common/cache_util.go

@@ -0,0 +1,33 @@
+package common
+
+import (
+	"context"
+
+	"github.com/eko/gocache/lib/v4/cache"
+	gethlog "github.com/ethereum/go-ethereum/log"
+	"github.com/ten-protocol/go-ten/go/common/log"
+)
+
+// GetCachedValue - returns the cached value for the provided key. If the key is not found, then invoke the 'onFailed' function
+// which returns the value, and cache it
+func GetCachedValue[V any](cache *cache.Cache[V], logger gethlog.Logger, key any, onFailed func(any) (V, error)) (V, error) {
+	value, err := cache.Get(context.Background(), key)
+	if err != nil {
+		// todo metrics for cache misses
+		b, err := onFailed(key)
+		if err != nil {
+			return b, err
+		}
+		CacheValue(cache, logger, key, b)
+		return b, err
+	}
+
+	return value, err
+}
+
+func CacheValue[V any](cache *cache.Cache[V], logger gethlog.Logger, key any, v V) {
+	err := cache.Set(context.Background(), key, v)
+	if err != nil {
+		logger.Error("Could not store value in cache", log.ErrKey, err)
+	}
+}

go/common/gethencoding/geth_encoding.go

@@ -5,9 +5,19 @@ import (
 	"fmt"
 	"math/big"
 	"strings"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/dgraph-io/ristretto"
+	"github.com/eko/gocache/lib/v4/cache"
+	ristretto_store "github.com/eko/gocache/store/ristretto/v4"
+
+	gethlog "github.com/ethereum/go-ethereum/log"
+	"github.com/ten-protocol/go-ten/go/common/log"
+	"github.com/ten-protocol/go-ten/go/enclave/storage"
 
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ten-protocol/go-ten/go/common"
 	"github.com/ten-protocol/go-ten/go/enclave/core"
 	"github.com/ten-protocol/go-ten/go/enclave/crypto"
@@ -33,6 +43,39 @@ const (
 	callFieldMaxPriorityFeePerGas = "maxpriorityfeepergas"
 )
 
+// EncodingService handles conversion to Geth data structures
+type EncodingService interface {
+	CreateEthHeaderForBatch(h *common.BatchHeader) (*types.Header, error)
+	CreateEthBlockFromBatch(b *core.Batch) (*types.Block, error)
+}
+
+type gethEncodingServiceImpl struct {
+	// conversion is expensive. Cache the converted headers. The key is the hash of the batch.
+	gethHeaderCache *cache.Cache[*types.Header]
+
+	storage storage.Storage
+	logger  gethlog.Logger
+}
+
+func NewGethEncodingService(storage storage.Storage, logger gethlog.Logger) EncodingService {
+	// todo (tudor) figure out the best values
+	ristrettoCache, err := ristretto.NewCache(&ristretto.Config{
+		NumCounters: 1000,    // number of keys to track frequency of 100.
+		MaxCost:     1 << 28, // maximum cost of cache (256MB).
+		BufferItems: 64,      // number of keys per Get buffer. Todo - what is this
+	})
+	if err != nil {
+		panic(err)
+	}
+	ristrettoStore := ristretto_store.NewRistretto(ristrettoCache)
+
+	return &gethEncodingServiceImpl{
+		gethHeaderCache: cache.New[*types.Header](ristrettoStore),
+		storage:         storage,
+		logger:          logger,
+	}
+}
+
 // ExtractEthCallMapString extracts the eth_call gethapi.TransactionArgs from an interface{}
 // it ensures that :
 // - All types are string
@@ -226,41 +269,94 @@ func ExtractEthCall(param interface{}) (*gethapi.TransactionArgs, error) {
 	return callMsg, nil
 }
 
-// CreateEthHeaderForBatch - the EVM requires an Ethereum "block" header.
-// In this function we are creating one from the Batch Header
-func CreateEthHeaderForBatch(h *common.BatchHeader, secret []byte) (*types.Header, error) {
-	// deterministically calculate private randomness that will be exposed to the evm
-	randomness := crypto.CalculateRootBatchEntropy(secret, h.Number)
+// CreateEthHeaderForBatch - the EVM requires an Ethereum header.
+// We convert the Batch headers to Ethereum headers to be able to use the Geth EVM.
+// Special care must be taken to maintain a valid chain of these converted headers.
+func (enc *gethEncodingServiceImpl) CreateEthHeaderForBatch(h *common.BatchHeader) (*types.Header, error) {
+	// wrap in a caching layer
+	return common.GetCachedValue(enc.gethHeaderCache, enc.logger, h.Hash(), func(a any) (*types.Header, error) {
+		// deterministically calculate the private randomness that will be exposed to the EVM
+		secret, err := enc.storage.FetchSecret()
+		if err != nil {
+			enc.logger.Crit("Could not fetch shared secret. Exiting.", log.ErrKey, err)
+		}
+		perBatchRandomness := crypto.CalculateRootBatchEntropy(secret[:], h.Number)
 
-	baseFee := uint64(0)
-	if h.BaseFee != nil {
-		baseFee = h.BaseFee.Uint64()
-	}
+		// calculate the converted hash of the parent, for a correct converted chain
+		// default to the genesis
+		convertedParentHash := common.GethGenesisParentHash
+
+		if h.SequencerOrderNo.Uint64() > common.L2GenesisSeqNo {
+			convertedParentHash, err = enc.storage.FetchConvertedHash(h.ParentHash)
+			if err != nil {
+				enc.logger.Error("Cannot find the converted value for the parent of", log.BatchSeqNoKey, h.SequencerOrderNo)
+				return nil, err
+			}
+		}
 
-	return &types.Header{
-		ParentHash:  h.ParentHash,
-		Root:        h.Root,
-		TxHash:      h.TxHash,
-		ReceiptHash: h.ReceiptHash,
-		Difficulty:  big.NewInt(0),
-		Number:      h.Number,
-		GasLimit:    h.GasLimit,
-		GasUsed:     h.GasUsed,
-		BaseFee:     big.NewInt(0).SetUint64(baseFee),
-		Coinbase:    h.Coinbase,
-		Time:        h.Time,
-		MixDigest:   randomness,
-		Nonce:       types.BlockNonce{},
-	}, nil
+		baseFee := uint64(0)
+		if h.BaseFee != nil {
+			baseFee = h.BaseFee.Uint64()
+		}
+
+		gethHeader := types.Header{
+			ParentHash:      convertedParentHash,
+			UncleHash:       gethcommon.Hash{},
+			Root:            h.Root,
+			TxHash:          h.TxHash,
+			ReceiptHash:     h.ReceiptHash,
+			Difficulty:      big.NewInt(0),
+			Number:          h.Number,
+			GasLimit:        h.GasLimit,
+			GasUsed:         h.GasUsed,
+			BaseFee:         big.NewInt(0).SetUint64(baseFee),
+			Coinbase:        h.Coinbase,
+			Time:            h.Time,
+			MixDigest:       perBatchRandomness,
+			Nonce:           types.BlockNonce{},
+			Extra:           h.SequencerOrderNo.Bytes(),
+			WithdrawalsHash: nil,
+			BlobGasUsed:     nil,
+			ExcessBlobGas:   nil,
+			Bloom:           types.Bloom{},
+		}
+		return &gethHeader, nil
+	})
 }
 
-func CreateEthBlockFromBatch(b *core.Batch) (*types.Block, error) {
-	blockHeader, err := CreateEthHeaderForBatch(b.Header, nil)
+// The Geth "Block" type doesn't expose the header directly.
+// This type is required for adjusting the header.
+type localBlock struct {
+	header       *types.Header
+	uncles       []*types.Header
+	transactions types.Transactions
+	withdrawals  types.Withdrawals
+
+	// caches
+	hash atomic.Value
+	size atomic.Value
+
+	// These fields are used by package eth to track
+	// inter-peer block relay.
+	ReceivedAt   time.Time
+	ReceivedFrom interface{}
+}
+
+func (enc *gethEncodingServiceImpl) CreateEthBlockFromBatch(b *core.Batch) (*types.Block, error) {
+	blockHeader, err := enc.CreateEthHeaderForBatch(b.Header)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create eth block from batch - %w", err)
 	}
 
-	return types.NewBlock(blockHeader, b.Transactions, nil, nil, trie.NewStackTrie(nil)), nil
+	// adjust the header of the returned block to make sure the hashes align
+	lb := localBlock{
+		header:       blockHeader,
+		uncles:       nil,
+		transactions: b.Transactions,
+		withdrawals:  nil,
+	}
+	block := *(*types.Block)(unsafe.Pointer(&lb))
+	return &block, nil
 }
 
 // DecodeParamBytes decodes the parameters byte array into a slice of interfaces

go/common/types.go

@@ -80,6 +80,8 @@ const (
 	HeightCommittedBlocks = 15
 )
 
+var GethGenesisParentHash = common.Hash{}
+
 // SystemError is the interface for the internal errors generated in the Enclave and consumed by the Host
 type SystemError interface {
 	Error() string

go/enclave/components/batch_executor.go

@@ -8,6 +8,8 @@ import (
 	"sort"
 	"sync"
 
+	"github.com/ten-protocol/go-ten/go/common/gethencoding"
+
 	"github.com/ten-protocol/go-ten/go/enclave/gas"
 	"github.com/ten-protocol/go-ten/go/enclave/storage"
 
@@ -34,6 +36,7 @@ var ErrNoTransactionsToProcess = fmt.Errorf("no transactions to process")
 // batchExecutor - the component responsible for executing batches
 type batchExecutor struct {
 	storage              storage.Storage
+	gethEncodingService  gethencoding.EncodingService
 	crossChainProcessors *crosschain.Processors
 	genesis              *genesis.Genesis
 	logger               gethlog.Logger
@@ -48,6 +51,7 @@ type batchExecutor struct {
 
 func NewBatchExecutor(
 	storage storage.Storage,
+	gethEncodingService gethencoding.EncodingService,
 	cc *crosschain.Processors,
 	genesis *genesis.Genesis,
 	gasOracle gas.Oracle,
@@ -57,6 +61,7 @@ func NewBatchExecutor(
 ) BatchExecutor {
 	return &batchExecutor{
 		storage:              storage,
+		gethEncodingService:  gethEncodingService,
 		crossChainProcessors: cc,
 		genesis:              genesis,
 		chainConfig:          chainConfig,
@@ -417,6 +422,7 @@ func (executor *batchExecutor) processTransactions(
 		stateDB,
 		batch.Header,
 		executor.storage,
+		executor.gethEncodingService,
 		cc,
 		tCount,
 		noBaseFee,