merge fix

changelog.md

@@ -2,6 +2,43 @@
 ---
 # Ten Testnet Change Log
 
+# July 2024-07-03 (v0.25.0)
+* This is an L2 deployment release meaning state will be lost in order to upgrade the network. Any dApps on the
+  network will need to be redeployed. The release predominantly addresses performance improvements for large numbers
+  of transactions in the network, and for event log queries. 
+* A full list of the PRs merged in this release is as below;
+  * `ef0bf45e` Fix for xchain messages (#1973)
+  * `0dc7d5c5` Fix for integration test (#1971)
+  * `186bc2c3` Host db performance fixes (#1969)
+  * `c0ea9b43` Bundle submission rework (#1966)
+  * `0b01bfd1` Assets fix and clear console warnings (#1959)
+  * `ddff74a5` Update db schema (#1967)
+  * `e1c91243` Rework event logs and transaction database schema (#1961)
+  * `291a698d` Add db query tool (#1963)
+  * `b1af061b` Initiate bridge deployment (#1965)
+  * `95f35150` Initiate bridge deployment (#1960)
+  * `447ff489` Placeholder commit for new gh action workflow (#1964)
+  * `9d6e7747` Fix for startup (#1962)
+  * `b537d2e6` Add indexed events (#1954)
+  * `c6e5a48f` Adds log constraints to docker containers not instantiated by the go client (#1956)
+  * `85e6cd0d` Ignore empty l1 head status from enclave (#1955)
+  * `a9485fef` Fixes for ethereum bridge and json representation of batch header (#1953)
+  * `e6453082` Deploy separate ten gateway for dexynth (#1949)
+  * `b8935c31` Network tests: util func for l1 transfers (#1948)
+  * `0f3d42bf` Network tests: start gateway synchronously to fix race (#1947)
+  * `9cee2fd4` Increase max message size grpc config (#1946)
+  * `c0bf4086` Small mem pool fixes (#1943)
+  * `b335bd1e` Avoid spamming stuck l1 transactions (#1941)
+  * `99a8aeee` Fix panic on uninitialised mem pool (#1940)
+  * `1daffac0` Replace health-check db query (#1938)
+  * `a880b169` Hardcode response to client version request (#1937)
+  * `5780140e` Check conversions (#1936)
+  * `b97d0b93` Fix flakyness errors (#1927)
+  * `6e432a1a` Add net_version support and test (#1934)
+  * `4f39832e` Fixing log spam (#1931)
+  * `f8b86b52` Fix log spam (#1929)
+  * `762dfef9` Fix for interval (#1926)
+
 # May 2024-05-20 (v0.24.0)
 * This is an L2 deployment release meaning state will be lost in order to upgrade the network. Any dApps on the 
 network will need to be redeployed. High level changes made in the release include;

contracts/src/management/ManagementContract.sol

@@ -61,6 +61,7 @@ contract ManagementContract is Initializable, OwnableUpgradeable {
     function initialize() public initializer {
         __Ownable_init(msg.sender);
         lastBatchSeqNo = 0;
+        rollups.nextFreeSequenceNumber = 1; // rollup 0 == nil hash
         merkleMessageBus = new MerkleTreeMessageBus.MerkleTreeMessageBus();
         messageBus = MessageBus.IMessageBus(address(merkleMessageBus));
 
@@ -72,14 +73,36 @@ contract ManagementContract is Initializable, OwnableUpgradeable {
         return (rol.Hash == rollupHash , rol);
     }
 
+    function GetRollupByNumber(uint256 number) view public returns(bool, Structs.MetaRollup memory) {
+        bytes32 hash = rollups.byOrder[number];
+        if (hash == 0x0) { // ensure we don't try to get rollup for hash zero as that would not pull anything, but the hash would match and return true
+            return (false, Structs.MetaRollup(0x0, "", 0));
+        }
+
+        return GetRollupByHash(hash);
+    }
+
+    function GetUniqueForkID(uint256 number) view public returns(bytes32, Structs.MetaRollup memory) {
+        (bool success, Structs.MetaRollup memory rollup) = GetRollupByNumber(number);
+        if (!success) {
+            return (0x0, rollup);
+        }
+
+        return (rollups.toUniqueForkID[number], rollup);
+    }
+
     function AppendRollup(Structs.MetaRollup calldata _r) internal {
         rollups.byHash[_r.Hash] = _r;
+        rollups.byOrder[rollups.nextFreeSequenceNumber] = _r.Hash;
+        rollups.toUniqueForkID[rollups.nextFreeSequenceNumber] = keccak256(abi.encode(_r.Hash, blockhash(block.number-1)));
+        rollups.nextFreeSequenceNumber++;
+
         if (_r.LastSequenceNumber > lastBatchSeqNo) {
             lastBatchSeqNo = _r.LastSequenceNumber;
         }
     }
 
-    function addCrossChainMessagesRoot(bytes32 _lastBatchHash, bytes32 blockHash, uint256 blockNum, bytes[] memory crossChainHashes, bytes calldata signature) external {
+    function addCrossChainMessagesRoot(bytes32 _lastBatchHash, bytes32 blockHash, uint256 blockNum, bytes[] memory crossChainHashes, bytes calldata signature, uint256 rollupNumber, bytes32 forkID) external {
         if (block.number > blockNum + 255) {
             revert("Block binding too old");
         }
@@ -88,6 +111,10 @@ contract ManagementContract is Initializable, OwnableUpgradeable {
             revert(string(abi.encodePacked("Invalid block binding:", Strings.toString(block.number),":", Strings.toString(uint256(blockHash)), ":", Strings.toString(uint256(blockhash(blockNum))))));
         }
 
+        if (rollups.toUniqueForkID[rollupNumber] != forkID) {
+            revert("Invalid forkID");
+        }
+
         address enclaveID = ECDSA.recover(keccak256(abi.encode(_lastBatchHash, blockHash, blockNum, crossChainHashes)), signature);
         require(attested[enclaveID], "enclaveID not attested"); //todo: only sequencer, rather than everyone who has attested.
 

contracts/src/management/Structs.sol

@@ -12,6 +12,9 @@ interface Structs {
 
     struct RollupStorage {
         mapping(bytes32=>MetaRollup) byHash;
+        mapping(uint256=>bytes32) byOrder;
+        mapping(uint256=>bytes32) toUniqueForkID;
+        uint256 nextFreeSequenceNumber;
     }
 
     struct HeaderCrossChainData {

go/common/docker/docker.go

@@ -104,10 +104,11 @@ func StartNewContainer(containerName, image string, cmds []string, ports []int,
 		Env:          envVars,
 	},
 		&container.HostConfig{
-			PortBindings: portBindings,
-			Mounts:       mountVolumes,
-			Resources:    container.Resources{Devices: deviceMapping},
-			LogConfig:    container.LogConfig{Type: "json-file", Config: logOptions},
+			PortBindings:  portBindings,
+			Mounts:        mountVolumes,
+			RestartPolicy: container.RestartPolicy{Name: "unless-stopped"},
+			Resources:     container.Resources{Devices: deviceMapping},
+			LogConfig:     container.LogConfig{Type: "json-file", Config: logOptions},
 		},
 		&network.NetworkingConfig{
 			EndpointsConfig: map[string]*network.EndpointSettings{

go/common/errutil/errors_util.go

@@ -25,6 +25,9 @@ var (
 	ErrAncestorBatchNotFound       = errors.New("parent for batch not found")
 	ErrCrossChainBundleRepublished = errors.New("root already added to the message bus")
 	ErrCrossChainBundleNoBatches   = errors.New("no batches for cross chain bundle")
+	ErrNoNextRollup                = errors.New("no next rollup")
+	ErrRollupForkMismatch          = errors.New("rollup fork mismatch")
+	ErrNoBundleToPublish           = errors.New("no bundle to publish")
 )
 
 // BlockRejectError is used as a standard format for error response from enclave for block submission errors

go/common/gethencoding/geth_encoding.go

@@ -11,9 +11,6 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/dgraph-io/ristretto"
-	"github.com/eko/gocache/lib/v4/cache"
-	ristretto_store "github.com/eko/gocache/store/ristretto/v4"
 	gethlog "github.com/ethereum/go-ethereum/log"
 	"github.com/ten-protocol/go-ten/go/common/log"
 	"github.com/ten-protocol/go-ten/go/enclave/storage"
@@ -51,29 +48,16 @@ type EncodingService interface {
 }
 
 type gethEncodingServiceImpl struct {
-	// conversion is expensive. Cache the converted headers. The key is the hash of the batch.
-	gethHeaderCache *cache.Cache[*types.Header]
-
-	storage storage.Storage
-	logger  gethlog.Logger
+	storage        storage.Storage
+	logger         gethlog.Logger
+	cachingService *storage.CacheService
 }
 
-func NewGethEncodingService(storage storage.Storage, logger gethlog.Logger) EncodingService {
-	// todo (tudor) figure out the best values
-	ristrettoCache, err := ristretto.NewCache(&ristretto.Config{
-		NumCounters: 5000, // number of keys to track frequency of.
-		MaxCost:     500,  // todo - this represents how many items.
-		BufferItems: 64,   // number of keys per Get buffer. Todo - what is this
-	})
-	if err != nil {
-		panic(err)
-	}
-	ristrettoStore := ristretto_store.NewRistretto(ristrettoCache)
-
+func NewGethEncodingService(storage storage.Storage, cachingService *storage.CacheService, logger gethlog.Logger) EncodingService {
 	return &gethEncodingServiceImpl{
-		gethHeaderCache: cache.New[*types.Header](ristrettoStore),
-		storage:         storage,
-		logger:          logger,
+		storage:        storage,
+		logger:         logger,
+		cachingService: cachingService,
 	}
 }
 
@@ -276,7 +260,7 @@ func ExtractEthCall(param interface{}) (*gethapi.TransactionArgs, error) {
 // Special care must be taken to maintain a valid chain of these converted headers.
 func (enc *gethEncodingServiceImpl) CreateEthHeaderForBatch(ctx context.Context, h *common.BatchHeader) (*types.Header, error) {
 	// wrap in a caching layer
-	return common.GetCachedValue(ctx, enc.gethHeaderCache, enc.logger, h.Hash(), func(a any) (*types.Header, error) {
+	return enc.cachingService.ReadConvertedHeader(ctx, h.Hash(), func(a any) (*types.Header, error) {
 		// deterministically calculate the private randomness that will be exposed to the EVM
 		secret, err := enc.storage.FetchSecret(ctx)
 		if err != nil {

go/common/host/services.go

@@ -22,6 +22,7 @@ const (
 	EnclaveServiceName         = "enclaves"
 	LogSubscriptionServiceName = "log-subs"
 	FilterAPIServiceName       = "filter-api"
+	CrossChainServiceName      = "cross-chain"
 )
 
 // The host has a number of services that encapsulate the various responsibilities of the host.
@@ -109,7 +110,7 @@ type L1Publisher interface {
 	PublishSecretResponse(secretResponse *common.ProducedSecretResponse) error
 
 	// PublishCrossChainBundle will create and publish a cross-chain bundle tx to the management contract
-	PublishCrossChainBundle(bundle *common.ExtCrossChainBundle) error
+	PublishCrossChainBundle(*common.ExtCrossChainBundle, *big.Int, gethcommon.Hash) error
 
 	FetchLatestSeqNo() (*big.Int, error)
 
@@ -119,7 +120,7 @@ type L1Publisher interface {
 	ResyncImportantContracts() error
 
 	// GetBundleRangeFromManagementContract returns the range of batches for which to build a bundle
-	GetBundleRangeFromManagementContract() (*big.Int, *big.Int, error)
+	GetBundleRangeFromManagementContract(lastRollupNumber *big.Int, lastRollupUID gethcommon.Hash) (*gethcommon.Hash, *big.Int, *big.Int, error)
 }
 
 // L2BatchRepository provides an interface for the host to request L2 batch data (live-streaming and historical)

go/common/log_events.go

@@ -43,7 +43,7 @@ func CreateAuthenticatedLogSubscriptionPayload(args []interface{}, vk *viewingke
 	if !ok {
 		return nil, fmt.Errorf("invalid subscription")
 	}
-	fc := FromCriteria(filterCriteria)
+	fc := SerializableFilterCriteria(filterCriteria)
 	logSubscription.Filter = &fc
 	return logSubscription, nil
 }
@@ -59,7 +59,7 @@ type FilterCriteriaJSON struct {
 	Topics    [][]common.Hash  `json:"topics"`
 }
 
-func FromCriteria(crit FilterCriteria) FilterCriteriaJSON {
+func SerializableFilterCriteria(crit FilterCriteria) FilterCriteriaJSON {
 	var from *rpc.BlockNumber
 	if crit.FromBlock != nil {
 		f := (rpc.BlockNumber)(crit.FromBlock.Int64())

go/common/query_types.go

@@ -49,7 +49,6 @@ type PublicTransaction struct {
 
 type PublicBatch struct {
 	SequencerOrderNo *big.Int              `json:"sequence"`
-	Hash             string                `json:"hash"`
 	FullHash         common.Hash           `json:"fullHash"`
 	Height           *big.Int              `json:"height"`
 	TxCount          *big.Int              `json:"txCount"`

go/enclave/enclave.go

@@ -114,8 +114,9 @@ func NewEnclave(
 	}
 
 	// Initialise the database
+	cachingService := storage.NewCacheService(logger)
 	chainConfig := ethchainadapter.ChainParams(big.NewInt(config.ObscuroChainID))
-	storage := storage.NewStorageFromConfig(config, chainConfig, logger)
+	storage := storage.NewStorageFromConfig(config, cachingService, chainConfig, logger)
 
 	// Initialise the Ethereum "Blockchain" structure that will allow us to validate incoming blocks
 	// todo (#1056) - valid block
@@ -160,7 +161,7 @@ func NewEnclave(
 
 	obscuroKey := crypto.GetObscuroKey(logger)
 
-	gethEncodingService := gethencoding.NewGethEncodingService(storage, logger)
+	gethEncodingService := gethencoding.NewGethEncodingService(storage, cachingService, logger)
 	dataEncryptionService := crypto.NewDataEncryptionService(logger)
 	dataCompressionService := compression.NewBrotliDataCompressionService()
 

go/enclave/genesis/genesis_test.go

@@ -43,7 +43,7 @@ func TestDefaultGenesis(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable to create temp db: %s", err)
 	}
-	storageDB := storage.NewStorage(backingDB, nil, gethlog.New())
+	storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New()), nil, gethlog.New())
 	stateDB, err := gen.applyAllocations(storageDB)
 	if err != nil {
 		t.Fatalf("unable to apply genesis allocations")
@@ -86,7 +86,7 @@ func TestCustomGenesis(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable to create temp db: %s", err)
 	}
-	storageDB := storage.NewStorage(backingDB, nil, gethlog.New())
+	storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New()), nil, gethlog.New())
 	stateDB, err := gen.applyAllocations(storageDB)
 	if err != nil {
 		t.Fatalf("unable to apply genesis allocations")