Rework how rollups are created (#1386)

* Sim tests passing.

* Small changes.

* Dump progress.

* Log messages.

* Added lock to stateDB.Commit.

* Test is more reliable

* Reversed order of fetching data when computing batch in order to make all fork batches fail for l1 block.

* Fixed linter issues.

* Removed dead code.

* Addressed PR comments.

* Working seq no based.

* Cleaned up management contract and rollup headers.

* Semi-Working version.

* Removed storage stuff.

* Compilation fix.

* Fix for enclave test.

* Fixes for linter.

* Rework for in mem rollup handling.

* Linter fix.

* Addressed PR review changes.

* Linter fixes.

* Change gas estimation.

* Fix bad log.

* Removed unused key.

* Removed old code.

* Fix for linter.

---------

Co-authored-by: StefanIliev545 <[email protected]>
Co-authored-by: StefanIliev545 <[email protected]>

contracts/src/management/ManagementContract.sol

@@ -15,125 +15,33 @@ contract ManagementContract {
     // TODO - Revisit the decision to store the host addresses in the smart contract.
     string[] private hostAddresses; // The addresses of all the Obscuro hosts on the network.
 
-    // tree holds a tree of rollups
-    Structs.Tree private tree;
-
     // networkSecretNotInitialized marks if the network secret has been initialized
     bool private networkSecretInitialized ;
 
     // isWithdrawalAvailable marks if the contract allows withdrawals or not
     bool private isWithdrawalAvailable;
 
+    uint256 public lastBatchSeqNo = 0;
+
+    Structs.RollupStorage private rollups;
     //The messageBus where messages can be sent to Obscuro
     MessageBus.IMessageBus public messageBus;
     constructor() {
         messageBus = new MessageBus.MessageBus();
         emit LogManagementContractCreated(address(messageBus));
     }
 
-    // InitializeTree starts the list and sets the initial values
-    function InitializeTree(Structs.MetaRollup memory r) public {
-        require(!tree.initialized, "cannot be initialized again");
-        tree.initialized = true;
-
-        // TreeElement starts a 1 and has no parent ( ParentID: 0 )
-        tree.rollups[1] = Structs.TreeElement(1, 0, r);
-        tree._HEAD = 1;
-        tree._nextID = 2;
-        tree.rollupsHashes[r.Hash] = 1;
-
-        // withdrawals are available at the start
-        isWithdrawalAvailable = true;
-    }
-
-    function GetRollupByID(uint256 rollupID) view public returns(bool, Structs.TreeElement memory) {
-        Structs.TreeElement memory rol = tree.rollups[rollupID];
-        return (rol.ElementID != 0 , rol);
-    }
-
-    function GetRollupByHash(bytes32 rollupHash) view public returns (bool, Structs.TreeElement memory) {
-        return GetRollupByID(tree.rollupsHashes[rollupHash]);
-    }
-
-    function GetHeadRollup() internal view returns ( Structs.TreeElement memory ) {
-        return tree.rollups[tree._HEAD];
-    }
-
-    function GetParentRollup(Structs.TreeElement memory element) view public returns( bool, Structs.TreeElement memory) {
-        return GetRollupByID(element.ParentID);
-    }
-
-    function AppendRollup(uint256 _parentID, Structs.MetaRollup calldata _r) internal {
-        // guarantee the storage ids are not compromised
-        uint rollupID = tree._nextID;
-        tree._nextID++;
-
-        // cannot append to non-existing parent rollups
-        (bool found, Structs.TreeElement memory parent) = GetRollupByID(_parentID);
-        require(found, "parent not found");
-
-        // store the rollup in an element
-        tree.rollups[rollupID] = Structs.TreeElement({ 
-            ElementID: rollupID, 
-            ParentID: _parentID,
-            rollup: _r
-        });
-
-        // mark the element as a child of parent
-        tree.rollupChildren[_parentID].push(rollupID);
-
-        // store the hashpointer
-        tree.rollupsHashes[_r.Hash] = rollupID;
-
-        // mark this as the head
-        if (parent.ElementID == tree._HEAD) {
-            tree._HEAD = rollupID;
-        }
+    function GetRollupByHash(bytes32 rollupHash) view public returns(bool, Structs.MetaRollup memory) {
+        Structs.MetaRollup memory rol = rollups.byHash[rollupHash];
+        return (rol.Hash == rollupHash , rol);
     }
 
-    // HasSecondCousinFork returns whether there is a fork in the current view of the rollups
-    // It works by:
-    // - Traversing up two levels ( from the HEAD to the grand father element )
-    // - Checking if there are siblings ( at the grand father level )
-    // - Checking if the siblings have children ( meaning that a fork expanded )
-    //
-    // Will return true when a rollup 6 or 6' with parent 5 or 5' is inserted
-    // 0 -> 1 -> 2 -> 3 -> 4 -> 5
-    //                  -> 4'-> 5'
-    //
-    function HasSecondCousinFork() view public returns (bool) {
-        Structs.TreeElement memory currentElement = GetHeadRollup();
-
-        // traverse up to the grandpa ( 2 levels up )
-        (bool foundParent, Structs.TreeElement memory parentElement) = GetParentRollup(currentElement);
-        require(foundParent, "no parent");
-        (bool foundGrandpa, Structs.TreeElement memory grandpaElement) = GetParentRollup(parentElement);
-        require(foundGrandpa, "no grand parent");
-
-        // follow each of the grandpa children until it's two levels deep
-        uint256[] memory childrenIDs = tree.rollupChildren[grandpaElement.ElementID];
-        for (uint256 i = 0; i < childrenIDs.length ; i++) {
-            (bool foundChild, Structs.TreeElement memory child) = GetRollupByID(childrenIDs[i]);
-
-            // no more children
-            if (!foundChild) {
-                return false;
-            }
-
-            // ignore the current tree
-            if (child.ElementID == parentElement.ElementID ) {
-                continue;
-            }
-
-            // if child has children then it's bad ( fork of depth 2 )
-            if (tree.rollupChildren[child.ElementID].length > 0) {
-                return true;
-            }
+    function AppendRollup(Structs.MetaRollup calldata _r) internal {
+        rollups.byHash[_r.Hash] = _r;
+        if (_r.LastSequenceNumber > lastBatchSeqNo) {
+            lastBatchSeqNo = _r.LastSequenceNumber;
         }
-
-        return false;
     }
-
     //
     //  -- End of Tree element list Library
     //
@@ -156,27 +64,7 @@ contract ManagementContract {
         // revert if the AggregatorID is not attested
         require(attested[r.AggregatorID], "aggregator not attested");
 
-        // if this is the first element initialize the tree structure
-        // TODO this should be moved to the network initialization
-        if (!tree.initialized) {
-            InitializeTree(r);
-            return;
-        }
-
-        (bool found, Structs.TreeElement memory parent) = GetRollupByHash(r.ParentHash);
-        require(found, "unable to find parent hash");
-
-        // don't check for forks at the start
-        if (tree._HEAD > 2) {
-            bool forkFound = HasSecondCousinFork();
-            if (forkFound) {
-                isWithdrawalAvailable = false;
-                // We keep accepting rollups just locks the contract
-                // require(!found, "detected a fork");
-            }
-        }
-
-        AppendRollup(parent.ElementID, r);
+        AppendRollup(r);
         pushCrossChainMessages(crossChainData);
     }
 

contracts/src/management/Structs.sol

@@ -6,36 +6,14 @@ import * as MessageBusStructs from "../messaging/Structs.sol";
 interface Structs {
      // MetaRollup is a rollup meta data
     struct MetaRollup{
-        bytes32 ParentHash;
         bytes32 Hash;
         address AggregatorID;
         bytes32 L1Block;
-        uint256 Number;
+        uint256 LastSequenceNumber;
     }
 
-    // TreeElement is an element of the Tree structure
-    struct TreeElement{
-        uint256 ElementID;
-        uint256 ParentID;
-        MetaRollup rollup;
-    }
-
-    // NonExisting - 0 (Constant)
-    // Tail - 1 (Constant)
-    // Head - X (Variable)
-    // Does not use rollup hashes as a storing ID as they can be compromised
-    struct Tree {
-        // rollups stores the Elements using incremental IDs
-        mapping(uint256 => TreeElement) rollups;
-        // map a rollup hash to a storage ID
-        mapping(bytes32 => uint256) rollupsHashes;
-        // map the children of a node
-        mapping(uint256 => uint256[]) rollupChildren;
-
-        uint256 _TAIL; // tail is always 1
-        uint256 _HEAD;
-        uint256 _nextID; // TODO use openzeppelin counters
-        bool initialized;
+    struct RollupStorage {
+        mapping(bytes32=>MetaRollup) byHash;
     }
 
     struct HeaderCrossChainData {

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/kamilsk/breaker v1.2.1
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
+	github.com/pkg/errors v0.9.1
 	github.com/sanity-io/litter v1.5.5
 	github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
 	github.com/stretchr/testify v1.8.3
@@ -78,7 +79,6 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/tsdb v0.7.1 // indirect
 	github.com/rjeczalik/notify v0.9.1 // indirect

go/common/enclave.go

@@ -111,7 +111,7 @@ type Enclave interface {
 
 	// CreateRollup - will create a new rollup by going through the sequencer if the node is a sequencer
 	// or panic otherwise.
-	CreateRollup() (*ExtRollup, SystemError)
+	CreateRollup(fromSeqNo uint64) (*ExtRollup, SystemError)
 
 	// DebugTraceTransaction returns the trace of a transaction
 	DebugTraceTransaction(hash gethcommon.Hash, config *tracers.TraceConfig) (json.RawMessage, SystemError)

go/common/headers.go

@@ -47,9 +47,6 @@ type BatchHeader struct {
 // RollupHeader is a public / plaintext struct that holds common properties of rollups.
 // All these fields are processed by the Management contract
 type RollupHeader struct {
-	ParentHash    L2RollupHash
-	Number        *big.Int
-	Time          uint64      `json:"timestamp"`
 	L1Proof       L1BlockHash // the L1 block hash used by the enclave to generate the current rollup
 	L1ProofNumber *big.Int    // the height of the proof - used by the management contract to check
 	Coinbase      common.Address
@@ -59,7 +56,7 @@ type RollupHeader struct {
 	PayloadHash common.Hash // The hash of the compressed batches. TODO
 	R, S        *big.Int    // signature values
 
-	HeadBatchHash L2BatchHash // todo - tudor - remove this after batch compression
+	LastBatchSeqNo uint64
 }
 
 // Hash returns the block hash of the header, which is simply the keccak256 hash of its

go/common/host/services.go

@@ -1,6 +1,8 @@
 package host
 
 import (
+	"math/big"
+
 	gethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/obscuronet/go-obscuro/go/common"
@@ -58,4 +60,6 @@ type L1Publisher interface {
 	PublishSecretResponse(secretResponse *common.ProducedSecretResponse) error
 
 	FetchLatestPeersList() ([]string, error)
+
+	FetchLatestSeqNo() (*big.Int, error)
 }

go/common/log/log.go

@@ -8,22 +8,21 @@ import (
 
 // These are the keys of the log entries
 const (
-	ErrKey          = "err"
-	CtrErrKey       = "ctr_err"
-	SubIDKey        = "subscription_id"
-	CfgKey          = "cfg"
-	TxKey           = "tx"
-	DurationKey     = "duration"
-	BatchHashKey    = "batch"
-	BatchHeightKey  = "batch_height"
-	RollupHashKey   = "rollup"
-	RollupHeightKey = "rollup_height"
-	CmpKey          = "component"
-	NodeIDKey       = "node_id"
-	NetworkIDKey    = "network_id"
-	BlockHeightKey  = "block_height"
-	BlockHashKey    = "block_hash"
-	PackageKey      = "package"
+	ErrKey         = "err"
+	CtrErrKey      = "ctr_err"
+	SubIDKey       = "subscription_id"
+	CfgKey         = "cfg"
+	TxKey          = "tx"
+	DurationKey    = "duration"
+	BatchHashKey   = "batch"
+	BatchHeightKey = "batch_height"
+	RollupHashKey  = "rollup"
+	CmpKey         = "component"
+	NodeIDKey      = "node_id"
+	NetworkIDKey   = "network_id"
+	BlockHeightKey = "block_height"
+	BlockHashKey   = "block_hash"
+	PackageKey     = "package"
 )
 
 // Logging is grouped by the component where it was initialised

go/common/rpc/converters.go

@@ -252,16 +252,13 @@ func ToRollupHeaderMsg(header *common.RollupHeader) *generated.RollupHeaderMsg {
 		return nil
 	}
 	headerMsg := generated.RollupHeaderMsg{
-		ParentHash:         header.ParentHash.Bytes(),
 		Proof:              header.L1Proof.Bytes(),
 		ProofNumber:        header.L1ProofNumber.Uint64(),
-		Number:             header.Number.Uint64(),
 		R:                  header.R.Bytes(),
 		S:                  header.S.Bytes(),
-		Time:               header.Time,
 		Coinbase:           header.Coinbase.Bytes(),
 		CrossChainMessages: ToCrossChainMsgs(header.CrossChainMessages),
-		HeadBatchHash:      header.HeadBatchHash.Bytes(),
+		LastBatchSeqNo:     header.LastBatchSeqNo,
 	}
 
 	return &headerMsg
@@ -289,15 +286,12 @@ func FromRollupHeaderMsg(header *generated.RollupHeaderMsg) *common.RollupHeader
 	r := &big.Int{}
 	s := &big.Int{}
 	return &common.RollupHeader{
-		ParentHash:         gethcommon.BytesToHash(header.ParentHash),
 		L1Proof:            gethcommon.BytesToHash(header.Proof),
 		L1ProofNumber:      big.NewInt(int64(header.ProofNumber)),
-		Number:             big.NewInt(int64(header.Number)),
 		R:                  r.SetBytes(header.R),
 		S:                  s.SetBytes(header.S),
-		Time:               header.Time,
 		Coinbase:           gethcommon.BytesToAddress(header.Coinbase),
 		CrossChainMessages: FromCrossChainMsgs(header.CrossChainMessages),
-		HeadBatchHash:      gethcommon.BytesToHash(header.HeadBatchHash),
+		LastBatchSeqNo:     header.LastBatchSeqNo,
 	}
 }