Tudor/aggressive header compression (#1502)

* add compression

* add compression

* reestablish batch broadcast

* clarify and document

* update the l1 proofs

* rename

* fixes

* fix tests

* fix

* more fixes

* more fixes

* more fixes

* more fixes

* add comment

* fix proto name

go/common/headers.go

@@ -6,10 +6,11 @@ import (
 	"math/big"
 	"sync"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/obscuronet/go-obscuro/contracts/generated/MessageBus"
 	"golang.org/x/crypto/sha3"
 )
@@ -83,6 +84,25 @@ type RollupHeader struct {
 	LastBatchSeqNo uint64
 }
 
+// CalldataRollupHeader contains all information necessary to reconstruct the batches included in the rollup.
+// This data structure is serialised, compressed, and encrypted, before being serialised again in the rollup.
+type CalldataRollupHeader struct {
+	FirstBatchSequence    *big.Int
+	FirstCanonBatchHeight *big.Int
+	FirstCanonParentHash  L2BatchHash
+
+	StartTime       uint64
+	BatchTimeDeltas [][]byte // todo - minimize assuming a default of 1 sec and then store only exceptions
+
+	L1HeightDeltas [][]byte // delta of the block height. Stored as a byte array because rlp can't encode negative numbers
+
+	// these fields are for debugging the compression. Uncomment if there are issues
+	// BatchHashes  []L2BatchHash
+	// BatchHeaders []*BatchHeader
+
+	ReOrgs [][]byte `rlp:"optional"` // sparse list of reorged headers - non null only for reorgs.
+}
+
 // MarshalJSON custom marshals the RollupHeader into a json
 func (r *RollupHeader) MarshalJSON() ([]byte, error) {
 	type Alias RollupHeader

go/common/rollups.go

@@ -6,10 +6,10 @@ import (
 
 // ExtRollup is an encrypted form of rollup used when passing the rollup around outside an enclave.
 type ExtRollup struct {
-	Header        *RollupHeader // the fields required by the management contract
-	BatchHeaders  []byte        // compressed batch headers
-	BatchPayloads []byte        // The batches included in the rollup, in external/encrypted form.
-	hash          atomic.Value
+	Header               *RollupHeader // the fields required by the management contract
+	CalldataRollupHeader []byte        // encrypted header useful for recreating the batches
+	BatchPayloads        []byte        // The transactions included in the rollup, in external/encrypted form.
+	hash                 atomic.Value
 }
 
 // Hash returns the keccak256 hash of the rollup's header.

go/common/rpc/converters.go

@@ -208,7 +208,7 @@ func ToExtRollupMsg(rollup *common.ExtRollup) generated.ExtRollupMsg {
 		return generated.ExtRollupMsg{}
 	}
 
-	return generated.ExtRollupMsg{Header: ToRollupHeaderMsg(rollup.Header), BatchPayloads: rollup.BatchPayloads, BatchHeaders: rollup.BatchHeaders}
+	return generated.ExtRollupMsg{Header: ToRollupHeaderMsg(rollup.Header), BatchPayloads: rollup.BatchPayloads, CalldataRollupHeader: rollup.CalldataRollupHeader}
 }
 
 func ToRollupHeaderMsg(header *common.RollupHeader) *generated.RollupHeaderMsg {
@@ -236,9 +236,9 @@ func FromExtRollupMsg(msg *generated.ExtRollupMsg) *common.ExtRollup {
 	}
 
 	return &common.ExtRollup{
-		Header:        FromRollupHeaderMsg(msg.Header),
-		BatchPayloads: msg.BatchPayloads,
-		BatchHeaders:  msg.BatchHeaders,
+		Header:               FromRollupHeaderMsg(msg.Header),
+		BatchPayloads:        msg.BatchPayloads,
+		CalldataRollupHeader: msg.CalldataRollupHeader,
 	}
 }
 

go/common/rpc/generated/enclave.proto

@@ -391,7 +391,7 @@ message BatchHeaderMsg {
 message ExtRollupMsg {
   RollupHeaderMsg header = 1;
   bytes batchPayloads = 2;
-  bytes batchHeaders = 3;
+  bytes calldataRollupHeader = 3;
 }
 
 message RollupHeaderMsg {

go/enclave/components/batch_executor.go

@@ -52,8 +52,7 @@ func NewBatchExecutor(storage storage.Storage, cc *crosschain.Processors, genesi
 func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext) (*ComputedBatch, error) {
 	defer executor.logger.Info("Batch context processed", log.DurationKey, measure.NewStopwatch())
 
-	// Block is loaded first since if its missing this batch might be based on l1 fork we dont know about
-	// and we want to filter out all fork batches based on not knowing the l1 block
+	// sanity check that the l1 block exists. We don't have to execute batches of forks.
 	block, err := executor.storage.FetchBlock(context.BlockPtr)
 	if errors.Is(err, errutil.ErrNotFound) {
 		return nil, errutil.ErrBlockForBatchNotFound
@@ -64,6 +63,7 @@ func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext) (*Co
 	// These variables will be used to create the new batch
 	parent, err := executor.storage.FetchBatch(context.ParentPtr)
 	if errors.Is(err, errutil.ErrNotFound) {
+		executor.logger.Error(fmt.Sprintf("can't find parent batch %s. Seq %d", context.ParentPtr, context.SequencerNo))
 		return nil, errutil.ErrAncestorBatchNotFound
 	}
 	if err != nil {
@@ -75,7 +75,8 @@ func (executor *batchExecutor) ComputeBatch(context *BatchExecutionContext) (*Co
 		var err error
 		parentBlock, err = executor.storage.FetchBlock(parent.Header.L1Proof)
 		if err != nil {
-			executor.logger.Crit(fmt.Sprintf("Could not retrieve a proof for batch %s", parent.Hash()), log.ErrKey, err)
+			executor.logger.Error(fmt.Sprintf("Could not retrieve a proof for batch %s", parent.Hash()), log.ErrKey, err)
+			return nil, err
 		}
 	}
 

go/enclave/components/batch_registry.go

@@ -104,7 +104,15 @@ func (br *batchRegistry) BatchesAfter(batchSeqNo uint64, rollupLimiter limiters.
 		}
 
 		batches = append(batches, batch)
-		br.logger.Info("Added batch to rollup", log.BatchHashKey, batch.Hash(), log.BatchSeqNoKey, batch.SeqNo())
+		br.logger.Info("Added batch to rollup", log.BatchHashKey, batch.Hash(), log.BatchSeqNoKey, batch.SeqNo(), log.BatchHeightKey, batch.Number(), "l1_proof", batch.Header.L1Proof)
+	}
+
+	// Sanity check that the rollup includes consecutive batches (according to the seqNo)
+	current := batches[0].SeqNo().Uint64()
+	for i, b := range batches {
+		if current+uint64(i) != b.SeqNo().Uint64() {
+			return nil, fmt.Errorf("created invalid rollup with batches out of sequence")
+		}
 	}
 
 	return batches, nil