compression fixes (#1559)

go/enclave/components/rollup_compression.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"math/big"
 
+	"golang.org/x/exp/slices"
+
 	"github.com/ethereum/go-ethereum/params"
 
 	"github.com/obscuronet/go-obscuro/go/common/errutil"
@@ -261,31 +263,30 @@ func (rc *RollupCompression) createIncompleteBatches(calldataRollupHeader *commo
 	startAtSeq := calldataRollupHeader.FirstBatchSequence.Int64()
 	currentHeight := calldataRollupHeader.FirstCanonBatchHeight.Int64() - 1
 	currentTime := int64(calldataRollupHeader.StartTime)
-	var currentL1Height *big.Int
 
 	rollupL1Block, err := rc.storage.FetchBlock(compressionL1Head)
+	if err != nil {
+		return nil, fmt.Errorf("can't find the block used for compression. Cause: %w", err)
+	}
+
+	l1Heights, err := rc.calculateL1HeightsFromDeltas(calldataRollupHeader, transactionsPerBatch)
+	if err != nil {
+		return nil, err
+	}
+
+	// a cache of the l1 blocks used by the current rollup, indexed by their height
+	l1BlocksAtHeight := make(map[uint64]*types.Block)
+	err = rc.calcL1AncestorsOfHeight(big.NewInt(int64(slices.Min(l1Heights))), rollupL1Block, l1BlocksAtHeight)
 	if err != nil {
 		return nil, err
 	}
 
 	for currentBatchIdx, batchTransactions := range transactionsPerBatch {
 		// the l1 proofs are stored as deltas, which compress well as it should be a series of 1s and 0s
-		// the first element is the actual height
-		l1Delta := big.NewInt(0)
-		err := l1Delta.GobDecode(calldataRollupHeader.L1HeightDeltas[currentBatchIdx])
-		if err != nil {
-			return nil, err
-		}
-		if currentBatchIdx == 0 {
-			currentL1Height = l1Delta
-		} else {
-			currentL1Height = big.NewInt(l1Delta.Int64() + currentL1Height.Int64())
-		}
-
 		// get the block with the currentL1Height, relative to the rollupL1Block
-		block, err := rc.getAncestorOfHeight(currentL1Height, rollupL1Block)
-		if err != nil {
-			return nil, err
+		block, f := l1BlocksAtHeight[l1Heights[currentBatchIdx]]
+		if !f {
+			return nil, fmt.Errorf("programming error. L1 block not retrieved")
 		}
 
 		// todo - this should be 1 second
@@ -348,15 +349,46 @@ func (rc *RollupCompression) createIncompleteBatches(calldataRollupHeader *commo
 	return incompleteBatches, nil
 }
 
-func (rc *RollupCompression) getAncestorOfHeight(ancestorHeight *big.Int, head *types.Block) (*types.Block, error) {
-	if head.NumberU64() == ancestorHeight.Uint64() {
-		return head, nil
-	}
-	p, err := rc.storage.FetchBlock(head.ParentHash())
+func (rc *RollupCompression) calculateL1HeightsFromDeltas(calldataRollupHeader *common.CalldataRollupHeader, transactionsPerBatch [][]*common.L2Tx) ([]uint64, error) {
+	referenceHeight := big.NewInt(0)
+	// the first element in the deltas is the actual height
+	err := referenceHeight.GobDecode(calldataRollupHeader.L1HeightDeltas[0])
 	if err != nil {
 		return nil, err
 	}
-	return rc.getAncestorOfHeight(ancestorHeight, p)
+
+	l1Heights := make([]uint64, 0)
+	l1Heights = append(l1Heights, referenceHeight.Uint64())
+	prevHeight := l1Heights[0]
+	for currentBatchIdx := range transactionsPerBatch {
+		// the l1 proofs are stored as deltas, which compress well as it should be a series of 1s and 0s
+		if currentBatchIdx > 0 {
+			l1Delta := big.NewInt(0)
+			err := l1Delta.GobDecode(calldataRollupHeader.L1HeightDeltas[currentBatchIdx])
+			if err != nil {
+				return nil, err
+			}
+			value := l1Delta.Int64() + int64(prevHeight)
+			if value < 0 {
+				rc.logger.Crit("Should not have a negative height")
+			}
+			l1Heights = append(l1Heights, uint64(value))
+			prevHeight = uint64(value)
+		}
+	}
+	return l1Heights, nil
+}
+
+func (rc *RollupCompression) calcL1AncestorsOfHeight(fromHeight *big.Int, toBlock *types.Block, path map[uint64]*types.Block) error {
+	path[toBlock.NumberU64()] = toBlock
+	if toBlock.NumberU64() == fromHeight.Uint64() {
+		return nil
+	}
+	p, err := rc.storage.FetchBlock(toBlock.ParentHash())
+	if err != nil {
+		return err
+	}
+	return rc.calcL1AncestorsOfHeight(fromHeight, p, path)
 }
 
 func (rc *RollupCompression) executeAndSaveIncompleteBatches(calldataRollupHeader *common.CalldataRollupHeader, incompleteBatches []*batchFromRollup) error { //nolint:gocognit
@@ -374,14 +406,27 @@ func (rc *RollupCompression) executeAndSaveIncompleteBatches(calldataRollupHeade
 		// check whether the batch is already stored in the database
 		b, err := rc.storage.FetchBatchBySeqNo(incompleteBatch.seqNo.Uint64())
 		if err == nil {
-			parentHash = b.Hash()
+			// chain to a parent only if the batch is not a reorg
+			if incompleteBatch.header == nil {
+				parentHash = b.Hash()
+			}
 			continue
 		}
 		if !errors.Is(err, errutil.ErrNotFound) {
 			return err
 		}
 
 		switch {
+		// this batch was re-orged
+		case incompleteBatch.header != nil:
+			err := rc.storage.StoreBatch(&core.Batch{
+				Header:       incompleteBatch.header,
+				Transactions: incompleteBatch.transactions,
+			})
+			if err != nil {
+				return err
+			}
+
 		// handle genesis
 		case incompleteBatch.seqNo.Uint64() == common.L2GenesisSeqNo:
 			genBatch, _, err := rc.batchExecutor.CreateGenesisState(
@@ -413,16 +458,6 @@ func (rc *RollupCompression) executeAndSaveIncompleteBatches(calldataRollupHeade
 			rc.logger.Info("Stored genesis", log.BatchHashKey, genBatch.Hash())
 			parentHash = genBatch.Hash()
 
-		// this batch was re-orged
-		case incompleteBatch.header != nil:
-			err := rc.storage.StoreBatch(&core.Batch{
-				Header:       incompleteBatch.header,
-				Transactions: incompleteBatch.transactions,
-			})
-			if err != nil {
-				return err
-			}
-
 		default:
 			// transforms the incompleteBatch into a BatchHeader by executing the transactions
 			// and then the info can be used to fill in the parent

go/enclave/components/rollup_consumer.go

@@ -58,21 +58,34 @@ func (rc *rollupConsumerImpl) ProcessRollupsInBlock(b *common.BlockAndReceipts)
 	if err != nil {
 		return err
 	}
-	if len(rollups) > 0 {
-		for _, rollup := range rollups {
-			// read batch data from rollup, verify and store it
-			internalHeader, err := rc.rollupCompression.ProcessExtRollup(rollup)
-			if err != nil {
-				rc.logger.Error("Failed processing rollup", log.RollupHashKey, rollup.Hash(), log.ErrKey, err)
-				// todo - issue challenge as a validator
-				return err
-			}
-			if err := rc.storage.StoreRollup(rollup, internalHeader); err != nil {
-				rc.logger.Error("Failed storing rollup", log.RollupHashKey, rollup.Hash(), log.ErrKey, err)
-				return err
-			}
+
+	for _, rollup := range rollups {
+		l1CompressionBlock, err := rc.storage.FetchBlock(rollup.Header.CompressionL1Head)
+		if err != nil {
+			rc.logger.Warn("Can't process rollup because the l1 block used for compression is not available", "block_hash", rollup.Header.CompressionL1Head, log.RollupHashKey, rollup.Hash(), log.ErrKey, err)
+			continue
+		}
+		canonicalBlockByHeight, err := rc.storage.FetchCanonicaBlockByHeight(l1CompressionBlock.Number())
+		if err != nil {
+			return err
+		}
+		if canonicalBlockByHeight.Hash() != l1CompressionBlock.Hash() {
+			rc.logger.Warn("Skipping rollup because it was compressed on top of a non-canonical rollup", "block_hash", rollup.Header.CompressionL1Head, log.RollupHashKey, rollup.Hash(), log.ErrKey, err)
+			continue
+		}
+		// read batch data from rollup, verify and store it
+		internalHeader, err := rc.rollupCompression.ProcessExtRollup(rollup)
+		if err != nil {
+			rc.logger.Error("Failed processing rollup", log.RollupHashKey, rollup.Hash(), log.ErrKey, err)
+			// todo - issue challenge as a validator
+			return err
+		}
+		if err := rc.storage.StoreRollup(rollup, internalHeader); err != nil {
+			rc.logger.Error("Failed storing rollup", log.RollupHashKey, rollup.Hash(), log.ErrKey, err)
+			return err
 		}
 	}
+
 	return nil
 }
 

go/host/enclave/guardian.go

@@ -555,7 +555,7 @@ func (g *Guardian) periodicRollupProduction() {
 			if time.Since(lastSuccessfulRollup) > g.rollupInterval || availBatchesSumSize >= g.maxRollupSize {
 				producedRollup, err := g.enclaveClient.CreateRollup(fromBatch)
 				if err != nil {
-					g.logger.Error("Unable to create rollup", log.BatchSeqNoKey, fromBatch)
+					g.logger.Error("Unable to create rollup", log.BatchSeqNoKey, fromBatch, log.ErrKey, err)
 					continue
 				}
 				// this method waits until the receipt is received

integration/simulation/simulation_in_mem_test.go

@@ -35,8 +35,8 @@ func TestInMemoryMonteCarloSimulation(t *testing.T) {
 		StartPort:                  integration.StartPortSimulationInMem,
 		IsInMem:                    true,
 		L1SetupData:                &params.L1SetupData{},
-		ReceiptTimeout:             30 * time.Second,
-		StoppingDelay:              10 * time.Second,
+		ReceiptTimeout:             5 * time.Second,
+		StoppingDelay:              4 * time.Second,
 		NodeWithInboundP2PDisabled: 2,
 	}