perf improvements

go/enclave/components/block_processor.go

@@ -95,6 +95,7 @@ func (bp *l1BlockProcessor) ingestBlock(block *common.L1Block) (*BlockIngestionT
 			bp.logger.Trace("parent not found",
 				"blkHeight", block.NumberU64(), log.BlockHashKey, block.Hash(),
 				"l1HeadHeight", prevL1Head.NumberU64(), "l1HeadHash", prevL1Head.Hash(),
+				log.ErrKey, err,
 			)
 			return nil, errutil.ErrBlockAncestorNotFound
 		}

go/enclave/storage/db_cache.go

@@ -0,0 +1,39 @@
+package storage
+
+import (
+	"context"
+
+	"github.com/eko/gocache/lib/v4/cache"
+	gethlog "github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/obscuronet/go-obscuro/go/common/log"
+)
+
+func getCachedValue[V any](cache *cache.Cache[[]byte], logger gethlog.Logger, key any, onFailed func(any) (V, error)) (V, error) {
+	value, err := cache.Get(context.Background(), key)
+	if err != nil {
+		// todo metrics for cache misses
+		b, err := onFailed(key)
+		if err != nil {
+			return b, err
+		}
+		cacheValue(cache, logger, key, b)
+		return b, err
+	}
+
+	v := new(V)
+	err = rlp.DecodeBytes(value, v)
+	return *v, err
+}
+
+func cacheValue(cache *cache.Cache[[]byte], logger gethlog.Logger, key any, v any) {
+	encoded, err := rlp.EncodeToBytes(v)
+	if err != nil {
+		logger.Error("Could not encode value to store in cache", log.ErrKey, err)
+		return
+	}
+	err = cache.Set(context.Background(), key, encoded)
+	if err != nil {
+		logger.Error("Could not store value in cache", log.ErrKey, err)
+	}
+}

go/enclave/storage/enclavedb/batch.go

@@ -2,6 +2,7 @@ package enclavedb
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -25,7 +26,7 @@ const (
 	txInsertValue = "(?,?,?,?,?,?)"
 
 	bInsert             = "insert into batch values (?,?,?,?,?,?,?,?,?)"
-	updateBatchExecuted = "update batch set is_executed=true where hash=?"
+	updateBatchExecuted = "update batch set is_executed=true where sequence=?"
 
 	selectBatch  = "select b.header, bb.content from batch b join batch_body bb on b.body=bb.hash"
 	selectHeader = "select b.header from batch b"
@@ -49,7 +50,7 @@ const (
 
 // WriteBatchAndTransactions - persists the batch and the transactions
 func WriteBatchAndTransactions(dbtx DBTransaction, batch *core.Batch) error {
-	bodyHash := batch.Header.TxHash.Bytes()
+	bodyHash := truncTo16(batch.Header.TxHash)
 
 	body, err := rlp.EncodeToBytes(batch.Transactions)
 	if err != nil {
@@ -64,27 +65,26 @@ func WriteBatchAndTransactions(dbtx DBTransaction, batch *core.Batch) error {
 
 	var parentBytes []byte
 	if batch.Number().Uint64() > 0 {
-		parentBytes = batch.Header.ParentHash.Bytes()
+		parentBytes = truncTo16(batch.Header.ParentHash)
 	}
 
-	// todo - this can be removed if the batches have no is_canonical
 	var isCanon bool
-	err = dbtx.GetDB().QueryRow(isCanonQuery, batch.Header.L1Proof.Bytes()).Scan(&isCanon)
+	err = dbtx.GetDB().QueryRow(isCanonQuery, truncTo16(batch.Header.L1Proof)).Scan(&isCanon)
 	if err != nil {
 		// if the block is not found, we assume it is non-canonical
 		// fmt.Printf("IsCanon %s err: %s\n", batch.Header.L1Proof, err)
 		isCanon = false
 	}
 
 	dbtx.ExecuteSQL(bInsert,
-		batch.Hash().Bytes(),                   // hash
-		parentBytes,                            // parent
 		batch.Header.SequencerOrderNo.Uint64(), // sequence
+		truncTo16(batch.Hash()),                // hash
+		parentBytes,                            // parent
 		batch.Header.Number.Uint64(),           // height
 		isCanon,                                // is_canonical
 		header,                                 // header blob
 		bodyHash,                               // reference to the batch body
-		batch.Header.L1Proof.Bytes(),           // l1_proof
+		truncTo16(batch.Header.L1Proof),        // l1_proof
 		false,                                  // executed
 	)
 
@@ -105,12 +105,12 @@ func WriteBatchAndTransactions(dbtx DBTransaction, batch *core.Batch) error {
 				return fmt.Errorf("unable to convert tx to message - %w", err)
 			}
 
-			args = append(args, transaction.Hash().Bytes()) // tx_hash
-			args = append(args, txBytes)                    // content
-			args = append(args, from.Bytes())               // sender_address
-			args = append(args, transaction.Nonce())        // nonce
-			args = append(args, i)                          // idx
-			args = append(args, bodyHash)                   // the batch body which contained it
+			args = append(args, truncTo16(transaction.Hash())) // tx_hash
+			args = append(args, txBytes)                       // content
+			args = append(args, from.Bytes())                  // sender_address
+			args = append(args, transaction.Nonce())           // nonce
+			args = append(args, i)                             // idx
+			args = append(args, bodyHash)                      // the batch body which contained it
 		}
 		dbtx.ExecuteSQL(insert, args...)
 	}
@@ -119,8 +119,8 @@ func WriteBatchAndTransactions(dbtx DBTransaction, batch *core.Batch) error {
 }
 
 // WriteBatchExecution - insert all receipts to the db
-func WriteBatchExecution(dbtx DBTransaction, hash common.L2BatchHash, receipts []*types.Receipt) error {
-	dbtx.ExecuteSQL(updateBatchExecuted, hash.Bytes())
+func WriteBatchExecution(dbtx DBTransaction, seqNo *big.Int, receipts []*types.Receipt) error {
+	dbtx.ExecuteSQL(updateBatchExecuted, seqNo.Uint64())
 
 	args := make([]any, 0)
 	for _, receipt := range receipts {
@@ -134,8 +134,8 @@ func WriteBatchExecution(dbtx DBTransaction, hash common.L2BatchHash, receipts [
 		args = append(args, executedTransactionID(&receipt.BlockHash, &receipt.TxHash)) // PK
 		args = append(args, receipt.ContractAddress.Bytes())                            // created_contract_address
 		args = append(args, receiptBytes)                                               // the serialised receipt
-		args = append(args, receipt.TxHash.Bytes())                                     // tx_hash
-		args = append(args, receipt.BlockHash.Bytes())                                  // batch_hash
+		args = append(args, truncTo16(receipt.TxHash))                                  // tx_hash
+		args = append(args, seqNo.Uint64())                                             // batch_seq
 	}
 	if len(args) > 0 {
 		insert := txExecInsert + strings.Repeat(txExecInsertValue+",", len(receipts))
@@ -150,23 +150,23 @@ func executedTransactionID(batchHash *common.L2BatchHash, txHash *common.L2TxHas
 	execTxID := make([]byte, 0)
 	execTxID = append(execTxID, batchHash.Bytes()...)
 	execTxID = append(execTxID, txHash.Bytes()...)
-	return execTxID
+	return truncTo16(sha256.Sum256(execTxID))
 }
 
 func ReadBatchBySeqNo(db *sql.DB, seqNo uint64) (*core.Batch, error) {
 	return fetchBatch(db, " where sequence=?", seqNo)
 }
 
 func ReadBatchByHash(db *sql.DB, hash common.L2BatchHash) (*core.Batch, error) {
-	return fetchBatch(db, " where b.hash=?", hash.Bytes())
+	return fetchBatch(db, " where b.hash=?", truncTo16(hash))
 }
 
 func ReadCanonicalBatchByHeight(db *sql.DB, height uint64) (*core.Batch, error) {
 	return fetchBatch(db, " where b.height=? and is_canonical=true", height)
 }
 
 func ReadBatchHeader(db *sql.DB, hash gethcommon.Hash) (*common.BatchHeader, error) {
-	return fetchBatchHeader(db, " where hash=?", hash.Bytes())
+	return fetchBatchHeader(db, " where hash=?", truncTo16(hash))
 }
 
 // todo - is there a better way to write this query?
@@ -175,7 +175,7 @@ func ReadCurrentHeadBatch(db *sql.DB) (*core.Batch, error) {
 }
 
 func ReadBatchesByBlock(db *sql.DB, hash common.L1BlockHash) ([]*core.Batch, error) {
-	return fetchBatches(db, " where b.l1_proof=? order by b.sequence", hash.Bytes())
+	return fetchBatches(db, " where b.l1_proof=? order by b.sequence", truncTo16(hash))
 }
 
 func ReadCurrentSequencerNo(db *sql.DB) (*big.Int, error) {
@@ -197,7 +197,7 @@ func ReadCurrentSequencerNo(db *sql.DB) (*big.Int, error) {
 
 func ReadHeadBatchForBlock(db *sql.DB, l1Hash common.L1BlockHash) (*core.Batch, error) {
 	query := " where b.is_canonical=true and b.is_executed=true and b.height=(select max(b1.height) from batch b1 where b1.is_canonical=true and b1.is_executed=true and b1.l1_proof=?)"
-	return fetchBatch(db, query, l1Hash.Bytes())
+	return fetchBatch(db, query, truncTo16(l1Hash))
 }
 
 func fetchBatch(db *sql.DB, whereQuery string, args ...any) (*core.Batch, error) {
@@ -355,11 +355,11 @@ func selectReceipts(db *sql.DB, config *params.ChainConfig, query string, args .
 // corresponding block body, so if the block body is not found it will return nil even
 // if the receipt itself is stored.
 func ReadReceiptsByBatchHash(db *sql.DB, hash common.L2BatchHash, config *params.ChainConfig) (types.Receipts, error) {
-	return selectReceipts(db, config, "where batch.hash = ?", hash.Bytes())
+	return selectReceipts(db, config, "where batch.hash = ?", truncTo16(hash))
 }
 
 func ReadReceipt(db *sql.DB, hash common.L2TxHash, config *params.ChainConfig) (*types.Receipt, error) {
-	row := db.QueryRow(queryReceipts+" where tx=?", hash.Bytes())
+	row := db.QueryRow(queryReceipts+" where tx=?", truncTo16(hash))
 	// receipt, tx, batch, height
 	var receiptData []byte
 	var txData []byte
@@ -394,7 +394,7 @@ func ReadReceipt(db *sql.DB, hash common.L2TxHash, config *params.ChainConfig) (
 }
 
 func ReadTransaction(db *sql.DB, txHash gethcommon.Hash) (*types.Transaction, gethcommon.Hash, uint64, uint64, error) {
-	row := db.QueryRow(selectTxQuery, txHash.Bytes())
+	row := db.QueryRow(selectTxQuery, truncTo16(txHash))
 
 	// tx, batch, height, idx
 	var txData []byte
@@ -452,7 +452,7 @@ func ReadUnexecutedBatches(db *sql.DB) ([]*core.Batch, error) {
 }
 
 func BatchWasExecuted(db *sql.DB, hash common.L2BatchHash) (bool, error) {
-	row := db.QueryRow(queryBatchWasExecuted, hash.Bytes())
+	row := db.QueryRow(queryBatchWasExecuted, truncTo16(hash))
 
 	var result bool
 	err := row.Scan(&result)

go/enclave/storage/enclavedb/block.go

@@ -39,14 +39,14 @@ func WriteBlock(dbtx DBTransaction, b *types.Header) error {
 
 	var parentBytes []byte
 	if b.Number.Uint64() > 1 {
-		parentBytes = b.ParentHash.Bytes()
+		parentBytes = truncTo16(b.ParentHash)
 	}
 	dbtx.ExecuteSQL(blockInsert,
-		b.Hash().Bytes(),
-		parentBytes,
-		true,
-		header,
-		b.Number.Uint64(),
+		truncTo16(b.Hash()), // hash
+		parentBytes,         // parent
+		true,                // is_canonical
+		header,              // header
+		b.Number.Uint64(),   // height
 	)
 	return nil
 }
@@ -70,19 +70,19 @@ func updateCanonicalValue(dbtx DBTransaction, isCanonical bool, values []common.
 	args := make([]any, 0)
 	args = append(args, isCanonical)
 	for _, value := range values {
-		args = append(args, value.Bytes())
+		args = append(args, truncTo16(value))
 	}
 	dbtx.ExecuteSQL(updateBlocks, args...)
 	dbtx.ExecuteSQL(updateBatches, args...)
 }
 
-func FetchBlockHeader(db *sql.DB, hash common.L2BatchHash) (*types.Header, error) {
-	return fetchBlockHeader(db, " where hash=?", hash.Bytes())
+func FetchBlockHeader(db *sql.DB, hash common.L1BlockHash) (*types.Header, error) {
+	return fetchBlockHeader(db, " where hash=?", truncTo16(hash))
 }
 
 // todo - remove this. For now creates a "block" but without a body.
-func FetchBlock(db *sql.DB, hash common.L2BatchHash) (*types.Block, error) {
-	return fetchBlock(db, " where hash=?", hash.Bytes())
+func FetchBlock(db *sql.DB, hash common.L1BlockHash) (*types.Block, error) {
+	return fetchBlock(db, " where hash=?", truncTo16(hash))
 }
 
 func FetchHeadBlock(db *sql.DB) (*types.Block, error) {
@@ -105,7 +105,7 @@ func WriteL1Messages(db *sql.DB, blockHash common.L1BlockHash, messages common.C
 			return err
 		}
 		args = append(args, data)
-		args = append(args, blockHash.Bytes())
+		args = append(args, truncTo16(blockHash))
 	}
 	if len(messages) > 0 {
 		_, err := db.Exec(insert, args...)
@@ -117,7 +117,7 @@ func WriteL1Messages(db *sql.DB, blockHash common.L1BlockHash, messages common.C
 func FetchL1Messages(db *sql.DB, blockHash common.L1BlockHash) (common.CrossChainMessages, error) {
 	var result common.CrossChainMessages
 	query := selectL1Msg + " where block = ?"
-	rows, err := db.Query(query, blockHash.Bytes())
+	rows, err := db.Query(query, truncTo16(blockHash))
 	if err != nil {
 		if errors.Is(err, sql.ErrNoRows) {
 			// make sure the error is converted to obscuro-wide not found error
@@ -152,11 +152,11 @@ func WriteRollup(dbtx DBTransaction, rollup *common.RollupHeader, internalHeader
 		return fmt.Errorf("could not encode batch header. Cause: %w", err)
 	}
 	dbtx.ExecuteSQL(rollupInsert,
-		rollup.Hash(),
+		truncTo16(rollup.Hash()),
 		internalHeader.FirstBatchSequence.Uint64(),
 		rollup.LastBatchSeqNo,
 		data,
-		rollup.CompressionL1Head.Bytes(),
+		truncTo16(rollup.CompressionL1Head),
 	)
 	return nil
 }
@@ -169,7 +169,7 @@ func FetchReorgedRollup(db *sql.DB, reorgedBlocks []common.L1BlockHash) (*common
 
 	args := make([]any, 0)
 	for _, value := range reorgedBlocks {
-		args = append(args, value.Bytes())
+		args = append(args, truncTo16(value))
 	}
 	rollup := new(common.L2BatchHash)
 	err := db.QueryRow(query, args...).Scan(&rollup)

go/enclave/storage/enclavedb/events.go

@@ -122,7 +122,7 @@ func writeLog(db *sql.DB, l *types.Log, receipt *types.Receipt, stateDB *state.S
 	}
 
 	return []any{
-		t0, t1, t2, t3, t4,
+		truncBTo16(t0), truncBTo16(t1), truncBTo16(t2), truncBTo16(t3), truncBTo16(t4),
 		data, l.Index, l.Address.Bytes(),
 		isLifecycle, a1, a2, a3, a4,
 		executedTransactionID(&receipt.BlockHash, &l.TxHash),
@@ -133,15 +133,15 @@ func FilterLogs(
 	db *sql.DB,
 	requestingAccount *gethcommon.Address,
 	fromBlock, toBlock *big.Int,
-	blockHash *common.L2BatchHash,
+	batchHash *common.L2BatchHash,
 	addresses []gethcommon.Address,
 	topics [][]gethcommon.Hash,
 ) ([]*types.Log, error) {
 	queryParams := []any{}
 	query := ""
-	if blockHash != nil {
+	if batchHash != nil {
 		query += " AND b.hash = ?"
-		queryParams = append(queryParams, blockHash.Bytes())
+		queryParams = append(queryParams, truncTo16(*batchHash))
 	}
 
 	// ignore negative numbers
@@ -170,7 +170,7 @@ func FilterLogs(
 				column := fmt.Sprintf("topic%d", i)
 				query += " AND " + column + " in (?" + strings.Repeat(",?", len(sub)-1) + ")"
 				for _, topic := range sub {
-					queryParams = append(queryParams, topic.Bytes())
+					queryParams = append(queryParams, truncTo16(topic))
 				}
 			}
 		}
@@ -184,7 +184,7 @@ func DebugGetLogs(db *sql.DB, txHash common.TxHash) ([]*tracers.DebugLogs, error
 
 	query := baseDebugEventsQuerySelect + " " + baseEventsJoin + "AND tx.hash = ?"
 
-	queryParams = append(queryParams, txHash.Bytes())
+	queryParams = append(queryParams, truncTo16(txHash))
 
 	result := make([]*tracers.DebugLogs, 0)
 
@@ -202,6 +202,7 @@ func DebugGetLogs(db *sql.DB, txHash common.TxHash) ([]*tracers.DebugLogs, error
 			LifecycleEvent: false,
 		}
 
+		// todo - read datablob
 		var t0, t1, t2, t3, t4 sql.NullString
 		var relAddress1, relAddress2, relAddress3, relAddress4 sql.NullByte
 		err = rows.Scan(

go/enclave/storage/enclavedb/utils.go

@@ -0,0 +1,16 @@
+package enclavedb
+
+import gethcommon "github.com/ethereum/go-ethereum/common"
+
+const truncHash = 16
+
+func truncTo16(hash gethcommon.Hash) []byte {
+	return truncBTo16(hash.Bytes())
+}
+
+func truncBTo16(bytes []byte) []byte {
+	b := bytes[0:truncHash]
+	c := make([]byte, truncHash)
+	copy(c, b)
+	return c
+}