From ce8d274ec4aa8a3ce4032daeda46a365242fac0c Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Tue, 30 Apr 2024 10:32:54 +0100 Subject: [PATCH 1/3] StateDB cleanup (#1890) * better cache for the statedb * fix merge * fix * remove cache * address pr comment --- go/enclave/components/batch_executor.go | 7 ++- go/enclave/components/batch_registry.go | 18 +++++++- go/enclave/components/interfaces.go | 6 ++- go/enclave/enclave.go | 16 +++---- go/enclave/events/subscription_manager.go | 19 +++++--- go/enclave/rpc/GetTransactionCount.go | 3 +- go/enclave/rpc/GetTransactionReceipt.go | 2 +- go/enclave/storage/storage.go | 56 +++++++++++++---------- 8 files changed, 80 insertions(+), 47 deletions(-) diff --git a/go/enclave/components/batch_executor.go b/go/enclave/components/batch_executor.go index 8213b1746a..cf4dea1fef 100644 --- a/go/enclave/components/batch_executor.go +++ b/go/enclave/components/batch_executor.go @@ -41,6 +41,7 @@ var ErrNoTransactionsToProcess = fmt.Errorf("no transactions to process") // batchExecutor - the component responsible for executing batches type batchExecutor struct { storage storage.Storage + batchRegistry BatchRegistry config config.EnclaveConfig gethEncodingService gethencoding.EncodingService crossChainProcessors *crosschain.Processors @@ -57,6 +58,7 @@ type batchExecutor struct { func NewBatchExecutor( storage storage.Storage, + batchRegistry BatchRegistry, config config.EnclaveConfig, gethEncodingService gethencoding.EncodingService, cc *crosschain.Processors, @@ -68,6 +70,7 @@ func NewBatchExecutor( ) BatchExecutor { return &batchExecutor{ storage: storage, + batchRegistry: batchRegistry, config: config, gethEncodingService: gethEncodingService, crossChainProcessors: cc, @@ -163,7 +166,7 @@ func (executor *batchExecutor) ComputeBatch(ctx context.Context, context *BatchE // Create a new batch based on the fromBlock of inclusion of the previous, including all new transactions batch := core.DeterministicEmptyBatch(parent.Header, block, context.AtTime, context.SequencerNo, context.BaseFee, context.Creator) - stateDB, err := executor.storage.CreateStateDB(ctx, batch.Header.ParentHash) + stateDB, err := executor.batchRegistry.GetBatchState(ctx, &batch.Header.ParentHash) if err != nil { return nil, fmt.Errorf("could not create stateDB. Cause: %w", err) } @@ -444,7 +447,7 @@ func (executor *batchExecutor) processTransactions( } else { // Exclude all errors excludedTransactions = append(excludedTransactions, tx.Tx) - executor.logger.Info("Excluding transaction from batch", log.TxKey, tx.Tx.Hash(), log.BatchHashKey, batch.Hash(), "cause", result) + executor.logger.Debug("Excluding transaction from batch", log.TxKey, tx.Tx.Hash(), log.BatchHashKey, batch.Hash(), "cause", result) } } sort.Sort(sortByTxIndex(txReceipts)) diff --git a/go/enclave/components/batch_registry.go b/go/enclave/components/batch_registry.go index a9bcd77659..026725ca01 100644 --- a/go/enclave/components/batch_registry.go +++ b/go/enclave/components/batch_registry.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "github.com/ten-protocol/go-ten/go/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ten-protocol/go-ten/go/enclave/storage" @@ -46,6 +48,7 @@ func NewBatchRegistry(storage storage.Storage, logger gethlog.Logger) BatchRegis } else { headBatchSeq = headBatch.SeqNo() } + return &batchRegistry{ storage: storage, headBatchSeq: headBatchSeq, @@ -154,6 +157,14 @@ func (br *batchRegistry) BatchesAfter(ctx context.Context, batchSeqNo uint64, up return resultBatches, resultBlocks, nil } +func (br *batchRegistry) GetBatchState(ctx context.Context, hash *common.L2BatchHash) (*state.StateDB, error) { + batch, err := br.storage.FetchBatch(ctx, *hash) + if err != nil { + return nil, err + } + return getBatchState(ctx, br.storage, batch) +} + func (br *batchRegistry) GetBatchStateAtHeight(ctx context.Context, blockNumber *gethrpc.BlockNumber) (*state.StateDB, error) { // We retrieve the batch of interest. batch, err := br.GetBatchAtHeight(ctx, *blockNumber) @@ -161,8 +172,11 @@ func (br *batchRegistry) GetBatchStateAtHeight(ctx context.Context, blockNumber return nil, err } - // We get that of the chain at that height - blockchainState, err := br.storage.CreateStateDB(ctx, batch.Hash()) + return getBatchState(ctx, br.storage, batch) +} + +func getBatchState(ctx context.Context, storage storage.Storage, batch *core.Batch) (*state.StateDB, error) { + blockchainState, err := storage.CreateStateDB(ctx, batch.Hash()) if err != nil { return nil, fmt.Errorf("could not create stateDB. Cause: %w", err) } diff --git a/go/enclave/components/interfaces.go b/go/enclave/components/interfaces.go index 1e507c7ad9..cb68b91478 100644 --- a/go/enclave/components/interfaces.go +++ b/go/enclave/components/interfaces.go @@ -85,10 +85,12 @@ type BatchRegistry interface { // BatchesAfter - Given a hash, will return batches following it until the head batch and the l1 blocks referenced by those batches BatchesAfter(ctx context.Context, batchSeqNo uint64, upToL1Height uint64, rollupLimiter limiters.RollupLimiter) ([]*core.Batch, []*types.Block, error) - // GetBatchStateAtHeight - creates a stateDB that represents the state committed when - // the batch with height matching the blockNumber was created and stored. + // GetBatchStateAtHeight - creates a stateDB for the block number GetBatchStateAtHeight(ctx context.Context, blockNumber *gethrpc.BlockNumber) (*state.StateDB, error) + // GetBatchState - creates a stateDB for the block hash + GetBatchState(ctx context.Context, hash *common.L2BatchHash) (*state.StateDB, error) + // GetBatchAtHeight - same as `GetBatchStateAtHeight`, but instead returns the full batch // rather than its stateDB only. GetBatchAtHeight(ctx context.Context, height gethrpc.BlockNumber) (*core.Batch, error) diff --git a/go/enclave/enclave.go b/go/enclave/enclave.go index 395c31c990..4216b30247 100644 --- a/go/enclave/enclave.go +++ b/go/enclave/enclave.go @@ -168,9 +168,9 @@ func NewEnclave( gasOracle := gas.NewGasOracle() blockProcessor := components.NewBlockProcessor(storage, crossChainProcessors, gasOracle, logger) - batchExecutor := components.NewBatchExecutor(storage, *config, gethEncodingService, crossChainProcessors, genesis, gasOracle, chainConfig, config.GasBatchExecutionLimit, logger) - sigVerifier, err := components.NewSignatureValidator(storage) registry := components.NewBatchRegistry(storage, logger) + batchExecutor := components.NewBatchExecutor(storage, registry, *config, gethEncodingService, crossChainProcessors, genesis, gasOracle, chainConfig, config.GasBatchExecutionLimit, logger) + sigVerifier, err := components.NewSignatureValidator(storage) rProducer := components.NewRollupProducer(enclaveKey.EnclaveID(), storage, registry, logger) if err != nil { logger.Crit("Could not initialise the signature validator", log.ErrKey, err) @@ -226,7 +226,7 @@ func NewEnclave( config.GasLocalExecutionCapFlag, ) rpcEncryptionManager := rpc.NewEncryptionManager(ecies.ImportECDSA(obscuroKey), storage, registry, crossChainProcessors, service, config, gasOracle, storage, blockProcessor, chain, logger) - subscriptionManager := events.NewSubscriptionManager(storage, config.ObscuroChainID, logger) + subscriptionManager := events.NewSubscriptionManager(storage, registry, config.ObscuroChainID, logger) // ensure cached chain state data is up-to-date using the persisted batch data err = restoreStateDBCache(context.Background(), storage, registry, batchExecutor, genesis, logger) @@ -671,7 +671,7 @@ func (e *enclaveImpl) GetCode(ctx context.Context, address gethcommon.Address, b return nil, responses.ToInternalError(fmt.Errorf("requested GetCode with the enclave stopping")) } - stateDB, err := e.storage.CreateStateDB(ctx, *batchHash) + stateDB, err := e.registry.GetBatchState(ctx, batchHash) if err != nil { return nil, responses.ToInternalError(fmt.Errorf("could not create stateDB. Cause: %w", err)) } @@ -904,7 +904,7 @@ func restoreStateDBCache(ctx context.Context, storage storage.Storage, registry } return fmt.Errorf("unexpected error fetching head batch to resync- %w", err) } - if !stateDBAvailableForBatch(ctx, storage, batch.Hash()) { + if !stateDBAvailableForBatch(ctx, registry, batch.Hash()) { logger.Info("state not available for latest batch after restart - rebuilding stateDB cache from batches") err = replayBatchesToValidState(ctx, storage, registry, producer, gen, logger) if err != nil { @@ -918,8 +918,8 @@ func restoreStateDBCache(ctx context.Context, storage storage.Storage, registry // batch in the chain and is used to query state at a certain height. // // This method checks if the stateDB data is available for a given batch hash (so it can be restored if not) -func stateDBAvailableForBatch(ctx context.Context, storage storage.Storage, hash common.L2BatchHash) bool { - _, err := storage.CreateStateDB(ctx, hash) +func stateDBAvailableForBatch(ctx context.Context, registry components.BatchRegistry, hash common.L2BatchHash) bool { + _, err := registry.GetBatchState(ctx, &hash) return err == nil } @@ -938,7 +938,7 @@ func replayBatchesToValidState(ctx context.Context, storage storage.Storage, reg return fmt.Errorf("no head batch found in DB but expected to replay batches - %w", err) } // loop backwards building a slice of all batches that don't have cached stateDB data available - for !stateDBAvailableForBatch(ctx, storage, batchToReplayFrom.Hash()) { + for !stateDBAvailableForBatch(ctx, registry, batchToReplayFrom.Hash()) { batchesToReplay = append(batchesToReplay, batchToReplayFrom) if batchToReplayFrom.NumberU64() == 0 { // no more parents to check, replaying from genesis diff --git a/go/enclave/events/subscription_manager.go b/go/enclave/events/subscription_manager.go index 3af2250c56..27e6515d41 100644 --- a/go/enclave/events/subscription_manager.go +++ b/go/enclave/events/subscription_manager.go @@ -6,6 +6,8 @@ import ( "fmt" "sync" + "github.com/ten-protocol/go-ten/go/enclave/components" + "github.com/ten-protocol/go-ten/go/enclave/vkhandler" gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc" @@ -36,7 +38,8 @@ type logSubscription struct { // SubscriptionManager manages the creation/deletion of subscriptions, and the filtering and encryption of logs for // active subscriptions. type SubscriptionManager struct { - storage storage.Storage + storage storage.Storage + registry components.BatchRegistry subscriptions map[gethrpc.ID]*logSubscription chainID int64 @@ -45,9 +48,10 @@ type SubscriptionManager struct { logger gethlog.Logger } -func NewSubscriptionManager(storage storage.Storage, chainID int64, logger gethlog.Logger) *SubscriptionManager { +func NewSubscriptionManager(storage storage.Storage, registry components.BatchRegistry, chainID int64, logger gethlog.Logger) *SubscriptionManager { return &SubscriptionManager{ - storage: storage, + storage: storage, + registry: registry, subscriptions: map[gethrpc.ID]*logSubscription{}, chainID: chainID, @@ -89,9 +93,9 @@ func (s *SubscriptionManager) RemoveSubscription(id gethrpc.ID) { } // FilterLogsForReceipt removes the logs that the sender of a transaction is not allowed to view -func FilterLogsForReceipt(ctx context.Context, receipt *types.Receipt, account *gethcommon.Address, storage storage.Storage) ([]*types.Log, error) { - filteredLogs := []*types.Log{} - stateDB, err := storage.CreateStateDB(ctx, receipt.BlockHash) +func FilterLogsForReceipt(ctx context.Context, receipt *types.Receipt, account *gethcommon.Address, registry components.BatchRegistry) ([]*types.Log, error) { + var filteredLogs []*types.Log + stateDB, err := registry.GetBatchState(ctx, &receipt.BlockHash) if err != nil { return nil, fmt.Errorf("could not create state DB to filter logs. Cause: %w", err) } @@ -130,7 +134,8 @@ func (s *SubscriptionManager) GetSubscribedLogsForBatch(ctx context.Context, bat } // the stateDb is needed to extract the user addresses from the topics - stateDB, err := s.storage.CreateStateDB(ctx, batch.Hash()) + h := batch.Hash() + stateDB, err := s.registry.GetBatchState(ctx, &h) if err != nil { return nil, fmt.Errorf("could not create state DB to filter logs. Cause: %w", err) } diff --git a/go/enclave/rpc/GetTransactionCount.go b/go/enclave/rpc/GetTransactionCount.go index 10018d2cd9..99989b32af 100644 --- a/go/enclave/rpc/GetTransactionCount.go +++ b/go/enclave/rpc/GetTransactionCount.go @@ -56,7 +56,8 @@ func GetTransactionCountExecute(builder *CallBuilder[uint64, string], rpc *Encry if err == nil { // todo - we should return an error when head state is not available, but for current test situations with race // conditions we allow it to return zero while head state is uninitialized - s, err := rpc.storage.CreateStateDB(builder.ctx, l2Head.Hash()) + h := l2Head.Hash() + s, err := rpc.registry.GetBatchState(builder.ctx, &h) if err != nil { return err } diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 6dabe91f3c..88c55f967d 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -74,7 +74,7 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri } // We filter out irrelevant logs. - txReceipt.Logs, err = events.FilterLogsForReceipt(builder.ctx, txReceipt, &txSigner, rpc.storage) + txReceipt.Logs, err = events.FilterLogsForReceipt(builder.ctx, txReceipt, &txSigner, rpc.registry) if err != nil { rpc.logger.Error("error filter logs ", log.TxKey, txHash, log.ErrKey, err) // this is a system error diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index 1938ad77b9..491d92461b 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -9,6 +9,9 @@ import ( "math/big" "time" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/triedb/hashdb" + "github.com/ethereum/go-ethereum/triedb" "github.com/dgraph-io/ristretto" @@ -24,11 +27,9 @@ import ( "github.com/ethereum/go-ethereum/rlp" gethcore "github.com/ethereum/go-ethereum/core" - gethcrypto "github.com/ethereum/go-ethereum/crypto" - "github.com/ten-protocol/go-ten/go/common/syserr" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + gethcrypto "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ten-protocol/go-ten/go/common" "github.com/ten-protocol/go-ten/go/common/log" @@ -65,7 +66,7 @@ type storageImpl struct { cachedSharedSecret *crypto.SharedEnclaveSecret - stateDB state.Database + stateCache state.Database chainConfig *params.ChainConfig logger gethlog.Logger } @@ -78,21 +79,28 @@ func NewStorageFromConfig(config *config.EnclaveConfig, chainConfig *params.Chai return NewStorage(backingDB, chainConfig, logger) } +var defaultCacheConfig = &gethcore.CacheConfig{ + TrieCleanLimit: 256, + TrieDirtyLimit: 256, + TrieTimeLimit: 5 * time.Minute, + SnapshotLimit: 256, + SnapshotWait: true, + StateScheme: rawdb.HashScheme, +} + +var trieDBConfig = &triedb.Config{ + Preimages: defaultCacheConfig.Preimages, + IsVerkle: false, + HashDB: &hashdb.Config{ + CleanCacheSize: defaultCacheConfig.TrieCleanLimit * 1024 * 1024, + }, +} + func NewStorage(backingDB enclavedb.EnclaveDB, chainConfig *params.ChainConfig, logger gethlog.Logger) Storage { - // these are the twice the default configs from geth - // todo - consider tweaking these independently on the validator and on the sequencer - // the validator probably need higher values on this cache? - cacheConfig := &gethcore.CacheConfig{ - TrieCleanLimit: 256 * 2, - TrieDirtyLimit: 256 * 2, - TrieTimeLimit: 5 * time.Minute, - SnapshotLimit: 256 * 2, - SnapshotWait: true, - } - - stateDB := state.NewDatabaseWithConfig(backingDB, &triedb.Config{ - Preimages: cacheConfig.Preimages, - }) + // Open trie database with provided config + triedb := triedb.NewDatabase(backingDB, trieDBConfig) + + stateDB := state.NewDatabaseWithNodeDB(backingDB, triedb) // todo (tudor) figure out the config ristrettoCache, err := ristretto.NewCache(&ristretto.Config{ @@ -106,7 +114,7 @@ func NewStorage(backingDB enclavedb.EnclaveDB, chainConfig *params.ChainConfig, ristrettoStore := ristretto_store.NewRistretto(ristrettoCache) return &storageImpl{ db: backingDB, - stateDB: stateDB, + stateCache: stateDB, chainConfig: chainConfig, blockCache: cache.New[*types.Block](ristrettoStore), batchCacheBySeqNo: cache.New[*core.Batch](ristrettoStore), @@ -117,11 +125,11 @@ func NewStorage(backingDB enclavedb.EnclaveDB, chainConfig *params.ChainConfig, } func (s *storageImpl) TrieDB() *triedb.Database { - return s.stateDB.TrieDB() + return s.stateCache.TrieDB() } func (s *storageImpl) StateDB() state.Database { - return s.stateDB + return s.stateCache } func (s *storageImpl) Close() error { @@ -325,16 +333,16 @@ func (s *storageImpl) CreateStateDB(ctx context.Context, batchHash common.L2Batc return nil, err } - statedb, err := state.New(batch.Header.Root, s.stateDB, nil) + statedb, err := state.New(batch.Header.Root, s.stateCache, nil) if err != nil { - return nil, syserr.NewInternalError(fmt.Errorf("could not create state DB for %s. Cause: %w", batch.Header.Root, err)) + return nil, fmt.Errorf("could not create state DB for %s. Cause: %w", batch.Header.Root, err) } return statedb, nil } func (s *storageImpl) EmptyStateDB() (*state.StateDB, error) { defer s.logDuration("EmptyStateDB", measure.NewStopwatch()) - statedb, err := state.New(types.EmptyRootHash, s.stateDB, nil) + statedb, err := state.New(types.EmptyRootHash, s.stateCache, nil) if err != nil { return nil, fmt.Errorf("could not create state DB. Cause: %w", err) } From c2a7b13033439148fd020ab25f4fef594fd52300 Mon Sep 17 00:00:00 2001 From: badgersrus <43809877+badgersrus@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:07:35 +0400 Subject: [PATCH 2/3] Remove redundant Host Volume code (#1887) * remove host volumes from host docker containers as we're using Postgres now --- go/node/docker_node.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/node/docker_node.go b/go/node/docker_node.go index cca9ad556d..eb67a175c4 100644 --- a/go/node/docker_node.go +++ b/go/node/docker_node.go @@ -10,7 +10,6 @@ import ( ) var ( - _hostDataDir = "/data" // this is how the directory is referenced within the host container _enclaveDataDir = "/enclavedata" // this is how the directory is references within the enclave container ) @@ -133,9 +132,7 @@ func (d *DockerNode) startHost() error { d.cfg.hostP2PPort, } - hostVolume := map[string]string{d.cfg.nodeName + "-host-volume": _hostDataDir} - - _, err := docker.StartNewContainer(d.cfg.nodeName+"-host", d.cfg.hostImage, cmd, exposedPorts, nil, nil, hostVolume) + _, err := docker.StartNewContainer(d.cfg.nodeName+"-host", d.cfg.hostImage, cmd, exposedPorts, nil, nil, nil) return err } From ac1b90a6beb5a9b9e701bb947faedaebedc42eab Mon Sep 17 00:00:00 2001 From: badgersrus <43809877+badgersrus@users.noreply.github.com> Date: Tue, 30 Apr 2024 14:34:20 +0400 Subject: [PATCH 3/3] New Tenscan APIs (#1879) * add new APIs required for tenscan and refactor existing code --- go/common/query_types.go | 6 +- go/host/rpc/clientapi/client_api_scan.go | 29 ++- go/host/storage/hostdb/batch.go | 134 ++++++++++--- go/host/storage/hostdb/batch_test.go | 112 +++++++++-- go/host/storage/hostdb/rollup.go | 116 ++++++++++- go/host/storage/hostdb/rollup_test.go | 180 +++++++++++++++++- go/host/storage/hostdb/sql_statements.go | 21 +- go/host/storage/hostdb/utils.go | 6 + go/host/storage/init/postgres/001_init.sql | 3 +- go/host/storage/init/postgres/postgres.go | 6 +- .../{host_sqlite_init.sql => 001_init.sql} | 7 +- go/host/storage/init/sqlite/sqlite.go | 2 +- go/host/storage/interfaces.go | 12 ++ go/host/storage/storage.go | 32 +++- go/obsclient/obsclient.go | 74 ++++++- go/rpc/client.go | 11 +- integration/manualtests/client_test.go | 2 +- integration/simulation/output_stats.go | 2 +- integration/simulation/utils.go | 2 +- integration/simulation/validate_chain.go | 6 +- integration/tenscan/tenscan_test.go | 75 +++++++- tools/tenscan/backend/obscuroscan_backend.go | 29 ++- .../webserver/webserver_routes_items.go | 80 +++++++- 23 files changed, 849 insertions(+), 98 deletions(-) rename go/host/storage/init/sqlite/{host_sqlite_init.sql => 001_init.sql} (88%) diff --git a/go/common/query_types.go b/go/common/query_types.go index 910b1d5e2f..eb6ed1f89c 100644 --- a/go/common/query_types.go +++ b/go/common/query_types.go @@ -49,7 +49,7 @@ type PublicTransaction struct { type PublicBatch struct { SequencerOrderNo *big.Int `json:"sequence"` - Hash []byte `json:"hash"` + Hash string `json:"hash"` FullHash common.Hash `json:"fullHash"` Height *big.Int `json:"height"` TxCount *big.Int `json:"txCount"` @@ -65,12 +65,12 @@ type PublicBatchDeprecated struct { type PublicRollup struct { ID *big.Int - Hash []byte + Hash string FirstSeq *big.Int LastSeq *big.Int Timestamp uint64 Header *RollupHeader - L1Hash []byte + L1Hash string } type PublicBlock struct { diff --git a/go/host/rpc/clientapi/client_api_scan.go b/go/host/rpc/clientapi/client_api_scan.go index 7bbde8b2cb..359f7ca1fc 100644 --- a/go/host/rpc/clientapi/client_api_scan.go +++ b/go/host/rpc/clientapi/client_api_scan.go @@ -29,6 +29,11 @@ func (s *ScanAPI) GetTotalContractCount(ctx context.Context) (*big.Int, error) { return s.host.EnclaveClient().GetTotalContractCount(ctx) } +// GetTransaction returns the transaction given its hash. +func (s *ScanAPI) GetTransaction(hash gethcommon.Hash) (*common.PublicTransaction, error) { + return s.host.Storage().FetchTransaction(hash) +} + // GetTotalTxCount returns the number of recorded transactions on the network. func (s *ScanAPI) GetTotalTransactionCount() (*big.Int, error) { return s.host.Storage().FetchTotalTxCount() @@ -65,8 +70,13 @@ func (s *ScanAPI) GetLatestBatch() (*common.BatchHeader, error) { } // GetBatchByHeight returns the `BatchHeader` with the given height -func (s *ScanAPI) GetBatchByHeight(height *big.Int) (*common.BatchHeader, error) { - return s.host.Storage().FetchBatchHeaderByHeight(height) +func (s *ScanAPI) GetBatchByHeight(height *big.Int) (*common.PublicBatch, error) { + return s.host.Storage().FetchBatchByHeight(height) +} + +// GetRollupBySeqNo returns the `PublicRollup` that contains the batch with the given sequence number +func (s *ScanAPI) GetRollupBySeqNo(seqNo uint64) (*common.PublicRollup, error) { + return s.host.Storage().FetchRollupBySeqNo(seqNo) } // GetRollupListing returns a paginated list of Rollups @@ -88,3 +98,18 @@ func (s *ScanAPI) GetPublicTransactionData(ctx context.Context, pagination *comm func (s *ScanAPI) GetBlockListing(pagination *common.QueryPagination) (*common.BlockListingResponse, error) { return s.host.Storage().FetchBlockListing(pagination) } + +// GetRollupByHash returns the public rollup data given its hash +func (s *ScanAPI) GetRollupByHash(rollupHash gethcommon.Hash) (*common.PublicRollup, error) { + return s.host.Storage().FetchRollupByHash(rollupHash) +} + +// GetRollupBatches returns the list of batches included in a rollup given its hash +func (s *ScanAPI) GetRollupBatches(rollupHash gethcommon.Hash) (*common.BatchListingResponse, error) { + return s.host.Storage().FetchRollupBatches(rollupHash) +} + +// GetBatchTransactions returns the public tx data of all txs present in a rollup given its hash +func (s *ScanAPI) GetBatchTransactions(batchHash gethcommon.Hash) (*common.TransactionListingResponse, error) { + return s.host.Storage().FetchBatchTransactions(batchHash) +} diff --git a/go/host/storage/hostdb/batch.go b/go/host/storage/hostdb/batch.go index 1eb0e7d740..2ff58a12cb 100644 --- a/go/host/storage/hostdb/batch.go +++ b/go/host/storage/hostdb/batch.go @@ -14,12 +14,14 @@ import ( const ( selectTxCount = "SELECT total FROM transaction_count WHERE id = 1" + selectTxSeqNo = "SELECT full_hash, b_sequence FROM transactions_host WHERE hash = " selectBatch = "SELECT sequence, full_hash, hash, height, ext_batch FROM batch_host" selectExtBatch = "SELECT ext_batch FROM batch_host" selectLatestBatch = "SELECT sequence, full_hash, hash, height, ext_batch FROM batch_host ORDER BY sequence DESC LIMIT 1" selectTxsAndBatch = "SELECT t.hash FROM transactions_host t JOIN batch_host b ON t.b_sequence = b.sequence WHERE b.full_hash = " selectBatchSeqByTx = "SELECT b_sequence FROM transactions_host WHERE hash = " - selectTxBySeq = "SELECT hash FROM transactions_host WHERE b_sequence = " + selectTxBySeq = "SELECT full_hash FROM transactions_host WHERE b_sequence = " + selectBatchTxs = "SELECT t.full_hash, b.sequence, b.height, b.ext_batch FROM transactions_host t JOIN batch_host b ON t.b_sequence = b.sequence" ) // AddBatch adds a batch and its header to the DB @@ -41,8 +43,8 @@ func AddBatch(dbtx *dbTransaction, statements *SQLStatements, batch *common.ExtB } if len(batch.TxHashes) > 0 { - for _, transaction := range batch.TxHashes { - _, err = dbtx.tx.Exec(statements.InsertTransactions, transaction.Bytes(), batch.SeqNo().Uint64()) + for _, txHash := range batch.TxHashes { + _, err = dbtx.tx.Exec(statements.InsertTransactions, truncTo16(txHash), txHash.Bytes(), batch.SeqNo().Uint64()) if err != nil { return fmt.Errorf("failed to insert transaction with hash: %d", err) } @@ -67,7 +69,7 @@ func AddBatch(dbtx *dbTransaction, statements *SQLStatements, batch *common.ExtB // GetBatchListing returns latest batches given a pagination. // For example, page 0, size 10 will return the latest 10 batches. func GetBatchListing(db HostDB, pagination *common.QueryPagination) (*common.BatchListingResponse, error) { - headBatch, err := GetCurrentHeadBatch(db.GetSQLDB()) + headBatch, err := GetCurrentHeadBatch(db) if err != nil { return nil, err } @@ -98,7 +100,7 @@ func GetBatchListing(db HostDB, pagination *common.QueryPagination) (*common.Bat // GetBatchListingDeprecated returns latest batches given a pagination. // For example, page 0, size 10 will return the latest 10 batches. func GetBatchListingDeprecated(db HostDB, pagination *common.QueryPagination) (*common.BatchListingResponseDeprecated, error) { - headBatch, err := GetCurrentHeadBatch(db.GetSQLDB()) + headBatch, err := GetCurrentHeadBatch(db) if err != nil { return nil, err } @@ -159,8 +161,8 @@ func GetBatchBySequenceNumber(db HostDB, seqNo uint64) (*common.ExtBatch, error) } // GetCurrentHeadBatch retrieves the current head batch with the largest sequence number (or height). -func GetCurrentHeadBatch(db *sql.DB) (*common.PublicBatch, error) { - return fetchHeadBatch(db) +func GetCurrentHeadBatch(db HostDB) (*common.PublicBatch, error) { + return fetchHeadBatch(db.GetSQLDB()) } // GetBatchHeader returns the batch header given the hash. @@ -181,8 +183,8 @@ func GetBatchHashByNumber(db HostDB, number *big.Int) (*gethcommon.Hash, error) } // GetHeadBatchHeader returns the latest batch header. -func GetHeadBatchHeader(db *sql.DB) (*common.BatchHeader, error) { - batch, err := fetchHeadBatch(db) +func GetHeadBatchHeader(db HostDB) (*common.BatchHeader, error) { + batch, err := fetchHeadBatch(db.GetSQLDB()) if err != nil { return nil, err } @@ -191,16 +193,15 @@ func GetHeadBatchHeader(db *sql.DB) (*common.BatchHeader, error) { // GetBatchNumber returns the height of the batch containing the given transaction hash. func GetBatchNumber(db HostDB, txHash gethcommon.Hash) (*big.Int, error) { - txBytes := txHash.Bytes() - batchHeight, err := fetchBatchNumber(db, txBytes) + batchHeight, err := fetchBatchNumber(db, truncTo16(txHash)) if err != nil { return nil, err } return batchHeight, nil } -// GetBatchTxs returns the transaction hashes of the batch with the given hash. -func GetBatchTxs(db HostDB, batchHash gethcommon.Hash) ([]gethcommon.Hash, error) { +// GetBatchTxHashes returns the transaction hashes of the batch with the given hash. +func GetBatchTxHashes(db HostDB, batchHash gethcommon.Hash) ([]gethcommon.Hash, error) { query := selectTxsAndBatch + db.GetSQLStatement().Placeholder rows, err := db.GetSQLDB().Query(query, batchHash) if err != nil { @@ -226,15 +227,41 @@ func GetBatchTxs(db HostDB, batchHash gethcommon.Hash) ([]gethcommon.Hash, error } // GetTotalTxCount returns the total number of batched transactions. -func GetTotalTxCount(db *sql.DB) (*big.Int, error) { +func GetTotalTxCount(db HostDB) (*big.Int, error) { var totalCount int - err := db.QueryRow(selectTxCount).Scan(&totalCount) + err := db.GetSQLDB().QueryRow(selectTxCount).Scan(&totalCount) if err != nil { return nil, fmt.Errorf("failed to retrieve total transaction count: %w", err) } return big.NewInt(int64(totalCount)), nil } +// GetTransaction returns a transaction given its hash +func GetTransaction(db HostDB, hash gethcommon.Hash) (*common.PublicTransaction, error) { + query := selectTxSeqNo + db.GetSQLStatement().Placeholder + + var fullHash []byte + var seq int + err := db.GetSQLDB().QueryRow(query, truncTo16(hash)).Scan(&fullHash, &seq) + if err != nil { + return nil, fmt.Errorf("failed to retrieve transaction sequence number: %w", err) + } + + batch, err := GetBatchBySequenceNumber(db, uint64(seq)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve batch by sequence number: %w", err) + } + + tx := &common.PublicTransaction{ + TransactionHash: gethcommon.BytesToHash(fullHash), + BatchHeight: batch.Header.Number, + BatchTimestamp: batch.Header.Time, + Finality: common.BatchFinal, + } + + return tx, nil +} + // GetPublicBatch returns the batch with the given hash. func GetPublicBatch(db HostDB, hash common.L2BatchHash) (*common.PublicBatch, error) { whereQuery := " WHERE b.hash=" + db.GetSQLStatement().Placeholder @@ -245,7 +272,7 @@ func GetPublicBatch(db HostDB, hash common.L2BatchHash) (*common.PublicBatch, er func GetBatchByTx(db HostDB, txHash gethcommon.Hash) (*common.ExtBatch, error) { var seqNo uint64 query := selectBatchSeqByTx + db.GetSQLStatement().Placeholder - err := db.GetSQLDB().QueryRow(query, txHash).Scan(&seqNo) + err := db.GetSQLDB().QueryRow(query, truncTo16(txHash)).Scan(&seqNo) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, errutil.ErrNotFound @@ -262,27 +289,35 @@ func GetBatchByHash(db HostDB, hash common.L2BatchHash) (*common.ExtBatch, error } // GetLatestBatch returns the head batch header -func GetLatestBatch(db *sql.DB) (*common.BatchHeader, error) { - headBatch, err := fetchHeadBatch(db) +func GetLatestBatch(db HostDB) (*common.BatchHeader, error) { + headBatch, err := fetchHeadBatch(db.GetSQLDB()) if err != nil { return nil, fmt.Errorf("failed to fetch head batch: %w", err) } return headBatch.Header, nil } +// GetBatchHeaderByHeight returns the batch header given the height +func GetBatchHeaderByHeight(db HostDB, height *big.Int) (*common.BatchHeader, error) { + whereQuery := " WHERE height=" + db.GetSQLStatement().Placeholder + return fetchBatchHeader(db.GetSQLDB(), whereQuery, height.Uint64()) +} + // GetBatchByHeight returns the batch header given the height -func GetBatchByHeight(db HostDB, height *big.Int) (*common.BatchHeader, error) { +func GetBatchByHeight(db HostDB, height *big.Int) (*common.PublicBatch, error) { whereQuery := " WHERE height=" + db.GetSQLStatement().Placeholder - headBatch, err := fetchBatchHeader(db.GetSQLDB(), whereQuery, height.Uint64()) - if err != nil { - return nil, fmt.Errorf("failed to batch header: %w", err) - } - return headBatch, nil + return fetchPublicBatch(db.GetSQLDB(), whereQuery, height.Uint64()) +} + +// GetBatchTransactions returns the TransactionListingResponse for a given batch hash +func GetBatchTransactions(db HostDB, batchHash gethcommon.Hash) (*common.TransactionListingResponse, error) { + whereQuery := " WHERE b.full_hash=" + db.GetSQLStatement().Placeholder + return fetchBatchTxs(db.GetSQLDB(), whereQuery, batchHash) } func fetchBatchHeader(db *sql.DB, whereQuery string, args ...any) (*common.BatchHeader, error) { var extBatch []byte - query := selectExtBatch + " " + whereQuery + query := selectExtBatch + whereQuery var err error if len(args) > 0 { err = db.QueryRow(query, args...).Scan(&extBatch) @@ -333,7 +368,7 @@ func fetchPublicBatch(db *sql.DB, whereQuery string, args ...any) (*common.Publi var heightInt64 int var extBatch []byte - query := selectBatch + " " + whereQuery + query := selectBatch + whereQuery var err error if len(args) > 0 { @@ -355,7 +390,7 @@ func fetchPublicBatch(db *sql.DB, whereQuery string, args ...any) (*common.Publi batch := &common.PublicBatch{ SequencerOrderNo: new(big.Int).SetInt64(int64(sequenceInt64)), - Hash: hash, + Hash: bytesToHexString(hash), FullHash: fullHash, Height: new(big.Int).SetInt64(int64(heightInt64)), TxCount: new(big.Int).SetInt64(int64(len(b.TxHashes))), @@ -419,7 +454,7 @@ func fetchHeadBatch(db *sql.DB) (*common.PublicBatch, error) { batch := &common.PublicBatch{ SequencerOrderNo: new(big.Int).SetInt64(int64(sequenceInt64)), - Hash: hash, + Hash: bytesToHexString(hash), FullHash: fullHash, Height: new(big.Int).SetInt64(int64(heightInt64)), TxCount: new(big.Int).SetInt64(int64(len(b.TxHashes))), @@ -454,3 +489,46 @@ func fetchTx(db HostDB, seqNo uint64) ([]common.TxHash, error) { return transactions, nil } + +func fetchBatchTxs(db *sql.DB, whereQuery string, batchHash gethcommon.Hash) (*common.TransactionListingResponse, error) { + query := selectBatchTxs + whereQuery + rows, err := db.Query(query, batchHash) + if err != nil { + return nil, fmt.Errorf("query execution for select batch txs failed: %w", err) + } + defer rows.Close() + + var transactions []common.PublicTransaction + for rows.Next() { + var ( + fullHash []byte + sequence int + height int + extBatch []byte + ) + err := rows.Scan(&fullHash, &sequence, &height, &extBatch) + if err != nil { + return nil, err + } + extBatchDecoded := new(common.ExtBatch) + if err := rlp.DecodeBytes(extBatch, extBatchDecoded); err != nil { + return nil, fmt.Errorf("could not decode batch. Cause: %w", err) + } + transaction := common.PublicTransaction{ + TransactionHash: gethcommon.BytesToHash(fullHash), + BatchHeight: big.NewInt(int64(height)), + BatchTimestamp: extBatchDecoded.Header.Time, + Finality: common.BatchFinal, + } + transactions = append(transactions, transaction) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error looping through transacion rows: %w", err) + } + + return &common.TransactionListingResponse{ + TransactionsData: transactions, + Total: uint64(len(transactions)), + }, nil +} diff --git a/go/host/storage/hostdb/batch_test.go b/go/host/storage/hostdb/batch_test.go index 86c861a2c9..1ea3b433ce 100644 --- a/go/host/storage/hostdb/batch_test.go +++ b/go/host/storage/hostdb/batch_test.go @@ -4,6 +4,7 @@ import ( "errors" "math/big" "testing" + "time" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -64,7 +65,7 @@ func TestHigherNumberBatchBecomesBatchHeader(t *testing.T) { //nolint:dupl dbtx.Write() - batchHeader, err := GetHeadBatchHeader(db.GetSQLDB()) + batchHeader, err := GetHeadBatchHeader(db) if err != nil { t.Errorf("stored batch but could not retrieve header. Cause: %s", err) } @@ -93,7 +94,7 @@ func TestLowerNumberBatchDoesNotBecomeBatchHeader(t *testing.T) { //nolint:dupl } dbtx.Write() - batchHeader, err := GetHeadBatchHeader(db.GetSQLDB()) + batchHeader, err := GetHeadBatchHeader(db) if err != nil { t.Errorf("stored batch but could not retrieve header. Cause: %s", err) } @@ -104,7 +105,7 @@ func TestLowerNumberBatchDoesNotBecomeBatchHeader(t *testing.T) { //nolint:dupl func TestHeadBatchHeaderIsNotSetInitially(t *testing.T) { db, _ := createSQLiteDB(t) - _, err := GetHeadBatchHeader(db.GetSQLDB()) + _, err := GetHeadBatchHeader(db) if !errors.Is(err, errutil.ErrNotFound) { t.Errorf("head batch was set, but no batchs had been written") } @@ -186,7 +187,7 @@ func TestCanRetrieveBatchTransactions(t *testing.T) { } dbtx.Write() - batchTxs, err := GetBatchTxs(db, batch.Header.Hash()) + batchTxs, err := GetBatchTxHashes(db, batch.Header.Hash()) if err != nil { t.Errorf("stored batch but could not retrieve headers transactions. Cause: %s", err) } @@ -219,7 +220,7 @@ func TestCanRetrieveTotalNumberOfTransactions(t *testing.T) { t.Errorf("could not store batch. Cause: %s", err) } - txHashesTwo := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} + txHashesTwo := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} batchTwo := createBatch(batchNumber+1, txHashesTwo) err = AddBatch(dbtx, db.GetSQLStatement(), &batchTwo) @@ -228,7 +229,7 @@ func TestCanRetrieveTotalNumberOfTransactions(t *testing.T) { } dbtx.Write() - totalTxs, err := GetTotalTxCount(db.GetSQLDB()) + totalTxs, err := GetTotalTxCount(db) if err != nil { t.Errorf("was not able to read total number of transactions. Cause: %s", err) } @@ -248,7 +249,7 @@ func TestGetLatestBatch(t *testing.T) { t.Errorf("could not store batch. Cause: %s", err) } - txHashesTwo := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} + txHashesTwo := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} batchTwo := createBatch(batchNumber+1, txHashesTwo) err = AddBatch(dbtx, db.GetSQLStatement(), &batchTwo) @@ -257,7 +258,7 @@ func TestGetLatestBatch(t *testing.T) { } dbtx.Write() - batch, err := GetLatestBatch(db.GetSQLDB()) + batch, err := GetLatestBatch(db) if err != nil { t.Errorf("was not able to read total number of transactions. Cause: %s", err) } @@ -267,6 +268,55 @@ func TestGetLatestBatch(t *testing.T) { } } +func TestGetTransaction(t *testing.T) { + db, _ := createSQLiteDB(t) + txHash1 := gethcommon.BytesToHash([]byte("magicStringOne")) + txHash2 := gethcommon.BytesToHash([]byte("magicStringOne")) + txHashes := []common.L2TxHash{txHash1, txHash2} + batchOne := createBatch(batchNumber, txHashes) + dbtx, _ := db.NewDBTransaction() + err := AddBatch(dbtx, db.GetSQLStatement(), &batchOne) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + dbtx.Write() + + tx, err := GetTransaction(db, txHash2) + if err != nil { + t.Errorf("was not able to get transaction. Cause: %s", err) + } + + if tx.BatchHeight.Cmp(big.NewInt(batchNumber)) != 0 { + t.Errorf("tx batch height was not retrieved correctly") + } + if tx.TransactionHash.Cmp(txHash2) != 0 { + t.Errorf("tx hash was not retrieved correctly") + } +} + +func TestGetBatchByHeight(t *testing.T) { + db, _ := createSQLiteDB(t) + batch1 := createBatch(batchNumber, []common.L2TxHash{}) + batch2 := createBatch(batchNumber+5, []common.L2TxHash{}) + dbtx, _ := db.NewDBTransaction() + err := AddBatch(dbtx, db.GetSQLStatement(), &batch1) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + err = AddBatch(dbtx, db.GetSQLStatement(), &batch2) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + dbtx.Write() + publicBatch, err := GetBatchByHeight(db, batch2.Header.Number) + if err != nil { + t.Errorf("stored batch but could not retrieve header. Cause: %s", err) + } + if batch2.Header.Number.Cmp(publicBatch.Header.Number) != 0 { + t.Errorf("batch header was not stored correctly") + } +} + func TestGetBatchListing(t *testing.T) { db, _ := createSQLiteDB(t) txHashesOne := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringOne")), gethcommon.BytesToHash([]byte("magicStringTwo"))} @@ -277,7 +327,7 @@ func TestGetBatchListing(t *testing.T) { t.Errorf("could not store batch. Cause: %s", err) } - txHashesTwo := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} + txHashesTwo := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} batchTwo := createBatch(batchNumber+1, txHashesTwo) err = AddBatch(dbtx, db.GetSQLStatement(), &batchTwo) @@ -285,7 +335,7 @@ func TestGetBatchListing(t *testing.T) { t.Errorf("could not store batch. Cause: %s", err) } - txHashesThree := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringFive")), gethcommon.BytesToHash([]byte("magicStringSix"))} + txHashesThree := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringFive")), gethcommon.BytesToHash([]byte("magicStringSix"))} batchThree := createBatch(batchNumber+2, txHashesThree) err = AddBatch(dbtx, db.GetSQLStatement(), &batchThree) @@ -359,7 +409,7 @@ func TestGetBatchListingDeprecated(t *testing.T) { t.Errorf("could not store batch. Cause: %s", err) } - txHashesTwo := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} + txHashesTwo := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} batchTwo := createBatch(batchNumber+1, txHashesTwo) err = AddBatch(dbtx, db.GetSQLStatement(), &batchTwo) @@ -367,7 +417,7 @@ func TestGetBatchListingDeprecated(t *testing.T) { t.Errorf("could not store batch. Cause: %s", err) } - txHashesThree := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringFive")), gethcommon.BytesToHash([]byte("magicStringSix"))} + txHashesThree := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringFive")), gethcommon.BytesToHash([]byte("magicStringSix"))} batchThree := createBatch(batchNumber+2, txHashesThree) err = AddBatch(dbtx, db.GetSQLStatement(), &batchThree) @@ -431,10 +481,48 @@ func TestGetBatchListingDeprecated(t *testing.T) { } } +func TestGetBatchTransactions(t *testing.T) { + db, _ := createSQLiteDB(t) + txHashesOne := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringOne")), gethcommon.BytesToHash([]byte("magicStringTwo"))} + batchOne := createBatch(batchNumber, txHashesOne) + dbtx, _ := db.NewDBTransaction() + err := AddBatch(dbtx, db.GetSQLStatement(), &batchOne) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + + txHashesTwo := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour")), gethcommon.BytesToHash([]byte("magicStringFive"))} + batchTwo := createBatch(batchNumber+1, txHashesTwo) + + err = AddBatch(dbtx, db.GetSQLStatement(), &batchTwo) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + + dbtx.Write() + + txListing1, err := GetBatchTransactions(db, batchOne.Header.Hash()) + if err != nil { + t.Errorf("stored batch but could not retrieve transactions. Cause: %s", err) + } + + if txListing1.Total != uint64(len(txHashesOne)) { + t.Errorf("batch transactions were not retrieved correctly") + } + txListing2, err := GetBatchTransactions(db, batchTwo.Header.Hash()) + if err != nil { + t.Errorf("stored batch but could not retrieve transactions. Cause: %s", err) + } + if txListing2.Total != uint64(len(txHashesTwo)) { + t.Errorf("batch transactions were not retrieved correctly") + } +} + func createBatch(batchNum int64, txHashes []common.L2BatchHash) common.ExtBatch { header := common.BatchHeader{ SequencerOrderNo: big.NewInt(batchNum), Number: big.NewInt(batchNum), + Time: uint64(time.Now().Unix()), } batch := common.ExtBatch{ Header: &header, diff --git a/go/host/storage/hostdb/rollup.go b/go/host/storage/hostdb/rollup.go index 6310c615bd..537dc7b832 100644 --- a/go/host/storage/hostdb/rollup.go +++ b/go/host/storage/hostdb/rollup.go @@ -14,8 +14,10 @@ import ( ) const ( - selectExtRollup = "SELECT ext_rollup from rollup_host r" - selectLatestRollup = "SELECT ext_rollup FROM rollup_host ORDER BY time_stamp DESC LIMIT 1" + selectExtRollup = "SELECT ext_rollup from rollup_host r" + selectLatestRollup = "SELECT ext_rollup FROM rollup_host ORDER BY time_stamp DESC LIMIT 1" + selectRollupBatches = "SELECT b.sequence, b.hash, b.full_hash, b.height, b.ext_batch FROM rollup_host r JOIN batch_host b ON r.start_seq <= b.sequence AND r.end_seq >= b.sequence" + selectPublicRollup = "SELECT id, hash, start_seq, end_seq, time_stamp, ext_rollup, compression_block FROM rollup_host" ) // AddRollup adds a rollup to the DB @@ -24,6 +26,7 @@ func AddRollup(dbtx *dbTransaction, statements *SQLStatements, rollup *common.Ex if err != nil { return fmt.Errorf("could not encode rollup: %w", err) } + _, err = dbtx.tx.Exec(statements.InsertRollup, truncTo16(rollup.Header.Hash()), // short hash metadata.FirstBatchSequence.Uint64(), // first batch sequence @@ -65,12 +68,12 @@ func GetRollupListing(db HostDB, pagination *common.QueryPagination) (*common.Ro rollup = common.PublicRollup{ ID: big.NewInt(int64(id)), - Hash: hash, + Hash: bytesToHexString(hash), FirstSeq: big.NewInt(int64(startSeq)), LastSeq: big.NewInt(int64(endSeq)), Timestamp: uint64(timeStamp), Header: extRollupDecoded.Header, - L1Hash: compressionBlock, + L1Hash: bytesToHexString(compressionBlock), } rollups = append(rollups, rollup) } @@ -102,14 +105,78 @@ func GetRollupHeaderByBlock(db HostDB, blockHash gethcommon.Hash) (*common.Rollu } // GetLatestRollup returns the latest rollup ordered by timestamp -func GetLatestRollup(db *sql.DB) (*common.RollupHeader, error) { - extRollup, err := fetchHeadRollup(db) +func GetLatestRollup(db HostDB) (*common.RollupHeader, error) { + extRollup, err := fetchHeadRollup(db.GetSQLDB()) if err != nil { return nil, fmt.Errorf("failed to fetch head rollup: %w", err) } return extRollup.Header, nil } +func GetRollupByHash(db HostDB, rollupHash gethcommon.Hash) (*common.PublicRollup, error) { + whereQuery := " WHERE hash=" + db.GetSQLStatement().Placeholder + return fetchPublicRollup(db.GetSQLDB(), whereQuery, truncTo16(rollupHash)) +} + +func GetRollupBySeqNo(db HostDB, seqNo uint64) (*common.PublicRollup, error) { + whereQuery := " WHERE " + db.GetSQLStatement().Placeholder + " BETWEEN start_seq AND end_seq" + return fetchPublicRollup(db.GetSQLDB(), whereQuery, seqNo) +} + +func GetRollupBatches(db HostDB, rollupHash gethcommon.Hash) (*common.BatchListingResponse, error) { + whereQuery := " WHERE r.hash=" + db.GetSQLStatement().Placeholder + orderQuery := " ORDER BY b.height DESC" + query := selectRollupBatches + whereQuery + orderQuery + rows, err := db.GetSQLDB().Query(query, truncTo16(rollupHash)) + if err != nil { + return nil, fmt.Errorf("query execution for select rollup batches failed: %w", err) + } + defer rows.Close() + + var batches []common.PublicBatch + for rows.Next() { + var ( + sequenceInt64 int + hash []byte + fullHash gethcommon.Hash + heightInt64 int + extBatch []byte + ) + err := rows.Scan(&sequenceInt64, &hash, &fullHash, &heightInt64, &extBatch) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, errutil.ErrNotFound + } + return nil, fmt.Errorf("failed to fetch rollup batches: %w", err) + } + var b common.ExtBatch + err = rlp.DecodeBytes(extBatch, &b) + if err != nil { + return nil, fmt.Errorf("could not decode ext batch. Cause: %w", err) + } + + batch := common.PublicBatch{ + SequencerOrderNo: new(big.Int).SetInt64(int64(sequenceInt64)), + Hash: bytesToHexString(hash), + FullHash: fullHash, + Height: new(big.Int).SetInt64(int64(heightInt64)), + TxCount: new(big.Int).SetInt64(int64(len(b.TxHashes))), + Header: b.Header, + EncryptedTxBlob: b.EncryptedTxBlob, + } + batches = append(batches, batch) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return &common.BatchListingResponse{ + BatchesData: batches, + Total: uint64(len(batches)), + }, nil +} + func fetchRollupHeader(db *sql.DB, whereQuery string, args ...any) (*common.RollupHeader, error) { rollup, err := fetchExtRollup(db, whereQuery, args...) if err != nil { @@ -159,3 +226,40 @@ func fetchHeadRollup(db *sql.DB) (*common.ExtRollup, error) { return &rollup, nil } + +func fetchPublicRollup(db *sql.DB, whereQuery string, args ...any) (*common.PublicRollup, error) { + query := selectPublicRollup + whereQuery + var rollup common.PublicRollup + var hash, extRollup, compressionblock []byte + var id, firstSeq, lastSeq, timestamp int + + err := db.QueryRow(query, args...).Scan( + &id, + &hash, + &firstSeq, + &lastSeq, + ×tamp, + &extRollup, + &compressionblock, + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, errutil.ErrNotFound + } + return nil, fmt.Errorf("failed to fetch rollup by hash: %w", err) + } + rollup.ID = big.NewInt(int64(id)) + rollup.Hash = bytesToHexString(hash) + rollup.FirstSeq = big.NewInt(int64(firstSeq)) + rollup.LastSeq = big.NewInt(int64(lastSeq)) + rollup.Timestamp = uint64(timestamp) + rollup.L1Hash = bytesToHexString(compressionblock) + + extRollupDecoded := new(common.ExtRollup) + if err := rlp.DecodeBytes(extRollup, extRollupDecoded); err != nil { + return nil, fmt.Errorf("could not decode rollup. Cause: %w", err) + } + + rollup.Header = extRollupDecoded.Header + return &rollup, nil +} diff --git a/go/host/storage/hostdb/rollup_test.go b/go/host/storage/hostdb/rollup_test.go index 1302966637..b72b3254b8 100644 --- a/go/host/storage/hostdb/rollup_test.go +++ b/go/host/storage/hostdb/rollup_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + gethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ten-protocol/go-ten/go/common" ) @@ -95,7 +97,7 @@ func TestGetLatestRollup(t *testing.T) { } dbtx.Write() - latestHeader, err := GetLatestRollup(db.GetSQLDB()) + latestHeader, err := GetLatestRollup(db) if err != nil { t.Errorf("could not get latest rollup. Cause: %s", err) } @@ -105,6 +107,55 @@ func TestGetLatestRollup(t *testing.T) { } } +func TestGetRollupBySeqNo(t *testing.T) { + db, err := createSQLiteDB(t) + if err != nil { + t.Fatalf("unable to initialise test db: %s", err) + } + + rollup1FirstSeq := int64(batchNumber - 10) + rollup1LastSeq := int64(batchNumber) + metadata1 := createRollupMetadata(rollup1FirstSeq) + rollup1 := createRollup(rollup1LastSeq) + block := common.L1Block{} + dbtx, _ := db.NewDBTransaction() + err = AddRollup(dbtx, db.GetSQLStatement(), &rollup1, &metadata1, &block) + if err != nil { + t.Errorf("could not store rollup. Cause: %s", err) + } + // Needed to increment the timestamp + time.Sleep(1 * time.Second) + + rollup2FirstSeq := int64(batchNumber + 1) // 778 + rollup2LastSeq := int64(batchNumber + 10) // 787 + metadata2 := createRollupMetadata(rollup2FirstSeq) + rollup2 := createRollup(rollup2LastSeq) + err = AddRollup(dbtx, db.GetSQLStatement(), &rollup2, &metadata2, &block) + if err != nil { + t.Errorf("could not store rollup 2. Cause: %s", err) + } + dbtx.Write() + + rollup, err := GetRollupBySeqNo(db, batchNumber+5) + if err != nil { + t.Errorf("could not get latest rollup. Cause: %s", err) + } + + // should fetch the second rollup + if rollup.LastSeq.Cmp(big.NewInt(int64(rollup2.Header.LastBatchSeqNo))) != 0 { + t.Errorf("rollup was not fetched correctly") + } + + rollup, err = GetRollupBySeqNo(db, batchNumber-5) + if err != nil { + t.Errorf("could not get latest rollup. Cause: %s", err) + } + // should fetch the first rollup + if rollup.LastSeq.Cmp(big.NewInt(int64(rollup1.Header.LastBatchSeqNo))) != 0 { + t.Errorf("rollup was not fetched correctly") + } +} + func TestGetRollupListing(t *testing.T) { db, err := createSQLiteDB(t) if err != nil { @@ -202,6 +253,133 @@ func TestGetRollupListing(t *testing.T) { } } +func TestGetRollupByHash(t *testing.T) { + db, err := createSQLiteDB(t) + if err != nil { + t.Fatalf("unable to initialise test db: %s", err) + } + + rollup1FirstSeq := int64(batchNumber - 10) + rollup1LastSeq := int64(batchNumber) + metadata1 := createRollupMetadata(rollup1FirstSeq) + rollup1 := createRollup(rollup1LastSeq) + block := common.L1Block{} + dbtx, _ := db.NewDBTransaction() + err = AddRollup(dbtx, db.GetSQLStatement(), &rollup1, &metadata1, &block) + if err != nil { + t.Errorf("could not store rollup. Cause: %s", err) + } + + rollup2FirstSeq := int64(batchNumber + 1) + rollup2LastSeq := int64(batchNumber + 10) + metadata2 := createRollupMetadata(rollup2FirstSeq) + rollup2 := createRollup(rollup2LastSeq) + err = AddRollup(dbtx, db.GetSQLStatement(), &rollup2, &metadata2, &block) + if err != nil { + t.Errorf("could not store rollup 2. Cause: %s", err) + } + dbtx.Write() + + publicRollup, err := GetRollupByHash(db, rollup2.Header.Hash()) + if err != nil { + t.Errorf("stored rollup but could not retrieve public rollup. Cause: %s", err) + } + + if publicRollup.FirstSeq.Cmp(big.NewInt(batchNumber+1)) != 0 { + t.Errorf("rollup was not stored correctly") + } + + if publicRollup.LastSeq.Cmp(big.NewInt(batchNumber+10)) != 0 { + t.Errorf("rollup was not stored correctly") + } +} + +func TestGetRollupBatches(t *testing.T) { + db, _ := createSQLiteDB(t) + txHashesOne := []common.L2TxHash{gethcommon.BytesToHash([]byte("magicStringOne")), gethcommon.BytesToHash([]byte("magicStringTwo"))} + batchOne := createBatch(batchNumber, txHashesOne) + dbtx, _ := db.NewDBTransaction() + err := AddBatch(dbtx, db.GetSQLStatement(), &batchOne) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + + txHashesTwo := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringThree")), gethcommon.BytesToHash([]byte("magicStringFour"))} + batchTwo := createBatch(batchNumber+1, txHashesTwo) + + err = AddBatch(dbtx, db.GetSQLStatement(), &batchTwo) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + + txHashesThree := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringFive")), gethcommon.BytesToHash([]byte("magicStringSix"))} + batchThree := createBatch(batchNumber+2, txHashesThree) + + err = AddBatch(dbtx, db.GetSQLStatement(), &batchThree) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + + txHashesFour := []gethcommon.Hash{gethcommon.BytesToHash([]byte("magicStringSeven")), gethcommon.BytesToHash([]byte("magicStringEight"))} + batchFour := createBatch(batchNumber+3, txHashesFour) + + err = AddBatch(dbtx, db.GetSQLStatement(), &batchFour) + if err != nil { + t.Errorf("could not store batch. Cause: %s", err) + } + + rollup1FirstSeq := int64(batchNumber) + rollup1LastSeq := int64(batchNumber + 1) + metadata1 := createRollupMetadata(rollup1FirstSeq) + rollup1 := createRollup(rollup1LastSeq) + block := common.L1Block{} + err = AddRollup(dbtx, db.GetSQLStatement(), &rollup1, &metadata1, &block) + if err != nil { + t.Errorf("could not store rollup. Cause: %s", err) + } + + rollup2FirstSeq := int64(batchNumber + 2) + rollup2LastSeq := int64(batchNumber + 3) + metadata2 := createRollupMetadata(rollup2FirstSeq) + rollup2 := createRollup(rollup2LastSeq) + err = AddRollup(dbtx, db.GetSQLStatement(), &rollup2, &metadata2, &block) + if err != nil { + t.Errorf("could not store rollup 2. Cause: %s", err) + } + dbtx.Write() + + // rollup one contains batches 1 & 2 + batchListing, err := GetRollupBatches(db, rollup1.Hash()) + if err != nil { + t.Errorf("could not get rollup batches. Cause: %s", err) + } + + // should be two elements + if big.NewInt(int64(batchListing.Total)).Cmp(big.NewInt(2)) != 0 { + t.Errorf("batch listing was not calculated correctly") + } + + // second element should be batch 1 as we're ordering by height descending + if batchListing.BatchesData[1].Header.SequencerOrderNo.Cmp(batchOne.SeqNo()) != 0 { + t.Errorf("batch listing was not returned correctly") + } + + // rollup one contains batches 3 & 4 + batchListing1, err := GetRollupBatches(db, rollup2.Hash()) + if err != nil { + t.Errorf("could not get rollup batches. Cause: %s", err) + } + + // should be two elements + if big.NewInt(int64(batchListing1.Total)).Cmp(big.NewInt(2)) != 0 { + t.Errorf("batch listing was not calculated correctly") + } + // second element should be batch 3 as we're ordering by height descending + if batchListing1.BatchesData[1].Header.SequencerOrderNo.Cmp(batchThree.SeqNo()) != 0 { + t.Errorf("batch listing was not returned correctly") + } +} + func createRollup(lastBatch int64) common.ExtRollup { header := common.RollupHeader{ LastBatchSeqNo: uint64(lastBatch), diff --git a/go/host/storage/hostdb/sql_statements.go b/go/host/storage/hostdb/sql_statements.go index bf3d80c9f0..cabb5a2553 100644 --- a/go/host/storage/hostdb/sql_statements.go +++ b/go/host/storage/hostdb/sql_statements.go @@ -2,20 +2,21 @@ package hostdb // SQLStatements struct holds SQL statements for a specific database type type SQLStatements struct { - InsertBatch string - InsertTransactions string - InsertTxCount string - InsertRollup string - InsertBlock string - SelectRollups string - SelectBlocks string - Placeholder string + InsertBatch string + InsertTransactions string + InsertTxCount string + InsertRollup string + InsertBlock string + SelectRollups string + SelectBlocks string + SelectRollupBatches string + Placeholder string } func SQLiteSQLStatements() *SQLStatements { return &SQLStatements{ InsertBatch: "INSERT INTO batch_host (sequence, full_hash, hash, height, ext_batch) VALUES (?, ?, ?, ?, ?)", - InsertTransactions: "REPLACE INTO transactions_host (hash, b_sequence) VALUES (?, ?)", + InsertTransactions: "REPLACE INTO transactions_host (hash, full_hash, b_sequence) VALUES (?, ?, ?)", InsertTxCount: "INSERT INTO transaction_count (id, total) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET total = EXCLUDED.total", InsertRollup: "INSERT INTO rollup_host (hash, start_seq, end_seq, time_stamp, ext_rollup, compression_block) values (?,?,?,?,?,?)", InsertBlock: "REPLACE INTO block_host (hash, header, rollup_hash) values (?,?,?)", @@ -28,7 +29,7 @@ func SQLiteSQLStatements() *SQLStatements { func PostgresSQLStatements() *SQLStatements { return &SQLStatements{ InsertBatch: "INSERT INTO batch_host (sequence, full_hash, hash, height, ext_batch) VALUES ($1, $2, $3, $4, $5)", - InsertTransactions: "INSERT INTO transactions_host (hash, b_sequence) VALUES ($1, $2) ON CONFLICT (hash) DO NOTHING", + InsertTransactions: "INSERT INTO transactions_host (hash, full_hash, b_sequence) VALUES ($1, $2, $3) ON CONFLICT (hash) DO NOTHING", InsertTxCount: "INSERT INTO transaction_count (id, total) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET total = EXCLUDED.total", InsertRollup: "INSERT INTO rollup_host (hash, start_seq, end_seq, time_stamp, ext_rollup, compression_block) values ($1, $2, $3, $4, $5, $6)", InsertBlock: "INSERT INTO block_host (hash, header, rollup_hash) VALUES ($1, $2, $3) ON CONFLICT (hash) DO NOTHING", diff --git a/go/host/storage/hostdb/utils.go b/go/host/storage/hostdb/utils.go index f98140a274..c1e75150f0 100644 --- a/go/host/storage/hostdb/utils.go +++ b/go/host/storage/hostdb/utils.go @@ -1,6 +1,8 @@ package hostdb import ( + "encoding/hex" + "fmt" "testing" gethcommon "github.com/ethereum/go-ethereum/common" @@ -66,3 +68,7 @@ func createSQLiteDB(t *testing.T) (HostDB, error) { } return NewHostDB(hostDB, SQLiteSQLStatements()) } + +func bytesToHexString(bytes []byte) string { + return fmt.Sprintf("0x%s", hex.EncodeToString(bytes)) +} diff --git a/go/host/storage/init/postgres/001_init.sql b/go/host/storage/init/postgres/001_init.sql index a37ae04929..09468d0ec7 100644 --- a/go/host/storage/init/postgres/001_init.sql +++ b/go/host/storage/init/postgres/001_init.sql @@ -37,9 +37,10 @@ CREATE INDEX IF NOT EXISTS IDX_BATCH_HEIGHT_HOST ON batch_host (height); CREATE TABLE IF NOT EXISTS transactions_host ( hash BYTEA PRIMARY KEY, + full_hash BYTEA NOT NULL UNIQUE, b_sequence INT, FOREIGN KEY (b_sequence) REFERENCES batch_host(sequence) - ); +); CREATE TABLE IF NOT EXISTS transaction_count ( diff --git a/go/host/storage/init/postgres/postgres.go b/go/host/storage/init/postgres/postgres.go index 7b2601e850..a64c872d9a 100644 --- a/go/host/storage/init/postgres/postgres.go +++ b/go/host/storage/init/postgres/postgres.go @@ -13,8 +13,8 @@ import ( ) const ( - defaultDatabase = "postgres" - maxDBPoolSize = 100 + defaultDatabase = "postgres" + maxDBConnections = 100 ) func CreatePostgresDBConnection(baseURL string, dbName string) (*sql.DB, error) { @@ -47,7 +47,7 @@ func CreatePostgresDBConnection(baseURL string, dbName string) (*sql.DB, error) dbURL = fmt.Sprintf("%s%s", baseURL, dbName) db, err = sql.Open("postgres", dbURL) - db.SetMaxOpenConns(maxDBPoolSize) + db.SetMaxOpenConns(maxDBConnections) if err != nil { return nil, fmt.Errorf("failed to connect to PostgreSQL database %s: %v", dbName, err) } diff --git a/go/host/storage/init/sqlite/host_sqlite_init.sql b/go/host/storage/init/sqlite/001_init.sql similarity index 88% rename from go/host/storage/init/sqlite/host_sqlite_init.sql rename to go/host/storage/init/sqlite/001_init.sql index 1a31ca8d53..6fb3352860 100644 --- a/go/host/storage/init/sqlite/host_sqlite_init.sql +++ b/go/host/storage/init/sqlite/001_init.sql @@ -27,7 +27,7 @@ create table if not exists batch_host ( sequence int primary key, full_hash binary(32) NOT NULL, - hash binary(16) NOT NULL unique, + hash binary(16) NOT NULL UNIQUE, height int NOT NULL, ext_batch mediumblob NOT NULL ); @@ -35,13 +35,14 @@ create index IDX_BATCH_HEIGHT_HOST on batch_host (height); create table if not exists transactions_host ( - hash binary(32) primary key, + hash binary(16) PRIMARY KEY, + full_hash binary(32) NOT NULL UNIQUE, b_sequence int REFERENCES batch_host ); create table if not exists transaction_count ( - id int NOT NULL primary key, + id int NOT NULL PRIMARY KEY, total int NOT NULL ); diff --git a/go/host/storage/init/sqlite/sqlite.go b/go/host/storage/init/sqlite/sqlite.go index 2aabe7f401..bc6903794b 100644 --- a/go/host/storage/init/sqlite/sqlite.go +++ b/go/host/storage/init/sqlite/sqlite.go @@ -14,7 +14,7 @@ import ( const ( tempDirName = "ten-persistence" - initFile = "host_sqlite_init.sql" + initFile = "001_init.sql" ) //go:embed *.sql diff --git a/go/host/storage/interfaces.go b/go/host/storage/interfaces.go index d40845d947..6e0feef121 100644 --- a/go/host/storage/interfaces.go +++ b/go/host/storage/interfaces.go @@ -40,8 +40,14 @@ type BatchResolver interface { FetchBatchListingDeprecated(pagination *common.QueryPagination) (*common.BatchListingResponseDeprecated, error) // FetchBatchHeaderByHeight returns the `BatchHeader` with the given height FetchBatchHeaderByHeight(height *big.Int) (*common.BatchHeader, error) + // FetchBatchByHeight returns the `PublicBatch` with the given height + FetchBatchByHeight(height *big.Int) (*common.PublicBatch, error) // FetchTotalTxCount returns the number of transactions in the DB FetchTotalTxCount() (*big.Int, error) + // FetchTransaction returns the transaction given its hash + FetchTransaction(hash gethcommon.Hash) (*common.PublicTransaction, error) + // FetchBatchTransactions returns a list of public transaction data within a given batch hash + FetchBatchTransactions(batchHash gethcommon.Hash) (*common.TransactionListingResponse, error) } type BlockResolver interface { @@ -55,4 +61,10 @@ type BlockResolver interface { FetchRollupListing(pagination *common.QueryPagination) (*common.RollupListingResponse, error) // FetchBlockListing returns a paginated list of blocks that include rollups FetchBlockListing(pagination *common.QueryPagination) (*common.BlockListingResponse, error) + // FetchRollupByHash returns the public rollup data given its hash + FetchRollupByHash(rollupHash gethcommon.Hash) (*common.PublicRollup, error) + // FetchRollupBySeqNo returns the public rollup given a seq number + FetchRollupBySeqNo(seqNo uint64) (*common.PublicRollup, error) + // FetchRollupBatches returns a list of public batch data within a given rollup hash + FetchRollupBatches(rollupHash gethcommon.Hash) (*common.BatchListingResponse, error) } diff --git a/go/host/storage/storage.go b/go/host/storage/storage.go index a3a7cba39f..5167f0bedd 100644 --- a/go/host/storage/storage.go +++ b/go/host/storage/storage.go @@ -103,7 +103,7 @@ func (s *storageImpl) FetchBatchHeaderByHash(hash gethcommon.Hash) (*common.Batc } func (s *storageImpl) FetchHeadBatchHeader() (*common.BatchHeader, error) { - return hostdb.GetHeadBatchHeader(s.db.GetSQLDB()) + return hostdb.GetHeadBatchHeader(s.db) } func (s *storageImpl) FetchPublicBatchByHash(batchHash common.L2BatchHash) (*common.PublicBatch, error) { @@ -119,10 +119,14 @@ func (s *storageImpl) FetchBatchByTx(txHash gethcommon.Hash) (*common.ExtBatch, } func (s *storageImpl) FetchLatestBatch() (*common.BatchHeader, error) { - return hostdb.GetLatestBatch(s.db.GetSQLDB()) + return hostdb.GetLatestBatch(s.db) } func (s *storageImpl) FetchBatchHeaderByHeight(height *big.Int) (*common.BatchHeader, error) { + return hostdb.GetBatchHeaderByHeight(s.db, height) +} + +func (s *storageImpl) FetchBatchByHeight(height *big.Int) (*common.PublicBatch, error) { return hostdb.GetBatchByHeight(s.db, height) } @@ -135,7 +139,7 @@ func (s *storageImpl) FetchBatchListingDeprecated(pagination *common.QueryPagina } func (s *storageImpl) FetchLatestRollupHeader() (*common.RollupHeader, error) { - return hostdb.GetLatestRollup(s.db.GetSQLDB()) + return hostdb.GetLatestRollup(s.db) } func (s *storageImpl) FetchRollupListing(pagination *common.QueryPagination) (*common.RollupListingResponse, error) { @@ -147,7 +151,27 @@ func (s *storageImpl) FetchBlockListing(pagination *common.QueryPagination) (*co } func (s *storageImpl) FetchTotalTxCount() (*big.Int, error) { - return hostdb.GetTotalTxCount(s.db.GetSQLDB()) + return hostdb.GetTotalTxCount(s.db) +} + +func (s *storageImpl) FetchTransaction(hash gethcommon.Hash) (*common.PublicTransaction, error) { + return hostdb.GetTransaction(s.db, hash) +} + +func (s *storageImpl) FetchRollupByHash(rollupHash gethcommon.Hash) (*common.PublicRollup, error) { + return hostdb.GetRollupByHash(s.db, rollupHash) +} + +func (s *storageImpl) FetchRollupBySeqNo(seqNo uint64) (*common.PublicRollup, error) { + return hostdb.GetRollupBySeqNo(s.db, seqNo) +} + +func (s *storageImpl) FetchRollupBatches(rollupHash gethcommon.Hash) (*common.BatchListingResponse, error) { + return hostdb.GetRollupBatches(s.db, rollupHash) +} + +func (s *storageImpl) FetchBatchTransactions(batchHash gethcommon.Hash) (*common.TransactionListingResponse, error) { + return hostdb.GetBatchTransactions(s.db, batchHash) } func (s *storageImpl) Close() error { diff --git a/go/obsclient/obsclient.go b/go/obsclient/obsclient.go index 30a68f47b4..29bed17426 100644 --- a/go/obsclient/obsclient.go +++ b/go/obsclient/obsclient.go @@ -56,8 +56,8 @@ func (oc *ObsClient) BatchNumber() (uint64, error) { return uint64(result), err } -// BatchByHash returns the batch with the given hash. -func (oc *ObsClient) BatchByHash(hash gethcommon.Hash) (*common.ExtBatch, error) { +// GetBatchByHash returns the batch with the given hash. +func (oc *ObsClient) GetBatchByHash(hash gethcommon.Hash) (*common.ExtBatch, error) { var batch *common.ExtBatch err := oc.rpcClient.Call(&batch, rpc.GetBatch, hash) if err == nil && batch == nil { @@ -66,8 +66,30 @@ func (oc *ObsClient) BatchByHash(hash gethcommon.Hash) (*common.ExtBatch, error) return batch, err } -// BatchHeaderByNumber returns the header of the rollup with the given number -func (oc *ObsClient) BatchHeaderByNumber(number *big.Int) (*common.BatchHeader, error) { +// GetBatchByHeight returns the batch with the given height. +func (oc *ObsClient) GetBatchByHeight(height *big.Int) (*common.PublicBatch, error) { + var batch *common.PublicBatch + + err := oc.rpcClient.Call(&batch, rpc.GetBatchByHeight, height) + if err == nil && batch == nil { + err = ethereum.NotFound + } + return batch, err +} + +// GetRollupBySeqNo returns the batch with the given height. +func (oc *ObsClient) GetRollupBySeqNo(seqNo uint64) (*common.PublicRollup, error) { + var rollup *common.PublicRollup + + err := oc.rpcClient.Call(&rollup, rpc.GetRollupBySeqNo, seqNo) + if err == nil && rollup == nil { + err = ethereum.NotFound + } + return rollup, err +} + +// GetBatchHeaderByNumber returns the header of the rollup with the given number +func (oc *ObsClient) GetBatchHeaderByNumber(number *big.Int) (*common.BatchHeader, error) { var batchHeader *common.BatchHeader err := oc.rpcClient.Call(&batchHeader, rpc.GetBatchByNumber, toBlockNumArg(number), false) if err == nil && batchHeader == nil { @@ -76,8 +98,8 @@ func (oc *ObsClient) BatchHeaderByNumber(number *big.Int) (*common.BatchHeader, return batchHeader, err } -// BatchHeaderByHash returns the block header with the given hash. -func (oc *ObsClient) BatchHeaderByHash(hash gethcommon.Hash) (*common.BatchHeader, error) { +// GetBatchHeaderByHash returns the block header with the given hash. +func (oc *ObsClient) GetBatchHeaderByHash(hash gethcommon.Hash) (*common.BatchHeader, error) { var batchHeader *common.BatchHeader err := oc.rpcClient.Call(&batchHeader, rpc.GetBatchByHash, hash, false) if err == nil && batchHeader == nil { @@ -86,6 +108,16 @@ func (oc *ObsClient) BatchHeaderByHash(hash gethcommon.Hash) (*common.BatchHeade return batchHeader, err } +// GetTransaction returns the transaction. +func (oc *ObsClient) GetTransaction(hash gethcommon.Hash) (*common.PublicTransaction, error) { + var tx *common.PublicTransaction + err := oc.rpcClient.Call(&tx, rpc.GetTransaction, hash) + if err == nil && tx == nil { + err = ethereum.NotFound + } + return tx, err +} + // Health returns the health of the node. func (oc *ObsClient) Health() (bool, error) { var healthy *hostcommon.HealthCheck @@ -189,6 +221,36 @@ func (oc *ObsClient) GetRollupListing(pagination *common.QueryPagination) (*comm return &result, nil } +// GetRollupByHash returns the public rollup data given its hash +func (oc *ObsClient) GetRollupByHash(hash gethcommon.Hash) (*common.PublicRollup, error) { + var rollup *common.PublicRollup + err := oc.rpcClient.Call(&rollup, rpc.GetRollupByHash, hash) + if err == nil && rollup == nil { + err = ethereum.NotFound + } + return rollup, err +} + +// GetRollupBatches returns a list of public batch data within a given rollup hash +func (oc *ObsClient) GetRollupBatches(hash gethcommon.Hash) (*common.BatchListingResponse, error) { + var batchListing *common.BatchListingResponse + err := oc.rpcClient.Call(&batchListing, rpc.GetRollupBatches, hash) + if err == nil && batchListing == nil { + err = ethereum.NotFound + } + return batchListing, err +} + +// GetBatchTransactions returns a list of public transaction data within a given batch has +func (oc *ObsClient) GetBatchTransactions(hash gethcommon.Hash) (*common.TransactionListingResponse, error) { + var txListing *common.TransactionListingResponse + err := oc.rpcClient.Call(&txListing, rpc.GetBatchTransactions, hash) + if err == nil && txListing == nil { + err = ethereum.NotFound + } + return txListing, err +} + // GetConfig returns the network config for obscuro func (oc *ObsClient) GetConfig() (*common.ObscuroNetworkInfo, error) { var result common.ObscuroNetworkInfo diff --git a/go/rpc/client.go b/go/rpc/client.go index d1e54964ce..89d404b0df 100644 --- a/go/rpc/client.go +++ b/go/rpc/client.go @@ -37,13 +37,18 @@ const ( GetTotalContractCount = "scan_getTotalContractCount" GetPublicTransactionData = "scan_getPublicTransactionData" GetBatchListing = "scan_getBatchListing" - GetBatchListingNew = "scan_getBatchListingNew" GetBlockListing = "scan_getBlockListing" - GetRollupListing = "scan_getRollupListing" GetBatch = "scan_getBatch" GetLatestBatch = "scan_getLatestBatch" - GetPublicBatchByHash = "scan_getPublicBatchByHash" GetBatchByHeight = "scan_getBatchByHeight" + GetTransaction = "scan_getTransaction" + + GetRollupListing = "scan_getRollupListing" + GetBatchListingNew = "scan_getBatchListingNew" + GetRollupByHash = "scan_getRollupByHash" + GetRollupBatches = "scan_getRollupBatches" + GetRollupBySeqNo = "scan_getRollupBySeqNo" + GetBatchTransactions = "scan_getBatchTransactions" ) // Client is used by client applications to interact with the Ten node diff --git a/integration/manualtests/client_test.go b/integration/manualtests/client_test.go index 73d067c51f..09959a6c1d 100644 --- a/integration/manualtests/client_test.go +++ b/integration/manualtests/client_test.go @@ -22,7 +22,7 @@ func TestClientGetRollup(t *testing.T) { obsClient := obsclient.NewObsClient(client) - batchHeader, err := obsClient.BatchHeaderByNumber(big.NewInt(4392)) + batchHeader, err := obsClient.GetBatchHeaderByNumber(big.NewInt(4392)) assert.Nil(t, err) var rollup *common.ExtRollup diff --git a/integration/simulation/output_stats.go b/integration/simulation/output_stats.go index fb810592d6..cf7a75a72b 100644 --- a/integration/simulation/output_stats.go +++ b/integration/simulation/output_stats.go @@ -73,7 +73,7 @@ func (o *OutputStats) countBlockChain() { o.l2RollupCountInHeaders++ - header, err = obscuroClient.BatchHeaderByHash(header.ParentHash) + header, err = obscuroClient.GetBatchHeaderByHash(header.ParentHash) if err != nil { testlog.Logger().Crit("could not retrieve rollup by hash.", log.ErrKey, err) } diff --git a/integration/simulation/utils.go b/integration/simulation/utils.go index 419fb23ab7..7fe5ed6400 100644 --- a/integration/simulation/utils.go +++ b/integration/simulation/utils.go @@ -52,7 +52,7 @@ func getHeadBatchHeader(client *obsclient.ObsClient) (*common.BatchHeader, error return nil, fmt.Errorf("simulation failed due to failed attempt to retrieve head rollup height. Cause: %w", err) } - headBatchHeader, err := client.BatchHeaderByNumber(big.NewInt(int64(headBatchHeight))) + headBatchHeader, err := client.GetBatchHeaderByNumber(big.NewInt(int64(headBatchHeight))) if err != nil { return nil, fmt.Errorf("simulation failed due to failed attempt to retrieve rollup with height %d. Cause: %w", headBatchHeight, err) } diff --git a/integration/simulation/validate_chain.go b/integration/simulation/validate_chain.go index cd406edb8a..89d60f57fb 100644 --- a/integration/simulation/validate_chain.go +++ b/integration/simulation/validate_chain.go @@ -430,7 +430,7 @@ func checkBlockchainOfObscuroNode(t *testing.T, rpcHandles *network.RPCHandles, return } // check that the headers are serialised and deserialised correctly, by recomputing a header's hash - parentHeader, err := obscuroClient.BatchHeaderByHash(headBatchHeader.ParentHash) + parentHeader, err := obscuroClient.GetBatchHeaderByHash(headBatchHeader.ParentHash) if err != nil { t.Errorf("could not retrieve parent of head batch") return @@ -443,7 +443,7 @@ func checkBlockchainOfObscuroNode(t *testing.T, rpcHandles *network.RPCHandles, func getLoggedWithdrawals(minObscuroHeight uint64, obscuroClient *obsclient.ObsClient, currentHeader *common.BatchHeader) *big.Int { totalAmountLogged := big.NewInt(0) for i := minObscuroHeight; i < currentHeader.Number.Uint64(); i++ { - header, err := obscuroClient.BatchHeaderByNumber(big.NewInt(int64(i))) + header, err := obscuroClient.GetBatchHeaderByNumber(big.NewInt(int64(i))) if err != nil { panic(err) } @@ -565,7 +565,7 @@ func extractWithdrawals(t *testing.T, obscuroClient *obsclient.ObsClient, nodeId } // note this retrieves batches currently. - newHeader, err := obscuroClient.BatchHeaderByHash(header.ParentHash) + newHeader, err := obscuroClient.GetBatchHeaderByHash(header.ParentHash) if err != nil { t.Errorf(fmt.Sprintf("Node %d: Could not retrieve batch header %s. Cause: %s", nodeIdx, header.ParentHash, err)) return diff --git a/integration/tenscan/tenscan_test.go b/integration/tenscan/tenscan_test.go index 21d0d6f3e9..3a75a19522 100644 --- a/integration/tenscan/tenscan_test.go +++ b/integration/tenscan/tenscan_test.go @@ -142,7 +142,7 @@ func TestTenscan(t *testing.T) { // check "hash" field is included in json response assert.Contains(t, string(body), "\"hash\"") - statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/new/batches/?offset=0&size=10", serverAddress)) + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/v2/batches/?offset=0&size=10", serverAddress)) assert.NoError(t, err) assert.Equal(t, 200, statusCode) @@ -160,6 +160,7 @@ func TestTenscan(t *testing.T) { // check "hash" field is included in json response assert.Contains(t, string(body), "\"hash\"") + // fetch block listing statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/blocks/?offset=0&size=10", serverAddress)) assert.NoError(t, err) assert.Equal(t, 200, statusCode) @@ -171,9 +172,8 @@ func TestTenscan(t *testing.T) { blocklistingObj := blockListing{} err = json.Unmarshal(body, &blocklistingObj) assert.NoError(t, err) - // assert.LessOrEqual(t, 9, len(blocklistingObj.Result.BlocksData)) - // assert.LessOrEqual(t, uint64(9), blocklistingObj.Result.Total) + // fetch batch by hash statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/batch/%s", serverAddress, batchlistingObj.Result.BatchesData[0].Header.Hash())) assert.NoError(t, err) assert.Equal(t, 200, statusCode) @@ -187,6 +187,75 @@ func TestTenscan(t *testing.T) { assert.NoError(t, err) assert.Equal(t, batchlistingObj.Result.BatchesData[0].Header.Hash(), batchObj.Item.Header.Hash()) + // fetch rollup listing + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/rollups/?offset=0&size=10", serverAddress)) + assert.NoError(t, err) + assert.Equal(t, 200, statusCode) + + type rollupListing struct { + Result common.RollupListingResponse `json:"result"` + } + + rollupListingObj := rollupListing{} + err = json.Unmarshal(body, &rollupListingObj) + assert.NoError(t, err) + assert.LessOrEqual(t, 4, len(rollupListingObj.Result.RollupsData)) + assert.LessOrEqual(t, uint64(4), rollupListingObj.Result.Total) + assert.Contains(t, string(body), "\"hash\"") + + // fetch batches in rollup + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/rollup/%s/batches", serverAddress, rollupListingObj.Result.RollupsData[0].Header.Hash())) + assert.NoError(t, err) + assert.Equal(t, 200, statusCode) + + err = json.Unmarshal(body, &batchlistingObj) + assert.NoError(t, err) + assert.True(t, batchlistingObj.Result.Total > 0) + + // fetch transaction listing + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/transactions/?offset=0&size=10", serverAddress)) + assert.NoError(t, err) + assert.Equal(t, 200, statusCode) + + type txListing struct { + Result common.TransactionListingResponse `json:"result"` + } + + txListingObj := txListing{} + err = json.Unmarshal(body, &txListingObj) + assert.NoError(t, err) + assert.LessOrEqual(t, 5, len(txListingObj.Result.TransactionsData)) + assert.LessOrEqual(t, uint64(5), txListingObj.Result.Total) + + // fetch batch by height from tx + batchHeight := txListingObj.Result.TransactionsData[0].BatchHeight + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/batch/height/%s", serverAddress, batchHeight)) + assert.NoError(t, err) + assert.Equal(t, 200, statusCode) + + type publicBatchFetch struct { + Item *common.PublicBatch `json:"item"` + } + + publicBatchObj := publicBatchFetch{} + err = json.Unmarshal(body, &publicBatchObj) + assert.NoError(t, err) + assert.True(t, publicBatchObj.Item.Height.Cmp(batchHeight) == 0) + + // fetch tx by hash + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/items/transaction/%s", serverAddress, txListingObj.Result.TransactionsData[0].TransactionHash)) + assert.NoError(t, err) + assert.Equal(t, 200, statusCode) + + type txFetch struct { + Item *common.PublicTransaction `json:"item"` + } + + txObj := txFetch{} + err = json.Unmarshal(body, &txObj) + assert.NoError(t, err) + assert.True(t, txObj.Item.Finality == common.BatchFinal) + statusCode, body, err = fasthttp.Get(nil, fmt.Sprintf("%s/info/obscuro/", serverAddress)) assert.NoError(t, err) assert.Equal(t, 200, statusCode) diff --git a/tools/tenscan/backend/obscuroscan_backend.go b/tools/tenscan/backend/obscuroscan_backend.go index 5f4d9bd127..e823cb2ce8 100644 --- a/tools/tenscan/backend/obscuroscan_backend.go +++ b/tools/tenscan/backend/obscuroscan_backend.go @@ -5,6 +5,7 @@ import ( "crypto/cipher" "encoding/base64" "fmt" + "math/big" "github.com/ethereum/go-ethereum/rlp" "github.com/ten-protocol/go-ten/go/common/compression" @@ -56,15 +57,23 @@ func (b *Backend) GetLatestRollupHeader() (*common.RollupHeader, error) { } func (b *Backend) GetBatchByHash(hash gethcommon.Hash) (*common.ExtBatch, error) { - return b.obsClient.BatchByHash(hash) + return b.obsClient.GetBatchByHash(hash) +} + +func (b *Backend) GetBatchByHeight(height *big.Int) (*common.PublicBatch, error) { + return b.obsClient.GetBatchByHeight(height) +} + +func (b *Backend) GetRollupBySeqNo(seqNo uint64) (*common.PublicRollup, error) { + return b.obsClient.GetRollupBySeqNo(seqNo) } func (b *Backend) GetBatchHeader(hash gethcommon.Hash) (*common.BatchHeader, error) { - return b.obsClient.BatchHeaderByHash(hash) + return b.obsClient.GetBatchHeaderByHash(hash) } -func (b *Backend) GetTransaction(_ gethcommon.Hash) (*common.L2Tx, error) { - return nil, fmt.Errorf("unable to get encrypted Tx") +func (b *Backend) GetTransaction(hash gethcommon.Hash) (*common.PublicTransaction, error) { + return b.obsClient.GetTransaction(hash) } func (b *Backend) GetPublicTransactions(offset uint64, size uint64) (*common.TransactionListingResponse, error) { @@ -102,6 +111,18 @@ func (b *Backend) GetRollupListing(offset uint64, size uint64) (*common.RollupLi }) } +func (b *Backend) GetRollupByHash(hash gethcommon.Hash) (*common.PublicRollup, error) { + return b.obsClient.GetRollupByHash(hash) +} + +func (b *Backend) GetRollupBatches(hash gethcommon.Hash) (*common.BatchListingResponse, error) { + return b.obsClient.GetRollupBatches(hash) +} + +func (b *Backend) GetBatchTransactions(hash gethcommon.Hash) (*common.TransactionListingResponse, error) { + return b.obsClient.GetBatchTransactions(hash) +} + func (b *Backend) DecryptTxBlob(payload string) ([]*common.L2Tx, error) { encryptedTxBytes, err := base64.StdEncoding.DecodeString(payload) if err != nil { diff --git a/tools/tenscan/backend/webserver/webserver_routes_items.go b/tools/tenscan/backend/webserver/webserver_routes_items.go index de58191bb3..b53878faef 100644 --- a/tools/tenscan/backend/webserver/webserver_routes_items.go +++ b/tools/tenscan/backend/webserver/webserver_routes_items.go @@ -2,6 +2,7 @@ package webserver import ( "fmt" + "math/big" "net/http" "strconv" @@ -13,13 +14,20 @@ func routeItems(r *gin.Engine, server *WebServer) { r.GET("/items/batch/latest/", server.getLatestBatch) r.GET("/items/batch/:hash", server.getBatch) r.GET("/items/rollup/latest/", server.getLatestRollupHeader) - r.GET("/items/rollups/", server.getRollupListing) // New r.GET("/items/batches/", server.getBatchListingDeprecated) - r.GET("/items/new/batches/", server.getBatchListingNew) r.GET("/items/blocks/", server.getBlockListing) // Deprecated r.GET("/items/transactions/", server.getPublicTransactions) r.GET("/info/obscuro/", server.getConfig) r.POST("/info/health/", server.getHealthStatus) + + r.GET("/items/rollups/", server.getRollupListing) // New + r.GET("/items/v2/batches/", server.getBatchListingNew) + r.GET("/items/rollup/:hash", server.getRollup) + r.GET("/items/rollup/:hash/batches", server.getRollupBatches) + r.GET("/items/batch/:hash/transactions", server.getBatchTransactions) + r.GET("/items/batch/height/:height", server.getBatchByHeight) + r.GET("/items/rollup/batch/:seq", server.getRollupBySeq) + r.GET("/items/transaction/:hash", server.getTransaction) } func (w *WebServer) getHealthStatus(c *gin.Context) { @@ -61,6 +69,38 @@ func (w *WebServer) getBatch(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"item": batch}) } +func (w *WebServer) getBatchByHeight(c *gin.Context) { + heightStr := c.Param("height") + + heightBigInt := new(big.Int) + heightBigInt.SetString(heightStr, 10) + batch, err := w.backend.GetBatchByHeight(heightBigInt) + if err != nil { + errorHandler(c, fmt.Errorf("unable to execute request %w", err), w.logger) + return + } + + c.JSON(http.StatusOK, gin.H{"item": batch}) +} + +func (w *WebServer) getRollupBySeq(c *gin.Context) { + seqNo := c.Param("seq") + + seq, err := strconv.ParseUint(seqNo, 10, 64) + if err != nil { + errorHandler(c, fmt.Errorf("unable to parse sequence number: %w", err), w.logger) + return + } + + batch, err := w.backend.GetRollupBySeqNo(seq) + if err != nil { + errorHandler(c, fmt.Errorf("unable to execute request %w", err), w.logger) + return + } + + c.JSON(http.StatusOK, gin.H{"item": batch}) +} + func (w *WebServer) getBatchHeader(c *gin.Context) { hash := c.Param("hash") parsedHash := gethcommon.HexToHash(hash) @@ -210,6 +250,42 @@ func (w *WebServer) getBlockListing(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"result": batchesListing}) } +func (w *WebServer) getRollup(c *gin.Context) { + hash := c.Param("hash") + parsedHash := gethcommon.HexToHash(hash) + rollup, err := w.backend.GetRollupByHash(parsedHash) + if err != nil { + errorHandler(c, fmt.Errorf("unable to execute request %w", err), w.logger) + return + } + + c.JSON(http.StatusOK, gin.H{"item": rollup}) +} + +func (w *WebServer) getRollupBatches(c *gin.Context) { + hash := c.Param("hash") + parsedHash := gethcommon.HexToHash(hash) + batchListing, err := w.backend.GetRollupBatches(parsedHash) + if err != nil { + errorHandler(c, fmt.Errorf("unable to execute request %w", err), w.logger) + return + } + + c.JSON(http.StatusOK, gin.H{"result": batchListing}) +} + +func (w *WebServer) getBatchTransactions(c *gin.Context) { + hash := c.Param("hash") + parsedHash := gethcommon.HexToHash(hash) + txListing, err := w.backend.GetBatchTransactions(parsedHash) + if err != nil { + errorHandler(c, fmt.Errorf("unable to execute request %w", err), w.logger) + return + } + + c.JSON(http.StatusOK, gin.H{"result": txListing}) +} + func (w *WebServer) getConfig(c *gin.Context) { config, err := w.backend.GetConfig() if err != nil {