From 63f046b774158cbf1aac363bc1154dc8cbc78507 Mon Sep 17 00:00:00 2001 From: Oliver Bundalo Date: Mon, 26 Feb 2024 22:50:37 +0100 Subject: [PATCH] Added block hash into block key --- blockchain/blockchain.go | 18 +++---- blockchain/blockchain_test.go | 54 +++++-------------- .../storagev2/leveldb/leveldb_perf_test.go | 12 ++--- blockchain/storagev2/leveldb/leveldb_test.go | 4 +- blockchain/storagev2/mdbx/mdbx_perf_test.go | 12 ++--- blockchain/storagev2/mdbx/mdbx_test.go | 4 +- blockchain/storagev2/storage_read.go | 20 +++---- blockchain/storagev2/storage_write.go | 22 +++++--- blockchain/storagev2/testing.go | 21 ++++---- command/regenesis/test_on_history.go | 2 +- 10 files changed, 74 insertions(+), 95 deletions(-) diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index ceee2c023d..88349bcca0 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -445,7 +445,7 @@ func (b *Blockchain) GetReceiptsByHash(hash types.Hash) ([]*types.Receipt, error return nil, err } - return b.db.ReadReceipts(n) + return b.db.ReadReceipts(n, hash) } // GetBodyByHash returns the body by their hash @@ -478,7 +478,7 @@ func (b *Blockchain) readHeader(hash types.Hash) (*types.Header, bool) { return nil, false } - hh, err := b.db.ReadHeader(n) + hh, err := b.db.ReadHeader(n, hash) if err != nil { return nil, false } @@ -497,7 +497,7 @@ func (b *Blockchain) readBody(hash types.Hash) (*types.Body, bool) { return nil, false } - bb, err := b.db.ReadBody(n) + bb, err := b.db.ReadBody(n, hash) if err != nil { b.logger.Error("failed to read body", "err", err) @@ -508,7 +508,7 @@ func (b *Blockchain) readBody(hash types.Hash) (*types.Body, bool) { if updated := b.recoverFromFieldsInTransactions(bb.Transactions); updated { batchWriter := b.db.NewWriter() - batchWriter.PutBody(n, bb) + batchWriter.PutBody(n, hash, bb) if err := batchWriter.WriteBatch(); err != nil { b.logger.Warn("failed to write body into storage", "hash", hash, "err", err) @@ -538,7 +538,7 @@ func (b *Blockchain) readTotalDifficulty(headerHash types.Hash) (*big.Int, bool) return nil, false } - dbDifficulty, ok := b.db.ReadTotalDifficulty(n) + dbDifficulty, ok := b.db.ReadTotalDifficulty(n, headerHash) if !ok { return nil, false } @@ -848,7 +848,7 @@ func (b *Blockchain) WriteFullBlock(fblock *types.FullBlock, source string) erro // write the receipts, do it only after the header has been written. // Otherwise, a client might ask for a header once the receipt is valid, // but before it is written into the storage - batchWriter.PutReceipts(block.Number(), fblock.Receipts) + batchWriter.PutReceipts(block.Number(), block.Hash(), fblock.Receipts) // update snapshot if err := b.consensus.ProcessHeaders([]*types.Header{header}); err != nil { @@ -919,7 +919,7 @@ func (b *Blockchain) WriteBlock(block *types.Block, source string) error { // write the receipts, do it only after the header has been written. // Otherwise, a client might ask for a header once the receipt is valid, // but before it is written into the storage - batchWriter.PutReceipts(block.Number(), blockReceipts) + batchWriter.PutReceipts(block.Number(), block.Hash(), blockReceipts) // update snapshot if err := b.consensus.ProcessHeaders([]*types.Header{header}); err != nil { @@ -1019,7 +1019,7 @@ func (b *Blockchain) writeBody(batchWriter *storagev2.Writer, block *types.Block } // Write the full body (txns + receipts) - batchWriter.PutBody(block.Number(), block.Body()) + batchWriter.PutBody(block.Number(), block.Hash(), block.Body()) // Write txn lookups (txHash -> block number) for _, txn := range block.Transactions { @@ -1211,7 +1211,7 @@ func (b *Blockchain) writeHeaderImpl( } batchWriter.PutHeader(header) - batchWriter.PutTotalDifficulty(header.Number, incomingTD) + batchWriter.PutTotalDifficulty(header.Number, header.Hash, incomingTD) batchWriter.PutForks(forks) batchWriter.PutBlockLookup(header.Hash, header.Number) diff --git a/blockchain/blockchain_test.go b/blockchain/blockchain_test.go index 0dfea125d9..b0a67828f7 100644 --- a/blockchain/blockchain_test.go +++ b/blockchain/blockchain_test.go @@ -1097,16 +1097,18 @@ func TestBlockchain_VerifyBlockParent(t *testing.T) { assert.ErrorIs(t, blockchain.verifyBlockParent(block), ErrParentNotFound) }) - t.Run("Parent hash mismatch", func(t *testing.T) { + t.Run("Invalid parent hash", func(t *testing.T) { t.Parallel() // Set up the storage callback storageCallback := func(storage *storagev2.Storage) { - h := emptyHeader + h := &types.Header{ + Hash: types.ZeroHash, + } w := storage.NewWriter() - w.PutBlockLookup(types.ZeroHash, h.Number) + w.PutBlockLookup(h.Hash, h.Number) w.PutHeader(h) err := w.WriteBatch() require.NoError(t, err) @@ -1128,38 +1130,6 @@ func TestBlockchain_VerifyBlockParent(t *testing.T) { assert.ErrorIs(t, blockchain.verifyBlockParent(block), ErrParentHashMismatch) }) - t.Run("Invalid block sequence", func(t *testing.T) { - t.Parallel() - - // Set up the storage callback - storageCallback := func(storage *storagev2.Storage) { - h := emptyHeader - - w := storage.NewWriter() - - w.PutBlockLookup(types.ZeroHash, h.Number) - w.PutHeader(h) - err := w.WriteBatch() - require.NoError(t, err) - } - - blockchain, err := NewMockBlockchain(map[TestCallbackType]interface{}{ - StorageCallback: storageCallback, - }) - if err != nil { - t.Fatalf("unable to instantiate new blockchain, %v", err) - } - - // Create a dummy block with a number much higher than the parent - block := &types.Block{ - Header: &types.Header{ - Number: 10, - }, - } - - assert.ErrorIs(t, blockchain.verifyBlockParent(block), ErrParentHashMismatch) - }) - t.Run("Invalid block sequence", func(t *testing.T) { t.Parallel() @@ -1205,7 +1175,7 @@ func TestBlockchain_VerifyBlockParent(t *testing.T) { w := storage.NewWriter() - w.PutBlockLookup(types.ZeroHash, h.Number) + w.PutBlockLookup(h.Hash, h.Number) w.PutHeader(h) err := w.WriteBatch() require.NoError(t, err) @@ -1664,7 +1634,7 @@ func TestBlockchain_WriteFullBlock(t *testing.T) { require.NoError(t, err) require.Equal(t, header.Number, n) - b, err := bc.db.ReadBody(header.Number) + b, err := bc.db.ReadBody(header.Number, header.Hash) require.NoError(t, err) require.NotNil(t, b) @@ -1672,7 +1642,7 @@ func TestBlockchain_WriteFullBlock(t *testing.T) { require.NoError(t, err) require.Equal(t, header.Number, l) - h, err := bc.db.ReadHeader(header.Number) + h, err := bc.db.ReadHeader(header.Number, header.Hash) require.NoError(t, err) require.NotNil(t, h) @@ -1684,11 +1654,11 @@ func TestBlockchain_WriteFullBlock(t *testing.T) { require.True(t, ok) require.Equal(t, header.Hash, ch) - td, ok := bc.db.ReadTotalDifficulty(header.Number) + td, ok := bc.db.ReadTotalDifficulty(header.Number, header.Hash) require.True(t, ok) require.NotNil(t, td) - r, err := bc.db.ReadReceipts(header.Number) + r, err := bc.db.ReadReceipts(header.Number, header.Hash) require.NoError(t, err) require.NotNil(t, r) } @@ -1810,9 +1780,9 @@ func blockWriter(tb testing.TB, numberOfBlocks uint64, blockTime, checkInterval } batchWriter.PutHeader(block.Block.Header) - batchWriter.PutBody(block.Block.Number(), block.Block.Body()) + batchWriter.PutBody(block.Block.Number(), block.Block.Hash(), block.Block.Body()) - batchWriter.PutReceipts(block.Block.Number(), receipts) + batchWriter.PutReceipts(block.Block.Number(), block.Block.Hash(), receipts) require.NoError(tb, blockchain.writeBatchAndUpdate(batchWriter, block.Block.Header, big.NewInt(0), false)) diff --git a/blockchain/storagev2/leveldb/leveldb_perf_test.go b/blockchain/storagev2/leveldb/leveldb_perf_test.go index b6d7de493e..9c5b641ca3 100644 --- a/blockchain/storagev2/leveldb/leveldb_perf_test.go +++ b/blockchain/storagev2/leveldb/leveldb_perf_test.go @@ -177,10 +177,10 @@ func prepareBatch(t *testing.T, s *storagev2.Storage, b *types.FullBlock) *stora } // Main DB sorted - batchWriter.PutBody(b.Block.Number(), b.Block.Body()) + batchWriter.PutBody(b.Block.Number(), b.Block.Hash(), b.Block.Body()) batchWriter.PutCanonicalHash(b.Block.Number(), b.Block.Hash()) batchWriter.PutHeader(b.Block.Header) - batchWriter.PutReceipts(b.Block.Number(), b.Receipts) + batchWriter.PutReceipts(b.Block.Number(), b.Block.Hash(), b.Receipts) return batchWriter } @@ -230,16 +230,16 @@ func TestReadBlockPerf(t *testing.T) { n := uint64(1 + rand.Intn(10000)) tn := time.Now().UTC() - _, err1 := s.ReadBody(n) h, ok := s.ReadCanonicalHash(n) - _, err3 := s.ReadHeader(n) - _, err4 := s.ReadReceipts(n) + _, err1 := s.ReadBody(n, h) + _, err3 := s.ReadHeader(n, h) + _, err4 := s.ReadReceipts(n, h) b, err5 := s.ReadBlockLookup(h) d := time.Since(tn) watchTime = watchTime + d.Milliseconds() - if err1 != nil || !ok || err3 != nil || err4 != nil || err5 != nil { + if !ok || err1 != nil || err3 != nil || err4 != nil || err5 != nil { t.Logf("\terror") } diff --git a/blockchain/storagev2/leveldb/leveldb_test.go b/blockchain/storagev2/leveldb/leveldb_test.go index 14d2ce1911..4992e9e90c 100644 --- a/blockchain/storagev2/leveldb/leveldb_test.go +++ b/blockchain/storagev2/leveldb/leveldb_test.go @@ -230,7 +230,7 @@ insertloop: case b := <-blockchain: batchWriter := s.NewWriter() - batchWriter.PutBody(b.Block.Number(), b.Block.Body()) + batchWriter.PutBody(b.Block.Number(), b.Block.Hash(), b.Block.Body()) for _, tx := range b.Block.Transactions { batchWriter.PutTxLookup(tx.Hash(), b.Block.Number()) @@ -239,7 +239,7 @@ insertloop: batchWriter.PutHeader(b.Block.Header) batchWriter.PutHeadNumber(uint64(i)) batchWriter.PutHeadHash(b.Block.Header.Hash) - batchWriter.PutReceipts(b.Block.Number(), b.Receipts) + batchWriter.PutReceipts(b.Block.Number(), b.Block.Hash(), b.Receipts) batchWriter.PutCanonicalHash(uint64(i), b.Block.Hash()) if err := batchWriter.WriteBatch(); err != nil { diff --git a/blockchain/storagev2/mdbx/mdbx_perf_test.go b/blockchain/storagev2/mdbx/mdbx_perf_test.go index 5f634690fb..a155211e18 100644 --- a/blockchain/storagev2/mdbx/mdbx_perf_test.go +++ b/blockchain/storagev2/mdbx/mdbx_perf_test.go @@ -152,10 +152,10 @@ func prepareBatch(t *testing.T, s *storagev2.Storage, b *types.FullBlock) *stora } // Main DB sorted - batchWriter.PutBody(b.Block.Number(), b.Block.Body()) + batchWriter.PutBody(b.Block.Number(), b.Block.Hash(), b.Block.Body()) batchWriter.PutCanonicalHash(b.Block.Number(), b.Block.Hash()) batchWriter.PutHeader(b.Block.Header) - batchWriter.PutReceipts(b.Block.Number(), b.Receipts) + batchWriter.PutReceipts(b.Block.Number(), b.Block.Hash(), b.Receipts) return batchWriter } @@ -205,16 +205,16 @@ func TestReadBlockPerf(t *testing.T) { n := uint64(1 + rand.Intn(10000)) tn := time.Now().UTC() - _, err1 := s.ReadBody(n) h, ok := s.ReadCanonicalHash(n) - _, err3 := s.ReadHeader(n) - _, err4 := s.ReadReceipts(n) + _, err1 := s.ReadBody(n, h) + _, err3 := s.ReadHeader(n, h) + _, err4 := s.ReadReceipts(n, h) b, err5 := s.ReadBlockLookup(h) d := time.Since(tn) watchTime = watchTime + d.Milliseconds() - if err1 != nil || !ok || err3 != nil || err4 != nil || err5 != nil { + if !ok || err1 != nil || err3 != nil || err4 != nil || err5 != nil { t.Logf("\terror") } diff --git a/blockchain/storagev2/mdbx/mdbx_test.go b/blockchain/storagev2/mdbx/mdbx_test.go index 4c481e3cb5..c62d91cb59 100644 --- a/blockchain/storagev2/mdbx/mdbx_test.go +++ b/blockchain/storagev2/mdbx/mdbx_test.go @@ -220,7 +220,7 @@ insertloop: case b := <-blockchain: batchWriter := s.NewWriter() - batchWriter.PutBody(b.Block.Number(), b.Block.Body()) + batchWriter.PutBody(b.Block.Number(), b.Block.Hash(), b.Block.Body()) for _, tx := range b.Block.Transactions { batchWriter.PutTxLookup(tx.Hash(), b.Block.Number()) @@ -229,7 +229,7 @@ insertloop: batchWriter.PutHeader(b.Block.Header) batchWriter.PutHeadNumber(uint64(i)) batchWriter.PutHeadHash(b.Block.Header.Hash) - batchWriter.PutReceipts(b.Block.Number(), b.Receipts) + batchWriter.PutReceipts(b.Block.Number(), b.Block.Hash(), b.Receipts) batchWriter.PutCanonicalHash(uint64(i), b.Block.Hash()) if err := batchWriter.WriteBatch(); err != nil { diff --git a/blockchain/storagev2/storage_read.go b/blockchain/storagev2/storage_read.go index 1ff2b01740..56103a6845 100644 --- a/blockchain/storagev2/storage_read.go +++ b/blockchain/storagev2/storage_read.go @@ -9,7 +9,7 @@ import ( // -- canonical hash -- -// ReadCanonicalHash gets the hash from the number of the canonical chain +// ReadCanonicalHash gets the hash from the number of the canonical block func (s *Storage) ReadCanonicalHash(n uint64) (types.Hash, bool) { data, ok := s.get(CANONICAL, common.EncodeUint64ToBytes(n)) if !ok { @@ -58,8 +58,8 @@ func (s *Storage) ReadForks() ([]types.Hash, error) { // DIFFICULTY // // ReadTotalDifficulty reads the difficulty -func (s *Storage) ReadTotalDifficulty(bn uint64) (*big.Int, bool) { - v, ok := s.get(DIFFICULTY, common.EncodeUint64ToBytes(bn)) +func (s *Storage) ReadTotalDifficulty(bn uint64, bh types.Hash) (*big.Int, bool) { + v, ok := s.get(DIFFICULTY, getKey(bn, bh)) if !ok { return nil, false } @@ -70,9 +70,9 @@ func (s *Storage) ReadTotalDifficulty(bn uint64) (*big.Int, bool) { // HEADER // // ReadHeader reads the header -func (s *Storage) ReadHeader(bn uint64) (*types.Header, error) { +func (s *Storage) ReadHeader(bn uint64, bh types.Hash) (*types.Header, error) { header := &types.Header{} - err := s.readRLP(HEADER, common.EncodeUint64ToBytes(bn), header) + err := s.readRLP(HEADER, getKey(bn, bh), header) return header, err } @@ -80,15 +80,15 @@ func (s *Storage) ReadHeader(bn uint64) (*types.Header, error) { // BODY // // ReadBody reads the body -func (s *Storage) ReadBody(bn uint64) (*types.Body, error) { +func (s *Storage) ReadBody(bn uint64, bh types.Hash) (*types.Body, error) { body := &types.Body{} - if err := s.readRLP(BODY, common.EncodeUint64ToBytes(bn), body); err != nil { + if err := s.readRLP(BODY, getKey(bn, bh), body); err != nil { return nil, err } // must read header because block number is needed in order to calculate each tx hash header := &types.Header{} - if err := s.readRLP(HEADER, common.EncodeUint64ToBytes(bn), header); err != nil { + if err := s.readRLP(HEADER, getKey(bn, bh), header); err != nil { return nil, err } @@ -102,9 +102,9 @@ func (s *Storage) ReadBody(bn uint64) (*types.Body, error) { // RECEIPTS // // ReadReceipts reads the receipts -func (s *Storage) ReadReceipts(bn uint64) ([]*types.Receipt, error) { +func (s *Storage) ReadReceipts(bn uint64, bh types.Hash) ([]*types.Receipt, error) { receipts := &types.Receipts{} - err := s.readRLP(RECEIPTS, common.EncodeUint64ToBytes(bn), receipts) + err := s.readRLP(RECEIPTS, getKey(bn, bh), receipts) return *receipts, err } diff --git a/blockchain/storagev2/storage_write.go b/blockchain/storagev2/storage_write.go index 36834abcab..e76c607a91 100644 --- a/blockchain/storagev2/storage_write.go +++ b/blockchain/storagev2/storage_write.go @@ -8,11 +8,13 @@ import ( ) func (w *Writer) PutHeader(h *types.Header) { - w.putRlp(HEADER, common.EncodeUint64ToBytes(h.Number), h) + // block_num_u64 + hash -> header (RLP) + w.putRlp(HEADER, getKey(h.Number, h.Hash), h) } -func (w *Writer) PutBody(bn uint64, body *types.Body) { - w.putRlp(BODY, common.EncodeUint64ToBytes(bn), body) +func (w *Writer) PutBody(bn uint64, bh types.Hash, body *types.Body) { + // block_num_u64 + hash -> body (RLP) + w.putRlp(BODY, getKey(bn, bh), body) } func (w *Writer) PutHeadHash(h types.Hash) { @@ -31,9 +33,9 @@ func (w *Writer) PutBlockLookup(hash types.Hash, bn uint64) { w.putIntoTable(BLOCK_LOOKUP, hash.Bytes(), common.EncodeUint64ToBytes(bn)) } -func (w *Writer) PutReceipts(bn uint64, receipts []*types.Receipt) { +func (w *Writer) PutReceipts(bn uint64, bh types.Hash, receipts []*types.Receipt) { rs := types.Receipts(receipts) - w.putRlp(RECEIPTS, common.EncodeUint64ToBytes(bn), &rs) + w.putRlp(RECEIPTS, getKey(bn, bh), &rs) } func (w *Writer) PutCanonicalHeader(h *types.Header, diff *big.Int) { @@ -42,15 +44,15 @@ func (w *Writer) PutCanonicalHeader(h *types.Header, diff *big.Int) { w.PutHeadNumber(h.Number) w.PutBlockLookup(h.Hash, h.Number) w.PutCanonicalHash(h.Number, h.Hash) - w.PutTotalDifficulty(h.Number, diff) + w.PutTotalDifficulty(h.Number, h.Hash, diff) } func (w *Writer) PutCanonicalHash(bn uint64, hash types.Hash) { w.putIntoTable(CANONICAL, common.EncodeUint64ToBytes(bn), hash.Bytes()) } -func (w *Writer) PutTotalDifficulty(bn uint64, diff *big.Int) { - w.putIntoTable(DIFFICULTY, common.EncodeUint64ToBytes(bn), diff.Bytes()) +func (w *Writer) PutTotalDifficulty(bn uint64, bh types.Hash, diff *big.Int) { + w.putIntoTable(DIFFICULTY, getKey(bn, bh), diff.Bytes()) } func (w *Writer) PutForks(forks []types.Hash) { @@ -97,3 +99,7 @@ func (w *Writer) getBatch(t uint8) Batch { return w.batch[MAINDB_INDEX] } + +func getKey(n uint64, h types.Hash) []byte { + return append(append(make([]byte, 0, 40), common.EncodeUint64ToBytes(n)...), h.Bytes()...) +} diff --git a/blockchain/storagev2/testing.go b/blockchain/storagev2/testing.go index d0acdbc216..30a725f11b 100644 --- a/blockchain/storagev2/testing.go +++ b/blockchain/storagev2/testing.go @@ -128,14 +128,15 @@ func testDifficulty(t *testing.T, m PlaceholderStorage) { Number: uint64(indx), ExtraData: []byte{}, } + h.ComputeHash() batch.PutHeader(h) batch.PutBlockLookup(h.Hash, h.Number) - batch.PutTotalDifficulty(h.Number, cc.Diff) + batch.PutTotalDifficulty(h.Number, h.Hash, cc.Diff) require.NoError(t, batch.WriteBatch()) - diff, ok := s.ReadTotalDifficulty(h.Number) + diff, ok := s.ReadTotalDifficulty(h.Number, h.Hash) assert.True(t, ok) if !reflect.DeepEqual(cc.Diff, diff) { @@ -232,7 +233,7 @@ func testHeader(t *testing.T, m PlaceholderStorage) { require.NoError(t, batch.WriteBatch()) - header1, err := s.ReadHeader(header.Number) + header1, err := s.ReadHeader(header.Number, header.Hash) assert.NoError(t, err) if !reflect.DeepEqual(header, header1) { @@ -254,6 +255,8 @@ func testBody(t *testing.T, m PlaceholderStorage) { ExtraData: []byte{}, // if not set it will fail } + header.ComputeHash() + batch := s.NewWriter() batch.PutHeader(header) @@ -292,11 +295,11 @@ func testBody(t *testing.T, m PlaceholderStorage) { batch2 := s.NewWriter() body0 := block.Body() - batch2.PutBody(header.Number, body0) + batch2.PutBody(header.Number, header.Hash, body0) require.NoError(t, batch2.WriteBatch()) - body1, err := s.ReadBody(header.Number) + body1, err := s.ReadBody(header.Number, header.Hash) assert.NoError(t, err) // NOTE: reflect.DeepEqual does not seem to work, check the hash of the transactions @@ -372,12 +375,12 @@ func testReceipts(t *testing.T, m PlaceholderStorage) { } batch.PutHeader(h) - batch.PutBody(h.Number, body) - batch.PutReceipts(h.Number, receipts) + batch.PutBody(h.Number, h.Hash, body) + batch.PutReceipts(h.Number, h.Hash, receipts) require.NoError(t, batch.WriteBatch()) - found, err := s.ReadReceipts(h.Number) + found, err := s.ReadReceipts(h.Number, h.Hash) assert.NoError(t, err) assert.True(t, reflect.DeepEqual(receipts, found)) } @@ -401,7 +404,7 @@ func testWriteCanonicalHeader(t *testing.T, m PlaceholderStorage) { require.NoError(t, batch.WriteBatch()) - hh, err := s.ReadHeader(h.Number) + hh, err := s.ReadHeader(h.Number, h.Hash) assert.NoError(t, err) if !reflect.DeepEqual(h, hh) { diff --git a/command/regenesis/test_on_history.go b/command/regenesis/test_on_history.go index 9fef4311eb..3dbf4c6f99 100644 --- a/command/regenesis/test_on_history.go +++ b/command/regenesis/test_on_history.go @@ -90,7 +90,7 @@ func HistoryTestCmd() *cobra.Command { return } - header, err := st.ReadHeader(bn) + header, err := st.ReadHeader(bn, canonicalHash) if !ok { outputter.SetError(fmt.Errorf("can't read header %w", err))