From 26f1514a2117ec548c55b7b33f2d1dc0cf7c568a Mon Sep 17 00:00:00 2001 From: tony Date: Tue, 26 Nov 2024 14:28:58 +0800 Subject: [PATCH 1/2] improve cache stats log and metric --- muxdb/cache.go | 61 +++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/muxdb/cache.go b/muxdb/cache.go index 0247818e0..ea73b47f7 100644 --- a/muxdb/cache.go +++ b/muxdb/cache.go @@ -58,17 +58,21 @@ func (c *cache) log() { last := c.lastLogTime.Swap(now) if now-last > int64(time.Second*20) { - logNode, hitNode, missNode, okNode := c.nodeStats.shouldLog("node cache stats") - logRoot, hitRoot, missRoot, okRoot := c.rootStats.shouldLog("root cache stats") - - if okNode || okRoot { - logNode() - metricCacheHitMissGaugeVec().SetWithLabel(hitNode, map[string]string{"type": "node", "event": "hit"}) - metricCacheHitMissGaugeVec().SetWithLabel(missNode, map[string]string{"type": "node", "event": "miss"}) - logRoot() - metricCacheHitMissGaugeVec().SetWithLabel(hitRoot, map[string]string{"type": "root", "event": "hit"}) - metricCacheHitMissGaugeVec().SetWithLabel(missRoot, map[string]string{"type": "root", "event": "miss"}) + shouldNode, hitNode, missNode := c.nodeStats.Stats() + shouldRoot, hitRoot, missRoot := c.rootStats.Stats() + + // log two categories together only one of the hit rate has + // changed compared to the last run, to avoid too many logs. + if shouldNode || shouldRoot { + logStats("node cache stats", hitNode, missNode) + logStats("root cache stats", hitRoot, missRoot) } + + // metrics will reported every 20 seconds + metricCacheHitMissGaugeVec().SetWithLabel(hitRoot, map[string]string{"type": "root", "event": "hit"}) + metricCacheHitMissGaugeVec().SetWithLabel(missRoot, map[string]string{"type": "root", "event": "miss"}) + metricCacheHitMissGaugeVec().SetWithLabel(hitNode, map[string]string{"type": "node", "event": "hit"}) + metricCacheHitMissGaugeVec().SetWithLabel(missNode, map[string]string{"type": "node", "event": "miss"}) } else { c.lastLogTime.CompareAndSwap(now, last) } @@ -178,28 +182,33 @@ type cacheStats struct { func (cs *cacheStats) Hit() int64 { return cs.hit.Add(1) } func (cs *cacheStats) Miss() int64 { return cs.miss.Add(1) } -func (cs *cacheStats) shouldLog(msg string) (func(), int64, int64, bool) { +func (cs *cacheStats) Stats() (bool, int64, int64) { hit := cs.hit.Load() miss := cs.miss.Load() lookups := hit + miss - hitrate := float64(hit) / float64(lookups) - flag := int32(hitrate * 1000) - return func() { - var str string - if lookups > 0 { - str = fmt.Sprintf("%.3f", hitrate) - } else { - str = "n/a" - } + hitRate := float64(0) + if lookups > 0 { + hitRate = float64(hit) / float64(lookups) + } + flag := int32(hitRate * 1000) + + return cs.flag.Swap(flag) != flag, hit, miss +} - logger.Info(msg, - "lookups", lookups, - "hitrate", str, - ) +func logStats(msg string, hit, miss int64) { + lookups := hit + miss + var str string + if lookups > 0 { + str = fmt.Sprintf("%.3f", float64(hit)/float64(lookups)) + } else { + str = "n/a" + } - cs.flag.Store(flag) - }, hit, miss, cs.flag.Load() != flag + logger.Info(msg, + "lookups", lookups, + "hitrate", str, + ) } type dummyCache struct{} From 0d55356f7230b4825c0048e0d1e2f5a63df431fe Mon Sep 17 00:00:00 2001 From: tony Date: Tue, 26 Nov 2024 15:18:39 +0800 Subject: [PATCH 2/2] totally removed SetBestBlockID --- bft/engine_test.go | 26 ++++++++++---------------- chain/block_reader_test.go | 11 +++++------ chain/chain_test.go | 5 ++--- chain/repository.go | 22 ---------------------- chain/repository_test.go | 17 ++++++++--------- 5 files changed, 25 insertions(+), 56 deletions(-) diff --git a/bft/engine_test.go b/bft/engine_test.go index 6fa736bed..d36494587 100644 --- a/bft/engine_test.go +++ b/bft/engine_test.go @@ -113,7 +113,7 @@ func (test *TestBFT) reCreateEngine() error { return nil } -func (test *TestBFT) newBlock(parentSummary *chain.BlockSummary, master genesis.DevAccount, shouldVote bool) (*chain.BlockSummary, error) { +func (test *TestBFT) newBlock(parentSummary *chain.BlockSummary, master genesis.DevAccount, shouldVote bool, asBest bool) (*chain.BlockSummary, error) { packer := packer.New(test.repo, test.stater, master.Address, &thor.Address{}, test.fc) flow, err := packer.Mock(parentSummary, parentSummary.Header.Timestamp()+thor.BlockInterval, parentSummary.Header.GasLimit()) if err != nil { @@ -134,7 +134,7 @@ func (test *TestBFT) newBlock(parentSummary *chain.BlockSummary, master genesis. return nil, err } - if err = test.repo.AddBlock(b, nil, conflicts, false); err != nil { + if err = test.repo.AddBlock(b, nil, conflicts, asBest); err != nil { return nil, err } @@ -155,13 +155,13 @@ func (test *TestBFT) fastForward(cnt int) error { acc := devAccounts[(int(parent.Header.Number())+1)%devCnt] var err error - parent, err = test.newBlock(parent, acc, true) + parent, err = test.newBlock(parent, acc, true, true) if err != nil { return err } } - return test.repo.SetBestBlockID(parent.Header.ID()) + return nil } func (test *TestBFT) fastForwardWithMinority(cnt int) error { @@ -172,13 +172,13 @@ func (test *TestBFT) fastForwardWithMinority(cnt int) error { acc := devAccounts[(int(parent.Header.Number())+1)%(devCnt/3)] var err error - parent, err = test.newBlock(parent, acc, true) + parent, err = test.newBlock(parent, acc, true, true) if err != nil { return err } } - return test.repo.SetBestBlockID(parent.Header.ID()) + return nil } func (test *TestBFT) buildBranch(cnt int) (*chain.Chain, error) { @@ -189,7 +189,7 @@ func (test *TestBFT) buildBranch(cnt int) (*chain.Chain, error) { acc := devAccounts[(int(parent.Header.Number())+1+4)%devCnt] var err error - parent, err = test.newBlock(parent, acc, true) + parent, err = test.newBlock(parent, acc, true, false) if err != nil { return nil, err } @@ -197,14 +197,14 @@ func (test *TestBFT) buildBranch(cnt int) (*chain.Chain, error) { return test.repo.NewChain(parent.Header.ID()), nil } -func (test *TestBFT) pack(parentID thor.Bytes32, shouldVote bool, best bool) (*chain.BlockSummary, error) { +func (test *TestBFT) pack(parentID thor.Bytes32, shouldVote bool, asBest bool) (*chain.BlockSummary, error) { acc := devAccounts[len(devAccounts)-1] parent, err := test.repo.GetBlockSummary(parentID) if err != nil { return nil, err } - blk, err := test.newBlock(parent, acc, shouldVote) + blk, err := test.newBlock(parent, acc, shouldVote, asBest) if err != nil { return nil, err } @@ -215,12 +215,6 @@ func (test *TestBFT) pack(parentID thor.Bytes32, shouldVote bool, best bool) (*c } } - if best { - if err := test.repo.SetBestBlockID(blk.Header.ID()); err != nil { - return nil, err - } - } - return test.repo.GetBlockSummary(blk.Header.ID()) } @@ -255,7 +249,7 @@ func TestNewBlock(t *testing.T) { PrivateKey: priv, } - summary, err := testBFT.newBlock(testBFT.repo.BestBlockSummary(), master, true) + summary, err := testBFT.newBlock(testBFT.repo.BestBlockSummary(), master, true, false) if err != nil { t.Fatal(err) } diff --git a/chain/block_reader_test.go b/chain/block_reader_test.go index 643b02478..804c91ce6 100644 --- a/chain/block_reader_test.go +++ b/chain/block_reader_test.go @@ -3,14 +3,13 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package chain_test +package chain import ( "testing" "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" ) func TestBlockReader(t *testing.T) { @@ -31,7 +30,7 @@ func TestBlockReader(t *testing.T) { br := repo.NewBlockReader(b2.Header().ID()) - var blks []*chain.ExtendedBlock + var blks []*ExtendedBlock for { r, err := br.Read() @@ -44,7 +43,7 @@ func TestBlockReader(t *testing.T) { blks = append(blks, r...) } - assert.Equal(t, []*chain.ExtendedBlock{ + assert.Equal(t, []*ExtendedBlock{ {block.Compose(b3.Header(), b3.Transactions()), false}, {block.Compose(b4.Header(), b4.Transactions()), false}}, blks) @@ -71,7 +70,7 @@ func TestBlockReaderFork(t *testing.T) { br := repo.NewBlockReader(b2x.Header().ID()) - var blks []*chain.ExtendedBlock + var blks []*ExtendedBlock for { r, err := br.Read() @@ -85,7 +84,7 @@ func TestBlockReaderFork(t *testing.T) { blks = append(blks, r...) } - assert.Equal(t, []*chain.ExtendedBlock{ + assert.Equal(t, []*ExtendedBlock{ {block.Compose(b2x.Header(), b2x.Transactions()), true}, {block.Compose(b2.Header(), b2.Transactions()), false}, {block.Compose(b3.Header(), b3.Transactions()), false}, diff --git a/chain/chain_test.go b/chain/chain_test.go index 98308a4e5..1b6a3f970 100644 --- a/chain/chain_test.go +++ b/chain/chain_test.go @@ -3,7 +3,7 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package chain_test +package chain import ( "testing" @@ -12,7 +12,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/test/datagen" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" @@ -31,7 +30,7 @@ func TestChain(t *testing.T) { _, repo := newTestRepo() b1 := newBlock(repo.GenesisBlock(), 10, tx1) - tx1Meta := &chain.TxMeta{BlockNum: 1, Index: 0, Reverted: false} + tx1Meta := &TxMeta{BlockNum: 1, Index: 0, Reverted: false} tx1Receipt := &tx.Receipt{} repo.AddBlock(b1, tx.Receipts{tx1Receipt}, 0, false) diff --git a/chain/repository.go b/chain/repository.go index 59b49f647..5fe131444 100644 --- a/chain/repository.go +++ b/chain/repository.go @@ -127,28 +127,6 @@ func (r *Repository) BestBlockSummary() *BlockSummary { return r.bestSummary.Load().(*BlockSummary) } -// SetBestBlockID set the given block id as best block id. -func (r *Repository) SetBestBlockID(id thor.Bytes32) (err error) { - defer func() { - if err == nil { - r.tick.Broadcast() - } - }() - summary, err := r.GetBlockSummary(id) - if err != nil { - return err - } - return r.setBestBlockSummary(summary) -} - -func (r *Repository) setBestBlockSummary(summary *BlockSummary) error { - if err := r.propStore.Put(bestBlockIDKey, summary.Header.ID().Bytes()); err != nil { - return err - } - r.bestSummary.Store(summary) - return nil -} - func (r *Repository) saveBlock(block *block.Block, receipts tx.Receipts, conflicts uint32, asBest bool) (*BlockSummary, error) { var ( header = block.Header() diff --git a/chain/repository_test.go b/chain/repository_test.go index b268909da..81bef17a5 100644 --- a/chain/repository_test.go +++ b/chain/repository_test.go @@ -3,7 +3,7 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package chain_test +package chain import ( "testing" @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" @@ -21,13 +20,13 @@ func M(args ...interface{}) []interface{} { return args } -func newTestRepo() (*muxdb.MuxDB, *chain.Repository) { +func newTestRepo() (*muxdb.MuxDB, *Repository) { db := muxdb.NewMem() b0 := new(block.Builder). ParentID(thor.Bytes32{0xff, 0xff, 0xff, 0xff}). Build() - repo, err := chain.NewRepository(db, b0) + repo, err := NewRepository(db, b0) if err != nil { panic(err) } @@ -49,11 +48,11 @@ func newBlock(parent *block.Block, ts uint64, txs ...*tx.Transaction) *block.Blo return b.WithSignature(sig) } -func TestRepository(t *testing.T) { +func TestRepositoryFunc(t *testing.T) { db, repo1 := newTestRepo() b0 := repo1.GenesisBlock() - repo1, err := chain.NewRepository(db, b0) + repo1, err := NewRepository(db, b0) if err != nil { panic(err) } @@ -69,11 +68,11 @@ func TestRepository(t *testing.T) { // best block not set, so still 0 assert.Equal(t, uint32(0), repo1.BestBlockSummary().Header.Number()) - repo1.SetBestBlockID(b1.Header().ID()) + assert.Nil(t, repo1.AddBlock(b1, tx.Receipts{receipt1}, 0, true)) assert.Equal(t, uint32(1), repo1.BestBlockSummary().Header.Number()) - repo2, _ := chain.NewRepository(db, b0) - for _, repo := range []*chain.Repository{repo1, repo2} { + repo2, _ := NewRepository(db, b0) + for _, repo := range []*Repository{repo1, repo2} { assert.Equal(t, b1.Header().ID(), repo.BestBlockSummary().Header.ID()) s, err := repo.GetBlockSummary(b1.Header().ID()) assert.Nil(t, err)