From 3b7079713e9ca5cf4a80fd29f5686e2de68e54d7 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 14:00:08 +0200
Subject: [PATCH 1/9] fix(rpc): validators endpoint fail during quorum rotation

---
 .../selectproposer/height_round_proposer.go   |  9 ++-
 internal/inspect/inspect.go                   |  2 +-
 internal/state/store.go                       | 77 ++++++++++++++-----
 node/node.go                                  |  2 +-
 4 files changed, 68 insertions(+), 22 deletions(-)

diff --git a/internal/consensus/versioned/selectproposer/height_round_proposer.go b/internal/consensus/versioned/selectproposer/height_round_proposer.go
index b8c2e0d1ac..fe3a0e6691 100644
--- a/internal/consensus/versioned/selectproposer/height_round_proposer.go
+++ b/internal/consensus/versioned/selectproposer/height_round_proposer.go
@@ -65,6 +65,8 @@ func NewHeightRoundProposerSelector(vset *types.ValidatorSet, currentHeight int6
 
 // proposerFromStore determines the proposer for the given height and round using current or previous block read from
 // the block store.
+//
+// Requires s.valSet to be set to correct validator set.
 func (s *heightRoundProposerSelector) proposerFromStore(height int64, round int32) error {
 	if s.bs == nil {
 		return fmt.Errorf("block store is nil")
@@ -85,12 +87,15 @@ func (s *heightRoundProposerSelector) proposerFromStore(height int64, round int3
 
 	meta := s.bs.LoadBlockMeta(height)
 	if meta != nil {
+		valSetHash := s.valSet.Hash()
 		// block already saved to store, just take the proposer
-		if !meta.Header.ValidatorsHash.Equal(s.valSet.Hash()) {
+		if !meta.Header.ValidatorsHash.Equal(valSetHash) {
 			// we loaded the same block, so quorum should be the same
 			s.logger.Error("quorum rotation detected but not expected",
 				"height", height,
-				"validators_hash", meta.Header.ValidatorsHash, "quorum_hash", s.valSet.QuorumHash,
+				"validators_hash", meta.Header.ValidatorsHash,
+				"expected_validators_hash", valSetHash,
+				"next_validators_hash", meta.Header.NextValidatorsHash,
 				"validators", s.valSet)
 
 			return fmt.Errorf("quorum hash mismatch at height %d", height)
diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go
index 4db926bdb3..4c3e067666 100644
--- a/internal/inspect/inspect.go
+++ b/internal/inspect/inspect.go
@@ -77,7 +77,7 @@ func NewFromConfig(logger log.Logger, cfg *config.Config) (*Inspector, error) {
 	if err != nil {
 		return nil, err
 	}
-	ss := state.NewStore(sDB)
+	ss := state.NewStoreWithLogger(sDB, logger.With("module", "state_store"))
 	return New(cfg.RPC, bs, ss, sinks, logger), nil
 }
 
diff --git a/internal/state/store.go b/internal/state/store.go
index 1bd53067ba..52b3a2d9ec 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -126,6 +126,11 @@ func NewStore(db dbm.DB) Store {
 	return dbStore{db, log.NewNopLogger()}
 }
 
+// NewStoreWithLogger creates the dbStore of the state pkg.
+func NewStoreWithLogger(db dbm.DB, logger log.Logger) Store {
+	return dbStore{db, logger}
+}
+
 // LoadState loads the State from the database.
 func (store dbStore) Load() (State, error) {
 	return store.loadState(stateKey)
@@ -498,12 +503,11 @@ func (store dbStore) SaveValidatorSets(lowerHeight, upperHeight int64, vals *typ
 	return batch.WriteSync()
 }
 
-//-----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 
-// LoadValidators loads the ValidatorSet for a given height and round 0.
-//
-// Returns ErrNoValSetForHeight if the validator set can't be found for this height.
-func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore) (*types.ValidatorSet, error) {
+// loadValidators is a helper that loads the validator set from height or last stored height.
+// It does NOT set a correct proposer.
+func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, error) {
 	valInfo, err := loadValidatorsInfo(store.db, height)
 	if err != nil {
 		return nil, ErrNoValSetForHeight{Height: height, Err: err}
@@ -511,6 +515,7 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 
 	if valInfo.ValidatorSet == nil {
 		lastStoredHeight := lastStoredHeightFor(height, valInfo.LastHeightChanged)
+		store.logger.Debug("Validator set is nil, loading last stored height", "height", height, "last_height_changed", valInfo.LastHeightChanged, "last_stored_height", lastStoredHeight)
 		valInfo, err = loadValidatorsInfo(store.db, lastStoredHeight)
 		if err != nil || valInfo.ValidatorSet == nil {
 			return nil,
@@ -522,6 +527,20 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		}
 	}
 
+	return valInfo, nil
+}
+
+// LoadValidators loads the ValidatorSet for a given height and round 0.
+//
+// Returns ErrNoValSetForHeight if the validator set can't be found for this height.
+func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore) (*types.ValidatorSet, error) {
+	store.logger.Debug("Loading validators", "height", height)
+
+	valInfo, err := store.loadValidators(height)
+	if err != nil {
+		return nil, ErrNoValSetForHeight{Height: height, Err: err}
+	}
+
 	valSet, err := types.ValidatorSetFromProto(valInfo.ValidatorSet)
 	if err != nil {
 		return nil, err
@@ -562,27 +581,48 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		return strategy.ValidatorSet(), nil
 	}
 
-	// If we have that height in the block store, we just take proposer from previous block and advance it.
+	// If we don't have that height in the block store, we just take proposer from previous block and advance it.
 	// We don't use current height block because we want to return proposer at round 0.
 	prevMeta := bs.LoadBlockMeta(height - 1)
 	if prevMeta == nil {
 		return nil, fmt.Errorf("could not find block meta for height %d", height-1)
 	}
-	// Configure proposer strategy; first set proposer from previous block
-	if err := valSet.SetProposer(prevMeta.Header.ProposerProTxHash); err != nil {
-		return nil, fmt.Errorf("could not set proposer: %w", err)
-	}
-	strategy, err := selectproposer.NewProposerSelector(cp, valSet, prevMeta.Header.Height, prevMeta.Round, bs, store.logger)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create validator scoring strategy: %w", err)
-	}
+	// Configure proposer strategy; first check if we have a quorum change
+	if !prevMeta.Header.ValidatorsHash.Equal(prevMeta.Header.NextValidatorsHash) {
+		// rotation happened - we select 1st validator as proposer
+		valSetHash := valSet.Hash()
+		if !prevMeta.Header.NextValidatorsHash.Equal(valSetHash) {
+			return nil, ErrNoValSetForHeight{
+				height,
+				fmt.Errorf("quorum hash mismatch at height %d, expected %X, got %X", height,
+					prevMeta.Header.NextValidatorsHash, valSetHash),
+			}
+		}
+
+		if err = valSet.SetProposer(valSet.GetByIndex(0).ProTxHash); err != nil {
+			return nil, ErrNoValSetForHeight{height, fmt.Errorf("could not set proposer: %w", err)}
+		}
+
+		return valSet, nil
+	} else {
 
-	// now, advance to (height,0)
-	if err := strategy.UpdateHeightRound(height, 0); err != nil {
-		return nil, fmt.Errorf("failed to update validator scores at height %d, round 0: %w", height, err)
+		// set proposer from previous block
+		if err := valSet.SetProposer(prevMeta.Header.ProposerProTxHash); err != nil {
+			return nil, fmt.Errorf("could not set proposer: %w", err)
+		}
+		strategy, err := selectproposer.NewProposerSelector(cp, valSet, prevMeta.Header.Height, prevMeta.Round, bs, store.logger)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create validator scoring strategy: %w", err)
+		}
+
+		// now, advance to (height,0)
+		if err := strategy.UpdateHeightRound(height, 0); err != nil {
+			return nil, fmt.Errorf("failed to update validator scores at height %d, round 0: %w", height, err)
+		}
+
+		return strategy.ValidatorSet(), nil
 	}
 
-	return strategy.ValidatorSet(), nil
 }
 
 func lastStoredHeightFor(height, lastHeightChanged int64) int64 {
@@ -643,6 +683,7 @@ func (store dbStore) saveValidatorsInfo(
 		return err
 	}
 
+	store.logger.Debug("saving validator set", "height", height, "last_height_changed", lastHeightChanged, "validators", valSet)
 	return batch.Set(validatorsKey(height), bz)
 }
 
diff --git a/node/node.go b/node/node.go
index 7daa8eee4b..4e36a2f3b5 100644
--- a/node/node.go
+++ b/node/node.go
@@ -136,7 +136,7 @@ func makeNode(
 	}
 	closers = append(closers, dbCloser)
 
-	stateStore := sm.NewStore(stateDB)
+	stateStore := sm.NewStoreWithLogger(stateDB, logger.With("module", "state_store"))
 
 	genDoc, err := genesisDocProvider()
 	if err != nil {

From e04fde0260b29aa84d47dadb476acce00b4cf0b6 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 14:58:23 +0200
Subject: [PATCH 2/9] test(state): load validators on quorum change

---
 .../selectproposer/height_proposer.go         |   2 +-
 .../selectproposer/height_round_proposer.go   |   2 +-
 internal/state/store.go                       |   2 +-
 internal/state/store_test.go                  | 100 +++++++++++++++++-
 4 files changed, 98 insertions(+), 8 deletions(-)

diff --git a/internal/consensus/versioned/selectproposer/height_proposer.go b/internal/consensus/versioned/selectproposer/height_proposer.go
index 4b2d87a691..4ac873e396 100644
--- a/internal/consensus/versioned/selectproposer/height_proposer.go
+++ b/internal/consensus/versioned/selectproposer/height_proposer.go
@@ -90,7 +90,7 @@ func (s *heightProposerSelector) proposerFromStore(height int64) error {
 				"validators_hash", meta.Header.ValidatorsHash, "quorum_hash", s.valSet.QuorumHash,
 				"validators", s.valSet)
 
-			return fmt.Errorf("quorum hash mismatch at height %d", height)
+			return fmt.Errorf("validator set hash mismatch at height %d", height)
 		}
 
 		proposer = meta.Header.ProposerProTxHash
diff --git a/internal/consensus/versioned/selectproposer/height_round_proposer.go b/internal/consensus/versioned/selectproposer/height_round_proposer.go
index fe3a0e6691..57088f75a4 100644
--- a/internal/consensus/versioned/selectproposer/height_round_proposer.go
+++ b/internal/consensus/versioned/selectproposer/height_round_proposer.go
@@ -98,7 +98,7 @@ func (s *heightRoundProposerSelector) proposerFromStore(height int64, round int3
 				"next_validators_hash", meta.Header.NextValidatorsHash,
 				"validators", s.valSet)
 
-			return fmt.Errorf("quorum hash mismatch at height %d", height)
+			return fmt.Errorf("validators hash mismatch at height %d", height)
 		}
 
 		proposer = meta.Header.ProposerProTxHash
diff --git a/internal/state/store.go b/internal/state/store.go
index 52b3a2d9ec..c84c0d6572 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -594,7 +594,7 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		if !prevMeta.Header.NextValidatorsHash.Equal(valSetHash) {
 			return nil, ErrNoValSetForHeight{
 				height,
-				fmt.Errorf("quorum hash mismatch at height %d, expected %X, got %X", height,
+				fmt.Errorf("next validators hash mismatch at height %d, expected %X, got %X", height,
 					prevMeta.Header.NextValidatorsHash, valSetHash),
 			}
 		}
diff --git a/internal/state/store_test.go b/internal/state/store_test.go
index a696f5a8cc..64b34f7f66 100644
--- a/internal/state/store_test.go
+++ b/internal/state/store_test.go
@@ -31,10 +31,18 @@ const (
 // mockBlockStoreForProposerSelector creates a mock block store that returns proposers based on the height.
 // It assumes every block ends in round 0 and the proposer is the next validator in the validator set.
 func mockBlockStoreForProposerSelector(t *testing.T, startHeight, endHeight int64, vals *types.ValidatorSet) selectproposer.BlockStore {
-	vals = vals.Copy()
-	valsHash := vals.Hash()
 	blockStore := mocks.NewBlockStore(t)
 	blockStore.On("Base").Return(startHeight).Maybe()
+	configureBlockMetaWithValidators(t, blockStore, startHeight, endHeight, vals)
+
+	return blockStore
+}
+
+// / configureBlockMetaWithValidators configures the block store to return proposers based on the height.
+func configureBlockMetaWithValidators(_ *testing.T, blockStore *mocks.BlockStore, startHeight, endHeight int64, vals *types.ValidatorSet) {
+	vals = vals.Copy()
+	valsHash := vals.Hash()
+
 	for h := startHeight; h <= endHeight; h++ {
 		blockStore.On("LoadBlockMeta", h).
 			Return(&types.BlockMeta{
@@ -47,8 +55,6 @@ func mockBlockStoreForProposerSelector(t *testing.T, startHeight, endHeight int6
 			}).Maybe()
 		vals.IncProposerIndex(1)
 	}
-
-	return blockStore
 }
 
 func TestStoreBootstrap(t *testing.T) {
@@ -106,7 +112,8 @@ func TestStoreLoadValidators(t *testing.T) {
 
 	// initialize block store - create mock validators for each height
 	blockStoreVS := expectedVS.Copy()
-	blockStore := mockBlockStoreForProposerSelector(t, 1, valSetCheckpointInterval, blockStoreVS.ValidatorSet())
+	blockStore := mocks.NewBlockStore(t)
+	configureBlockMetaWithValidators(t, blockStore, 1, valSetCheckpointInterval, blockStoreVS.ValidatorSet())
 
 	// 1) LoadValidators loads validators using a height where they were last changed
 	// Note that only the current validators at height h are saved
@@ -167,6 +174,89 @@ func TestStoreLoadValidators(t *testing.T) {
 	require.Equal(t, expected, valsAtCheckpoint)
 }
 
+// / Given a set of blocks in the block store and two validator sets that rotate,
+// / When we load the validators during quorum rotation,
+// / Then we receive the correct validators for each height.
+func TestStoreLoadValidatorsOnRotation(t *testing.T) {
+	const startHeight = int64(1)
+
+	testCases := []struct {
+		rotations int64
+		nVals     int
+		nValSets  int64
+	}{
+		{1, 3, 3},
+		{2, 3, 2},
+		{3, 2, 4},
+	}
+
+	for _, tc := range testCases {
+		t.Run(fmt.Sprintf("rotations=%d,nVals=%d,nValSets=%d", tc.rotations, tc.nVals, tc.nValSets), func(t *testing.T) {
+			rotations := tc.rotations
+			nVals := tc.nVals
+			nValSets := tc.nValSets
+			uncommittedHeight := startHeight + rotations*int64(nVals)
+
+			stateDB := dbm.NewMemDB()
+			stateStore := sm.NewStore(stateDB)
+			validators := make([]*types.ValidatorSet, nValSets)
+			for i := int64(0); i < nValSets; i++ {
+				validators[i], _ = types.RandValidatorSet(nVals)
+				t.Logf("Validator set %d: %v", i, validators[i].Hash())
+			}
+
+			blockStore := mocks.NewBlockStore(t)
+			blockStore.On("Base").Return(startHeight).Maybe()
+
+			// configure block store and state store to return correct validators and block meta
+			for i := int64(0); i < rotations; i++ {
+				nextQuorumHeight := startHeight + (i+1)*int64(nVals)
+
+				err := stateStore.SaveValidatorSets(startHeight+i*int64(nVals), nextQuorumHeight-1, validators[i%nValSets])
+				require.NoError(t, err)
+
+				vals := validators[i%nValSets].Copy()
+				// reset proposer
+				require.NoError(t, vals.SetProposer(vals.GetByIndex(0).ProTxHash))
+				valsHash := vals.Hash()
+				nextValsHash := valsHash // we only change it in last block
+
+				// configure block store to return correct validator sets and proposers
+				for h := startHeight + i*int64(nVals); h < nextQuorumHeight; h++ {
+					if h == nextQuorumHeight-1 {
+						nextValsHash = validators[(i+1)%nValSets].Hash()
+					}
+					blockStore.On("LoadBlockMeta", h).
+						Return(&types.BlockMeta{
+							Header: types.Header{
+								Height:             h,
+								ProposerProTxHash:  vals.Proposer().ProTxHash,
+								ValidatorsHash:     valsHash,
+								NextValidatorsHash: nextValsHash,
+							},
+						}).Maybe()
+					vals.IncProposerIndex(1)
+				}
+
+				assert.LessOrEqual(t, nextQuorumHeight, uncommittedHeight, "we should not save block at height {}", uncommittedHeight)
+			}
+
+			// now, we are at height 10, and we should rotate to next validators
+			// we don't have the last block yet, but we already have validator set for the next height
+			blockStore.On("LoadBlockMeta", uncommittedHeight).Return(nil).Maybe()
+
+			expectedValidators := validators[rotations%nValSets]
+			err := stateStore.SaveValidatorSets(uncommittedHeight, uncommittedHeight, expectedValidators)
+			require.NoError(t, err)
+
+			// We should get correct validator set from the store
+			readVals, err := stateStore.LoadValidators(uncommittedHeight, blockStore)
+			require.NoError(t, err)
+			assert.Equal(t, expectedValidators, readVals)
+		})
+	}
+}
+
 // This benchmarks the speed of loading validators from different heights if there is no validator set change.
 // NOTE: This isn't too indicative of validator retrieval speed as the db is always (regardless of height) only
 // performing two operations: 1) retrieve validator info at height x, which has a last validator set change of 1

From 655159696329aab6ce1de0b8c60e803d4a062800 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 15:50:51 +0200
Subject: [PATCH 3/9] refactor: NewStore now accepts optional logger

---
 .../selectproposer/proposer_selector.go       |  1 -
 internal/inspect/inspect.go                   |  2 +-
 internal/state/store.go                       | 83 +++++--------------
 internal/state/store_test.go                  | 28 +++++--
 node/node.go                                  |  2 +-
 5 files changed, 45 insertions(+), 71 deletions(-)

diff --git a/internal/consensus/versioned/selectproposer/proposer_selector.go b/internal/consensus/versioned/selectproposer/proposer_selector.go
index 9895dfd467..403a8e4c49 100644
--- a/internal/consensus/versioned/selectproposer/proposer_selector.go
+++ b/internal/consensus/versioned/selectproposer/proposer_selector.go
@@ -60,7 +60,6 @@ func NewProposerSelector(cp types.ConsensusParams, valSet *types.ValidatorSet, v
 	case int32(tmtypes.VersionParams_CONSENSUS_VERSION_0):
 		return NewHeightProposerSelector(valSet, valsetHeight, bs, logger)
 	case int32(tmtypes.VersionParams_CONSENSUS_VERSION_1):
-
 		return NewHeightRoundProposerSelector(valSet, valsetHeight, valsetRound, bs, logger)
 	default:
 		return nil, fmt.Errorf("unknown consensus version: %v", cp.Version.ConsensusVersion)
diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go
index 4c3e067666..20444118c1 100644
--- a/internal/inspect/inspect.go
+++ b/internal/inspect/inspect.go
@@ -77,7 +77,7 @@ func NewFromConfig(logger log.Logger, cfg *config.Config) (*Inspector, error) {
 	if err != nil {
 		return nil, err
 	}
-	ss := state.NewStoreWithLogger(sDB, logger.With("module", "state_store"))
+	ss := state.NewStore(sDB, logger.With("module", "state_store"))
 	return New(cfg.RPC, bs, ss, sinks, logger), nil
 }
 
diff --git a/internal/state/store.go b/internal/state/store.go
index c84c0d6572..86fcb26f3f 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -122,13 +122,12 @@ type dbStore struct {
 var _ Store = (*dbStore)(nil)
 
 // NewStore creates the dbStore of the state pkg.
-func NewStore(db dbm.DB) Store {
-	return dbStore{db, log.NewNopLogger()}
-}
+func NewStore(db dbm.DB, logger ...log.Logger) Store {
+	if len(logger) != 1 || logger[0] == nil {
+		logger = []log.Logger{log.NewNopLogger()}
+	}
 
-// NewStoreWithLogger creates the dbStore of the state pkg.
-func NewStoreWithLogger(db dbm.DB, logger log.Logger) Store {
-	return dbStore{db, logger}
+	return dbStore{db, logger[0]}
 }
 
 // LoadState loads the State from the database.
@@ -503,11 +502,12 @@ func (store dbStore) SaveValidatorSets(lowerHeight, upperHeight int64, vals *typ
 	return batch.WriteSync()
 }
 
-// -----------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
 
-// loadValidators is a helper that loads the validator set from height or last stored height.
-// It does NOT set a correct proposer.
-func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, error) {
+// LoadValidators loads the ValidatorSet for a given height and round 0.
+//
+// Returns ErrNoValSetForHeight if the validator set can't be found for this height.
+func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore) (*types.ValidatorSet, error) {
 	valInfo, err := loadValidatorsInfo(store.db, height)
 	if err != nil {
 		return nil, ErrNoValSetForHeight{Height: height, Err: err}
@@ -515,7 +515,6 @@ func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, erro
 
 	if valInfo.ValidatorSet == nil {
 		lastStoredHeight := lastStoredHeightFor(height, valInfo.LastHeightChanged)
-		store.logger.Debug("Validator set is nil, loading last stored height", "height", height, "last_height_changed", valInfo.LastHeightChanged, "last_stored_height", lastStoredHeight)
 		valInfo, err = loadValidatorsInfo(store.db, lastStoredHeight)
 		if err != nil || valInfo.ValidatorSet == nil {
 			return nil,
@@ -527,20 +526,6 @@ func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, erro
 		}
 	}
 
-	return valInfo, nil
-}
-
-// LoadValidators loads the ValidatorSet for a given height and round 0.
-//
-// Returns ErrNoValSetForHeight if the validator set can't be found for this height.
-func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore) (*types.ValidatorSet, error) {
-	store.logger.Debug("Loading validators", "height", height)
-
-	valInfo, err := store.loadValidators(height)
-	if err != nil {
-		return nil, ErrNoValSetForHeight{Height: height, Err: err}
-	}
-
 	valSet, err := types.ValidatorSetFromProto(valInfo.ValidatorSet)
 	if err != nil {
 		return nil, err
@@ -581,48 +566,27 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		return strategy.ValidatorSet(), nil
 	}
 
-	// If we don't have that height in the block store, we just take proposer from previous block and advance it.
+	// If we have that height in the block store, we just take proposer from previous block and advance it.
 	// We don't use current height block because we want to return proposer at round 0.
 	prevMeta := bs.LoadBlockMeta(height - 1)
 	if prevMeta == nil {
 		return nil, fmt.Errorf("could not find block meta for height %d", height-1)
 	}
-	// Configure proposer strategy; first check if we have a quorum change
-	if !prevMeta.Header.ValidatorsHash.Equal(prevMeta.Header.NextValidatorsHash) {
-		// rotation happened - we select 1st validator as proposer
-		valSetHash := valSet.Hash()
-		if !prevMeta.Header.NextValidatorsHash.Equal(valSetHash) {
-			return nil, ErrNoValSetForHeight{
-				height,
-				fmt.Errorf("next validators hash mismatch at height %d, expected %X, got %X", height,
-					prevMeta.Header.NextValidatorsHash, valSetHash),
-			}
-		}
-
-		if err = valSet.SetProposer(valSet.GetByIndex(0).ProTxHash); err != nil {
-			return nil, ErrNoValSetForHeight{height, fmt.Errorf("could not set proposer: %w", err)}
-		}
-
-		return valSet, nil
-	} else {
-
-		// set proposer from previous block
-		if err := valSet.SetProposer(prevMeta.Header.ProposerProTxHash); err != nil {
-			return nil, fmt.Errorf("could not set proposer: %w", err)
-		}
-		strategy, err := selectproposer.NewProposerSelector(cp, valSet, prevMeta.Header.Height, prevMeta.Round, bs, store.logger)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create validator scoring strategy: %w", err)
-		}
-
-		// now, advance to (height,0)
-		if err := strategy.UpdateHeightRound(height, 0); err != nil {
-			return nil, fmt.Errorf("failed to update validator scores at height %d, round 0: %w", height, err)
-		}
+	// Configure proposer strategy; first set proposer from previous block
+	if err := valSet.SetProposer(prevMeta.Header.ProposerProTxHash); err != nil {
+		return nil, fmt.Errorf("could not set proposer: %w", err)
+	}
+	strategy, err := selectproposer.NewProposerSelector(cp, valSet, prevMeta.Header.Height, prevMeta.Round, bs, store.logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create validator scoring strategy: %w", err)
+	}
 
-		return strategy.ValidatorSet(), nil
+	// now, advance to (height,0)
+	if err := strategy.UpdateHeightRound(height, 0); err != nil {
+		return nil, fmt.Errorf("failed to update validator scores at height %d, round 0: %w", height, err)
 	}
 
+	return strategy.ValidatorSet(), nil
 }
 
 func lastStoredHeightFor(height, lastHeightChanged int64) int64 {
@@ -683,7 +647,6 @@ func (store dbStore) saveValidatorsInfo(
 		return err
 	}
 
-	store.logger.Debug("saving validator set", "height", height, "last_height_changed", lastHeightChanged, "validators", valSet)
 	return batch.Set(validatorsKey(height), bz)
 }
 
diff --git a/internal/state/store_test.go b/internal/state/store_test.go
index 64b34f7f66..cf69ac01fa 100644
--- a/internal/state/store_test.go
+++ b/internal/state/store_test.go
@@ -181,24 +181,30 @@ func TestStoreLoadValidatorsOnRotation(t *testing.T) {
 	const startHeight = int64(1)
 
 	testCases := []struct {
-		rotations int64
-		nVals     int
-		nValSets  int64
+		rotations        int64
+		nVals            int
+		nValSets         int64
+		consensusVersion int32
 	}{
-		{1, 3, 3},
-		{2, 3, 2},
-		{3, 2, 4},
+		{1, 3, 3, 0},
+		{1, 3, 3, 1},
+		{2, 3, 2, 0},
+		{2, 3, 2, 1},
+		{3, 2, 4, 0},
+		{3, 2, 4, 1},
 	}
 
 	for _, tc := range testCases {
-		t.Run(fmt.Sprintf("rotations=%d,nVals=%d,nValSets=%d", tc.rotations, tc.nVals, tc.nValSets), func(t *testing.T) {
+		t.Run(fmt.Sprintf("rotations=%d,nVals=%d,nValSets=%d,ver=%d",
+			tc.rotations, tc.nVals, tc.nValSets, tc.consensusVersion), func(t *testing.T) {
 			rotations := tc.rotations
 			nVals := tc.nVals
 			nValSets := tc.nValSets
 			uncommittedHeight := startHeight + rotations*int64(nVals)
 
 			stateDB := dbm.NewMemDB()
-			stateStore := sm.NewStore(stateDB)
+			stateStore := sm.NewStore(stateDB, log.NewTestingLoggerWithLevel(t, log.LogLevelDebug))
+
 			validators := make([]*types.ValidatorSet, nValSets)
 			for i := int64(0); i < nValSets; i++ {
 				validators[i], _ = types.RandValidatorSet(nVals)
@@ -236,6 +242,12 @@ func TestStoreLoadValidatorsOnRotation(t *testing.T) {
 							},
 						}).Maybe()
 					vals.IncProposerIndex(1)
+
+					// set consensus version
+					state := makeRandomStateFromValidatorSet(vals, h+1, startHeight+i*int64(nVals), blockStore)
+					state.LastHeightConsensusParamsChanged = h + 1
+					state.ConsensusParams.Version.ConsensusVersion = tc.consensusVersion
+					require.NoError(t, stateStore.Save(state))
 				}
 
 				assert.LessOrEqual(t, nextQuorumHeight, uncommittedHeight, "we should not save block at height {}", uncommittedHeight)
diff --git a/node/node.go b/node/node.go
index 4e36a2f3b5..f69b7d196f 100644
--- a/node/node.go
+++ b/node/node.go
@@ -136,7 +136,7 @@ func makeNode(
 	}
 	closers = append(closers, dbCloser)
 
-	stateStore := sm.NewStoreWithLogger(stateDB, logger.With("module", "state_store"))
+	stateStore := sm.NewStore(stateDB, logger.With("module", "state_store"))
 
 	genDoc, err := genesisDocProvider()
 	if err != nil {

From f5db91dab746e3795e60eafe4d47775ab3afb937 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 15:56:31 +0200
Subject: [PATCH 4/9] revert: restore code removed by mistake

---
 internal/state/store.go | 72 ++++++++++++++++++++++++++++++-----------
 1 file changed, 54 insertions(+), 18 deletions(-)

diff --git a/internal/state/store.go b/internal/state/store.go
index 86fcb26f3f..ed2d4ee3c0 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -502,12 +502,11 @@ func (store dbStore) SaveValidatorSets(lowerHeight, upperHeight int64, vals *typ
 	return batch.WriteSync()
 }
 
-//-----------------------------------------------------------------------------
+// -----------------------------------------------------------------------------
 
-// LoadValidators loads the ValidatorSet for a given height and round 0.
-//
-// Returns ErrNoValSetForHeight if the validator set can't be found for this height.
-func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore) (*types.ValidatorSet, error) {
+// loadValidators is a helper that loads the validator set from height or last stored height.
+// It does NOT set a correct proposer.
+func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, error) {
 	valInfo, err := loadValidatorsInfo(store.db, height)
 	if err != nil {
 		return nil, ErrNoValSetForHeight{Height: height, Err: err}
@@ -515,6 +514,7 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 
 	if valInfo.ValidatorSet == nil {
 		lastStoredHeight := lastStoredHeightFor(height, valInfo.LastHeightChanged)
+		store.logger.Debug("Validator set is nil, loading last stored height", "height", height, "last_height_changed", valInfo.LastHeightChanged, "last_stored_height", lastStoredHeight)
 		valInfo, err = loadValidatorsInfo(store.db, lastStoredHeight)
 		if err != nil || valInfo.ValidatorSet == nil {
 			return nil,
@@ -526,6 +526,20 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		}
 	}
 
+	return valInfo, nil
+}
+
+// LoadValidators loads the ValidatorSet for a given height and round 0.
+//
+// Returns ErrNoValSetForHeight if the validator set can't be found for this height.
+func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore) (*types.ValidatorSet, error) {
+	store.logger.Debug("Loading validators", "height", height)
+
+	valInfo, err := store.loadValidators(height)
+	if err != nil {
+		return nil, ErrNoValSetForHeight{Height: height, Err: err}
+	}
+
 	valSet, err := types.ValidatorSetFromProto(valInfo.ValidatorSet)
 	if err != nil {
 		return nil, err
@@ -566,27 +580,48 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		return strategy.ValidatorSet(), nil
 	}
 
-	// If we have that height in the block store, we just take proposer from previous block and advance it.
+	// If we don't have that height in the block store, we just take proposer from previous block and advance it.
 	// We don't use current height block because we want to return proposer at round 0.
 	prevMeta := bs.LoadBlockMeta(height - 1)
 	if prevMeta == nil {
 		return nil, fmt.Errorf("could not find block meta for height %d", height-1)
 	}
-	// Configure proposer strategy; first set proposer from previous block
-	if err := valSet.SetProposer(prevMeta.Header.ProposerProTxHash); err != nil {
-		return nil, fmt.Errorf("could not set proposer: %w", err)
-	}
-	strategy, err := selectproposer.NewProposerSelector(cp, valSet, prevMeta.Header.Height, prevMeta.Round, bs, store.logger)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create validator scoring strategy: %w", err)
-	}
+	// Configure proposer strategy; first check if we have a quorum change
+	if !prevMeta.Header.ValidatorsHash.Equal(prevMeta.Header.NextValidatorsHash) {
+		// rotation happened - we select 1st validator as proposer
+		valSetHash := valSet.Hash()
+		if !prevMeta.Header.NextValidatorsHash.Equal(valSetHash) {
+			return nil, ErrNoValSetForHeight{
+				height,
+				fmt.Errorf("next validators hash mismatch at height %d, expected %X, got %X", height,
+					prevMeta.Header.NextValidatorsHash, valSetHash),
+			}
+		}
 
-	// now, advance to (height,0)
-	if err := strategy.UpdateHeightRound(height, 0); err != nil {
-		return nil, fmt.Errorf("failed to update validator scores at height %d, round 0: %w", height, err)
+		if err = valSet.SetProposer(valSet.GetByIndex(0).ProTxHash); err != nil {
+			return nil, ErrNoValSetForHeight{height, fmt.Errorf("could not set proposer: %w", err)}
+		}
+
+		return valSet, nil
+	} else {
+
+		// set proposer from previous block
+		if err := valSet.SetProposer(prevMeta.Header.ProposerProTxHash); err != nil {
+			return nil, fmt.Errorf("could not set proposer: %w", err)
+		}
+		strategy, err := selectproposer.NewProposerSelector(cp, valSet, prevMeta.Header.Height, prevMeta.Round, bs, store.logger)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create validator scoring strategy: %w", err)
+		}
+
+		// now, advance to (height,0)
+		if err := strategy.UpdateHeightRound(height, 0); err != nil {
+			return nil, fmt.Errorf("failed to update validator scores at height %d, round 0: %w", height, err)
+		}
+
+		return strategy.ValidatorSet(), nil
 	}
 
-	return strategy.ValidatorSet(), nil
 }
 
 func lastStoredHeightFor(height, lastHeightChanged int64) int64 {
@@ -647,6 +682,7 @@ func (store dbStore) saveValidatorsInfo(
 		return err
 	}
 
+	store.logger.Debug("saving validator set", "height", height, "last_height_changed", lastHeightChanged, "validators", valSet)
 	return batch.Set(validatorsKey(height), bz)
 }
 

From 400420b4a6f86b375bf8a50d31a50ef85c7dcc23 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 16:37:17 +0200
Subject: [PATCH 5/9] chore: rabbit's feedback

---
 internal/state/store.go      | 13 ++++++-------
 internal/state/store_test.go |  8 ++++----
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/internal/state/store.go b/internal/state/store.go
index ed2d4ee3c0..3fce06073f 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -509,7 +509,7 @@ func (store dbStore) SaveValidatorSets(lowerHeight, upperHeight int64, vals *typ
 func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, error) {
 	valInfo, err := loadValidatorsInfo(store.db, height)
 	if err != nil {
-		return nil, ErrNoValSetForHeight{Height: height, Err: err}
+		return nil, ErrNoValSetForHeight{Height: height, Err: fmt.Errorf("failed to load validators info: %w", err)}
 	}
 
 	if valInfo.ValidatorSet == nil {
@@ -517,12 +517,11 @@ func (store dbStore) loadValidators(height int64) (*tmstate.ValidatorsInfo, erro
 		store.logger.Debug("Validator set is nil, loading last stored height", "height", height, "last_height_changed", valInfo.LastHeightChanged, "last_stored_height", lastStoredHeight)
 		valInfo, err = loadValidatorsInfo(store.db, lastStoredHeight)
 		if err != nil || valInfo.ValidatorSet == nil {
-			return nil,
-				fmt.Errorf("couldn't find validators at height %d (height %d was originally requested): %w",
-					lastStoredHeight,
-					height,
-					err,
-				)
+			return nil, ErrNoValSetForHeight{Height: height, Err: fmt.Errorf("couldn't find validators at height %d (height %d was originally requested): %w",
+				lastStoredHeight,
+				height,
+				err,
+			)}
 		}
 	}
 
diff --git a/internal/state/store_test.go b/internal/state/store_test.go
index cf69ac01fa..215e5467a8 100644
--- a/internal/state/store_test.go
+++ b/internal/state/store_test.go
@@ -38,7 +38,7 @@ func mockBlockStoreForProposerSelector(t *testing.T, startHeight, endHeight int6
 	return blockStore
 }
 
-// / configureBlockMetaWithValidators configures the block store to return proposers based on the height.
+// configureBlockMetaWithValidators configures the block store to return proposers based on the height.
 func configureBlockMetaWithValidators(_ *testing.T, blockStore *mocks.BlockStore, startHeight, endHeight int64, vals *types.ValidatorSet) {
 	vals = vals.Copy()
 	valsHash := vals.Hash()
@@ -174,9 +174,9 @@ func TestStoreLoadValidators(t *testing.T) {
 	require.Equal(t, expected, valsAtCheckpoint)
 }
 
-// / Given a set of blocks in the block store and two validator sets that rotate,
-// / When we load the validators during quorum rotation,
-// / Then we receive the correct validators for each height.
+// Given a set of blocks in the block store and two validator sets that rotate,
+// When we load the validators during quorum rotation,
+// Then we receive the correct validators for each height.
 func TestStoreLoadValidatorsOnRotation(t *testing.T) {
 	const startHeight = int64(1)
 

From ada43a1273553328e95cebd5741d09ffe7c1cf8d Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 16:38:42 +0200
Subject: [PATCH 6/9] fix: TestStoreLoadValidators fails

---
 internal/state/store_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/internal/state/store_test.go b/internal/state/store_test.go
index 215e5467a8..28980d6737 100644
--- a/internal/state/store_test.go
+++ b/internal/state/store_test.go
@@ -112,8 +112,7 @@ func TestStoreLoadValidators(t *testing.T) {
 
 	// initialize block store - create mock validators for each height
 	blockStoreVS := expectedVS.Copy()
-	blockStore := mocks.NewBlockStore(t)
-	configureBlockMetaWithValidators(t, blockStore, 1, valSetCheckpointInterval, blockStoreVS.ValidatorSet())
+	blockStore := mockBlockStoreForProposerSelector(t, 1, valSetCheckpointInterval, blockStoreVS.ValidatorSet())
 
 	// 1) LoadValidators loads validators using a height where they were last changed
 	// Note that only the current validators at height h are saved

From cbe1a7683d8811aa4826091ac6ddd66a9dbc2fb0 Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Tue, 15 Oct 2024 17:54:56 +0200
Subject: [PATCH 7/9] fix: test

---
 internal/state/store_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/state/store_test.go b/internal/state/store_test.go
index 28980d6737..4dd2562b60 100644
--- a/internal/state/store_test.go
+++ b/internal/state/store_test.go
@@ -147,9 +147,9 @@ func TestStoreLoadValidators(t *testing.T) {
 	// check that a request will go back to the last checkpoint
 	_, err = stateStore.LoadValidators(valSetCheckpointInterval+1, blockStore)
 	require.Error(t, err)
-	require.Equal(t, fmt.Sprintf("couldn't find validators at height %d (height %d was originally requested): "+
+	require.ErrorContains(t, err, fmt.Sprintf("couldn't find validators at height %d (height %d was originally requested): "+
 		"value retrieved from db is empty",
-		valSetCheckpointInterval, valSetCheckpointInterval+1), err.Error())
+		valSetCheckpointInterval, valSetCheckpointInterval+1))
 
 	// now save a validator set at that checkpoint
 	err = stateStore.Save(makeRandomStateFromValidatorSet(vals, valSetCheckpointInterval, 1, blockStore))

From fe916397ccc458619a51fcec10e56de51794753a Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Sat, 2 Nov 2024 23:12:44 +0100
Subject: [PATCH 8/9] chore(state): document NewStore() logger param

---
 internal/state/store.go | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/internal/state/store.go b/internal/state/store.go
index 3fce06073f..d969272b35 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -122,15 +122,31 @@ type dbStore struct {
 var _ Store = (*dbStore)(nil)
 
 // NewStore creates the dbStore of the state pkg.
+//
+// ## Parameters
+//
+//   - `db` - the database to use
+//   - `logger` - the logger to use; optional, defaults to a nop logger if not provided; if more than one is provided,
+//     it will panic
+//
+// ##Panics
+//
+// If more than one logger is provided.
 func NewStore(db dbm.DB, logger ...log.Logger) Store {
-	if len(logger) != 1 || logger[0] == nil {
+	// To avoid changing the API, we use `logger ...log.Logger` in function signature, so that old code can
+	// provide only `db`. In this case, we use NopLogger.
+	if logger[0] == nil {
 		logger = []log.Logger{log.NewNopLogger()}
 	}
 
+	if len(logger) > 1 {
+		panic("NewStore(): maximum one logger is allowed")
+	}
+
 	return dbStore{db, logger[0]}
 }
 
-// LoadState loads the State from the database.
+// Load loads the State from the database.
 func (store dbStore) Load() (State, error) {
 	return store.loadState(stateKey)
 }

From b671c742c2c42ce52df3a4e182788e0e430030ce Mon Sep 17 00:00:00 2001
From: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Date: Mon, 4 Nov 2024 08:35:03 +0100
Subject: [PATCH 9/9] fix: potential panic in NewStore

---
 internal/state/store.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/internal/state/store.go b/internal/state/store.go
index d969272b35..991912e904 100644
--- a/internal/state/store.go
+++ b/internal/state/store.go
@@ -135,7 +135,7 @@ var _ Store = (*dbStore)(nil)
 func NewStore(db dbm.DB, logger ...log.Logger) Store {
 	// To avoid changing the API, we use `logger ...log.Logger` in function signature, so that old code can
 	// provide only `db`. In this case, we use NopLogger.
-	if logger[0] == nil {
+	if len(logger) == 0 || logger[0] == nil {
 		logger = []log.Logger{log.NewNopLogger()}
 	}
 
@@ -607,8 +607,8 @@ func (store dbStore) LoadValidators(height int64, bs selectproposer.BlockStore)
 		valSetHash := valSet.Hash()
 		if !prevMeta.Header.NextValidatorsHash.Equal(valSetHash) {
 			return nil, ErrNoValSetForHeight{
-				height,
-				fmt.Errorf("next validators hash mismatch at height %d, expected %X, got %X", height,
+				Height: height,
+				Err: fmt.Errorf("next validators hash mismatch at height %d, expected %X, got %X", height,
 					prevMeta.Header.NextValidatorsHash, valSetHash),
 			}
 		}