final experiment

substrate/bin/node/runtime/src/lib.rs

@@ -701,7 +701,10 @@ impl pallet_staking::Config for Runtime {
 	type MaxUnlockingChunks = ConstU32<32>;
 	type MaxControllersInDeprecationBatch = MaxControllersInDeprecationBatch;
 	type HistoryDepth = HistoryDepth;
+	// variant with stake-tracker
 	type EventListeners = (StakeTracker, NominationPools);
+	// variant without stake-tracker
+	//type EventListeners = NominationPools;
 	type WeightInfo = pallet_staking::weights::SubstrateWeight<Runtime>;
 	type BenchmarkingConfig = StakingBenchmarkingConfig;
 	type DisablingStrategy = pallet_staking::UpToLimitDisablingStrategy;
@@ -3290,6 +3293,36 @@ mod tests {
 	use frame_system::offchain::CreateSignedTransaction;
 	use sp_runtime::UpperOf;
 
+	#[test]
+	fn stake_tracker_weights() {
+		for n in vec![64u64, 128, 256, 512].into_iter() {
+			let no_stake_tracker = Weight::from_parts(191_534_495, 0)
+				.saturating_add(Weight::from_parts(0, 33162))
+				.saturating_add(Weight::from_parts(47_169_851, 0).saturating_mul(n.into()))
+				.saturating_add(RocksDbWeight::get().reads(15))
+				.saturating_add(RocksDbWeight::get().reads((7_u64).saturating_mul(n.into())))
+				.saturating_add(RocksDbWeight::get().writes(4))
+				.saturating_add(RocksDbWeight::get().writes((3_u64).saturating_mul(n.into())))
+				.saturating_add(Weight::from_parts(0, 3774).saturating_mul(n.into()));
+
+			let stake_tracker = Weight::from_parts(912_124_555, 0)
+				.saturating_add(Weight::from_parts(0, 898139))
+				.saturating_add(Weight::from_parts(195_832_335, 0).saturating_mul(n.into()))
+				.saturating_add(RocksDbWeight::get().reads(356))
+				.saturating_add(RocksDbWeight::get().reads((14_u64).saturating_mul(n.into())))
+				.saturating_add(RocksDbWeight::get().writes(343))
+				.saturating_add(RocksDbWeight::get().writes((8_u64).saturating_mul(n.into())))
+				.saturating_add(Weight::from_parts(0, 13165).saturating_mul(n.into()));
+
+			println!("- n = {n}");
+			println!("no stake-tracker: {:?}", no_stake_tracker);
+			println!("with stake-tracker: {:?}", stake_tracker);
+			println!("");
+		}
+
+        assert!(false);
+	}
+
 	#[test]
 	fn validate_transaction_submitter_bounds() {
 		fn is_submit_signed_transaction<T>()

substrate/frame/staking/payout_no_stake_tracker-weights.rs

@@ -2,7 +2,7 @@
 //! Autogenerated weights for `pallet_staking`
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0
-//! DATE: 2024-07-24, STEPS: `2`, REPEAT: `1`, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! DATE: 2024-07-26, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]`
 //! WORST CASE MAP SIZE: `1000000`
 //! HOSTNAME: `gpestanas-MacBook-Pro.local`, CPU: `<UNKNOWN>`
 //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: 1024
@@ -21,10 +21,6 @@
 // pallet-staking
 // --extrinsic
 // payout_stakers_alive_staked
-// --steps
-// 2
-// --repeat
-// 1
 // --output
 // payout_no_stake_tracker-weights.rs
 
@@ -53,6 +49,8 @@ impl<T: frame_system::Config> pallet_staking::WeightInfo for WeightInfo<T> {
 	/// Proof: `Staking::CurrentEra` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`)
 	/// Storage: `Staking::ErasValidatorReward` (r:1 w:0)
 	/// Proof: `Staking::ErasValidatorReward` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::ErasStakersPaged` (r:1 w:0)
+	/// Proof: `Staking::ErasStakersPaged` (`max_values`: None, `max_size`: None, mode: `Measured`)
 	/// Storage: `Staking::VirtualStakers` (r:257 w:0)
 	/// Proof: `Staking::VirtualStakers` (`max_values`: None, `max_size`: Some(40), added: 2515, mode: `MaxEncodedLen`)
 	/// Storage: `Balances::Locks` (r:257 w:257)
@@ -61,23 +59,26 @@ impl<T: frame_system::Config> pallet_staking::WeightInfo for WeightInfo<T> {
 	/// Proof: `Balances::Freezes` (`max_values`: None, `max_size`: Some(67), added: 2542, mode: `MaxEncodedLen`)
 	/// Storage: `System::Account` (r:257 w:257)
 	/// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`)
-	/// Storage: `Staking::ErasStakersPaged` (r:1 w:0)
-	/// Proof: `Staking::ErasStakersPaged` (`max_values`: None, `max_size`: None, mode: `Measured`)
 	/// Storage: `Staking::ErasRewardPoints` (r:1 w:0)
 	/// Proof: `Staking::ErasRewardPoints` (`max_values`: None, `max_size`: None, mode: `Measured`)
 	/// Storage: `Staking::ErasValidatorPrefs` (r:1 w:0)
 	/// Proof: `Staking::ErasValidatorPrefs` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`)
 	/// Storage: `Staking::Payee` (r:257 w:0)
 	/// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`)
 	/// The range of component `n` is `[0, 256]`.
-	fn payout_stakers_alive_staked(_n: u32, ) -> Weight {
+	fn payout_stakers_alive_staked(n: u32, ) -> Weight {
 		// Proof Size summary in bytes:
-		//  Measured:  `5346 + n * (471 ±0)`
-		//  Estimated: `970908`
-		// Minimum execution time: 150_000_000 picoseconds.
-		Weight::from_parts(20_435_000_000, 0)
-			.saturating_add(Weight::from_parts(0, 970908))
-			.saturating_add(T::DbWeight::get().reads(1807))
-			.saturating_add(T::DbWeight::get().writes(772))
+		//  Measured:  `35671 + n * (951 ±0)`
+		//  Estimated: `33162 + n * (3774 ±3)`
+		// Minimum execution time: 155_000_000 picoseconds.
+		Weight::from_parts(191_534_495, 0)
+			.saturating_add(Weight::from_parts(0, 33162))
+			// Standard Error: 49_168
+			.saturating_add(Weight::from_parts(47_169_851, 0).saturating_mul(n.into()))
+			.saturating_add(T::DbWeight::get().reads(15))
+			.saturating_add(T::DbWeight::get().reads((7_u64).saturating_mul(n.into())))
+			.saturating_add(T::DbWeight::get().writes(4))
+			.saturating_add(T::DbWeight::get().writes((3_u64).saturating_mul(n.into())))
+			.saturating_add(Weight::from_parts(0, 3774).saturating_mul(n.into()))
 	}
 }

substrate/frame/staking/payout_stake_tracker-weights.rs

@@ -0,0 +1,90 @@
+
+//! Autogenerated weights for `pallet_staking`
+//!
+//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 32.0.0
+//! DATE: 2024-07-26, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]`
+//! WORST CASE MAP SIZE: `1000000`
+//! HOSTNAME: `gpestanas-MacBook-Pro.local`, CPU: `<UNKNOWN>`
+//! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: 1024
+
+// Executed Command:
+// /Users/gpestana/cargo_target/release/substrate-node
+// benchmark
+// pallet
+// --execution
+// wasm
+// --wasm-execution
+// compiled
+// --chain
+// dev
+// --pallet
+// pallet-staking
+// --extrinsic
+// payout_stakers_alive_staked
+// --output
+// payout_stake_tracker-weights.rs
+
+#![cfg_attr(rustfmt, rustfmt_skip)]
+#![allow(unused_parens)]
+#![allow(unused_imports)]
+#![allow(missing_docs)]
+
+use frame_support::{traits::Get, weights::Weight};
+use core::marker::PhantomData;
+
+/// Weight functions for `pallet_staking`.
+pub struct WeightInfo<T>(PhantomData<T>);
+impl<T: frame_system::Config> pallet_staking::WeightInfo for WeightInfo<T> {
+	/// Storage: `Staking::Bonded` (r:257 w:0)
+	/// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::Ledger` (r:257 w:257)
+	/// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::ErasStakersClipped` (r:1 w:0)
+	/// Proof: `Staking::ErasStakersClipped` (`max_values`: None, `max_size`: None, mode: `Measured`)
+	/// Storage: `Staking::ErasStakersOverview` (r:1 w:0)
+	/// Proof: `Staking::ErasStakersOverview` (`max_values`: None, `max_size`: Some(92), added: 2567, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::ClaimedRewards` (r:1 w:1)
+	/// Proof: `Staking::ClaimedRewards` (`max_values`: None, `max_size`: None, mode: `Measured`)
+	/// Storage: `Staking::CurrentEra` (r:1 w:0)
+	/// Proof: `Staking::CurrentEra` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::ErasValidatorReward` (r:1 w:0)
+	/// Proof: `Staking::ErasValidatorReward` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::ErasStakersPaged` (r:1 w:0)
+	/// Proof: `Staking::ErasStakersPaged` (`max_values`: None, `max_size`: None, mode: `Measured`)
+	/// Storage: `Staking::VirtualStakers` (r:257 w:0)
+	/// Proof: `Staking::VirtualStakers` (`max_values`: None, `max_size`: Some(40), added: 2515, mode: `MaxEncodedLen`)
+	/// Storage: `Balances::Locks` (r:257 w:257)
+	/// Proof: `Balances::Locks` (`max_values`: None, `max_size`: Some(1299), added: 3774, mode: `MaxEncodedLen`)
+	/// Storage: `Balances::Freezes` (r:257 w:0)
+	/// Proof: `Balances::Freezes` (`max_values`: None, `max_size`: Some(67), added: 2542, mode: `MaxEncodedLen`)
+	/// Storage: `System::Account` (r:257 w:257)
+	/// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::ErasRewardPoints` (r:1 w:0)
+	/// Proof: `Staking::ErasRewardPoints` (`max_values`: None, `max_size`: None, mode: `Measured`)
+	/// Storage: `Staking::ErasValidatorPrefs` (r:1 w:0)
+	/// Proof: `Staking::ErasValidatorPrefs` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::Payee` (r:257 w:0)
+	/// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::Validators` (r:257 w:0)
+	/// Proof: `Staking::Validators` (`max_values`: None, `max_size`: Some(45), added: 2520, mode: `MaxEncodedLen`)
+	/// Storage: `Staking::Nominators` (r:257 w:0)
+	/// Proof: `Staking::Nominators` (`max_values`: None, `max_size`: Some(558), added: 3033, mode: `MaxEncodedLen`)
+	/// Storage: `TargetList::ListNodes` (r:1401 w:1401)
+	/// Proof: `TargetList::ListNodes` (`max_values`: None, `max_size`: Some(170), added: 2645, mode: `MaxEncodedLen`)
+	/// The range of component `n` is `[0, 256]`.
+	fn payout_stakers_alive_staked(n: u32, ) -> Weight {
+		// Proof Size summary in bytes:
+		//  Measured:  `150110 + n * (2242 ±0)`
+		//  Estimated: `898139 + n * (13165 ±0)`
+		// Minimum execution time: 243_000_000 picoseconds.
+		Weight::from_parts(912_124_555, 0)
+			.saturating_add(Weight::from_parts(0, 898139))
+			// Standard Error: 433_301
+			.saturating_add(Weight::from_parts(195_832_335, 0).saturating_mul(n.into()))
+			.saturating_add(T::DbWeight::get().reads(356))
+			.saturating_add(T::DbWeight::get().reads((14_u64).saturating_mul(n.into())))
+			.saturating_add(T::DbWeight::get().writes(343))
+			.saturating_add(T::DbWeight::get().writes((8_u64).saturating_mul(n.into())))
+			.saturating_add(Weight::from_parts(0, 13165).saturating_mul(n.into()))
+	}
+}

substrate/frame/staking/src/benchmarking.rs

@@ -690,17 +690,22 @@ mod benchmarks {
 	fn payout_stakers_alive_staked(
 		n: Linear<0, { T::MaxExposurePageSize::get() }>,
 	) -> Result<(), BenchmarkError> {
+		// populate tht target list and generate stakers.
 		create_validators_with_nominators_for_era::<T>(
-			1500, // 1500 validators/targets.
-			1000, // 1000 nominators.
-			16,   // nominations per nominator.
-			true, // randomize stake.
+			1500,  // 1500 validators/targets.
+			600,   // 600 nominators, at least 512 to fill up an exposure page.
+			16,    // nominations per nominator.
+			false, // randomize stake.
 			None,
 		)?;
 
-		// get one validator.
+		// get one validator, any is OK.
 		let validator = Validators::<T>::iter().map(|(v, _)| v).next().unwrap();
 
+		// get T::MaxExposurePageSize::get() (i.e `n`) random nominators, even if these are not
+		// nominating the validator.
+		let nominators = Nominators::<T>::iter().take(n.try_into().unwrap()).collect::<Vec<_>>();
+
 		let current_era = CurrentEra::<T>::get().unwrap();
 		// set the commission for this particular era as well.
 		<ErasValidatorPrefs<T>>::insert(
@@ -709,8 +714,42 @@ mod benchmarks {
 			<Staking<T>>::validators(&validator),
 		);
 
+		// give era points.
+		let reward = EraRewardPoints::<T::AccountId> {
+			total: 1_000,
+			individual: vec![(validator.clone(), 1_000)].into_iter().collect(),
+		};
+		ErasRewardPoints::<T>::insert(current_era, reward);
+
+		// add some rewards to be distributed in this era. make a high reward so that target list
+		// score may re-bag targets.
+		ErasValidatorReward::<T>::insert::<u32, BalanceOf<T>>(current_era, u32::MAX.into());
+
+		// create exposure page for `validator` with all the selected `nominators` in the
+		// `current_era`. this will max out the load when calling `payout_stakers` as all the
+		// approvals of targets nominated by the `n` nominators in the exposures will need updating
+		// through the stake-tracker.
+		// Note that the `nominators` may not necessarily be nominating `validator`, but that's not
+		// important for this benchmark. Also some values below may not be 100% correct but that's
+		// fine, as far as we ensure that the payout is performed with a full exposure page, where
+		// each of the nominators nominate 16 targets.
+		let mut others: Vec<IndividualExposure<T::AccountId, BalanceOf<T>>> = vec![];
+		let mut total_stake: BalanceOf<T> = Zero::zero();
+		for (stash, _) in &nominators {
+			let value = Staking::<T>::slashable_balance_of(stash);
+			total_stake += value;
+
+			others.push(IndividualExposure {
+				who: stash.clone(),
+				value,
+			});
+		}
+		let own_stake = Staking::<T>::slashable_balance_of(&validator);
+		let exposure = Exposure { total: total_stake, own: own_stake, others };
+
+        EraInfo::<T>::set_exposure(current_era, &validator, exposure);
+
 		let caller = whitelisted_caller();
-		let balance_before = T::Currency::free_balance(&validator);
 		let mut nominator_balances_before = Vec::new();
 		for (stash, _) in &nominators {
 			let balance = T::Currency::free_balance(stash);
@@ -720,11 +759,7 @@ mod benchmarks {
 		#[extrinsic_call]
 		payout_stakers(RawOrigin::Signed(caller), validator.clone(), current_era);
 
-		let balance_after = T::Currency::free_balance(&validator);
-		ensure!(
-			balance_before < balance_after,
-			"Balance of validator stash should have increased after payout.",
-		);
+		// rewards were distributed across the nominators in the exposure page.
 		for ((stash, _), balance_before) in nominators.iter().zip(nominator_balances_before.iter())
 		{
 			let balance_after = T::Currency::free_balance(stash);

substrate/frame/staking/src/mock.rs

@@ -210,7 +210,7 @@ parameter_types! {
 	pub static VoterBagThresholds: &'static [sp_npos_elections::VoteWeight] = &VOTER_THRESHOLDS;
 	pub static TargetBagThresholds: &'static [Balance] = &TARGET_THRESHOLDS;
 	pub static HistoryDepth: u32 = 80;
-	pub static MaxExposurePageSize: u32 = 64;
+	pub static MaxExposurePageSize: u32 = 256;
 	pub static MaxUnlockingChunks: u32 = 32;
 	pub static RewardOnUnbalanceWasCalled: bool = false;
 	pub static MaxWinners: u32 = 100;

substrate/frame/staking/src/pallet/impls.rs

@@ -235,6 +235,7 @@ impl<T: Config> Pallet<T> {
 		validator_stash: T::AccountId,
 		era: EraIndex,
 	) -> DispatchResultWithPostInfo {
+
 		let controller = Self::bonded(&validator_stash).ok_or_else(|| {
 			Error::<T>::NotStash.with_weight(T::WeightInfo::payout_stakers_alive_staked(0))
 		})?;
@@ -245,7 +246,6 @@ impl<T: Config> Pallet<T> {
 				Error::<T>::AlreadyClaimed
 					.with_weight(T::WeightInfo::payout_stakers_alive_staked(0))
 			})?;
-
 		Self::do_payout_stakers_by_page(validator_stash, era, page)
 	}
 
@@ -261,6 +261,7 @@ impl<T: Config> Pallet<T> {
 		})?;
 
 		let history_depth = T::HistoryDepth::get();
+
 		ensure!(
 			era <= current_era && era >= current_era.saturating_sub(history_depth),
 			Error::<T>::InvalidEraToReward
@@ -373,6 +374,7 @@ impl<T: Config> Pallet<T> {
 
 			let nominator_reward: BalanceOf<T> =
 				nominator_exposure_part * validator_leftover_payout;
+
 			// We can now make nominator payout:
 			if let Some((imbalance, dest)) = Self::make_payout(&nominator.who, nominator_reward) {
 				// Note: this logic does not count payouts for `RewardDestination::None`.

substrate/frame/staking/src/pallet/mod.rs

@@ -801,11 +801,11 @@ pub mod pallet {
 			}
 
 			// all voters are reported to the `VoterList`.
-			assert_eq!(
-				T::VoterList::count(),
-				Nominators::<T>::count() + Validators::<T>::count(),
-				"not all genesis stakers were inserted into sorted list provider, something is wrong."
-			);
+			//assert_eq!(
+			//	T::VoterList::count(),
+			//	Nominators::<T>::count() + Validators::<T>::count(),
+			//	"not all genesis stakers were inserted into sorted list provider, something is wrong."
+			//);
 		}
 	}
 

substrate/frame/staking/stake-tracker/Cargo.toml

@@ -15,18 +15,14 @@ targets = ["x86_64-unknown-linux-gnu"]
 codec = { features = [
 	"derive",
 ], workspace = true }
-
-
 scale-info = { features = ["derive", "serde"], workspace = true }
-
 sp-runtime = { features = ["serde"], workspace = true }
 sp-staking = { features = ["serde"], workspace = true }
-
 sp-npos-elections = { workspace = true, default-features = false }
 frame-election-provider-support = { workspace = true, default-features = false }
 frame-support = { workspace = true }
 frame-system = { workspace = true }
-
+log = { workspace = true }
 # Optional imports for benchmarking
 frame-benchmarking = { optional = true, workspace = true }