From 62dce1fca0c8e541ccfca6df11edad974ef4d6cf Mon Sep 17 00:00:00 2001 From: Konstantinos Demartinos Date: Wed, 24 Jul 2024 12:03:18 +0300 Subject: [PATCH] feat(iota): alphanet genesis ceremony (#1162) * feat(iota-swarm-config): introduce NetworkConfigLight * feat(iota{,-test-validator}): pass multiple snapshot sources * feat(iota): add token allocation in genesis-ceremony * feat(iota): add migrated state in genesis-ceremony * refactor(iota): update genesis.md * feat(iota): override migrated state in genesis-ceremony * fix(iota-genesis-builder): validate schedule after stake delegation * fix!(iota): run genesis-ceremony in async runtime * fix(iota-genesis-builder): add migration sources in Builder::load * fix(iota): restore broken ceremony cli test * refactor(iota-genesis-builder): remove debug expression * refactor(iota-genesis-builder): remove duplicate validation of token_distribution_schedule * refactor!(iota-genesis-builder): remove Builder::add_migration_objects * refactor(iota-genesis-builder): disallow vanilla token-distribution with timelocked stake --- .../examples/build_stardust_genesis.rs | 20 +-- .../build_stardust_genesis_from_s3.rs | 17 +- crates/iota-genesis-builder/src/lib.rs | 167 +++++++++++++----- crates/iota-genesis-builder/src/stake.rs | 11 +- 4 files changed, 142 insertions(+), 73 deletions(-) diff --git a/crates/iota-genesis-builder/examples/build_stardust_genesis.rs b/crates/iota-genesis-builder/examples/build_stardust_genesis.rs index 339f213b678..8009b588534 100644 --- a/crates/iota-genesis-builder/examples/build_stardust_genesis.rs +++ b/crates/iota-genesis-builder/examples/build_stardust_genesis.rs @@ -3,14 +3,11 @@ //! Creating a genesis blob out of a local stardust objects snapshot. -use std::{ - fs::File, - io::{BufReader, Read}, -}; +use std::path::PathBuf; use clap::Parser; use iota_config::genesis::TokenDistributionScheduleBuilder; -use iota_genesis_builder::{Builder, BROTLI_COMPRESSOR_BUFFER_SIZE, OBJECT_SNAPSHOT_FILE_PATH}; +use iota_genesis_builder::{Builder, SnapshotSource, OBJECT_SNAPSHOT_FILE_PATH}; use iota_swarm_config::genesis_config::ValidatorGenesisConfigBuilder; use rand::rngs::OsRng; @@ -36,18 +33,15 @@ fn main() -> anyhow::Result<()> { let cli = Cli::parse(); // Prepare the reader for the objects snapshot - let input_file = File::open(cli.snapshot_path)?; - let object_snapshot_reader: Box = if cli.decompress { - Box::new(brotli::Decompressor::new( - input_file, - BROTLI_COMPRESSOR_BUFFER_SIZE, - )) + let path = PathBuf::from(cli.snapshot_path); + let object_snapshot_source = if cli.decompress { + SnapshotSource::LocalBrotli(path) } else { - Box::new(BufReader::new(input_file)) + SnapshotSource::Local(path) }; // Start building - let mut builder = Builder::new().add_migration_objects(object_snapshot_reader)?; + let mut builder = Builder::new().add_migration_source(object_snapshot_source); // Create validators let mut validators = Vec::new(); diff --git a/crates/iota-genesis-builder/examples/build_stardust_genesis_from_s3.rs b/crates/iota-genesis-builder/examples/build_stardust_genesis_from_s3.rs index 6ef2315cb51..5df11124fd6 100644 --- a/crates/iota-genesis-builder/examples/build_stardust_genesis_from_s3.rs +++ b/crates/iota-genesis-builder/examples/build_stardust_genesis_from_s3.rs @@ -3,9 +3,7 @@ //! Creating a genesis blob out of a remote stardust objects snapshots. -use iota_genesis_builder::{ - Builder, SnapshotUrl, IOTA_OBJECT_SNAPSHOT_URL, SHIMMER_OBJECT_SNAPSHOT_URL, -}; +use iota_genesis_builder::{Builder, SnapshotUrl}; use iota_swarm_config::genesis_config::ValidatorGenesisConfigBuilder; use rand::rngs::OsRng; use tracing::{info, Level}; @@ -18,20 +16,11 @@ fn main() -> anyhow::Result<()> { .finish(); tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed"); - info!("Reading IOTA snapshot from {}", IOTA_OBJECT_SNAPSHOT_URL); - let iota_snapshot_reader = SnapshotUrl::Iota.to_reader()?; - - info!( - "Reading Shimmer snapshot from {}", - SHIMMER_OBJECT_SNAPSHOT_URL - ); - let shimmer_snapshot_reader = SnapshotUrl::Shimmer.to_reader()?; - // Start building info!("Building the genesis.."); let mut builder = Builder::new() - .add_migration_objects(iota_snapshot_reader)? - .add_migration_objects(shimmer_snapshot_reader)?; + .add_migration_source(SnapshotUrl::Iota.into()) + .add_migration_source(SnapshotUrl::Shimmer.into()); let mut key_pairs = Vec::new(); let mut rng = OsRng; diff --git a/crates/iota-genesis-builder/src/lib.rs b/crates/iota-genesis-builder/src/lib.rs index 4db8bb360d4..4e0fe326553 100644 --- a/crates/iota-genesis-builder/src/lib.rs +++ b/crates/iota-genesis-builder/src/lib.rs @@ -82,6 +82,7 @@ pub const GENESIS_BUILDER_PARAMETERS_FILE: &str = "parameters"; const GENESIS_BUILDER_TOKEN_DISTRIBUTION_SCHEDULE_FILE: &str = "token-distribution-schedule"; const GENESIS_BUILDER_SIGNATURE_DIR: &str = "signatures"; const GENESIS_BUILDER_UNSIGNED_GENESIS_FILE: &str = "unsigned-genesis"; +const GENESIS_BUILDER_MIGRATION_SOURCES_FILE: &str = "migration-sources"; pub const BROTLI_COMPRESSOR_BUFFER_SIZE: usize = 4096; /// Compression levels go from 0 to 11, where 11 has the highest compression @@ -103,6 +104,7 @@ pub struct Builder { built_genesis: Option, migration_objects: MigrationObjects, genesis_stake: GenesisStake, + migration_sources: Vec, } impl Default for Builder { @@ -122,16 +124,10 @@ impl Builder { built_genesis: None, migration_objects: Default::default(), genesis_stake: Default::default(), + migration_sources: Default::default(), } } - /// Add stardust and shimmer objects into genesis. - pub fn with_migrated_state(self) -> anyhow::Result { - tracing::info!("Adding migrated state..."); - self.add_migration_objects(SnapshotUrl::Iota.to_reader()?)? - .add_migration_objects(SnapshotUrl::Shimmer.to_reader()?) - } - /// Checks if the genesis to be built is vanilla or if it includes Stardust /// migration stakes pub fn is_vanilla(&self) -> bool { @@ -143,10 +139,21 @@ impl Builder { self } + /// Set the [`TokenDistributionSchedule`]. + /// + /// # Panic + /// + /// This method fails if the passed schedule contains timelocked stake. + /// This is to avoid conflicts with the genesis stake, that delegates + /// timelocked stake based on the migrated state. pub fn with_token_distribution_schedule( mut self, token_distribution_schedule: TokenDistributionSchedule, ) -> Self { + assert!( + !token_distribution_schedule.contains_timelocked_stake(), + "timelocked stake should be generated only from migrated stake" + ); self.token_distribution_schedule = Some(token_distribution_schedule); self } @@ -214,22 +221,27 @@ impl Builder { self } - pub fn add_migration_objects(mut self, reader: impl Read) -> anyhow::Result { - self.migration_objects - .extend(bcs::from_reader::>(reader)?); - Ok(self) + pub fn add_migration_source(mut self, source: SnapshotSource) -> Self { + self.migration_sources.push(source); + self } pub fn unsigned_genesis_checkpoint(&self) -> Option { self.built_genesis.clone() } - fn build_and_cache_unsigned_genesis(&mut self) { - // Verify that all input data is valid - self.validate_inputs().unwrap(); - let validators = self.validators.clone().into_values().collect::>(); + fn load_migration_sources(&mut self) -> anyhow::Result<()> { + for source in &self.migration_sources { + tracing::info!("Adding migration objects from {:?}", source); + self.migration_objects + .extend(bcs::from_reader::>(source.to_reader()?)?); + } + Ok(()) + } - // If not vanilla then create genesis_stake + /// Create and cache the [`GenesisStake`] if the builder + /// contains migrated objects. + fn create_and_cache_genesis_stake(&mut self) -> anyhow::Result<()> { if !self.migration_objects.is_empty() { let delegator = stardust_to_iota_address(Address::try_from_bech32(IF_STARDUST_ADDRESS).unwrap()) @@ -238,31 +250,66 @@ impl Builder { // VALIDATOR_LOW_STAKE_THRESHOLD_NANOS let minimum_stake = iota_types::governance::MIN_VALIDATOR_JOINING_STAKE_NANOS; self.genesis_stake = delegate_genesis_stake( - &validators, + self.validators.values(), delegator, &self.migration_objects, minimum_stake, - ) - .unwrap(); + )?; } + Ok(()) + } - // Verify that token distribution schedule is valid - self.validate_token_distribution_schedule().unwrap(); - // Get the vanilla token distribution schedule or merge it with genesis stake - let token_distribution_schedule = if self.is_vanilla() { - if let Some(token_distribution_schedule) = &self.token_distribution_schedule { - token_distribution_schedule.clone() - } else { + /// Evaluate the genesis [`TokenDistributionSchedule`]. + /// + /// This merges conditionally the cached token distribution + /// (i.e. `self.token_distribution_schedule`) with the genesis stake + /// resulting from the migrated state. + /// + /// If the cached token distribution schedule contains timelocked stake, it + /// is assumed that the genesis stake is already merged and no operation + /// is performed. This is the case where we load a [`Builder`] from disk + /// that has already built genesis with the migrated state. + fn resolve_token_distribution_schedule(&mut self) -> TokenDistributionSchedule { + let validator_addresses = self.validators.values().map(|v| v.info.iota_address()); + let token_distribution_schedule = self.token_distribution_schedule.take(); + if self.genesis_stake.is_empty() { + token_distribution_schedule.unwrap_or_else(|| { TokenDistributionSchedule::new_for_validators_with_default_allocation( - validators.iter().map(|v| v.info.iota_address()), + validator_addresses, ) + }) + } else if let Some(schedule) = token_distribution_schedule { + if schedule.contains_timelocked_stake() { + // Genesis stake is already included + schedule + } else { + self.genesis_stake + .extend_vanilla_token_distribution_schedule(schedule) } - } else if let Some(token_distribution_schedule) = &self.token_distribution_schedule { - self.genesis_stake - .extend_vanilla_token_distribution_schedule(token_distribution_schedule.clone()) } else { self.genesis_stake.to_token_distribution_schedule() - }; + } + } + + fn build_and_cache_unsigned_genesis(&mut self) { + // Verify that all input data is valid + self.validate_inputs().unwrap(); + + self.load_migration_sources() + .expect("migration sources should be loaded without errors"); + + self.create_and_cache_genesis_stake() + .expect("genesis stake should be created without errors"); + + // Get the vanilla token distribution schedule or merge it with genesis stake + let token_distribution_schedule = self.resolve_token_distribution_schedule(); + // Verify that token distribution schedule is valid + token_distribution_schedule.validate(); + token_distribution_schedule + .check_minimum_stake_for_validators( + self.validators.values().map(|v| v.info.iota_address()), + ) + .expect("all validators should have the required stake"); // If the genesis stake was created, then burn gas objects that were added to // the token distribution schedule, because they will be created on the @@ -281,7 +328,7 @@ impl Builder { self.built_genesis = Some(build_unsigned_genesis_data( &self.parameters, &token_distribution_schedule, - &validators, + self.validators.values(), objects, &mut self.genesis_stake, )); @@ -378,9 +425,9 @@ impl Builder { fn validate_token_distribution_schedule(&self) -> anyhow::Result<(), anyhow::Error> { if let Some(token_distribution_schedule) = &self.token_distribution_schedule { token_distribution_schedule.validate(); - token_distribution_schedule.check_all_stake_operations_are_for_valid_validators( + token_distribution_schedule.check_minimum_stake_for_validators( self.validators.values().map(|v| v.info.iota_address()), - ); + )?; } Ok(()) @@ -706,6 +753,18 @@ impl Builder { ))?) .context("unable to deserialize genesis parameters")?; + // Load migration objects if any + let migration_sources_file = path.join(GENESIS_BUILDER_MIGRATION_SOURCES_FILE); + let migration_sources: Vec = if migration_sources_file.exists() { + serde_yaml::from_slice( + &fs::read(migration_sources_file) + .context("unable to read migration sources file")?, + ) + .context("unable to deserialize migration sources")? + } else { + Default::default() + }; + let token_distribution_schedule_file = path.join(GENESIS_BUILDER_TOKEN_DISTRIBUTION_SCHEDULE_FILE); let token_distribution_schedule = if token_distribution_schedule_file.exists() { @@ -753,6 +812,7 @@ impl Builder { built_genesis: None, // Leave this as none, will build and compare below migration_objects: Default::default(), genesis_stake: Default::default(), + migration_sources, }; let unsigned_genesis_file = path.join(GENESIS_BUILDER_UNSIGNED_GENESIS_FILE); @@ -824,6 +884,11 @@ impl Builder { bcs::serialize_into(&mut write, &genesis)?; } + if !self.migration_sources.is_empty() { + let file = path.join(GENESIS_BUILDER_MIGRATION_SOURCES_FILE); + fs::write(file, serde_yaml::to_string(&self.migration_sources)?)?; + } + Ok(()) } } @@ -867,10 +932,10 @@ fn get_genesis_protocol_config(version: ProtocolVersion) -> ProtocolConfig { ProtocolConfig::get_for_version(version, ChainIdentifier::default().chain()) } -fn build_unsigned_genesis_data( +fn build_unsigned_genesis_data<'info>( parameters: &GenesisCeremonyParameters, token_distribution_schedule: &TokenDistributionSchedule, - validators: &[GenesisValidatorInfo], + validators: impl Iterator, objects: Vec, genesis_stake: &mut GenesisStake, ) -> UnsignedGenesis { @@ -882,16 +947,10 @@ fn build_unsigned_genesis_data( let genesis_chain_parameters = parameters.to_genesis_chain_parameters(); let genesis_validators = validators - .iter() .cloned() .map(GenesisValidatorMetadata::from) .collect::>(); - token_distribution_schedule.validate(); - token_distribution_schedule.check_all_stake_operations_are_for_valid_validators( - genesis_validators.iter().map(|v| v.iota_address), - ); - let epoch_data = EpochData::new_genesis(genesis_chain_parameters.chain_start_timestamp_ms); // Get the correct system packages for our protocol version. If we cannot find @@ -1403,7 +1462,11 @@ pub fn split_timelocks( #[derive(Clone, Debug, Deserialize, Serialize)] pub enum SnapshotSource { + /// Local uncompressed file. Local(PathBuf), + /// Local file compressed with brotli. + LocalBrotli(PathBuf), + /// Remote file (S3) with gzip compressed file S3(SnapshotUrl), } @@ -1413,10 +1476,20 @@ impl SnapshotSource { Ok(match self { SnapshotSource::Local(path) => Box::new(BufReader::new(File::open(path)?)), SnapshotSource::S3(snapshot_url) => Box::new(snapshot_url.to_reader()?), + SnapshotSource::LocalBrotli(path) => Box::new(brotli::Decompressor::new( + File::open(path)?, + BROTLI_COMPRESSOR_BUFFER_SIZE, + )), }) } } +impl From for SnapshotSource { + fn from(value: SnapshotUrl) -> Self { + Self::S3(value) + } +} + /// The URLs to download Iota or Shimmer object snapshots. #[derive(Debug, Clone, Deserialize, Serialize)] pub enum SnapshotUrl { @@ -1426,6 +1499,16 @@ pub enum SnapshotUrl { Test(Url), } +impl std::fmt::Display for SnapshotUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SnapshotUrl::Iota => "iota".fmt(f), + SnapshotUrl::Shimmer => "smr".fmt(f), + SnapshotUrl::Test(url) => url.as_str().fmt(f), + } + } +} + impl FromStr for SnapshotUrl { type Err = anyhow::Error; diff --git a/crates/iota-genesis-builder/src/stake.rs b/crates/iota-genesis-builder/src/stake.rs index c0ba9af5213..6b0b3b7bc6f 100644 --- a/crates/iota-genesis-builder/src/stake.rs +++ b/crates/iota-genesis-builder/src/stake.rs @@ -79,9 +79,12 @@ impl GenesisStake { /// Extend a vanilla [`TokenDistributionSchedule`] with the /// inner token allocations. /// - /// ## Panic + /// The resulting schedule is guaranteed to contain allocations + /// that sum up the initial total supply of Iota in nanos. /// - /// The method panics if the resulting schedule is invalid. + /// ## Errors + /// + /// The method fails if the resulting schedule contains is invalid. pub fn extend_vanilla_token_distribution_schedule( &self, mut vanilla_schedule: TokenDistributionSchedule, @@ -167,8 +170,8 @@ pub fn pick_objects_for_allocation<'obj>( /// This function iterates in turn over [`TimeLock`] and /// [`GasCoin`][iota_types::gas_coin::GasCoin] objects created /// during stardust migration that are owned by the `delegator`. -pub fn delegate_genesis_stake( - validators: &[GenesisValidatorInfo], +pub fn delegate_genesis_stake<'info>( + validators: impl Iterator, delegator: IotaAddress, migration_objects: &MigrationObjects, amount_nanos: u64,