Merge branch 'main' into bf-view-timeout

Cargo.toml

@@ -3,6 +3,7 @@ members = [
   # Binaries
   "core/bin/block_reverter",
   "core/bin/contract-verifier",
+  "core/bin/custom_genesis_export",
   "core/bin/external_node",
   "core/bin/merkle_tree_consistency_checker",
   "core/bin/snapshots_creator",

core/bin/custom_genesis_export/.gitignore

@@ -0,0 +1 @@
+*.bin

core/bin/custom_genesis_export/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "custom_genesis_export"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+[dependencies]
+clap = { workspace = true, features = ["derive"] }
+futures.workspace = true
+sqlx = { workspace = true, features = [
+    "runtime-tokio",
+    "tls-native-tls",
+    "macros",
+    "postgres",
+] }
+tokio = { workspace = true, features = ["full"] }
+
+zksync_types.workspace = true
+zksync_node_genesis.workspace = true
+zksync_contracts.workspace = true
+zksync_core_leftovers.workspace = true
+zksync_protobuf_config.workspace = true
+zksync_dal.workspace = true
+anyhow.workspace = true
+bincode.workspace = true

core/bin/custom_genesis_export/README.md

@@ -0,0 +1,60 @@
+# Custom Genesis Export
+
+The `custom_genesis_export` tool allows exporting a state from a zkSync PostgreSQL database in a format that can be
+included as a custom genesis state for a new chain.
+
+This is particularly useful in data migration scenarios where a large existing state needs to be applied to a newly
+created chain.
+
+A typical workflow could be:
+
+- Run a chain locally, not connected to the real L1, and add all required data to it.
+- Export the data using the `custom_genesis_export` tool.
+- Create a new chain connected to the real ecosystem using the exported data.
+
+## How it works
+
+The tool exports all entries from `storage_logs`, and `factory_deps`, except those related to the system context. The
+data is then written to a binary file using the Rust standard library following a simple serialisation format.
+
+`custom_genesis_export` can be built using the following command:
+
+```shell
+cargo build --release -p custom_genesis_export
+```
+
+And then executed using the following command, where:
+
+- `database-url` is the URL of the PostgreSQL database.
+- `genesis-config-path` is the path to the `genesis.yaml` configuration file, used to set up a new chain (located in the
+  `file_based` directory).
+- `output-path` is the path to the generated binary output file.
+
+```shell
+custom_genesis_export --database-url=postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_validium --genesis-config-path=/Users/ischasny/Dev/zksync-era/etc/env/file_based/genesis.yaml --output-path=export.bin
+```
+
+> Please make sure that the database is not written into before running data export.
+
+After the export is completed, the tool will make the following updates to the `genesis.yaml` file in-place:
+
+- Update `genesis_root_hash`, `rollup_last_leaf_index`, and `genesis_commitment` to match the contents of the export
+  file.
+- Add a `custom_genesis_state_path` property pointing to the data export.
+
+The modified genesis file can be used to bootstrap an ecosystem or initialize new chains. The data export will be
+automatically recognized by the server during the execution of `zkstack ecosystem init ...` and
+`zkstack chain create ...` commands.
+
+### Running considerations
+
+- All chains within the same ecosystem must be bootstrapped from the same genesis state. This is enforced at the
+  protocol level. If two chains require different states, this can only be achieved by bringing the chain into the
+  ecosystem through governance voting.
+  - If a chain is added to the ecosystem via a vote, ensure no assets are minted on the old bridge, as this would create
+    discrepancies with the new one. One should set gas prices to zero when generating a state to account for that.
+- To calculate genesis parameters, the tool must load all VM logs into RAM. This is due to implementation specifics. For
+  larger states, ensure the VM has sufficient RAM capacity.
+- After the import, block numbers for all VM logs will be reset to zero - if the imported data has been indexed based on
+  block number, such indexes will break.
+- External Nodes will have to be bootstrapped from data snapshot (i.e. genesis can't be generated locally).

core/bin/custom_genesis_export/src/main.rs

@@ -0,0 +1,141 @@
+extern crate core;
+
+use std::{fs, fs::File, io::BufWriter, path::PathBuf, str::FromStr};
+
+use clap::Parser;
+use zksync_contracts::BaseSystemContractsHashes;
+use zksync_core_leftovers::temp_config_store::read_yaml_repr;
+use zksync_dal::{custom_genesis_export_dal::GenesisState, ConnectionPool, Core, CoreDal};
+use zksync_node_genesis::{make_genesis_batch_params, utils::get_deduped_log_queries};
+use zksync_protobuf_config::encode_yaml_repr;
+use zksync_types::{url::SensitiveUrl, StorageLog};
+
+#[derive(Debug, Parser)]
+#[command(name = "Custom genesis export tool", author = "Matter Labs")]
+struct Args {
+    /// PostgreSQL connection string for the database to export.
+    #[arg(short, long)]
+    database_url: Option<String>,
+
+    /// Output file path.
+    #[arg(short, long, default_value = "genesis_export.bin")]
+    output_path: PathBuf,
+
+    /// Path to the genesis.yaml
+    #[arg(short, long)]
+    genesis_config_path: PathBuf,
+}
+
+/// The `custom_genesis_export` tool allows exporting storage logs and factory dependencies
+/// from the ZKSync PostgreSQL database in a way that they can be used as a custom genesis state for a new chain.
+///
+/// Inputs:
+///     * `database_url` - URL to the PostgreSQL database.
+///     * `output` - Path to the output file.
+///     * `genesis_config_path` - Path to the `genesis.yaml` configuration file, which will be used to set up a new chain (located in the `file_based` directory).
+///
+/// Given the inputs above, `custom_genesis_export` will perform the following:
+///     * Read storage logs, and factory dependencies; filter out those related to the system context,
+///       and save the remaining data to the output file.
+///     * Calculate the new `genesis_root_hash`, `rollup_last_leaf_index`, and `genesis_commitment`, then update these
+///       in-place in the provided `genesis.yaml`. Additionally, the tool will add a `custom_genesis_state_path` property
+///       pointing to the genesis export.
+///
+/// Note: To calculate the new genesis parameters, the current implementation requires loading all storage logs
+/// into RAM. This is necessary due to the specific sorting and filtering that need to be applied.
+/// For larger states, keep this in mind and ensure you have a machine with sufficient RAM.
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let args = Args::parse();
+
+    let mut out = BufWriter::new(File::create(&args.output_path)?);
+
+    println!(
+        "Export file: {}",
+        args.output_path.canonicalize()?.display(),
+    );
+
+    println!("Connecting to source database...");
+
+    let db_url = args.database_url.or_else(|| std::env::var("DATABASE_URL").ok()).expect("Specify the database connection string in either a CLI argument or in the DATABASE_URL environment variable.");
+    // we need only 1 DB connection at most for data export
+    let connection_pool_builder =
+        ConnectionPool::<Core>::builder(SensitiveUrl::from_str(db_url.as_str())?, 1);
+    let connection_pool = connection_pool_builder.build().await?;
+
+    let mut storage = connection_pool.connection().await?;
+    let mut transaction = storage.start_transaction().await?;
+
+    println!("Connected to source database.");
+
+    let storage_logs = transaction
+        .custom_genesis_export_dal()
+        .get_storage_logs()
+        .await?;
+    let factory_deps = transaction
+        .custom_genesis_export_dal()
+        .get_factory_deps()
+        .await?;
+
+    transaction.commit().await?;
+
+    println!(
+        "Loaded {} storage logs {} factory deps from source database.",
+        storage_logs.len(),
+        factory_deps.len()
+    );
+
+    let storage_logs_for_genesis: Vec<StorageLog> =
+        storage_logs.iter().map(StorageLog::from).collect();
+
+    bincode::serialize_into(
+        &mut out,
+        &GenesisState {
+            storage_logs,
+            factory_deps,
+        },
+    )?;
+
+    println!(
+        "Saved genesis state into the file {}.",
+        args.output_path.display()
+    );
+    println!("Calculating new genesis parameters");
+
+    let mut genesis_config = read_yaml_repr::<zksync_protobuf_config::proto::genesis::Genesis>(
+        &args.genesis_config_path,
+    )?;
+
+    let base_system_contract_hashes = BaseSystemContractsHashes {
+        bootloader: genesis_config
+            .bootloader_hash
+            .ok_or(anyhow::anyhow!("No bootloader_hash specified"))?,
+        default_aa: genesis_config
+            .default_aa_hash
+            .ok_or(anyhow::anyhow!("No default_aa_hash specified"))?,
+        evm_emulator: genesis_config.evm_emulator_hash,
+    };
+
+    let (genesis_batch_params, _) = make_genesis_batch_params(
+        get_deduped_log_queries(&storage_logs_for_genesis),
+        base_system_contract_hashes,
+        genesis_config
+            .protocol_version
+            .ok_or(anyhow::anyhow!("No bootloader_hash specified"))?
+            .minor,
+    );
+
+    genesis_config.genesis_root_hash = Some(genesis_batch_params.root_hash);
+    genesis_config.rollup_last_leaf_index = Some(genesis_batch_params.rollup_last_leaf_index);
+    genesis_config.genesis_commitment = Some(genesis_batch_params.commitment);
+    genesis_config.custom_genesis_state_path =
+        args.output_path.canonicalize()?.to_str().map(String::from);
+
+    let bytes =
+        encode_yaml_repr::<zksync_protobuf_config::proto::genesis::Genesis>(&genesis_config)?;
+    fs::write(&args.genesis_config_path, &bytes)?;
+
+    println!("Done.");
+
+    Ok(())
+}

core/lib/config/src/configs/genesis.rs

@@ -32,6 +32,7 @@ pub struct GenesisConfig {
     pub fee_account: Address,
     pub dummy_verifier: bool,
     pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode,
+    pub custom_genesis_state_path: Option<String>,
 }
 
 impl GenesisConfig {
@@ -60,6 +61,7 @@ impl GenesisConfig {
             l2_chain_id: L2ChainId::default(),
             dummy_verifier: false,
             l1_batch_commit_data_generator_mode: L1BatchCommitmentMode::Rollup,
+            custom_genesis_state_path: None,
         }
     }
 }

core/lib/config/src/testonly.rs

@@ -746,6 +746,7 @@ impl Distribution<configs::GenesisConfig> for EncodeDist {
                 0 => L1BatchCommitmentMode::Rollup,
                 _ => L1BatchCommitmentMode::Validium,
             },
+            custom_genesis_state_path: None,
         }
     }
 }