diff --git a/.github/workflows/new-build-core-template.yml b/.github/workflows/new-build-core-template.yml index ab3854733e62..557d8455a31d 100644 --- a/.github/workflows/new-build-core-template.yml +++ b/.github/workflows/new-build-core-template.yml @@ -197,10 +197,8 @@ jobs: shell: bash run: | echo PLATFORM=$(echo ${{ matrix.platforms }} | tr '/' '-') >> $GITHUB_ENV - echo IMAGE_TAG_SHA=$(git rev-parse --short HEAD) >> $GITHUB_ENV - # Support for custom tag suffix if [ -n "${{ inputs.image_tag_suffix }}" ]; then - echo IMAGE_TAG_SHA_TS="${{ inputs.image_tag_suffix }}" >> $GITHUB_ENV + echo IMAGE_TAG_SHA_TS="${{ env.IMAGE_TAG_SUFFIX }}" >> $GITHUB_ENV else echo IMAGE_TAG_SHA_TS=$(git rev-parse --short HEAD)-$(date +%s) >> $GITHUB_ENV fi diff --git a/Cargo.lock b/Cargo.lock index c92a4690e221..3df58a9634e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,6 +256,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "ark-ff" version = "0.3.0" @@ -4402,6 +4408,16 @@ dependencies = [ "serde", ] +[[package]] +name = "iri-string" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0f0a572e8ffe56e2ff4f769f32ffe919282c3916799f8b68688b6030063bea" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -5718,6 +5734,46 @@ dependencies = [ "memchr", ] +[[package]] +name = "octocrab" +version = "0.41.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2dfd11f6efbd39491d71a3864496f0b6f45e2d01b73b26c55d631c4e0dafaef" +dependencies = [ + "arc-swap", + "async-trait", + "base64 0.22.1", + "bytes", + "cfg-if", + "chrono", + "either", + "futures 0.3.31", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.0", + "hyper-rustls 0.27.3", + "hyper-timeout 0.5.1", + "hyper-util", + "jsonwebtoken", + "once_cell", + "percent-encoding", + "pin-project", + "secrecy 0.10.3", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "snafu", + "tokio", + "tower 0.5.1", + "tower-http 0.6.1", + "tracing", + "url", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -7779,6 +7835,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -8442,6 +8507,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck 0.5.0", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.85", +] + [[package]] name = "snapshots_creator" version = "0.1.0" @@ -9037,7 +9123,7 @@ dependencies = [ "pbkdf2", "regex", "schnorrkel", - "secrecy", + "secrecy 0.8.0", "sha2 0.10.8", "sp-core-hashing", "subxt", @@ -9714,6 +9800,7 @@ dependencies = [ "pin-project-lite", "sync_wrapper 0.1.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -9739,6 +9826,25 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower 0.5.1", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -11017,7 +11123,7 @@ dependencies = [ "ethabi", "hex", "num_enum 0.7.3", - "secrecy", + "secrecy 0.8.0", "serde", "serde_json", "serde_with", @@ -11147,7 +11253,7 @@ version = "0.1.0" dependencies = [ "anyhow", "rand 0.8.5", - "secrecy", + "secrecy 0.8.0", "serde", "serde_json", "tracing", @@ -11350,7 +11456,7 @@ dependencies = [ "test-casing", "tokio", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "vise", "zksync_dal", @@ -11383,8 +11489,12 @@ dependencies = [ "assert_matches", "chrono", "ethabi", + "futures-util", "hex", + "octocrab", "regex", + "reqwest 0.12.9", + "rustls 0.23.16", "semver 1.0.23", "serde", "serde_json", @@ -12019,7 +12129,7 @@ dependencies = [ "thread_local", "tokio", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "vise", "zk_evm 0.150.7", @@ -12053,7 +12163,7 @@ dependencies = [ "anyhow", "async-trait", "rand 0.8.5", - "secrecy", + "secrecy 0.8.0", "semver 1.0.23", "tempfile", "test-casing", @@ -12336,10 +12446,9 @@ dependencies = [ "serde_json", "tokio", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "vise", - "zksync_basic_types", "zksync_config", "zksync_contracts", "zksync_dal", @@ -12396,7 +12505,7 @@ dependencies = [ "hex", "prost 0.12.6", "rand 0.8.5", - "secrecy", + "secrecy 0.8.0", "serde_json", "serde_yaml", "tracing", diff --git a/Cargo.toml b/Cargo.toml index af7620a5216f..44a00196fb76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ ethabi = "18.0.0" flate2 = "1.0.28" fraction = "0.15.3" futures = "0.3" +futures-util = "0.3" glob = "0.3" google-cloud-auth = "0.16.0" google-cloud-storage = "0.20.0" @@ -142,6 +143,7 @@ mini-moka = "0.10.0" num = "0.4.0" num_cpus = "1.13" num_enum = "0.7.2" +octocrab = "0.41" once_cell = "1" opentelemetry = "0.24.0" opentelemetry_sdk = "0.24.0" diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 88f25256c40d..ab86c147977d 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -55,7 +55,9 @@ async fn main() -> anyhow::Result<()> { let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool) .await .context("failed initializing contract verifier")?; + let update_task = contract_verifier.sync_compiler_versions_task(); let tasks = vec![ + tokio::spawn(update_task), tokio::spawn(contract_verifier.run(stop_receiver.clone(), opt.jobs_number)), tokio::spawn( PrometheusExporterConfig::pull(prometheus_config.listener_port).run(stop_receiver), diff --git a/core/lib/config/src/configs/proof_data_handler.rs b/core/lib/config/src/configs/proof_data_handler.rs index 1d8703df51aa..443d602b8126 100644 --- a/core/lib/config/src/configs/proof_data_handler.rs +++ b/core/lib/config/src/configs/proof_data_handler.rs @@ -9,9 +9,12 @@ pub struct TeeConfig { pub tee_support: bool, /// All batches before this one are considered to be processed. pub first_tee_processed_batch: L1BatchNumber, - /// Timeout in seconds for retrying TEE proof generation if it fails. Retries continue - /// indefinitely until successful. + /// Timeout in seconds for retrying the preparation of input for TEE proof generation if it + /// previously failed (e.g., due to a transient network issue) or if it was picked by a TEE + /// prover but the TEE proof was not submitted within that time. pub tee_proof_generation_timeout_in_secs: u16, + /// Timeout in hours after which a batch will be permanently ignored if repeated retries failed. + pub tee_batch_permanently_ignored_timeout_in_hours: u16, } impl Default for TeeConfig { @@ -21,6 +24,8 @@ impl Default for TeeConfig { first_tee_processed_batch: Self::default_first_tee_processed_batch(), tee_proof_generation_timeout_in_secs: Self::default_tee_proof_generation_timeout_in_secs(), + tee_batch_permanently_ignored_timeout_in_hours: + Self::default_tee_batch_permanently_ignored_timeout_in_hours(), } } } @@ -35,12 +40,20 @@ impl TeeConfig { } pub fn default_tee_proof_generation_timeout_in_secs() -> u16 { - 600 + 60 + } + + pub fn default_tee_batch_permanently_ignored_timeout_in_hours() -> u16 { + 10 * 24 } pub fn tee_proof_generation_timeout(&self) -> Duration { Duration::from_secs(self.tee_proof_generation_timeout_in_secs.into()) } + + pub fn tee_batch_permanently_ignored_timeout(&self) -> Duration { + Duration::from_secs(3600 * u64::from(self.tee_batch_permanently_ignored_timeout_in_hours)) + } } #[derive(Debug, Deserialize, Clone, PartialEq)] diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index c24d47f27b33..8a9e96c96a5e 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -681,6 +681,7 @@ impl Distribution for EncodeDist { tee_support: self.sample(rng), first_tee_processed_batch: L1BatchNumber(rng.gen()), tee_proof_generation_timeout_in_secs: self.sample(rng), + tee_batch_permanently_ignored_timeout_in_hours: self.sample(rng), }, } } diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 6ccd6422d7da..c2cf97826561 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -28,8 +28,12 @@ hex.workspace = true serde = { workspace = true, features = ["derive"] } tempfile.workspace = true regex.workspace = true +reqwest.workspace = true tracing.workspace = true semver.workspace = true +octocrab = { workspace = true, features = ["stream"] } +futures-util.workspace = true +rustls.workspace = true [dev-dependencies] zksync_node_test_utils.workspace = true diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 7dc5d47d4562..284d9921a674 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; +use resolver::{GitHubCompilerResolver, ResolverMultiplexer}; use tokio::time; use zksync_dal::{contract_verification_dal::DeployedContractData, ConnectionPool, Core, CoreDal}; use zksync_queued_job_processor::{async_trait, JobProcessor}; @@ -121,12 +122,20 @@ impl ContractVerifier { compilation_timeout: Duration, connection_pool: ConnectionPool, ) -> anyhow::Result { - Self::with_resolver( - compilation_timeout, - connection_pool, - Arc::::default(), - ) - .await + let env_resolver = Arc::::default(); + let gh_resolver = Arc::new(GitHubCompilerResolver::new().await?); + let mut resolver = ResolverMultiplexer::new(env_resolver); + + // Killer switch: if anything goes wrong with GH resolver, we can disable it without having to rollback. + // TODO: Remove once GH resolver is proven to be stable. + let disable_gh_resolver = std::env::var("DISABLE_GITHUB_RESOLVER").is_ok(); + if !disable_gh_resolver { + resolver = resolver.with_resolver(gh_resolver); + } else { + tracing::warn!("GitHub resolver was disabled via DISABLE_GITHUB_RESOLVER env variable") + } + + Self::with_resolver(compilation_timeout, connection_pool, Arc::new(resolver)).await } async fn with_resolver( @@ -134,21 +143,42 @@ impl ContractVerifier { connection_pool: ConnectionPool, compiler_resolver: Arc, ) -> anyhow::Result { - let this = Self { + Self::sync_compiler_versions(compiler_resolver.as_ref(), &connection_pool).await?; + Ok(Self { compilation_timeout, contract_deployer: zksync_contracts::deployer_contract(), connection_pool, compiler_resolver, - }; - this.sync_compiler_versions().await?; - Ok(this) + }) + } + + /// Returns a future that would periodically update the supported compiler versions + /// in the database. + pub fn sync_compiler_versions_task( + &self, + ) -> impl std::future::Future> { + const UPDATE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour. + + let resolver = self.compiler_resolver.clone(); + let pool = self.connection_pool.clone(); + async move { + loop { + tracing::info!("Updating compiler versions"); + if let Err(err) = Self::sync_compiler_versions(resolver.as_ref(), &pool).await { + tracing::error!("Failed to sync compiler versions: {:?}", err); + } + tokio::time::sleep(UPDATE_INTERVAL).await; + } + } } /// Synchronizes compiler versions. #[tracing::instrument(level = "debug", skip_all)] - async fn sync_compiler_versions(&self) -> anyhow::Result<()> { - let supported_versions = self - .compiler_resolver + async fn sync_compiler_versions( + resolver: &dyn CompilerResolver, + pool: &ConnectionPool, + ) -> anyhow::Result<()> { + let supported_versions = resolver .supported_versions() .await .context("cannot get supported compilers")?; @@ -163,26 +193,23 @@ impl ContractVerifier { "persisting supported compiler versions" ); - let mut storage = self - .connection_pool - .connection_tagged("contract_verifier") - .await?; + let mut storage = pool.connection_tagged("contract_verifier").await?; let mut transaction = storage.start_transaction().await?; transaction .contract_verification_dal() - .set_zksolc_versions(&supported_versions.zksolc) + .set_zksolc_versions(&supported_versions.zksolc.into_iter().collect::>()) .await?; transaction .contract_verification_dal() - .set_solc_versions(&supported_versions.solc) + .set_solc_versions(&supported_versions.solc.into_iter().collect::>()) .await?; transaction .contract_verification_dal() - .set_zkvyper_versions(&supported_versions.zkvyper) + .set_zkvyper_versions(&supported_versions.zkvyper.into_iter().collect::>()) .await?; transaction .contract_verification_dal() - .set_vyper_versions(&supported_versions.vyper) + .set_vyper_versions(&supported_versions.vyper.into_iter().collect::>()) .await?; transaction.commit().await?; Ok(()) diff --git a/core/lib/contract_verifier/src/resolver/env.rs b/core/lib/contract_verifier/src/resolver/env.rs new file mode 100644 index 000000000000..798efde64348 --- /dev/null +++ b/core/lib/contract_verifier/src/resolver/env.rs @@ -0,0 +1,132 @@ +use std::{collections::HashSet, path::PathBuf}; + +use anyhow::Context as _; +use tokio::fs; +use zksync_queued_job_processor::async_trait; +use zksync_utils::env::Workspace; + +use crate::{ + compilers::{Solc, SolcInput, Vyper, VyperInput, ZkSolc, ZkSolcInput, ZkVyper}, + error::ContractVerifierError, + resolver::{ + Compiler, CompilerPaths, CompilerResolver, CompilerType, SupportedCompilerVersions, + }, + ZkCompilerVersions, +}; + +/// Default [`CompilerResolver`] using pre-downloaded compilers in the `/etc` subdirectories (relative to the workspace). +#[derive(Debug)] +pub(crate) struct EnvCompilerResolver { + home_dir: PathBuf, +} + +impl Default for EnvCompilerResolver { + fn default() -> Self { + Self { + home_dir: Workspace::locate().core(), + } + } +} + +impl EnvCompilerResolver { + async fn read_dir(&self, dir: &str) -> anyhow::Result> { + let mut dir_entries = fs::read_dir(self.home_dir.join(dir)) + .await + .context("failed reading dir")?; + let mut versions = HashSet::new(); + while let Some(entry) = dir_entries.next_entry().await? { + let Ok(file_type) = entry.file_type().await else { + continue; + }; + if file_type.is_dir() { + if let Ok(name) = entry.file_name().into_string() { + versions.insert(name); + } + } + } + Ok(versions) + } +} + +#[async_trait] +impl CompilerResolver for EnvCompilerResolver { + async fn supported_versions(&self) -> anyhow::Result { + let versions = SupportedCompilerVersions { + solc: self + .read_dir("etc/solc-bin") + .await + .context("failed reading solc dir")?, + zksolc: self + .read_dir("etc/zksolc-bin") + .await + .context("failed reading zksolc dir")?, + vyper: self + .read_dir("etc/vyper-bin") + .await + .context("failed reading vyper dir")?, + zkvyper: self + .read_dir("etc/zkvyper-bin") + .await + .context("failed reading zkvyper dir")?, + }; + tracing::info!("EnvResolver supported versions: {:?}", versions); + + Ok(versions) + } + + async fn resolve_solc( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + let solc_path = CompilerType::Solc.bin_path(&self.home_dir, version).await?; + Ok(Box::new(Solc::new(solc_path))) + } + + async fn resolve_zksolc( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + let zksolc_version = &version.zk; + let zksolc_path = CompilerType::ZkSolc + .bin_path(&self.home_dir, zksolc_version) + .await?; + let solc_path = CompilerType::Solc + .bin_path(&self.home_dir, &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: solc_path, + zk: zksolc_path, + }; + Ok(Box::new(ZkSolc::new( + compiler_paths, + zksolc_version.to_owned(), + ))) + } + + async fn resolve_vyper( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + let vyper_path = CompilerType::Vyper + .bin_path(&self.home_dir, version) + .await?; + Ok(Box::new(Vyper::new(vyper_path))) + } + + async fn resolve_zkvyper( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + let zkvyper_path = CompilerType::ZkVyper + .bin_path(&self.home_dir, &version.zk) + .await?; + let vyper_path = CompilerType::Vyper + .bin_path(&self.home_dir, &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: vyper_path, + zk: zkvyper_path, + }; + Ok(Box::new(ZkVyper::new(compiler_paths))) + } +} diff --git a/core/lib/contract_verifier/src/resolver/github/gh_api.rs b/core/lib/contract_verifier/src/resolver/github/gh_api.rs new file mode 100644 index 000000000000..8c9ac6723249 --- /dev/null +++ b/core/lib/contract_verifier/src/resolver/github/gh_api.rs @@ -0,0 +1,240 @@ +//! A thin wrapper over the GitHub API for the purposes of the contract verifier. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use anyhow::Context as _; +use futures_util::TryStreamExt as _; +use octocrab::service::middleware::retry::RetryConfig; + +/// Representation of releases of the compiler. +/// The main difference from the `CompilerType` used in the `resolver` module is that +/// we treat `ZkVmSolc` differently, as it's stored in a different repository. +#[derive(Debug, Clone, Copy)] +pub(super) enum CompilerGitHubRelease { + /// "Upstream" Solidity + Solc, + /// "Upstream" Vyper + Vyper, + /// ZkSync's fork of the Solidity compiler + /// Used as a dependency for ZkSolc + ZkVmSolc, + /// Solidity compiler for EraVM + ZkSolc, + /// Vyper compiler for EraVM + ZkVyper, +} + +impl CompilerGitHubRelease { + fn organization(self) -> &'static str { + match self { + Self::Solc => "ethereum", + Self::Vyper => "vyperlang", + Self::ZkVmSolc => "matter-labs", + Self::ZkSolc => "matter-labs", + Self::ZkVyper => "matter-labs", + } + } + + fn repo(self) -> &'static str { + match self { + Self::Solc => "solidity", + Self::Vyper => "vyper", + Self::ZkVmSolc => "era-solidity", + Self::ZkSolc => "era-compiler-solidity", + Self::ZkVyper => "era-compiler-vyper", + } + } + + /// Check if version is blacklisted, e.g. it shouldn't be available in the contract verifier. + fn is_version_blacklisted(self, version: &str) -> bool { + match self { + Self::Solc => { + let Ok(version) = semver::Version::parse(version) else { + tracing::error!( + "Incorrect version passed to blacklist check: {self:?}:{version}" + ); + return true; + }; + // The earliest supported version is 0.4.10. + version < semver::Version::new(0, 4, 10) + } + Self::Vyper => { + let Ok(version) = semver::Version::parse(version) else { + tracing::error!( + "Incorrect version passed to blacklist check: {self:?}:{version}" + ); + return true; + }; + + // Versions below `0.3` are not supported. + if version < semver::Version::new(0, 3, 0) { + return true; + } + + // In `0.3.x` we only allow `0.3.3`, `0.3.9`, and `0.3.10`. + if version.minor == 3 { + return !matches!(version.patch, 3 | 9 | 10); + } + + false + } + _ => false, + } + } + + fn extract_version(self, tag_name: &str) -> Option { + match self { + Self::Solc | Self::Vyper => { + // Solidity and Vyper releases are tagged with version numbers in form of `vX.Y.Z`. + tag_name + .strip_prefix('v') + .filter(|v| semver::Version::parse(v).is_ok()) + .map(|v| v.to_string()) + } + Self::ZkVmSolc => { + // ZkVmSolc releases are tagged with version numbers in form of `X.Y.Z-A.B.C`, where + // `X.Y.Z` is the version of the Solidity compiler, and `A.B.C` is the version of the ZkSync fork. + if let Some((main, fork)) = tag_name.split_once('-') { + if semver::Version::parse(main).is_ok() && semver::Version::parse(fork).is_ok() + { + // In contract verifier, our fork is prefixed with `zkVM-`. + return Some(format!("zkVM-{tag_name}")); + } + } + None + } + Self::ZkSolc | Self::ZkVyper => { + // zksolc and zkvyper releases are tagged with version numbers in form of `X.Y.Z` (without 'v'). + if semver::Version::parse(tag_name).is_ok() { + Some(tag_name.to_string()) + } else { + None + } + } + } + } + + fn match_asset(&self, asset_url: &str) -> bool { + match self { + Self::Solc => asset_url.contains("solc-static-linux"), + Self::Vyper => asset_url.contains(".linux"), + Self::ZkVmSolc => asset_url.contains("solc-linux-amd64"), + Self::ZkSolc => asset_url.contains("zksolc-linux-amd64-musl"), + Self::ZkVyper => asset_url.contains("zkvyper-linux-amd64-musl"), + } + } +} + +/// A thin wrapper over the GitHub API for the purposes of the contract verifier. +#[derive(Debug)] +pub(super) struct GitHubApi { + client: Arc, +} + +impl GitHubApi { + /// Creates a new instance of the GitHub API wrapper. + pub(super) fn new() -> Self { + // Octocrab requires rustls to be configured. + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .ok(); + + let client = Arc::new( + octocrab::Octocrab::builder() + .add_retry_config(Self::retry_config()) + .set_connect_timeout(Some(Self::connect_timeout())) + .set_read_timeout(Some(Self::read_timeout())) + .build() + .unwrap(), + ); + Self { client } + } + + fn retry_config() -> RetryConfig { + RetryConfig::Simple(4) + } + + fn connect_timeout() -> Duration { + Duration::from_secs(10) + } + + fn read_timeout() -> Duration { + Duration::from_secs(60) + } + + /// Returns versions for both upstream and our fork of solc. + pub async fn solc_versions(&self) -> anyhow::Result> { + let mut versions = self + .extract_versions(CompilerGitHubRelease::Solc) + .await + .context("Can't fetch upstream solc versions")?; + versions.extend( + self.extract_versions(CompilerGitHubRelease::ZkVmSolc) + .await + .context("Can't fetch zkVM solc versions")?, + ); + Ok(versions) + } + + pub async fn zksolc_versions(&self) -> anyhow::Result> { + self.extract_versions(CompilerGitHubRelease::ZkSolc).await + } + + pub async fn vyper_versions(&self) -> anyhow::Result> { + self.extract_versions(CompilerGitHubRelease::Vyper).await + } + + pub async fn zkvyper_versions(&self) -> anyhow::Result> { + self.extract_versions(CompilerGitHubRelease::ZkVyper).await + } + + /// Will scan all the releases for a specific compiler. + async fn extract_versions( + &self, + compiler: CompilerGitHubRelease, + ) -> anyhow::Result> { + // Create a stream over all the versions to not worry about pagination. + let releases = self + .client + .repos(compiler.organization(), compiler.repo()) + .releases() + .list() + .per_page(100) + .send() + .await? + .into_stream(&self.client); + tokio::pin!(releases); + + // Go through all the releases, filter ones that match the version. + // For matching versions, find a suitable asset and store its URL. + let mut versions = HashMap::new(); + while let Some(release) = releases.try_next().await? { + // Skip pre-releases. + if release.prerelease { + continue; + } + + if let Some(version) = compiler.extract_version(&release.tag_name) { + if compiler.is_version_blacklisted(&version) { + tracing::debug!("Skipping {compiler:?}:{version} due to blacklist"); + continue; + } + + let mut found = false; + for asset in release.assets { + if compiler.match_asset(asset.browser_download_url.as_str()) { + tracing::info!("Discovered release {compiler:?}:{version}"); + versions.insert(version.clone(), asset.browser_download_url.clone()); + found = true; + break; + } + } + if !found { + tracing::warn!("Didn't find a matching artifact for {compiler:?}:{version}"); + } + } + } + + Ok(versions) + } +} diff --git a/core/lib/contract_verifier/src/resolver/github/mod.rs b/core/lib/contract_verifier/src/resolver/github/mod.rs new file mode 100644 index 000000000000..a50d0151b7ff --- /dev/null +++ b/core/lib/contract_verifier/src/resolver/github/mod.rs @@ -0,0 +1,311 @@ +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; + +use anyhow::Context as _; +use tokio::{io::AsyncWriteExt as _, sync::RwLock}; +use zksync_queued_job_processor::async_trait; + +use self::gh_api::GitHubApi; +use crate::{ + compilers::{Solc, SolcInput, Vyper, VyperInput, ZkSolc, ZkSolcInput, ZkVyper}, + error::ContractVerifierError, + resolver::{ + Compiler, CompilerPaths, CompilerResolver, CompilerType, SupportedCompilerVersions, + }, + ZkCompilerVersions, +}; + +mod gh_api; + +/// [`CompilerResolver`] that can dynamically download missing compilers from GitHub releases. +/// +/// Note: this resolver does not interact with [`EnvCompilerResolver`](super::EnvCompilerResolver). +/// This is important for the context of zksolc/zkvyper, as there we have two separate compilers +/// required for compilation. This resolver will download both of them, even if one of the versions +/// is available in the `EnvCompilerResolver`. +#[derive(Debug)] +pub(crate) struct GitHubCompilerResolver { + /// We expect that contract-verifier will be running in docker without any persistent storage, + /// so we explicitly don't expect any artifacts to survive restart. + artifacts_dir: tempfile::TempDir, + gh_client: GitHubApi, + client: reqwest::Client, + supported_versions: RwLock, + /// List of downloads performed right now. + /// `broadcast` receiver can be used to wait until the download is finished. + active_downloads: RwLock>>, +} + +#[derive(Debug)] +struct SupportedVersions { + /// Holds versions for both upstream and zkVM solc. + solc_versions: HashMap, + zksolc_versions: HashMap, + vyper_versions: HashMap, + zkvyper_versions: HashMap, + last_updated: Instant, +} + +impl Default for SupportedVersions { + fn default() -> Self { + Self::new() + } +} + +impl SupportedVersions { + // Note: We assume that contract verifier will run the task to update supported versions + // rarely, but we still want to protect ourselves from accidentally spamming GitHub API. + // So, this interval is smaller than the expected time between updates (this way we don't + // run into an issue where intervals are slightly out of sync, causing a delay in "real" + // update time). + const CACHE_INTERVAL: Duration = Duration::from_secs(10 * 60); // 10 minutes + + fn new() -> Self { + Self { + solc_versions: HashMap::new(), + zksolc_versions: HashMap::new(), + vyper_versions: HashMap::new(), + zkvyper_versions: HashMap::new(), + last_updated: Instant::now(), + } + } + + fn is_outdated(&self) -> bool { + self.last_updated.elapsed() > Self::CACHE_INTERVAL + } + + async fn update(&mut self, gh_client: &GitHubApi) -> anyhow::Result<()> { + // Non-atomic update is fine here: the fields are independent, so if + // at least one update succeeds, it's worth persisting. We won't be changing + // the last update timestamp in case of failure though, so it will be retried + // next time. + self.solc_versions = gh_client + .solc_versions() + .await + .context("failed fetching solc versions")?; + self.zksolc_versions = gh_client + .zksolc_versions() + .await + .context("failed fetching zksolc versions")?; + self.vyper_versions = gh_client + .vyper_versions() + .await + .context("failed fetching vyper versions")?; + self.zkvyper_versions = gh_client + .zkvyper_versions() + .await + .context("failed fetching zkvyper versions")?; + self.last_updated = Instant::now(); + Ok(()) + } + + async fn update_if_needed(&mut self, gh_client: &GitHubApi) -> anyhow::Result<()> { + if self.is_outdated() { + tracing::info!("GH compiler versions cache outdated, updating"); + self.update(gh_client).await?; + } + Ok(()) + } +} + +impl GitHubCompilerResolver { + pub async fn new() -> anyhow::Result { + let artifacts_dir = tempfile::tempdir().context("failed creating temp dir")?; + let gh_client = GitHubApi::new(); + let mut supported_versions = SupportedVersions::default(); + if let Err(err) = supported_versions.update(&gh_client).await { + // We don't want the resolver to fail at creation if versions can't be fetched. + // It shouldn't bring down the whole application, so the expectation here is that + // the versions will be fetched later. + tracing::error!("failed syncing compiler versions at start: {:?}", err); + } + + Ok(Self { + artifacts_dir, + gh_client, + client: reqwest::Client::new(), + supported_versions: RwLock::new(supported_versions), + active_downloads: RwLock::default(), + }) + } +} + +impl GitHubCompilerResolver { + async fn download_version_if_needed( + &self, + compiler: CompilerType, + version: &str, + ) -> anyhow::Result<()> { + // We need to check the lock first, because the compiler may still be downloading. + // We must hold the lock until we know if we need to download the compiler. + let mut lock = self.active_downloads.write().await; + if let Some(rx) = lock.get(&(compiler, version.to_string())) { + let mut rx = rx.resubscribe(); + drop(lock); + tracing::debug!( + "Waiting for {}:{} download to finish", + compiler.as_str(), + version + ); + rx.recv().await?; + return Ok(()); + } + + if compiler.exists(self.artifacts_dir.path(), version).await? { + tracing::debug!("Compiler {}:{} exists", compiler.as_str(), version); + return Ok(()); + } + + // Mark the compiler as downloading. + let (tx, rx) = tokio::sync::broadcast::channel(1); + lock.insert((compiler, version.to_string()), rx); + drop(lock); + + tracing::info!("Downloading {}:{}", compiler.as_str(), version); + let lock = self.supported_versions.read().await; + let versions = match compiler { + CompilerType::Solc => &lock.solc_versions, + CompilerType::ZkSolc => &lock.zksolc_versions, + CompilerType::Vyper => &lock.vyper_versions, + CompilerType::ZkVyper => &lock.zkvyper_versions, + }; + + let version_url = versions + .get(version) + .ok_or_else(|| { + ContractVerifierError::UnknownCompilerVersion("solc", version.to_owned()) + })? + .clone(); + drop(lock); + let path = compiler.bin_path_unchecked(self.artifacts_dir.path(), version); + + let response = self.client.get(version_url).send().await?; + let body = response.bytes().await?; + + tracing::info!("Saving {}:{} to {:?}", compiler.as_str(), version, path); + + tokio::fs::create_dir_all(path.parent().unwrap()) + .await + .context("failed to create dir")?; + + let mut file = tokio::fs::File::create_new(path) + .await + .context("failed to create file")?; + file.write_all(&body) + .await + .context("failed to write to file")?; + file.flush().await.context("failed to flush file")?; + + // On UNIX-like systems, make file executable. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = file.metadata().await?.permissions(); + perms.set_mode(0o700); // Only owner can execute and access. + file.set_permissions(perms).await?; + } + + tracing::info!("Finished downloading {}:{}", compiler.as_str(), version); + + // Notify other waiters that the compiler is downloaded. + tx.send(()).ok(); + let mut lock = self.active_downloads.write().await; + lock.remove(&(compiler, version.to_string())); + drop(lock); + + Ok(()) + } +} + +#[async_trait] +impl CompilerResolver for GitHubCompilerResolver { + async fn supported_versions(&self) -> anyhow::Result { + let mut lock = self.supported_versions.write().await; + lock.update_if_needed(&self.gh_client).await?; + + let versions = SupportedCompilerVersions { + solc: lock.solc_versions.keys().cloned().collect(), + zksolc: lock.zksolc_versions.keys().cloned().collect(), + vyper: lock.vyper_versions.keys().cloned().collect(), + zkvyper: lock.zkvyper_versions.keys().cloned().collect(), + }; + tracing::info!("GitHubResolver supported versions: {:?}", versions); + Ok(versions) + } + + async fn resolve_solc( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Solc, version) + .await?; + + let solc_path = CompilerType::Solc + .bin_path(self.artifacts_dir.path(), version) + .await?; + Ok(Box::new(Solc::new(solc_path))) + } + + async fn resolve_zksolc( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Solc, &version.base) + .await?; + self.download_version_if_needed(CompilerType::ZkSolc, &version.zk) + .await?; + + let zksolc_version = &version.zk; + let zksolc_path = CompilerType::ZkSolc + .bin_path(self.artifacts_dir.path(), zksolc_version) + .await?; + let solc_path = CompilerType::Solc + .bin_path(self.artifacts_dir.path(), &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: solc_path, + zk: zksolc_path, + }; + Ok(Box::new(ZkSolc::new( + compiler_paths, + zksolc_version.to_owned(), + ))) + } + + async fn resolve_vyper( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Vyper, version) + .await?; + + let vyper_path = CompilerType::Vyper + .bin_path(self.artifacts_dir.path(), version) + .await?; + Ok(Box::new(Vyper::new(vyper_path))) + } + + async fn resolve_zkvyper( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Vyper, &version.base) + .await?; + self.download_version_if_needed(CompilerType::ZkVyper, &version.zk) + .await?; + + let zkvyper_path = CompilerType::ZkVyper + .bin_path(self.artifacts_dir.path(), &version.zk) + .await?; + let vyper_path = CompilerType::Vyper + .bin_path(self.artifacts_dir.path(), &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: vyper_path, + zk: zkvyper_path, + }; + Ok(Box::new(ZkVyper::new(compiler_paths))) + } +} diff --git a/core/lib/contract_verifier/src/resolver.rs b/core/lib/contract_verifier/src/resolver/mod.rs similarity index 51% rename from core/lib/contract_verifier/src/resolver.rs rename to core/lib/contract_verifier/src/resolver/mod.rs index 018da12a152a..a9d2bcf9049d 100644 --- a/core/lib/contract_verifier/src/resolver.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -1,21 +1,26 @@ use std::{ + collections::HashSet, fmt, path::{Path, PathBuf}, + sync::Arc, }; use anyhow::Context as _; use tokio::fs; use zksync_queued_job_processor::async_trait; use zksync_types::contract_verification_api::CompilationArtifacts; -use zksync_utils::env::Workspace; +pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; use crate::{ - compilers::{Solc, SolcInput, Vyper, VyperInput, ZkSolc, ZkSolcInput, ZkVyper}, + compilers::{SolcInput, VyperInput, ZkSolcInput}, error::ContractVerifierError, ZkCompilerVersions, }; -#[derive(Debug, Clone, Copy)] +mod env; +mod github; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum CompilerType { Solc, ZkSolc, @@ -48,6 +53,14 @@ impl CompilerType { .join(self.as_str()) } + async fn exists(self, home_dir: &Path, version: &str) -> Result { + let path = self.bin_path_unchecked(home_dir, version); + let exists = fs::try_exists(&path) + .await + .with_context(|| format!("failed accessing `{}`", self.as_str()))?; + Ok(exists) + } + async fn bin_path( self, home_dir: &Path, @@ -68,12 +81,25 @@ impl CompilerType { } /// Compiler versions supported by a [`CompilerResolver`]. -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) struct SupportedCompilerVersions { - pub solc: Vec, - pub zksolc: Vec, - pub vyper: Vec, - pub zkvyper: Vec, + /// Note: solc can have two "flavors": "upstream" solc (e.g. "real" solc used for L1 development), + /// and "zksync" solc (e.g. ZKsync fork of the solc used by `zksolc`). + /// They both are considered as "solc", but they have different versioning scheme, e.g. + /// "upstream" solc can have version `0.8.0`, while "zksync" solc can have version `zkVM-0.8.0-1.0.1`. + pub solc: HashSet, + pub zksolc: HashSet, + pub vyper: HashSet, + pub zkvyper: HashSet, +} + +impl SupportedCompilerVersions { + fn merge(&mut self, other: SupportedCompilerVersions) { + self.solc.extend(other.solc); + self.zksolc.extend(other.zksolc); + self.vyper.extend(other.vyper); + self.zkvyper.extend(other.zkvyper); + } } impl SupportedCompilerVersions { @@ -138,116 +164,115 @@ pub(crate) trait Compiler: Send + fmt::Debug { ) -> Result; } -/// Default [`CompilerResolver`] using pre-downloaded compilers in the `/etc` subdirectories (relative to the workspace). #[derive(Debug)] -pub(crate) struct EnvCompilerResolver { - home_dir: PathBuf, +pub struct ResolverMultiplexer { + resolvers: Vec>, } -impl Default for EnvCompilerResolver { - fn default() -> Self { +impl ResolverMultiplexer { + pub fn new(resolver: Arc) -> Self { Self { - home_dir: Workspace::locate().core(), + resolvers: vec![resolver], } } -} -impl EnvCompilerResolver { - async fn read_dir(&self, dir: &str) -> anyhow::Result> { - let mut dir_entries = fs::read_dir(self.home_dir.join(dir)) - .await - .context("failed reading dir")?; - let mut versions = vec![]; - while let Some(entry) = dir_entries.next_entry().await? { - let Ok(file_type) = entry.file_type().await else { - continue; - }; - if file_type.is_dir() { - if let Ok(name) = entry.file_name().into_string() { - versions.push(name); - } - } - } - Ok(versions) + pub fn with_resolver(mut self, resolver: Arc) -> Self { + self.resolvers.push(resolver); + self } } #[async_trait] -impl CompilerResolver for EnvCompilerResolver { +impl CompilerResolver for ResolverMultiplexer { async fn supported_versions(&self) -> anyhow::Result { - Ok(SupportedCompilerVersions { - solc: self - .read_dir("etc/solc-bin") - .await - .context("failed reading solc dir")?, - zksolc: self - .read_dir("etc/zksolc-bin") - .await - .context("failed reading zksolc dir")?, - vyper: self - .read_dir("etc/vyper-bin") - .await - .context("failed reading vyper dir")?, - zkvyper: self - .read_dir("etc/zkvyper-bin") - .await - .context("failed reading zkvyper dir")?, - }) + let mut versions = SupportedCompilerVersions::default(); + for resolver in &self.resolvers { + versions.merge(resolver.supported_versions().await?); + } + Ok(versions) } + /// Resolves a `solc` compiler. async fn resolve_solc( &self, version: &str, ) -> Result>, ContractVerifierError> { - let solc_path = CompilerType::Solc.bin_path(&self.home_dir, version).await?; - Ok(Box::new(Solc::new(solc_path))) + for resolver in &self.resolvers { + match resolver.resolve_solc(version).await { + Ok(compiler) => return Ok(compiler), + Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + continue; + } + Err(err) => return Err(err), + } + } + Err(ContractVerifierError::UnknownCompilerVersion( + "solc", + version.to_owned(), + )) } + /// Resolves a `zksolc` compiler. async fn resolve_zksolc( &self, version: &ZkCompilerVersions, ) -> Result>, ContractVerifierError> { - let zksolc_version = &version.zk; - let zksolc_path = CompilerType::ZkSolc - .bin_path(&self.home_dir, zksolc_version) - .await?; - let solc_path = CompilerType::Solc - .bin_path(&self.home_dir, &version.base) - .await?; - let compiler_paths = CompilerPaths { - base: solc_path, - zk: zksolc_path, - }; - Ok(Box::new(ZkSolc::new( - compiler_paths, - zksolc_version.to_owned(), - ))) + let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( + "zksolc", + version.zk.to_owned(), + )); + for resolver in &self.resolvers { + match resolver.resolve_zksolc(version).await { + Ok(compiler) => return Ok(compiler), + err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + last_error = err; + continue; + } + Err(err) => return Err(err), + } + } + last_error } + /// Resolves a `vyper` compiler. async fn resolve_vyper( &self, version: &str, ) -> Result>, ContractVerifierError> { - let vyper_path = CompilerType::Vyper - .bin_path(&self.home_dir, version) - .await?; - Ok(Box::new(Vyper::new(vyper_path))) + for resolver in &self.resolvers { + match resolver.resolve_vyper(version).await { + Ok(compiler) => return Ok(compiler), + Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + continue; + } + Err(err) => return Err(err), + } + } + Err(ContractVerifierError::UnknownCompilerVersion( + "vyper", + version.to_owned(), + )) } + /// Resolves a `zkvyper` compiler. async fn resolve_zkvyper( &self, version: &ZkCompilerVersions, ) -> Result>, ContractVerifierError> { - let zkvyper_path = CompilerType::ZkVyper - .bin_path(&self.home_dir, &version.zk) - .await?; - let vyper_path = CompilerType::Vyper - .bin_path(&self.home_dir, &version.base) - .await?; - let compiler_paths = CompilerPaths { - base: vyper_path, - zk: zkvyper_path, - }; - Ok(Box::new(ZkVyper::new(compiler_paths))) + let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( + "zkvyper", + version.zk.to_owned(), + )); + for resolver in &self.resolvers { + match resolver.resolve_zkvyper(version).await { + Ok(compiler) => return Ok(compiler), + err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + last_error = err; + continue; + } + Err(err) => return Err(err), + } + } + last_error } } diff --git a/core/lib/contract_verifier/src/tests/mod.rs b/core/lib/contract_verifier/src/tests/mod.rs index 395d467542dc..f66732675ce6 100644 --- a/core/lib/contract_verifier/src/tests/mod.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -1,6 +1,9 @@ //! Tests for the contract verifier. -use std::{collections::HashMap, iter}; +use std::{ + collections::{HashMap, HashSet}, + iter, +}; use test_casing::{test_casing, Product}; use tokio::sync::watch; @@ -289,10 +292,10 @@ impl Compiler for MockCompilerResolver { impl CompilerResolver for MockCompilerResolver { async fn supported_versions(&self) -> anyhow::Result { Ok(SupportedCompilerVersions { - solc: vec![SOLC_VERSION.to_owned()], - zksolc: vec![ZKSOLC_VERSION.to_owned()], - vyper: vec![], - zkvyper: vec![], + solc: [SOLC_VERSION.to_owned()].into_iter().collect(), + zksolc: [ZKSOLC_VERSION.to_owned()].into_iter().collect(), + vyper: HashSet::default(), + zkvyper: HashSet::default(), }) } diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index 831ba96a7fd3..05a4ae6ecb80 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -28,16 +28,16 @@ struct TestCompilerVersions { } impl TestCompilerVersions { - fn new(mut versions: SupportedCompilerVersions) -> Option { + fn new(versions: SupportedCompilerVersions) -> Option { let solc = versions .solc .into_iter() .find(|ver| !ver.starts_with("zkVM"))?; Some(Self { solc, - zksolc: versions.zksolc.pop()?, - vyper: versions.vyper.pop()?, - zkvyper: versions.zkvyper.pop()?, + zksolc: versions.zksolc.into_iter().next()?, + vyper: versions.vyper.into_iter().next()?, + zkvyper: versions.zkvyper.into_iter().next()?, }) } diff --git a/core/lib/dal/.sqlx/query-cee7a608bd77815e9582531383481b01395cfd2a3e95fb4593229bd878163320.json b/core/lib/dal/.sqlx/query-e46c99b23db91800b27c717100f8203a62629904bc4956249e690a8ad7a48983.json similarity index 50% rename from core/lib/dal/.sqlx/query-cee7a608bd77815e9582531383481b01395cfd2a3e95fb4593229bd878163320.json rename to core/lib/dal/.sqlx/query-e46c99b23db91800b27c717100f8203a62629904bc4956249e690a8ad7a48983.json index 4b219bfee0a5..7ca2c9e7e9fa 100644 --- a/core/lib/dal/.sqlx/query-cee7a608bd77815e9582531383481b01395cfd2a3e95fb4593229bd878163320.json +++ b/core/lib/dal/.sqlx/query-e46c99b23db91800b27c717100f8203a62629904bc4956249e690a8ad7a48983.json @@ -1,12 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n WITH upsert AS (\n SELECT\n p.l1_batch_number\n FROM\n proof_generation_details p\n LEFT JOIN\n tee_proof_generation_details tee\n ON\n p.l1_batch_number = tee.l1_batch_number\n AND tee.tee_type = $1\n WHERE\n (\n p.l1_batch_number >= $5\n AND p.vm_run_data_blob_url IS NOT NULL\n AND p.proof_gen_data_blob_url IS NOT NULL\n )\n AND (\n tee.l1_batch_number IS NULL\n OR (\n tee.status = $3\n OR (\n tee.status = $2\n AND tee.prover_taken_at < NOW() - $4::INTERVAL\n )\n )\n )\n FETCH FIRST ROW ONLY\n )\n \n INSERT INTO\n tee_proof_generation_details (\n l1_batch_number, tee_type, status, created_at, updated_at, prover_taken_at\n )\n SELECT\n l1_batch_number,\n $1,\n $2,\n NOW(),\n NOW(),\n NOW()\n FROM\n upsert\n ON CONFLICT (l1_batch_number, tee_type) DO\n UPDATE\n SET\n status = $2,\n updated_at = NOW(),\n prover_taken_at = NOW()\n RETURNING\n l1_batch_number\n ", + "query": "\n WITH upsert AS (\n SELECT\n p.l1_batch_number\n FROM\n proof_generation_details p\n LEFT JOIN\n tee_proof_generation_details tee\n ON\n p.l1_batch_number = tee.l1_batch_number\n AND tee.tee_type = $1\n WHERE\n (\n p.l1_batch_number >= $5\n AND p.vm_run_data_blob_url IS NOT NULL\n AND p.proof_gen_data_blob_url IS NOT NULL\n )\n AND (\n tee.l1_batch_number IS NULL\n OR (\n (tee.status = $2 OR tee.status = $3)\n AND tee.prover_taken_at < NOW() - $4::INTERVAL\n )\n )\n FETCH FIRST ROW ONLY\n )\n \n INSERT INTO\n tee_proof_generation_details (\n l1_batch_number, tee_type, status, created_at, updated_at, prover_taken_at\n )\n SELECT\n l1_batch_number,\n $1,\n $2,\n NOW(),\n NOW(),\n NOW()\n FROM\n upsert\n ON CONFLICT (l1_batch_number, tee_type) DO\n UPDATE\n SET\n status = $2,\n updated_at = NOW(),\n prover_taken_at = NOW()\n RETURNING\n l1_batch_number,\n created_at\n ", "describe": { "columns": [ { "ordinal": 0, "name": "l1_batch_number", "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamp" } ], "parameters": { @@ -19,8 +24,9 @@ ] }, "nullable": [ + false, false ] }, - "hash": "cee7a608bd77815e9582531383481b01395cfd2a3e95fb4593229bd878163320" + "hash": "e46c99b23db91800b27c717100f8203a62629904bc4956249e690a8ad7a48983" } diff --git a/core/lib/dal/doc/TeeProofGenerationDal.md b/core/lib/dal/doc/TeeProofGenerationDal.md index fcfa379816c7..d9ae70aeb2fd 100644 --- a/core/lib/dal/doc/TeeProofGenerationDal.md +++ b/core/lib/dal/doc/TeeProofGenerationDal.md @@ -11,9 +11,11 @@ title: Status Diagram --- stateDiagram-v2 -[*] --> unpicked : insert_tee_proof_generation_job -unpicked --> picked_by_prover : lock_batch_for_proving +[*] --> picked_by_prover : lock picked_by_prover --> generated : save_proof_artifacts_metadata -picked_by_prover --> unpicked : unlock_batch +picked_by_prover --> permanently_ignored : unlock_batch +picked_by_prover --> failed : unlock_batch +failed --> picked_by_prover : lock +permanently_ignored --> [*] generated --> [*] ``` diff --git a/core/lib/dal/migrations/20240930110000_tee_add_permanently_ignored_state.down.sql b/core/lib/dal/migrations/20240930110000_tee_add_permanently_ignored_state.down.sql new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/core/lib/dal/migrations/20240930110000_tee_add_permanently_ignored_state.up.sql b/core/lib/dal/migrations/20240930110000_tee_add_permanently_ignored_state.up.sql new file mode 100644 index 000000000000..12a21d1728c8 --- /dev/null +++ b/core/lib/dal/migrations/20240930110000_tee_add_permanently_ignored_state.up.sql @@ -0,0 +1,8 @@ +-- There were manually added tee_proof_generation_details entries with status 'permanently_ignore'. + +UPDATE tee_proof_generation_details SET status = 'permanently_ignored' WHERE status = 'permanently_ignore'; + +-- Entries with the status 'unpicked' were not used at all after the migration to the logic +-- introduced in https://github.com/matter-labs/zksync-era/pull/3017. This was overlooked. + +DELETE FROM tee_proof_generation_details WHERE status = 'unpicked'; diff --git a/core/lib/dal/src/models/storage_tee_proof.rs b/core/lib/dal/src/models/storage_tee_proof.rs index 5c93361e7df1..6f80c59511f9 100644 --- a/core/lib/dal/src/models/storage_tee_proof.rs +++ b/core/lib/dal/src/models/storage_tee_proof.rs @@ -1,4 +1,7 @@ -use chrono::NaiveDateTime; +use chrono::{DateTime, NaiveDateTime, Utc}; +use zksync_types::L1BatchNumber; + +use crate::tee_proof_generation_dal::LockedBatch; #[derive(Debug, Clone, sqlx::FromRow)] pub struct StorageTeeProof { @@ -8,3 +11,18 @@ pub struct StorageTeeProof { pub updated_at: NaiveDateTime, pub attestation: Option>, } + +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct StorageLockedBatch { + pub l1_batch_number: i64, + pub created_at: NaiveDateTime, +} + +impl From for LockedBatch { + fn from(tx: StorageLockedBatch) -> LockedBatch { + LockedBatch { + l1_batch_number: L1BatchNumber::from(tx.l1_batch_number as u32), + created_at: DateTime::::from_naive_utc_and_offset(tx.created_at, Utc), + } + } +} diff --git a/core/lib/dal/src/tee_proof_generation_dal.rs b/core/lib/dal/src/tee_proof_generation_dal.rs index 755d02769101..4d19c3ff0c8b 100644 --- a/core/lib/dal/src/tee_proof_generation_dal.rs +++ b/core/lib/dal/src/tee_proof_generation_dal.rs @@ -1,6 +1,7 @@ #![doc = include_str!("../doc/TeeProofGenerationDal.md")] use std::time::Duration; +use chrono::{DateTime, Utc}; use strum::{Display, EnumString}; use zksync_db_connection::{ connection::Connection, @@ -10,21 +11,47 @@ use zksync_db_connection::{ }; use zksync_types::{tee_types::TeeType, L1BatchNumber}; -use crate::{models::storage_tee_proof::StorageTeeProof, Core}; +use crate::{ + models::storage_tee_proof::{StorageLockedBatch, StorageTeeProof}, + Core, +}; #[derive(Debug)] pub struct TeeProofGenerationDal<'a, 'c> { pub(crate) storage: &'a mut Connection<'c, Core>, } -#[derive(Debug, EnumString, Display)] -enum TeeProofGenerationJobStatus { - #[strum(serialize = "unpicked")] - Unpicked, +#[derive(Debug, Clone, Copy, EnumString, Display)] +pub enum TeeProofGenerationJobStatus { + /// The batch has been picked by a TEE prover and is currently being processed. #[strum(serialize = "picked_by_prover")] PickedByProver, + /// The proof has been successfully generated and submitted for the batch. #[strum(serialize = "generated")] Generated, + /// The proof generation for the batch has failed, which can happen if its inputs (GCS blob + /// files) are incomplete or the API is unavailable. Failed batches are retried for a specified + /// period, as defined in the configuration. + #[strum(serialize = "failed")] + Failed, + /// The batch will not be processed again because the proof generation has been failing for an + /// extended period, as specified in the configuration. + #[strum(serialize = "permanently_ignored")] + PermanentlyIgnored, +} + +/// Represents a locked batch picked by a TEE prover. A batch is locked when taken by a TEE prover +/// ([TeeProofGenerationJobStatus::PickedByProver]). It can transition to one of three states: +/// 1. [TeeProofGenerationJobStatus::Generated]. +/// 2. [TeeProofGenerationJobStatus::Failed]. +/// 3. [TeeProofGenerationJobStatus::PermanentlyIgnored]. +#[derive(Clone, Debug)] +pub struct LockedBatch { + /// Locked batch number. + pub l1_batch_number: L1BatchNumber, + /// The creation time of the job for this batch. It is used to determine if the batch should + /// transition to [TeeProofGenerationJobStatus::PermanentlyIgnored] or [TeeProofGenerationJobStatus::Failed]. + pub created_at: DateTime, } impl TeeProofGenerationDal<'_, '_> { @@ -33,10 +60,11 @@ impl TeeProofGenerationDal<'_, '_> { tee_type: TeeType, processing_timeout: Duration, min_batch_number: L1BatchNumber, - ) -> DalResult> { + ) -> DalResult> { let processing_timeout = pg_interval_from_duration(processing_timeout); let min_batch_number = i64::from(min_batch_number.0); - sqlx::query!( + let locked_batch = sqlx::query_as!( + StorageLockedBatch, r#" WITH upsert AS ( SELECT @@ -57,11 +85,8 @@ impl TeeProofGenerationDal<'_, '_> { AND ( tee.l1_batch_number IS NULL OR ( - tee.status = $3 - OR ( - tee.status = $2 - AND tee.prover_taken_at < NOW() - $4::INTERVAL - ) + (tee.status = $2 OR tee.status = $3) + AND tee.prover_taken_at < NOW() - $4::INTERVAL ) ) FETCH FIRST ROW ONLY @@ -87,11 +112,12 @@ impl TeeProofGenerationDal<'_, '_> { updated_at = NOW(), prover_taken_at = NOW() RETURNING - l1_batch_number + l1_batch_number, + created_at "#, tee_type.to_string(), TeeProofGenerationJobStatus::PickedByProver.to_string(), - TeeProofGenerationJobStatus::Unpicked.to_string(), + TeeProofGenerationJobStatus::Failed.to_string(), processing_timeout, min_batch_number ) @@ -100,14 +126,17 @@ impl TeeProofGenerationDal<'_, '_> { .with_arg("processing_timeout", &processing_timeout) .with_arg("l1_batch_number", &min_batch_number) .fetch_optional(self.storage) - .await - .map(|record| record.map(|record| L1BatchNumber(record.l1_batch_number as u32))) + .await? + .map(Into::into); + + Ok(locked_batch) } pub async fn unlock_batch( &mut self, l1_batch_number: L1BatchNumber, tee_type: TeeType, + status: TeeProofGenerationJobStatus, ) -> DalResult<()> { let batch_number = i64::from(l1_batch_number.0); sqlx::query!( @@ -120,7 +149,7 @@ impl TeeProofGenerationDal<'_, '_> { l1_batch_number = $2 AND tee_type = $3 "#, - TeeProofGenerationJobStatus::Unpicked.to_string(), + status.to_string(), batch_number, tee_type.to_string() ) @@ -266,7 +295,7 @@ impl TeeProofGenerationDal<'_, '_> { "#, batch_number, tee_type.to_string(), - TeeProofGenerationJobStatus::Unpicked.to_string(), + TeeProofGenerationJobStatus::PickedByProver.to_string(), ); let instrumentation = Instrumented::new("insert_tee_proof_generation_job") .with_arg("l1_batch_number", &batch_number) @@ -281,7 +310,7 @@ impl TeeProofGenerationDal<'_, '_> { } /// For testing purposes only. - pub async fn get_oldest_unpicked_batch(&mut self) -> DalResult> { + pub async fn get_oldest_picked_by_prover_batch(&mut self) -> DalResult> { let query = sqlx::query!( r#" SELECT @@ -295,7 +324,7 @@ impl TeeProofGenerationDal<'_, '_> { LIMIT 1 "#, - TeeProofGenerationJobStatus::Unpicked.to_string(), + TeeProofGenerationJobStatus::PickedByProver.to_string(), ); let batch_number = Instrumented::new("get_oldest_unpicked_batch") .with(query) diff --git a/core/lib/env_config/src/proof_data_handler.rs b/core/lib/env_config/src/proof_data_handler.rs index 47848585e769..65fd1d516de3 100644 --- a/core/lib/env_config/src/proof_data_handler.rs +++ b/core/lib/env_config/src/proof_data_handler.rs @@ -29,6 +29,7 @@ mod tests { tee_support: true, first_tee_processed_batch: L1BatchNumber(1337), tee_proof_generation_timeout_in_secs: 600, + tee_batch_permanently_ignored_timeout_in_hours: 240, }, } } @@ -41,6 +42,7 @@ mod tests { PROOF_DATA_HANDLER_TEE_SUPPORT="true" PROOF_DATA_HANDLER_FIRST_TEE_PROCESSED_BATCH="1337" PROOF_DATA_HANDLER_TEE_PROOF_GENERATION_TIMEOUT_IN_SECS="600" + PROOF_DATA_HANDLER_TEE_BATCH_PERMANENTLY_IGNORED_TIMEOUT_IN_HOURS="240" "#; let mut lock = MUTEX.lock(); lock.set_env(config); diff --git a/core/lib/object_store/src/retries.rs b/core/lib/object_store/src/retries.rs index 2cccbb17c2bb..16d2c1cd55f1 100644 --- a/core/lib/object_store/src/retries.rs +++ b/core/lib/object_store/src/retries.rs @@ -53,7 +53,6 @@ impl Request<'_> { backoff_secs *= 2; } Err(err) => { - tracing::warn!(%err, "Failed request with a fatal error"); break Err(err); } } diff --git a/core/lib/protobuf_config/src/proof_data_handler.rs b/core/lib/protobuf_config/src/proof_data_handler.rs index c01e163bd771..92a9c90bbb64 100644 --- a/core/lib/protobuf_config/src/proof_data_handler.rs +++ b/core/lib/protobuf_config/src/proof_data_handler.rs @@ -29,6 +29,12 @@ impl ProtoRepr for proto::ProofDataHandler { .unwrap_or_else( configs::TeeConfig::default_tee_proof_generation_timeout_in_secs, ), + tee_batch_permanently_ignored_timeout_in_hours: self + .tee_batch_permanently_ignored_timeout_in_hours + .map(|x| x as u16) + .unwrap_or_else( + configs::TeeConfig::default_tee_batch_permanently_ignored_timeout_in_hours, + ), }, }) } @@ -42,6 +48,11 @@ impl ProtoRepr for proto::ProofDataHandler { tee_proof_generation_timeout_in_secs: Some( this.tee_config.tee_proof_generation_timeout_in_secs.into(), ), + tee_batch_permanently_ignored_timeout_in_hours: Some( + this.tee_config + .tee_batch_permanently_ignored_timeout_in_hours + .into(), + ), } } } diff --git a/core/lib/protobuf_config/src/proto/config/prover.proto b/core/lib/protobuf_config/src/proto/config/prover.proto index 392834d25f3d..64735713fcab 100644 --- a/core/lib/protobuf_config/src/proto/config/prover.proto +++ b/core/lib/protobuf_config/src/proto/config/prover.proto @@ -110,4 +110,5 @@ message ProofDataHandler { optional bool tee_support = 3; // optional optional uint64 first_tee_processed_batch = 4; // optional optional uint32 tee_proof_generation_timeout_in_secs = 5; // optional + optional uint32 tee_batch_permanently_ignored_timeout_in_hours = 6; // optional } diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 5f81e889b537..b5d2b3276527 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -4,7 +4,6 @@ use serde_json::Value; use serde_with::{hex::Hex, serde_as}; use strum::Display; use zksync_basic_types::{ - tee_types::TeeType, web3::{AccessList, Bytes, Index}, Bloom, L1BatchNumber, H160, H256, H64, U256, U64, }; @@ -16,6 +15,7 @@ pub use crate::transaction_request::{ use crate::{ debug_flat_call::{DebugCallFlat, ResultDebugCallFlat}, protocol_version::L1VerifierConfig, + tee_types::TeeType, Address, L2BlockNumber, ProtocolVersionId, }; diff --git a/core/node/proof_data_handler/Cargo.toml b/core/node/proof_data_handler/Cargo.toml index 639266a2be96..0bd1501277b7 100644 --- a/core/node/proof_data_handler/Cargo.toml +++ b/core/node/proof_data_handler/Cargo.toml @@ -27,9 +27,7 @@ tracing.workspace = true [dev-dependencies] hyper.workspace = true -chrono.workspace = true zksync_multivm.workspace = true serde_json.workspace = true tower.workspace = true -zksync_basic_types.workspace = true zksync_contracts.workspace = true diff --git a/core/node/proof_data_handler/src/tee_request_processor.rs b/core/node/proof_data_handler/src/tee_request_processor.rs index ee5be844b981..971b94fe315f 100644 --- a/core/node/proof_data_handler/src/tee_request_processor.rs +++ b/core/node/proof_data_handler/src/tee_request_processor.rs @@ -1,9 +1,12 @@ use std::sync::Arc; use axum::{extract::Path, Json}; -use chrono::Utc; +use chrono::{Duration as ChronoDuration, Utc}; use zksync_config::configs::ProofDataHandlerConfig; -use zksync_dal::{ConnectionPool, Core, CoreDal}; +use zksync_dal::{ + tee_proof_generation_dal::{LockedBatch, TeeProofGenerationJobStatus}, + ConnectionPool, Core, CoreDal, +}; use zksync_object_store::{ObjectStore, ObjectStoreError}; use zksync_prover_interface::{ api::{ @@ -48,49 +51,62 @@ impl TeeRequestProcessor { ) -> Result>, RequestProcessorError> { tracing::info!("Received request for proof generation data: {:?}", request); - let mut min_batch_number = self.config.tee_config.first_tee_processed_batch; - let mut missing_range: Option<(L1BatchNumber, L1BatchNumber)> = None; + let batch_ignored_timeout = ChronoDuration::from_std( + self.config + .tee_config + .tee_batch_permanently_ignored_timeout(), + ) + .map_err(|err| { + RequestProcessorError::GeneralError(format!( + "Failed to convert batch_ignored_timeout: {}", + err + )) + })?; + let min_batch_number = self.config.tee_config.first_tee_processed_batch; - let result = loop { - let Some(l1_batch_number) = self + loop { + let Some(locked_batch) = self .lock_batch_for_proving(request.tee_type, min_batch_number) .await? else { - // No job available - return Ok(None); + break Ok(None); // no job available }; + let batch_number = locked_batch.l1_batch_number; match self - .tee_verifier_input_for_existing_batch(l1_batch_number) + .tee_verifier_input_for_existing_batch(batch_number) .await { Ok(input) => { break Ok(Some(Json(TeeProofGenerationDataResponse(Box::new(input))))); } Err(RequestProcessorError::ObjectStore(ObjectStoreError::KeyNotFound(_))) => { - missing_range = match missing_range { - Some((start, _)) => Some((start, l1_batch_number)), - None => Some((l1_batch_number, l1_batch_number)), + let duration = Utc::now().signed_duration_since(locked_batch.created_at); + let status = if duration > batch_ignored_timeout { + TeeProofGenerationJobStatus::PermanentlyIgnored + } else { + TeeProofGenerationJobStatus::Failed }; - self.unlock_batch(l1_batch_number, request.tee_type).await?; - min_batch_number = l1_batch_number + 1; + self.unlock_batch(batch_number, request.tee_type, status) + .await?; + tracing::warn!( + "Assigned status {} to batch {} created at {}", + status, + batch_number, + locked_batch.created_at + ); } Err(err) => { - self.unlock_batch(l1_batch_number, request.tee_type).await?; + self.unlock_batch( + batch_number, + request.tee_type, + TeeProofGenerationJobStatus::Failed, + ) + .await?; break Err(err); } } - }; - - if let Some((start, end)) = missing_range { - tracing::warn!( - "Blobs for batch numbers {} to {} not found in the object store. Marked as unpicked.", - start, - end - ); } - - result } #[tracing::instrument(skip(self))] @@ -158,7 +174,7 @@ impl TeeRequestProcessor { &self, tee_type: TeeType, min_batch_number: L1BatchNumber, - ) -> Result, RequestProcessorError> { + ) -> Result, RequestProcessorError> { self.pool .connection_tagged("tee_request_processor") .await? @@ -176,12 +192,13 @@ impl TeeRequestProcessor { &self, l1_batch_number: L1BatchNumber, tee_type: TeeType, + status: TeeProofGenerationJobStatus, ) -> Result<(), RequestProcessorError> { self.pool .connection_tagged("tee_request_processor") .await? .tee_proof_generation_dal() - .unlock_batch(l1_batch_number, tee_type) + .unlock_batch(l1_batch_number, tee_type, status) .await?; Ok(()) } diff --git a/core/node/proof_data_handler/src/tests.rs b/core/node/proof_data_handler/src/tests.rs index 87c6bff8a1f4..dae2ef8cd0c0 100644 --- a/core/node/proof_data_handler/src/tests.rs +++ b/core/node/proof_data_handler/src/tests.rs @@ -6,12 +6,13 @@ use axum::{ }; use serde_json::json; use tower::ServiceExt; -use zksync_basic_types::L2ChainId; use zksync_config::configs::{ProofDataHandlerConfig, TeeConfig}; use zksync_dal::{ConnectionPool, CoreDal}; use zksync_object_store::MockObjectStore; use zksync_prover_interface::api::SubmitTeeProofRequest; -use zksync_types::{commitment::L1BatchCommitmentMode, tee_types::TeeType, L1BatchNumber}; +use zksync_types::{ + commitment::L1BatchCommitmentMode, tee_types::TeeType, L1BatchNumber, L2ChainId, +}; use crate::create_proof_processing_router; @@ -29,6 +30,7 @@ async fn request_tee_proof_inputs() { tee_support: true, first_tee_processed_batch: L1BatchNumber(0), tee_proof_generation_timeout_in_secs: 600, + tee_batch_permanently_ignored_timeout_in_hours: 10 * 24, }, }, L1BatchCommitmentMode::Rollup, @@ -88,6 +90,7 @@ async fn submit_tee_proof() { tee_support: true, first_tee_processed_batch: L1BatchNumber(0), tee_proof_generation_timeout_in_secs: 600, + tee_batch_permanently_ignored_timeout_in_hours: 10 * 24, }, }, L1BatchCommitmentMode::Rollup, @@ -119,7 +122,7 @@ async fn submit_tee_proof() { let mut proof_db_conn = db_conn_pool.connection().await.unwrap(); let oldest_batch_number = proof_db_conn .tee_proof_generation_dal() - .get_oldest_unpicked_batch() + .get_oldest_picked_by_prover_batch() .await .unwrap(); @@ -156,7 +159,7 @@ async fn mock_tee_batch_status( // there should not be any batches awaiting proof in the db yet - let oldest_batch_number = proof_dal.get_oldest_unpicked_batch().await.unwrap(); + let oldest_batch_number = proof_dal.get_oldest_picked_by_prover_batch().await.unwrap(); assert!(oldest_batch_number.is_none()); // mock SQL table with relevant information about the status of TEE proof generation @@ -169,7 +172,7 @@ async fn mock_tee_batch_status( // now, there should be one batch in the db awaiting proof let oldest_batch_number = proof_dal - .get_oldest_unpicked_batch() + .get_oldest_picked_by_prover_batch() .await .unwrap() .unwrap(); diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index c0dd8638c8d9..ca34a5190d3a 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -79,6 +79,10 @@ - [Fee Model](specs/zk_evm/fee_model.md) - [Precompiles](specs/zk_evm/precompiles.md) - [System Contracts](specs/zk_evm/system_contracts.md) +- [Interop](specs/interop/overview.md) + - [Interop Messages](specs/interop/interopmessages.md) + - [Bundles and Calls](specs/interop/bundlesandcalls.md) + - [Interop Transactions](specs/interop/interoptransactions.md) # Announcements diff --git a/docs/src/guides/advanced/16_decentralization.md b/docs/src/guides/advanced/16_decentralization.md index 6037235ea064..a5f889a813d0 100644 --- a/docs/src/guides/advanced/16_decentralization.md +++ b/docs/src/guides/advanced/16_decentralization.md @@ -8,7 +8,7 @@ and enabled as follows: Run the following to generate consensus secrets: ``` -docker run --entrypoint /usr/bin/zksync_external_node "matterlabs/external-node:2.0-v25.1.0" generate-secrets > consensus_secrets.yaml +docker run --entrypoint /usr/bin/zksync_external_node "matterlabs/external-node:2.0-v25.0.0" generate-secrets > consensus_secrets.yaml chmod 600 consensus_secrets.yaml ``` @@ -55,12 +55,12 @@ long) to achieve that: ## Gitops repo config -If you are using the matterlabs gitops repo to configure the main node, it is even more complicated becase the +If you are using the matterlabs gitops repo to configure the main node, it is even more complicated because the `consensus_config.yaml` file is rendered from a helm chart. See the [example](https://github.com/matter-labs/gitops-kubernetes/blob/main/apps/environments/mainnet2/server-v2/server-v2-core.yaml), to see where you have to paste the content of the `consensus_config.yaml` file. -You need to embed the `consenus_secrets.yaml` file into a kubernetes config: +You need to embed the `consensus_secrets.yaml` file into a kubernetes config: ```yaml apiVersion: v1 diff --git a/docs/src/guides/build-docker.md b/docs/src/guides/build-docker.md index 5dd9cff022b9..6b0608275d8b 100644 --- a/docs/src/guides/build-docker.md +++ b/docs/src/guides/build-docker.md @@ -10,7 +10,8 @@ Install prerequisites: see ## Build docker files -You may build all images with [Makefile](../../docker/Makefile) located in [docker](../../docker) directory in this repository +You may build all images with [Makefile](../../docker/Makefile) located in [docker](../../docker) directory in this +repository > All commands should be run from the root directory of the repository diff --git a/docs/src/specs/img/autoexecution.png b/docs/src/specs/img/autoexecution.png new file mode 100644 index 000000000000..bef85e282012 Binary files /dev/null and b/docs/src/specs/img/autoexecution.png differ diff --git a/docs/src/specs/img/callride.png b/docs/src/specs/img/callride.png new file mode 100644 index 000000000000..568aab9c7242 Binary files /dev/null and b/docs/src/specs/img/callride.png differ diff --git a/docs/src/specs/img/chainop.png b/docs/src/specs/img/chainop.png new file mode 100644 index 000000000000..bd18e98b0e1a Binary files /dev/null and b/docs/src/specs/img/chainop.png differ diff --git a/docs/src/specs/img/finaltx.png b/docs/src/specs/img/finaltx.png new file mode 100644 index 000000000000..c3bd287c0465 Binary files /dev/null and b/docs/src/specs/img/finaltx.png differ diff --git a/docs/src/specs/img/gateway.png b/docs/src/specs/img/gateway.png new file mode 100644 index 000000000000..4d18d38ba338 Binary files /dev/null and b/docs/src/specs/img/gateway.png differ diff --git a/docs/src/specs/img/globalroot.png b/docs/src/specs/img/globalroot.png new file mode 100644 index 000000000000..3152cbec4e81 Binary files /dev/null and b/docs/src/specs/img/globalroot.png differ diff --git a/docs/src/specs/img/interopcall.png b/docs/src/specs/img/interopcall.png new file mode 100644 index 000000000000..b683122f1363 Binary files /dev/null and b/docs/src/specs/img/interopcall.png differ diff --git a/docs/src/specs/img/interopcallbundle.png b/docs/src/specs/img/interopcallbundle.png new file mode 100644 index 000000000000..fc82f02bc69c Binary files /dev/null and b/docs/src/specs/img/interopcallbundle.png differ diff --git a/docs/src/specs/img/interopmsg.png b/docs/src/specs/img/interopmsg.png new file mode 100644 index 000000000000..5469b1a4e3ff Binary files /dev/null and b/docs/src/specs/img/interopmsg.png differ diff --git a/docs/src/specs/img/interoptx.png b/docs/src/specs/img/interoptx.png new file mode 100644 index 000000000000..3d9fe295fe4e Binary files /dev/null and b/docs/src/specs/img/interoptx.png differ diff --git a/docs/src/specs/img/ipointers.png b/docs/src/specs/img/ipointers.png new file mode 100644 index 000000000000..60c29e601237 Binary files /dev/null and b/docs/src/specs/img/ipointers.png differ diff --git a/docs/src/specs/img/levelsofinterop.png b/docs/src/specs/img/levelsofinterop.png new file mode 100644 index 000000000000..5ee15946935d Binary files /dev/null and b/docs/src/specs/img/levelsofinterop.png differ diff --git a/docs/src/specs/img/msgdotsender.png b/docs/src/specs/img/msgdotsender.png new file mode 100644 index 000000000000..b411690785db Binary files /dev/null and b/docs/src/specs/img/msgdotsender.png differ diff --git a/docs/src/specs/img/paymastertx.png b/docs/src/specs/img/paymastertx.png new file mode 100644 index 000000000000..b3f25936b8dd Binary files /dev/null and b/docs/src/specs/img/paymastertx.png differ diff --git a/docs/src/specs/img/proofmerklepath.png b/docs/src/specs/img/proofmerklepath.png new file mode 100644 index 000000000000..beaaa175e1d0 Binary files /dev/null and b/docs/src/specs/img/proofmerklepath.png differ diff --git a/docs/src/specs/img/retryexample.png b/docs/src/specs/img/retryexample.png new file mode 100644 index 000000000000..6c79ef637340 Binary files /dev/null and b/docs/src/specs/img/retryexample.png differ diff --git a/docs/src/specs/img/sendtol1.png b/docs/src/specs/img/sendtol1.png new file mode 100644 index 000000000000..b98e51879f34 Binary files /dev/null and b/docs/src/specs/img/sendtol1.png differ diff --git a/docs/src/specs/img/verifyinteropmsg.png b/docs/src/specs/img/verifyinteropmsg.png new file mode 100644 index 000000000000..c59e5607f7da Binary files /dev/null and b/docs/src/specs/img/verifyinteropmsg.png differ diff --git a/docs/src/specs/interop/README.md b/docs/src/specs/interop/README.md new file mode 100644 index 000000000000..88e3d514b562 --- /dev/null +++ b/docs/src/specs/interop/README.md @@ -0,0 +1,6 @@ +# Interop + +- [Overview](./overview.md) +- [Interop Messages](./interopmessages.md) +- [Bundles and Calls](./bundlesandcalls.md) +- [Interop Transactions](./interoptransactions.md) diff --git a/docs/src/specs/interop/bundlesandcalls.md b/docs/src/specs/interop/bundlesandcalls.md new file mode 100644 index 000000000000..e49e10e8eed2 --- /dev/null +++ b/docs/src/specs/interop/bundlesandcalls.md @@ -0,0 +1,278 @@ +# Bundles and Calls + +## Basics Calls + +Interop Calls are the next level of interfaces, built on top of Interop Messages, enabling you to call contracts on +other chains. + +![interopcall.png](../img/interopcall.png) + +At this level, the system handles replay protection—once a call is successfully executed, it cannot be executed again +(eliminating the need for your own nullifiers or similar mechanisms). + +Additionally, these calls originate from aliased accounts, simplifying permission management (more details on this +below). + +Cancellations and retries are managed at the next level (Bundles), which are covered in the following section. + +### Interface + +On the sending side, the interface provides the option to send this "call" to the destination contract. + +```solidity +struct InteropCall { + address sourceSender, + address destinationAddress, + uint256 destinationChainId, + calldata data, + uint256 value +} +contract InteropCenter { + // On source chain. + // Sends a 'single' basic internal call to destination chain & address. + // Internally, it starts a bundle, adds this call and sends it over. + function sendCall(destinationChain, destinationAddress, calldata, msgValue) returns bytes32 bundleId; +} +``` + +In return, you receive a `bundleId` (we’ll explain bundles later, but for now, think of it as a unique identifier for +your call). + +On the destination chain, you can execute the call using the execute method: + +```solidity +contract InteropCenter { + // Executes a given bundle. + // interopMessage is the message that contains your bundle as payload. + // If it fails, it can be called again. + function executeInteropBundle(interopMessage, proof); + + // If the bundle didn't execute succesfully yet, it can be marked as cancelled. + // See details below. + function cancelInteropBundle(interopMessage, proof); +} + +``` + +You can retrieve the `interopMessage` (which contains your entire payload) from the Gateway, or you can construct it +yourself using L1 data. + +Under the hood, this process calls the `destinationAddress` with the specified calldata. + +This leads to an important question: **Who is the msg.sender for this call?** + +## `msg.sender` of the Destination Call + +The `msg.sender` on the destination chain will be the **AliasedAccount** — an address created as a hash of the original +sender and the original source chain. + +(Normally, we’d like to use `sourceAccount@sourceChain`, but since Ethereum limits the size of addresses to 20 bytes, we +compute the Keccak hash of the string above and use this as the address.) + +One way to think about it is this: You (as account `0x5bFF1...` on chain A) can send a call to a contract on a +destination chain, and for that contract, it will appear as if the call came locally from the address +`keccak(0x5bFF1 || A)`. This means you are effectively "controlling" such an account address on **every ZK Chain** by +sending interop messages from the `0x5bFF1...` account on chain A. + +![msgdotsender.png](../img/msgdotsender.png) + +## Simple Example + +Imagine you have contracts on chains B, C, and D, and you’d like them to send "reports" to the Headquarters (HQ) +contract on chain A every time a customer makes a purchase. + +```solidity +// Deployed on chains B, C, D. +contract Shop { + /// Called by the customers when they buy something. + function buy(uint256 itemPrice) { + // handle payment etc. + ... + // report to HQ + InteropCenter(INTEROP_ADDRESS).sendCall( + 324, // chain id of chain A, + 0xc425.., // HQ contract on chain A, + createCalldata("reportSales(uint256)", itemPrice), // calldata + 0, // no value + ); + } +} + +// Deployed on chain A +contract HQ { + // List of shops + mapping (address => bool) shops; + mapping (address => uint256) sales; + function addShop(address addressOnChain, uint256 chainId) onlyOwner { + // Adding aliased accounts. + shops[address(keccak(addressOnChain || chainId))] = true; + } + + function reportSales(uint256 itemPrice) { + // only allow calls from our shops (their aliased accounts). + require(shops[msg.sender]); + sales[msg.sender] += itemPrice; + } +} +``` + +#### Who is paying for gas? How does this Call get to the destination chain + +At this level, the **InteropCall** acts like a hitchhiker — it relies on someone (anyone) to pick it up, execute it, and +pay for the gas! + +![callride.png](../img/callride.png) + +While any transaction on the destination chain can simply call `InteropCenter.executeInteropBundle`, if you don’t want +to rely on hitchhiking, you can create one yourself. We’ll discuss this in the section about **Interop Transactions**. + +## Bundles + +Before we proceed to discuss **InteropTransactions**, there is one more layer in between: **InteropBundles**. + +![interopcallbundle.png](../img/interopcallbundle.png) + +**Bundles Offer:** + +- **Shared Fate**: All calls in the bundle either succeed or fail together. +- **Retries**: If a bundle fails, it can be retried (e.g., with more gas). +- **Cancellations**: If a bundle has not been successfully executed yet, it can be cancelled. + +If you look closely at the interface we used earlier, you’ll notice that we were already discussing the execution of +**Bundles** rather than single calls. So, let’s dive into what bundles are and the role they fulfill. + +The primary purpose of a bundle is to ensure that a given list of calls is executed in a specific order and has a shared +fate (i.e., either all succeed or all fail). + +In this sense, you can think of a bundle as a **"multicall"**, but with two key differences: + +1. You cannot "unbundle" items—an individual `InteropCall` cannot be run independently; it is tightly tied to the + bundle. + +2. Each `InteropCall` within a bundle can use a different aliased account, enabling separate permissions for each call. + +```solidity +contract InteropCenter { + struct InteropBundle { + // Calls have to be done in this order. + InteropCall calls[]; + uint256 destinationChain; + + // If not set - anyone can execute it. + address executionAddresses[]; + // Who can 'cancel' this bundle. + address cancellationAddress; + } + + // Starts a new bundle. + // All the calls that will be added to this bundle (potentially by different contracts) + // will have a 'shared fate'. + // The whole bundle must be going to a single destination chain. + function startBundle(destinationChain) returns bundleId; + // Adds a new call to the opened bundle. + // Returns the messageId of this single message in the bundle. + function addToBundle(bundleId, destinationAddress, calldata, msgValue) return msgHash; + // Finishes a given bundle, and sends it. + function finishAndSendBundle(bundleId) return msgHash; +} +``` + +### Cross Chain Swap Example + +Imagine you want to perform a swap on chain B, exchanging USDC for PEPE, but all your assets are currently on chain A. + +This process would typically involve four steps: + +1. Transfer USDC from chain A to chain B. +2. Set allowance for the swap. +3. Execute the swap. +4. Transfer PEPE back to chain A. + +Each of these steps is a separate "call," but you need them to execute in exactly this order and, ideally, atomically. +If the swap fails, you wouldn’t want the allowance to remain set on the destination chain. + +Below is an example of how this process could look (note that the code is pseudocode; we’ll explain the helper methods +required to make it work in a later section). + +```solidity +bundleId = InteropCenter(INTEROP_CENTER).startBundle(chainD); +// This will 'burn' the 1k USDC, create the special interopCall +// when this call is executed on chainD, it will mint 1k USDC there. +// BUT - this interopCall is tied to this bundle id. +USDCBridge.transferWithBundle( + bundleId, + chainD, + aliasedAccount(this(account), block.chain_id), + 1000); + + +// This will create interopCall to set allowance. +InteropCenter.addToBundle(bundleId, + USDCOnDestinationChain, + createCalldata("approve", 1000, poolOnDestinationChain), + 0); +// This will create interopCall to do the swap. +InteropCenter.addToBundle(bundleId, + poolOnDestinationChain, + createCalldata("swap", "USDC_PEPE", 1000, ...), + 0) +// And this will be the interopcall to transfer all the assets back. +InteropCenter.addToBundle(bundleId, + pepeBridgeOnDestinationChain, + createCalldata("transferAll", block.chain_id, this(account)), + 0) + + +bundleHash = interopCenter.finishAndSendBundle(bundleId); +``` + +In the code above, we created a bundle that anyone can execute on the destination chain. This bundle will handle the +entire process: minting, approving, swapping, and transferring back. + +### Bundle Restrictions + +When starting a bundle, if you specify the `executionAddress`, only that account will be able to execute the bundle on +the destination chain. If no `executionAddress` is specified, anyone can trigger the execution. + +## Retries and Cancellations + +If bundle execution fails — whether due to a contract error or running out of gas—none of its calls will be applied. The +bundle can be re-run on the **destination chain** without requiring any updates or notifications to the source chain. +More details about retries and gas will be covered in the next level, **Interop Transactions**. + +This process can be likened to a "hitchhiker" (or in the case of a bundle, a group of hitchhikers) — if the car they’re +traveling in doesn’t reach the destination, they simply find another ride rather than returning home. + +However, there are cases where the bundle should be cancelled. Cancellation can be performed by the +`cancellationAddress` specified in the bundle itself. + +#### For our cross chain swap example + +1. Call `cancelInteropBundle(interopMessage, proof)` on the destination chain. + - A helper method for this will be introduced in the later section. +2. When cancellation occurs, the destination chain will generate an `InteropMessage` containing cancellation + information. +3. Using the proof from this method, the user can call the USDC bridge to recover their assets: + +```solidity +USDCBridge.recoverFailedTransfer(bundleId, cancellationMessage, proof); +``` + +### Some details on our approach + +#### Destination Contract + +- On ElasticChain, the destination contract does not need to know it is being called via an interop call. Requests + arrive from `aliased accounts'. + +#### Batching + +- ElasticChain supports bundling of messages, ensuring shared fate and strict order. + +#### Execution Permissions + +- ElasticChain allows restricting who can execute the call or bundle on the destination chain. + +#### Cancellations + +- ElasticChain supports restricting who can cancel. Cancellation can happen at any time. diff --git a/docs/src/specs/interop/interopmessages.md b/docs/src/specs/interop/interopmessages.md new file mode 100644 index 000000000000..cabfd0e56750 --- /dev/null +++ b/docs/src/specs/interop/interopmessages.md @@ -0,0 +1,181 @@ +# Interop Messages + +In this section, we’re going to cover the lowest level of the interop stack: **Interop Messages** — the interface that +forms the foundation for everything else. + +We’ll explore the details of the interface, its use cases, and how it compares to similar interfaces from +Superchain/Optimism. + +This is an advanced document. While most users and app developers typically interact with higher levels of interop, it’s +still valuable to understand how the internals work. + +## Basics + +![interopmsg.png](../img/interopmsg.png) + +Interop Messages are the lowest level of our stack. + +An **InteropMessage** contains data and offers two methods: + +- Send a message +- Verify that a given message was sent on some chain + +Notice that the message itself doesn’t have any ‘destination chain’ or address—it is simply a payload that a user (or +contract) is creating. Think of it as a broadcast. + +The `InteropCenter` is a contract that is pre-deployed on all chains at a fixed address `0x00..1234`. + +```solidity +contract InteropCenter { + // Sends interop message. Can be called by anyone. + // Returns the unique interopHash. + function sendInteropMessage(bytes data) returns interopHash; + + // Interop message - uniquely identified by the hash of the payload. + struct InteropMessage { + bytes data; + address sender; // filled by InteropCenter + uint256 sourceChainId; // filled by InteropCenter + uint256 messageNum; // a 'nonce' to guarantee different hashes. + } + + // Verifies if such interop message was ever producted. + function verifyInteropMessage(bytes32 interopHash, Proof merkleProof) return bool; +} +``` + +When you call `sendInteropMessage`, the `InteropCenter` adds additional fields, such as your sender address, source +chain ID, and messageNum (a nonce ensuring the hash of this structure is globally unique). It then returns the +`interopHash`. + +This `interopHash` serves as a globally unique identifier that can be used on any chain in the network to call +`verifyInteropMessage`. + +![A message created on one chain can be verified on any other chain.](../img/verifyinteropmsg.png) + +#### How do I get the proof + +You’ll notice that **verifyInteropMessage** has a second argument — a proof that you need to provide. This proof is a +Merkle tree proof (more details below). You can obtain it by querying the Settlement Layer (Gateway) or generating it +off-chain by examining the Gateway state on L1. + +#### How does the interop message differ from other layers (InteropTransactions, InteropCalls) + +As the most basic layer, an interop message doesn’t include any advanced features — it lacks support for selecting +destination chains, nullifiers/replay, cancellation, and more. + +If you need these capabilities, consider integrating with a higher layer of interop, such as Call or Bundle, which +provide these additional functionalities. + +## Simple Use Case + +Before we dive into the details of how the system works, let’s look at a simple use case for a DApp that decides to use +InteropMessage. + +For this example, imagine a basic cross-chain contract where the `signup()` method can be called on chains B, C, and D +only if someone has first called `signup_open()` on chain A. + +```solidity +// Contract deployed on chain A. +contract SignupManager { + public bytes32 sigup_open_msg_hash; + function signup_open() onlyOwner { + // We are open for business + signup_open_msg_hash = InteropCenter(INTEROP_CENTER_ADDRESS).sendInteropMessage("We are open"); + } +} + +// Contract deployed on all other chains. +contract SignupContract { + public bool signupIsOpen; + // Anyone can call it. + function openSignup(InteropMessage message, InteropProof proof) { + InteropCenter(INTEROP_CENTER_ADDRESS).verifyInteropMessage(keccak(message), proof); + require(message.sourceChainId == CHAIN_A_ID); + require(message.sender == SIGNUP_MANAGER_ON_CHAIN_A); + require(message.data == "We are open"); + signupIsOpen = true; + } + + function signup() { + require(signupIsOpen); + signedUpUser[msg.sender] = true; + } +} +``` + +In the example above, the `signupManager` on chain A calls the `signup_open` method. After that, any user on other +chains can retrieve the `signup_open_msg_hash`, obtain the necessary proof from the Gateway (or another source), and +call the `openSignup` function on any destination chain. + +## Deeper Technical Dive + +Let’s break down what happens inside the InteropCenter when a new interop message is created: + +```solidity +function sendInteropMessage(bytes data) { + messageNum += 1; + msg = InteropMessage({data, msg.sender, block.chain_id, messageNum}); + // Does L2->L1 Messaging. + sendToL1(abi.encode(msg)); + return keccak(msg); +} +``` + +As you can see, it populates the necessary data and then calls the `sendToL1` method. + +The `sendToL1` method is part of a system contract that gathers all messages during a batch, constructs a Merkle tree +from them at the end of the batch, and sends this tree to the SettlementLayer (Gateway) when the batch is committed. + +![sendtol1.png](../img/sendtol1.png) + +The Gateway will verify the hashes of the messages to ensure it has received the correct preimages. Once the proof for +the batch is submitted (or more accurately, during the "execute" step), it will add the root of the Merkle tree to its +`globalRoot`. + +![globalroot.png](../img/globalroot.png) + +The `globalRoot` is the root of the Merkle tree that includes all messages from all chains. Each chain regularly reads +the globalRoot value from the Gateway to stay synchronized. + +![gateway.png](../img/gateway.png) + +If a user wants to call `verifyInteropMessage` on a chain, they first need to query the Gateway for the Merkle path from +the batch they are interested in up to the `globalRoot`. Once they have this path, they can provide it as an argument +when calling a method on the destination chain (such as the `openSignup` method in our example). + +![proofmerklepath.png](../img/proofmerklepath.png) + +#### What if the Gateway doesn’t respond + +If the Gateway doesn’t respond, users can manually re-create the Merkle proof using data available on L1. Every +interopMessage is also sent to L1. + +#### Global roots change frequently + +Yes, global roots update continuously as new chains prove their blocks. However, chains retain historical global roots +for a reasonable period (around 24 hours) to ensure that recently generated Merkle paths remain valid. + +#### Is this secure? Could a chain operator, like Chain D, use a different global root + +Yes, it’s secure. If a malicious operator on Chain D attempted to use a different global root, they wouldn’t be able to +submit the proof for their new batch to the Gateway. This is because the proof’s public inputs must include the valid +global root. + +#### What if the Gateway is malicious + +If the Gateway behaves maliciously, it wouldn’t be able to submit its batches to L1, as the proof would fail +verification. A separate section will cover interop transaction security in more detail. + +### Other Features + +#### Dependency Set + +- In ElasticChain, this is implicitly handled by the Gateway. Any chain that is part of the global root can exchange + messages with any other chain, effectively forming an undirected graph. + +#### Timestamps and Expiration + +- In ElasticChain, older messages become increasingly difficult to validate as it becomes harder to gather the data + required to construct a Merkle proof. Expiration is also being considered for this reason, but the specifics are yet + to be determined. diff --git a/docs/src/specs/interop/interoptransactions.md b/docs/src/specs/interop/interoptransactions.md new file mode 100644 index 000000000000..feccc49ea91a --- /dev/null +++ b/docs/src/specs/interop/interoptransactions.md @@ -0,0 +1,196 @@ +# Interop Transactions + +## Basics + +The **InteropTransaction** sits at the top of our interop stack, acting as the “delivery” mechanism for **Interop +Bundles**. + +Think of it like a car that picks up our "hitchhiker" bundles and carries them to their destination. + +![interoptx.png](../img/interoptx.png) + +**Note:** Interop Transactions aren’t the only way to execute a bundle. Once an interop bundle is created on the source +chain, users can simply send a regular transaction on the destination chain to execute it. + +However, this approach can be inconvenient as it requires users to have funds on the destination chain to cover gas fees +and to configure the necessary network settings (like the RPC address). + +**InteropTransactions** simplify this process by handling everything from the source chain. They allow you to select +which **interopBundle** to execute, specify gas details (such as gas amount and gas price), and determine who will cover +the gas costs. This can be achieved using tokens on the source chain or through a paymaster. + +Once configured, the transaction will automatically execute, either by the chain operator, the gateway, or off-chain +tools. + +An **InteropTransaction** contains two pointers to bundles: + +- **feesBundle**: Holds interop calls to cover fees. +- **bundleHash**: Contains the main execution. + +![ipointers.png](../img/ipointers.png) + +## Interface + +The function `sendInteropTransaction` provides all the options. For simpler use cases, refer to the helper methods +defined later in the article. + +```solidity +contract InteropCenter { + /// Creates a transaction that will attempt to execute a given Bundle on the destination chain. + /// Such transaction can be 'picked up' by the destination chain automatically. + /// This function covers all the cases - we expect most users to use the helper + /// functions defined later. + function sendInteropTransaction( + destinationChain, + bundleHash, // the main bundle that you want to execute on destination chain + gasLimit, // gasLimit & price for execution + gasPrice, + feesBundleHash, // this is the bundle that contains the calls to pay for gas + destinationPaymaster, // optionally - you can use a paymaster on destination chain + destinationPaymasterInput); // with specific params + + + struct InteropTransaction { + address sourceChainSender + uint256 destinationChain + uint256 gasLimit; + uint256 gasPrice; + uint256 value; + bytes32 bundleHash; + bytes32 feesBundleHash; + address destinationPaymaster; + bytes destinationPaymasterInput; + } +} +``` + +After creating the **InteropBundle**, you can simply call `sendInteropTransaction` to create the complete transaction +that will execute the bundle. + +## Retries + +If your transaction fails to execute the bundle (e.g., due to a low gas limit) or isn’t included at all (e.g., due to +too low gasPrice), you can send another transaction to **attempt to execute the same bundle again**. + +Simply call `sendInteropTransaction` again with updated gas settings. + +### Example of Retrying + +Here’s a concrete example: Suppose you created a bundle to perform a swap that includes transferring 100 ETH, executing +the swap, and transferring some tokens back. + +You attempted to send the interop transaction with a low gas limit (e.g., 100). Since you didn’t have any base tokens on +the destination chain, you created a separate bundle to transfer a small fee (e.g., 0.0001) to cover the gas. + +You sent your first interop transaction to the destination chain, but it failed due to insufficient gas. However, your +“fee bundle” was successfully executed, as it covered the gas cost for the failed attempt. + +Now, you have two options: either cancel the execution bundle (the one with 100 ETH) or retry. + +To retry, you decide to set a higher gas limit (e.g., 10,000) and create another fee transfer (e.g., 0.01) but use **the +same execution bundle** as before. + +This time, the transaction succeeds — the swap completes on the destination chain, and the resulting tokens are +successfully transferred back to the source chain. + +![retryexample.png](../img/retryexample.png) + +## Fees & Restrictions + +Using an **InteropBundle** for fee payments offers flexibility, allowing users to transfer a small amount to cover the +fees while keeping the main assets in the execution bundle itself. + +### Restrictions + +This flexibility comes with trade-offs, similar to the validation phases in **Account Abstraction** or **ERC4337**, +primarily designed to prevent DoS attacks. Key restrictions include: + +- **Lower gas limits** +- **Limited access to specific slots** + +Additionally, when the `INTEROP_CENTER` constructs an **InteropTransaction**, it enforces extra restrictions on +**feePaymentBundles**: + +- **Restricted Executors**: + Only your **AliasedAccount** on the receiving side can execute the `feePaymentBundle`. + +This restriction is crucial for security, preventing others from executing your **fee bundle**, which could cause your +transaction to fail and prevent the **execution bundle** from processing. + +### **Types of Fees** + +#### Using the Destination Chain’s Base Token + +The simplest scenario is when you (as the sender) already have the destination chain’s base token available on the +source chain. + +For example: + +- If you are sending a transaction from **Era** (base token: ETH) to **Sophon** (base token: SOPH) and already have SOPH + on ERA, you can use it for the fee. + +To make this easier, we’ll provide a helper function: + +```solidity +contract InteropCenter { + // Creates InteropTransaction to the destination chain with payment with base token. + // Before calling, you have to 'approve' InteropCenter to the ERC20/Bridge that holds the destination chain's base tokens. + // or if the destination chain's tokens are the same as yours, just attach value to this call. + function sendInteropTxMinimal( + destinationChain, + bundleHash, // the main bundle that you want to execute on destination chain + gasLimit, // gasLimit & price for execution + gasPrice, + ); + } +``` + +#### Using paymaster on the destination chain + +If you don’t have the base token from the destination chain (e.g., SOPH in our example) on your source chain, you’ll +need to use a paymaster on the destination chain instead. + +In this case, you’ll send the token you do have (e.g., USDC) to the destination chain as part of the **feeBundleHash**. +Once there, you’ll use it to pay the paymaster on the destination chain to cover your gas fees. + +Your **InteropTransaction** would look like this: + +![paymastertx.png](../img/paymastertx.png) + +## **Automatic Execution** + +One of the main advantages of **InteropTransactions** is that they execute automatically. As the sender on the source +chain, you don’t need to worry about technical details like RPC addresses or obtaining proofs — it’s all handled for +you. + +After creating an **InteropTransaction**, it can be relayed to the destination chain by anyone. The transaction already +includes a signature (also known as an interop message proof), making it fully self-contained and ready to send without +requiring additional permissions. + +Typically, the destination chain’s operator will handle and include incoming **InteropTransactions**. However, if they +don’t, the **Gateway** or other participants can step in to prepare and send them. + +You can also use the available tools to create and send the destination transaction yourself. Since the transaction is +self-contained, it doesn’t require additional funds or signatures to execute. + +![Usually destination chain operator will keep querying gateway to see if there are any messages for their chain.](../img/autoexecution.png) + +Once they see the message, they can request the proof from the **Gateway** and also fetch the **InteropBundles** +contained within the message (along with their respective proofs). + +![Operator getting necessary data from Gateway.](../img/chainop.png) + +As the final step, the operator can use the received data to create a regular transaction, which can then be sent to +their chain. + +![Creating the final transaction to send to the destination chain](../img/finaltx.png) + +The steps above don’t require any special permissions and can be executed by anyone. + +While the **Gateway** was used above for tasks like providing proofs, if the Gateway becomes malicious, all this +information can still be constructed off-chain using data available on L1. + +### How it Works Under the hood + +We’ll modify the default account to accept interop proofs as signatures, seamlessly integrating with the existing ZKSync +native **Account Abstraction** model. diff --git a/docs/src/specs/interop/overview.md b/docs/src/specs/interop/overview.md new file mode 100644 index 000000000000..8ca28723e03a --- /dev/null +++ b/docs/src/specs/interop/overview.md @@ -0,0 +1,166 @@ +# Intro Guide to Interop + +## What is Interop + +Interop is a way to communicate and transact between two ZK Stack chains. It allows you to: + +**1. Observe messages:** Track when an interop message (think of it as a special event) is created on the source chain. + +**2. Send assets:** Transfer ERC20 tokens and other assets between chains. + +**3. Execute calls:** Call a contract on a remote chain with specific calldata and value. + +With interop, you automatically get an account (a.k.a. aliasedAccount) on each chain, which you can control from the +source chain. + +**4. Execute bundles of calls:** Group multiple remote calls into a single bundle, ensuring all of them execute at once. + +**5. Execute transactions:** Create transactions on the source chain, which will automatically get executed on the +destination chain, with options to choose from various cross-chain Paymaster solutions to handle gas fees. + +## How to Use Interop + +Here’s a simple example of calling a contract on a destination chain: + +```solidity +cast send source-chain-rpc.com INTEROP_CENTER_ADDRESS sendInteropWithSingleCall( + 0x1fa72e78 // destination_chain_id, + 0xb4AB2FF34fa... // destination_contract, + 0x29723511000000... // destination_calldata, + 0, // value + 100_000, // gasLimit + 250_000_000, // gasPrice + ) +``` + +While this looks very similar to a 'regular' call, there are some nuances, especially around handling failures and +errors. + +Let’s explore these key details together. + +## Common Questions and Considerations + +#### 1. Who pays for gas + +When using this method, your account must hold `gasLimit * gasPrice` worth of destination chain tokens on the source +chain. + +For example, if you’re sending the request from Era and the destination chain is Sophon (with SOPH tokens), you’ll need +SOPH tokens available on Era. + +Additional payment options are available, which will be covered in later sections. + +#### 2. How does the destination contract know it’s from me + +The destination contract will see `msg.sender` as `keccak(source_account, source_chain)[:20]`. + +Ideally, we would use something like `source_account@source_chain` (similar to an email format), but since Ethereum +addresses are limited to 20 bytes, we use a Keccak hash to fit this constraint. + +#### 3. Who executes it on the destination chain + +The call is auto-executed on the destination chain. As a user, you don’t need to take any additional actions. + +#### 4. What if it runs out of gas or the gasPrice is set too low + +In either scenario, you can retry the transaction using the `retryInteropTransaction` method: + +```solidity + cast send source-chain.com INTEROP_CENTER_ADDRESS retryInteropTransaction( + 0x2654.. // previous interop transaction hash from above + 200_000, // new gasLimit + 300_000_000 // new gasPrice + ) +``` + +**Important** : Depending on your use case, it’s crucial to retry the transaction rather than creating a new one with +`sendInteropWithSingleCall`. + +For example, if your call involves transferring a large amount of assets, initiating a new `sendInteropWithSingleCall` +could result in freezing or burning those assets again. + +#### 5. What if my assets were burned during the transaction, but it failed on the destination chain? How do I get them back + +If your transaction fails on the destination chain, you can either: + +1. Retry the transaction with more gas or a higher gas limit (refer to the retry method above). + +2. Cancel the transaction using the following method: + +```solidity +cast send source-chain INTEROP_CENTER_ADDRESS cancelInteropTransaction( + 0x2654.., // previous interop transaction + 100_000, // gasLimit (cancellation also requires gas, but only to mark it as cancelled) + 300_000_000 // gasPrice +) +``` + +After cancellation, call the claimFailedDeposit method on the source chain contracts to recover the burned assets. Note +that the details for this step may vary depending on the contract specifics. + +## Complex Scenario + +#### 6. What if I want to transfer USDC to the Sophon chain, swap it for PEPE coin, and transfer the results back + +To accomplish this, you’ll need to: + +- Create multiple **InteropCalls** (e.g., transferring USDC, executing the swap). +- Combine these calls into a single **InteropBundle**. +- Execute the **InteropTransaction** on the destination chain. + +The step-by-step process and exact details will be covered in the next section. + +## Technical Details + +### How is Interop Different from a Bridge + +Bridges generally fall into two categories: Native and Third-Party. + +#### 1. Native Bridges + +Native bridges enable asset transfers “up and down” (from L2 to L1 and vice versa). In contrast, interop allows direct +transfers between different L2s. + +Instead of doing a "round trip" (L2 → L1 → another L2), interop lets you move assets directly between two L2s, saving +both time and cost. + +#### 2. Third-Party Bridging + +Third-party bridges enable transfers between two L2s, but they rely on their own liquidity. While you, as the user, +receive assets on the destination chain instantly, these assets come from the bridge’s liquidity pool. + +Bridge operators then rebalance using native bridging, which requires maintaining token reserves on both sides. This +adds costs for the bridge operators, often resulting in higher fees for users. + +The good news is that third-party bridges can use interop to improve their token transfers by utilizing the +**InteropMessage** layer. + +More details on this will follow below. + +### How Fast is It + +Interop speed is determined by its lowest level: **InteropMessage** propagation speed. This essentially depends on how +quickly the destination chain can confirm that the message created by the source chain is valid. + +- **Default Mode:** To prioritize security, the default interop mode waits for a ZK proof to validate the message, which + typically takes around 10 minutes. + +- **Fast Mode (Planned):** We are developing an alternative **INTEROP_CENTER** contract (using a different address but + the same interface) that will operate within 1 second. However, this faster mode comes with additional risks, similar + to the approach used by optimistic chains. + +### 4 Levels of Interop + +When analyzing interop, it can be broken into four levels, allowing you to choose the appropriate level for integration: + +- **InteropMessages:** The lowest level, directly used by third-party bridges and other protocols. + +- **InteropCall:** A medium level, designed for use by "library" contracts. + +- **InteropCallBundle:** A higher level, intended for use by "user-visible" contracts. + +- **InteropTransaction:** The highest level, designed for use in UX and frontends. + +![levelsofinterop.png](../img/levelsofinterop.png) + +We will be covering the details of each layer in the next section. diff --git a/etc/env/base/proof_data_handler.toml b/etc/env/base/proof_data_handler.toml index b56ac26fb177..767d1d16da2e 100644 --- a/etc/env/base/proof_data_handler.toml +++ b/etc/env/base/proof_data_handler.toml @@ -1,5 +1,6 @@ [proof_data_handler] http_port = 3320 proof_generation_timeout_in_secs = 18000 -tee_proof_generation_timeout_in_secs = 600 +tee_proof_generation_timeout_in_secs = 60 +tee_batch_permanently_ignored_timeout_in_hours = 240 tee_support = true diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index a4005e9477a8..23e8b3ee420c 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -166,7 +166,8 @@ witness_vector_generator: data_handler: http_port: 3320 proof_generation_timeout_in_secs: 18000 - tee_proof_generation_timeout_in_secs: 600 + tee_proof_generation_timeout_in_secs: 60 + tee_batch_permanently_ignored_timeout_in_hours: 240 tee_support: true prover_gateway: api_url: http://127.0.0.1:3320 diff --git a/install b/install new file mode 100755 index 000000000000..8bf36687e20d --- /dev/null +++ b/install @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -eo pipefail + +echo "Installing foundryup-zksync..." + +BASE_DIR="${XDG_CONFIG_HOME:-$HOME}" +FOUNDRY_DIR="${FOUNDRY_DIR-"$BASE_DIR/.foundry"}" +FOUNDRY_BIN_DIR="$FOUNDRY_DIR/bin" +FOUNDRY_MAN_DIR="$FOUNDRY_DIR/share/man/man1" + +BIN_URL="https://raw.githubusercontent.com/matter-labs/foundry-zksync/main/foundryup-zksync/foundryup-zksync" +BIN_PATH="$FOUNDRY_BIN_DIR/foundryup-zksync" + +# Create the .foundry bin directory and foundryup binary if it doesn't exist. +mkdir -p "$FOUNDRY_BIN_DIR" +curl -sSf -L "$BIN_URL" -o "$BIN_PATH" +chmod +x "$BIN_PATH" + +# Create the man directory for future man files if it doesn't exist. +mkdir -p "$FOUNDRY_MAN_DIR" + +# Store the correct profile file (i.e. .profile for bash or .zshenv for ZSH). +case $SHELL in +*/zsh) + PROFILE="${ZDOTDIR-"$HOME"}/.zshenv" + PREF_SHELL=zsh + ;; +*/bash) + PROFILE=$HOME/.bashrc + PREF_SHELL=bash + ;; +*/fish) + PROFILE=$HOME/.config/fish/config.fish + PREF_SHELL=fish + ;; +*/ash) + PROFILE=$HOME/.profile + PREF_SHELL=ash + ;; +*) + echo "foundryup-zksync: could not detect shell, manually add ${FOUNDRY_BIN_DIR} to your PATH." + exit 1 +esac + +# Only add foundryup-zksync if it isn't already in PATH. +if [[ ":$PATH:" != *":${FOUNDRY_BIN_DIR}:"* ]]; then + # Add the foundryup directory to the path and ensure the old PATH variables remain. + # If the shell is fish, echo fish_add_path instead of export. + if [[ "$PREF_SHELL" == "fish" ]]; then + echo >> "$PROFILE" && echo "fish_add_path -a $FOUNDRY_BIN_DIR" >> "$PROFILE" + else + echo >> "$PROFILE" && echo "export PATH=\"\$PATH:$FOUNDRY_BIN_DIR\"" >> "$PROFILE" + fi +fi + +# Warn MacOS users that they may need to manually install libusb via Homebrew: +if [[ "$OSTYPE" =~ ^darwin ]] && [[ ! -f /usr/local/opt/libusb/lib/libusb-1.0.0.dylib && ! -f /opt/homebrew/opt/libusb/lib/libusb-1.0.0.dylib ]]; then + echo && echo "warning: libusb not found. You may need to install it manually on MacOS via Homebrew (brew install libusb)." +fi + +echo +echo "Detected your preferred shell is $PREF_SHELL and added foundryup-zksync to PATH." +echo "Run 'source $PROFILE' or start a new terminal session to use foundryup-zksync." +echo "Then, simply run 'foundryup-zksync' to install foundry-zksync." diff --git a/prover/Cargo.lock b/prover/Cargo.lock index af249b435a6b..559304d653e7 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -7881,6 +7881,7 @@ dependencies = [ "zksync_queued_job_processor", "zksync_types", "zksync_utils", + "zksync_vlog", ] [[package]] diff --git a/prover/crates/bin/circuit_prover/Cargo.toml b/prover/crates/bin/circuit_prover/Cargo.toml index d7b7a8ca80fd..8fecc7a7a6a1 100644 --- a/prover/crates/bin/circuit_prover/Cargo.toml +++ b/prover/crates/bin/circuit_prover/Cargo.toml @@ -33,6 +33,7 @@ zksync_core_leftovers.workspace = true zksync_utils.workspace = true zksync_circuit_prover_service.workspace = true zksync_prover_job_processor.workspace = true +zksync_vlog.workspace = true vise.workspace = true shivini = { workspace = true, features = [ diff --git a/prover/crates/bin/circuit_prover/src/main.rs b/prover/crates/bin/circuit_prover/src/main.rs index e115d1510657..a445ceca3abe 100644 --- a/prover/crates/bin/circuit_prover/src/main.rs +++ b/prover/crates/bin/circuit_prover/src/main.rs @@ -20,6 +20,7 @@ use zksync_prover_dal::{ConnectionPool, Prover}; use zksync_prover_fri_types::PROVER_PROTOCOL_SEMANTIC_VERSION; use zksync_prover_keystore::keystore::Keystore; use zksync_utils::wait_for_tasks::ManagedTasks; +use zksync_vlog::prometheus::PrometheusExporterConfig; /// On most commodity hardware, WVG can take ~30 seconds to complete. /// GPU processing is ~1 second. @@ -82,7 +83,10 @@ async fn main() -> anyhow::Result<()> { let cancellation_token = CancellationToken::new(); - let mut tasks = vec![]; + let exporter_config = PrometheusExporterConfig::pull(prover_config.prometheus_port); + let (metrics_stop_sender, metrics_stop_receiver) = tokio::sync::watch::channel(false); + + let mut tasks = vec![tokio::spawn(exporter_config.run(metrics_stop_receiver))]; let (witness_vector_sender, witness_vector_receiver) = tokio::sync::mpsc::channel(CHANNEL_SIZE); @@ -142,7 +146,9 @@ async fn main() -> anyhow::Result<()> { .shutdown_time .observe(shutdown_time.elapsed()); PROVER_BINARY_METRICS.run_time.observe(start_time.elapsed()); - + metrics_stop_sender + .send(true) + .context("failed to stop metrics")?; Ok(()) } /// Loads configs necessary for proving. diff --git a/zkstack_cli/crates/config/src/consts.rs b/zkstack_cli/crates/config/src/consts.rs index 360e3fffa3b1..1332d59037f4 100644 --- a/zkstack_cli/crates/config/src/consts.rs +++ b/zkstack_cli/crates/config/src/consts.rs @@ -63,9 +63,10 @@ pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; /// Default port for the explorer data fetcher service pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; -pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; -pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; -pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker"; +pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api:v2.50.8"; +pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = + "matterlabs/block-explorer-data-fetcher:v2.50.8"; +pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker:v2.50.8"; /// Interval (in milliseconds) for polling new batches to process in explorer app pub const EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL: u64 = 1000; diff --git a/zkstack_cli/crates/zkstack/src/consts.rs b/zkstack_cli/crates/zkstack/src/consts.rs index b7c4d2a20709..f5fbf0b0c9bb 100644 --- a/zkstack_cli/crates/zkstack/src/consts.rs +++ b/zkstack_cli/crates/zkstack/src/consts.rs @@ -12,7 +12,7 @@ pub const L2_BASE_TOKEN_ADDRESS: &str = "0x0000000000000000000000000000000000008 /// Path to the JS runtime config for the block-explorer-app docker container to be mounted to pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; -pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; +pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app:v2.50.8"; /// Path to the JS runtime config for the dapp-portal docker container to be mounted to pub const PORTAL_DOCKER_CONFIG_PATH: &str = "/usr/src/app/dist/config.js"; pub const PORTAL_DOCKER_IMAGE: &str = "matterlabs/dapp-portal";