From a1f2b0a21941d1b3a8cd4853dcf472e910e6db4e Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 15 Nov 2024 13:51:59 +0400 Subject: [PATCH 1/5] feat(contract-verifier): Download compilers from GH --- Cargo.lock | 140 +++++++++- Cargo.toml | 2 + core/lib/contract_verifier/Cargo.toml | 4 + core/lib/contract_verifier/src/lib.rs | 20 +- .../lib/contract_verifier/src/resolver/env.rs | 132 ++++++++++ .../src/resolver/github/gh_api.rs | 240 ++++++++++++++++++ .../src/resolver/github/mod.rs | 235 +++++++++++++++++ .../src/{resolver.rs => resolver/mod.rs} | 194 ++++++++------ core/lib/contract_verifier/src/tests/mod.rs | 13 +- core/lib/contract_verifier/src/tests/real.rs | 8 +- zkstack_cli/crates/config/src/consts.rs | 7 +- zkstack_cli/crates/zkstack/src/consts.rs | 2 +- 12 files changed, 884 insertions(+), 113 deletions(-) create mode 100644 core/lib/contract_verifier/src/resolver/env.rs create mode 100644 core/lib/contract_verifier/src/resolver/github/gh_api.rs create mode 100644 core/lib/contract_verifier/src/resolver/github/mod.rs rename core/lib/contract_verifier/src/{resolver.rs => resolver/mod.rs} (51%) diff --git a/Cargo.lock b/Cargo.lock index eb93300b1729..f27bd256d8cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -196,6 +196,12 @@ version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "ark-ff" version = "0.3.0" @@ -3863,6 +3869,7 @@ dependencies = [ "hyper-util", "log", "rustls 0.23.16", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -4111,6 +4118,16 @@ dependencies = [ "serde", ] +[[package]] +name = "iri-string" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0f0a572e8ffe56e2ff4f769f32ffe919282c3916799f8b68688b6030063bea" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" @@ -5411,6 +5428,46 @@ dependencies = [ "memchr", ] +[[package]] +name = "octocrab" +version = "0.41.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2dfd11f6efbd39491d71a3864496f0b6f45e2d01b73b26c55d631c4e0dafaef" +dependencies = [ + "arc-swap", + "async-trait", + "base64 0.22.1", + "bytes", + "cfg-if", + "chrono", + "either", + "futures 0.3.31", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.0", + "hyper-rustls 0.27.3", + "hyper-timeout 0.5.1", + "hyper-util", + "jsonwebtoken", + "once_cell", + "percent-encoding", + "pin-project", + "secrecy 0.10.3", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "snafu", + "tokio", + "tower 0.5.1", + "tower-http 0.6.1", + "tracing", + "url", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -6981,6 +7038,19 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -7318,6 +7388,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -7965,6 +8044,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck 0.5.0", + "proc-macro2 1.0.89", + "quote 1.0.37", + "syn 2.0.85", +] + [[package]] name = "snapshots_creator" version = "0.1.0" @@ -8546,7 +8646,7 @@ dependencies = [ "pbkdf2", "regex", "schnorrkel", - "secrecy", + "secrecy 0.8.0", "sha2 0.10.8", "sp-core-hashing", "subxt", @@ -9178,6 +9278,7 @@ dependencies = [ "pin-project-lite", "sync_wrapper 0.1.2", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -9203,6 +9304,25 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +dependencies = [ + "bitflags 2.6.0", + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower 0.5.1", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -10454,7 +10574,7 @@ dependencies = [ "ethabi", "hex", "num_enum 0.7.3", - "secrecy", + "secrecy 0.8.0", "serde", "serde_json", "serde_with", @@ -10583,7 +10703,7 @@ version = "0.1.0" dependencies = [ "anyhow", "rand 0.8.5", - "secrecy", + "secrecy 0.8.0", "serde", "serde_json", "tracing", @@ -10786,7 +10906,7 @@ dependencies = [ "test-casing", "tokio", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "vise", "zksync_dal", @@ -10820,8 +10940,12 @@ dependencies = [ "assert_matches", "chrono", "ethabi", + "futures-util", "hex", + "octocrab", "regex", + "reqwest 0.12.9", + "rustls 0.23.16", "semver 1.0.23", "serde", "serde_json", @@ -11458,7 +11582,7 @@ dependencies = [ "thread_local", "tokio", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "vise", "zk_evm 0.150.7", @@ -11492,7 +11616,7 @@ dependencies = [ "anyhow", "async-trait", "rand 0.8.5", - "secrecy", + "secrecy 0.8.0", "semver 1.0.23", "tempfile", "test-casing", @@ -11775,7 +11899,7 @@ dependencies = [ "serde_json", "tokio", "tower 0.4.13", - "tower-http", + "tower-http 0.5.2", "tracing", "vise", "zksync_basic_types", @@ -11835,7 +11959,7 @@ dependencies = [ "hex", "prost 0.12.6", "rand 0.8.5", - "secrecy", + "secrecy 0.8.0", "serde_json", "serde_yaml", "tracing", diff --git a/Cargo.toml b/Cargo.toml index e491c64605bc..75b603aea326 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ ethabi = "18.0.0" flate2 = "1.0.28" fraction = "0.15.3" futures = "0.3" +futures-util = "0.3" glob = "0.3" google-cloud-auth = "0.16.0" google-cloud-storage = "0.20.0" @@ -142,6 +143,7 @@ mini-moka = "0.10.0" num = "0.4.0" num_cpus = "1.13" num_enum = "0.7.2" +octocrab = "0.41" once_cell = "1" opentelemetry = "0.24.0" opentelemetry_sdk = "0.24.0" diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 6ccd6422d7da..c2cf97826561 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -28,8 +28,12 @@ hex.workspace = true serde = { workspace = true, features = ["derive"] } tempfile.workspace = true regex.workspace = true +reqwest.workspace = true tracing.workspace = true semver.workspace = true +octocrab = { workspace = true, features = ["stream"] } +futures-util.workspace = true +rustls.workspace = true [dev-dependencies] zksync_node_test_utils.workspace = true diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index e8bf05c72e81..6d20917508fb 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -9,6 +9,7 @@ use std::{ use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; +use resolver::{GitHubCompilerResolver, ResolverMultiplexer}; use tokio::time; use zksync_dal::{contract_verification_dal::DeployedContractData, ConnectionPool, Core, CoreDal}; use zksync_queued_job_processor::{async_trait, JobProcessor}; @@ -121,12 +122,11 @@ impl ContractVerifier { compilation_timeout: Duration, connection_pool: ConnectionPool, ) -> anyhow::Result { - Self::with_resolver( - compilation_timeout, - connection_pool, - Arc::::default(), - ) - .await + let env_resolver = Arc::::default(); + let gh_resolver = Arc::new(GitHubCompilerResolver::new().await?); + let resolver = Arc::new(ResolverMultiplexer::new(env_resolver).with_resolver(gh_resolver)); + + Self::with_resolver(compilation_timeout, connection_pool, resolver).await } async fn with_resolver( @@ -170,19 +170,19 @@ impl ContractVerifier { let mut transaction = storage.start_transaction().await?; transaction .contract_verification_dal() - .set_zksolc_versions(&supported_versions.zksolc) + .set_zksolc_versions(&supported_versions.zksolc.into_iter().collect::>()) .await?; transaction .contract_verification_dal() - .set_solc_versions(&supported_versions.solc) + .set_solc_versions(&supported_versions.solc.into_iter().collect::>()) .await?; transaction .contract_verification_dal() - .set_zkvyper_versions(&supported_versions.zkvyper) + .set_zkvyper_versions(&supported_versions.zkvyper.into_iter().collect::>()) .await?; transaction .contract_verification_dal() - .set_vyper_versions(&supported_versions.vyper) + .set_vyper_versions(&supported_versions.vyper.into_iter().collect::>()) .await?; transaction.commit().await?; Ok(()) diff --git a/core/lib/contract_verifier/src/resolver/env.rs b/core/lib/contract_verifier/src/resolver/env.rs new file mode 100644 index 000000000000..798efde64348 --- /dev/null +++ b/core/lib/contract_verifier/src/resolver/env.rs @@ -0,0 +1,132 @@ +use std::{collections::HashSet, path::PathBuf}; + +use anyhow::Context as _; +use tokio::fs; +use zksync_queued_job_processor::async_trait; +use zksync_utils::env::Workspace; + +use crate::{ + compilers::{Solc, SolcInput, Vyper, VyperInput, ZkSolc, ZkSolcInput, ZkVyper}, + error::ContractVerifierError, + resolver::{ + Compiler, CompilerPaths, CompilerResolver, CompilerType, SupportedCompilerVersions, + }, + ZkCompilerVersions, +}; + +/// Default [`CompilerResolver`] using pre-downloaded compilers in the `/etc` subdirectories (relative to the workspace). +#[derive(Debug)] +pub(crate) struct EnvCompilerResolver { + home_dir: PathBuf, +} + +impl Default for EnvCompilerResolver { + fn default() -> Self { + Self { + home_dir: Workspace::locate().core(), + } + } +} + +impl EnvCompilerResolver { + async fn read_dir(&self, dir: &str) -> anyhow::Result> { + let mut dir_entries = fs::read_dir(self.home_dir.join(dir)) + .await + .context("failed reading dir")?; + let mut versions = HashSet::new(); + while let Some(entry) = dir_entries.next_entry().await? { + let Ok(file_type) = entry.file_type().await else { + continue; + }; + if file_type.is_dir() { + if let Ok(name) = entry.file_name().into_string() { + versions.insert(name); + } + } + } + Ok(versions) + } +} + +#[async_trait] +impl CompilerResolver for EnvCompilerResolver { + async fn supported_versions(&self) -> anyhow::Result { + let versions = SupportedCompilerVersions { + solc: self + .read_dir("etc/solc-bin") + .await + .context("failed reading solc dir")?, + zksolc: self + .read_dir("etc/zksolc-bin") + .await + .context("failed reading zksolc dir")?, + vyper: self + .read_dir("etc/vyper-bin") + .await + .context("failed reading vyper dir")?, + zkvyper: self + .read_dir("etc/zkvyper-bin") + .await + .context("failed reading zkvyper dir")?, + }; + tracing::info!("EnvResolver supported versions: {:?}", versions); + + Ok(versions) + } + + async fn resolve_solc( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + let solc_path = CompilerType::Solc.bin_path(&self.home_dir, version).await?; + Ok(Box::new(Solc::new(solc_path))) + } + + async fn resolve_zksolc( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + let zksolc_version = &version.zk; + let zksolc_path = CompilerType::ZkSolc + .bin_path(&self.home_dir, zksolc_version) + .await?; + let solc_path = CompilerType::Solc + .bin_path(&self.home_dir, &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: solc_path, + zk: zksolc_path, + }; + Ok(Box::new(ZkSolc::new( + compiler_paths, + zksolc_version.to_owned(), + ))) + } + + async fn resolve_vyper( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + let vyper_path = CompilerType::Vyper + .bin_path(&self.home_dir, version) + .await?; + Ok(Box::new(Vyper::new(vyper_path))) + } + + async fn resolve_zkvyper( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + let zkvyper_path = CompilerType::ZkVyper + .bin_path(&self.home_dir, &version.zk) + .await?; + let vyper_path = CompilerType::Vyper + .bin_path(&self.home_dir, &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: vyper_path, + zk: zkvyper_path, + }; + Ok(Box::new(ZkVyper::new(compiler_paths))) + } +} diff --git a/core/lib/contract_verifier/src/resolver/github/gh_api.rs b/core/lib/contract_verifier/src/resolver/github/gh_api.rs new file mode 100644 index 000000000000..23b801d440d1 --- /dev/null +++ b/core/lib/contract_verifier/src/resolver/github/gh_api.rs @@ -0,0 +1,240 @@ +//! A thin wrapper over the GitHub API for the purposes of the contract verifier. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use anyhow::Context as _; +use futures_util::TryStreamExt as _; +use octocrab::service::middleware::retry::RetryConfig; + +/// Representation of releases of the compiler. +/// The main difference from the `CompilerType` used in the `resolver` module is that +/// we treat `ZkVmSolc` differently, as it's stored in a different repository. +#[derive(Debug, Clone, Copy)] +pub(super) enum CompilerGitHubRelease { + /// "Upstream" Solidity + Solc, + /// "Upstream" Vyper + Vyper, + /// ZkSync's fork of the Solidity compiler + /// Used as a dependency for ZkSolc + ZkVmSolc, + /// Solidity compiler for EraVM + ZkSolc, + /// Vyper compiler for EraVM + ZkVyper, +} + +impl CompilerGitHubRelease { + fn organization(self) -> &'static str { + match self { + CompilerGitHubRelease::Solc => "ethereum", + CompilerGitHubRelease::Vyper => "vyperlang", + CompilerGitHubRelease::ZkVmSolc => "matter-labs", + CompilerGitHubRelease::ZkSolc => "matter-labs", + CompilerGitHubRelease::ZkVyper => "matter-labs", + } + } + + fn repo(self) -> &'static str { + match self { + CompilerGitHubRelease::Solc => "solidity", + CompilerGitHubRelease::Vyper => "vyper", + CompilerGitHubRelease::ZkVmSolc => "era-solidity", + CompilerGitHubRelease::ZkSolc => "era-compiler-solidity", + CompilerGitHubRelease::ZkVyper => "era-compiler-vyper", + } + } + + /// Check if version is blacklisted, e.g. it shouldn't be available in the contract verifier. + fn is_version_blacklisted(self, version: &str) -> bool { + match self { + CompilerGitHubRelease::Solc => { + let Ok(version) = semver::Version::parse(version) else { + tracing::error!( + "Incorrect version passed to blacklist check: {self:?}:{version}" + ); + return true; + }; + // The earliest supported version is 0.4.10. + version < semver::Version::new(0, 4, 10) + } + CompilerGitHubRelease::Vyper => { + let Ok(version) = semver::Version::parse(version) else { + tracing::error!( + "Incorrect version passed to blacklist check: {self:?}:{version}" + ); + return true; + }; + + // Versions below `0.3` are not supported. + if version < semver::Version::new(0, 3, 0) { + return true; + } + + // In `0.3.x` we only allow `0.3.3`, `0.3.9`, and `0.3.10`. + if version.minor == 3 { + return !matches!(version.patch, 3 | 9 | 10); + } + + false + } + _ => false, + } + } + + fn extract_version(self, tag_name: &str) -> Option { + match self { + CompilerGitHubRelease::Solc | CompilerGitHubRelease::Vyper => { + // Solidity and Vyper releases are tagged with version numbers in form of `vX.Y.Z`. + tag_name + .strip_prefix('v') + .filter(|v| semver::Version::parse(v).is_ok()) + .map(|v| v.to_string()) + } + CompilerGitHubRelease::ZkVmSolc => { + // ZkVmSolc releases are tagged with version numbers in form of `X.Y.Z-A.B.C`, where + // `X.Y.Z` is the version of the Solidity compiler, and `A.B.C` is the version of the ZkSync fork. + if let Some((main, fork)) = tag_name.split_once('-') { + if semver::Version::parse(main).is_ok() && semver::Version::parse(fork).is_ok() + { + // In contract verifier, our fork is prefixed with `zkVM-`. + return Some(format!("zkVM-{tag_name}")); + } + } + None + } + CompilerGitHubRelease::ZkSolc | CompilerGitHubRelease::ZkVyper => { + // zksolc and zkvyper releases are tagged with version numbers in form of `X.Y.Z` (without 'v'). + if semver::Version::parse(tag_name).is_ok() { + Some(tag_name.to_string()) + } else { + None + } + } + } + } + + fn match_asset(&self, asset_url: &str) -> bool { + match self { + CompilerGitHubRelease::Solc => asset_url.contains("solc-static-linux"), + CompilerGitHubRelease::Vyper => asset_url.contains(".linux"), + CompilerGitHubRelease::ZkVmSolc => asset_url.contains("solc-linux-amd64"), + CompilerGitHubRelease::ZkSolc => asset_url.contains("zksolc-linux-amd64-musl"), + CompilerGitHubRelease::ZkVyper => asset_url.contains("zkvyper-linux-amd64-musl"), + } + } +} + +/// A thin wrapper over the GitHub API for the purposes of the contract verifier. +#[derive(Debug)] +pub(super) struct GitHubApi { + client: Arc, +} + +impl GitHubApi { + /// Creates a new instance of the GitHub API wrapper. + pub(super) fn new() -> Self { + // Octocrab requires rustls to be configured. + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .ok(); + + let client = Arc::new( + octocrab::Octocrab::builder() + .add_retry_config(Self::retry_config()) + .set_connect_timeout(Some(Self::connect_timeout())) + .set_read_timeout(Some(Self::read_timeout())) + .build() + .unwrap(), + ); + Self { client } + } + + fn retry_config() -> RetryConfig { + RetryConfig::Simple(4) + } + + fn connect_timeout() -> Duration { + Duration::from_secs(10) + } + + fn read_timeout() -> Duration { + Duration::from_secs(60) + } + + /// Returns versions for both upstream and our fork of solc. + pub async fn solc_versions(&self) -> anyhow::Result> { + let mut versions = self + .extract_versions(CompilerGitHubRelease::Solc) + .await + .context("Can't fetch upstream solc versions")?; + versions.extend( + self.extract_versions(CompilerGitHubRelease::ZkVmSolc) + .await + .context("Can't fetch zkVM solc versions")?, + ); + Ok(versions) + } + + pub async fn zksolc_versions(&self) -> anyhow::Result> { + self.extract_versions(CompilerGitHubRelease::ZkSolc).await + } + + pub async fn vyper_versions(&self) -> anyhow::Result> { + self.extract_versions(CompilerGitHubRelease::Vyper).await + } + + pub async fn zkvyper_versions(&self) -> anyhow::Result> { + self.extract_versions(CompilerGitHubRelease::ZkVyper).await + } + + /// Will scan all the releases for a specific compiler. + async fn extract_versions( + &self, + compiler: CompilerGitHubRelease, + ) -> anyhow::Result> { + // Create a stream over all the versions to not worry about pagination. + let releases = self + .client + .repos(compiler.organization(), compiler.repo()) + .releases() + .list() + .per_page(100) + .send() + .await? + .into_stream(&self.client); + tokio::pin!(releases); + + // Go through all the releases, filter ones that match the version. + // For matching versions, find a suitable asset and store its URL. + let mut versions = HashMap::new(); + while let Some(release) = releases.try_next().await? { + // Skip pre-releases. + if release.prerelease { + continue; + } + + if let Some(version) = compiler.extract_version(&release.tag_name) { + if compiler.is_version_blacklisted(&version) { + tracing::debug!("Skipping {compiler:?}:{version} due to blacklist"); + continue; + } + + let mut found = false; + for asset in release.assets { + if compiler.match_asset(asset.browser_download_url.as_str()) { + tracing::info!("Discovered release {compiler:?}:{version}"); + versions.insert(version.clone(), asset.browser_download_url.clone()); + found = true; + break; + } + } + if !found { + tracing::warn!("Didn't find a matching artifact for {compiler:?}:{version}"); + } + } + } + + Ok(versions) + } +} diff --git a/core/lib/contract_verifier/src/resolver/github/mod.rs b/core/lib/contract_verifier/src/resolver/github/mod.rs new file mode 100644 index 000000000000..73e78744e3af --- /dev/null +++ b/core/lib/contract_verifier/src/resolver/github/mod.rs @@ -0,0 +1,235 @@ +use std::collections::HashMap; + +use anyhow::Context as _; +use tokio::io::AsyncWriteExt as _; +use zksync_queued_job_processor::async_trait; + +use self::gh_api::GitHubApi; +use crate::{ + compilers::{Solc, SolcInput, Vyper, VyperInput, ZkSolc, ZkSolcInput, ZkVyper}, + error::ContractVerifierError, + resolver::{ + Compiler, CompilerPaths, CompilerResolver, CompilerType, SupportedCompilerVersions, + }, + ZkCompilerVersions, +}; + +mod gh_api; + +/// [`CompilerResolver`] that can dynamically download missing compilers from GitHub releases. +/// +/// Note: this resolver does not interact with [`EnvCompilerResolver`](super::EnvCompilerResolver). +/// This is important for the context of zksolc/zkvyper, as there we have two separate compilers +/// required for compilation. This resolver will download both of them, even if one of the versions +/// is available in the `EnvCompilerResolver`. +#[derive(Debug)] +pub(crate) struct GitHubCompilerResolver { + /// We expect that contract-verifier will be running in docker without any persistent storage, + /// so we explicitly don't expect any artifacts to survive restart. + artifacts_dir: tempfile::TempDir, + gh_client: GitHubApi, + client: reqwest::Client, + /// Holds versions for both upstream and zkVM solc. + solc_versions: HashMap, + zksolc_versions: HashMap, + vyper_versions: HashMap, + zkvyper_versions: HashMap, +} + +impl GitHubCompilerResolver { + pub async fn new() -> anyhow::Result { + let artifacts_dir = tempfile::tempdir().context("failed creating temp dir")?; + let gh_client = GitHubApi::new(); + + let mut self_ = Self { + artifacts_dir, + gh_client, + client: reqwest::Client::new(), + solc_versions: HashMap::new(), + zksolc_versions: HashMap::new(), + vyper_versions: HashMap::new(), + zkvyper_versions: HashMap::new(), + }; + if let Err(err) = self_.sync_compiler_versions().await { + // We don't want the resolver to fail at creation if versions can't be fetched. + // It shouldn't bring down the whole application, so the expectation here is that + // the versions will be fetched later. + tracing::error!("failed syncing compiler versions at start: {:?}", err); + } + + Ok(self_) + } +} + +impl GitHubCompilerResolver { + async fn sync_compiler_versions(&mut self) -> anyhow::Result<()> { + self.solc_versions = self + .gh_client + .solc_versions() + .await + .context("failed fetching solc versions")?; + self.zksolc_versions = self + .gh_client + .zksolc_versions() + .await + .context("failed fetching zksolc versions")?; + self.vyper_versions = self + .gh_client + .vyper_versions() + .await + .context("failed fetching vyper versions")?; + self.zkvyper_versions = self + .gh_client + .zkvyper_versions() + .await + .context("failed fetching zkvyper versions")?; + Ok(()) + } + + async fn download_version_if_needed( + &self, + compiler: CompilerType, + version: &str, + ) -> anyhow::Result<()> { + if compiler.exists(self.artifacts_dir.path(), version).await? { + tracing::debug!("Compiler {}:{} exists", compiler.as_str(), version); + return Ok(()); + } + + tracing::info!("Downloading {}:{}", compiler.as_str(), version); + let versions = match compiler { + CompilerType::Solc => &self.solc_versions, + CompilerType::ZkSolc => &self.zksolc_versions, + CompilerType::Vyper => &self.vyper_versions, + CompilerType::ZkVyper => &self.zkvyper_versions, + }; + + let version_url = versions + .get(version) + .ok_or_else(|| { + ContractVerifierError::UnknownCompilerVersion("solc", version.to_owned()) + })? + .clone(); + let path = compiler.bin_path_unchecked(self.artifacts_dir.path(), version); + + let response = self.client.get(version_url).send().await?; + let body = response.bytes().await?; + + tracing::info!("Saving {}:{} to {:?}", compiler.as_str(), version, path); + + tokio::fs::create_dir_all(path.parent().unwrap()) + .await + .map_err(|e| anyhow::anyhow!("failed to create dir: {}", e))?; + + let mut file = tokio::fs::File::create(path) + .await + .map_err(|e| anyhow::anyhow!("failed to create file: {}", e))?; + file.write_all(&body) + .await + .map_err(|e| anyhow::anyhow!("failed to write to file: {}", e))?; + file.flush() + .await + .map_err(|e| anyhow::anyhow!("failed to flush file: {}", e))?; + + // On UNIX-like systems, make file executable. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = file.metadata().await?.permissions(); + perms.set_mode(0o700); // Only owner can execute and access. + file.set_permissions(perms).await?; + } + + tracing::info!("Finished downloading {}:{}", compiler.as_str(), version); + + Ok(()) + } +} + +#[async_trait] +impl CompilerResolver for GitHubCompilerResolver { + async fn supported_versions(&self) -> anyhow::Result { + let versions = SupportedCompilerVersions { + solc: self.solc_versions.keys().cloned().collect(), + zksolc: self.zksolc_versions.keys().cloned().collect(), + vyper: self.vyper_versions.keys().cloned().collect(), + zkvyper: self.zkvyper_versions.keys().cloned().collect(), + }; + tracing::info!("GitHubResolver supported versions: {:?}", versions); + Ok(versions) + } + + async fn resolve_solc( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Solc, version) + .await?; + + let solc_path = CompilerType::Solc + .bin_path(self.artifacts_dir.path(), version) + .await?; + Ok(Box::new(Solc::new(solc_path))) + } + + async fn resolve_zksolc( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Solc, &version.base) + .await?; + self.download_version_if_needed(CompilerType::ZkSolc, &version.zk) + .await?; + + let zksolc_version = &version.zk; + let zksolc_path = CompilerType::ZkSolc + .bin_path(self.artifacts_dir.path(), zksolc_version) + .await?; + let solc_path = CompilerType::Solc + .bin_path(self.artifacts_dir.path(), &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: solc_path, + zk: zksolc_path, + }; + Ok(Box::new(ZkSolc::new( + compiler_paths, + zksolc_version.to_owned(), + ))) + } + + async fn resolve_vyper( + &self, + version: &str, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Vyper, version) + .await?; + + let vyper_path = CompilerType::Vyper + .bin_path(self.artifacts_dir.path(), version) + .await?; + Ok(Box::new(Vyper::new(vyper_path))) + } + + async fn resolve_zkvyper( + &self, + version: &ZkCompilerVersions, + ) -> Result>, ContractVerifierError> { + self.download_version_if_needed(CompilerType::Vyper, &version.base) + .await?; + self.download_version_if_needed(CompilerType::ZkVyper, &version.zk) + .await?; + + let zkvyper_path = CompilerType::ZkVyper + .bin_path(self.artifacts_dir.path(), &version.zk) + .await?; + let vyper_path = CompilerType::Vyper + .bin_path(self.artifacts_dir.path(), &version.base) + .await?; + let compiler_paths = CompilerPaths { + base: vyper_path, + zk: zkvyper_path, + }; + Ok(Box::new(ZkVyper::new(compiler_paths))) + } +} diff --git a/core/lib/contract_verifier/src/resolver.rs b/core/lib/contract_verifier/src/resolver/mod.rs similarity index 51% rename from core/lib/contract_verifier/src/resolver.rs rename to core/lib/contract_verifier/src/resolver/mod.rs index 018da12a152a..a4941c25b26e 100644 --- a/core/lib/contract_verifier/src/resolver.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -1,20 +1,26 @@ use std::{ + collections::HashSet, fmt, path::{Path, PathBuf}, + sync::Arc, }; use anyhow::Context as _; use tokio::fs; use zksync_queued_job_processor::async_trait; use zksync_types::contract_verification_api::CompilationArtifacts; -use zksync_utils::env::Workspace; use crate::{ - compilers::{Solc, SolcInput, Vyper, VyperInput, ZkSolc, ZkSolcInput, ZkVyper}, + compilers::{SolcInput, VyperInput, ZkSolcInput}, error::ContractVerifierError, ZkCompilerVersions, }; +pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; + +mod env; +mod github; + #[derive(Debug, Clone, Copy)] enum CompilerType { Solc, @@ -48,6 +54,14 @@ impl CompilerType { .join(self.as_str()) } + async fn exists(self, home_dir: &Path, version: &str) -> Result { + let path = self.bin_path_unchecked(home_dir, version); + let exists = fs::try_exists(&path) + .await + .with_context(|| format!("failed accessing `{}`", self.as_str()))?; + Ok(exists) + } + async fn bin_path( self, home_dir: &Path, @@ -68,12 +82,25 @@ impl CompilerType { } /// Compiler versions supported by a [`CompilerResolver`]. -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) struct SupportedCompilerVersions { - pub solc: Vec, - pub zksolc: Vec, - pub vyper: Vec, - pub zkvyper: Vec, + /// Note: solc can have two "flavors": "upstream" solc (e.g. "real" solc used for L1 development), + /// and "zksync" solc (e.g. ZKsync fork of the solc used by `zksolc`). + /// They both are considered as "solc", but they have different versioning scheme, e.g. + /// "upstream" solc can have version `0.8.0`, while "zksync" solc can have version `zkVM-0.8.0-1.0.1`. + pub solc: HashSet, + pub zksolc: HashSet, + pub vyper: HashSet, + pub zkvyper: HashSet, +} + +impl SupportedCompilerVersions { + fn merge(&mut self, other: SupportedCompilerVersions) { + self.solc.extend(other.solc); + self.zksolc.extend(other.zksolc); + self.vyper.extend(other.vyper); + self.zkvyper.extend(other.zkvyper); + } } impl SupportedCompilerVersions { @@ -138,116 +165,119 @@ pub(crate) trait Compiler: Send + fmt::Debug { ) -> Result; } -/// Default [`CompilerResolver`] using pre-downloaded compilers in the `/etc` subdirectories (relative to the workspace). #[derive(Debug)] -pub(crate) struct EnvCompilerResolver { - home_dir: PathBuf, +pub struct ResolverMultiplexer { + resolvers: Vec>, } -impl Default for EnvCompilerResolver { - fn default() -> Self { +impl ResolverMultiplexer { + pub fn new(resolver: Arc) -> Self { Self { - home_dir: Workspace::locate().core(), + resolvers: vec![resolver], } } -} -impl EnvCompilerResolver { - async fn read_dir(&self, dir: &str) -> anyhow::Result> { - let mut dir_entries = fs::read_dir(self.home_dir.join(dir)) - .await - .context("failed reading dir")?; - let mut versions = vec![]; - while let Some(entry) = dir_entries.next_entry().await? { - let Ok(file_type) = entry.file_type().await else { - continue; - }; - if file_type.is_dir() { - if let Ok(name) = entry.file_name().into_string() { - versions.push(name); - } - } - } - Ok(versions) + pub fn with_resolver(mut self, resolver: Arc) -> Self { + self.resolvers.push(resolver); + self } } #[async_trait] -impl CompilerResolver for EnvCompilerResolver { +impl CompilerResolver for ResolverMultiplexer { async fn supported_versions(&self) -> anyhow::Result { - Ok(SupportedCompilerVersions { - solc: self - .read_dir("etc/solc-bin") - .await - .context("failed reading solc dir")?, - zksolc: self - .read_dir("etc/zksolc-bin") - .await - .context("failed reading zksolc dir")?, - vyper: self - .read_dir("etc/vyper-bin") - .await - .context("failed reading vyper dir")?, - zkvyper: self - .read_dir("etc/zkvyper-bin") - .await - .context("failed reading zkvyper dir")?, - }) + let mut versions = SupportedCompilerVersions::default(); + for resolver in &self.resolvers { + versions.merge(resolver.supported_versions().await?); + } + Ok(versions) } + /// Resolves a `solc` compiler. async fn resolve_solc( &self, version: &str, ) -> Result>, ContractVerifierError> { - let solc_path = CompilerType::Solc.bin_path(&self.home_dir, version).await?; - Ok(Box::new(Solc::new(solc_path))) + let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( + "solc", + version.to_owned(), + )); + for resolver in &self.resolvers { + match resolver.resolve_solc(version).await { + Ok(compiler) => return Ok(compiler), + err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + last_error = err; + continue; + } + Err(err) => return Err(err), + } + } + last_error } + /// Resolves a `zksolc` compiler. async fn resolve_zksolc( &self, version: &ZkCompilerVersions, ) -> Result>, ContractVerifierError> { - let zksolc_version = &version.zk; - let zksolc_path = CompilerType::ZkSolc - .bin_path(&self.home_dir, zksolc_version) - .await?; - let solc_path = CompilerType::Solc - .bin_path(&self.home_dir, &version.base) - .await?; - let compiler_paths = CompilerPaths { - base: solc_path, - zk: zksolc_path, - }; - Ok(Box::new(ZkSolc::new( - compiler_paths, - zksolc_version.to_owned(), - ))) + let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( + "zksolc", + version.zk.to_owned(), + )); + for resolver in &self.resolvers { + match resolver.resolve_zksolc(version).await { + Ok(compiler) => return Ok(compiler), + err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + last_error = err; + continue; + } + Err(err) => return Err(err), + } + } + last_error } + /// Resolves a `vyper` compiler. async fn resolve_vyper( &self, version: &str, ) -> Result>, ContractVerifierError> { - let vyper_path = CompilerType::Vyper - .bin_path(&self.home_dir, version) - .await?; - Ok(Box::new(Vyper::new(vyper_path))) + let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( + "vyper", + version.to_owned(), + )); + for resolver in &self.resolvers { + match resolver.resolve_vyper(version).await { + Ok(compiler) => return Ok(compiler), + err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + last_error = err; + continue; + } + Err(err) => return Err(err), + } + } + last_error } + /// Resolves a `zkvyper` compiler. async fn resolve_zkvyper( &self, version: &ZkCompilerVersions, ) -> Result>, ContractVerifierError> { - let zkvyper_path = CompilerType::ZkVyper - .bin_path(&self.home_dir, &version.zk) - .await?; - let vyper_path = CompilerType::Vyper - .bin_path(&self.home_dir, &version.base) - .await?; - let compiler_paths = CompilerPaths { - base: vyper_path, - zk: zkvyper_path, - }; - Ok(Box::new(ZkVyper::new(compiler_paths))) + let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( + "zkvyper", + version.zk.to_owned(), + )); + for resolver in &self.resolvers { + match resolver.resolve_zkvyper(version).await { + Ok(compiler) => return Ok(compiler), + err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { + last_error = err; + continue; + } + Err(err) => return Err(err), + } + } + last_error } } diff --git a/core/lib/contract_verifier/src/tests/mod.rs b/core/lib/contract_verifier/src/tests/mod.rs index 2aad39a12e0e..80fc304fdd11 100644 --- a/core/lib/contract_verifier/src/tests/mod.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -1,6 +1,9 @@ //! Tests for the contract verifier. -use std::{collections::HashMap, iter}; +use std::{ + collections::{HashMap, HashSet}, + iter, +}; use test_casing::{test_casing, Product}; use tokio::sync::watch; @@ -289,10 +292,10 @@ impl Compiler for MockCompilerResolver { impl CompilerResolver for MockCompilerResolver { async fn supported_versions(&self) -> anyhow::Result { Ok(SupportedCompilerVersions { - solc: vec![SOLC_VERSION.to_owned()], - zksolc: vec![ZKSOLC_VERSION.to_owned()], - vyper: vec![], - zkvyper: vec![], + solc: [SOLC_VERSION.to_owned()].into_iter().collect(), + zksolc: [ZKSOLC_VERSION.to_owned()].into_iter().collect(), + vyper: HashSet::default(), + zkvyper: HashSet::default(), }) } diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs index 4282e6de4ef8..168285279967 100644 --- a/core/lib/contract_verifier/src/tests/real.rs +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -28,16 +28,16 @@ struct TestCompilerVersions { } impl TestCompilerVersions { - fn new(mut versions: SupportedCompilerVersions) -> Option { + fn new(versions: SupportedCompilerVersions) -> Option { let solc = versions .solc .into_iter() .find(|ver| !ver.starts_with("zkVM"))?; Some(Self { solc, - zksolc: versions.zksolc.pop()?, - vyper: versions.vyper.pop()?, - zkvyper: versions.zkvyper.pop()?, + zksolc: versions.zksolc.into_iter().next()?, + vyper: versions.vyper.into_iter().next()?, + zkvyper: versions.zkvyper.into_iter().next()?, }) } diff --git a/zkstack_cli/crates/config/src/consts.rs b/zkstack_cli/crates/config/src/consts.rs index f462ce33b8f8..c3efb4ac3e96 100644 --- a/zkstack_cli/crates/config/src/consts.rs +++ b/zkstack_cli/crates/config/src/consts.rs @@ -63,9 +63,10 @@ pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; /// Default port for the explorer data fetcher service pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; -pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; -pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; -pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker"; +pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api:v2.50.8"; +pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = + "matterlabs/block-explorer-data-fetcher:v2.50.8"; +pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker:v2.50.8"; /// Interval (in milliseconds) for polling new batches to process in explorer app pub const EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL: u64 = 1000; diff --git a/zkstack_cli/crates/zkstack/src/consts.rs b/zkstack_cli/crates/zkstack/src/consts.rs index b7c4d2a20709..f5fbf0b0c9bb 100644 --- a/zkstack_cli/crates/zkstack/src/consts.rs +++ b/zkstack_cli/crates/zkstack/src/consts.rs @@ -12,7 +12,7 @@ pub const L2_BASE_TOKEN_ADDRESS: &str = "0x0000000000000000000000000000000000008 /// Path to the JS runtime config for the block-explorer-app docker container to be mounted to pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; -pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; +pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app:v2.50.8"; /// Path to the JS runtime config for the dapp-portal docker container to be mounted to pub const PORTAL_DOCKER_CONFIG_PATH: &str = "/usr/src/app/dist/config.js"; pub const PORTAL_DOCKER_IMAGE: &str = "matterlabs/dapp-portal"; From 877c0c82f66d9c3489c5281328cd3ae7f50d6a0e Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 15 Nov 2024 14:35:05 +0400 Subject: [PATCH 2/5] Periodically update compiler versions --- core/bin/contract-verifier/src/main.rs | 2 + core/lib/contract_verifier/src/lib.rs | 40 ++++-- .../src/resolver/github/mod.rs | 116 ++++++++++++------ 3 files changed, 111 insertions(+), 47 deletions(-) diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 88f25256c40d..ab86c147977d 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -55,7 +55,9 @@ async fn main() -> anyhow::Result<()> { let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool) .await .context("failed initializing contract verifier")?; + let update_task = contract_verifier.sync_compiler_versions_task(); let tasks = vec![ + tokio::spawn(update_task), tokio::spawn(contract_verifier.run(stop_receiver.clone(), opt.jobs_number)), tokio::spawn( PrometheusExporterConfig::pull(prometheus_config.listener_port).run(stop_receiver), diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 6d20917508fb..13c39de0d7db 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -134,21 +134,42 @@ impl ContractVerifier { connection_pool: ConnectionPool, compiler_resolver: Arc, ) -> anyhow::Result { - let this = Self { + Self::sync_compiler_versions(compiler_resolver.as_ref(), &connection_pool).await?; + Ok(Self { compilation_timeout, contract_deployer: zksync_contracts::deployer_contract(), connection_pool, compiler_resolver, - }; - this.sync_compiler_versions().await?; - Ok(this) + }) + } + + /// Returns a future that would periodically update the supported compiler versions + /// in the database. + pub fn sync_compiler_versions_task( + &self, + ) -> impl std::future::Future> { + const UPDATE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hour. + + let resolver = self.compiler_resolver.clone(); + let pool = self.connection_pool.clone(); + async move { + loop { + tracing::info!("Updating compiler versions"); + if let Err(err) = Self::sync_compiler_versions(resolver.as_ref(), &pool).await { + tracing::error!("Failed to sync compiler versions: {:?}", err); + } + tokio::time::sleep(UPDATE_INTERVAL).await; + } + } } /// Synchronizes compiler versions. #[tracing::instrument(level = "debug", skip_all)] - async fn sync_compiler_versions(&self) -> anyhow::Result<()> { - let supported_versions = self - .compiler_resolver + async fn sync_compiler_versions( + resolver: &dyn CompilerResolver, + pool: &ConnectionPool, + ) -> anyhow::Result<()> { + let supported_versions = resolver .supported_versions() .await .context("cannot get supported compilers")?; @@ -163,10 +184,7 @@ impl ContractVerifier { "persisting supported compiler versions" ); - let mut storage = self - .connection_pool - .connection_tagged("contract_verifier") - .await?; + let mut storage = pool.connection_tagged("contract_verifier").await?; let mut transaction = storage.start_transaction().await?; transaction .contract_verification_dal() diff --git a/core/lib/contract_verifier/src/resolver/github/mod.rs b/core/lib/contract_verifier/src/resolver/github/mod.rs index 73e78744e3af..d73cdda57a2b 100644 --- a/core/lib/contract_verifier/src/resolver/github/mod.rs +++ b/core/lib/contract_verifier/src/resolver/github/mod.rs @@ -1,7 +1,10 @@ -use std::collections::HashMap; +use std::{ + collections::HashMap, + time::{Duration, Instant}, +}; use anyhow::Context as _; -use tokio::io::AsyncWriteExt as _; +use tokio::{io::AsyncWriteExt as _, sync::RwLock}; use zksync_queued_job_processor::async_trait; use self::gh_api::GitHubApi; @@ -29,63 +32,99 @@ pub(crate) struct GitHubCompilerResolver { artifacts_dir: tempfile::TempDir, gh_client: GitHubApi, client: reqwest::Client, + supported_versions: RwLock, +} + +#[derive(Debug)] +struct SupportedVersions { /// Holds versions for both upstream and zkVM solc. solc_versions: HashMap, zksolc_versions: HashMap, vyper_versions: HashMap, zkvyper_versions: HashMap, + last_updated: Instant, } -impl GitHubCompilerResolver { - pub async fn new() -> anyhow::Result { - let artifacts_dir = tempfile::tempdir().context("failed creating temp dir")?; - let gh_client = GitHubApi::new(); +impl Default for SupportedVersions { + fn default() -> Self { + Self::new() + } +} - let mut self_ = Self { - artifacts_dir, - gh_client, - client: reqwest::Client::new(), +impl SupportedVersions { + // Note: We assume that contract verifier will run the task to update supported versions + // rarely, but we still want to protect ourselves from accidentally spamming GitHub API. + // So, this interval is smaller than the expected time between updates (this way we don't + // run into an issue where intervals are slightly out of sync, causing a delay in "real" + // update tim). + const CACHE_INTERVAL: Duration = Duration::from_secs(10 * 60); // 10 minutes + + fn new() -> Self { + Self { solc_versions: HashMap::new(), zksolc_versions: HashMap::new(), vyper_versions: HashMap::new(), zkvyper_versions: HashMap::new(), - }; - if let Err(err) = self_.sync_compiler_versions().await { - // We don't want the resolver to fail at creation if versions can't be fetched. - // It shouldn't bring down the whole application, so the expectation here is that - // the versions will be fetched later. - tracing::error!("failed syncing compiler versions at start: {:?}", err); + last_updated: Instant::now(), } + } - Ok(self_) + fn is_outdated(&self) -> bool { + self.last_updated.elapsed() > Self::CACHE_INTERVAL } -} -impl GitHubCompilerResolver { - async fn sync_compiler_versions(&mut self) -> anyhow::Result<()> { - self.solc_versions = self - .gh_client + async fn update(&mut self, gh_client: &GitHubApi) -> anyhow::Result<()> { + self.solc_versions = gh_client .solc_versions() .await .context("failed fetching solc versions")?; - self.zksolc_versions = self - .gh_client + self.zksolc_versions = gh_client .zksolc_versions() .await .context("failed fetching zksolc versions")?; - self.vyper_versions = self - .gh_client + self.vyper_versions = gh_client .vyper_versions() .await .context("failed fetching vyper versions")?; - self.zkvyper_versions = self - .gh_client + self.zkvyper_versions = gh_client .zkvyper_versions() .await .context("failed fetching zkvyper versions")?; + self.last_updated = Instant::now(); Ok(()) } + async fn update_if_needed(&mut self, gh_client: &GitHubApi) -> anyhow::Result<()> { + if self.is_outdated() { + tracing::info!("GH compiler versions cache outdated, updating"); + self.update(gh_client).await?; + } + Ok(()) + } +} + +impl GitHubCompilerResolver { + pub async fn new() -> anyhow::Result { + let artifacts_dir = tempfile::tempdir().context("failed creating temp dir")?; + let gh_client = GitHubApi::new(); + let mut supported_versions = SupportedVersions::default(); + if let Err(err) = supported_versions.update(&gh_client).await { + // We don't want the resolver to fail at creation if versions can't be fetched. + // It shouldn't bring down the whole application, so the expectation here is that + // the versions will be fetched later. + tracing::error!("failed syncing compiler versions at start: {:?}", err); + } + + Ok(Self { + artifacts_dir, + gh_client, + client: reqwest::Client::new(), + supported_versions: RwLock::new(supported_versions), + }) + } +} + +impl GitHubCompilerResolver { async fn download_version_if_needed( &self, compiler: CompilerType, @@ -97,11 +136,12 @@ impl GitHubCompilerResolver { } tracing::info!("Downloading {}:{}", compiler.as_str(), version); + let lock = self.supported_versions.read().await; let versions = match compiler { - CompilerType::Solc => &self.solc_versions, - CompilerType::ZkSolc => &self.zksolc_versions, - CompilerType::Vyper => &self.vyper_versions, - CompilerType::ZkVyper => &self.zkvyper_versions, + CompilerType::Solc => &lock.solc_versions, + CompilerType::ZkSolc => &lock.zksolc_versions, + CompilerType::Vyper => &lock.vyper_versions, + CompilerType::ZkVyper => &lock.zkvyper_versions, }; let version_url = versions @@ -110,6 +150,7 @@ impl GitHubCompilerResolver { ContractVerifierError::UnknownCompilerVersion("solc", version.to_owned()) })? .clone(); + drop(lock); let path = compiler.bin_path_unchecked(self.artifacts_dir.path(), version); let response = self.client.get(version_url).send().await?; @@ -149,11 +190,14 @@ impl GitHubCompilerResolver { #[async_trait] impl CompilerResolver for GitHubCompilerResolver { async fn supported_versions(&self) -> anyhow::Result { + let mut lock = self.supported_versions.write().await; + lock.update_if_needed(&self.gh_client).await?; + let versions = SupportedCompilerVersions { - solc: self.solc_versions.keys().cloned().collect(), - zksolc: self.zksolc_versions.keys().cloned().collect(), - vyper: self.vyper_versions.keys().cloned().collect(), - zkvyper: self.zkvyper_versions.keys().cloned().collect(), + solc: lock.solc_versions.keys().cloned().collect(), + zksolc: lock.zksolc_versions.keys().cloned().collect(), + vyper: lock.vyper_versions.keys().cloned().collect(), + zkvyper: lock.zkvyper_versions.keys().cloned().collect(), }; tracing::info!("GitHubResolver supported versions: {:?}", versions); Ok(versions) From 1885ee47933aeff9727ccb445a16fd1a3665bcbe Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 15 Nov 2024 14:42:32 +0400 Subject: [PATCH 3/5] Add kill switch --- core/lib/contract_verifier/src/lib.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 13c39de0d7db..aaa9c126641e 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -124,9 +124,18 @@ impl ContractVerifier { ) -> anyhow::Result { let env_resolver = Arc::::default(); let gh_resolver = Arc::new(GitHubCompilerResolver::new().await?); - let resolver = Arc::new(ResolverMultiplexer::new(env_resolver).with_resolver(gh_resolver)); + let mut resolver = ResolverMultiplexer::new(env_resolver); - Self::with_resolver(compilation_timeout, connection_pool, resolver).await + // Killer switch: if anything goes wrong with GH resolver, we can disable it without having to rollback. + // TODO: Remove once GH resolver is proven to be stable. + let disable_gh_resolver = std::env::var("DISABLE_GITHUB_RESOLVER").is_ok(); + if !disable_gh_resolver { + resolver = resolver.with_resolver(gh_resolver); + } else { + tracing::warn!("GitHub resolver was disabled via DISABLE_GITHUB_RESOLVER env variable") + } + + Self::with_resolver(compilation_timeout, connection_pool, Arc::new(resolver)).await } async fn with_resolver( From ca3e5a2121d08371a1ca2f2309c0e8847d63b7b9 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 15 Nov 2024 14:52:41 +0400 Subject: [PATCH 4/5] fmt --- core/lib/contract_verifier/src/resolver/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/lib/contract_verifier/src/resolver/mod.rs b/core/lib/contract_verifier/src/resolver/mod.rs index a4941c25b26e..52d17d39e054 100644 --- a/core/lib/contract_verifier/src/resolver/mod.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -10,14 +10,13 @@ use tokio::fs; use zksync_queued_job_processor::async_trait; use zksync_types::contract_verification_api::CompilationArtifacts; +pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; use crate::{ compilers::{SolcInput, VyperInput, ZkSolcInput}, error::ContractVerifierError, ZkCompilerVersions, }; -pub(crate) use self::{env::EnvCompilerResolver, github::GitHubCompilerResolver}; - mod env; mod github; From 9a2082d4eafcc892eb053643fd0c430c6b3e2e89 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 15 Nov 2024 17:41:41 +0400 Subject: [PATCH 5/5] Review comments --- .../src/resolver/github/gh_api.rs | 40 ++++++++-------- .../src/resolver/github/mod.rs | 48 +++++++++++++++---- .../lib/contract_verifier/src/resolver/mod.rs | 26 +++++----- 3 files changed, 71 insertions(+), 43 deletions(-) diff --git a/core/lib/contract_verifier/src/resolver/github/gh_api.rs b/core/lib/contract_verifier/src/resolver/github/gh_api.rs index 23b801d440d1..8c9ac6723249 100644 --- a/core/lib/contract_verifier/src/resolver/github/gh_api.rs +++ b/core/lib/contract_verifier/src/resolver/github/gh_api.rs @@ -27,28 +27,28 @@ pub(super) enum CompilerGitHubRelease { impl CompilerGitHubRelease { fn organization(self) -> &'static str { match self { - CompilerGitHubRelease::Solc => "ethereum", - CompilerGitHubRelease::Vyper => "vyperlang", - CompilerGitHubRelease::ZkVmSolc => "matter-labs", - CompilerGitHubRelease::ZkSolc => "matter-labs", - CompilerGitHubRelease::ZkVyper => "matter-labs", + Self::Solc => "ethereum", + Self::Vyper => "vyperlang", + Self::ZkVmSolc => "matter-labs", + Self::ZkSolc => "matter-labs", + Self::ZkVyper => "matter-labs", } } fn repo(self) -> &'static str { match self { - CompilerGitHubRelease::Solc => "solidity", - CompilerGitHubRelease::Vyper => "vyper", - CompilerGitHubRelease::ZkVmSolc => "era-solidity", - CompilerGitHubRelease::ZkSolc => "era-compiler-solidity", - CompilerGitHubRelease::ZkVyper => "era-compiler-vyper", + Self::Solc => "solidity", + Self::Vyper => "vyper", + Self::ZkVmSolc => "era-solidity", + Self::ZkSolc => "era-compiler-solidity", + Self::ZkVyper => "era-compiler-vyper", } } /// Check if version is blacklisted, e.g. it shouldn't be available in the contract verifier. fn is_version_blacklisted(self, version: &str) -> bool { match self { - CompilerGitHubRelease::Solc => { + Self::Solc => { let Ok(version) = semver::Version::parse(version) else { tracing::error!( "Incorrect version passed to blacklist check: {self:?}:{version}" @@ -58,7 +58,7 @@ impl CompilerGitHubRelease { // The earliest supported version is 0.4.10. version < semver::Version::new(0, 4, 10) } - CompilerGitHubRelease::Vyper => { + Self::Vyper => { let Ok(version) = semver::Version::parse(version) else { tracing::error!( "Incorrect version passed to blacklist check: {self:?}:{version}" @@ -84,14 +84,14 @@ impl CompilerGitHubRelease { fn extract_version(self, tag_name: &str) -> Option { match self { - CompilerGitHubRelease::Solc | CompilerGitHubRelease::Vyper => { + Self::Solc | Self::Vyper => { // Solidity and Vyper releases are tagged with version numbers in form of `vX.Y.Z`. tag_name .strip_prefix('v') .filter(|v| semver::Version::parse(v).is_ok()) .map(|v| v.to_string()) } - CompilerGitHubRelease::ZkVmSolc => { + Self::ZkVmSolc => { // ZkVmSolc releases are tagged with version numbers in form of `X.Y.Z-A.B.C`, where // `X.Y.Z` is the version of the Solidity compiler, and `A.B.C` is the version of the ZkSync fork. if let Some((main, fork)) = tag_name.split_once('-') { @@ -103,7 +103,7 @@ impl CompilerGitHubRelease { } None } - CompilerGitHubRelease::ZkSolc | CompilerGitHubRelease::ZkVyper => { + Self::ZkSolc | Self::ZkVyper => { // zksolc and zkvyper releases are tagged with version numbers in form of `X.Y.Z` (without 'v'). if semver::Version::parse(tag_name).is_ok() { Some(tag_name.to_string()) @@ -116,11 +116,11 @@ impl CompilerGitHubRelease { fn match_asset(&self, asset_url: &str) -> bool { match self { - CompilerGitHubRelease::Solc => asset_url.contains("solc-static-linux"), - CompilerGitHubRelease::Vyper => asset_url.contains(".linux"), - CompilerGitHubRelease::ZkVmSolc => asset_url.contains("solc-linux-amd64"), - CompilerGitHubRelease::ZkSolc => asset_url.contains("zksolc-linux-amd64-musl"), - CompilerGitHubRelease::ZkVyper => asset_url.contains("zkvyper-linux-amd64-musl"), + Self::Solc => asset_url.contains("solc-static-linux"), + Self::Vyper => asset_url.contains(".linux"), + Self::ZkVmSolc => asset_url.contains("solc-linux-amd64"), + Self::ZkSolc => asset_url.contains("zksolc-linux-amd64-musl"), + Self::ZkVyper => asset_url.contains("zkvyper-linux-amd64-musl"), } } } diff --git a/core/lib/contract_verifier/src/resolver/github/mod.rs b/core/lib/contract_verifier/src/resolver/github/mod.rs index d73cdda57a2b..a50d0151b7ff 100644 --- a/core/lib/contract_verifier/src/resolver/github/mod.rs +++ b/core/lib/contract_verifier/src/resolver/github/mod.rs @@ -33,6 +33,9 @@ pub(crate) struct GitHubCompilerResolver { gh_client: GitHubApi, client: reqwest::Client, supported_versions: RwLock, + /// List of downloads performed right now. + /// `broadcast` receiver can be used to wait until the download is finished. + active_downloads: RwLock>>, } #[derive(Debug)] @@ -56,7 +59,7 @@ impl SupportedVersions { // rarely, but we still want to protect ourselves from accidentally spamming GitHub API. // So, this interval is smaller than the expected time between updates (this way we don't // run into an issue where intervals are slightly out of sync, causing a delay in "real" - // update tim). + // update time). const CACHE_INTERVAL: Duration = Duration::from_secs(10 * 60); // 10 minutes fn new() -> Self { @@ -74,6 +77,10 @@ impl SupportedVersions { } async fn update(&mut self, gh_client: &GitHubApi) -> anyhow::Result<()> { + // Non-atomic update is fine here: the fields are independent, so if + // at least one update succeeds, it's worth persisting. We won't be changing + // the last update timestamp in case of failure though, so it will be retried + // next time. self.solc_versions = gh_client .solc_versions() .await @@ -120,6 +127,7 @@ impl GitHubCompilerResolver { gh_client, client: reqwest::Client::new(), supported_versions: RwLock::new(supported_versions), + active_downloads: RwLock::default(), }) } } @@ -130,11 +138,31 @@ impl GitHubCompilerResolver { compiler: CompilerType, version: &str, ) -> anyhow::Result<()> { + // We need to check the lock first, because the compiler may still be downloading. + // We must hold the lock until we know if we need to download the compiler. + let mut lock = self.active_downloads.write().await; + if let Some(rx) = lock.get(&(compiler, version.to_string())) { + let mut rx = rx.resubscribe(); + drop(lock); + tracing::debug!( + "Waiting for {}:{} download to finish", + compiler.as_str(), + version + ); + rx.recv().await?; + return Ok(()); + } + if compiler.exists(self.artifacts_dir.path(), version).await? { tracing::debug!("Compiler {}:{} exists", compiler.as_str(), version); return Ok(()); } + // Mark the compiler as downloading. + let (tx, rx) = tokio::sync::broadcast::channel(1); + lock.insert((compiler, version.to_string()), rx); + drop(lock); + tracing::info!("Downloading {}:{}", compiler.as_str(), version); let lock = self.supported_versions.read().await; let versions = match compiler { @@ -160,17 +188,15 @@ impl GitHubCompilerResolver { tokio::fs::create_dir_all(path.parent().unwrap()) .await - .map_err(|e| anyhow::anyhow!("failed to create dir: {}", e))?; + .context("failed to create dir")?; - let mut file = tokio::fs::File::create(path) + let mut file = tokio::fs::File::create_new(path) .await - .map_err(|e| anyhow::anyhow!("failed to create file: {}", e))?; + .context("failed to create file")?; file.write_all(&body) .await - .map_err(|e| anyhow::anyhow!("failed to write to file: {}", e))?; - file.flush() - .await - .map_err(|e| anyhow::anyhow!("failed to flush file: {}", e))?; + .context("failed to write to file")?; + file.flush().await.context("failed to flush file")?; // On UNIX-like systems, make file executable. #[cfg(unix)] @@ -183,6 +209,12 @@ impl GitHubCompilerResolver { tracing::info!("Finished downloading {}:{}", compiler.as_str(), version); + // Notify other waiters that the compiler is downloaded. + tx.send(()).ok(); + let mut lock = self.active_downloads.write().await; + lock.remove(&(compiler, version.to_string())); + drop(lock); + Ok(()) } } diff --git a/core/lib/contract_verifier/src/resolver/mod.rs b/core/lib/contract_verifier/src/resolver/mod.rs index 52d17d39e054..a9d2bcf9049d 100644 --- a/core/lib/contract_verifier/src/resolver/mod.rs +++ b/core/lib/contract_verifier/src/resolver/mod.rs @@ -20,7 +20,7 @@ use crate::{ mod env; mod github; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] enum CompilerType { Solc, ZkSolc, @@ -197,21 +197,19 @@ impl CompilerResolver for ResolverMultiplexer { &self, version: &str, ) -> Result>, ContractVerifierError> { - let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( - "solc", - version.to_owned(), - )); for resolver in &self.resolvers { match resolver.resolve_solc(version).await { Ok(compiler) => return Ok(compiler), - err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { - last_error = err; + Err(ContractVerifierError::UnknownCompilerVersion(..)) => { continue; } Err(err) => return Err(err), } } - last_error + Err(ContractVerifierError::UnknownCompilerVersion( + "solc", + version.to_owned(), + )) } /// Resolves a `zksolc` compiler. @@ -241,21 +239,19 @@ impl CompilerResolver for ResolverMultiplexer { &self, version: &str, ) -> Result>, ContractVerifierError> { - let mut last_error = Err(ContractVerifierError::UnknownCompilerVersion( - "vyper", - version.to_owned(), - )); for resolver in &self.resolvers { match resolver.resolve_vyper(version).await { Ok(compiler) => return Ok(compiler), - err @ Err(ContractVerifierError::UnknownCompilerVersion(..)) => { - last_error = err; + Err(ContractVerifierError::UnknownCompilerVersion(..)) => { continue; } Err(err) => return Err(err), } } - last_error + Err(ContractVerifierError::UnknownCompilerVersion( + "vyper", + version.to_owned(), + )) } /// Resolves a `zkvyper` compiler.