Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extra Features #353

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
3606fc1
feat(vm_executor): Add new histogram metric for gas per tx in vm_exec…
doyleish Nov 8, 2024
e874fbc
ci: Try to get setup_2\26 key from cache (#3235)
alexandrst88 Nov 8, 2024
5bc70c8
ci: Change schedule to monthly and add grouping to renovate (#3239)
alexandrst88 Nov 8, 2024
25ec1c8
chore(deps): update trufflesecurity/trufflehog digest to 781157a (#2881)
renovate[bot] Nov 8, 2024
fd27507
refactor(vm): Narrow down factory deps returned in execution result (…
slowli Nov 8, 2024
4509179
feat(contract-verifier): Adapt contract verifier API for EVM bytecode…
slowli Nov 8, 2024
94ecb52
Merge pull request #330 from lambdaclass/eigen-client-clippy-and-doc
juanbono Nov 8, 2024
1bfff0e
fix(prover): Remove unneeded dependencies, add default for graceful_s…
yorik Nov 8, 2024
342b476
Merge pull request #331 from lambdaclass/eigen-client-merge-main
juanbono Nov 8, 2024
077bc13
Merge branch 'main' into eigen-client-extra-features
juanbono Nov 8, 2024
c98726a
Update integration readme (#335)
gianbelinche Nov 11, 2024
991c340
update contracts submodule
juan518munoz Nov 12, 2024
f55a424
Merge pull request #337 from lambdaclass/eigen-client-contracts-submo…
juanbono Nov 12, 2024
f2fb12c
feat(eigen-client-extra-features): Remove kzgpad (#339)
gianbelinche Nov 15, 2024
b825d4f
Create eigendaservicemanager type (#340)
gianbelinche Nov 15, 2024
9561bc3
Change match to if (#341)
gianbelinche Nov 15, 2024
986a1d2
feat(eigen-client-eigen-features): Move dependencies (#342)
gianbelinche Nov 15, 2024
09ecfdd
feat(eigen-client-extra-features): Backon (#344)
gianbelinche Nov 19, 2024
a2ddcc3
Remove memstore (#346)
gianbelinche Nov 19, 2024
7d369a8
Add reserved to proto (#343)
gianbelinche Nov 21, 2024
a9c31a7
Remove extra config (#351)
gianbelinche Nov 21, 2024
6e1de82
feat(eigen-client-extra-features): Kzg bn254 crate (#352)
gianbelinche Nov 21, 2024
8d0b908
feat(eigen-client-extra-features): Remove Alloy Dependency (#349)
gianbelinche Nov 21, 2024
43a3e5c
Merge branch 'eigen-client-extra-features' into eigen-client-m0-with-…
gianbelinche Nov 22, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/build-tee-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,3 @@ jobs:
docker push "${repo}/${tag}"
done
done
3 changes: 1 addition & 2 deletions .github/workflows/ci-common-reusable.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
run: |
run_retried docker-compose -f ${RUNNER_COMPOSE_FILE} pull
docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk postgres
- name: Install zkstack
run: |
ci_run ./zkstack_cli/zkstackup/install -g --path ./zkstack_cli/zkstackup/zkstackup
Expand All @@ -38,4 +38,3 @@ jobs:
# `zk lint prover` = cargo clippy, which does cargo check behind the scenes, which is a lightweight version of cargo build
- name: Lints
run: ci_run zkstack dev lint -t rs --check

2 changes: 1 addition & 1 deletion .github/workflows/ci-prover-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
- name: Kill prover & start compressor
run: |
sudo ./bin/prover_checkers/kill_prover
ci_run zkstack prover run --component=compressor --docker=false &>prover_logs/compressor.log &
- name: Wait for batch to be executed on L1
env:
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/new-build-contract-verifier-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,11 @@ jobs:
- name: Download setup key
shell: bash
run: |
run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
if [ -f "/setup_2^26.key" ]; then
cp '/setup_2^26.key' './setup_2^26.key'
else
run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
fi
- name: Set env vars
shell: bash
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/new-build-core-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,11 @@ jobs:
- name: Download setup key
shell: bash
run: |
run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
if [ -f "/setup_2^26.key" ]; then
cp '/setup_2^26.key' './setup_2^26.key'
else
run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
fi
- name: Set env vars
shell: bash
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/new-build-prover-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,6 @@ jobs:
if: matrix.components == 'proof-fri-gpu-compressor'
run: |
run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
# We need to run this only when ERA_BELLMAN_CUDA_RELEASE is not available
# In our case it happens only when PR is created from fork
- name: Wait for runner IP to be not rate-limited against GH API
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/secrets_scanner.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ jobs:
with:
fetch-depth: 0
- name: TruffleHog OSS
uses: trufflesecurity/trufflehog@0c66d30c1f4075cee1aada2e1ab46dabb1b0071a
uses: trufflesecurity/trufflehog@781157ae368b2218a0a56b889387dd26faa20f97
with:
path: ./
base: ${{ github.event.repository.default_branch }}
Expand Down
5 changes: 5 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ google-cloud-storage = "0.20.0"
governor = "0.4.2"
hex = "0.4"
http = "1.1"
http-body-util = "0.1.2"
httpmock = "0.7.0"
hyper = "1.3"
insta = "1.29.0"
Expand Down Expand Up @@ -219,6 +220,10 @@ pbjson-types = "0.6.0"

# Eigen
tokio-stream = "0.1.16"
rust-kzg-bn254 = "0.2.1"
ark-bn254 = "0.5.0"
num-bigint = "0.4.6"
serial_test = "3.1.1"

# Here and below:
# We *always* pin the latest version of protocol to disallow accidental changes in the execution logic.
Expand Down
5 changes: 1 addition & 4 deletions core/bin/contract-verifier/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ publish = false

[dependencies]
zksync_dal.workspace = true
zksync_env_config.workspace = true
zksync_config = { workspace = true, features = ["observability_ext"] }
zksync_contract_verifier_lib.workspace = true
zksync_queued_job_processor.workspace = true
Expand All @@ -21,8 +20,6 @@ zksync_vlog.workspace = true
zksync_core_leftovers.workspace = true

anyhow.workspace = true
clap = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["full"] }
futures.workspace = true
ctrlc.workspace = true
structopt.workspace = true
tracing.workspace = true
53 changes: 18 additions & 35 deletions core/bin/contract-verifier/src/main.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use std::{cell::RefCell, time::Duration};
use std::{path::PathBuf, time::Duration};

use anyhow::Context;
use futures::{channel::mpsc, executor::block_on, SinkExt, StreamExt};
use structopt::StructOpt;
use anyhow::Context as _;
use clap::Parser;
use tokio::sync::watch;
use zksync_config::configs::PrometheusConfig;
use zksync_contract_verifier_lib::ContractVerifier;
Expand All @@ -12,27 +11,31 @@ use zksync_queued_job_processor::JobProcessor;
use zksync_utils::wait_for_tasks::ManagedTasks;
use zksync_vlog::prometheus::PrometheusExporterConfig;

#[derive(StructOpt)]
#[structopt(name = "ZKsync contract code verifier", author = "Matter Labs")]
#[derive(Debug, Parser)]
#[command(name = "ZKsync contract code verifier", author = "Matter Labs")]
struct Opt {
/// Number of jobs to process. If None, runs indefinitely.
#[structopt(long)]
#[arg(long)]
jobs_number: Option<usize>,
/// Path to the configuration file.
#[structopt(long)]
config_path: Option<std::path::PathBuf>,
#[arg(long)]
config_path: Option<PathBuf>,
/// Path to the secrets file.
#[structopt(long)]
secrets_path: Option<std::path::PathBuf>,
#[arg(long)]
secrets_path: Option<PathBuf>,
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
let opt = Opt::from_args();
let opt = Opt::parse();

let general_config = load_general_config(opt.config_path).context("general config")?;
let database_secrets = load_database_secrets(opt.secrets_path).context("database secrets")?;
let observability_config = general_config
.observability
.context("ObservabilityConfig")?;
let _observability_guard = observability_config.install()?;

let database_secrets = load_database_secrets(opt.secrets_path).context("database secrets")?;
let verifier_config = general_config
.contract_verifier
.context("ContractVerifierConfig")?;
Expand All @@ -46,33 +49,13 @@ async fn main() -> anyhow::Result<()> {
.context("Master DB URL is absent")?,
)
.build()
.await
.unwrap();

let observability_config = general_config
.observability
.context("ObservabilityConfig")?;

let _observability_guard = observability_config.install()?;
.await?;

let (stop_sender, stop_receiver) = watch::channel(false);
let (stop_signal_sender, mut stop_signal_receiver) = mpsc::channel(256);
{
let stop_signal_sender = RefCell::new(stop_signal_sender.clone());
ctrlc::set_handler(move || {
let mut sender = stop_signal_sender.borrow_mut();
block_on(sender.send(true)).expect("Ctrl+C signal send");
})
.expect("Error setting Ctrl+C handler");
}

let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool)
.await
.context("failed initializing contract verifier")?;
let tasks = vec![
// TODO PLA-335: Leftovers after the prover DB split.
// The prover connection pool is not used by the contract verifier, but we need to pass it
// since `JobProcessor` trait requires it.
tokio::spawn(contract_verifier.run(stop_receiver.clone(), opt.jobs_number)),
tokio::spawn(
PrometheusExporterConfig::pull(prometheus_config.listener_port).run(stop_receiver),
Expand All @@ -82,7 +65,7 @@ async fn main() -> anyhow::Result<()> {
let mut tasks = ManagedTasks::new(tasks);
tokio::select! {
() = tasks.wait_single() => {},
_ = stop_signal_receiver.next() => {
_ = tokio::signal::ctrl_c() => {
tracing::info!("Stop signal received, shutting down");
},
};
Expand Down
26 changes: 3 additions & 23 deletions core/lib/config/src/configs/da_client/eigen.rs
Original file line number Diff line number Diff line change
@@ -1,30 +1,8 @@
use serde::Deserialize;
use zksync_basic_types::secrets::PrivateKey;

pub const EIGEN_MEMSTORE_CLIENT_NAME: &str = "MemStore";
pub const EIGEN_DISPERSER_CLIENT_NAME: &str = "Disperser";

#[derive(Clone, Debug, PartialEq, Deserialize)]
pub enum EigenConfig {
MemStore(MemStoreConfig),
Disperser(DisperserConfig),
}

/// Configuration for the EigenDA in-memory client.
#[derive(Clone, Debug, PartialEq, Deserialize, Default)]
pub struct MemStoreConfig {
pub max_blob_size_bytes: u64,
/// Blob expiration time in seconds
pub blob_expiration: u64,
/// Latency in milliseconds for get operations
pub get_latency: u64,
/// Latency in milliseconds for put operations
pub put_latency: u64,
}

/// Configuration for the EigenDA remote disperser client.
#[derive(Clone, Debug, PartialEq, Deserialize, Default)]
pub struct DisperserConfig {
pub struct EigenConfig {
/// URL of the Disperser RPC server
pub disperser_rpc: String,
/// Block height needed to reach in order to consider the blob finalized
Expand All @@ -48,6 +26,8 @@ pub struct DisperserConfig {
pub verify_cert: bool,
/// Path to the file containing the points used for KZG
pub path_to_points: String,
/// Chain ID of the Ethereum network
pub chain_id: u64,
}

#[derive(Clone, Debug, PartialEq)]
Expand Down
Loading
Loading