From bcfef146efa1e56e0850563b81f7eb324753ac42 Mon Sep 17 00:00:00 2001 From: Ho Date: Tue, 29 Aug 2023 20:44:24 +0800 Subject: [PATCH] refactor codes to be compatible with "use scroll l2 trace for super circuit testing in zkevm-circuits" feature (#249) * wip: refactor according to new CircuitBuilder * dump zkevm-circuits' version and pass compile * wip: refactor builder * update lock * Update mock.rs * bump version of zkevm-circuits * refactor circuit builder and ccc * post merging fixes * bump zkevm-circuits temporary disable missed circuit params * bump circuit version refine ccc with new circuibuilder constructor * clippy and fmt * bump zkevm * develop branch uses zkevm-circuits/decelop (#248) * run * upgrade zkevm-circuits * fix tx circuit ccc * Merge remote-tracking branch 'origin/main' into develop_pre * resume missed changes --------- Co-authored-by: Zhang Zhuo --- Cargo.lock | 22 +- prover/Cargo.toml | 12 +- prover/src/utils.rs | 2 +- prover/src/zkevm/capacity_checker.rs | 80 +++-- prover/src/zkevm/circuit.rs | 4 +- prover/src/zkevm/circuit/builder.rs | 513 +++++++++------------------ types/Cargo.toml | 2 +- types/src/eth.rs | 241 ------------- types/src/lib.rs | 8 +- 9 files changed, 240 insertions(+), 644 deletions(-) delete mode 100644 types/src/eth.rs diff --git a/Cargo.lock b/Cargo.lock index c8f96d384..28f22d527 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "ark-std", "env_logger 0.10.0", @@ -419,7 +419,7 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bus-mapping" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -433,6 +433,7 @@ dependencies = [ "lazy_static", "log", "mock", + "mpt-zktrie", "once_cell", "poseidon-circuit", "rand", @@ -1122,7 +1123,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "ethers-core 0.17.0", "ethers-signers", @@ -1387,7 +1388,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "eth-types", "geth-utils", @@ -1600,7 +1601,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "digest 0.7.6", "eth-types", @@ -1640,7 +1641,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "env_logger 0.9.3", "gobuild 0.1.0-alpha.2 (git+https://github.com/scroll-tech/gobuild.git)", @@ -2246,7 +2247,7 @@ dependencies = [ [[package]] name = "keccak256" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "env_logger 0.9.3", "eth-types", @@ -2446,7 +2447,7 @@ dependencies = [ [[package]] name = "mock" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "eth-types", "ethers-core 0.17.0", @@ -2461,9 +2462,8 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ - "bus-mapping", "eth-types", "halo2-mpt-circuits", "halo2_proofs", @@ -4712,7 +4712,7 @@ checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" [[package]] name = "zkevm-circuits" version = "0.1.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.7.5#aa9a9aff698a5b253d1f3c29ea3d3006364777bf" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?branch=develop#a74ba4211e6178d623279abc5a8e61231189f173" dependencies = [ "array-init", "bus-mapping", diff --git a/prover/Cargo.toml b/prover/Cargo.toml index c3878671b..425741bf5 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -8,12 +8,12 @@ edition = "2021" [dependencies] halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02" } -aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5" } -bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5" } -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5" } -zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } -mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5" } -mock = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5" } +aggregator = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } +bus-mapping = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } +zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["test","scroll","scroll-trace","shanghai"] } +mpt-zktrie = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } +mock = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } snark-verifier = { git = "https://github.com/scroll-tech/snark-verifier", tag = "v0.1.2" } snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", tag = "v0.1.2" } diff --git a/prover/src/utils.rs b/prover/src/utils.rs index aa663aef3..637dae66f 100644 --- a/prover/src/utils.rs +++ b/prover/src/utils.rs @@ -24,7 +24,7 @@ use std::{ str::FromStr, sync::Once, }; -use types::eth::{BlockTrace, BlockTraceJsonRpcResult}; +use types::{eth::BlockTrace, BlockTraceJsonRpcResult}; use zkevm_circuits::evm_circuit::witness::Block; pub static LOGGER: Once = Once::new(); diff --git a/prover/src/zkevm/capacity_checker.rs b/prover/src/zkevm/capacity_checker.rs index 8bde1f0dd..b257b41cd 100644 --- a/prover/src/zkevm/capacity_checker.rs +++ b/prover/src/zkevm/capacity_checker.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use super::circuit::{ MAX_BYTECODE, MAX_CALLDATA, MAX_EXP_STEPS, MAX_KECCAK_ROWS, MAX_MPT_ROWS, MAX_POSEIDON_ROWS, MAX_RWS, MAX_VERTICLE_ROWS, @@ -7,9 +5,13 @@ use super::circuit::{ use super::circuit::{ block_traces_to_witness_block_with_updated_state, calculate_row_usage_of_witness_block, - fill_zktrie_state_from_proofs, + get_super_circuit_params, +}; +use bus_mapping::{ + circuit_input_builder::{self, CircuitInputBuilder}, + state_db::{CodeDB, StateDB}, }; -use eth_types::H256; +use eth_types::{ToWord, H256}; use itertools::Itertools; use mpt_zktrie::state::ZktrieState; use serde_derive::{Deserialize, Serialize}; @@ -52,7 +54,7 @@ impl RowUsage { MAX_BYTECODE, // bytecode MAX_RWS, // copy MAX_KECCAK_ROWS, // keccak - MAX_CALLDATA, // tx + MAX_VERTICLE_ROWS, // tx MAX_CALLDATA, // rlp 7 * MAX_EXP_STEPS, // exp MAX_KECCAK_ROWS, // modexp @@ -118,9 +120,7 @@ pub struct CircuitCapacityChecker { pub light_mode: bool, pub acc_row_usage: RowUsage, pub row_usages: Vec, - pub state: Option, - // poseidon codehash to code len - pub codelen: HashMap, + pub builder_ctx: Option<(CodeDB, StateDB, ZktrieState)>, } // Currently TxTrace is same as BlockTrace, with "transactions" and "executionResults" should be of @@ -139,44 +139,51 @@ impl CircuitCapacityChecker { Self { acc_row_usage: RowUsage::new(), row_usages: Vec::new(), - state: None, light_mode: true, - codelen: HashMap::new(), + builder_ctx: None, } } pub fn reset(&mut self) { - self.state = None; + self.builder_ctx = None; self.acc_row_usage = RowUsage::new(); self.row_usages = Vec::new(); - self.codelen = HashMap::new(); } pub fn estimate_circuit_capacity( &mut self, txs: &[TxTrace], ) -> Result { assert!(!txs.is_empty()); - if self.state.is_none() { - self.state = Some(ZktrieState::construct(txs[0].storage_trace.root_before)); - } - let traces = txs; - let state = self.state.as_mut().unwrap(); - fill_zktrie_state_from_proofs(state, traces, self.light_mode)?; - let (witness_block, codedb) = - block_traces_to_witness_block_with_updated_state(traces, state, self.light_mode)?; - let mut rows = calculate_row_usage_of_witness_block(&witness_block)?; - - // Dedup bytecode row usage for bytecode circuit / poseidon circuit - for (hash, bytes) in &codedb.0 { - if self.codelen.contains_key(hash) { - assert_eq!(rows[2].name, "bytecode"); - rows[2].row_num_real -= bytes.len() + 1; - assert_eq!(rows[10].name, "poseidon"); - rows[10].row_num_real -= bytes.len() / (31 * 2) * 9; - } else { - self.codelen.insert(*hash, bytes.len()); - } - } - + let mut estimate_builder = if let Some((code_db, sdb, mpt_state)) = self.builder_ctx.take() + { + // here we create a new builder for another (sealed) witness block + // this builder inherit the current execution state (sdb/cdb) of + // the previous one and do not use zktrie state, + // notice the prev_root in current builder may be not invalid (since the state has + // changed but we may not update it in light mode) + let mut builder_block = + circuit_input_builder::Block::from_headers(&[], get_super_circuit_params()); + builder_block.chain_id = txs[0].chain_id; + builder_block.start_l1_queue_index = txs[0].start_l1_queue_index; + builder_block.prev_state_root = H256(*mpt_state.root()).to_word(); + let mut builder = + CircuitInputBuilder::new_with_trie_state(sdb, code_db, mpt_state, &builder_block); + builder.add_more_l2_trace(&txs[0], txs.len() > 1, true)?; + builder + } else { + CircuitInputBuilder::new_from_l2_trace( + get_super_circuit_params(), + &txs[0], + txs.len() > 1, + true, + )? + }; + let traces = &txs[1..]; + let witness_block = block_traces_to_witness_block_with_updated_state( + traces, + &mut estimate_builder, + self.light_mode, + )?; + let rows = calculate_row_usage_of_witness_block(&witness_block)?; let row_usage_details: Vec = rows .into_iter() .map(|x| SubCircuitRowUsage { @@ -187,6 +194,11 @@ impl CircuitCapacityChecker { let tx_row_usage = RowUsage::from_row_usage_details(row_usage_details); self.row_usages.push(tx_row_usage.clone()); self.acc_row_usage.add(&tx_row_usage); + self.builder_ctx.replace(( + estimate_builder.code_db, + estimate_builder.sdb, + estimate_builder.mpt_init_state, + )); Ok(self.acc_row_usage.normalize()) } } diff --git a/prover/src/zkevm/circuit.rs b/prover/src/zkevm/circuit.rs index 9b3a33c65..186355def 100644 --- a/prover/src/zkevm/circuit.rs +++ b/prover/src/zkevm/circuit.rs @@ -13,8 +13,8 @@ use crate::utils::read_env_var; pub use self::builder::{ block_traces_to_padding_witness_block, block_traces_to_witness_block, block_traces_to_witness_block_with_updated_state, calculate_row_usage_of_trace, - calculate_row_usage_of_witness_block, check_batch_capacity, fill_zktrie_state_from_proofs, - normalize_withdraw_proof, storage_trace_to_padding_witness_block, WitnessBlock, + calculate_row_usage_of_witness_block, check_batch_capacity, get_super_circuit_params, + normalize_withdraw_proof, WitnessBlock, }; pub use builder::{ MAX_BYTECODE, MAX_CALLDATA, MAX_EXP_STEPS, MAX_INNER_BLOCKS, MAX_KECCAK_ROWS, MAX_MPT_ROWS, diff --git a/prover/src/zkevm/circuit/builder.rs b/prover/src/zkevm/circuit/builder.rs index 367901b93..7f56ac472 100644 --- a/prover/src/zkevm/circuit/builder.rs +++ b/prover/src/zkevm/circuit/builder.rs @@ -2,24 +2,19 @@ use super::{TargetCircuit, AUTO_TRUNCATE, CHAIN_ID}; use crate::config::INNER_DEGREE; use anyhow::{bail, Result}; use bus_mapping::{ - circuit_input_builder::{ - self, BlockHead, CircuitInputBuilder, CircuitsParams, PrecompileEcParams, - }, - state_db::{Account, CodeDB, StateDB}, + circuit_input_builder::{self, CircuitInputBuilder, CircuitsParams, PrecompileEcParams}, + state_db::{CodeDB, StateDB}, }; -use eth_types::{evm_types::opcode_ids::OpcodeId, ToAddress, ToBigEndian, H256}; -use ethers_core::types::{Bytes, U256}; +use eth_types::{ToBigEndian, ToWord, H256}; use halo2_proofs::halo2curves::bn256::Fr; -use is_even::IsEven; use itertools::Itertools; use mpt_zktrie::state::ZktrieState; -use std::{ - collections::{hash_map::Entry, HashMap}, - time::Instant, -}; -use types::eth::{BlockTrace, EthBlock, ExecStep, StorageTrace}; +use std::{collections::HashMap, time::Instant}; +use types::eth::{BlockTrace, StorageTrace}; use zkevm_circuits::{ - evm_circuit::witness::{block_apply_mpt_state, block_convert_with_l1_queue_index, Block}, + evm_circuit::witness::{ + block_apply_mpt_state, block_convert, block_convert_with_l1_queue_index, Block, + }, util::SubCircuit, witness::WithdrawProof, }; @@ -41,7 +36,8 @@ pub const MAX_PRECOMPILE_EC_ADD: usize = 50; pub const MAX_PRECOMPILE_EC_MUL: usize = 50; pub const MAX_PRECOMPILE_EC_PAIRING: usize = 2; -fn get_super_circuit_params() -> CircuitsParams { +/// default params for super circuit +pub fn get_super_circuit_params() -> CircuitsParams { CircuitsParams { max_evm_rows: MAX_RWS, max_rws: MAX_RWS, @@ -183,57 +179,48 @@ pub fn check_batch_capacity(block_traces: &mut Vec) -> Result<()> { Ok(()) } -pub fn fill_zktrie_state_from_proofs( - zktrie_state: &mut ZktrieState, - block_traces: &[BlockTrace], - light_mode: bool, -) -> Result<()> { - log::debug!( - "building partial statedb, old root {}, light_mode {}", - hex::encode(zktrie_state.root()), - light_mode - ); - let account_proofs = block_traces.iter().flat_map(|block| { - log::trace!("account proof for block {:?}:", block.header.number); - block.storage_trace.proofs.iter().flat_map(|kv_map| { - kv_map - .iter() - .map(|(k, bts)| (k, bts.iter().map(Bytes::as_ref))) - }) - }); - let storage_proofs = block_traces.iter().flat_map(|block| { - log::trace!("storage proof for block {:?}:", block.header.number); - block - .storage_trace - .storage_proofs - .iter() - .flat_map(|(k, kv_map)| { - kv_map - .iter() - .map(move |(sk, bts)| (k, sk, bts.iter().map(Bytes::as_ref))) - }) - }); - let additional_proofs = block_traces.iter().flat_map(|block| { - log::trace!("storage proof for block {:?}:", block.header.number); - log::trace!("additional proof for block {:?}:", block.header.number); - block - .storage_trace - .deletion_proofs - .iter() - .map(Bytes::as_ref) - }); - zktrie_state.update_statedb_from_proofs( - account_proofs.clone(), - storage_proofs.clone(), - additional_proofs.clone(), - )?; - if !light_mode { - zktrie_state.update_nodes_from_proofs(account_proofs, storage_proofs, additional_proofs)?; +// prepare an empty builder which can updated by more trace +// from the default settings +// only require the prev state root being provided +// any initial zktrie state can be also set +fn prepare_default_builder( + old_root: H256, + initial_mpt_state: Option, +) -> CircuitInputBuilder { + let mut builder_block = + circuit_input_builder::Block::from_headers(&[], get_super_circuit_params()); + builder_block.chain_id = *CHAIN_ID; + builder_block.prev_state_root = old_root.to_word(); + let code_db = CodeDB::new(); + + if let Some(mpt_state) = initial_mpt_state { + assert_eq!( + H256::from_slice(mpt_state.root()), + old_root, + "the provided zktrie state must be the prev state" + ); + let state_db = StateDB::from(&mpt_state); + let mut builder = CircuitInputBuilder::new(state_db, code_db, &builder_block); + builder.mpt_init_state = mpt_state; + builder + } else { + CircuitInputBuilder::new(StateDB::new(), code_db, &builder_block) + } +} + +// check if block traces match preset parameters +fn validite_block_traces(block_traces: &[BlockTrace]) -> Result<()> { + let chain_id = block_traces + .iter() + .map(|block_trace| block_trace.chain_id) + .next() + .unwrap_or(*CHAIN_ID); + if *CHAIN_ID != chain_id { + bail!( + "CHAIN_ID env var is wrong. chain id in trace {chain_id}, CHAIN_ID {}", + *CHAIN_ID + ); } - log::debug!( - "building partial statedb done, root {}", - hex::encode(zktrie_state.root()) - ); Ok(()) } @@ -259,14 +246,21 @@ pub fn block_traces_to_witness_block(block_traces: &[BlockTrace]) -> Result 1, + false, + )?; + block_traces_to_witness_block_with_updated_state(&block_traces[1..], &mut builder, false) + } } pub fn block_traces_to_padding_witness_block(block_traces: &[BlockTrace]) -> Result> { @@ -274,139 +268,124 @@ pub fn block_traces_to_padding_witness_block(block_traces: &[BlockTrace]) -> Res "block_traces_to_padding_witness_block, input len {:?}", block_traces.len() ); - let chain_id = block_traces - .iter() - .map(|block_trace| block_trace.chain_id) - .next() - .unwrap_or(*CHAIN_ID); - if *CHAIN_ID != chain_id { - bail!( - "CHAIN_ID env var is wrong. chain id in trace {chain_id}, CHAIN_ID {}", - *CHAIN_ID - ); - } - let old_root = if block_traces.is_empty() { - eth_types::Hash::zero() + validite_block_traces(block_traces)?; + + // the only purpose here it to get the final zktrie state and + // proof for withdraw root + let mut padding_builder = if block_traces.is_empty() { + log::debug!("preparing default builder"); + prepare_default_builder(H256::zero(), None) } else { - block_traces[0].storage_trace.root_before + let start_l1_queue_index = block_traces[0].start_l1_queue_index; + log::debug!( + "new from l2 trace, block num {:?}", + block_traces[0].header.number + ); + let mut builder = CircuitInputBuilder::new_from_l2_trace( + get_super_circuit_params(), + &block_traces[0], + block_traces.len() > 1, + false, + )?; + for (idx, block_trace) in block_traces[1..].iter().enumerate() { + log::debug!( + "adding more l2 trace block_trace idx {}, block num {:?}", + idx + 1, + block_trace.header.number + ); + builder.add_more_l2_trace( + block_trace, + idx + 2 == block_traces.len(), //not typo, we use 1..end of the traces only + false, + )?; + } + builder.finalize_building()?; + let mut witness_block = block_convert_with_l1_queue_index::( + &builder.block, + &builder.code_db, + start_l1_queue_index, + )?; + log::debug!( + "witness_block built with circuits_params {:?} for padding", + witness_block.circuits_params + ); + // so we have the finalized state which contain withdraw proof + block_apply_mpt_state(&mut witness_block, &builder.mpt_init_state); + let old_root = H256(*builder.mpt_init_state.root()); + prepare_default_builder(old_root, Some(builder.mpt_init_state)) }; - let mut state = ZktrieState::construct(old_root); - fill_zktrie_state_from_proofs(&mut state, block_traces, false)?; - - // the only purpose here it to get the updated zktrie state - let prev_witness_block = - block_traces_to_witness_block_with_updated_state(block_traces, &mut state, false)?.0; // TODO: when prev_witness_block.tx.is_empty(), the `withdraw_proof` here should be a subset of // storage proofs of prev block - let storage_trace = normalize_withdraw_proof(&prev_witness_block.mpt_updates.withdraw_proof); - storage_trace_to_padding_witness_block(storage_trace) -} + padding_builder.finalize_building()?; -pub fn storage_trace_to_padding_witness_block(storage_trace: StorageTrace) -> Result> { - log::debug!( - "withdraw proof {}", - serde_json::to_string_pretty(&storage_trace)? - ); + let mut padding_block = block_convert(&padding_builder.block, &padding_builder.code_db)?; + // drag the withdraw proof from zktrie state + block_apply_mpt_state(&mut padding_block, &padding_builder.mpt_init_state); - let mut state = ZktrieState::construct(storage_trace.root_before); - let dummy_chunk_traces = vec![BlockTrace { - chain_id: *CHAIN_ID, - storage_trace, - ..Default::default() - }]; - fill_zktrie_state_from_proofs(&mut state, &dummy_chunk_traces, false)?; - Ok(block_traces_to_witness_block_with_updated_state(&[], &mut state, false)?.0) + Ok(padding_block) } +/// update the builder with another batch of trace and then *FINALIZE* it +/// (so the buidler CAN NOT be update any more) +/// light_mode skip the time consuming calculation on mpt root for each +/// tx, currently used in row estimation pub fn block_traces_to_witness_block_with_updated_state( block_traces: &[BlockTrace], - zktrie_state: &mut ZktrieState, - light_mode: bool, // light_mode used in row estimation -) -> Result<(Block, CodeDB)> { - let chain_id = block_traces - .iter() - .map(|block_trace| block_trace.chain_id) - .next() - .unwrap_or(*CHAIN_ID); - // total l1 msgs popped before this chunk - let start_l1_queue_index = block_traces - .iter() - .map(|block_trace| block_trace.start_l1_queue_index) - .next() - .unwrap_or(0); - if *CHAIN_ID != chain_id { - bail!( - "CHAIN_ID env var is wrong. chain id in trace {chain_id}, CHAIN_ID {}", - *CHAIN_ID + builder: &mut CircuitInputBuilder, + light_mode: bool, +) -> Result> { + let metric = |builder: &CircuitInputBuilder, idx: usize| -> Result<(), bus_mapping::Error> { + let t = Instant::now(); + let block = block_convert_with_l1_queue_index::( + &builder.block, + &builder.code_db, + builder.block.start_l1_queue_index, + )?; + log::debug!("block convert time {:?}", t.elapsed()); + let rows = ::Inner::min_num_rows_block(&block); + log::debug!( + "after block {}, tx num {:?}, tx len sum {}, rows needed {:?}. estimate time: {:?}", + idx, + builder.block.txs().len(), + builder + .block + .txs() + .iter() + .map(|t| t.input.len()) + .sum::(), + rows, + t.elapsed() ); - } + Ok(()) + }; - let mut state_db: StateDB = zktrie_state.state().clone(); + // TODO: enable this switch + let per_block_metric = false; - let (zero_coinbase_exist, _) = state_db.get_account(&Default::default()); - if !zero_coinbase_exist { - state_db.set_account(&Default::default(), Account::zero()); - } + let initial_blk_index = if builder.block.txs.is_empty() { + 0 + } else { + if per_block_metric { + metric(builder, 0)?; + } + 1 + }; - let code_db = build_codedb(&state_db, block_traces)?; - let circuit_params = get_super_circuit_params(); - let mut builder_block = circuit_input_builder::Block::from_headers(&[], circuit_params); - builder_block.chain_id = chain_id; - builder_block.prev_state_root = U256::from(zktrie_state.root()); - let mut builder = CircuitInputBuilder::new(state_db.clone(), code_db.clone(), &builder_block); for (idx, block_trace) in block_traces.iter().enumerate() { let is_last = idx == block_traces.len() - 1; - let eth_block: EthBlock = block_trace.clone().into(); - - let mut geth_trace = Vec::new(); - for result in &block_trace.execution_results { - geth_trace.push(result.into()); - } - // TODO: Get the history_hashes. - let mut header = BlockHead::new_with_l1_queue_index( - chain_id, - block_trace.start_l1_queue_index, - Vec::new(), - ð_block, - )?; - // override zeroed minder field with additional "coinbase" field in blocktrace - if let Some(address) = block_trace.coinbase.address { - header.coinbase = address; - } - let block_num = header.number.as_u64(); - builder.block.start_l1_queue_index = start_l1_queue_index; // the chunk's start_l1_queue_index - builder.block.headers.insert(block_num, header); - builder.handle_block_inner(ð_block, geth_trace.as_slice(), false, is_last)?; - log::debug!("handle_block_inner done for block {:?}", block_num); - let per_block_metric = false; + log::debug!( + "add_more_l2_trace idx {idx}, block num {:?}", + block_trace.header.number + ); + builder.add_more_l2_trace(block_trace, !is_last, false)?; if per_block_metric { - let t = Instant::now(); - let block = block_convert_with_l1_queue_index::( - &builder.block, - &builder.code_db, - start_l1_queue_index, - )?; - log::debug!("block convert time {:?}", t.elapsed()); - let rows = ::Inner::min_num_rows_block(&block); - log::debug!( - "after block {}, tx num {:?}, tx len sum {}, rows needed {:?}. estimate time: {:?}", - idx, - builder.block.txs().len(), - builder - .block - .txs() - .iter() - .map(|t| t.input.len()) - .sum::(), - rows, - t.elapsed() - ); + metric(builder, idx + initial_blk_index)?; } } - builder.set_value_ops_call_context_rwc_eor(); - builder.set_end_block()?; + builder.finalize_building()?; + let start_l1_queue_index = builder.block.start_l1_queue_index; log::debug!("converting builder.block to witness block"); let mut witness_block = @@ -416,176 +395,16 @@ pub fn block_traces_to_witness_block_with_updated_state( witness_block.circuits_params ); - if !light_mode && zktrie_state.root() != &[0u8; 32] { + if !light_mode && builder.mpt_init_state.root() != &[0u8; 32] { log::debug!("block_apply_mpt_state"); - block_apply_mpt_state(&mut witness_block, zktrie_state); + block_apply_mpt_state(&mut witness_block, &builder.mpt_init_state); log::debug!("block_apply_mpt_state done"); } - zktrie_state.set_state(builder.sdb.clone()); log::debug!( "finish replay trie updates, root {}", - hex::encode(zktrie_state.root()) + hex::encode(builder.mpt_init_state.root()) ); - Ok((witness_block, code_db)) -} - -pub fn decode_bytecode(bytecode: &str) -> Result> { - let mut stripped = if let Some(stripped) = bytecode.strip_prefix("0x") { - stripped.to_string() - } else { - bytecode.to_string() - }; - - let bytecode_len = stripped.len() as u64; - if !bytecode_len.is_even() { - stripped = format!("0{stripped}"); - } - - hex::decode(stripped).map_err(|e| e.into()) -} - -fn trace_code( - cdb: &mut CodeDB, - code_hash: Option, - code: Bytes, - step: &ExecStep, - sdb: &StateDB, - stack_pos: usize, -) { - // first, try to read from sdb - let stack = step - .stack - .as_ref() - .expect("should have stack in call context"); - let addr = stack[stack.len() - stack_pos - 1].to_address(); //stack N-stack_pos - - let code_hash = code_hash.or_else(|| { - let (_existed, acc_data) = sdb.get_account(&addr); - if acc_data.code_hash != CodeDB::empty_code_hash() && !code.is_empty() { - // they must be same - Some(acc_data.code_hash) - } else { - // let us re-calculate it - None - } - }); - let code_hash = match code_hash { - Some(code_hash) => { - if code_hash.is_zero() { - CodeDB::hash(&code) - } else { - if log::log_enabled!(log::Level::Trace) { - assert_eq!( - code_hash, - CodeDB::hash(&code), - "bytecode len {:?}, step {:?}", - code.len(), - step - ); - } - code_hash - } - } - None => { - let hash = CodeDB::hash(&code); - log::debug!( - "hash_code done: addr {addr:?}, size {}, hash {hash:?}", - &code.len() - ); - hash - } - }; - - cdb.0.entry(code_hash).or_insert_with(|| { - log::trace!( - "trace code addr {:?}, size {} hash {:?}", - addr, - &code.len(), - code_hash - ); - code.to_vec() - }); -} - -pub fn build_codedb(sdb: &StateDB, blocks: &[BlockTrace]) -> Result { - let mut cdb = CodeDB::new(); - log::debug!("building codedb"); - - cdb.insert(Vec::new()); - - for block in blocks.iter().rev() { - log::debug!("build_codedb for block {:?}", block.header.number); - for (er_idx, execution_result) in block.execution_results.iter().enumerate() { - if let Some(bytecode) = &execution_result.byte_code { - let bytecode = decode_bytecode(bytecode)?.to_vec(); - - let code_hash = execution_result - .to - .as_ref() - .and_then(|t| t.poseidon_code_hash) - .unwrap_or_else(|| CodeDB::hash(&bytecode)); - let code_hash = if code_hash.is_zero() { - CodeDB::hash(&bytecode) - } else { - code_hash - }; - if let Entry::Vacant(e) = cdb.0.entry(code_hash) { - e.insert(bytecode); - //log::debug!("inserted tx bytecode {:?} {:?}", code_hash, hash); - } - if execution_result.account_created.is_none() { - //assert_eq!(Some(hash), execution_result.code_hash); - } - } - - for step in execution_result.exec_steps.iter().rev() { - if let Some(data) = &step.extra_data { - match step.op { - OpcodeId::CALL - | OpcodeId::CALLCODE - | OpcodeId::DELEGATECALL - | OpcodeId::STATICCALL => { - let code_idx = if block.transactions[er_idx].to.is_none() { - 0 - } else { - 1 - }; - let callee_code = data.get_code_at(code_idx); - if callee_code.is_none() { - bail!("invalid trace: cannot get code of call: {:?}", step); - } - let code_hash = match step.op { - OpcodeId::CALL | OpcodeId::CALLCODE => data.get_code_hash_at(1), - OpcodeId::STATICCALL => data.get_code_hash_at(0), - _ => None, - }; - trace_code(&mut cdb, code_hash, callee_code.unwrap(), step, sdb, 1); - } - OpcodeId::CREATE | OpcodeId::CREATE2 => { - // notice we do not need to insert code for CREATE, - // bustmapping do this job - } - OpcodeId::EXTCODESIZE | OpcodeId::EXTCODECOPY => { - let code = data.get_code_at(0); - if code.is_none() { - bail!("invalid trace: cannot get code of ext: {:?}", step); - } - trace_code(&mut cdb, None, code.unwrap(), step, sdb, 0); - } - - _ => {} - } - } - } - } - } - - log::debug!("building codedb done"); - for (k, v) in &cdb.0 { - assert!(!k.is_zero()); - log::trace!("codedb codehash {:?}, len {}", k, v.len()); - } - Ok(cdb) + Ok(witness_block) } pub fn normalize_withdraw_proof(proof: &WithdrawProof) -> StorageTrace { diff --git a/types/Cargo.toml b/types/Cargo.toml index e31d0e5f1..f5a7164ff 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -4,7 +4,7 @@ version = "0.7.5" edition = "2021" [dependencies] -eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.7.5" } +eth-types = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop" } base64 = "0.13.0" blake2 = "0.10.3" ethers-core = "0.17.0" diff --git a/types/src/eth.rs b/types/src/eth.rs deleted file mode 100644 index 382f453c8..000000000 --- a/types/src/eth.rs +++ /dev/null @@ -1,241 +0,0 @@ -use eth_types::{ - evm_types::{Gas, GasCost, OpcodeId, ProgramCounter, Stack, Storage}, - Block, GethExecStep, GethExecTrace, Hash, Transaction, Word, H256, -}; -use ethers_core::types::{Address, Bytes, U256, U64}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Deserialize, Serialize, Default, Debug, Clone)] -pub struct BlockTrace { - #[serde(rename = "chainID", default)] - pub chain_id: u64, - pub coinbase: AccountProofWrapper, - pub header: EthBlock, - pub transactions: Vec, - #[serde(rename = "executionResults")] - pub execution_results: Vec, - #[serde(rename = "storageTrace")] - pub storage_trace: StorageTrace, - #[serde(rename = "txStorageTraces", default)] - pub tx_storage_trace: Vec, - #[serde(rename = "startL1QueueIndex", default)] - pub start_l1_queue_index: u64, - // #[serde(rename = "mptwitness", default)] - // pub mpt_witness: Vec, -} - -#[derive(Deserialize, Serialize, Default, Debug, Clone)] -pub struct BlockTraceJsonRpcResult { - pub result: BlockTrace, -} - -impl From for EthBlock { - fn from(mut b: BlockTrace) -> Self { - let mut txs = Vec::new(); - for (idx, tx_data) in b.transactions.iter_mut().enumerate() { - let tx_idx = Some(U64::from(idx)); - let tx = tx_data.to_eth_tx(b.header.hash, b.header.number, tx_idx); - txs.push(tx) - } - EthBlock { - transactions: txs, - difficulty: 0.into(), - ..b.header - } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct TransactionTrace { - // FIXME after traces upgraded - #[serde(default, rename = "txHash")] - pub tx_hash: H256, - #[serde(rename = "type")] - pub type_: u8, - pub nonce: u64, - pub gas: u64, - #[serde(rename = "gasPrice")] - pub gas_price: U256, - pub from: Address, - pub to: Option
, - #[serde(rename = "chainId")] - pub chain_id: U256, - pub value: U256, - pub data: Bytes, - #[serde(rename = "isCreate")] - pub is_create: bool, - pub v: U64, - pub r: U256, - pub s: U256, -} - -impl TransactionTrace { - pub fn to_eth_tx( - &self, - block_hash: Option, - block_number: Option, - transaction_index: Option, - ) -> Transaction { - Transaction { - hash: self.tx_hash, - nonce: U256::from(self.nonce), - block_hash, - block_number, - transaction_index, - from: self.from, - to: self.to, - value: self.value, - gas_price: Some(self.gas_price), - gas: U256::from(self.gas), - input: self.data.clone(), - v: self.v, - r: self.r, - s: self.s, - transaction_type: Some(U64::from(self.type_ as u64)), - access_list: None, - max_priority_fee_per_gas: None, - max_fee_per_gas: None, - chain_id: Some(self.chain_id), - other: Default::default(), - } - } -} - -pub type AccountTrieProofs = HashMap>; -pub type StorageTrieProofs = HashMap>>; - -#[derive(Deserialize, Serialize, Default, Debug, Clone)] -pub struct StorageTrace { - #[serde(rename = "rootBefore")] - pub root_before: Hash, - #[serde(rename = "rootAfter")] - pub root_after: Hash, - pub proofs: Option, - #[serde(rename = "storageProofs", default)] - pub storage_proofs: StorageTrieProofs, - #[serde(rename = "deletionProofs", default)] - pub deletion_proofs: Vec, -} - -pub type EthBlock = Block; - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct ExecutionResult { - #[serde(rename = "l1DataFee", default)] - pub l1_fee: U256, - pub gas: u64, - pub failed: bool, - #[serde(rename = "returnValue", default)] - pub return_value: String, - pub from: Option, - pub to: Option, - #[serde(rename = "accountAfter", default)] - pub account_after: Vec, - #[serde(rename = "accountCreated")] - pub account_created: Option, - #[serde(rename = "poseidonCodeHash")] - pub code_hash: Option, - #[serde(rename = "byteCode")] - pub byte_code: Option, - #[serde(rename = "structLogs")] - pub exec_steps: Vec, -} - -impl From<&ExecutionResult> for GethExecTrace { - fn from(e: &ExecutionResult) -> Self { - let mut struct_logs = Vec::new(); - for exec_step in &e.exec_steps { - let step = exec_step.into(); - struct_logs.push(step) - } - GethExecTrace { - l1_fee: e.l1_fee.as_u64(), - gas: Gas(e.gas), - failed: e.failed, - return_value: e.return_value.clone(), - struct_logs, - } - } -} - -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct ExecStep { - pub pc: u64, - pub op: OpcodeId, - pub gas: u64, - #[serde(rename = "gasCost")] - pub gas_cost: u64, - #[serde(default)] - pub refund: u64, - pub depth: isize, - pub error: Option, - pub stack: Option>, - pub memory: Option>, - pub storage: Option>, - #[serde(rename = "extraData")] - pub extra_data: Option, -} - -impl From<&ExecStep> for GethExecStep { - fn from(e: &ExecStep) -> Self { - let stack = e.stack.clone().map_or_else(Stack::new, Stack::from); - let storage = e.storage.clone().map_or_else(Storage::empty, Storage::from); - - GethExecStep { - pc: ProgramCounter(e.pc as usize), - // FIXME - op: e.op, - gas: Gas(e.gas), - gas_cost: GasCost(e.gas_cost), - refund: Gas(e.refund), - depth: e.depth as u16, - error: e.error.clone(), - stack, - memory: Default::default(), - storage, - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct ExtraData { - #[serde(rename = "codeList")] - pub code_list: Option>, - #[serde(rename = "proofList")] - pub proof_list: Option>, -} - -impl ExtraData { - pub fn get_code_at(&self, i: usize) -> Option { - self.code_list.as_ref().and_then(|c| c.get(i)).cloned() - } - - pub fn get_code_hash_at(&self, i: usize) -> Option { - self.get_proof_at(i).and_then(|a| a.poseidon_code_hash) - } - - pub fn get_proof_at(&self, i: usize) -> Option { - self.proof_list.as_ref().and_then(|p| p.get(i)).cloned() - } -} - -#[derive(Serialize, Deserialize, Clone, Default, Debug)] -pub struct AccountProofWrapper { - pub address: Option
, - pub nonce: Option, - pub balance: Option, - #[serde(rename = "keccakCodeHash")] - pub keccak_code_hash: Option, - #[serde(rename = "poseidonCodeHash")] - pub poseidon_code_hash: Option, - pub proof: Option>, - pub storage: Option, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct StorageProofWrapper { - pub key: Option, - pub value: Option, - pub proof: Option>, -} diff --git a/types/src/lib.rs b/types/src/lib.rs index 8c63fd288..8635d740a 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -1,4 +1,10 @@ -pub mod eth; +pub use eth_types::l2_types as eth; +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize, Default, Debug, Clone)] +pub struct BlockTraceJsonRpcResult { + pub result: eth::BlockTrace, +} pub mod base64 { use base64::{decode, encode};