diff --git a/Cargo.lock b/Cargo.lock index e886773392d0..31fc4ed7a5e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1313,14 +1313,14 @@ dependencies = [ [[package]] name = "circuit_encodings" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67617688c66640c84f9b98ff26d48f7898dca4faeb45241a4f21ec333788e7b" +checksum = "f5128d4b8fbb27ac453f573a95601058e74487bdafd22a3168cded66bf340c28" dependencies = [ "derivative", "serde", - "zk_evm 0.150.5", - "zkevm_circuits 0.150.5", + "zk_evm 0.150.6", + "zkevm_circuits 0.150.6", ] [[package]] @@ -1380,11 +1380,11 @@ dependencies = [ [[package]] name = "circuit_sequencer_api" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21017310971d4a051e4a52ad70eed11d1ae69defeca8314f73a3a4bad16705a9" +checksum = "093d0c2c0b39144ddb4e1e88d73d95067ce34ec7750808b2eed01edbb510b88e" dependencies = [ - "circuit_encodings 0.150.5", + "circuit_encodings 0.150.6", "derivative", "rayon", "serde", @@ -9342,9 +9342,9 @@ dependencies = [ [[package]] name = "zk_evm" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6e69931f24db5cf333b714721e8d80ff88bfdb7da8c3dc7882612ffddb8d27" +checksum = "c14bda6c101389145cd01fac900f1392876bc0284d98faf7f376237baa2cb19d" dependencies = [ "anyhow", "lazy_static", @@ -9352,7 +9352,7 @@ dependencies = [ "serde", "serde_json", "static_assertions", - "zk_evm_abstractions 0.150.5", + "zk_evm_abstractions 0.150.6", ] [[package]] @@ -9383,15 +9383,15 @@ dependencies = [ [[package]] name = "zk_evm_abstractions" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6b0720261ab55490fe3a96e96de30d5d7b277940b52ea7f52dbf564eb1748" +checksum = "a008f2442fc6a508bdd1f902380242cb6ff11b8b27acdac2677c6d9f75cbb004" dependencies = [ "anyhow", "num_enum 0.6.1", "serde", "static_assertions", - "zkevm_opcode_defs 0.150.5", + "zkevm_opcode_defs 0.150.6", ] [[package]] @@ -9440,9 +9440,9 @@ dependencies = [ [[package]] name = "zkevm_circuits" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784fa7cfb51e17c5ced112bca43da30b3468b2347b7af0427ad9638759fb140e" +checksum = "1f68518aedd5358b17224771bb78bacd912cf66011aeda98b1f887cfb9e0972f" dependencies = [ "arrayvec 0.7.6", "boojum", @@ -9454,7 +9454,7 @@ dependencies = [ "seq-macro", "serde", "smallvec", - "zkevm_opcode_defs 0.150.5", + "zkevm_opcode_defs 0.150.6", "zksync_cs_derive", ] @@ -9502,9 +9502,9 @@ dependencies = [ [[package]] name = "zkevm_opcode_defs" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79055eae1b6c1ab80793ed9d77d2964c9c896afa4b5dfed278cf58cd10acfe8f" +checksum = "762b5f1c1b283c5388995a85d40a05aef1c14f50eb904998b7e9364739f5b899" dependencies = [ "bitflags 2.6.0", "blake2 0.10.6", @@ -9628,7 +9628,7 @@ dependencies = [ "anyhow", "circuit_sequencer_api 0.140.3", "circuit_sequencer_api 0.141.2", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "futures 0.3.30", "itertools 0.10.5", "num_cpus", @@ -9640,7 +9640,7 @@ dependencies = [ "vise", "zk_evm 0.133.0", "zk_evm 0.141.0", - "zk_evm 0.150.5", + "zk_evm 0.150.6", "zksync_contracts", "zksync_dal", "zksync_eth_client", @@ -10355,9 +10355,9 @@ dependencies = [ [[package]] name = "zksync_kzg" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb8a9c76c172a6d639855ee342b9a670e3ba472f5ae302f771b1c3ee777dc88" +checksum = "6c006b6b7a27cc50ff0c515b6d0b197dbb907bbf65d1d2ea42fc3ed21b315642" dependencies = [ "boojum", "derivative", @@ -10367,7 +10367,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "zkevm_circuits 0.150.5", + "zkevm_circuits 0.150.6", ] [[package]] @@ -10494,7 +10494,7 @@ dependencies = [ "circuit_sequencer_api 0.140.3", "circuit_sequencer_api 0.141.2", "circuit_sequencer_api 0.142.2", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "ethabi", "hex", "itertools 0.10.5", @@ -10508,7 +10508,7 @@ dependencies = [ "zk_evm 0.133.0", "zk_evm 0.140.0", "zk_evm 0.141.0", - "zk_evm 0.150.5", + "zk_evm 0.150.6", "zksync_contracts", "zksync_eth_signer", "zksync_system_constants", @@ -10549,7 +10549,7 @@ dependencies = [ "tower-http", "tracing", "vise", - "zk_evm 0.150.5", + "zk_evm 0.150.6", "zksync_config", "zksync_consensus_roles", "zksync_contracts", @@ -10777,6 +10777,7 @@ dependencies = [ "anyhow", "assert_matches", "async-trait", + "backon", "chrono", "futures 0.3.30", "once_cell", @@ -10945,7 +10946,7 @@ version = "0.1.0" dependencies = [ "bincode", "chrono", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "serde", "serde_json", "serde_with", @@ -11305,19 +11306,19 @@ dependencies = [ [[package]] name = "zksync_vm2" version = "0.2.1" -source = "git+https://github.com/matter-labs/vm2.git?rev=a233d44bbe61dc6a758a754c3b78fe4f83e56699#a233d44bbe61dc6a758a754c3b78fe4f83e56699" +source = "git+https://github.com/matter-labs/vm2.git?rev=df5bec3d04d64d434f9b0ccb285ba4681008f7b3#df5bec3d04d64d434f9b0ccb285ba4681008f7b3" dependencies = [ "enum_dispatch", "primitive-types", - "zk_evm_abstractions 0.150.5", - "zkevm_opcode_defs 0.150.5", + "zk_evm_abstractions 0.150.6", + "zkevm_opcode_defs 0.150.6", "zksync_vm2_interface", ] [[package]] name = "zksync_vm2_interface" version = "0.2.1" -source = "git+https://github.com/matter-labs/vm2.git?rev=a233d44bbe61dc6a758a754c3b78fe4f83e56699#a233d44bbe61dc6a758a754c3b78fe4f83e56699" +source = "git+https://github.com/matter-labs/vm2.git?rev=df5bec3d04d64d434f9b0ccb285ba4681008f7b3#df5bec3d04d64d434f9b0ccb285ba4681008f7b3" dependencies = [ "primitive-types", ] diff --git a/Cargo.toml b/Cargo.toml index 940d5dd036b7..f1e70e7f3028 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -218,18 +218,18 @@ circuit_sequencer_api_1_3_3 = { package = "circuit_sequencer_api", version = "0. circuit_sequencer_api_1_4_0 = { package = "circuit_sequencer_api", version = "0.140" } circuit_sequencer_api_1_4_1 = { package = "circuit_sequencer_api", version = "0.141" } circuit_sequencer_api_1_4_2 = { package = "circuit_sequencer_api", version = "0.142" } -circuit_sequencer_api_1_5_0 = { package = "circuit_sequencer_api", version = "=0.150.5" } +circuit_sequencer_api_1_5_0 = { package = "circuit_sequencer_api", version = "=0.150.6" } crypto_codegen = { package = "zksync_solidity_vk_codegen", version = "=0.30.1" } -kzg = { package = "zksync_kzg", version = "=0.150.5" } +kzg = { package = "zksync_kzg", version = "=0.150.6" } zk_evm = { version = "=0.133.0" } zk_evm_1_3_1 = { package = "zk_evm", version = "0.131.0-rc.2" } zk_evm_1_3_3 = { package = "zk_evm", version = "0.133" } zk_evm_1_4_0 = { package = "zk_evm", version = "0.140" } zk_evm_1_4_1 = { package = "zk_evm", version = "0.141" } -zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.5" } +zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.6" } # New VM; pinned to a specific commit because of instability -zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "a233d44bbe61dc6a758a754c3b78fe4f83e56699" } +zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "df5bec3d04d64d434f9b0ccb285ba4681008f7b3" } # Consensus dependencies. zksync_concurrency = "=0.5.0" diff --git a/core/lib/dal/.sqlx/query-180cc8d88563a42423ca1d4b92181f4625ebd593aa4cd2bae79bcc0637387d78.json b/core/lib/dal/.sqlx/query-180cc8d88563a42423ca1d4b92181f4625ebd593aa4cd2bae79bcc0637387d78.json new file mode 100644 index 000000000000..b40bdca666b8 --- /dev/null +++ b/core/lib/dal/.sqlx/query-180cc8d88563a42423ca1d4b92181f4625ebd593aa4cd2bae79bcc0637387d78.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM l1_batches\n WHERE\n number > $1\n AND NOT is_sealed\n RETURNING number\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "number", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "180cc8d88563a42423ca1d4b92181f4625ebd593aa4cd2bae79bcc0637387d78" +} diff --git a/core/lib/dal/src/blocks_dal.rs b/core/lib/dal/src/blocks_dal.rs index bf1b48130c40..f71dc68ce757 100644 --- a/core/lib/dal/src/blocks_dal.rs +++ b/core/lib/dal/src/blocks_dal.rs @@ -2058,6 +2058,37 @@ impl BlocksDal<'_, '_> { Ok(()) } + /// Deletes the unsealed L1 batch from the storage. Expects the caller to make sure there are no + /// associated L2 blocks. + /// + /// Accepts `batch_to_keep` as a safety mechanism. + pub async fn delete_unsealed_l1_batch( + &mut self, + batch_to_keep: L1BatchNumber, + ) -> DalResult<()> { + let deleted_row = sqlx::query!( + r#" + DELETE FROM l1_batches + WHERE + number > $1 + AND NOT is_sealed + RETURNING number + "#, + i64::from(batch_to_keep.0) + ) + .instrument("delete_unsealed_l1_batch") + .with_arg("batch_to_keep", &batch_to_keep) + .fetch_optional(self.storage) + .await?; + if let Some(deleted_row) = deleted_row { + tracing::info!( + l1_batch_number = %deleted_row.number, + "Deleted unsealed batch" + ); + } + Ok(()) + } + /// Deletes all L1 batches from the storage so that the specified batch number is the last one left. pub async fn delete_l1_batches(&mut self, last_batch_to_keep: L1BatchNumber) -> DalResult<()> { self.delete_l1_batches_inner(Some(last_batch_to_keep)).await @@ -2184,6 +2215,20 @@ impl BlocksDal<'_, '_> { Ok(Some((L2BlockNumber(min as u32), L2BlockNumber(max as u32)))) } + /// Returns `true` if there exists a non-sealed batch (i.e. there is one+ stored L2 block that isn't assigned + /// to any batch yet). + pub async fn pending_batch_exists(&mut self) -> DalResult { + let count = sqlx::query_scalar!( + "SELECT COUNT(miniblocks.number) FROM miniblocks WHERE l1_batch_number IS NULL" + ) + .instrument("pending_batch_exists") + .fetch_one(self.storage) + .await? + .unwrap_or(0); + + Ok(count != 0) + } + // methods used for measuring Eth tx stage transition latencies // and emitting metrics base on these measured data pub async fn oldest_uncommitted_batch_timestamp(&mut self) -> DalResult> { diff --git a/core/lib/multivm/src/versions/testonly/bytecode_publishing.rs b/core/lib/multivm/src/versions/testonly/bytecode_publishing.rs index 33af7be8cc6f..346241a96245 100644 --- a/core/lib/multivm/src/versions/testonly/bytecode_publishing.rs +++ b/core/lib/multivm/src/versions/testonly/bytecode_publishing.rs @@ -1,4 +1,4 @@ -use zksync_test_account::{DeployContractsTx, TxType}; +use zksync_test_account::TxType; use super::{read_test_contract, tester::VmTesterBuilder, TestedVm}; use crate::{ @@ -20,8 +20,16 @@ pub(crate) fn test_bytecode_publishing() { let compressed_bytecode = bytecode::compress(counter.clone()).unwrap().compressed; - let DeployContractsTx { tx, .. } = account.get_deploy_tx(&counter, None, TxType::L2); - vm.vm.push_transaction(tx); + let tx = account.get_deploy_tx(&counter, None, TxType::L2).tx; + assert_eq!(tx.execute.factory_deps.len(), 1); // The deployed bytecode is the only dependency + let push_result = vm.vm.push_transaction(tx); + assert_eq!(push_result.compressed_bytecodes.len(), 1); + assert_eq!(push_result.compressed_bytecodes[0].original, counter); + assert_eq!( + push_result.compressed_bytecodes[0].compressed, + compressed_bytecode + ); + let result = vm.vm.execute(VmExecutionMode::OneTx); assert!(!result.result.is_failed(), "Transaction wasn't successful"); diff --git a/core/lib/multivm/src/versions/vm_1_3_2/vm.rs b/core/lib/multivm/src/versions/vm_1_3_2/vm.rs index 89196788a762..31457fc9676a 100644 --- a/core/lib/multivm/src/versions/vm_1_3_2/vm.rs +++ b/core/lib/multivm/src/versions/vm_1_3_2/vm.rs @@ -8,8 +8,9 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, - L2BlockEnv, SystemEnv, TxExecutionMode, VmExecutionMode, VmExecutionResultAndLogs, - VmFactory, VmInterface, VmInterfaceHistoryEnabled, VmMemoryMetrics, + L2BlockEnv, PushTransactionResult, SystemEnv, TxExecutionMode, VmExecutionMode, + VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, + VmMemoryMetrics, }, tracers::old::TracerDispatcher, utils::bytecode, @@ -44,13 +45,17 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - fn push_transaction(&mut self, tx: Transaction) { - crate::vm_1_3_2::vm_with_bootloader::push_transaction_to_bootloader_memory( - &mut self.vm, - &tx, - self.system_env.execution_mode.glue_into(), - None, - ) + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { + let compressed_bytecodes = + crate::vm_1_3_2::vm_with_bootloader::push_transaction_to_bootloader_memory( + &mut self.vm, + &tx, + self.system_env.execution_mode.glue_into(), + None, + ); + PushTransactionResult { + compressed_bytecodes: compressed_bytecodes.into(), + } } fn inspect( diff --git a/core/lib/multivm/src/versions/vm_1_3_2/vm_with_bootloader.rs b/core/lib/multivm/src/versions/vm_1_3_2/vm_with_bootloader.rs index d1acdf7708e8..fd4d483fba5e 100644 --- a/core/lib/multivm/src/versions/vm_1_3_2/vm_with_bootloader.rs +++ b/core/lib/multivm/src/versions/vm_1_3_2/vm_with_bootloader.rs @@ -442,7 +442,7 @@ pub fn get_bootloader_memory( let mut previous_compressed: usize = 0; let mut already_included_txs_size = 0; for (tx_index_in_block, tx) in txs.into_iter().enumerate() { - let compressed_bytecodes = predefined_compressed_bytecodes[tx_index_in_block].clone(); + let compressed_bytecodes = &predefined_compressed_bytecodes[tx_index_in_block]; let mut total_compressed_len_words = 0; for i in compressed_bytecodes.iter() { @@ -475,7 +475,7 @@ pub fn push_transaction_to_bootloader_memory( tx: &Transaction, execution_mode: TxExecutionMode, explicit_compressed_bytecodes: Option>, -) { +) -> Vec { let tx: TransactionData = tx.clone().into(); let block_gas_per_pubdata_byte = vm.block_context.context.block_gas_price_per_pubdata(); let overhead = tx.overhead_gas(block_gas_per_pubdata_byte as u32); @@ -485,7 +485,7 @@ pub fn push_transaction_to_bootloader_memory( execution_mode, overhead, explicit_compressed_bytecodes, - ); + ) } pub fn push_raw_transaction_to_bootloader_memory( @@ -494,7 +494,7 @@ pub fn push_raw_transaction_to_bootloader_memory>, -) { +) -> Vec { let tx_index_in_block = vm.bootloader_state.free_tx_index(); let already_included_txs_size = vm.bootloader_state.free_tx_offset(); @@ -555,7 +555,7 @@ pub fn push_raw_transaction_to_bootloader_memory, + compressed_bytecodes: &[CompressedBytecodeInfo], ) -> Vec<(usize, U256)> { let overhead_gas = tx.overhead_gas(block_gas_per_pubdata); let trusted_gas_limit = tx.trusted_gas_limit(block_gas_per_pubdata); @@ -604,7 +605,7 @@ pub(crate) fn get_bootloader_memory_for_encoded_tx( predefined_overhead: u32, trusted_gas_limit: u32, previous_compressed_bytecode_size: usize, - compressed_bytecodes: Vec, + compressed_bytecodes: &[CompressedBytecodeInfo], ) -> Vec<(usize, U256)> { let mut memory: Vec<(usize, U256)> = Vec::default(); let bootloader_description_offset = @@ -640,8 +641,8 @@ pub(crate) fn get_bootloader_memory_for_encoded_tx( COMPRESSED_BYTECODES_OFFSET + 1 + previous_compressed_bytecode_size; let memory_addition: Vec<_> = compressed_bytecodes - .into_iter() - .flat_map(|x| bytecode::encode_call(&x)) + .iter() + .flat_map(bytecode::encode_call) .collect(); let memory_addition = bytes_to_be_words(memory_addition); diff --git a/core/lib/multivm/src/versions/vm_1_4_1/vm.rs b/core/lib/multivm/src/versions/vm_1_4_1/vm.rs index 4122ee94e66a..1c38958bb318 100644 --- a/core/lib/multivm/src/versions/vm_1_4_1/vm.rs +++ b/core/lib/multivm/src/versions/vm_1_4_1/vm.rs @@ -9,7 +9,7 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, + FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, }, utils::events::extract_l2tol1logs_from_l1_messenger, @@ -81,9 +81,14 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { self.push_transaction_with_compression(tx, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } /// Execute VM with custom tracers. diff --git a/core/lib/multivm/src/versions/vm_1_4_2/vm.rs b/core/lib/multivm/src/versions/vm_1_4_2/vm.rs index fe2015debd2b..ca69a191e26f 100644 --- a/core/lib/multivm/src/versions/vm_1_4_2/vm.rs +++ b/core/lib/multivm/src/versions/vm_1_4_2/vm.rs @@ -11,7 +11,7 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, + FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, }, utils::events::extract_l2tol1logs_from_l1_messenger, @@ -83,9 +83,14 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { self.push_transaction_with_compression(tx, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } /// Execute VM with custom tracers. diff --git a/core/lib/multivm/src/versions/vm_boojum_integration/vm.rs b/core/lib/multivm/src/versions/vm_boojum_integration/vm.rs index ebc0a511d203..bfd055a5cc84 100644 --- a/core/lib/multivm/src/versions/vm_boojum_integration/vm.rs +++ b/core/lib/multivm/src/versions/vm_boojum_integration/vm.rs @@ -9,7 +9,7 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, + FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, }, utils::events::extract_l2tol1logs_from_l1_messenger, @@ -81,9 +81,14 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult { self.push_transaction_with_compression(tx, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } /// Execute VM with custom tracers. diff --git a/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs b/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs index b48ec7eacb0b..f588f20ab25d 100644 --- a/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs +++ b/core/lib/multivm/src/versions/vm_fast/circuits_tracer.rs @@ -1,5 +1,5 @@ use circuit_sequencer_api_1_5_0::{geometry_config::get_geometry_config, toolset::GeometryConfig}; -use zksync_vm2::interface::{CycleStats, Opcode, OpcodeType, StateInterface, Tracer}; +use zksync_vm2::interface::{CycleStats, GlobalStateInterface, Opcode, OpcodeType, Tracer}; use zksync_vm_interface::CircuitStatistic; use crate::vm_latest::tracers::circuits_capacity::*; @@ -24,7 +24,7 @@ pub struct CircuitsTracer { } impl Tracer for CircuitsTracer { - fn after_instruction(&mut self, _state: &mut S) { + fn after_instruction(&mut self, _: &mut S) { self.main_vm_cycles += 1; match OP::VALUE { diff --git a/core/lib/multivm/src/versions/vm_fast/vm.rs b/core/lib/multivm/src/versions/vm_fast/vm.rs index ac3346c8eeaa..d89bc5987eff 100644 --- a/core/lib/multivm/src/versions/vm_fast/vm.rs +++ b/core/lib/multivm/src/versions/vm_fast/vm.rs @@ -13,7 +13,7 @@ use zksync_types::{ BYTES_PER_ENUMERATION_INDEX, }, AccountTreeId, StorageKey, StorageLog, StorageLogKind, StorageLogWithPreviousValue, - BOOTLOADER_ADDRESS, H160, H256, KNOWN_CODES_STORAGE_ADDRESS, L1_MESSENGER_ADDRESS, + Transaction, BOOTLOADER_ADDRESS, H160, H256, KNOWN_CODES_STORAGE_ADDRESS, L1_MESSENGER_ADDRESS, L2_BASE_TOKEN_ADDRESS, U256, }; use zksync_utils::{bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; @@ -35,10 +35,10 @@ use crate::{ interface::{ storage::{ImmutableStorageView, ReadStorage, StoragePtr, StorageView}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - ExecutionResult, FinishedL1Batch, Halt, L1BatchEnv, L2BlockEnv, Refunds, SystemEnv, - TxRevertReason, VmEvent, VmExecutionLogs, VmExecutionMode, VmExecutionResultAndLogs, - VmExecutionStatistics, VmFactory, VmInterface, VmInterfaceHistoryEnabled, VmRevertReason, - VmTrackingContracts, + ExecutionResult, FinishedL1Batch, Halt, L1BatchEnv, L2BlockEnv, PushTransactionResult, + Refunds, SystemEnv, TxRevertReason, VmEvent, VmExecutionLogs, VmExecutionMode, + VmExecutionResultAndLogs, VmExecutionStatistics, VmFactory, VmInterface, + VmInterfaceHistoryEnabled, VmRevertReason, VmTrackingContracts, }, is_supported_by_fast_vm, utils::events::extract_l2tol1logs_from_l1_messenger, @@ -574,8 +574,14 @@ where impl VmInterface for Vm { type TracerDispatcher = Tr; - fn push_transaction(&mut self, tx: zksync_types::Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { self.push_transaction_inner(tx, 0, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } fn inspect( diff --git a/core/lib/multivm/src/versions/vm_latest/vm.rs b/core/lib/multivm/src/versions/vm_latest/vm.rs index f4cc1580e935..3a36b008e884 100644 --- a/core/lib/multivm/src/versions/vm_latest/vm.rs +++ b/core/lib/multivm/src/versions/vm_latest/vm.rs @@ -13,7 +13,7 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, + FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, VmTrackingContracts, }, @@ -134,9 +134,14 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { self.push_transaction_with_compression(tx, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } /// Execute VM with custom tracers. diff --git a/core/lib/multivm/src/versions/vm_m5/vm.rs b/core/lib/multivm/src/versions/vm_m5/vm.rs index 5a26506f3463..3d57d1cd5439 100644 --- a/core/lib/multivm/src/versions/vm_m5/vm.rs +++ b/core/lib/multivm/src/versions/vm_m5/vm.rs @@ -5,8 +5,9 @@ use crate::{ glue::{history_mode::HistoryMode, GlueInto}, interface::{ storage::StoragePtr, BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, L2BlockEnv, - SystemEnv, TxExecutionMode, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, - VmInterface, VmInterfaceHistoryEnabled, VmMemoryMetrics, + PushTransactionResult, SystemEnv, TxExecutionMode, VmExecutionMode, + VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, + VmMemoryMetrics, }, vm_m5::{ storage::Storage, @@ -60,12 +61,15 @@ impl VmInterface for Vm { /// Tracers are not supported for here we use `()` as a placeholder type TracerDispatcher = (); - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { crate::vm_m5::vm_with_bootloader::push_transaction_to_bootloader_memory( &mut self.vm, &tx, self.system_env.execution_mode.glue_into(), - ) + ); + PushTransactionResult { + compressed_bytecodes: (&[]).into(), // bytecode compression isn't supported + } } fn inspect( diff --git a/core/lib/multivm/src/versions/vm_m6/vm.rs b/core/lib/multivm/src/versions/vm_m6/vm.rs index 1fdc8ae64f80..1ee6aa618220 100644 --- a/core/lib/multivm/src/versions/vm_m6/vm.rs +++ b/core/lib/multivm/src/versions/vm_m6/vm.rs @@ -7,7 +7,7 @@ use crate::{ glue::{history_mode::HistoryMode, GlueInto}, interface::{ storage::StoragePtr, BytecodeCompressionError, BytecodeCompressionResult, FinishedL1Batch, - L1BatchEnv, L2BlockEnv, SystemEnv, TxExecutionMode, VmExecutionMode, + L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, TxExecutionMode, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, VmMemoryMetrics, }, @@ -72,13 +72,17 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - fn push_transaction(&mut self, tx: Transaction) { - crate::vm_m6::vm_with_bootloader::push_transaction_to_bootloader_memory( - &mut self.vm, - &tx, - self.system_env.execution_mode.glue_into(), - None, - ) + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult { + let compressed_bytecodes = + crate::vm_m6::vm_with_bootloader::push_transaction_to_bootloader_memory( + &mut self.vm, + &tx, + self.system_env.execution_mode.glue_into(), + None, + ); + PushTransactionResult { + compressed_bytecodes: compressed_bytecodes.into(), + } } fn inspect( diff --git a/core/lib/multivm/src/versions/vm_m6/vm_with_bootloader.rs b/core/lib/multivm/src/versions/vm_m6/vm_with_bootloader.rs index 7a9fbb73fe49..ae44e721b0d7 100644 --- a/core/lib/multivm/src/versions/vm_m6/vm_with_bootloader.rs +++ b/core/lib/multivm/src/versions/vm_m6/vm_with_bootloader.rs @@ -491,7 +491,7 @@ fn get_bootloader_memory_v1( predefined_refunds[tx_index_in_block], block_gas_price_per_pubdata as u32, previous_compressed, - compressed_bytecodes, + &compressed_bytecodes, ); previous_compressed += total_compressed_len; @@ -536,7 +536,7 @@ fn get_bootloader_memory_v2( predefined_refunds[tx_index_in_block], block_gas_price_per_pubdata as u32, previous_compressed, - compressed_bytecodes, + &compressed_bytecodes, ); previous_compressed += total_compressed_len_words; @@ -554,7 +554,7 @@ pub fn push_transaction_to_bootloader_memory( tx: &Transaction, execution_mode: TxExecutionMode, explicit_compressed_bytecodes: Option>, -) { +) -> Vec { let tx: TransactionData = tx.clone().into(); let block_gas_per_pubdata_byte = vm.block_context.context.block_gas_price_per_pubdata(); let overhead = tx.overhead_gas(block_gas_per_pubdata_byte as u32); @@ -564,7 +564,7 @@ pub fn push_transaction_to_bootloader_memory( execution_mode, overhead, explicit_compressed_bytecodes, - ); + ) } pub fn push_raw_transaction_to_bootloader_memory( @@ -573,7 +573,7 @@ pub fn push_raw_transaction_to_bootloader_memory( execution_mode: TxExecutionMode, predefined_overhead: u32, explicit_compressed_bytecodes: Option>, -) { +) -> Vec { match vm.vm_subversion { MultiVMSubversion::V1 => push_raw_transaction_to_bootloader_memory_v1( vm, @@ -599,7 +599,7 @@ fn push_raw_transaction_to_bootloader_memory_v1( execution_mode: TxExecutionMode, predefined_overhead: u32, explicit_compressed_bytecodes: Option>, -) { +) -> Vec { let tx_index_in_block = vm.bootloader_state.free_tx_index(); let already_included_txs_size = vm.bootloader_state.free_tx_offset(); @@ -651,7 +651,7 @@ fn push_raw_transaction_to_bootloader_memory_v1( predefined_overhead, trusted_ergs_limit, previous_bytecodes, - compressed_bytecodes, + &compressed_bytecodes, ); vm.state.memory.populate_page( @@ -661,6 +661,7 @@ fn push_raw_transaction_to_bootloader_memory_v1( ); vm.bootloader_state.add_tx_data(encoded_tx_size); vm.bootloader_state.add_compressed_bytecode(compressed_len); + compressed_bytecodes } // Bytecode compression bug fixed @@ -670,7 +671,7 @@ fn push_raw_transaction_to_bootloader_memory_v2( execution_mode: TxExecutionMode, predefined_overhead: u32, explicit_compressed_bytecodes: Option>, -) { +) -> Vec { let tx_index_in_block = vm.bootloader_state.free_tx_index(); let already_included_txs_size = vm.bootloader_state.free_tx_offset(); @@ -730,7 +731,7 @@ fn push_raw_transaction_to_bootloader_memory_v2( predefined_overhead, trusted_ergs_limit, previous_bytecodes, - compressed_bytecodes, + &compressed_bytecodes, ); vm.state.memory.populate_page( @@ -741,6 +742,7 @@ fn push_raw_transaction_to_bootloader_memory_v2( vm.bootloader_state.add_tx_data(encoded_tx_size); vm.bootloader_state .add_compressed_bytecode(compressed_bytecodes_encoding_len_words); + compressed_bytecodes } #[allow(clippy::too_many_arguments)] @@ -752,7 +754,7 @@ fn get_bootloader_memory_for_tx( predefined_refund: u32, block_gas_per_pubdata: u32, previous_compressed_bytecode_size: usize, - compressed_bytecodes: Vec, + compressed_bytecodes: &[CompressedBytecodeInfo], ) -> Vec<(usize, U256)> { let overhead_gas = tx.overhead_gas(block_gas_per_pubdata); let trusted_gas_limit = tx.trusted_gas_limit(block_gas_per_pubdata); @@ -779,7 +781,7 @@ pub(crate) fn get_bootloader_memory_for_encoded_tx( predefined_overhead: u32, trusted_gas_limit: u32, previous_compressed_bytecode_size: usize, - compressed_bytecodes: Vec, + compressed_bytecodes: &[CompressedBytecodeInfo], ) -> Vec<(usize, U256)> { let mut memory: Vec<(usize, U256)> = Vec::default(); let bootloader_description_offset = @@ -815,8 +817,8 @@ pub(crate) fn get_bootloader_memory_for_encoded_tx( COMPRESSED_BYTECODES_OFFSET + 1 + previous_compressed_bytecode_size; let memory_addition: Vec<_> = compressed_bytecodes - .into_iter() - .flat_map(|x| bytecode::encode_call(&x)) + .iter() + .flat_map(bytecode::encode_call) .collect(); let memory_addition = bytes_to_be_words(memory_addition); diff --git a/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs b/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs index d87fd4d104da..2bcd68bec044 100644 --- a/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs +++ b/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs @@ -6,7 +6,7 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, + FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, }, vm_latest::HistoryEnabled, @@ -74,9 +74,14 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { - self.push_transaction_with_compression(tx, true) + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { + self.push_transaction_with_compression(tx, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } /// Execute VM with custom tracers. diff --git a/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs b/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs index 28c09590f2ad..497128c64bd9 100644 --- a/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs +++ b/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs @@ -6,7 +6,7 @@ use crate::{ interface::{ storage::{StoragePtr, WriteStorage}, BytecodeCompressionError, BytecodeCompressionResult, CurrentExecutionState, - FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, + FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, }, vm_latest::HistoryEnabled, @@ -74,9 +74,14 @@ impl Vm { impl VmInterface for Vm { type TracerDispatcher = TracerDispatcher; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { - self.push_transaction_with_compression(tx, true) + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { + self.push_transaction_with_compression(tx, true); + PushTransactionResult { + compressed_bytecodes: self + .bootloader_state + .get_last_tx_compressed_bytecodes() + .into(), + } } /// Execute VM with custom tracers. diff --git a/core/lib/multivm/src/vm_instance.rs b/core/lib/multivm/src/vm_instance.rs index 7e17e0db1d76..46756312609e 100644 --- a/core/lib/multivm/src/vm_instance.rs +++ b/core/lib/multivm/src/vm_instance.rs @@ -8,8 +8,8 @@ use crate::{ interface::{ storage::{ImmutableStorageView, ReadStorage, StoragePtr, StorageView}, utils::ShadowVm, - BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, - VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, + BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, + SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceHistoryEnabled, VmMemoryMetrics, }, tracers::TracerDispatcher, @@ -55,8 +55,7 @@ macro_rules! dispatch_legacy_vm { impl VmInterface for LegacyVmInstance { type TracerDispatcher = TracerDispatcher, H>; - /// Push tx into memory for the future execution - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { dispatch_legacy_vm!(self.push_transaction(tx)) } @@ -247,8 +246,8 @@ impl VmInterface for FastVmInsta Tr, ); - fn push_transaction(&mut self, tx: Transaction) { - dispatch_fast_vm!(self.push_transaction(tx)); + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { + dispatch_fast_vm!(self.push_transaction(tx)) } fn inspect( diff --git a/core/lib/vm_interface/src/lib.rs b/core/lib/vm_interface/src/lib.rs index 645e3e7c856e..e0287483067a 100644 --- a/core/lib/vm_interface/src/lib.rs +++ b/core/lib/vm_interface/src/lib.rs @@ -30,10 +30,10 @@ pub use crate::{ outputs::{ BatchTransactionExecutionResult, BootloaderMemory, Call, CallType, CircuitStatistic, CompressedBytecodeInfo, CurrentExecutionState, DeduplicatedWritesMetrics, - ExecutionResult, FinishedL1Batch, L2Block, OneshotTransactionExecutionResult, Refunds, - TransactionExecutionMetrics, TransactionExecutionResult, TxExecutionStatus, VmEvent, - VmExecutionLogs, VmExecutionMetrics, VmExecutionResultAndLogs, VmExecutionStatistics, - VmMemoryMetrics, + ExecutionResult, FinishedL1Batch, L2Block, OneshotTransactionExecutionResult, + PushTransactionResult, Refunds, TransactionExecutionMetrics, + TransactionExecutionResult, TxExecutionStatus, VmEvent, VmExecutionLogs, + VmExecutionMetrics, VmExecutionResultAndLogs, VmExecutionStatistics, VmMemoryMetrics, }, tracer, }, diff --git a/core/lib/vm_interface/src/types/outputs/mod.rs b/core/lib/vm_interface/src/types/outputs/mod.rs index 1fa1cd5d1688..fe25801dd12e 100644 --- a/core/lib/vm_interface/src/types/outputs/mod.rs +++ b/core/lib/vm_interface/src/types/outputs/mod.rs @@ -1,3 +1,5 @@ +use std::borrow::Cow; + pub use self::{ bytecode::CompressedBytecodeInfo, execution_result::{ @@ -20,3 +22,14 @@ mod execution_state; mod finished_l1batch; mod l2_block; mod statistic; + +/// Result of pushing a transaction to the VM state without executing it. +#[derive(Debug)] +pub struct PushTransactionResult<'a> { + /// Compressed bytecodes for the transaction. If the VM doesn't support bytecode compression, returns + /// an empty slice. + /// + /// Importantly, these bytecodes are not guaranteed to be published by the transaction; + /// e.g., it may run out of gas during publication. + pub compressed_bytecodes: Cow<'a, [CompressedBytecodeInfo]>, +} diff --git a/core/lib/vm_interface/src/utils/dump.rs b/core/lib/vm_interface/src/utils/dump.rs index 288c6445494d..522a455a11ba 100644 --- a/core/lib/vm_interface/src/utils/dump.rs +++ b/core/lib/vm_interface/src/utils/dump.rs @@ -5,9 +5,9 @@ use zksync_types::{block::L2BlockExecutionData, L1BatchNumber, L2BlockNumber, Tr use crate::{ storage::{ReadStorage, StoragePtr, StorageSnapshot, StorageView}, - BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, L2BlockEnv, SystemEnv, VmExecutionMode, - VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceExt, VmInterfaceHistoryEnabled, - VmTrackingContracts, + BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, L2BlockEnv, PushTransactionResult, + SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, VmInterfaceExt, + VmInterfaceHistoryEnabled, VmTrackingContracts, }; fn create_storage_snapshot( @@ -142,9 +142,9 @@ impl DumpingVm { impl VmInterface for DumpingVm { type TracerDispatcher = Vm::TracerDispatcher; - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult { self.record_transaction(tx.clone()); - self.inner.push_transaction(tx); + self.inner.push_transaction(tx) } fn inspect( diff --git a/core/lib/vm_interface/src/utils/shadow.rs b/core/lib/vm_interface/src/utils/shadow.rs index 92eb65a810f7..8cdc899238e4 100644 --- a/core/lib/vm_interface/src/utils/shadow.rs +++ b/core/lib/vm_interface/src/utils/shadow.rs @@ -11,8 +11,8 @@ use super::dump::{DumpingVm, VmDump}; use crate::{ storage::{ReadStorage, StoragePtr, StorageView}, BytecodeCompressionResult, CurrentExecutionState, FinishedL1Batch, L1BatchEnv, L2BlockEnv, - SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, VmInterface, - VmInterfaceHistoryEnabled, VmTrackingContracts, + PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, VmFactory, + VmInterface, VmInterfaceHistoryEnabled, VmTrackingContracts, }; /// Handler for VM divergences. @@ -163,11 +163,30 @@ where ::TracerDispatcher, ); - fn push_transaction(&mut self, tx: Transaction) { + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_> { + let main_result = self.main.push_transaction(tx.clone()); + // Extend lifetime to `'static` so that the result isn't mutably borrowed from the main VM. + // Unfortunately, there's no way to express that this borrow is actually immutable, which would allow not extending the lifetime unless there's a divergence. + let main_result: PushTransactionResult<'static> = PushTransactionResult { + compressed_bytecodes: main_result.compressed_bytecodes.into_owned().into(), + }; + if let Some(shadow) = self.shadow.get_mut() { - shadow.vm.push_transaction(tx.clone()); + let tx_repr = format!("{tx:?}"); // includes little data, so is OK to call proactively + let shadow_result = shadow.vm.push_transaction(tx); + + let mut errors = DivergenceErrors::new(); + errors.check_match( + "bytecodes", + &main_result.compressed_bytecodes, + &shadow_result.compressed_bytecodes, + ); + if let Err(err) = errors.into_result() { + let ctx = format!("pushing transaction {tx_repr}"); + self.report(err.context(ctx)); + } } - self.main.push_transaction(tx); + main_result } fn inspect( diff --git a/core/lib/vm_interface/src/vm.rs b/core/lib/vm_interface/src/vm.rs index 37e33a92b509..3a06d7f80cbe 100644 --- a/core/lib/vm_interface/src/vm.rs +++ b/core/lib/vm_interface/src/vm.rs @@ -15,15 +15,20 @@ use zksync_types::{Transaction, H256}; use crate::{ storage::StoragePtr, BytecodeCompressionResult, FinishedL1Batch, L1BatchEnv, L2BlockEnv, - SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, + PushTransactionResult, SystemEnv, VmExecutionMode, VmExecutionResultAndLogs, }; pub trait VmInterface { /// Lifetime is used to be able to define `Option<&mut _>` as a dispatcher. type TracerDispatcher: Default; - /// Push transaction to bootloader memory. - fn push_transaction(&mut self, tx: Transaction); + /// Pushes a transaction to bootloader memory for future execution with bytecode compression (if it's supported by the VM). + /// + /// # Return value + /// + /// Returns preprocessing results, such as compressed bytecodes. The results may borrow from the VM state, + /// so you may want to inspect results before next operations with the VM, or clone the necessary parts. + fn push_transaction(&mut self, tx: Transaction) -> PushTransactionResult<'_>; /// Executes the next VM step (either next transaction or bootloader or the whole batch) /// with custom tracers. diff --git a/core/node/consensus/src/testonly.rs b/core/node/consensus/src/testonly.rs index 98c0d6b08131..4ebcf5c9a617 100644 --- a/core/node/consensus/src/testonly.rs +++ b/core/node/consensus/src/testonly.rs @@ -219,11 +219,10 @@ impl StateKeeper { .wait(IoCursor::for_fetcher(&mut conn.0)) .await? .context("IoCursor::new()")?; - let batch_sealed = ctx - .wait(conn.0.blocks_dal().get_unsealed_l1_batch()) + let pending_batch = ctx + .wait(conn.0.blocks_dal().pending_batch_exists()) .await? - .context("get_unsealed_l1_batch()")? - .is_none(); + .context("pending_batch_exists()")?; let (actions_sender, actions_queue) = ActionQueue::new(); let addr = sync::watch::channel(None).0; let sync_state = SyncState::default(); @@ -259,7 +258,7 @@ impl StateKeeper { last_batch: cursor.l1_batch, last_block: cursor.next_l2_block - 1, last_timestamp: cursor.prev_l2_block_timestamp, - batch_sealed, + batch_sealed: !pending_batch, next_priority_op: PriorityOpId(1), actions_sender, sync_state: sync_state.clone(), diff --git a/core/node/node_sync/Cargo.toml b/core/node/node_sync/Cargo.toml index ccfc8dd8a4e9..9c5b0c000700 100644 --- a/core/node/node_sync/Cargo.toml +++ b/core/node/node_sync/Cargo.toml @@ -43,3 +43,4 @@ zksync_node_test_utils.workspace = true assert_matches.workspace = true once_cell.workspace = true test-casing.workspace = true +backon.workspace = true diff --git a/core/node/node_sync/src/external_io.rs b/core/node/node_sync/src/external_io.rs index 10fb2925015f..5e3a5ce9f46e 100644 --- a/core/node/node_sync/src/external_io.rs +++ b/core/node/node_sync/src/external_io.rs @@ -155,6 +155,14 @@ impl StateKeeperIO for ExternalIO { ) })?; let Some(mut pending_l2_block_header) = pending_l2_block_header else { + tracing::info!( + l1_batch_number = %cursor.l1_batch, + "No pending L2 blocks found; pruning unsealed batch if exists as we need at least one L2 block to initialize" + ); + storage + .blocks_dal() + .delete_unsealed_l1_batch(cursor.l1_batch - 1) + .await?; return Ok((cursor, None)); }; diff --git a/core/node/node_sync/src/fetcher.rs b/core/node/node_sync/src/fetcher.rs index 3f8558ed0ac5..51b9f7c7a060 100644 --- a/core/node/node_sync/src/fetcher.rs +++ b/core/node/node_sync/src/fetcher.rs @@ -114,8 +114,8 @@ impl IoCursorExt for IoCursor { let mut this = Self::new(storage).await?; // It's important to know whether we have opened a new batch already or just sealed the previous one. // Depending on it, we must either insert `OpenBatch` item into the queue, or not. - let unsealed_batch = storage.blocks_dal().get_unsealed_l1_batch().await?; - if unsealed_batch.is_none() { + let was_new_batch_open = storage.blocks_dal().pending_batch_exists().await?; + if !was_new_batch_open { this.l1_batch -= 1; // Should continue from the last L1 batch present in the storage } Ok(this) @@ -201,35 +201,3 @@ impl IoCursorExt for IoCursor { new_actions } } - -#[cfg(test)] -mod tests { - use zksync_dal::{ConnectionPool, Core, CoreDal}; - use zksync_node_genesis::{insert_genesis_batch, GenesisParams}; - use zksync_state_keeper::io::IoCursor; - use zksync_types::{block::UnsealedL1BatchHeader, L1BatchNumber}; - - use crate::fetcher::IoCursorExt; - - #[tokio::test] - async fn io_cursor_recognizes_empty_unsealed_batch() -> anyhow::Result<()> { - let pool = ConnectionPool::::test_pool().await; - let mut conn = pool.connection().await.unwrap(); - insert_genesis_batch(&mut conn, &GenesisParams::mock()) - .await - .unwrap(); - conn.blocks_dal() - .insert_l1_batch(UnsealedL1BatchHeader { - number: L1BatchNumber(1), - timestamp: 1, - protocol_version: None, - fee_address: Default::default(), - fee_input: Default::default(), - }) - .await?; - - let io_cursor = IoCursor::for_fetcher(&mut conn).await?; - assert_eq!(io_cursor.l1_batch, L1BatchNumber(1)); - Ok(()) - } -} diff --git a/core/node/node_sync/src/sync_action.rs b/core/node/node_sync/src/sync_action.rs index 8cb90d24fe84..e3fd56ae9bb0 100644 --- a/core/node/node_sync/src/sync_action.rs +++ b/core/node/node_sync/src/sync_action.rs @@ -33,6 +33,18 @@ impl ActionQueueSender { Ok(()) } + /// Pushes a single action into the queue without checking validity of the sequence. + /// + /// Useful to simulate situations where only a part of the sequence was executed on the node. + #[cfg(test)] + pub async fn push_action_unchecked(&self, action: SyncAction) -> anyhow::Result<()> { + self.0 + .send(action) + .await + .map_err(|_| anyhow::anyhow!("node action processor stopped"))?; + Ok(()) + } + /// Checks whether the action sequence is valid. /// Returned error is meant to be used as a panic message, since an invalid sequence represents an unrecoverable /// error. This function itself does not panic for the ease of testing. diff --git a/core/node/node_sync/src/tests.rs b/core/node/node_sync/src/tests.rs index 3f5791cdf24c..1ae148709b22 100644 --- a/core/node/node_sync/src/tests.rs +++ b/core/node/node_sync/src/tests.rs @@ -2,6 +2,7 @@ use std::{iter, sync::Arc, time::Duration}; +use backon::{ConstantBuilder, Retryable}; use test_casing::test_casing; use tokio::{sync::watch, task::JoinHandle}; use zksync_contracts::BaseSystemContractsHashes; @@ -18,7 +19,7 @@ use zksync_state_keeper::{ }; use zksync_types::{ api, - block::L2BlockHasher, + block::{L2BlockHasher, UnsealedL1BatchHeader}, fee_model::{BatchFeeInput, PubdataIndependentBatchFeeModelInput}, snapshots::SnapshotRecoveryStatus, Address, L1BatchNumber, L2BlockNumber, L2ChainId, ProtocolVersionId, Transaction, H256, @@ -652,3 +653,101 @@ async fn external_io_with_multiple_l1_batches() { assert_eq!(fictive_l2_block.timestamp, 2); assert_eq!(fictive_l2_block.l2_tx_count, 0); } + +async fn wait_for_batch_to_be_open( + pool: &ConnectionPool, + number: L1BatchNumber, +) -> anyhow::Result { + (|| async { + let mut storage = pool.connection().await.unwrap(); + let unsealed_batch = storage.blocks_dal().get_unsealed_l1_batch().await?; + + if let Some(unsealed_batch) = unsealed_batch { + if unsealed_batch.number == number { + Ok(unsealed_batch) + } else { + Err(anyhow::anyhow!("L1 batch #{number} is not open yet")) + } + } else { + Err(anyhow::anyhow!("No unsealed L1 batch found yet")) + } + }) + .retry( + &ConstantBuilder::default() + .with_delay(Duration::from_millis(200)) + .with_max_times(20), + ) + .await +} + +#[tokio::test] +async fn external_io_empty_unsealed_batch() { + let pool = ConnectionPool::::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + ensure_genesis(&mut storage).await; + drop(storage); + + let open_batch_one = open_l1_batch(1, 1, 1); + let tx = create_l2_transaction(10, 100); + let tx_hash = tx.hash(); + let tx = FetchedTransaction::new(tx.into()); + let open_batch_two = open_l1_batch(2, 2, 3); + let fictive_l2_block = SyncAction::L2Block { + params: L2BlockParams { + timestamp: 2, + virtual_blocks: 0, + }, + number: L2BlockNumber(2), + }; + let actions1 = vec![open_batch_one, tx.into(), SyncAction::SealL2Block]; + let actions2 = vec![fictive_l2_block, SyncAction::SealBatch]; + + let (actions_sender, action_queue) = ActionQueue::new(); + let client = MockMainNodeClient::default(); + let state_keeper = + StateKeeperHandles::new(pool.clone(), client, action_queue, &[&[tx_hash]]).await; + actions_sender.push_actions(actions1).await.unwrap(); + actions_sender.push_actions(actions2).await.unwrap(); + // Unchecked insert of batch #2 to simulate restart in the middle of processing an action sequence + // In other words batch #2 is inserted completely empty with no blocks/txs present in it + actions_sender + .push_action_unchecked(open_batch_two.clone()) + .await + .unwrap(); + // Wait until the L2 block is sealed. + state_keeper.wait_for_local_block(L2BlockNumber(2)).await; + + // Wait until L1 batch #2 is opened and persisted. + let unsealed_batch = wait_for_batch_to_be_open(&pool, L1BatchNumber(2)) + .await + .unwrap(); + assert_eq!(unsealed_batch.number, L1BatchNumber(2)); + assert_eq!(unsealed_batch.timestamp, 2); + + // Prepare the rest of batch #2 + let tx = create_l2_transaction(20, 200); + let tx_hash = tx.hash(); + let tx = FetchedTransaction::new(tx.into()); + let fictive_l2_block = SyncAction::L2Block { + params: L2BlockParams { + timestamp: 4, + virtual_blocks: 0, + }, + number: L2BlockNumber(4), + }; + let actions1 = vec![open_batch_two, tx.into(), SyncAction::SealL2Block]; + let actions2 = vec![fictive_l2_block, SyncAction::SealBatch]; + + // Restart state keeper + let (actions_sender, action_queue) = ActionQueue::new(); + let client = MockMainNodeClient::default(); + let state_keeper = + StateKeeperHandles::new(pool.clone(), client, action_queue, &[&[tx_hash]]).await; + actions_sender.push_actions(actions1).await.unwrap(); + actions_sender.push_actions(actions2).await.unwrap(); + + let hash_task = tokio::spawn(mock_l1_batch_hash_computation(pool.clone(), 1)); + // Wait until the block #4 is sealed. + state_keeper.wait_for_local_block(L2BlockNumber(4)).await; + hash_task.await.unwrap(); +} diff --git a/core/tests/vm-benchmark/src/vm.rs b/core/tests/vm-benchmark/src/vm.rs index 30e2321298fe..dddef0de82fe 100644 --- a/core/tests/vm-benchmark/src/vm.rs +++ b/core/tests/vm-benchmark/src/vm.rs @@ -103,7 +103,7 @@ impl BenchmarkingVmFactory for Fast impl vm_fast::Tracer for InstructionCount { fn before_instruction< OP: zksync_vm2::interface::OpcodeType, - S: zksync_vm2::interface::StateInterface, + S: zksync_vm2::interface::GlobalStateInterface, >( &mut self, _: &mut S, diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 1d584a473d96..1408f2b23cd9 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -304,7 +304,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" dependencies = [ - "bindgen 0.69.4", + "bindgen", "cc", "cmake", "dunce", @@ -461,29 +461,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.59.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "clap 2.34.0", - "env_logger 0.9.3", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2 1.0.85", - "quote 1.0.36", - "regex", - "rustc-hash", - "shlex", - "which", -] - [[package]] name = "bindgen" version = "0.69.4" @@ -674,9 +651,9 @@ dependencies = [ [[package]] name = "boojum-cuda" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f9a6d958dd58a0899737e5a1fc6597aefcf7980bf8be5be5329e701cbd45ca" +checksum = "98c681a3f867afe40bcc188e5cb5260bbf5699531823affa3cbe28f7ca9b7bc9" dependencies = [ "boojum", "cmake", @@ -822,11 +799,11 @@ dependencies = [ [[package]] name = "circuit_definitions" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b532214f063e5e0ee5c0fc1d3afd56dec541efa68b8985f14cc55cc324f4c48" +checksum = "492404ea63c934d8e894325f0a741723bf91cd035cb34a92fddd8617c4a00fd3" dependencies = [ - "circuit_encodings 0.150.5", + "circuit_encodings 0.150.6", "crossbeam", "derivative", "seq-macro", @@ -872,14 +849,14 @@ dependencies = [ [[package]] name = "circuit_encodings" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67617688c66640c84f9b98ff26d48f7898dca4faeb45241a4f21ec333788e7b" +checksum = "f5128d4b8fbb27ac453f573a95601058e74487bdafd22a3168cded66bf340c28" dependencies = [ "derivative", "serde", - "zk_evm 0.150.5", - "zkevm_circuits 0.150.5", + "zk_evm 0.150.6", + "zkevm_circuits 0.150.6", ] [[package]] @@ -939,11 +916,11 @@ dependencies = [ [[package]] name = "circuit_sequencer_api" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21017310971d4a051e4a52ad70eed11d1ae69defeca8314f73a3a4bad16705a9" +checksum = "093d0c2c0b39144ddb4e1e88d73d95067ce34ec7750808b2eed01edbb510b88e" dependencies = [ - "circuit_encodings 0.150.5", + "circuit_encodings 0.150.6", "derivative", "rayon", "serde", @@ -1764,9 +1741,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "era_cudart" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f0d6e329b2c11d134c3140951209be968ef316ed64ddde75640eaed7f10264" +checksum = "c1e1990fee6e9d25b40524ce53ca7977a211155a17bc7277f4dd354633e4fc22" dependencies = [ "bitflags 2.6.0", "era_cudart_sys", @@ -1775,9 +1752,9 @@ dependencies = [ [[package]] name = "era_cudart_sys" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "060e8186234c7a281021fb95614e06e94e1fc7ab78938360a5c27af0f8fc6105" +checksum = "d84e8d300c28cd91ceb56340f66da8607409f44a45f5e694e23723630db8c852" dependencies = [ "serde_json", ] @@ -4103,12 +4080,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pem" version = "3.0.4" @@ -5711,9 +5682,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" [[package]] name = "shivini" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb6d928451f0779f14da02ee9d51d4bde560328edc6471f0d5c5c11954345c4" +checksum = "92776ca824f49c255a7417939706d759e0fd3dd4217420d01da68beae04f0bd6" dependencies = [ "bincode", "blake2 0.10.6", @@ -7479,9 +7450,9 @@ dependencies = [ [[package]] name = "zk_evm" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6e69931f24db5cf333b714721e8d80ff88bfdb7da8c3dc7882612ffddb8d27" +checksum = "c14bda6c101389145cd01fac900f1392876bc0284d98faf7f376237baa2cb19d" dependencies = [ "anyhow", "lazy_static", @@ -7489,7 +7460,7 @@ dependencies = [ "serde", "serde_json", "static_assertions", - "zk_evm_abstractions 0.150.5", + "zk_evm_abstractions 0.150.6", ] [[package]] @@ -7520,22 +7491,22 @@ dependencies = [ [[package]] name = "zk_evm_abstractions" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6b0720261ab55490fe3a96e96de30d5d7b277940b52ea7f52dbf564eb1748" +checksum = "a008f2442fc6a508bdd1f902380242cb6ff11b8b27acdac2677c6d9f75cbb004" dependencies = [ "anyhow", "num_enum 0.6.1", "serde", "static_assertions", - "zkevm_opcode_defs 0.150.5", + "zkevm_opcode_defs 0.150.6", ] [[package]] name = "zkevm-assembly" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e99106038062537c05b4e6e7754d1bbba28ba16185a3e5ee5ad22e2f8be883bb" +checksum = "5dc743ac7b0d618536dc3ace798fd4b8af78b057884afda5785c7970e15d62d0" dependencies = [ "env_logger 0.9.3", "hex", @@ -7548,7 +7519,7 @@ dependencies = [ "smallvec", "structopt", "thiserror", - "zkevm_opcode_defs 0.150.5", + "zkevm_opcode_defs 0.150.6", ] [[package]] @@ -7597,9 +7568,9 @@ dependencies = [ [[package]] name = "zkevm_circuits" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784fa7cfb51e17c5ced112bca43da30b3468b2347b7af0427ad9638759fb140e" +checksum = "1f68518aedd5358b17224771bb78bacd912cf66011aeda98b1f887cfb9e0972f" dependencies = [ "arrayvec 0.7.4", "boojum", @@ -7611,7 +7582,7 @@ dependencies = [ "seq-macro", "serde", "smallvec", - "zkevm_opcode_defs 0.150.5", + "zkevm_opcode_defs 0.150.6", "zksync_cs_derive", ] @@ -7659,9 +7630,9 @@ dependencies = [ [[package]] name = "zkevm_opcode_defs" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79055eae1b6c1ab80793ed9d77d2964c9c896afa4b5dfed278cf58cd10acfe8f" +checksum = "762b5f1c1b283c5388995a85d40a05aef1c14f50eb904998b7e9364739f5b899" dependencies = [ "bitflags 2.6.0", "blake2 0.10.6", @@ -7676,13 +7647,13 @@ dependencies = [ [[package]] name = "zkevm_test_harness" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "550f82d3b7448c35168dc13bfadbccd5fd306097b6e1ea01793151c1c9137a36" +checksum = "73ad3e73d290a38a35dd245fd68cb6f498a8a8da4a52f846e88da3d3c31a34fd" dependencies = [ "bincode", "circuit_definitions", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "codegen", "crossbeam", "derivative", @@ -7703,11 +7674,10 @@ dependencies = [ [[package]] name = "zksync-gpu-ffi" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86511b3957adfe415ecdbd1ee01c51aa3ca131a607e61ca024976312f613b0f9" +checksum = "d555e24b853359c5b076c52f9ff9e0ed62a7edc8c2f82f93517c524410c21ecb" dependencies = [ - "bindgen 0.59.2", "cmake", "crossbeam", "derivative", @@ -7719,9 +7689,9 @@ dependencies = [ [[package]] name = "zksync-gpu-prover" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e4c00f2db603d1b696bc2e9d822bb4c087050de5b65559067fc2232786cbc93" +checksum = "615dad34e5fe678ec3b3e029af3f19313bebb1b771a8ce963c9ab9a8cc3879d3" dependencies = [ "bit-vec", "cfg-if", @@ -7736,9 +7706,9 @@ dependencies = [ [[package]] name = "zksync-wrapper-prover" -version = "0.150.9" +version = "0.151.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58df1ec10e0d5eb58563bb01abda5ed185c9b9621502e361848ca40eb7868ac" +checksum = "80721b2da2643bd43f664ac65673ee078e6973c0a88d75b73bfaeac8e1bf5432" dependencies = [ "circuit_definitions", "zkevm_test_harness", @@ -8106,9 +8076,9 @@ dependencies = [ [[package]] name = "zksync_kzg" -version = "0.150.5" +version = "0.150.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb8a9c76c172a6d639855ee342b9a670e3ba472f5ae302f771b1c3ee777dc88" +checksum = "6c006b6b7a27cc50ff0c515b6d0b197dbb907bbf65d1d2ea42fc3ed21b315642" dependencies = [ "boojum", "derivative", @@ -8118,7 +8088,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "zkevm_circuits 0.150.5", + "zkevm_circuits 0.150.6", ] [[package]] @@ -8154,7 +8124,7 @@ dependencies = [ "circuit_sequencer_api 0.140.3", "circuit_sequencer_api 0.141.2", "circuit_sequencer_api 0.142.2", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "ethabi", "hex", "itertools 0.10.5", @@ -8166,7 +8136,7 @@ dependencies = [ "zk_evm 0.133.0", "zk_evm 0.140.0", "zk_evm 0.141.0", - "zk_evm 0.150.5", + "zk_evm 0.150.6", "zksync_contracts", "zksync_system_constants", "zksync_types", @@ -8218,7 +8188,7 @@ dependencies = [ "anyhow", "async-trait", "bincode", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "clap 4.5.4", "ctrlc", "futures 0.3.30", @@ -8444,7 +8414,7 @@ name = "zksync_prover_interface" version = "0.1.0" dependencies = [ "chrono", - "circuit_sequencer_api 0.150.5", + "circuit_sequencer_api 0.150.6", "serde", "serde_with", "strum", @@ -8639,19 +8609,19 @@ dependencies = [ [[package]] name = "zksync_vm2" version = "0.2.1" -source = "git+https://github.com/matter-labs/vm2.git?rev=a233d44bbe61dc6a758a754c3b78fe4f83e56699#a233d44bbe61dc6a758a754c3b78fe4f83e56699" +source = "git+https://github.com/matter-labs/vm2.git?rev=df5bec3d04d64d434f9b0ccb285ba4681008f7b3#df5bec3d04d64d434f9b0ccb285ba4681008f7b3" dependencies = [ "enum_dispatch", "primitive-types", - "zk_evm_abstractions 0.150.5", - "zkevm_opcode_defs 0.150.5", + "zk_evm_abstractions 0.150.6", + "zkevm_opcode_defs 0.150.6", "zksync_vm2_interface", ] [[package]] name = "zksync_vm2_interface" version = "0.2.1" -source = "git+https://github.com/matter-labs/vm2.git?rev=a233d44bbe61dc6a758a754c3b78fe4f83e56699#a233d44bbe61dc6a758a754c3b78fe4f83e56699" +source = "git+https://github.com/matter-labs/vm2.git?rev=df5bec3d04d64d434f9b0ccb285ba4681008f7b3#df5bec3d04d64d434f9b0ccb285ba4681008f7b3" dependencies = [ "primitive-types", ] diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 742eee649de1..61169dd43636 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -62,13 +62,13 @@ url = "2.5.2" vise = "0.2.0" # Proving dependencies -circuit_definitions = "=0.150.5" -circuit_sequencer_api = "=0.150.5" -zkevm_test_harness = "=0.150.5" +circuit_definitions = "=0.150.6" +circuit_sequencer_api = "=0.150.6" +zkevm_test_harness = "=0.150.6" # GPU proving dependencies -wrapper_prover = { package = "zksync-wrapper-prover", version = "=0.150.9" } -shivini = "=0.150.9" +wrapper_prover = { package = "zksync-wrapper-prover", version = "=0.151.0" } +shivini = "=0.151.0" # Core workspace dependencies zksync_multivm = { path = "../core/lib/multivm", version = "0.1.0" } diff --git a/prover/crates/bin/witness_generator/src/rounds/basic_circuits/utils.rs b/prover/crates/bin/witness_generator/src/rounds/basic_circuits/utils.rs index a8bc59bd45e5..31dc54814103 100644 --- a/prover/crates/bin/witness_generator/src/rounds/basic_circuits/utils.rs +++ b/prover/crates/bin/witness_generator/src/rounds/basic_circuits/utils.rs @@ -5,7 +5,7 @@ use std::{ }; use circuit_definitions::{ - circuit_definitions::base_layer::{ZkSyncBaseLayerCircuit, ZkSyncBaseLayerStorage}, + circuit_definitions::base_layer::ZkSyncBaseLayerStorage, encodings::recursion_request::RecursionQueueSimulator, zkevm_circuits::fsm_input_output::ClosedFormInputCompactFormWitness, }; @@ -21,7 +21,7 @@ use zksync_multivm::{ zk_evm_latest::ethereum_types::Address, }; use zksync_object_store::ObjectStore; -use zksync_prover_fri_types::{keys::ClosedFormInputKey, CircuitAuxData}; +use zksync_prover_fri_types::keys::ClosedFormInputKey; use zksync_prover_interface::inputs::WitnessInputData; use zksync_system_constants::BOOTLOADER_ADDRESS; use zksync_types::L1BatchNumber; @@ -31,8 +31,7 @@ use crate::{ rounds::basic_circuits::Witness, storage_oracle::StorageOracle, utils::{ - expand_bootloader_contents, save_circuit, save_ram_premutation_queue_witness, - ClosedFormInputWrapper, KZG_TRUSTED_SETUP_FILE, + expand_bootloader_contents, save_circuit, ClosedFormInputWrapper, KZG_TRUSTED_SETUP_FILE, }, witness::WitnessStorage, }; @@ -64,17 +63,38 @@ pub(super) async fn generate_witness( let (circuit_sender, mut circuit_receiver) = tokio::sync::mpsc::channel(1); let (queue_sender, mut queue_receiver) = tokio::sync::mpsc::channel(1); - let (ram_permutation_queue_sender, mut ram_permutation_queue_receiver) = - tokio::sync::mpsc::channel(1); let make_circuits_span = tracing::info_span!("make_circuits"); let make_circuits_span_copy = make_circuits_span.clone(); + + use std::{sync::mpsc::sync_channel, thread}; + let (artifacts_sender, artifacts_receiver) = sync_channel(1); + + let artifacts_receiver_handle = thread::spawn(move || { + let span = tracing::info_span!(parent: make_circuits_span_copy, "make_circuits_blocking"); + + while let Ok(artifact) = artifacts_receiver.recv() { + match artifact { + WitnessGenerationArtifact::BaseLayerCircuit(circuit) => { + let parent_span = span.clone(); + tracing::info_span!(parent: parent_span, "send_circuit").in_scope(|| { + circuit_sender + .blocking_send(circuit) + .expect("failed to send circuit from harness"); + }); + } + WitnessGenerationArtifact::RecursionQueue((a, b, c)) => queue_sender + .blocking_send((a as u8, b, c)) + .expect("failed to send recursion queue from harness"), + _ => {} + } + } + }); + // Blocking call from harness that does the CPU heavy lifting. // Provides circuits and recursion queue via callback functions and returns scheduler witnesses. // Circuits are "streamed" one by one as they're being generated. let make_circuits_handle = tokio::task::spawn_blocking(move || { - let span = tracing::info_span!(parent: make_circuits_span_copy, "make_circuits_blocking"); - let witness_storage = WitnessStorage::new(input.vm_run_data.witness_block_state); let storage_view = StorageView::new(witness_storage).to_rc_ptr(); @@ -91,33 +111,11 @@ pub(super) async fn generate_witness( .to_str() .expect("Path to KZG trusted setup is not a UTF-8 string"); - let artifacts_callback = |artifact: WitnessGenerationArtifact| match artifact { - WitnessGenerationArtifact::BaseLayerCircuit(circuit) => { - let parent_span = span.clone(); - tracing::info_span!(parent: parent_span, "send_circuit").in_scope(|| { - circuit_sender - .blocking_send(circuit) - .expect("failed to send circuit from harness"); - }); - } - WitnessGenerationArtifact::RecursionQueue((a, b, c)) => queue_sender - .blocking_send((a as u8, b, c)) - .expect("failed to send recursion queue from harness"), - a @ WitnessGenerationArtifact::MemoryQueueWitness(_) => { - let parent_span = span.clone(); - tracing::info_span!(parent: parent_span, "send_ram_permutation_queue_witness") - .in_scope(|| { - ram_permutation_queue_sender - .blocking_send(a) - .expect("failed to send ram permutation queue sitness from harness"); - }); - } - }; - let evm_emulator_code_hash = input.vm_run_data.evm_emulator_code_hash; // By convention, default AA is used instead of the EVM emulator if the latter is disabled. let evm_emulator_code_hash = evm_emulator_code_hash.unwrap_or(input.vm_run_data.default_account_code_hash); + let (scheduler_witness, block_witness) = zkevm_test_harness::external_calls::run( Address::zero(), BOOTLOADER_ADDRESS, @@ -132,9 +130,9 @@ pub(super) async fn generate_witness( geometry_config, storage_oracle, tree, - path, + path.to_owned(), input.eip_4844_blobs.blobs(), - artifacts_callback, + artifacts_sender, ); (scheduler_witness, block_witness) }) @@ -153,8 +151,6 @@ pub(super) async fn generate_witness( // If the order is tampered with, proving will fail (as the proof would be computed for a different sequence of instruction). let mut circuit_sequence = 0; - let mut ram_circuit_sequence = 0; - while let Some(circuit) = circuit_receiver .recv() .instrument(tracing::info_span!("wait_for_circuit")) @@ -169,26 +165,9 @@ pub(super) async fn generate_witness( .await .expect("failed to get permit for running save circuit task"); - let partial_circuit_aux_data = match &circuit { - ZkSyncBaseLayerCircuit::RAMPermutation(_) => { - let circuit_subsequence_number = ram_circuit_sequence; - ram_circuit_sequence += 1; - Some(CircuitAuxData { - circuit_subsequence_number, - }) - } - _ => None, - }; - save_circuit_handles.push(tokio::task::spawn(async move { - let (circuit_id, circuit_url) = save_circuit( - block_number, - circuit, - sequence, - partial_circuit_aux_data, - object_store, - ) - .await; + let (circuit_id, circuit_url) = + save_circuit(block_number, circuit, sequence, object_store).await; drop(permit); (circuit_id, circuit_url) })); @@ -196,57 +175,6 @@ pub(super) async fn generate_witness( } .instrument(save_circuits_span); - let mut save_ram_queue_witness_handles = vec![]; - - let save_ram_queue_witness_span = tracing::info_span!("save_circuits"); - - // Future which receives part of RAM permutation circuits witnesses and saves them async. - // Uses semaphore because these artifacts are of significant size - let ram_queue_witness_receiver_handle = async { - let mut sorted_sequence = 0; - let mut unsorted_sequence = 0; - - while let Some(witness_artifact) = ram_permutation_queue_receiver - .recv() - .instrument(tracing::info_span!("wait_for_ram_witness")) - .await - { - let object_store = object_store.clone(); - let semaphore = semaphore.clone(); - let permit = semaphore - .acquire_owned() - .await - .expect("failed to get permit for running save ram permutation queue witness task"); - let (is_sorted, witness, sequence) = match witness_artifact { - WitnessGenerationArtifact::MemoryQueueWitness((witness, sorted)) => { - let sequence = if sorted { - let sequence = sorted_sequence; - sorted_sequence += 1; - sequence - } else { - let sequence = unsorted_sequence; - unsorted_sequence += 1; - sequence - }; - (sorted, witness, sequence) - } - _ => panic!("Invalid artifact received"), - }; - save_ram_queue_witness_handles.push(tokio::task::spawn(async move { - let _ = save_ram_premutation_queue_witness( - block_number, - sequence, - is_sorted, - witness, - object_store, - ) - .await; - drop(permit); - })); - } - } - .instrument(save_ram_queue_witness_span); - let mut save_queue_handles = vec![]; let save_queues_span = tracing::info_span!("save_queues"); @@ -272,11 +200,10 @@ pub(super) async fn generate_witness( } .instrument(save_queues_span); - let (witnesses, _, _, _) = tokio::join!( + let (witnesses, _, _) = tokio::join!( make_circuits_handle, circuit_receiver_handle, - queue_receiver_handle, - ram_queue_witness_receiver_handle + queue_receiver_handle ); let (mut scheduler_witness, block_aux_witness) = witnesses.unwrap(); @@ -301,11 +228,7 @@ pub(super) async fn generate_witness( .filter(|(circuit_id, _, _)| circuits_present.contains(circuit_id)) .collect(); - let _: Vec<_> = futures::future::join_all(save_ram_queue_witness_handles) - .await - .into_iter() - .map(|result| result.expect("failed to save ram permutation queue witness")) - .collect(); + artifacts_receiver_handle.join().unwrap(); scheduler_witness.previous_block_meta_hash = input.previous_batch_metadata.meta_hash.0; scheduler_witness.previous_block_aux_hash = input.previous_batch_metadata.aux_hash.0; diff --git a/prover/crates/bin/witness_generator/src/utils.rs b/prover/crates/bin/witness_generator/src/utils.rs index 8524bdae9ff0..ea631f19cd85 100644 --- a/prover/crates/bin/witness_generator/src/utils.rs +++ b/prover/crates/bin/witness_generator/src/utils.rs @@ -3,10 +3,7 @@ use std::{ sync::Arc, }; -use circuit_definitions::{ - circuit_definitions::base_layer::ZkSyncBaseLayerCircuit, - encodings::memory_query::MemoryQueueStateWitnesses, -}; +use circuit_definitions::circuit_definitions::base_layer::ZkSyncBaseLayerCircuit; use once_cell::sync::Lazy; use zkevm_test_harness::boojum::field::goldilocks::GoldilocksField; use zksync_multivm::utils::get_used_bootloader_memory_bytes; @@ -24,8 +21,8 @@ use zksync_prover_fri_types::{ encodings::recursion_request::RecursionQueueSimulator, zkevm_circuits::scheduler::input::SchedulerCircuitInstanceWitness, }, - keys::{AggregationsKey, ClosedFormInputKey, FriCircuitKey, RamPermutationQueueWitnessKey}, - CircuitAuxData, CircuitWrapper, FriProofWrapper, RamPermutationQueueWitness, + keys::{AggregationsKey, ClosedFormInputKey, FriCircuitKey}, + CircuitWrapper, FriProofWrapper, }; use zksync_types::{basic_fri_types::AggregationRound, L1BatchNumber, ProtocolVersionId, U256}; @@ -121,7 +118,6 @@ pub async fn save_circuit( block_number: L1BatchNumber, circuit: ZkSyncBaseLayerCircuit, sequence_number: usize, - aux_data_for_partial_circuit: Option, object_store: Arc, ) -> (u8, String) { let circuit_id = circuit.numeric_circuit_type(); @@ -133,43 +129,12 @@ pub async fn save_circuit( depth: 0, }; - let blob_url = if let Some(aux_data_for_partial_circuit) = aux_data_for_partial_circuit { - object_store - .put( - circuit_key, - &CircuitWrapper::BasePartial((circuit, aux_data_for_partial_circuit)), - ) - .await - .unwrap() - } else { - object_store - .put(circuit_key, &CircuitWrapper::Base(circuit)) - .await - .unwrap() - }; - (circuit_id, blob_url) -} - -#[tracing::instrument( - skip_all, - fields(l1_batch = %block_number) -)] -pub async fn save_ram_premutation_queue_witness( - block_number: L1BatchNumber, - circuit_subsequence_number: usize, - is_sorted: bool, - witness: MemoryQueueStateWitnesses, - object_store: Arc, -) -> String { - let witness_key = RamPermutationQueueWitnessKey { - block_number, - circuit_subsequence_number, - is_sorted, - }; - object_store - .put(witness_key, &RamPermutationQueueWitness { witness }) + let blob_url = object_store + .put(circuit_key, &CircuitWrapper::Base(circuit)) .await - .unwrap() + .unwrap(); + + (circuit_id, blob_url) } #[tracing::instrument( diff --git a/prover/crates/lib/keystore/src/keystore.rs b/prover/crates/lib/keystore/src/keystore.rs index 6225943e3cd7..ab3b115bc635 100644 --- a/prover/crates/lib/keystore/src/keystore.rs +++ b/prover/crates/lib/keystore/src/keystore.rs @@ -470,6 +470,7 @@ impl Keystore { } /// Async loads mapping of all circuits to setup key, if successful + #[cfg(feature = "gpu")] pub async fn load_all_setup_key_mapping( &self, ) -> anyhow::Result>> {